From 68467bdf4d76ba2c57289209b0ffd6ba599e2080 Mon Sep 17 00:00:00 2001
From: rockerBOO <rockerboo@gmail.com>
Date: Thu, 11 Apr 2024 17:33:19 -0400
Subject: [PATCH] Fix unset or invalid LR from making a param_group

---
 networks/dylora.py  | 4 ++--
 networks/lora.py    | 5 +++--
 networks/lora_fa.py | 4 ++--
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/networks/dylora.py b/networks/dylora.py
index dc5c7cb35..0546fc7ae 100644
--- a/networks/dylora.py
+++ b/networks/dylora.py
@@ -412,8 +412,8 @@ def prepare_optimizer_params(
         text_encoder_lr,
         unet_lr,
         default_lr,
-        unet_loraplus_ratio=None,
         text_encoder_loraplus_ratio=None,
+        unet_loraplus_ratio=None,
         loraplus_ratio=None
     ):
         self.requires_grad_(True)
@@ -441,7 +441,7 @@ def assemble_params(loras, lr, ratio):
                     else:
                         param_data["lr"] = lr
 
-                if ("lr" in param_data) and (param_data["lr"] == 0):
+                if param_data.get("lr", None) == 0 or param_data.get("lr", None) is None:
                     continue
 
                 params.append(param_data)
diff --git a/networks/lora.py b/networks/lora.py
index 6cb05bcb0..d74608fea 100644
--- a/networks/lora.py
+++ b/networks/lora.py
@@ -1040,8 +1040,8 @@ def prepare_optimizer_params(
         text_encoder_lr,
         unet_lr,
         default_lr,
-        unet_loraplus_ratio=None,
         text_encoder_loraplus_ratio=None,
+        unet_loraplus_ratio=None,
         loraplus_ratio=None
     ):
         self.requires_grad_(True)
@@ -1069,7 +1069,8 @@ def assemble_params(loras, lr, ratio):
                     else:
                         param_data["lr"] = lr
 
-                if ("lr" in param_data) and (param_data["lr"] == 0):
+                if param_data.get("lr", None) == 0 or param_data.get("lr", None) is None:
+                    print("NO LR skipping!")
                     continue
 
                 params.append(param_data)
diff --git a/networks/lora_fa.py b/networks/lora_fa.py
index 2eff86d6c..9a608118a 100644
--- a/networks/lora_fa.py
+++ b/networks/lora_fa.py
@@ -1038,8 +1038,8 @@ def prepare_optimizer_params(
         text_encoder_lr,
         unet_lr,
         default_lr,
-        unet_loraplus_ratio=None,
         text_encoder_loraplus_ratio=None,
+        unet_loraplus_ratio=None,
         loraplus_ratio=None
     ):
         self.requires_grad_(True)
@@ -1067,7 +1067,7 @@ def assemble_params(loras, lr, ratio):
                     else:
                         param_data["lr"] = lr
 
-                if ("lr" in param_data) and (param_data["lr"] == 0):
+                if param_data.get("lr", None) == 0 or param_data.get("lr", None) is None:
                     continue
 
                 params.append(param_data)