Merge branch 'dev' into finer-settings-freezing-control

14 files changed, 212 insertions, 40 deletions

diff --git a/extensions-builtin/Lora/lyco_helpers.py b/extensions-builtin/Lora/lyco_helpers.py
index 279b34bc..1679a0ce 100644
--- a/extensions-builtin/Lora/lyco_helpers.py
+++ b/extensions-builtin/Lora/lyco_helpers.py
@@ -19,3 +19,50 @@ def rebuild_cp_decomposition(up, down, mid):
     up = up.reshape(up.size(0), -1)

     down = down.reshape(down.size(0), -1)

     return torch.einsum('n m k l, i n, m j -> i j k l', mid, up, down)

+

+

+# copied from https://github.com/KohakuBlueleaf/LyCORIS/blob/dev/lycoris/modules/lokr.py

+def factorization(dimension: int, factor:int=-1) -> tuple[int, int]:

+    '''

+    return a tuple of two value of input dimension decomposed by the number closest to factor

+    second value is higher or equal than first value.

+

+    In LoRA with Kroneckor Product, first value is a value for weight scale.

+    secon value is a value for weight.

+

+    Becuase of non-commutative property, A⊗B ≠ B⊗A. Meaning of two matrices is slightly different.

+

+    examples)

+    factor

+        -1               2                4               8               16               ...

+    127 -> 1, 127   127 -> 1, 127    127 -> 1, 127   127 -> 1, 127   127 -> 1, 127

+    128 -> 8, 16    128 -> 2, 64     128 -> 4, 32    128 -> 8, 16    128 -> 8, 16

+    250 -> 10, 25   250 -> 2, 125    250 -> 2, 125   250 -> 5, 50    250 -> 10, 25

+    360 -> 8, 45    360 -> 2, 180    360 -> 4, 90    360 -> 8, 45    360 -> 12, 30

+    512 -> 16, 32   512 -> 2, 256    512 -> 4, 128   512 -> 8, 64    512 -> 16, 32

+    1024 -> 32, 32  1024 -> 2, 512   1024 -> 4, 256  1024 -> 8, 128  1024 -> 16, 64

+    '''

+

+    if factor > 0 and (dimension % factor) == 0:

+        m = factor

+        n = dimension // factor

+        if m > n:

+            n, m = m, n

+        return m, n

+    if factor < 0:

+        factor = dimension

+    m, n = 1, dimension

+    length = m + n

+    while m<n:

+        new_m = m + 1

+        while dimension%new_m != 0:

+            new_m += 1

+        new_n = dimension // new_m

+        if new_m + new_n > length or new_m>factor:

+            break

+        else:

+            m, n = new_m, new_n

+    if m > n:

+        n, m = m, n

+    return m, n

+

diff --git a/extensions-builtin/Lora/network.py b/extensions-builtin/Lora/network.py
index 6021fd8d..a62e5eff 100644
--- a/extensions-builtin/Lora/network.py
+++ b/extensions-builtin/Lora/network.py
@@ -137,7 +137,7 @@ class NetworkModule:
     def finalize_updown(self, updown, orig_weight, output_shape, ex_bias=None):

         if self.bias is not None:

             updown = updown.reshape(self.bias.shape)

-            updown += self.bias.to(orig_weight.device, dtype=orig_weight.dtype)

+            updown += self.bias.to(orig_weight.device, dtype=updown.dtype)

             updown = updown.reshape(output_shape)

 

         if len(output_shape) == 4:

diff --git a/extensions-builtin/Lora/network_full.py b/extensions-builtin/Lora/network_full.py
index bf6930e9..f221c95f 100644
--- a/extensions-builtin/Lora/network_full.py
+++ b/extensions-builtin/Lora/network_full.py
@@ -18,9 +18,9 @@ class NetworkModuleFull(network.NetworkModule):
 

     def calc_updown(self, orig_weight):

         output_shape = self.weight.shape

-        updown = self.weight.to(orig_weight.device, dtype=orig_weight.dtype)

+        updown = self.weight.to(orig_weight.device)

         if self.ex_bias is not None:

-            ex_bias = self.ex_bias.to(orig_weight.device, dtype=orig_weight.dtype)

+            ex_bias = self.ex_bias.to(orig_weight.device)

         else:

             ex_bias = None

 

diff --git a/extensions-builtin/Lora/network_glora.py b/extensions-builtin/Lora/network_glora.py
index 492d4870..efe5c681 100644
--- a/extensions-builtin/Lora/network_glora.py
+++ b/extensions-builtin/Lora/network_glora.py
@@ -22,12 +22,12 @@ class NetworkModuleGLora(network.NetworkModule):
         self.w2b = weights.w["b2.weight"]
 
     def calc_updown(self, orig_weight):
-        w1a = self.w1a.to(orig_weight.device, dtype=orig_weight.dtype)
-        w1b = self.w1b.to(orig_weight.device, dtype=orig_weight.dtype)
-        w2a = self.w2a.to(orig_weight.device, dtype=orig_weight.dtype)
-        w2b = self.w2b.to(orig_weight.device, dtype=orig_weight.dtype)
+        w1a = self.w1a.to(orig_weight.device)
+        w1b = self.w1b.to(orig_weight.device)
+        w2a = self.w2a.to(orig_weight.device)
+        w2b = self.w2b.to(orig_weight.device)
 
         output_shape = [w1a.size(0), w1b.size(1)]
-        updown = ((w2b @ w1b) + ((orig_weight @ w2a) @ w1a))
+        updown = ((w2b @ w1b) + ((orig_weight.to(dtype = w1a.dtype) @ w2a) @ w1a))
 
         return self.finalize_updown(updown, orig_weight, output_shape)
diff --git a/extensions-builtin/Lora/network_hada.py b/extensions-builtin/Lora/network_hada.py
index 5fcb0695..d95a0fd1 100644
--- a/extensions-builtin/Lora/network_hada.py
+++ b/extensions-builtin/Lora/network_hada.py
@@ -27,16 +27,16 @@ class NetworkModuleHada(network.NetworkModule):
         self.t2 = weights.w.get("hada_t2")

 

     def calc_updown(self, orig_weight):

-        w1a = self.w1a.to(orig_weight.device, dtype=orig_weight.dtype)

-        w1b = self.w1b.to(orig_weight.device, dtype=orig_weight.dtype)

-        w2a = self.w2a.to(orig_weight.device, dtype=orig_weight.dtype)

-        w2b = self.w2b.to(orig_weight.device, dtype=orig_weight.dtype)

+        w1a = self.w1a.to(orig_weight.device)

+        w1b = self.w1b.to(orig_weight.device)

+        w2a = self.w2a.to(orig_weight.device)

+        w2b = self.w2b.to(orig_weight.device)

 

         output_shape = [w1a.size(0), w1b.size(1)]

 

         if self.t1 is not None:

             output_shape = [w1a.size(1), w1b.size(1)]

-            t1 = self.t1.to(orig_weight.device, dtype=orig_weight.dtype)

+            t1 = self.t1.to(orig_weight.device)

             updown1 = lyco_helpers.make_weight_cp(t1, w1a, w1b)

             output_shape += t1.shape[2:]

         else:

@@ -45,7 +45,7 @@ class NetworkModuleHada(network.NetworkModule):
             updown1 = lyco_helpers.rebuild_conventional(w1a, w1b, output_shape)

 

         if self.t2 is not None:

-            t2 = self.t2.to(orig_weight.device, dtype=orig_weight.dtype)

+            t2 = self.t2.to(orig_weight.device)

             updown2 = lyco_helpers.make_weight_cp(t2, w2a, w2b)

         else:

             updown2 = lyco_helpers.rebuild_conventional(w2a, w2b, output_shape)

diff --git a/extensions-builtin/Lora/network_ia3.py b/extensions-builtin/Lora/network_ia3.py
index 7edc4249..96faeaf3 100644
--- a/extensions-builtin/Lora/network_ia3.py
+++ b/extensions-builtin/Lora/network_ia3.py
@@ -17,7 +17,7 @@ class NetworkModuleIa3(network.NetworkModule):
         self.on_input = weights.w["on_input"].item()

 

     def calc_updown(self, orig_weight):

-        w = self.w.to(orig_weight.device, dtype=orig_weight.dtype)

+        w = self.w.to(orig_weight.device)

 

         output_shape = [w.size(0), orig_weight.size(1)]

         if self.on_input:

diff --git a/extensions-builtin/Lora/network_lokr.py b/extensions-builtin/Lora/network_lokr.py
index 340acdab..fcdaeafd 100644
--- a/extensions-builtin/Lora/network_lokr.py
+++ b/extensions-builtin/Lora/network_lokr.py
@@ -37,22 +37,22 @@ class NetworkModuleLokr(network.NetworkModule):
 

     def calc_updown(self, orig_weight):

         if self.w1 is not None:

-            w1 = self.w1.to(orig_weight.device, dtype=orig_weight.dtype)

+            w1 = self.w1.to(orig_weight.device)

         else:

-            w1a = self.w1a.to(orig_weight.device, dtype=orig_weight.dtype)

-            w1b = self.w1b.to(orig_weight.device, dtype=orig_weight.dtype)

+            w1a = self.w1a.to(orig_weight.device)

+            w1b = self.w1b.to(orig_weight.device)

             w1 = w1a @ w1b

 

         if self.w2 is not None:

-            w2 = self.w2.to(orig_weight.device, dtype=orig_weight.dtype)

+            w2 = self.w2.to(orig_weight.device)

         elif self.t2 is None:

-            w2a = self.w2a.to(orig_weight.device, dtype=orig_weight.dtype)

-            w2b = self.w2b.to(orig_weight.device, dtype=orig_weight.dtype)

+            w2a = self.w2a.to(orig_weight.device)

+            w2b = self.w2b.to(orig_weight.device)

             w2 = w2a @ w2b

         else:

-            t2 = self.t2.to(orig_weight.device, dtype=orig_weight.dtype)

-            w2a = self.w2a.to(orig_weight.device, dtype=orig_weight.dtype)

-            w2b = self.w2b.to(orig_weight.device, dtype=orig_weight.dtype)

+            t2 = self.t2.to(orig_weight.device)

+            w2a = self.w2a.to(orig_weight.device)

+            w2b = self.w2b.to(orig_weight.device)

             w2 = lyco_helpers.make_weight_cp(t2, w2a, w2b)

 

         output_shape = [w1.size(0) * w2.size(0), w1.size(1) * w2.size(1)]

diff --git a/extensions-builtin/Lora/network_lora.py b/extensions-builtin/Lora/network_lora.py
index 26c0a72c..4cc40295 100644
--- a/extensions-builtin/Lora/network_lora.py
+++ b/extensions-builtin/Lora/network_lora.py
@@ -61,13 +61,13 @@ class NetworkModuleLora(network.NetworkModule):
         return module

 

     def calc_updown(self, orig_weight):

-        up = self.up_model.weight.to(orig_weight.device, dtype=orig_weight.dtype)

-        down = self.down_model.weight.to(orig_weight.device, dtype=orig_weight.dtype)

+        up = self.up_model.weight.to(orig_weight.device)

+        down = self.down_model.weight.to(orig_weight.device)

 

         output_shape = [up.size(0), down.size(1)]

         if self.mid_model is not None:

             # cp-decomposition

-            mid = self.mid_model.weight.to(orig_weight.device, dtype=orig_weight.dtype)

+            mid = self.mid_model.weight.to(orig_weight.device)

             updown = lyco_helpers.rebuild_cp_decomposition(up, down, mid)

             output_shape += mid.shape[2:]

         else:

diff --git a/extensions-builtin/Lora/network_norm.py b/extensions-builtin/Lora/network_norm.py
index ce450158..d25afcbb 100644
--- a/extensions-builtin/Lora/network_norm.py
+++ b/extensions-builtin/Lora/network_norm.py
@@ -18,10 +18,10 @@ class NetworkModuleNorm(network.NetworkModule):
 
     def calc_updown(self, orig_weight):
         output_shape = self.w_norm.shape
-        updown = self.w_norm.to(orig_weight.device, dtype=orig_weight.dtype)
+        updown = self.w_norm.to(orig_weight.device)
 
         if self.b_norm is not None:
-            ex_bias = self.b_norm.to(orig_weight.device, dtype=orig_weight.dtype)
+            ex_bias = self.b_norm.to(orig_weight.device)
         else:
             ex_bias = None
 
diff --git a/extensions-builtin/Lora/network_oft.py b/extensions-builtin/Lora/network_oft.py
new file mode 100644
index 00000000..fa647020
--- /dev/null
+++ b/extensions-builtin/Lora/network_oft.py
@@ -0,0 +1,82 @@
+import torch
+import network
+from lyco_helpers import factorization
+from einops import rearrange
+
+
+class ModuleTypeOFT(network.ModuleType):
+    def create_module(self, net: network.Network, weights: network.NetworkWeights):
+        if all(x in weights.w for x in ["oft_blocks"]) or all(x in weights.w for x in ["oft_diag"]):
+            return NetworkModuleOFT(net, weights)
+
+        return None
+
+# Supports both kohya-ss' implementation of COFT  https://github.com/kohya-ss/sd-scripts/blob/main/networks/oft.py
+# and KohakuBlueleaf's implementation of OFT/COFT https://github.com/KohakuBlueleaf/LyCORIS/blob/dev/lycoris/modules/diag_oft.py
+class NetworkModuleOFT(network.NetworkModule):
+    def __init__(self,  net: network.Network, weights: network.NetworkWeights):
+
+        super().__init__(net, weights)
+
+        self.lin_module = None
+        self.org_module: list[torch.Module] = [self.sd_module]
+
+        self.scale = 1.0
+
+        # kohya-ss
+        if "oft_blocks" in weights.w.keys():
+            self.is_kohya = True
+            self.oft_blocks = weights.w["oft_blocks"] # (num_blocks, block_size, block_size)
+            self.alpha = weights.w["alpha"] # alpha is constraint
+            self.dim = self.oft_blocks.shape[0] # lora dim
+        # LyCORIS
+        elif "oft_diag" in weights.w.keys():
+            self.is_kohya = False
+            self.oft_blocks = weights.w["oft_diag"]
+            # self.alpha is unused
+            self.dim = self.oft_blocks.shape[1] # (num_blocks, block_size, block_size)
+
+        is_linear = type(self.sd_module) in [torch.nn.Linear, torch.nn.modules.linear.NonDynamicallyQuantizableLinear]
+        is_conv = type(self.sd_module) in [torch.nn.Conv2d]
+        is_other_linear = type(self.sd_module) in [torch.nn.MultiheadAttention] # unsupported
+
+        if is_linear:
+            self.out_dim = self.sd_module.out_features
+        elif is_conv:
+            self.out_dim = self.sd_module.out_channels
+        elif is_other_linear:
+            self.out_dim = self.sd_module.embed_dim
+
+        if self.is_kohya:
+            self.constraint = self.alpha * self.out_dim
+            self.num_blocks = self.dim
+            self.block_size = self.out_dim // self.dim
+        else:
+            self.constraint = None
+            self.block_size, self.num_blocks = factorization(self.out_dim, self.dim)
+
+    def calc_updown(self, orig_weight):
+        oft_blocks = self.oft_blocks.to(orig_weight.device, dtype=orig_weight.dtype)
+        eye = torch.eye(self.block_size, device=self.oft_blocks.device)
+
+        if self.is_kohya:
+            block_Q = oft_blocks - oft_blocks.transpose(1, 2) # ensure skew-symmetric orthogonal matrix
+            norm_Q = torch.norm(block_Q.flatten())
+            new_norm_Q = torch.clamp(norm_Q, max=self.constraint)
+            block_Q = block_Q * ((new_norm_Q + 1e-8) / (norm_Q + 1e-8))
+            oft_blocks = torch.matmul(eye + block_Q, (eye - block_Q).float().inverse())
+
+        R = oft_blocks.to(orig_weight.device, dtype=orig_weight.dtype)
+
+        # This errors out for MultiheadAttention, might need to be handled up-stream
+        merged_weight = rearrange(orig_weight, '(k n) ... -> k n ...', k=self.num_blocks, n=self.block_size)
+        merged_weight = torch.einsum(
+            'k n m, k n ... -> k m ...',
+            R,
+            merged_weight
+        )
+        merged_weight = rearrange(merged_weight, 'k m ... -> (k m) ...')
+
+        updown = merged_weight.to(orig_weight.device, dtype=orig_weight.dtype) - orig_weight
+        output_shape = orig_weight.shape
+        return self.finalize_updown(updown, orig_weight, output_shape)
diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py
index 60d8dec4..72ebd624 100644
--- a/extensions-builtin/Lora/networks.py
+++ b/extensions-builtin/Lora/networks.py
@@ -1,3 +1,4 @@
+import gradio as gr

 import logging

 import os

 import re

@@ -11,6 +12,7 @@ import network_ia3
 import network_lokr

 import network_full

 import network_norm

+import network_oft

 

 import torch

 from typing import Union

@@ -28,6 +30,7 @@ module_types = [
     network_full.ModuleTypeFull(),

     network_norm.ModuleTypeNorm(),

     network_glora.ModuleTypeGLora(),

+    network_oft.ModuleTypeOFT(),

 ]

 

 

@@ -157,7 +160,8 @@ def load_network(name, network_on_disk):
     bundle_embeddings = {}

 

     for key_network, weight in sd.items():

-        key_network_without_network_parts, network_part = key_network.split(".", 1)

+        key_network_without_network_parts, _, network_part = key_network.partition(".")

+

         if key_network_without_network_parts == "bundle_emb":

             emb_name, vec_name = network_part.split(".", 1)

             emb_dict = bundle_embeddings.get(emb_name, {})

@@ -189,6 +193,17 @@ def load_network(name, network_on_disk):
                 key = key_network_without_network_parts.replace("lora_te1_text_model", "transformer_text_model")

                 sd_module = shared.sd_model.network_layer_mapping.get(key, None)

 

+        # kohya_ss OFT module

+        elif sd_module is None and "oft_unet" in key_network_without_network_parts:

+            key = key_network_without_network_parts.replace("oft_unet", "diffusion_model")

+            sd_module = shared.sd_model.network_layer_mapping.get(key, None)

+

+        # KohakuBlueLeaf OFT module

+        if sd_module is None and "oft_diag" in key:

+            key = key_network_without_network_parts.replace("lora_unet", "diffusion_model")

+            key = key_network_without_network_parts.replace("lora_te1_text_model", "0_transformer_text_model")

+            sd_module = shared.sd_model.network_layer_mapping.get(key, None)

+

         if sd_module is None:

             keys_failed_to_match[key_network] = key

             continue

@@ -300,7 +315,12 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No
                 emb_db.skipped_embeddings[name] = embedding

 

     if failed_to_load_networks:

-        sd_hijack.model_hijack.comments.append("Networks not found: " + ", ".join(failed_to_load_networks))

+        lora_not_found_message = f'Lora not found: {", ".join(failed_to_load_networks)}'

+        sd_hijack.model_hijack.comments.append(lora_not_found_message)

+        if shared.opts.lora_not_found_warning_console:

+            print(f'\n{lora_not_found_message}\n')

+        if shared.opts.lora_not_found_gradio_warning:

+            gr.Warning(lora_not_found_message)

 

     purge_networks_from_memory()

 

@@ -375,18 +395,26 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn
             if module is not None and hasattr(self, 'weight'):

                 try:

                     with torch.no_grad():

-                        updown, ex_bias = module.calc_updown(self.weight)

+                        if getattr(self, 'fp16_weight', None) is None:

+                            weight = self.weight

+                            bias = self.bias

+                        else:

+                            weight = self.fp16_weight.clone().to(self.weight.device)

+                            bias = getattr(self, 'fp16_bias', None)

+                            if bias is not None:

+                                bias = bias.clone().to(self.bias.device)

+                        updown, ex_bias = module.calc_updown(weight)

 

-                        if len(self.weight.shape) == 4 and self.weight.shape[1] == 9:

+                        if len(weight.shape) == 4 and weight.shape[1] == 9:

                             # inpainting model. zero pad updown to make channel[1]  4 to 9

                             updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5))

 

-                        self.weight += updown

+                        self.weight.copy_((weight.to(dtype=updown.dtype) + updown).to(dtype=self.weight.dtype))

                         if ex_bias is not None and hasattr(self, 'bias'):

                             if self.bias is None:

-                                self.bias = torch.nn.Parameter(ex_bias)

+                                self.bias = torch.nn.Parameter(ex_bias).to(self.weight.dtype)

                             else:

-                                self.bias += ex_bias

+                                self.bias.copy_((bias + ex_bias).to(dtype=self.bias.dtype))

                 except RuntimeError as e:

                     logging.debug(f"Network {net.name} layer {network_layer_name}: {e}")

                     extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1

diff --git a/extensions-builtin/Lora/scripts/lora_script.py b/extensions-builtin/Lora/scripts/lora_script.py
index ef23968c..1518f7e5 100644
--- a/extensions-builtin/Lora/scripts/lora_script.py
+++ b/extensions-builtin/Lora/scripts/lora_script.py
@@ -39,6 +39,8 @@ shared.options_templates.update(shared.options_section(('extra_networks', "Extra
     "lora_show_all": shared.OptionInfo(False, "Always show all networks on the Lora page").info("otherwise, those detected as for incompatible version of Stable Diffusion will be hidden"),

     "lora_hide_unknown_for_versions": shared.OptionInfo([], "Hide networks of unknown versions for model versions", gr.CheckboxGroup, {"choices": ["SD1", "SD2", "SDXL"]}),

     "lora_in_memory_limit": shared.OptionInfo(0, "Number of Lora networks to keep cached in memory", gr.Number, {"precision": 0}),

+    "lora_not_found_warning_console": shared.OptionInfo(False, "Lora not found warning in console"),

+    "lora_not_found_gradio_warning": shared.OptionInfo(False, "Lora not found warning popup in webui"),

 }))

 

 

diff --git a/extensions-builtin/Lora/ui_edit_user_metadata.py b/extensions-builtin/Lora/ui_edit_user_metadata.py
index c7011909..3160aecf 100644
--- a/extensions-builtin/Lora/ui_edit_user_metadata.py
+++ b/extensions-builtin/Lora/ui_edit_user_metadata.py
@@ -54,12 +54,13 @@ class LoraUserMetadataEditor(ui_extra_networks_user_metadata.UserMetadataEditor)
         self.slider_preferred_weight = None

         self.edit_notes = None

 

-    def save_lora_user_metadata(self, name, desc, sd_version, activation_text, preferred_weight, notes):

+    def save_lora_user_metadata(self, name, desc, sd_version, activation_text, preferred_weight, negative_text, notes):

         user_metadata = self.get_user_metadata(name)

         user_metadata["description"] = desc

         user_metadata["sd version"] = sd_version

         user_metadata["activation text"] = activation_text

         user_metadata["preferred weight"] = preferred_weight

+        user_metadata["negative text"] = negative_text

         user_metadata["notes"] = notes

 

         self.write_user_metadata(name, user_metadata)

@@ -127,6 +128,7 @@ class LoraUserMetadataEditor(ui_extra_networks_user_metadata.UserMetadataEditor)
             gr.HighlightedText.update(value=gradio_tags, visible=True if tags else False),

             user_metadata.get('activation text', ''),

             float(user_metadata.get('preferred weight', 0.0)),

+            user_metadata.get('negative text', ''),

             gr.update(visible=True if tags else False),

             gr.update(value=self.generate_random_prompt_from_tags(tags), visible=True if tags else False),

         ]

@@ -162,7 +164,7 @@ class LoraUserMetadataEditor(ui_extra_networks_user_metadata.UserMetadataEditor)
         self.taginfo = gr.HighlightedText(label="Training dataset tags")

         self.edit_activation_text = gr.Text(label='Activation text', info="Will be added to prompt along with Lora")

         self.slider_preferred_weight = gr.Slider(label='Preferred weight', info="Set to 0 to disable", minimum=0.0, maximum=2.0, step=0.01)

-

+        self.edit_negative_text = gr.Text(label='Negative prompt', info="Will be added to negative prompts")

         with gr.Row() as row_random_prompt:

             with gr.Column(scale=8):

                 random_prompt = gr.Textbox(label='Random prompt', lines=4, max_lines=4, interactive=False)

@@ -198,6 +200,7 @@ class LoraUserMetadataEditor(ui_extra_networks_user_metadata.UserMetadataEditor)
             self.taginfo,

             self.edit_activation_text,

             self.slider_preferred_weight,

+            self.edit_negative_text,

             row_random_prompt,

             random_prompt,

         ]

@@ -211,7 +214,9 @@ class LoraUserMetadataEditor(ui_extra_networks_user_metadata.UserMetadataEditor)
             self.select_sd_version,

             self.edit_activation_text,

             self.slider_preferred_weight,

+            self.edit_negative_text,

             self.edit_notes,

         ]

 

+

         self.setup_save_handler(self.button_save, self.save_lora_user_metadata, edited_components)

diff --git a/extensions-builtin/Lora/ui_extra_networks_lora.py b/extensions-builtin/Lora/ui_extra_networks_lora.py
index 55409a78..e714fac4 100644
--- a/extensions-builtin/Lora/ui_extra_networks_lora.py
+++ b/extensions-builtin/Lora/ui_extra_networks_lora.py
@@ -17,6 +17,8 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
 

     def create_item(self, name, index=None, enable_filter=True):

         lora_on_disk = networks.available_networks.get(name)

+        if lora_on_disk is None:

+            return

 

         path, ext = os.path.splitext(lora_on_disk.filename)

 

@@ -43,6 +45,11 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
         if activation_text:

             item["prompt"] += " + " + quote_js(" " + activation_text)

 

+        negative_prompt = item["user_metadata"].get("negative text")

+        item["negative_prompt"] = quote_js("")

+        if negative_prompt:

+            item["negative_prompt"] = quote_js('(' + negative_prompt + ':1)')

+

         sd_version = item["user_metadata"].get("sd version")

         if sd_version in network.SdVersion.__members__:

             item["sd_version"] = sd_version

@@ -66,9 +73,10 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
         return item

 

     def list_items(self):

-        for index, name in enumerate(networks.available_networks):

+        # instantiate a list to protect against concurrent modification

+        names = list(networks.available_networks)

+        for index, name in enumerate(names):

             item = self.create_item(name, index)

-

             if item is not None:

                 yield item