24 files changed, 315 insertions, 169 deletions

diff --git a/modules/devices.py b/modules/devices.py
index 655ca1d3..52c3e7cd 100644
--- a/modules/devices.py
+++ b/modules/devices.py
@@ -1,21 +1,17 @@
-import sys, os, shlex
+import sys
 import contextlib
 import torch
 from modules import errors
-from packaging import version
+
+if sys.platform == "darwin":
+    from modules import mac_specific
 
 
-# has_mps is only available in nightly pytorch (for now) and macOS 12.3+.
-# check `getattr` and try it for compatibility
 def has_mps() -> bool:
-    if not getattr(torch, 'has_mps', False):
-        return False
-    try:
-        torch.zeros(1).to(torch.device("mps"))
-        return True
-    except Exception:
+    if sys.platform != "darwin":
         return False
-
+    else:
+        return mac_specific.has_mps
 
 def extract_device_id(args, name):
     for x in range(len(args)):
@@ -154,56 +150,3 @@ def test_for_nans(x, where):
     message += " Use --disable-nan-check commandline argument to disable this check."
 
     raise NansException(message)
-
-
-# MPS workaround for https://github.com/pytorch/pytorch/issues/79383
-orig_tensor_to = torch.Tensor.to
-def tensor_to_fix(self, *args, **kwargs):
-    if self.device.type != 'mps' and \
-       ((len(args) > 0 and isinstance(args[0], torch.device) and args[0].type == 'mps') or \
-       (isinstance(kwargs.get('device'), torch.device) and kwargs['device'].type == 'mps')):
-        self = self.contiguous()
-    return orig_tensor_to(self, *args, **kwargs)
-
-
-# MPS workaround for https://github.com/pytorch/pytorch/issues/80800 
-orig_layer_norm = torch.nn.functional.layer_norm
-def layer_norm_fix(*args, **kwargs):
-    if len(args) > 0 and isinstance(args[0], torch.Tensor) and args[0].device.type == 'mps':
-        args = list(args)
-        args[0] = args[0].contiguous()
-    return orig_layer_norm(*args, **kwargs)
-
-
-# MPS workaround for https://github.com/pytorch/pytorch/issues/90532
-orig_tensor_numpy = torch.Tensor.numpy
-def numpy_fix(self, *args, **kwargs):
-    if self.requires_grad:
-        self = self.detach()
-    return orig_tensor_numpy(self, *args, **kwargs)
-
-
-# MPS workaround for https://github.com/pytorch/pytorch/issues/89784
-orig_cumsum = torch.cumsum
-orig_Tensor_cumsum = torch.Tensor.cumsum
-def cumsum_fix(input, cumsum_func, *args, **kwargs):
-    if input.device.type == 'mps':
-        output_dtype = kwargs.get('dtype', input.dtype)
-        if output_dtype == torch.int64:
-            return cumsum_func(input.cpu(), *args, **kwargs).to(input.device)
-        elif cumsum_needs_bool_fix and output_dtype == torch.bool or cumsum_needs_int_fix and (output_dtype == torch.int8 or output_dtype == torch.int16):
-            return cumsum_func(input.to(torch.int32), *args, **kwargs).to(torch.int64)
-    return cumsum_func(input, *args, **kwargs)
-
-
-if has_mps():
-    if version.parse(torch.__version__) < version.parse("1.13"):
-        # PyTorch 1.13 doesn't need these fixes but unfortunately is slower and has regressions that prevent training from working
-        torch.Tensor.to = tensor_to_fix
-        torch.nn.functional.layer_norm = layer_norm_fix
-        torch.Tensor.numpy = numpy_fix
-    elif version.parse(torch.__version__) > version.parse("1.13.1"):
-        cumsum_needs_int_fix = not torch.Tensor([1,2]).to(torch.device("mps")).equal(torch.ShortTensor([1,1]).to(torch.device("mps")).cumsum(0))
-        cumsum_needs_bool_fix = not torch.BoolTensor([True,True]).to(device=torch.device("mps"), dtype=torch.int64).equal(torch.BoolTensor([True,False]).to(torch.device("mps")).cumsum(0))
-        torch.cumsum = lambda input, *args, **kwargs: ( cumsum_fix(input, orig_cumsum, *args, **kwargs) )
-        torch.Tensor.cumsum = lambda self, *args, **kwargs: ( cumsum_fix(self, orig_Tensor_cumsum, *args, **kwargs) )
diff --git a/modules/esrgan_model_arch.py b/modules/esrgan_model_arch.py
index bc9ceb2a..1b52b0f5 100644
--- a/modules/esrgan_model_arch.py
+++ b/modules/esrgan_model_arch.py
@@ -1,5 +1,6 @@
 # this file is adapted from https://github.com/victorca25/iNNfer

 

+from collections import OrderedDict

 import math

 import functools

 import torch

diff --git a/modules/extensions.py b/modules/extensions.py
index 5e12b1aa..3eef9eaf 100644
--- a/modules/extensions.py
+++ b/modules/extensions.py
@@ -2,6 +2,7 @@ import os
 import sys

 import traceback

 

+import time

 import git

 

 from modules import paths, shared

@@ -25,6 +26,7 @@ class Extension:
         self.status = ''

         self.can_update = False

         self.is_builtin = is_builtin

+        self.version = ''

 

         repo = None

         try:

@@ -40,6 +42,10 @@ class Extension:
             try:

                 self.remote = next(repo.remote().urls, None)

                 self.status = 'unknown'

+                head = repo.head.commit

+                ts = time.asctime(time.gmtime(repo.head.commit.committed_date))

+                self.version = f'{head.hexsha[:8]} ({ts})'

+

             except Exception:

                 self.remote = None

 

diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py
index fc9e17aa..89dc23bf 100644
--- a/modules/generation_parameters_copypaste.py
+++ b/modules/generation_parameters_copypaste.py
@@ -74,8 +74,8 @@ def image_from_url_text(filedata):
     return image

 

 

-def add_paste_fields(tabname, init_img, fields):

-    paste_fields[tabname] = {"init_img": init_img, "fields": fields}

+def add_paste_fields(tabname, init_img, fields, override_settings_component=None):

+    paste_fields[tabname] = {"init_img": init_img, "fields": fields, "override_settings_component": override_settings_component}

 

     # backwards compatibility for existing extensions

     import modules.ui

@@ -110,6 +110,7 @@ def connect_paste_params_buttons():
     for binding in registered_param_bindings:

         destination_image_component = paste_fields[binding.tabname]["init_img"]

         fields = paste_fields[binding.tabname]["fields"]

+        override_settings_component = binding.override_settings_component or paste_fields[binding.tabname]["override_settings_component"]

 

         destination_width_component = next(iter([field for field, name in fields if name == "Size-1"] if fields else []), None)

         destination_height_component = next(iter([field for field, name in fields if name == "Size-2"] if fields else []), None)

@@ -130,7 +131,7 @@ def connect_paste_params_buttons():
             )

 

         if binding.source_text_component is not None and fields is not None:

-            connect_paste(binding.paste_button, fields, binding.source_text_component, binding.override_settings_component, binding.tabname)

+            connect_paste(binding.paste_button, fields, binding.source_text_component, override_settings_component, binding.tabname)

 

         if binding.source_tabname is not None and fields is not None:

             paste_field_names = ['Prompt', 'Negative prompt', 'Steps', 'Face restoration'] + (["Seed"] if shared.opts.send_seed else [])

diff --git a/modules/hashes.py b/modules/hashes.py
index 819362a3..83272a07 100644
--- a/modules/hashes.py
+++ b/modules/hashes.py
@@ -4,6 +4,7 @@ import os.path
 

 import filelock

 

+from modules import shared

 from modules.paths import data_path

 

 

@@ -68,6 +69,9 @@ def sha256(filename, title):
     if sha256_value is not None:

         return sha256_value

 

+    if shared.cmd_opts.no_hashing:

+        return None

+

     print(f"Calculating sha256 for {filename}: ", end='')

     sha256_value = calculate_sha256(filename)

     print(f"{sha256_value}")

diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py
index 503534e2..a15bae18 100644
--- a/modules/hypernetworks/hypernetwork.py
+++ b/modules/hypernetworks/hypernetwork.py
@@ -307,7 +307,7 @@ class Hypernetwork:
     def shorthash(self):

         sha256 = hashes.sha256(self.filename, f'hypernet/{self.name}')

 

-        return sha256[0:10]

+        return sha256[0:10] if sha256 else None

 

 

 def list_hypernetworks(path):

@@ -380,8 +380,8 @@ def apply_single_hypernetwork(hypernetwork, context_k, context_v, layer=None):
         layer.hyper_k = hypernetwork_layers[0]

         layer.hyper_v = hypernetwork_layers[1]

 

-    context_k = hypernetwork_layers[0](context_k)

-    context_v = hypernetwork_layers[1](context_v)

+    context_k = devices.cond_cast_unet(hypernetwork_layers[0](devices.cond_cast_float(context_k)))

+    context_v = devices.cond_cast_unet(hypernetwork_layers[1](devices.cond_cast_float(context_v)))

     return context_k, context_v

 

 

diff --git a/modules/images.py b/modules/images.py
index ae3cdaf4..38404de3 100644
--- a/modules/images.py
+++ b/modules/images.py
@@ -16,8 +16,9 @@ from PIL import Image, ImageFont, ImageDraw, PngImagePlugin
 from fonts.ttf import Roboto

 import string

 import json

+import hashlib

 

-from modules import sd_samplers, shared, script_callbacks

+from modules import sd_samplers, shared, script_callbacks, errors

 from modules.shared import opts, cmd_opts

 

 LANCZOS = (Image.Resampling.LANCZOS if hasattr(Image, 'Resampling') else Image.LANCZOS)

@@ -130,7 +131,7 @@ class GridAnnotation:
         self.size = None

 

 

-def draw_grid_annotations(im, width, height, hor_texts, ver_texts):

+def draw_grid_annotations(im, width, height, hor_texts, ver_texts, margin=0):

     def wrap(drawing, text, font, line_length):

         lines = ['']

         for word in text.split():

@@ -194,32 +195,35 @@ def draw_grid_annotations(im, width, height, hor_texts, ver_texts):
             line.allowed_width = allowed_width

 

     hor_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing for lines in hor_texts]

-    ver_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing * len(lines) for lines in

-                        ver_texts]

+    ver_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing * len(lines) for lines in ver_texts]

 

     pad_top = 0 if sum(hor_text_heights) == 0 else max(hor_text_heights) + line_spacing * 2

 

-    result = Image.new("RGB", (im.width + pad_left, im.height + pad_top), "white")

-    result.paste(im, (pad_left, pad_top))

+    result = Image.new("RGB", (im.width + pad_left + margin * (cols-1), im.height + pad_top + margin * (rows-1)), "white")

+

+    for row in range(rows):

+        for col in range(cols):

+            cell = im.crop((width * col, height * row, width * (col+1), height * (row+1)))

+            result.paste(cell, (pad_left + (width + margin) * col, pad_top + (height + margin) * row))

 

     d = ImageDraw.Draw(result)

 

     for col in range(cols):

-        x = pad_left + width * col + width / 2

+        x = pad_left + (width + margin) * col + width / 2

         y = pad_top / 2 - hor_text_heights[col] / 2

 

         draw_texts(d, x, y, hor_texts[col], fnt, fontsize)

 

     for row in range(rows):

         x = pad_left / 2

-        y = pad_top + height * row + height / 2 - ver_text_heights[row] / 2

+        y = pad_top + (height + margin) * row + height / 2 - ver_text_heights[row] / 2

 

         draw_texts(d, x, y, ver_texts[row], fnt, fontsize)

 

     return result

 

 

-def draw_prompt_matrix(im, width, height, all_prompts):

+def draw_prompt_matrix(im, width, height, all_prompts, margin=0):

     prompts = all_prompts[1:]

     boundary = math.ceil(len(prompts) / 2)

 

@@ -229,7 +233,7 @@ def draw_prompt_matrix(im, width, height, all_prompts):
     hor_texts = [[GridAnnotation(x, is_active=pos & (1 << i) != 0) for i, x in enumerate(prompts_horiz)] for pos in range(1 << len(prompts_horiz))]

     ver_texts = [[GridAnnotation(x, is_active=pos & (1 << i) != 0) for i, x in enumerate(prompts_vert)] for pos in range(1 << len(prompts_vert))]

 

-    return draw_grid_annotations(im, width, height, hor_texts, ver_texts)

+    return draw_grid_annotations(im, width, height, hor_texts, ver_texts, margin)

 

 

 def resize_image(resize_mode, im, width, height, upscaler_name=None):

@@ -340,6 +344,7 @@ class FilenameGenerator:
         'date': lambda self: datetime.datetime.now().strftime('%Y-%m-%d'),

         'datetime': lambda self, *args: self.datetime(*args),  # accepts formats: [datetime], [datetime<Format>], [datetime<Format><Time Zone>]

         'job_timestamp': lambda self: getattr(self.p, "job_timestamp", shared.state.job_timestamp),

+        'prompt_hash': lambda self: hashlib.sha256(self.prompt.encode()).hexdigest()[0:8],

         'prompt': lambda self: sanitize_filename_part(self.prompt),

         'prompt_no_styles': lambda self: self.prompt_no_style(),

         'prompt_spaces': lambda self: sanitize_filename_part(self.prompt, replace_spaces=False),

@@ -548,6 +553,8 @@ def save_image(image, path, basename, seed=None, prompt=None, extension='png', i
         elif extension.lower() in (".jpg", ".jpeg", ".webp"):

             if image_to_save.mode == 'RGBA':

                 image_to_save = image_to_save.convert("RGB")

+            elif image_to_save.mode == 'I;16':

+                image_to_save = image_to_save.point(lambda p: p * 0.0038910505836576).convert("RGB" if extension.lower() == ".webp" else "L")

 

             image_to_save.save(temp_file_path, format=image_format, quality=opts.jpeg_quality)

 

@@ -570,17 +577,19 @@ def save_image(image, path, basename, seed=None, prompt=None, extension='png', i
 

     image.already_saved_as = fullfn

 

-    target_side_length = 4000

-    oversize = image.width > target_side_length or image.height > target_side_length

-    if opts.export_for_4chan and (oversize or os.stat(fullfn).st_size > 4 * 1024 * 1024):

+    oversize = image.width > opts.target_side_length or image.height > opts.target_side_length

+    if opts.export_for_4chan and (oversize or os.stat(fullfn).st_size > opts.img_downscale_threshold * 1024 * 1024):

         ratio = image.width / image.height

 

         if oversize and ratio > 1:

-            image = image.resize((target_side_length, image.height * target_side_length // image.width), LANCZOS)

+            image = image.resize((opts.target_side_length, image.height * opts.target_side_length // image.width), LANCZOS)

         elif oversize:

-            image = image.resize((image.width * target_side_length // image.height, target_side_length), LANCZOS)

+            image = image.resize((image.width * opts.target_side_length // image.height, opts.target_side_length), LANCZOS)

 

-        _atomically_save_image(image, fullfn_without_extension, ".jpg")

+        try:

+            _atomically_save_image(image, fullfn_without_extension, ".jpg")

+        except Exception as e:

+            errors.display(e, "saving image as downscaled JPG")

 

     if opts.save_txt and info is not None:

         txt_fullfn = f"{fullfn_without_extension}.txt"

diff --git a/modules/img2img.py b/modules/img2img.py
index f813299c..c973b770 100644
--- a/modules/img2img.py
+++ b/modules/img2img.py
@@ -73,10 +73,12 @@ def process_batch(p, input_dir, output_dir, inpaint_mask_dir, args):
 

             if not save_normally:

                 os.makedirs(output_dir, exist_ok=True)

+                if processed_image.mode == 'RGBA':

+                    processed_image = processed_image.convert("RGB")

                 processed_image.save(os.path.join(output_dir, filename))

 

 

-def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_styles, init_img, sketch, init_img_with_mask, inpaint_color_sketch, inpaint_color_sketch_orig, init_img_inpaint, init_mask_inpaint, steps: int, sampler_index: int, mask_blur: int, mask_alpha: float, inpainting_fill: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, denoising_strength: float, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, seed_enable_extras: bool, height: int, width: int, resize_mode: int, inpaint_full_res: bool, inpaint_full_res_padding: int, inpainting_mask_invert: int, img2img_batch_input_dir: str, img2img_batch_output_dir: str, img2img_batch_inpaint_mask_dir: str, override_settings_texts, *args):

+def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_styles, init_img, sketch, init_img_with_mask, inpaint_color_sketch, inpaint_color_sketch_orig, init_img_inpaint, init_mask_inpaint, steps: int, sampler_index: int, mask_blur: int, mask_alpha: float, inpainting_fill: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, denoising_strength: float, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, seed_enable_extras: bool, height: int, width: int, resize_mode: int, inpaint_full_res: bool, inpaint_full_res_padding: int, inpainting_mask_invert: int, img2img_batch_input_dir: str, img2img_batch_output_dir: str, img2img_batch_inpaint_mask_dir: str, override_settings_texts, *args):

     override_settings = create_override_settings_dict(override_settings_texts)

 

     is_batch = mode == 5

@@ -142,6 +144,7 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s
         inpainting_fill=inpainting_fill,

         resize_mode=resize_mode,

         denoising_strength=denoising_strength,

+        image_cfg_scale=image_cfg_scale,

         inpaint_full_res=inpaint_full_res,

         inpaint_full_res_padding=inpaint_full_res_padding,

         inpainting_mask_invert=inpainting_mask_invert,

diff --git a/modules/mac_specific.py b/modules/mac_specific.py
new file mode 100644
index 00000000..ddcea53b
--- /dev/null
+++ b/modules/mac_specific.py
@@ -0,0 +1,53 @@
+import torch
+from modules import paths
+from modules.sd_hijack_utils import CondFunc
+from packaging import version
+
+
+# has_mps is only available in nightly pytorch (for now) and macOS 12.3+.
+# check `getattr` and try it for compatibility
+def check_for_mps() -> bool:
+    if not getattr(torch, 'has_mps', False):
+        return False
+    try:
+        torch.zeros(1).to(torch.device("mps"))
+        return True
+    except Exception:
+        return False
+has_mps = check_for_mps()
+
+
+# MPS workaround for https://github.com/pytorch/pytorch/issues/89784
+def cumsum_fix(input, cumsum_func, *args, **kwargs):
+    if input.device.type == 'mps':
+        output_dtype = kwargs.get('dtype', input.dtype)
+        if output_dtype == torch.int64:
+            return cumsum_func(input.cpu(), *args, **kwargs).to(input.device)
+        elif cumsum_needs_bool_fix and output_dtype == torch.bool or cumsum_needs_int_fix and (output_dtype == torch.int8 or output_dtype == torch.int16):
+            return cumsum_func(input.to(torch.int32), *args, **kwargs).to(torch.int64)
+    return cumsum_func(input, *args, **kwargs)
+
+
+if has_mps:
+    # MPS fix for randn in torchsde
+    CondFunc('torchsde._brownian.brownian_interval._randn', lambda _, size, dtype, device, seed: torch.randn(size, dtype=dtype, device=torch.device("cpu"), generator=torch.Generator(torch.device("cpu")).manual_seed(int(seed))).to(device), lambda _, size, dtype, device, seed: device.type == 'mps')
+
+    if version.parse(torch.__version__) < version.parse("1.13"):
+        # PyTorch 1.13 doesn't need these fixes but unfortunately is slower and has regressions that prevent training from working
+
+        # MPS workaround for https://github.com/pytorch/pytorch/issues/79383
+        CondFunc('torch.Tensor.to', lambda orig_func, self, *args, **kwargs: orig_func(self.contiguous(), *args, **kwargs),
+                                                          lambda _, self, *args, **kwargs: self.device.type != 'mps' and (args and isinstance(args[0], torch.device) and args[0].type == 'mps' or isinstance(kwargs.get('device'), torch.device) and kwargs['device'].type == 'mps'))
+        # MPS workaround for https://github.com/pytorch/pytorch/issues/80800 
+        CondFunc('torch.nn.functional.layer_norm', lambda orig_func, *args, **kwargs: orig_func(*([args[0].contiguous()] + list(args[1:])), **kwargs),
+                                                                                        lambda _, *args, **kwargs: args and isinstance(args[0], torch.Tensor) and args[0].device.type == 'mps')
+        # MPS workaround for https://github.com/pytorch/pytorch/issues/90532
+        CondFunc('torch.Tensor.numpy', lambda orig_func, self, *args, **kwargs: orig_func(self.detach(), *args, **kwargs), lambda _, self, *args, **kwargs: self.requires_grad)
+    elif version.parse(torch.__version__) > version.parse("1.13.1"):
+        cumsum_needs_int_fix = not torch.Tensor([1,2]).to(torch.device("mps")).equal(torch.ShortTensor([1,1]).to(torch.device("mps")).cumsum(0))
+        cumsum_needs_bool_fix = not torch.BoolTensor([True,True]).to(device=torch.device("mps"), dtype=torch.int64).equal(torch.BoolTensor([True,False]).to(torch.device("mps")).cumsum(0))
+        cumsum_fix_func = lambda orig_func, input, *args, **kwargs: cumsum_fix(input, orig_func, *args, **kwargs)
+        CondFunc('torch.cumsum', cumsum_fix_func, None)
+        CondFunc('torch.Tensor.cumsum', cumsum_fix_func, None)
+        CondFunc('torch.narrow', lambda orig_func, *args, **kwargs: orig_func(*args, **kwargs).clone(), None)
+
diff --git a/modules/modelloader.py b/modules/modelloader.py
index e9aa514e..fc3f6249 100644
--- a/modules/modelloader.py
+++ b/modules/modelloader.py
@@ -45,6 +45,9 @@ def load_models(model_path: str, model_url: str = None, command_path: str = None
                     full_path = file
                     if os.path.isdir(full_path):
                         continue
+                    if os.path.islink(full_path) and not os.path.exists(full_path):
+                        print(f"Skipping broken symlink: {full_path}")
+                        continue
                     if ext_blacklist is not None and any([full_path.endswith(x) for x in ext_blacklist]):
                         continue
                     if len(ext_filter) != 0:
diff --git a/modules/processing.py b/modules/processing.py
index e544c2e1..2009d3bf 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -186,7 +186,7 @@ class StableDiffusionProcessing:
         return conditioning

 

     def edit_image_conditioning(self, source_image):

-        conditioning_image = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(source_image))

+        conditioning_image = self.sd_model.encode_first_stage(source_image).mode()

 

         return conditioning_image

 

@@ -268,6 +268,7 @@ class Processed:
         self.height = p.height

         self.sampler_name = p.sampler_name

         self.cfg_scale = p.cfg_scale

+        self.image_cfg_scale = getattr(p, 'image_cfg_scale', None)

         self.steps = p.steps

         self.batch_size = p.batch_size

         self.restore_faces = p.restore_faces

@@ -445,6 +446,7 @@ def create_infotext(p, all_prompts, all_seeds, all_subseeds, comments=None, iter
         "Steps": p.steps,

         "Sampler": p.sampler_name,

         "CFG scale": p.cfg_scale,

+        "Image CFG scale": getattr(p, 'image_cfg_scale', None),

         "Seed": all_seeds[index],

         "Face restoration": (opts.face_restoration_model if p.restore_faces else None),

         "Size": f"{p.width}x{p.height}",

@@ -541,8 +543,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
     if os.path.exists(cmd_opts.embeddings_dir) and not p.do_not_reload_embeddings:

         model_hijack.embedding_db.load_textual_inversion_embeddings()

 

-    _, extra_network_data = extra_networks.parse_prompts(p.all_prompts[0:1])

-

     if p.scripts is not None:

         p.scripts.process(p)

 

@@ -580,13 +580,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
             if shared.opts.live_previews_enable and opts.show_progress_type == "Approx NN":

                 sd_vae_approx.model()

 

-            if not p.disable_extra_networks:

-                extra_networks.activate(p, extra_network_data)

-

-        with open(os.path.join(paths.data_path, "params.txt"), "w", encoding="utf8") as file:

-            processed = Processed(p, [], p.seed, "")

-            file.write(processed.infotext(p, 0))

-

         if state.job_count == -1:

             state.job_count = p.n_iter

 

@@ -607,11 +600,24 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
             if len(prompts) == 0:

                 break

 

-            prompts, _ = extra_networks.parse_prompts(prompts)

+            prompts, extra_network_data = extra_networks.parse_prompts(prompts)

+

+            if not p.disable_extra_networks:

+                with devices.autocast():

+                    extra_networks.activate(p, extra_network_data)

 

             if p.scripts is not None:

                 p.scripts.process_batch(p, batch_number=n, prompts=prompts, seeds=seeds, subseeds=subseeds)

 

+            # params.txt should be saved after scripts.process_batch, since the

+            # infotext could be modified by that callback

+            # Example: a wildcard processed by process_batch sets an extra model

+            # strength, which is saved as "Model Strength: 1.0" in the infotext

+            if n == 0:

+                with open(os.path.join(paths.data_path, "params.txt"), "w", encoding="utf8") as file:

+                    processed = Processed(p, [], p.seed, "")

+                    file.write(processed.infotext(p, 0))

+

             uc = get_conds_with_caching(prompt_parser.get_learned_conditioning, negative_prompts, p.steps, cached_uc)

             c = get_conds_with_caching(prompt_parser.get_multicond_learned_conditioning, prompts, p.steps, cached_c)

 

@@ -901,12 +907,13 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
 class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):

     sampler = None

 

-    def __init__(self, init_images: list = None, resize_mode: int = 0, denoising_strength: float = 0.75, mask: Any = None, mask_blur: int = 4, inpainting_fill: int = 0, inpaint_full_res: bool = True, inpaint_full_res_padding: int = 0, inpainting_mask_invert: int = 0, initial_noise_multiplier: float = None, **kwargs):

+    def __init__(self, init_images: list = None, resize_mode: int = 0, denoising_strength: float = 0.75, image_cfg_scale: float = None, mask: Any = None, mask_blur: int = 4, inpainting_fill: int = 0, inpaint_full_res: bool = True, inpaint_full_res_padding: int = 0, inpainting_mask_invert: int = 0, initial_noise_multiplier: float = None, **kwargs):

         super().__init__(**kwargs)

 

         self.init_images = init_images

         self.resize_mode: int = resize_mode

         self.denoising_strength: float = denoising_strength

+        self.image_cfg_scale: float = image_cfg_scale if shared.sd_model.cond_stage_key == "edit" else None

         self.init_latent = None

         self.image_mask = mask

         self.latent_mask = None

diff --git a/modules/script_callbacks.py b/modules/script_callbacks.py
index 4bb45ec7..edd0e2a7 100644
--- a/modules/script_callbacks.py
+++ b/modules/script_callbacks.py
@@ -46,6 +46,18 @@ class CFGDenoiserParams:
         """Total number of sampling steps planned"""

 

 

+class CFGDenoisedParams:

+    def __init__(self, x, sampling_step, total_sampling_steps):

+        self.x = x

+        """Latent image representation in the process of being denoised"""

+

+        self.sampling_step = sampling_step

+        """Current Sampling step number"""

+

+        self.total_sampling_steps = total_sampling_steps

+        """Total number of sampling steps planned"""

+

+

 class UiTrainTabParams:

     def __init__(self, txt2img_preview_params):

         self.txt2img_preview_params = txt2img_preview_params

@@ -68,6 +80,7 @@ callback_map = dict(
     callbacks_before_image_saved=[],

     callbacks_image_saved=[],

     callbacks_cfg_denoiser=[],

+    callbacks_cfg_denoised=[],

     callbacks_before_component=[],

     callbacks_after_component=[],

     callbacks_image_grid=[],

@@ -150,6 +163,14 @@ def cfg_denoiser_callback(params: CFGDenoiserParams):
             report_exception(c, 'cfg_denoiser_callback')

 

 

+def cfg_denoised_callback(params: CFGDenoisedParams):

+    for c in callback_map['callbacks_cfg_denoised']:

+        try:

+            c.callback(params)

+        except Exception:

+            report_exception(c, 'cfg_denoised_callback')

+

+

 def before_component_callback(component, **kwargs):

     for c in callback_map['callbacks_before_component']:

         try:

@@ -283,6 +304,14 @@ def on_cfg_denoiser(callback):
     add_callback(callback_map['callbacks_cfg_denoiser'], callback)

 

 

+def on_cfg_denoised(callback):

+    """register a function to be called in the kdiffussion cfg_denoiser method after building the inner model inputs.

+    The callback is called with one argument:

+        - params: CFGDenoisedParams - parameters to be passed to the inner model and sampling state details.

+    """

+    add_callback(callback_map['callbacks_cfg_denoised'], callback)

+

+

 def on_before_component(callback):

     """register a function to be called before a component is created.

     The callback is called with arguments:

diff --git a/modules/sd_disable_initialization.py b/modules/sd_disable_initialization.py
index e90aa9fe..c4a09d15 100644
--- a/modules/sd_disable_initialization.py
+++ b/modules/sd_disable_initialization.py
@@ -20,8 +20,9 @@ class DisableInitialization:
     ```

     """

 

-    def __init__(self):

+    def __init__(self, disable_clip=True):

         self.replaced = []

+        self.disable_clip = disable_clip

 

     def replace(self, obj, field, func):

         original = getattr(obj, field, None)

@@ -75,12 +76,14 @@ class DisableInitialization:
         self.replace(torch.nn.init, 'kaiming_uniform_', do_nothing)

         self.replace(torch.nn.init, '_no_grad_normal_', do_nothing)

         self.replace(torch.nn.init, '_no_grad_uniform_', do_nothing)

-        self.create_model_and_transforms = self.replace(open_clip, 'create_model_and_transforms', create_model_and_transforms_without_pretrained)

-        self.CLIPTextModel_from_pretrained = self.replace(ldm.modules.encoders.modules.CLIPTextModel, 'from_pretrained', CLIPTextModel_from_pretrained)

-        self.transformers_modeling_utils_load_pretrained_model = self.replace(transformers.modeling_utils.PreTrainedModel, '_load_pretrained_model', transformers_modeling_utils_load_pretrained_model)

-        self.transformers_tokenization_utils_base_cached_file = self.replace(transformers.tokenization_utils_base, 'cached_file', transformers_tokenization_utils_base_cached_file)

-        self.transformers_configuration_utils_cached_file = self.replace(transformers.configuration_utils, 'cached_file', transformers_configuration_utils_cached_file)

-        self.transformers_utils_hub_get_from_cache = self.replace(transformers.utils.hub, 'get_from_cache', transformers_utils_hub_get_from_cache)

+

+        if self.disable_clip:

+            self.create_model_and_transforms = self.replace(open_clip, 'create_model_and_transforms', create_model_and_transforms_without_pretrained)

+            self.CLIPTextModel_from_pretrained = self.replace(ldm.modules.encoders.modules.CLIPTextModel, 'from_pretrained', CLIPTextModel_from_pretrained)

+            self.transformers_modeling_utils_load_pretrained_model = self.replace(transformers.modeling_utils.PreTrainedModel, '_load_pretrained_model', transformers_modeling_utils_load_pretrained_model)

+            self.transformers_tokenization_utils_base_cached_file = self.replace(transformers.tokenization_utils_base, 'cached_file', transformers_tokenization_utils_base_cached_file)

+            self.transformers_configuration_utils_cached_file = self.replace(transformers.configuration_utils, 'cached_file', transformers_configuration_utils_cached_file)

+            self.transformers_utils_hub_get_from_cache = self.replace(transformers.utils.hub, 'get_from_cache', transformers_utils_hub_get_from_cache)

 

     def __exit__(self, exc_type, exc_val, exc_tb):

         for obj, field, original in self.replaced:

diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index 8fdc5990..fca418cd 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -104,6 +104,9 @@ class StableDiffusionModelHijack:
             m.cond_stage_model.model.token_embedding = EmbeddingsWithFixes(m.cond_stage_model.model.token_embedding, self)

             m.cond_stage_model = sd_hijack_open_clip.FrozenOpenCLIPEmbedderWithCustomWords(m.cond_stage_model, self)

 

+        if m.cond_stage_key == "edit":

+            sd_hijack_unet.hijack_ddpm_edit()

+

         self.optimization_method = apply_optimizations()

 

         self.clip = m.cond_stage_model

diff --git a/modules/sd_hijack_inpainting.py b/modules/sd_hijack_inpainting.py
index 478cd499..55a2ce4d 100644
--- a/modules/sd_hijack_inpainting.py
+++ b/modules/sd_hijack_inpainting.py
@@ -11,6 +11,7 @@ import ldm.models.diffusion.plms
 from ldm.models.diffusion.ddpm import LatentDiffusion
 from ldm.models.diffusion.plms import PLMSSampler