From 8a34671fe91e142bce9e5556cca2258b3be9dd6e Mon Sep 17 00:00:00 2001
From: MrCheeze <fishycheeze@yahoo.ca>
Date: Fri, 24 Mar 2023 22:48:16 -0400
Subject: Add support for the Variations models (unclip-h and unclip-l)

---
 modules/sd_samplers_kdiffusion.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

(limited to 'modules/sd_samplers_kdiffusion.py')

diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py
index 93f0e55a..e9f08518 100644
--- a/modules/sd_samplers_kdiffusion.py
+++ b/modules/sd_samplers_kdiffusion.py
@@ -92,14 +92,21 @@ class CFGDenoiser(torch.nn.Module):
         batch_size = len(conds_list)
         repeats = [len(conds_list[i]) for i in range(batch_size)]
 
+        if shared.sd_model.model.conditioning_key == "crossattn-adm":
+            image_uncond = torch.zeros_like(image_cond)
+            make_condition_dict = lambda c_crossattn, c_adm: {"c_crossattn": c_crossattn, "c_adm": c_adm} 
+        else:
+            image_uncond = image_cond
+            make_condition_dict = lambda c_crossattn, c_concat: {"c_crossattn": c_crossattn, "c_concat": [c_concat]} 
+
         if not is_edit_model:
             x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x])
             sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma])
-            image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond])
+            image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_uncond])
         else:
             x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x] + [x])
             sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma] + [sigma])
-            image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond] + [torch.zeros_like(self.init_latent)])
+            image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_uncond] + [torch.zeros_like(self.init_latent)])
 
         denoiser_params = CFGDenoiserParams(x_in, image_cond_in, sigma_in, state.sampling_step, state.sampling_steps, tensor, uncond)
         cfg_denoiser_callback(denoiser_params)
@@ -116,13 +123,13 @@ class CFGDenoiser(torch.nn.Module):
                 cond_in = torch.cat([tensor, uncond, uncond])
 
             if shared.batch_cond_uncond:
-                x_out = self.inner_model(x_in, sigma_in, cond={"c_crossattn": [cond_in], "c_concat": [image_cond_in]})
+                x_out = self.inner_model(x_in, sigma_in, cond=make_condition_dict([cond_in], image_cond_in))
             else:
                 x_out = torch.zeros_like(x_in)
                 for batch_offset in range(0, x_out.shape[0], batch_size):
                     a = batch_offset
                     b = a + batch_size
-                    x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond={"c_crossattn": [cond_in[a:b]], "c_concat": [image_cond_in[a:b]]})
+                    x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond=make_condition_dict([cond_in[a:b]], image_cond_in[a:b]))
         else:
             x_out = torch.zeros_like(x_in)
             batch_size = batch_size*2 if shared.batch_cond_uncond else batch_size
@@ -135,9 +142,9 @@ class CFGDenoiser(torch.nn.Module):
                 else:
                     c_crossattn = torch.cat([tensor[a:b]], uncond)
 
-                x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond={"c_crossattn": c_crossattn, "c_concat": [image_cond_in[a:b]]})
+                x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond=make_condition_dict(c_crossattn, image_cond_in[a:b]))
 
-            x_out[-uncond.shape[0]:] = self.inner_model(x_in[-uncond.shape[0]:], sigma_in[-uncond.shape[0]:], cond={"c_crossattn": [uncond], "c_concat": [image_cond_in[-uncond.shape[0]:]]})
+            x_out[-uncond.shape[0]:] = self.inner_model(x_in[-uncond.shape[0]:], sigma_in[-uncond.shape[0]:], cond=make_condition_dict([uncond], image_cond_in[-uncond.shape[0]:]))
 
         denoised_params = CFGDenoisedParams(x_out, state.sampling_step, state.sampling_steps)
         cfg_denoised_callback(denoised_params)
-- 
cgit v1.2.3


From 42082e8a3239c1c32cd9e2a03a20b610af857b51 Mon Sep 17 00:00:00 2001
From: devdn <zboodles2@gmail.com>
Date: Tue, 28 Mar 2023 18:18:28 -0400
Subject: performance increase

---
 modules/processing.py             |  4 +++-
 modules/sd_samplers_kdiffusion.py | 22 +++++++++++++++++-----
 modules/shared.py                 |  1 +
 scripts/xyz_grid.py               |  1 +
 4 files changed, 22 insertions(+), 6 deletions(-)

(limited to 'modules/sd_samplers_kdiffusion.py')

diff --git a/modules/processing.py b/modules/processing.py
index 6d9c6a8d..9f00ce3c 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -105,7 +105,7 @@ class StableDiffusionProcessing:
     """
     The first set of paramaters: sd_models -> do_not_reload_embeddings represent the minimum required to create a StableDiffusionProcessing
     """
-    def __init__(self, sd_model=None, outpath_samples=None, outpath_grids=None, prompt: str = "", styles: List[str] = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, seed_enable_extras: bool = True, sampler_name: str = None, batch_size: int = 1, n_iter: int = 1, steps: int = 50, cfg_scale: float = 7.0, width: int = 512, height: int = 512, restore_faces: bool = False, tiling: bool = False, do_not_save_samples: bool = False, do_not_save_grid: bool = False, extra_generation_params: Dict[Any, Any] = None, overlay_images: Any = None, negative_prompt: str = None, eta: float = None, do_not_reload_embeddings: bool = False, denoising_strength: float = 0, ddim_discretize: str = None, s_churn: float = 0.0, s_tmax: float = None, s_tmin: float = 0.0, s_noise: float = 1.0, override_settings: Dict[str, Any] = None, override_settings_restore_afterwards: bool = True, sampler_index: int = None, script_args: list = None):
+    def __init__(self, sd_model=None, outpath_samples=None, outpath_grids=None, prompt: str = "", styles: List[str] = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, seed_enable_extras: bool = True, sampler_name: str = None, batch_size: int = 1, n_iter: int = 1, steps: int = 50, cfg_scale: float = 7.0, width: int = 512, height: int = 512, restore_faces: bool = False, tiling: bool = False, do_not_save_samples: bool = False, do_not_save_grid: bool = False, extra_generation_params: Dict[Any, Any] = None, overlay_images: Any = None, negative_prompt: str = None, eta: float = None, do_not_reload_embeddings: bool = False, denoising_strength: float = 0, ddim_discretize: str = None, s_min_uncond: float = 0.0, s_churn: float = 0.0, s_tmax: float = None, s_tmin: float = 0.0, s_noise: float = 1.0, override_settings: Dict[str, Any] = None, override_settings_restore_afterwards: bool = True, sampler_index: int = None, script_args: list = None):
         if sampler_index is not None:
             print("sampler_index argument for StableDiffusionProcessing does not do anything; use sampler_name", file=sys.stderr)
 
@@ -140,6 +140,7 @@ class StableDiffusionProcessing:
         self.denoising_strength: float = denoising_strength
         self.sampler_noise_scheduler_override = None
         self.ddim_discretize = ddim_discretize or opts.ddim_discretize
+        self.s_min_uncond = s_min_uncond or opts.s_min_uncond
         self.s_churn = s_churn or opts.s_churn
         self.s_tmin = s_tmin or opts.s_tmin
         self.s_tmax = s_tmax or float('inf')  # not representable as a standard ui option
@@ -162,6 +163,7 @@ class StableDiffusionProcessing:
         self.all_seeds = None
         self.all_subseeds = None
         self.iteration = 0
+        
 
     @property
     def sd_model(self):
diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py
index e9f08518..6a54ce32 100644
--- a/modules/sd_samplers_kdiffusion.py
+++ b/modules/sd_samplers_kdiffusion.py
@@ -76,7 +76,7 @@ class CFGDenoiser(torch.nn.Module):
 
         return denoised
 
-    def forward(self, x, sigma, uncond, cond, cond_scale, image_cond):
+    def forward(self, x, sigma, uncond, cond, cond_scale, s_min_uncond, image_cond):
         if state.interrupted or state.skipped:
             raise sd_samplers_common.InterruptedException
 
@@ -116,6 +116,12 @@ class CFGDenoiser(torch.nn.Module):
         tensor = denoiser_params.text_cond
         uncond = denoiser_params.text_uncond
 
+        sigma_thresh = s_min_uncond
+        if(torch.dot(sigma,sigma) < sigma.shape[0] * (sigma_thresh*sigma_thresh) and not is_edit_model):
+            uncond = torch.zeros([0,0,uncond.shape[2]])
+            x_in=x_in[:x_in.shape[0]//2]
+            sigma_in=sigma_in[:sigma_in.shape[0]//2]
+
         if tensor.shape[1] == uncond.shape[1]:
             if not is_edit_model:
                 cond_in = torch.cat([tensor, uncond])
@@ -144,7 +150,8 @@ class CFGDenoiser(torch.nn.Module):
 
                 x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond=make_condition_dict(c_crossattn, image_cond_in[a:b]))
 
-            x_out[-uncond.shape[0]:] = self.inner_model(x_in[-uncond.shape[0]:], sigma_in[-uncond.shape[0]:], cond=make_condition_dict([uncond], image_cond_in[-uncond.shape[0]:]))
+            if uncond.shape[0]:
+                x_out[-uncond.shape[0]:] = self.inner_model(x_in[-uncond.shape[0]:], sigma_in[-uncond.shape[0]:], cond=make_condition_dict([uncond], image_cond_in[-uncond.shape[0]:]))
 
         denoised_params = CFGDenoisedParams(x_out, state.sampling_step, state.sampling_steps)
         cfg_denoised_callback(denoised_params)
@@ -157,7 +164,10 @@ class CFGDenoiser(torch.nn.Module):
             sd_samplers_common.store_latent(x_out[-uncond.shape[0]:])
 
         if not is_edit_model:
-            denoised = self.combine_denoised(x_out, conds_list, uncond, cond_scale)
+            if uncond.shape[0]:
+                denoised = self.combine_denoised(x_out, conds_list, uncond, cond_scale)
+            else:
+                denoised = x_out
         else:
             denoised = self.combine_denoised_for_edit_model(x_out, cond_scale)
 
@@ -165,7 +175,6 @@ class CFGDenoiser(torch.nn.Module):
             denoised = self.init_latent * self.mask + self.nmask * denoised
 
         self.step += 1
-
         return denoised
 
 
@@ -244,6 +253,7 @@ class KDiffusionSampler:
         self.model_wrap_cfg.step = 0
         self.model_wrap_cfg.image_cfg_scale = getattr(p, 'image_cfg_scale', None)
         self.eta = p.eta if p.eta is not None else opts.eta_ancestral
+        self.s_min_uncond = getattr(p, 's_min_uncond', 0.0)
 
         k_diffusion.sampling.torch = TorchHijack(self.sampler_noises if self.sampler_noises is not None else [])
 
@@ -326,6 +336,7 @@ class KDiffusionSampler:
             'image_cond': image_conditioning, 
             'uncond': unconditional_conditioning, 
             'cond_scale': p.cfg_scale,
+            's_min_uncond': self.s_min_uncond
         }
 
         samples = self.launch_sampling(t_enc + 1, lambda: self.func(self.model_wrap_cfg, xi, extra_args=extra_args, disable=False, callback=self.callback_state, **extra_params_kwargs))
@@ -359,7 +370,8 @@ class KDiffusionSampler:
             'cond': conditioning, 
             'image_cond': image_conditioning, 
             'uncond': unconditional_conditioning, 
-            'cond_scale': p.cfg_scale
+            'cond_scale': p.cfg_scale,
+            's_min_uncond': self.s_min_uncond
         }, disable=False, callback=self.callback_state, **extra_params_kwargs))
 
         return samples
diff --git a/modules/shared.py b/modules/shared.py
index 5fd0eecb..0bdd30b8 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -405,6 +405,7 @@ options_templates.update(options_section(('sampler-params', "Sampler parameters"
     "eta_ancestral": OptionInfo(1.0, "eta (noise multiplier) for ancestral samplers", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}),
     "ddim_discretize": OptionInfo('uniform', "img2img DDIM discretize", gr.Radio, {"choices": ['uniform', 'quad']}),
     's_churn': OptionInfo(0.0, "sigma churn", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}),
+    's_min_uncond': OptionInfo(0, "minimum sigma to use unconditioned guidance", gr.Slider, {"minimum": 0.0, "maximum": 2.0, "step": 0.01}),
     's_tmin':  OptionInfo(0.0, "sigma tmin",  gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}),
     's_noise': OptionInfo(1.0, "sigma noise", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}),
     'eta_noise_seed_delta': OptionInfo(0, "Eta noise seed delta", gr.Number, {"precision": 0}),
diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py
index 3895a795..d6a44b1c 100644
--- a/scripts/xyz_grid.py
+++ b/scripts/xyz_grid.py
@@ -212,6 +212,7 @@ axis_options = [
     AxisOptionTxt2Img("Sampler", str, apply_sampler, format_value=format_value, confirm=confirm_samplers, choices=lambda: [x.name for x in sd_samplers.samplers]),
     AxisOptionImg2Img("Sampler", str, apply_sampler, format_value=format_value, confirm=confirm_samplers, choices=lambda: [x.name for x in sd_samplers.samplers_for_img2img]),
     AxisOption("Checkpoint name", str, apply_checkpoint, format_value=format_value, confirm=confirm_checkpoints, cost=1.0, choices=lambda: list(sd_models.checkpoints_list)),
+    AxisOption("Negative Guidance minimum sigma", float, apply_field("s_min_uncond")),
     AxisOption("Sigma Churn", float, apply_field("s_churn")),
     AxisOption("Sigma min", float, apply_field("s_tmin")),
     AxisOption("Sigma max", float, apply_field("s_tmax")),
-- 
cgit v1.2.3


From 44e8e9c36807d4a71c2fc84129ebcf5ba4f77f21 Mon Sep 17 00:00:00 2001
From: devdn <zboodles2@gmail.com>
Date: Thu, 30 Mar 2023 00:54:28 -0400
Subject: fix live preview & alternate uncond guidance for better quality

---
 modules/sd_samplers_kdiffusion.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

(limited to 'modules/sd_samplers_kdiffusion.py')

diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py
index 6a54ce32..17d24df4 100644
--- a/modules/sd_samplers_kdiffusion.py
+++ b/modules/sd_samplers_kdiffusion.py
@@ -116,11 +116,13 @@ class CFGDenoiser(torch.nn.Module):
         tensor = denoiser_params.text_cond
         uncond = denoiser_params.text_uncond
 
-        sigma_thresh = s_min_uncond
-        if(torch.dot(sigma,sigma) < sigma.shape[0] * (sigma_thresh*sigma_thresh) and not is_edit_model):
-            uncond = torch.zeros([0,0,uncond.shape[2]])
-            x_in=x_in[:x_in.shape[0]//2]
-            sigma_in=sigma_in[:sigma_in.shape[0]//2]
+        if self.step % 2 and s_min_uncond > 0 and not is_edit_model:
+            # alternating uncond allows for higher thresholds without the quality loss normally expected from raising it
+            sigma_threshold = s_min_uncond
+            if(torch.dot(sigma,sigma) < sigma.shape[0] * (sigma_threshold*sigma_threshold) ):
+                uncond = torch.zeros([0,0,uncond.shape[2]])
+                x_in=x_in[:x_in.shape[0]//2]
+                sigma_in=sigma_in[:sigma_in.shape[0]//2]
 
         if tensor.shape[1] == uncond.shape[1]:
             if not is_edit_model:
@@ -159,7 +161,7 @@ class CFGDenoiser(torch.nn.Module):
         devices.test_for_nans(x_out, "unet")
 
         if opts.live_preview_content == "Prompt":
-            sd_samplers_common.store_latent(x_out[0:uncond.shape[0]])
+            sd_samplers_common.store_latent(x_out[0:x_out.shape[0]-uncond.shape[0]])
         elif opts.live_preview_content == "Negative prompt":
             sd_samplers_common.store_latent(x_out[-uncond.shape[0]:])
 
-- 
cgit v1.2.3


From d40e44ade479f7bba30d5317381cbc58c861775b Mon Sep 17 00:00:00 2001
From: Deciare <1689220+deciare@users.noreply.github.com>
Date: Tue, 18 Apr 2023 23:18:58 -0400
Subject: Option to use CPU for random number generation.

Makes a given manual seed generate the same images across different
platforms, independently of the GPU architecture in use.

Fixes #9613.
---
 modules/devices.py                | 8 ++++++--
 modules/sd_samplers_common.py     | 9 +++++++++
 modules/sd_samplers_kdiffusion.py | 2 +-
 modules/shared.py                 | 1 +
 4 files changed, 17 insertions(+), 3 deletions(-)

(limited to 'modules/sd_samplers_kdiffusion.py')

diff --git a/modules/devices.py b/modules/devices.py
index 52c3e7cd..3bc86a6a 100644
--- a/modules/devices.py
+++ b/modules/devices.py
@@ -92,14 +92,18 @@ def cond_cast_float(input):
 
 
 def randn(seed, shape):
+    from modules.shared import opts
+
     torch.manual_seed(seed)
-    if device.type == 'mps':
+    if opts.use_cpu_randn or device.type == 'mps':
         return torch.randn(shape, device=cpu).to(device)
     return torch.randn(shape, device=device)
 
 
 def randn_without_seed(shape):
-    if device.type == 'mps':
+    from modules.shared import opts
+
+    if opts.use_cpu_randn or device.type == 'mps':
         return torch.randn(shape, device=cpu).to(device)
     return torch.randn(shape, device=device)
 
diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py
index a1aac7cf..e6a372d5 100644
--- a/modules/sd_samplers_common.py
+++ b/modules/sd_samplers_common.py
@@ -60,3 +60,12 @@ def store_latent(decoded):
 
 class InterruptedException(BaseException):
     pass
+
+if opts.use_cpu_randn:
+    import torchsde._brownian.brownian_interval
+
+    def torchsde_randn(size, dtype, device, seed):
+        generator = torch.Generator(devices.cpu).manual_seed(int(seed))
+        return torch.randn(size, dtype=dtype, device=devices.cpu, generator=generator).to(device)
+
+    torchsde._brownian.brownian_interval._randn = torchsde_randn
diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py
index e9f08518..13f4567a 100644
--- a/modules/sd_samplers_kdiffusion.py
+++ b/modules/sd_samplers_kdiffusion.py
@@ -190,7 +190,7 @@ class TorchHijack:
             if noise.shape == x.shape:
                 return noise
 
-        if x.device.type == 'mps':
+        if opts.use_cpu_randn or x.device.type == 'mps':
             return torch.randn_like(x, device=devices.cpu).to(x.device)
         else:
             return torch.randn_like(x)
diff --git a/modules/shared.py b/modules/shared.py
index 5fd0eecb..59b037d5 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -331,6 +331,7 @@ options_templates.update(options_section(('sd', "Stable Diffusion"), {
     "comma_padding_backtrack": OptionInfo(20, "Increase coherency by padding from the last comma within n tokens when using more than 75 tokens", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1 }),
     "CLIP_stop_at_last_layers": OptionInfo(1, "Clip skip", gr.Slider, {"minimum": 1, "maximum": 12, "step": 1}),
     "upcast_attn": OptionInfo(False, "Upcast cross attention layer to float32"),
+    "use_cpu_randn": OptionInfo(False, "Use CPU for random number generation to make manual seeds generate the same image across platforms. This may change existing seeds."),
 }))
 
 options_templates.update(options_section(('compatibility', "Compatibility"), {
-- 
cgit v1.2.3


From 5fe0dd79beaa5ef737ff85254ee9870f60ae9464 Mon Sep 17 00:00:00 2001
From: AUTOMATIC <16777216c@gmail.com>
Date: Sat, 29 Apr 2023 11:29:37 +0300
Subject: rename CPU RNG to RNG source in settings, add infotext and parameters
 copypaste support to RNG source

---
 modules/devices.py                         | 4 ++--
 modules/generation_parameters_copypaste.py | 5 +++++
 modules/processing.py                      | 3 ++-
 modules/sd_samplers_common.py              | 3 ++-
 modules/sd_samplers_kdiffusion.py          | 2 +-
 modules/shared.py                          | 2 +-
 6 files changed, 13 insertions(+), 6 deletions(-)

(limited to 'modules/sd_samplers_kdiffusion.py')

diff --git a/modules/devices.py b/modules/devices.py
index 3bc86a6a..c705a3cb 100644
--- a/modules/devices.py
+++ b/modules/devices.py
@@ -95,7 +95,7 @@ def randn(seed, shape):
     from modules.shared import opts
 
     torch.manual_seed(seed)
-    if opts.use_cpu_randn or device.type == 'mps':
+    if opts.randn_source == "CPU" or device.type == 'mps':
         return torch.randn(shape, device=cpu).to(device)
     return torch.randn(shape, device=device)
 
@@ -103,7 +103,7 @@ def randn(seed, shape):
 def randn_without_seed(shape):
     from modules.shared import opts
 
-    if opts.use_cpu_randn or device.type == 'mps':
+    if opts.randn_source == "CPU" or device.type == 'mps':
         return torch.randn(shape, device=cpu).to(device)
     return torch.randn(shape, device=device)
 
diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py
index 6df76858..e7269363 100644
--- a/modules/generation_parameters_copypaste.py
+++ b/modules/generation_parameters_copypaste.py
@@ -284,6 +284,10 @@ Steps: 20, Sampler: Euler a, CFG scale: 7, Seed: 965400086, Size: 512x512, Model
 
     restore_old_hires_fix_params(res)
 
+    # Missing RNG means the default was set, which is GPU RNG
+    if "RNG" not in res:
+        res["RNG"] = "GPU"
+
     return res
 
 
@@ -304,6 +308,7 @@ infotext_to_setting_name_mapping = [
     ('UniPC skip type', 'uni_pc_skip_type'),
     ('UniPC order', 'uni_pc_order'),
     ('UniPC lower order final', 'uni_pc_lower_order_final'),
+    ('RNG', 'randn_source'),
 ]
 
 
diff --git a/modules/processing.py b/modules/processing.py
index 5556afc5..7bac154d 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -477,7 +477,8 @@ def create_infotext(p, all_prompts, all_seeds, all_subseeds, comments=None, iter
         "Conditional mask weight": getattr(p, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) if p.is_using_inpainting_conditioning else None,
         "Clip skip": None if clip_skip <= 1 else clip_skip,
         "ENSD": None if opts.eta_noise_seed_delta == 0 else opts.eta_noise_seed_delta,
-        "Init image hash": getattr(p, 'init_img_hash', None)
+        "Init image hash": getattr(p, 'init_img_hash', None),
+        "RNG": (opts.randn_source if opts.randn_source != "GPU" else None)
     }
 
     generation_params.update(p.extra_generation_params)
diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py
index e6a372d5..bc074238 100644
--- a/modules/sd_samplers_common.py
+++ b/modules/sd_samplers_common.py
@@ -61,7 +61,8 @@ def store_latent(decoded):
 class InterruptedException(BaseException):
     pass
 
-if opts.use_cpu_randn:
+
+if opts.randn_source == "CPU":
     import torchsde._brownian.brownian_interval
 
     def torchsde_randn(size, dtype, device, seed):
diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py
index 13f4567a..a547d1b5 100644
--- a/modules/sd_samplers_kdiffusion.py
+++ b/modules/sd_samplers_kdiffusion.py
@@ -190,7 +190,7 @@ class TorchHijack:
             if noise.shape == x.shape:
                 return noise
 
-        if opts.use_cpu_randn or x.device.type == 'mps':
+        if opts.randn_source == "CPU" or x.device.type == 'mps':
             return torch.randn_like(x, device=devices.cpu).to(x.device)
         else:
             return torch.randn_like(x)
diff --git a/modules/shared.py b/modules/shared.py
index b5b401fe..73704889 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -334,7 +334,7 @@ options_templates.update(options_section(('sd', "Stable Diffusion"), {
     "comma_padding_backtrack": OptionInfo(20, "Increase coherency by padding from the last comma within n tokens when using more than 75 tokens", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1 }),
     "CLIP_stop_at_last_layers": OptionInfo(1, "Clip skip", gr.Slider, {"minimum": 1, "maximum": 12, "step": 1}),
     "upcast_attn": OptionInfo(False, "Upcast cross attention layer to float32"),
-    "use_cpu_randn": OptionInfo(False, "Use CPU for random number generation to make manual seeds generate the same image across platforms. This may change existing seeds."),
+    "randn_source": OptionInfo("GPU", "Random number generator source. Changes seeds drastically. Use CPU to produce the same picture across different vidocard vendors.", gr.Radio, {"choices": ["GPU", "CPU"]}),
 }))
 
 options_templates.update(options_section(('compatibility', "Compatibility"), {
-- 
cgit v1.2.3


From 1d11e896984c883f6a0debb3abaef945595cbc70 Mon Sep 17 00:00:00 2001
From: AUTOMATIC <16777216c@gmail.com>
Date: Sat, 29 Apr 2023 15:57:09 +0300
Subject: rework Negative Guidance minimum sigma to work with AND, add infotext
 and copypaste parameters support

---
 javascript/hints.js                        |  3 ++-
 modules/generation_parameters_copypaste.py |  1 +
 modules/processing.py                      |  3 ++-
 modules/sd_samplers_kdiffusion.py          | 43 +++++++++++++++++-------------
 4 files changed, 30 insertions(+), 20 deletions(-)

(limited to 'modules/sd_samplers_kdiffusion.py')

diff --git a/javascript/hints.js b/javascript/hints.js
index c6bae360..44d418da 100644
--- a/javascript/hints.js
+++ b/javascript/hints.js
@@ -111,7 +111,8 @@ titles = {
     "Resize height to": "Resizes image to this height. If 0, height is inferred from either of two nearby sliders.",
     "Multiplier for extra networks": "When adding extra network such as Hypernetwork or Lora to prompt, use this multiplier for it.",
     "Discard weights with matching name": "Regular expression; if weights's name matches it, the weights is not written to the resulting checkpoint. Use ^model_ema to discard EMA weights.",
-    "Extra networks tab order": "Comma-separated list of tab names; tabs listed here will appear in the extra networks UI first and in order lsited."
+    "Extra networks tab order": "Comma-separated list of tab names; tabs listed here will appear in the extra networks UI first and in order lsited.",
+    "Negative Guidance minimum sigma": "Skip negative prompt for steps where image is already mostly denoised; the higher this value, the more skips there will be; provides increased performance in exchange for minor quality reduction."
 }
 
 
diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py
index e7269363..99f1a0d3 100644
--- a/modules/generation_parameters_copypaste.py
+++ b/modules/generation_parameters_copypaste.py
@@ -309,6 +309,7 @@ infotext_to_setting_name_mapping = [
     ('UniPC order', 'uni_pc_order'),
     ('UniPC lower order final', 'uni_pc_lower_order_final'),
     ('RNG', 'randn_source'),
+    ('NGMS', 's_min_uncond'),
 ]
 
 
diff --git a/modules/processing.py b/modules/processing.py
index 04a06290..c50784f4 100644
--- a/modules/processing.py
+++ b/modules/processing.py
@@ -480,7 +480,8 @@ def create_infotext(p, all_prompts, all_seeds, all_subseeds, comments=None, iter
         "Clip skip": None if clip_skip <= 1 else clip_skip,
         "ENSD": None if opts.eta_noise_seed_delta == 0 else opts.eta_noise_seed_delta,
         "Init image hash": getattr(p, 'init_img_hash', None),
-        "RNG": (opts.randn_source if opts.randn_source != "GPU" else None)
+        "RNG": opts.randn_source if opts.randn_source != "GPU" else None,
+        "NGMS": None if p.s_min_uncond == 0 else p.s_min_uncond,
     }
 
     generation_params.update(p.extra_generation_params)
diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py
index d42d5fcf..f8aaac59 100644
--- a/modules/sd_samplers_kdiffusion.py
+++ b/modules/sd_samplers_kdiffusion.py
@@ -115,20 +115,21 @@ class CFGDenoiser(torch.nn.Module):
         sigma_in = denoiser_params.sigma
         tensor = denoiser_params.text_cond
         uncond = denoiser_params.text_uncond
+        skip_uncond = False
 
-        if self.step % 2 and s_min_uncond > 0 and not is_edit_model:
-            # alternating uncond allows for higher thresholds without the quality loss normally expected from raising it
-            sigma_threshold = s_min_uncond
-            if(torch.dot(sigma,sigma) < sigma.shape[0] * (sigma_threshold*sigma_threshold) ):
-                uncond = torch.zeros([0,0,uncond.shape[2]])
-                x_in=x_in[:x_in.shape[0]//2]
-                sigma_in=sigma_in[:sigma_in.shape[0]//2]
+        # alternating uncond allows for higher thresholds without the quality loss normally expected from raising it
+        if self.step % 2 and s_min_uncond > 0 and sigma[0] < s_min_uncond and not is_edit_model:
+            skip_uncond = True
+            x_in = x_in[:-batch_size]
+            sigma_in = sigma_in[:-batch_size]
 
-        if tensor.shape[1] == uncond.shape[1]:
-            if not is_edit_model:
-                cond_in = torch.cat([tensor, uncond])
-            else:
+        if tensor.shape[1] == uncond.shape[1] or skip_uncond:
+            if is_edit_model:
                 cond_in = torch.cat([tensor, uncond, uncond])
+            elif skip_uncond:
+                cond_in = tensor
+            else:
+                cond_in = torch.cat([tensor, uncond])
 
             if shared.batch_cond_uncond:
                 x_out = self.inner_model(x_in, sigma_in, cond=make_condition_dict([cond_in], image_cond_in))
@@ -152,9 +153,15 @@ class CFGDenoiser(torch.nn.Module):
 
                 x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond=make_condition_dict(c_crossattn, image_cond_in[a:b]))
 
-            if uncond.shape[0]:
+            if not skip_uncond:
                 x_out[-uncond.shape[0]:] = self.inner_model(x_in[-uncond.shape[0]:], sigma_in[-uncond.shape[0]:], cond=make_condition_dict([uncond], image_cond_in[-uncond.shape[0]:]))
 
+        if skip_uncond:
+            #x_out = torch.cat([x_out, x_out[0:batch_size]])  # we skipped uncond denoising, so we put cond-denoised image to where the uncond-denoised image should be
+            denoised_image_indexes = [x[0][0] for x in conds_list]
+            fake_uncond = torch.cat([x_out[i:i+1] for i in denoised_image_indexes])
+            x_out = torch.cat([x_out, fake_uncond])
+
         denoised_params = CFGDenoisedParams(x_out, state.sampling_step, state.sampling_steps)
         cfg_denoised_callback(denoised_params)
 
@@ -165,13 +172,12 @@ class CFGDenoiser(torch.nn.Module):
         elif opts.live_preview_content == "Negative prompt":
             sd_samplers_common.store_latent(x_out[-uncond.shape[0]:])
 
-        if not is_edit_model:
-            if uncond.shape[0]:
-                denoised = self.combine_denoised(x_out, conds_list, uncond, cond_scale)
-            else:
-                denoised = x_out
-        else:
+        if is_edit_model:
             denoised = self.combine_denoised_for_edit_model(x_out, cond_scale)
+        elif skip_uncond:
+            denoised = self.combine_denoised(x_out, conds_list, uncond, 1.0)
+        else:
+            denoised = self.combine_denoised(x_out, conds_list, uncond, cond_scale)
 
         if self.mask is not None:
             denoised = self.init_latent * self.mask + self.nmask * denoised
@@ -221,6 +227,7 @@ class KDiffusionSampler:
         self.eta = None
         self.config = None
         self.last_latent = None
+        self.s_min_uncond = None
 
         self.conditioning_key = sd_model.model.conditioning_key
 
-- 
cgit v1.2.3


From 737b73a820584b8035fcc37fe35993bec867f326 Mon Sep 17 00:00:00 2001
From: AUTOMATIC <16777216c@gmail.com>
Date: Sat, 29 Apr 2023 16:05:20 +0300
Subject: some extra lines I forgot to add for previous commit

---
 modules/sd_samplers_kdiffusion.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'modules/sd_samplers_kdiffusion.py')

diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py
index f8aaac59..136aa8e5 100644
--- a/modules/sd_samplers_kdiffusion.py
+++ b/modules/sd_samplers_kdiffusion.py
@@ -156,11 +156,10 @@ class CFGDenoiser(torch.nn.Module):
             if not skip_uncond:
                 x_out[-uncond.shape[0]:] = self.inner_model(x_in[-uncond.shape[0]:], sigma_in[-uncond.shape[0]:], cond=make_condition_dict([uncond], image_cond_in[-uncond.shape[0]:]))
 
+        denoised_image_indexes = [x[0][0] for x in conds_list]
         if skip_uncond:
-            #x_out = torch.cat([x_out, x_out[0:batch_size]])  # we skipped uncond denoising, so we put cond-denoised image to where the uncond-denoised image should be
-            denoised_image_indexes = [x[0][0] for x in conds_list]
             fake_uncond = torch.cat([x_out[i:i+1] for i in denoised_image_indexes])
-            x_out = torch.cat([x_out, fake_uncond])
+            x_out = torch.cat([x_out, fake_uncond])  # we skipped uncond denoising, so we put cond-denoised image to where the uncond-denoised image should be
 
         denoised_params = CFGDenoisedParams(x_out, state.sampling_step, state.sampling_steps)
         cfg_denoised_callback(denoised_params)
-- 
cgit v1.2.3


From 8863b31d83b527d041ca45e23b9af99e2346081a Mon Sep 17 00:00:00 2001
From: AUTOMATIC <16777216c@gmail.com>
Date: Sat, 29 Apr 2023 16:06:20 +0300
Subject: use correct images for previews when using AND (see #9491)

---
 modules/sd_samplers_kdiffusion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'modules/sd_samplers_kdiffusion.py')

diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py
index 136aa8e5..eb98e599 100644
--- a/modules/sd_samplers_kdiffusion.py
+++ b/modules/sd_samplers_kdiffusion.py
@@ -167,7 +167,7 @@ class CFGDenoiser(torch.nn.Module):
         devices.test_for_nans(x_out, "unet")
 
         if opts.live_preview_content == "Prompt":
-            sd_samplers_common.store_latent(x_out[0:x_out.shape[0]-uncond.shape[0]])
+            sd_samplers_common.store_latent(torch.cat([x_out[i:i+1] for i in denoised_image_indexes]))
         elif opts.live_preview_content == "Negative prompt":
             sd_samplers_common.store_latent(x_out[-uncond.shape[0]:])
 
-- 
cgit v1.2.3