From a8d4213317c6970aa3ca8cbeeaacb07b936b591c Mon Sep 17 00:00:00 2001
From: AUTOMATIC1111 <16777216c@gmail.com>
Date: Sat, 22 Jul 2023 17:08:45 +0300
Subject: add --log-startup option to print detailed startup progress
---
modules/cmd_args.py | 1 +
1 file changed, 1 insertion(+)
(limited to 'modules/cmd_args.py')
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index e401f641..dd5fadc4 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -13,6 +13,7 @@ parser.add_argument("--reinstall-xformers", action='store_true', help="launch.py
parser.add_argument("--reinstall-torch", action='store_true', help="launch.py argument: install the appropriate version of torch even if you have some version already installed")
parser.add_argument("--update-check", action='store_true', help="launch.py argument: check for updates at startup")
parser.add_argument("--test-server", action='store_true', help="launch.py argument: configure server for testing")
+parser.add_argument("--log-startup", action='store_true', help="launch.py argument: print a detailed log of what's happening at startup")
parser.add_argument("--skip-prepare-environment", action='store_true', help="launch.py argument: skip all environment preparation")
parser.add_argument("--skip-install", action='store_true', help="launch.py argument: skip installation of packages")
parser.add_argument("--do-not-download-clip", action='store_true', help="do not download CLIP model even if it's not included in the checkpoint")
--
cgit v1.2.3
From 0a89cd1a584b1584a0609c0ba27fb35c434b0b68 Mon Sep 17 00:00:00 2001
From: AUTOMATIC1111 <16777216c@gmail.com>
Date: Mon, 24 Jul 2023 22:08:08 +0300
Subject: Use less RAM when creating models
---
modules/cmd_args.py | 1 +
modules/sd_disable_initialization.py | 106 +++++++++++++++++++++++++++++++++--
modules/sd_models.py | 16 ++++--
webui.py | 4 +-
4 files changed, 114 insertions(+), 13 deletions(-)
(limited to 'modules/cmd_args.py')
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index dd5fadc4..cb4ec5f7 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -67,6 +67,7 @@ parser.add_argument("--opt-sdp-no-mem-attention", action='store_true', help="pre
parser.add_argument("--disable-opt-split-attention", action='store_true', help="prefer no cross-attention layer optimization for automatic choice of optimization")
parser.add_argument("--disable-nan-check", action='store_true', help="do not check if produced images/latent spaces have nans; useful for running without a checkpoint in CI")
parser.add_argument("--use-cpu", nargs='+', help="use CPU as torch device for specified modules", default=[], type=str.lower)
+parser.add_argument("--disable-model-loading-ram-optimization", action='store_true', help="disable an optimization that reduces RAM use when loading a model")
parser.add_argument("--listen", action='store_true', help="launch gradio with 0.0.0.0 as server name, allowing to respond to network requests")
parser.add_argument("--port", type=int, help="launch gradio with given server port, you need root/admin rights for ports < 1024, defaults to 7860 if available", default=None)
parser.add_argument("--show-negative-prompt", action='store_true', help="does not do anything", default=False)
diff --git a/modules/sd_disable_initialization.py b/modules/sd_disable_initialization.py
index 9fc89dc6..695c5736 100644
--- a/modules/sd_disable_initialization.py
+++ b/modules/sd_disable_initialization.py
@@ -3,8 +3,31 @@ import open_clip
import torch
import transformers.utils.hub
+from modules import shared
-class DisableInitialization:
+
+class ReplaceHelper:
+ def __init__(self):
+ self.replaced = []
+
+ def replace(self, obj, field, func):
+ original = getattr(obj, field, None)
+ if original is None:
+ return None
+
+ self.replaced.append((obj, field, original))
+ setattr(obj, field, func)
+
+ return original
+
+ def restore(self):
+ for obj, field, original in self.replaced:
+ setattr(obj, field, original)
+
+ self.replaced.clear()
+
+
+class DisableInitialization(ReplaceHelper):
"""
When an object of this class enters a `with` block, it starts:
- preventing torch's layer initialization functions from working
@@ -21,7 +44,7 @@ class DisableInitialization:
"""
def __init__(self, disable_clip=True):
- self.replaced = []
+ super().__init__()
self.disable_clip = disable_clip
def replace(self, obj, field, func):
@@ -86,8 +109,81 @@ class DisableInitialization:
self.transformers_utils_hub_get_from_cache = self.replace(transformers.utils.hub, 'get_from_cache', transformers_utils_hub_get_from_cache)
def __exit__(self, exc_type, exc_val, exc_tb):
- for obj, field, original in self.replaced:
- setattr(obj, field, original)
+ self.restore()
- self.replaced.clear()
+class InitializeOnMeta(ReplaceHelper):
+ """
+ Context manager that causes all parameters for linear/conv2d/mha layers to be allocated on meta device,
+ which results in those parameters having no values and taking no memory. model.to() will be broken and
+ will need to be repaired by using LoadStateDictOnMeta below when loading params from state dict.
+
+ Usage:
+ ```
+ with sd_disable_initialization.InitializeOnMeta():
+ sd_model = instantiate_from_config(sd_config.model)
+ ```
+ """
+
+ def __enter__(self):
+ if shared.cmd_opts.disable_model_loading_ram_optimization:
+ return
+
+ def set_device(x):
+ x["device"] = "meta"
+ return x
+
+ linear_init = self.replace(torch.nn.Linear, '__init__', lambda *args, **kwargs: linear_init(*args, **set_device(kwargs)))
+ conv2d_init = self.replace(torch.nn.Conv2d, '__init__', lambda *args, **kwargs: conv2d_init(*args, **set_device(kwargs)))
+ mha_init = self.replace(torch.nn.MultiheadAttention, '__init__', lambda *args, **kwargs: mha_init(*args, **set_device(kwargs)))
+ self.replace(torch.nn.Module, 'to', lambda *args, **kwargs: None)
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.restore()
+
+
+class LoadStateDictOnMeta(ReplaceHelper):
+ """
+ Context manager that allows to read parameters from state_dict into a model that has some of its parameters in the meta device.
+ As those parameters are read from state_dict, they will be deleted from it, so by the end state_dict will be mostly empty, to save memory.
+ Meant to be used together with InitializeOnMeta above.
+
+ Usage:
+ ```
+ with sd_disable_initialization.LoadStateDictOnMeta(state_dict):
+ model.load_state_dict(state_dict, strict=False)
+ ```
+ """
+
+ def __init__(self, state_dict, device):
+ super().__init__()
+ self.state_dict = state_dict
+ self.device = device
+
+ def __enter__(self):
+ if shared.cmd_opts.disable_model_loading_ram_optimization:
+ return
+
+ sd = self.state_dict
+ device = self.device
+
+ def load_from_state_dict(original, self, state_dict, prefix, *args, **kwargs):
+ params = [(name, param) for name, param in self._parameters.items() if param is not None and param.is_meta]
+
+ for name, param in params:
+ if param.is_meta:
+ self._parameters[name] = torch.nn.parameter.Parameter(torch.zeros_like(param, device=device), requires_grad=param.requires_grad)
+
+ original(self, state_dict, prefix, *args, **kwargs)
+
+ for name, _ in params:
+ key = prefix + name
+ if key in sd:
+ del sd[key]
+
+ linear_load_from_state_dict = self.replace(torch.nn.Linear, '_load_from_state_dict', lambda *args, **kwargs: load_from_state_dict(linear_load_from_state_dict, *args, **kwargs))
+ conv2d_load_from_state_dict = self.replace(torch.nn.Conv2d, '_load_from_state_dict', lambda *args, **kwargs: load_from_state_dict(conv2d_load_from_state_dict, *args, **kwargs))
+ mha_load_from_state_dict = self.replace(torch.nn.MultiheadAttention, '_load_from_state_dict', lambda *args, **kwargs: load_from_state_dict(mha_load_from_state_dict, *args, **kwargs))
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.restore()
diff --git a/modules/sd_models.py b/modules/sd_models.py
index fb31a793..acb1e817 100644
--- a/modules/sd_models.py
+++ b/modules/sd_models.py
@@ -460,7 +460,6 @@ def get_empty_cond(sd_model):
return sd_model.cond_stage_model([""])
-
def load_model(checkpoint_info=None, already_loaded_state_dict=None):
from modules import lowvram, sd_hijack
checkpoint_info = checkpoint_info or select_checkpoint()
@@ -495,19 +494,24 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None):
sd_model = None
try:
with sd_disable_initialization.DisableInitialization(disable_clip=clip_is_included_into_sd or shared.cmd_opts.do_not_download_clip):
- sd_model = instantiate_from_config(sd_config.model)
- except Exception:
- pass
+ with sd_disable_initialization.InitializeOnMeta():
+ sd_model = instantiate_from_config(sd_config.model)
+
+ except Exception as e:
+ errors.display(e, "creating model quickly", full_traceback=True)
if sd_model is None:
print('Failed to create model quickly; will retry using slow method.', file=sys.stderr)
- sd_model = instantiate_from_config(sd_config.model)
+
+ with sd_disable_initialization.InitializeOnMeta():
+ sd_model = instantiate_from_config(sd_config.model)
sd_model.used_config = checkpoint_config
timer.record("create model")
- load_model_weights(sd_model, checkpoint_info, state_dict, timer)
+ with sd_disable_initialization.LoadStateDictOnMeta(state_dict, devices.cpu):
+ load_model_weights(sd_model, checkpoint_info, state_dict, timer)
if shared.cmd_opts.lowvram or shared.cmd_opts.medvram:
lowvram.setup_for_low_vram(sd_model, shared.cmd_opts.medvram)
diff --git a/webui.py b/webui.py
index 2314735f..51248c39 100644
--- a/webui.py
+++ b/webui.py
@@ -320,9 +320,9 @@ def initialize_rest(*, reload_script_modules=False):
if modules.sd_hijack.current_optimizer is None:
modules.sd_hijack.apply_optimizations()
- Thread(target=load_model).start()
+ devices.first_time_calculation()
- Thread(target=devices.first_time_calculation).start()
+ Thread(target=load_model).start()
shared.reload_hypernetworks()
startup_timer.record("reload hypernetworks")
--
cgit v1.2.3
From bbfff771d7337707bf501b27f98da2f7a7c06f73 Mon Sep 17 00:00:00 2001
From: w-e-w <40751091+w-e-w@users.noreply.github.com>
Date: Sat, 29 Jul 2023 01:07:35 +0900
Subject: --disable-all-extensions --disable-extra-extensions
---
modules/cmd_args.py | 2 ++
modules/extensions.py | 10 +++++++---
modules/ui_extensions.py | 18 +++++++++++-------
3 files changed, 20 insertions(+), 10 deletions(-)
(limited to 'modules/cmd_args.py')
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index dd5fadc4..1262f1a4 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -111,3 +111,5 @@ parser.add_argument('--subpath', type=str, help='customize the subpath for gradi
parser.add_argument('--add-stop-route', action='store_true', help='add /_stop route to stop server')
parser.add_argument('--api-server-stop', action='store_true', help='enable server stop/restart/kill via api')
parser.add_argument('--timeout-keep-alive', type=int, default=30, help='set timeout_keep_alive for uvicorn')
+parser.add_argument("--disable-all-extensions", action='store_true', help="prevent all extensions from running regardless of any other settings", default=False)
+parser.add_argument("--disable-extra-extensions", action='store_true', help=" prevent all extensions except built-in from running regardless of any other settings", default=False)
diff --git a/modules/extensions.py b/modules/extensions.py
index 3ad5ed53..e4633af4 100644
--- a/modules/extensions.py
+++ b/modules/extensions.py
@@ -11,9 +11,9 @@ os.makedirs(extensions_dir, exist_ok=True)
def active():
- if shared.opts.disable_all_extensions == "all":
+ if shared.cmd_opts.disable_all_extensions or shared.opts.disable_all_extensions == "all":
return []
- elif shared.opts.disable_all_extensions == "extra":
+ elif shared.cmd_opts.disable_extra_extensions or shared.opts.disable_all_extensions == "extra":
return [x for x in extensions if x.enabled and x.is_builtin]
else:
return [x for x in extensions if x.enabled]
@@ -141,8 +141,12 @@ def list_extensions():
if not os.path.isdir(extensions_dir):
return
- if shared.opts.disable_all_extensions == "all":
+ if shared.cmd_opts.disable_all_extensions:
+ print("*** \"--disable-all-extensions\" arg was used, will not load any extensions ***")
+ elif shared.opts.disable_all_extensions == "all":
print("*** \"Disable all extensions\" option was set, will not load any extensions ***")
+ elif shared.cmd_opts.disable_extra_extensions:
+ print("*** \"--disable-extra-extensions\" arg was used, will only load built-in extensions ***")
elif shared.opts.disable_all_extensions == "extra":
print("*** \"Disable all extensions\" option was set, will only load built-in extensions ***")
diff --git a/modules/ui_extensions.py b/modules/ui_extensions.py
index f3e4fba7..bd28bfcf 100644
--- a/modules/ui_extensions.py
+++ b/modules/ui_extensions.py
@@ -164,7 +164,7 @@ def extension_table():
ext_status = ext.status
style = ""
- if shared.opts.disable_all_extensions == "extra" and not ext.is_builtin or shared.opts.disable_all_extensions == "all":
+ if shared.cmd_opts.disable_extra_extensions and not ext.is_builtin or shared.opts.disable_all_extensions == "extra" and not ext.is_builtin or shared.cmd_opts.disable_all_extensions or shared.opts.disable_all_extensions == "all":
style = STYLE_PRIMARY
version_link = ext.version
@@ -537,12 +537,16 @@ def create_ui():
extensions_update_list = gr.Text(elem_id="extensions_update_list", visible=False).style(container=False)
html = ""
- if shared.opts.disable_all_extensions != "none":
- html = """
-
- "Disable all extensions" was set, change it to "none" to load all extensions again
-
- """
+
+ if shared.cmd_opts.disable_all_extensions or shared.cmd_opts.disable_extra_extensions or shared.opts.disable_all_extensions != "none":
+ if shared.cmd_opts.disable_all_extensions:
+ msg = '"--disable-all-extensions" was used, remove it to load all extensions again'
+ elif shared.opts.disable_all_extensions != "none":
+ msg = '"Disable all extensions" was set, change it to "none" to load all extensions again'
+ elif shared.cmd_opts.disable_extra_extensions:
+ msg = '"--disable-extra-extensions" was used, remove it to load all extensions again'
+ html = f'{msg}'
+
info = gr.HTML(html)
extensions_table = gr.HTML('Loading...')
ui.load(fn=extension_table, inputs=[], outputs=[extensions_table])
--
cgit v1.2.3
From edfae9e78af23bdd6161c55c7ec88533de8925f8 Mon Sep 17 00:00:00 2001
From: AUTOMATIC1111 <16777216c@gmail.com>
Date: Wed, 9 Aug 2023 20:49:33 +0300
Subject: add --loglevel commandline argument for logging remove the
progressbar for extension installation in favor of logging output
---
modules/cmd_args.py | 1 +
modules/initialize_util.py | 12 ------------
modules/launch_utils.py | 9 +++++----
modules/logging_config.py | 16 ++++++++++++++++
webui.py | 2 --
5 files changed, 22 insertions(+), 18 deletions(-)
create mode 100644 modules/logging_config.py
(limited to 'modules/cmd_args.py')
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index 64f21e01..b0a11538 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -16,6 +16,7 @@ parser.add_argument("--test-server", action='store_true', help="launch.py argume
parser.add_argument("--log-startup", action='store_true', help="launch.py argument: print a detailed log of what's happening at startup")
parser.add_argument("--skip-prepare-environment", action='store_true', help="launch.py argument: skip all environment preparation")
parser.add_argument("--skip-install", action='store_true', help="launch.py argument: skip installation of packages")
+parser.add_argument("--loglevel", type=str, help="log level; one of: CRITICAL, ERROR, WARNING, INFO, DEBUG", default=None)
parser.add_argument("--do-not-download-clip", action='store_true', help="do not download CLIP model even if it's not included in the checkpoint")
parser.add_argument("--data-dir", type=str, default=os.path.dirname(os.path.dirname(os.path.realpath(__file__))), help="base path where all user data is stored")
parser.add_argument("--config", type=str, default=sd_default_config, help="path to config which constructs model",)
diff --git a/modules/initialize_util.py b/modules/initialize_util.py
index e59bd3c4..d8370576 100644
--- a/modules/initialize_util.py
+++ b/modules/initialize_util.py
@@ -1,5 +1,4 @@
import json
-import logging
import os
import signal
import sys
@@ -7,17 +6,6 @@ import re
from modules.timer import startup_timer
-def setup_logging():
- # We can't use cmd_opts for this because it will not have been initialized at this point.
- log_level = os.environ.get("SD_WEBUI_LOG_LEVEL")
- if log_level:
- log_level = getattr(logging, log_level.upper(), None) or logging.INFO
- logging.basicConfig(
- level=log_level,
- format='%(asctime)s %(levelname)s [%(name)s] %(message)s',
- datefmt='%Y-%m-%d %H:%M:%S',
- )
-
def gradio_server_name():
from modules.shared_cmd_options import cmd_opts
diff --git a/modules/launch_utils.py b/modules/launch_utils.py
index 7143f144..90c00dd2 100644
--- a/modules/launch_utils.py
+++ b/modules/launch_utils.py
@@ -1,4 +1,5 @@
# this scripts installs necessary requirements and launches main program in webui.py
+import logging
import re
import subprocess
import os
@@ -11,8 +12,10 @@ from functools import lru_cache
from modules import cmd_args, errors
from modules.paths_internal import script_path, extensions_dir
from modules.timer import startup_timer
+from modules import logging_config
args, _ = cmd_args.parser.parse_known_args()
+logging_config.setup_logging(args.loglevel)
python = sys.executable
git = os.environ.get('GIT', "git")
@@ -248,10 +251,8 @@ def run_extensions_installers(settings_file):
return
with startup_timer.subcategory("run extensions installers"):
- import tqdm
- progress_bar = tqdm.tqdm(list_extensions(settings_file))
- for dirname_extension in progress_bar:
- progress_bar.set_description(f"Installing {dirname_extension}")
+ for dirname_extension in list_extensions(settings_file):
+ logging.debug(f"Installing {dirname_extension}")
path = os.path.join(extensions_dir, dirname_extension)
diff --git a/modules/logging_config.py b/modules/logging_config.py
new file mode 100644
index 00000000..7db23d4b
--- /dev/null
+++ b/modules/logging_config.py
@@ -0,0 +1,16 @@
+import os
+import logging
+
+
+def setup_logging(loglevel):
+ if loglevel is None:
+ loglevel = os.environ.get("SD_WEBUI_LOG_LEVEL")
+
+ if loglevel:
+ log_level = getattr(logging, loglevel.upper(), None) or logging.INFO
+ logging.basicConfig(
+ level=log_level,
+ format='%(asctime)s %(levelname)s [%(name)s] %(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S',
+ )
+
diff --git a/webui.py b/webui.py
index 738b3bef..5c827dae 100644
--- a/webui.py
+++ b/webui.py
@@ -10,8 +10,6 @@ from modules import initialize
startup_timer = timer.startup_timer
startup_timer.record("launcher")
-initialize_util.setup_logging()
-
initialize.imports()
initialize.check_versions()
--
cgit v1.2.3
From 0cf85b24df4c5d46461baea78fb233d721d6e1b1 Mon Sep 17 00:00:00 2001
From: w-e-w <40751091+w-e-w@users.noreply.github.com>
Date: Wed, 16 Aug 2023 20:18:46 +0900
Subject: auto add data-dir to gradio-allowed-path
---
modules/cmd_args.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
(limited to 'modules/cmd_args.py')
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index b0a11538..f360f484 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -81,7 +81,7 @@ parser.add_argument("--gradio-auth", type=str, help='set gradio authentication l
parser.add_argument("--gradio-auth-path", type=str, help='set gradio authentication file path ex. "/path/to/auth/file" same auth format as --gradio-auth', default=None)
parser.add_argument("--gradio-img2img-tool", type=str, help='does not do anything')
parser.add_argument("--gradio-inpaint-tool", type=str, help="does not do anything")
-parser.add_argument("--gradio-allowed-path", action='append', help="add path to gradio's allowed_paths, make it possible to serve files from it")
+parser.add_argument("--gradio-allowed-path", action='append', help="add path to gradio's allowed_paths, make it possible to serve files from it", default=[data_path])
parser.add_argument("--opt-channelslast", action='store_true', help="change memory type for stable diffusion to channels last")
parser.add_argument("--styles-file", type=str, help="filename to use for styles", default=os.path.join(data_path, 'styles.csv'))
parser.add_argument("--autolaunch", action='store_true', help="open the webui URL in the system's default browser upon launch", default=False)
--
cgit v1.2.3
From dfd6ea3fcaf2eb701af61136a290132303a729d5 Mon Sep 17 00:00:00 2001
From: AUTOMATIC1111 <16777216c@gmail.com>
Date: Mon, 21 Aug 2023 15:07:10 +0300
Subject: ditch --always-batch-cond-uncond in favor of an UI setting
---
modules/cmd_args.py | 2 +-
modules/sd_samplers_cfg_denoiser.py | 4 ++--
modules/shared.py | 2 +-
modules/shared_options.py | 3 ++-
4 files changed, 6 insertions(+), 5 deletions(-)
(limited to 'modules/cmd_args.py')
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index f360f484..9f8e5b30 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -37,7 +37,7 @@ parser.add_argument("--allow-code", action='store_true', help="allow custom scri
parser.add_argument("--medvram", action='store_true', help="enable stable diffusion model optimizations for sacrificing a little speed for low VRM usage")
parser.add_argument("--lowvram", action='store_true', help="enable stable diffusion model optimizations for sacrificing a lot of speed for very low VRM usage")
parser.add_argument("--lowram", action='store_true', help="load stable diffusion checkpoint weights to VRAM instead of RAM")
-parser.add_argument("--always-batch-cond-uncond", action='store_true', help="disables cond/uncond batching that is enabled to save memory with --medvram or --lowvram")
+parser.add_argument("--always-batch-cond-uncond", action='store_true', help="does not do anything")
parser.add_argument("--unload-gfpgan", action='store_true', help="does not do anything.")
parser.add_argument("--precision", type=str, help="evaluate at this precision", choices=["full", "autocast"], default="autocast")
parser.add_argument("--upcast-sampling", action='store_true', help="upcast sampling. No effect with --no-half. Usually produces similar results to --no-half with better performance while using less memory.")
diff --git a/modules/sd_samplers_cfg_denoiser.py b/modules/sd_samplers_cfg_denoiser.py
index bc9b97e4..b8101d38 100644
--- a/modules/sd_samplers_cfg_denoiser.py
+++ b/modules/sd_samplers_cfg_denoiser.py
@@ -165,7 +165,7 @@ class CFGDenoiser(torch.nn.Module):
else:
cond_in = catenate_conds([tensor, uncond])
- if shared.batch_cond_uncond:
+ if shared.opts.batch_cond_uncond:
x_out = self.inner_model(x_in, sigma_in, cond=make_condition_dict(cond_in, image_cond_in))
else:
x_out = torch.zeros_like(x_in)
@@ -175,7 +175,7 @@ class CFGDenoiser(torch.nn.Module):
x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond=make_condition_dict(subscript_cond(cond_in, a, b), image_cond_in[a:b]))
else:
x_out = torch.zeros_like(x_in)
- batch_size = batch_size*2 if shared.batch_cond_uncond else batch_size
+ batch_size = batch_size*2 if shared.opts.batch_cond_uncond else batch_size
for batch_offset in range(0, tensor.shape[0], batch_size):
a = batch_offset
b = min(a + batch_size, tensor.shape[0])
diff --git a/modules/shared.py b/modules/shared.py
index d9d01484..0c57b712 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -10,7 +10,7 @@ from modules import util
cmd_opts = shared_cmd_options.cmd_opts
parser = shared_cmd_options.parser
-batch_cond_uncond = cmd_opts.always_batch_cond_uncond or not (cmd_opts.lowvram or cmd_opts.medvram)
+batch_cond_uncond = True # old field, unused now in favor of shared.opts.batch_cond_uncond
parallel_processing_allowed = not cmd_opts.lowvram and not cmd_opts.medvram
styles_filename = cmd_opts.styles_file
config_filename = cmd_opts.ui_settings_file
diff --git a/modules/shared_options.py b/modules/shared_options.py
index 6f1a738d..095cf479 100644
--- a/modules/shared_options.py
+++ b/modules/shared_options.py
@@ -192,7 +192,8 @@ options_templates.update(options_section(('optimizations', "Optimizations"), {
"token_merging_ratio_img2img": OptionInfo(0.0, "Token merging ratio for img2img", gr.Slider, {"minimum": 0.0, "maximum": 0.9, "step": 0.1}).info("only applies if non-zero and overrides above"),
"token_merging_ratio_hr": OptionInfo(0.0, "Token merging ratio for high-res pass", gr.Slider, {"minimum": 0.0, "maximum": 0.9, "step": 0.1}, infotext='Token merging ratio hr').info("only applies if non-zero and overrides above"),
"pad_cond_uncond": OptionInfo(False, "Pad prompt/negative prompt to be same length", infotext='Pad conds').info("improves performance when prompt and negative prompt have different lengths; changes seeds"),
- "persistent_cond_cache": OptionInfo(True, "Persistent cond cache").info("Do not recalculate conds from prompts if prompts have not changed since previous calculation"),
+ "persistent_cond_cache": OptionInfo(True, "Persistent cond cache").info("do not recalculate conds from prompts if prompts have not changed since previous calculation"),
+ "batch_cond_uncond": OptionInfo(True, "Batch cond/uncond").info("do both conditional and unconditional denoising in one batch; uses a bit more VRAM during sampling, but improves speed; previously this was controlled by --always-batch-cond-uncond comandline argument"),
}))
options_templates.update(options_section(('compatibility', "Compatibility"), {
--
cgit v1.2.3
From 016554e43740e0b7ded75e89255de81270de9d6c Mon Sep 17 00:00:00 2001
From: AUTOMATIC1111 <16777216c@gmail.com>
Date: Tue, 22 Aug 2023 18:49:08 +0300
Subject: add --medvram-sdxl
---
modules/cmd_args.py | 1 +
modules/interrogate.py | 5 ++---
modules/lowvram.py | 18 ++++++++++++++++--
modules/sd_models.py | 16 ++++++++--------
modules/sd_unet.py | 2 +-
modules/sd_vae.py | 4 ++--
modules/shared.py | 2 +-
7 files changed, 31 insertions(+), 17 deletions(-)
(limited to 'modules/cmd_args.py')
diff --git a/modules/cmd_args.py b/modules/cmd_args.py
index 9f8e5b30..f0f361bd 100644
--- a/modules/cmd_args.py
+++ b/modules/cmd_args.py
@@ -35,6 +35,7 @@ parser.add_argument("--hypernetwork-dir", type=str, default=os.path.join(models_
parser.add_argument("--localizations-dir", type=str, default=os.path.join(script_path, 'localizations'), help="localizations directory")
parser.add_argument("--allow-code", action='store_true', help="allow custom script execution from webui")
parser.add_argument("--medvram", action='store_true', help="enable stable diffusion model optimizations for sacrificing a little speed for low VRM usage")
+parser.add_argument("--medvram-sdxl", action='store_true', help="enable --medvram optimization just for SDXL models")
parser.add_argument("--lowvram", action='store_true', help="enable stable diffusion model optimizations for sacrificing a lot of speed for very low VRM usage")
parser.add_argument("--lowram", action='store_true', help="load stable diffusion checkpoint weights to VRAM instead of RAM")
parser.add_argument("--always-batch-cond-uncond", action='store_true', help="does not do anything")
diff --git a/modules/interrogate.py b/modules/interrogate.py
index a3ae1dd5..3045560d 100644
--- a/modules/interrogate.py
+++ b/modules/interrogate.py
@@ -186,9 +186,8 @@ class InterrogateModels:
res = ""
shared.state.begin(job="interrogate")
try:
- if shared.cmd_opts.lowvram or shared.cmd_opts.medvram:
- lowvram.send_everything_to_cpu()
- devices.torch_gc()
+ lowvram.send_everything_to_cpu()
+ devices.torch_gc()
self.load()
diff --git a/modules/lowvram.py b/modules/lowvram.py
index 96f52b7b..45701046 100644
--- a/modules/lowvram.py
+++ b/modules/lowvram.py
@@ -1,5 +1,5 @@
import torch
-from modules import devices
+from modules import devices, shared
module_in_gpu = None
cpu = torch.device("cpu")
@@ -14,6 +14,20 @@ def send_everything_to_cpu():
module_in_gpu = None
+def is_needed(sd_model):
+ return shared.cmd_opts.lowvram or shared.cmd_opts.medvram or shared.cmd_opts.medvram_sdxl and hasattr(sd_model, 'conditioner')
+
+
+def apply(sd_model):
+ enable = is_needed(sd_model)
+ shared.parallel_processing_allowed = not enable
+
+ if enable:
+ setup_for_low_vram(sd_model, not shared.cmd_opts.lowvram)
+ else:
+ sd_model.lowvram = False
+
+
def setup_for_low_vram(sd_model, use_medvram):
if getattr(sd_model, 'lowvram', False):
return
@@ -130,4 +144,4 @@ def setup_for_low_vram(sd_model, use_medvram):
def is_enabled(sd_model):
- return getattr(sd_model, 'lowvram', False)
+ return sd_model.lowvram
diff --git a/modules/sd_models.py b/modules/sd_models.py
index 27d15e66..4331853a 100644
--- a/modules/sd_models.py
+++ b/modules/sd_models.py
@@ -517,7 +517,7 @@ def get_empty_cond(sd_model):
def send_model_to_cpu(m):
- if shared.cmd_opts.lowvram or shared.cmd_opts.medvram:
+ if m.lowvram:
lowvram.send_everything_to_cpu()
else:
m.to(devices.cpu)
@@ -525,17 +525,17 @@ def send_model_to_cpu(m):
devices.torch_gc()
-def model_target_device():
- if shared.cmd_opts.lowvram or shared.cmd_opts.medvram:
+def model_target_device(m):
+ if lowvram.is_needed(m):
return devices.cpu
else:
return devices.device
def send_model_to_device(m):
- if shared.cmd_opts.lowvram or shared.cmd_opts.medvram:
- lowvram.setup_for_low_vram(m, shared.cmd_opts.medvram)
- else:
+ lowvram.apply(m)
+
+ if not m.lowvram:
m.to(shared.device)
@@ -601,7 +601,7 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None):
'': torch.float16,
}
- with sd_disable_initialization.LoadStateDictOnMeta(state_dict, device=model_target_device(), weight_dtype_conversion=weight_dtype_conversion):
+ with sd_disable_initialization.LoadStateDictOnMeta(state_dict, device=model_target_device(sd_model), weight_dtype_conversion=weight_dtype_conversion):
load_model_weights(sd_model, checkpoint_info, state_dict, timer)
timer.record("load weights from state dict")
@@ -743,7 +743,7 @@ def reload_model_weights(sd_model=None, info=None):
script_callbacks.model_loaded_callback(sd_model)
timer.record("script callbacks")
- if not shared.cmd_opts.lowvram and not shared.cmd_opts.medvram:
+ if not sd_model.lowvram:
sd_model.to(devices.device)
timer.record("move model to device")
diff --git a/modules/sd_unet.py b/modules/sd_unet.py
index 6d708ad2..5525cfbc 100644
--- a/modules/sd_unet.py
+++ b/modules/sd_unet.py
@@ -47,7 +47,7 @@ def apply_unet(option=None):
if current_unet_option is None:
current_unet = None
- if not (shared.cmd_opts.lowvram or shared.cmd_opts.medvram):
+ if not shared.sd_model.lowvram:
shared.sd_model.model.diffusion_model.to(devices.device)
return
diff --git a/modules/sd_vae.py b/modules/sd_vae.py
index ee118656..669097da 100644
--- a/modules/sd_vae.py
+++ b/modules/sd_vae.py
@@ -263,7 +263,7 @@ def reload_vae_weights(sd_model=None, vae_file=unspecified):
if loaded_vae_file == vae_file:
return
- if shared.cmd_opts.lowvram or shared.cmd_opts.medvram:
+ if sd_model.lowvram:
lowvram.send_everything_to_cpu()
else:
sd_model.to(devices.cpu)
@@ -275,7 +275,7 @@ def reload_vae_weights(sd_model=None, vae_file=unspecified):
sd_hijack.model_hijack.hijack(sd_model)
script_callbacks.model_loaded_callback(sd_model)
- if not shared.cmd_opts.lowvram and not shared.cmd_opts.medvram:
+ if not sd_model.lowvram:
sd_model.to(devices.device)
print("VAE weights loaded.")
diff --git a/modules/shared.py b/modules/shared.py
index 0c57b712..f321159d 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -11,7 +11,7 @@ cmd_opts = shared_cmd_options.cmd_opts
parser = shared_cmd_options.parser
batch_cond_uncond = True # old field, unused now in favor of shared.opts.batch_cond_uncond
-parallel_processing_allowed = not cmd_opts.lowvram and not cmd_opts.medvram
+parallel_processing_allowed = True
styles_filename = cmd_opts.styles_file
config_filename = cmd_opts.ui_settings_file
hide_dirs = {"visible": not cmd_opts.hide_ui_dir_config}
--
cgit v1.2.3