From 0c3feb202c5714abd50d879c1db2cd9a71ce93e3 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Tue, 10 Jan 2023 14:08:29 +0300 Subject: disable torch weight initialization and CLIP downloading/reading checkpoint to speedup creating sd model from config --- modules/sd_disable_initialization.py | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 modules/sd_disable_initialization.py (limited to 'modules/sd_disable_initialization.py') diff --git a/modules/sd_disable_initialization.py b/modules/sd_disable_initialization.py new file mode 100644 index 00000000..c9a3b5e4 --- /dev/null +++ b/modules/sd_disable_initialization.py @@ -0,0 +1,44 @@ +import ldm.modules.encoders.modules +import open_clip +import torch + + +class DisableInitialization: + """ + When an object of this class enters a `with` block, it starts preventing torch's layer initialization + functions from working, and changes CLIP and OpenCLIP to not download model weights. When it leaves, + reverts everything to how it was. + + Use like this: + ``` + with DisableInitialization(): + do_things() + ``` + """ + + def __enter__(self): + def do_nothing(*args, **kwargs): + pass + + def create_model_and_transforms_without_pretrained(*args, pretrained=None, **kwargs): + return self.create_model_and_transforms(*args, pretrained=None, **kwargs) + + def CLIPTextModel_from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs): + return self.CLIPTextModel_from_pretrained(None, *model_args, config=pretrained_model_name_or_path, state_dict={}, **kwargs) + + self.init_kaiming_uniform = torch.nn.init.kaiming_uniform_ + self.init_no_grad_normal = torch.nn.init._no_grad_normal_ + self.create_model_and_transforms = open_clip.create_model_and_transforms + self.CLIPTextModel_from_pretrained = ldm.modules.encoders.modules.CLIPTextModel.from_pretrained + + torch.nn.init.kaiming_uniform_ = do_nothing + torch.nn.init._no_grad_normal_ = do_nothing + open_clip.create_model_and_transforms = create_model_and_transforms_without_pretrained + ldm.modules.encoders.modules.CLIPTextModel.from_pretrained = CLIPTextModel_from_pretrained + + def __exit__(self, exc_type, exc_val, exc_tb): + torch.nn.init.kaiming_uniform_ = self.init_kaiming_uniform + torch.nn.init._no_grad_normal_ = self.init_no_grad_normal + open_clip.create_model_and_transforms = self.create_model_and_transforms + ldm.modules.encoders.modules.CLIPTextModel.from_pretrained = self.CLIPTextModel_from_pretrained + -- cgit v1.2.3 From ce3f639ec8758ce2bc90483336361d2dc25acd3a Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Tue, 10 Jan 2023 16:51:04 +0300 Subject: add more stuff to ignore when creating model from config prevent .vae.safetensors files from being listed as stable diffusion models --- modules/sd_disable_initialization.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) (limited to 'modules/sd_disable_initialization.py') diff --git a/modules/sd_disable_initialization.py b/modules/sd_disable_initialization.py index c9a3b5e4..9942bd7e 100644 --- a/modules/sd_disable_initialization.py +++ b/modules/sd_disable_initialization.py @@ -1,15 +1,19 @@ import ldm.modules.encoders.modules import open_clip import torch +import transformers.utils.hub class DisableInitialization: """ - When an object of this class enters a `with` block, it starts preventing torch's layer initialization - functions from working, and changes CLIP and OpenCLIP to not download model weights. When it leaves, - reverts everything to how it was. + When an object of this class enters a `with` block, it starts: + - preventing torch's layer initialization functions from working + - changes CLIP and OpenCLIP to not download model weights + - changes CLIP to not make requests to check if there is a new version of a file you already have - Use like this: + When it leaves the block, it reverts everything to how it was before. + + Use it like this: ``` with DisableInitialization(): do_things() @@ -26,19 +30,36 @@ class DisableInitialization: def CLIPTextModel_from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs): return self.CLIPTextModel_from_pretrained(None, *model_args, config=pretrained_model_name_or_path, state_dict={}, **kwargs) + def transformers_utils_hub_get_from_cache(url, *args, local_files_only=False, **kwargs): + + # this file is always 404, prevent making request + if url == 'https://huggingface.co/openai/clip-vit-large-patch14/resolve/main/added_tokens.json': + raise transformers.utils.hub.EntryNotFoundError + + try: + return self.transformers_utils_hub_get_from_cache(url, *args, local_files_only=True, **kwargs) + except Exception as e: + return self.transformers_utils_hub_get_from_cache(url, *args, local_files_only=False, **kwargs) + self.init_kaiming_uniform = torch.nn.init.kaiming_uniform_ self.init_no_grad_normal = torch.nn.init._no_grad_normal_ + self.init_no_grad_uniform_ = torch.nn.init._no_grad_uniform_ self.create_model_and_transforms = open_clip.create_model_and_transforms self.CLIPTextModel_from_pretrained = ldm.modules.encoders.modules.CLIPTextModel.from_pretrained + self.transformers_utils_hub_get_from_cache = transformers.utils.hub.get_from_cache torch.nn.init.kaiming_uniform_ = do_nothing torch.nn.init._no_grad_normal_ = do_nothing + torch.nn.init._no_grad_uniform_ = do_nothing open_clip.create_model_and_transforms = create_model_and_transforms_without_pretrained ldm.modules.encoders.modules.CLIPTextModel.from_pretrained = CLIPTextModel_from_pretrained + transformers.utils.hub.get_from_cache = transformers_utils_hub_get_from_cache def __exit__(self, exc_type, exc_val, exc_tb): torch.nn.init.kaiming_uniform_ = self.init_kaiming_uniform torch.nn.init._no_grad_normal_ = self.init_no_grad_normal + torch.nn.init._no_grad_uniform_ = self.init_no_grad_uniform_ open_clip.create_model_and_transforms = self.create_model_and_transforms ldm.modules.encoders.modules.CLIPTextModel.from_pretrained = self.CLIPTextModel_from_pretrained + transformers.utils.hub.get_from_cache = self.transformers_utils_hub_get_from_cache -- cgit v1.2.3 From 0f8603a55988d22616b17140e6c4a7e9d0736af5 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Tue, 10 Jan 2023 17:46:59 +0300 Subject: add support for transformers==4.25.1 add fallback for when quick model creation fails --- modules/sd_disable_initialization.py | 42 ++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) (limited to 'modules/sd_disable_initialization.py') diff --git a/modules/sd_disable_initialization.py b/modules/sd_disable_initialization.py index 9942bd7e..088ac24b 100644 --- a/modules/sd_disable_initialization.py +++ b/modules/sd_disable_initialization.py @@ -30,30 +30,53 @@ class DisableInitialization: def CLIPTextModel_from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs): return self.CLIPTextModel_from_pretrained(None, *model_args, config=pretrained_model_name_or_path, state_dict={}, **kwargs) - def transformers_utils_hub_get_from_cache(url, *args, local_files_only=False, **kwargs): + def transformers_modeling_utils_load_pretrained_model(*args, **kwargs): + args = args[0:3] + ('/', ) + args[4:] # resolved_archive_file; must set it to something to prevent what seems to be a bug + return self.transformers_modeling_utils_load_pretrained_model(*args, **kwargs) + + def transformers_utils_hub_get_file_from_cache(original, url, *args, **kwargs): # this file is always 404, prevent making request if url == 'https://huggingface.co/openai/clip-vit-large-patch14/resolve/main/added_tokens.json': raise transformers.utils.hub.EntryNotFoundError try: - return self.transformers_utils_hub_get_from_cache(url, *args, local_files_only=True, **kwargs) + return original(url, *args, local_files_only=True, **kwargs) except Exception as e: - return self.transformers_utils_hub_get_from_cache(url, *args, local_files_only=False, **kwargs) + return original(url, *args, local_files_only=False, **kwargs) + + def transformers_utils_hub_get_from_cache(url, *args, local_files_only=False, **kwargs): + return transformers_utils_hub_get_file_from_cache(self.transformers_utils_hub_get_from_cache, url, *args, **kwargs) + + def transformers_tokenization_utils_base_cached_file(url, *args, local_files_only=False, **kwargs): + return transformers_utils_hub_get_file_from_cache(self.transformers_tokenization_utils_base_cached_file, url, *args, **kwargs) + + def transformers_configuration_utils_cached_file(url, *args, local_files_only=False, **kwargs): + return transformers_utils_hub_get_file_from_cache(self.transformers_configuration_utils_cached_file, url, *args, **kwargs) self.init_kaiming_uniform = torch.nn.init.kaiming_uniform_ self.init_no_grad_normal = torch.nn.init._no_grad_normal_ self.init_no_grad_uniform_ = torch.nn.init._no_grad_uniform_ self.create_model_and_transforms = open_clip.create_model_and_transforms self.CLIPTextModel_from_pretrained = ldm.modules.encoders.modules.CLIPTextModel.from_pretrained - self.transformers_utils_hub_get_from_cache = transformers.utils.hub.get_from_cache + self.transformers_modeling_utils_load_pretrained_model = getattr(transformers.modeling_utils.PreTrainedModel, '_load_pretrained_model', None) + self.transformers_tokenization_utils_base_cached_file = getattr(transformers.tokenization_utils_base, 'cached_file', None) + self.transformers_configuration_utils_cached_file = getattr(transformers.configuration_utils, 'cached_file', None) + self.transformers_utils_hub_get_from_cache = getattr(transformers.utils.hub, 'get_from_cache', None) torch.nn.init.kaiming_uniform_ = do_nothing torch.nn.init._no_grad_normal_ = do_nothing torch.nn.init._no_grad_uniform_ = do_nothing open_clip.create_model_and_transforms = create_model_and_transforms_without_pretrained ldm.modules.encoders.modules.CLIPTextModel.from_pretrained = CLIPTextModel_from_pretrained - transformers.utils.hub.get_from_cache = transformers_utils_hub_get_from_cache + if self.transformers_modeling_utils_load_pretrained_model is not None: + transformers.modeling_utils.PreTrainedModel._load_pretrained_model = transformers_modeling_utils_load_pretrained_model + if self.transformers_tokenization_utils_base_cached_file is not None: + transformers.tokenization_utils_base.cached_file = transformers_tokenization_utils_base_cached_file + if self.transformers_configuration_utils_cached_file is not None: + transformers.configuration_utils.cached_file = transformers_configuration_utils_cached_file + if self.transformers_utils_hub_get_from_cache is not None: + transformers.utils.hub.get_from_cache = transformers_utils_hub_get_from_cache def __exit__(self, exc_type, exc_val, exc_tb): torch.nn.init.kaiming_uniform_ = self.init_kaiming_uniform @@ -61,5 +84,12 @@ class DisableInitialization: torch.nn.init._no_grad_uniform_ = self.init_no_grad_uniform_ open_clip.create_model_and_transforms = self.create_model_and_transforms ldm.modules.encoders.modules.CLIPTextModel.from_pretrained = self.CLIPTextModel_from_pretrained - transformers.utils.hub.get_from_cache = self.transformers_utils_hub_get_from_cache + if self.transformers_modeling_utils_load_pretrained_model is not None: + transformers.modeling_utils.PreTrainedModel._load_pretrained_model = self.transformers_modeling_utils_load_pretrained_model + if self.transformers_tokenization_utils_base_cached_file is not None: + transformers.utils.hub.cached_file = self.transformers_tokenization_utils_base_cached_file + if self.transformers_configuration_utils_cached_file is not None: + transformers.utils.hub.cached_file = self.transformers_configuration_utils_cached_file + if self.transformers_utils_hub_get_from_cache is not None: + transformers.utils.hub.get_from_cache = self.transformers_utils_hub_get_from_cache -- cgit v1.2.3 From 4bd490727e156ff53107d53416d6b89be86f2a62 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Wed, 11 Jan 2023 18:54:04 +0300 Subject: fix for an error caused by skipping initialization, for realsies this time: TypeError: expected str, bytes or os.PathLike object, not NoneType --- modules/sd_disable_initialization.py | 71 ++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 39 deletions(-) (limited to 'modules/sd_disable_initialization.py') diff --git a/modules/sd_disable_initialization.py b/modules/sd_disable_initialization.py index 088ac24b..c72d8efc 100644 --- a/modules/sd_disable_initialization.py +++ b/modules/sd_disable_initialization.py @@ -20,6 +20,19 @@ class DisableInitialization: ``` """ + def __init__(self): + self.replaced = [] + + def replace(self, obj, field, func): + original = getattr(obj, field, None) + if original is None: + return None + + self.replaced.append((obj, field, original)) + setattr(obj, field, func) + + return original + def __enter__(self): def do_nothing(*args, **kwargs): pass @@ -37,11 +50,14 @@ class DisableInitialization: def transformers_utils_hub_get_file_from_cache(original, url, *args, **kwargs): # this file is always 404, prevent making request - if url == 'https://huggingface.co/openai/clip-vit-large-patch14/resolve/main/added_tokens.json': - raise transformers.utils.hub.EntryNotFoundError + if url == 'https://huggingface.co/openai/clip-vit-large-patch14/resolve/main/added_tokens.json' or url == 'openai/clip-vit-large-patch14' and args[0] == 'added_tokens.json': + return None try: - return original(url, *args, local_files_only=True, **kwargs) + res = original(url, *args, local_files_only=True, **kwargs) + if res is None: + res = original(url, *args, local_files_only=False, **kwargs) + return res except Exception as e: return original(url, *args, local_files_only=False, **kwargs) @@ -54,42 +70,19 @@ class DisableInitialization: def transformers_configuration_utils_cached_file(url, *args, local_files_only=False, **kwargs): return transformers_utils_hub_get_file_from_cache(self.transformers_configuration_utils_cached_file, url, *args, **kwargs) - self.init_kaiming_uniform = torch.nn.init.kaiming_uniform_ - self.init_no_grad_normal = torch.nn.init._no_grad_normal_ - self.init_no_grad_uniform_ = torch.nn.init._no_grad_uniform_ - self.create_model_and_transforms = open_clip.create_model_and_transforms - self.CLIPTextModel_from_pretrained = ldm.modules.encoders.modules.CLIPTextModel.from_pretrained - self.transformers_modeling_utils_load_pretrained_model = getattr(transformers.modeling_utils.PreTrainedModel, '_load_pretrained_model', None) - self.transformers_tokenization_utils_base_cached_file = getattr(transformers.tokenization_utils_base, 'cached_file', None) - self.transformers_configuration_utils_cached_file = getattr(transformers.configuration_utils, 'cached_file', None) - self.transformers_utils_hub_get_from_cache = getattr(transformers.utils.hub, 'get_from_cache', None) - - torch.nn.init.kaiming_uniform_ = do_nothing - torch.nn.init._no_grad_normal_ = do_nothing - torch.nn.init._no_grad_uniform_ = do_nothing - open_clip.create_model_and_transforms = create_model_and_transforms_without_pretrained - ldm.modules.encoders.modules.CLIPTextModel.from_pretrained = CLIPTextModel_from_pretrained - if self.transformers_modeling_utils_load_pretrained_model is not None: - transformers.modeling_utils.PreTrainedModel._load_pretrained_model = transformers_modeling_utils_load_pretrained_model - if self.transformers_tokenization_utils_base_cached_file is not None: - transformers.tokenization_utils_base.cached_file = transformers_tokenization_utils_base_cached_file - if self.transformers_configuration_utils_cached_file is not None: - transformers.configuration_utils.cached_file = transformers_configuration_utils_cached_file - if self.transformers_utils_hub_get_from_cache is not None: - transformers.utils.hub.get_from_cache = transformers_utils_hub_get_from_cache + self.replace(torch.nn.init, 'kaiming_uniform_', do_nothing) + self.replace(torch.nn.init, '_no_grad_normal_', do_nothing) + self.replace(torch.nn.init, '_no_grad_uniform_', do_nothing) + self.create_model_and_transforms = self.replace(open_clip, 'create_model_and_transforms', create_model_and_transforms_without_pretrained) + self.CLIPTextModel_from_pretrained = self.replace(ldm.modules.encoders.modules.CLIPTextModel, 'from_pretrained', CLIPTextModel_from_pretrained) + self.transformers_modeling_utils_load_pretrained_model = self.replace(transformers.modeling_utils.PreTrainedModel, '_load_pretrained_model', transformers_modeling_utils_load_pretrained_model) + self.transformers_tokenization_utils_base_cached_file = self.replace(transformers.tokenization_utils_base, 'cached_file', transformers_tokenization_utils_base_cached_file) + self.transformers_configuration_utils_cached_file = self.replace(transformers.configuration_utils, 'cached_file', transformers_configuration_utils_cached_file) + self.transformers_utils_hub_get_from_cache = self.replace(transformers.utils.hub, 'get_from_cache', transformers_utils_hub_get_from_cache) def __exit__(self, exc_type, exc_val, exc_tb): - torch.nn.init.kaiming_uniform_ = self.init_kaiming_uniform - torch.nn.init._no_grad_normal_ = self.init_no_grad_normal - torch.nn.init._no_grad_uniform_ = self.init_no_grad_uniform_ - open_clip.create_model_and_transforms = self.create_model_and_transforms - ldm.modules.encoders.modules.CLIPTextModel.from_pretrained = self.CLIPTextModel_from_pretrained - if self.transformers_modeling_utils_load_pretrained_model is not None: - transformers.modeling_utils.PreTrainedModel._load_pretrained_model = self.transformers_modeling_utils_load_pretrained_model - if self.transformers_tokenization_utils_base_cached_file is not None: - transformers.utils.hub.cached_file = self.transformers_tokenization_utils_base_cached_file - if self.transformers_configuration_utils_cached_file is not None: - transformers.utils.hub.cached_file = self.transformers_configuration_utils_cached_file - if self.transformers_utils_hub_get_from_cache is not None: - transformers.utils.hub.get_from_cache = self.transformers_utils_hub_get_from_cache + for obj, field, original in self.replaced: + setattr(obj, field, original) + + self.replaced.clear() -- cgit v1.2.3