From 7157e5d064741fa57ca81a2c6432a651f21ee82f Mon Sep 17 00:00:00 2001 From: Patryk Wychowaniec Date: Thu, 20 Oct 2022 19:22:59 +0200 Subject: interrogate: Fix CLIP-interrogation on CPU Currently, trying to perform CLIP interrogation on a CPU fails, saying: ``` RuntimeError: "slow_conv2d_cpu" not implemented for 'Half' ``` This merge request fixes this issue by detecting whether the target device is CPU and, if so, force-enabling `--no-half` and passing `device="cpu"` to `clip.load()` (which then does some extra tricks to ensure it works correctly on CPU). --- modules/interrogate.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'modules/interrogate.py') diff --git a/modules/interrogate.py b/modules/interrogate.py index 64b91eb4..65b05d34 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -28,9 +28,11 @@ class InterrogateModels: clip_preprocess = None categories = None dtype = None + running_on_cpu = None def __init__(self, content_dir): self.categories = [] + self.running_on_cpu = devices.device_interrogate == torch.device("cpu") if os.path.exists(content_dir): for filename in os.listdir(content_dir): @@ -53,7 +55,11 @@ class InterrogateModels: def load_clip_model(self): import clip - model, preprocess = clip.load(clip_model_name) + if self.running_on_cpu: + model, preprocess = clip.load(clip_model_name, device="cpu") + else: + model, preprocess = clip.load(clip_model_name) + model.eval() model = model.to(devices.device_interrogate) @@ -62,14 +68,14 @@ class InterrogateModels: def load(self): if self.blip_model is None: self.blip_model = self.load_blip_model() - if not shared.cmd_opts.no_half: + if not shared.cmd_opts.no_half and not self.running_on_cpu: self.blip_model = self.blip_model.half() self.blip_model = self.blip_model.to(devices.device_interrogate) if self.clip_model is None: self.clip_model, self.clip_preprocess = self.load_clip_model() - if not shared.cmd_opts.no_half: + if not shared.cmd_opts.no_half and not self.running_on_cpu: self.clip_model = self.clip_model.half() self.clip_model = self.clip_model.to(devices.device_interrogate) -- cgit v1.2.3 From d587586d3be2de061238defb8a556f03743287f6 Mon Sep 17 00:00:00 2001 From: mawr Date: Mon, 31 Oct 2022 00:14:07 +0300 Subject: Added "--clip-models-path" switch to avoid using default "~/.cache/clip" and enable to run under unprivileged user without homedir --- modules/interrogate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'modules/interrogate.py') diff --git a/modules/interrogate.py b/modules/interrogate.py index 65b05d34..9769aa34 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -56,9 +56,9 @@ class InterrogateModels: import clip if self.running_on_cpu: - model, preprocess = clip.load(clip_model_name, device="cpu") + model, preprocess = clip.load(clip_model_name, device="cpu", download_root=shared.cmd_opts.clip_models_path) else: - model, preprocess = clip.load(clip_model_name) + model, preprocess = clip.load(clip_model_name, download_root=shared.cmd_opts.clip_models_path) model.eval() model = model.to(devices.device_interrogate) -- cgit v1.2.3 From 745f1e8f8008ea4906b0f5eb8b8f71d205fedf9e Mon Sep 17 00:00:00 2001 From: "Tiago F. Santos" Date: Tue, 22 Nov 2022 12:48:25 +0000 Subject: [CLIP interrogator] use local file, if available --- modules/interrogate.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'modules/interrogate.py') diff --git a/modules/interrogate.py b/modules/interrogate.py index 9769aa34..1a9c758e 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -14,6 +14,7 @@ import modules.shared as shared from modules import devices, paths, lowvram blip_image_eval_size = 384 +blip_model_local = os.path.join('models', 'Interrogator', 'BLIP_model.pth') blip_model_url = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_caption_capfilt_large.pth' clip_model_name = 'ViT-L/14' @@ -47,7 +48,13 @@ class InterrogateModels: def load_blip_model(self): import models.blip - blip_model = models.blip.blip_decoder(pretrained=blip_model_url, image_size=blip_image_eval_size, vit='base', med_config=os.path.join(paths.paths["BLIP"], "configs", "med_config.json")) + if not os.path.isfile(blip_model_local): + print("Downloading BLIP...") + import requests as req + open(blip_model_local, 'wb').write(req.get(blip_model_url, allow_redirects=True).content) + print("BLIP downloaded to", blip_model_local + '.') + + blip_model = models.blip.blip_decoder(pretrained=blip_model_local, image_size=blip_image_eval_size, vit='base', med_config=os.path.join(paths.paths["BLIP"], "configs", "med_config.json")) blip_model.eval() return blip_model -- cgit v1.2.3 From a2ae5a655518b150a34b95d7afecc87a43280406 Mon Sep 17 00:00:00 2001 From: "Tiago F. Santos" Date: Thu, 24 Nov 2022 13:04:45 +0000 Subject: [interrogator] mkdir check --- modules/interrogate.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'modules/interrogate.py') diff --git a/modules/interrogate.py b/modules/interrogate.py index 1a9c758e..f177a5a8 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -14,7 +14,8 @@ import modules.shared as shared from modules import devices, paths, lowvram blip_image_eval_size = 384 -blip_model_local = os.path.join('models', 'Interrogator', 'BLIP_model.pth') +blip_local_dir = os.path.join('models', 'Interrogator') +blip_local_file = os.path.join(blip_local_dir, 'model_base_caption_capfilt_large.pth') blip_model_url = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_caption_capfilt_large.pth' clip_model_name = 'ViT-L/14' @@ -48,13 +49,16 @@ class InterrogateModels: def load_blip_model(self): import models.blip - if not os.path.isfile(blip_model_local): + if not os.path.isfile(blip_local_file): + if not os.path.isdir(blip_local_dir): + os.mkdir(blip_local_dir) + print("Downloading BLIP...") - import requests as req - open(blip_model_local, 'wb').write(req.get(blip_model_url, allow_redirects=True).content) - print("BLIP downloaded to", blip_model_local + '.') + from requests import get as reqget + open(blip_local_file, 'wb').write(reqget(blip_model_url, allow_redirects=True).content) + print("BLIP downloaded to", blip_local_file + '.') - blip_model = models.blip.blip_decoder(pretrained=blip_model_local, image_size=blip_image_eval_size, vit='base', med_config=os.path.join(paths.paths["BLIP"], "configs", "med_config.json")) + blip_model = models.blip.blip_decoder(pretrained=blip_local_file, image_size=blip_image_eval_size, vit='base', med_config=os.path.join(paths.paths["BLIP"], "configs", "med_config.json")) blip_model.eval() return blip_model -- cgit v1.2.3 From 4d5f1691dda971ec7b461dd880426300fd54ccee Mon Sep 17 00:00:00 2001 From: brkirch Date: Mon, 28 Nov 2022 21:36:35 -0500 Subject: Use devices.autocast instead of torch.autocast --- modules/interrogate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'modules/interrogate.py') diff --git a/modules/interrogate.py b/modules/interrogate.py index 9769aa34..40c6b082 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -148,8 +148,7 @@ class InterrogateModels: clip_image = self.clip_preprocess(pil_image).unsqueeze(0).type(self.dtype).to(devices.device_interrogate) - precision_scope = torch.autocast if shared.cmd_opts.precision == "autocast" else contextlib.nullcontext - with torch.no_grad(), precision_scope("cuda"): + with torch.no_grad(), devices.autocast(): image_features = self.clip_model.encode_image(clip_image).type(self.dtype) image_features /= image_features.norm(dim=-1, keepdim=True) -- cgit v1.2.3 From 4b0dc206edbad90affe609ac0bf2e9be7e197674 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sat, 3 Dec 2022 18:45:51 +0300 Subject: use modelloader for #4956 --- modules/interrogate.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) (limited to 'modules/interrogate.py') diff --git a/modules/interrogate.py b/modules/interrogate.py index 3a09b366..0068b81c 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -1,4 +1,3 @@ -import contextlib import os import sys import traceback @@ -11,12 +10,9 @@ from torchvision import transforms from torchvision.transforms.functional import InterpolationMode import modules.shared as shared -from modules import devices, paths, lowvram +from modules import devices, paths, lowvram, modelloader blip_image_eval_size = 384 -blip_local_dir = os.path.join('models', 'Interrogator') -blip_local_file = os.path.join(blip_local_dir, 'model_base_caption_capfilt_large.pth') -blip_model_url = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_caption_capfilt_large.pth' clip_model_name = 'ViT-L/14' Category = namedtuple("Category", ["name", "topn", "items"]) @@ -49,16 +45,14 @@ class InterrogateModels: def load_blip_model(self): import models.blip - if not os.path.isfile(blip_local_file): - if not os.path.isdir(blip_local_dir): - os.mkdir(blip_local_dir) + files = modelloader.load_models( + model_path=os.path.join(paths.models_path, "BLIP"), + model_url='https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_caption_capfilt_large.pth', + ext_filter=[".pth"], + download_name='model_base_caption_capfilt_large.pth', + ) - print("Downloading BLIP...") - from requests import get as reqget - open(blip_local_file, 'wb').write(reqget(blip_model_url, allow_redirects=True).content) - print("BLIP downloaded to", blip_local_file + '.') - - blip_model = models.blip.blip_decoder(pretrained=blip_local_file, image_size=blip_image_eval_size, vit='base', med_config=os.path.join(paths.paths["BLIP"], "configs", "med_config.json")) + blip_model = models.blip.blip_decoder(pretrained=files[0], image_size=blip_image_eval_size, vit='base', med_config=os.path.join(paths.paths["BLIP"], "configs", "med_config.json")) blip_model.eval() return blip_model -- cgit v1.2.3 From 3bf5591efe9a9f219c6088be322a87adc4f48f95 Mon Sep 17 00:00:00 2001 From: Yuval Aboulafia Date: Sat, 24 Dec 2022 21:35:29 +0200 Subject: fix F541 f-string without any placeholders --- modules/interrogate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/interrogate.py') diff --git a/modules/interrogate.py b/modules/interrogate.py index 0068b81c..46935210 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -172,7 +172,7 @@ class InterrogateModels: res += ", " + match except Exception: - print(f"Error interrogating", file=sys.stderr) + print("Error interrogating", file=sys.stderr) print(traceback.format_exc(), file=sys.stderr) res += "" -- cgit v1.2.3 From 65be1df7bb55b21a3d76630a397c820218cbd12a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 31 Dec 2022 07:46:04 -0500 Subject: initialize result so not to cause exception on empty results --- modules/interrogate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/interrogate.py') diff --git a/modules/interrogate.py b/modules/interrogate.py index 46935210..6f761c5a 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -135,7 +135,7 @@ class InterrogateModels: return caption[0] def interrogate(self, pil_image): - res = None + res = "" try: -- cgit v1.2.3 From d8d206c1685d1e7027d4af82ed18d106f41d1cc4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 3 Jan 2023 11:01:04 -0500 Subject: add state to interrogate --- modules/interrogate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'modules/interrogate.py') diff --git a/modules/interrogate.py b/modules/interrogate.py index 6f761c5a..738d8ff7 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -136,7 +136,8 @@ class InterrogateModels: def interrogate(self, pil_image): res = "" - + shared.state.begin() + shared.state.job = 'interrogate' try: if shared.cmd_opts.lowvram or shared.cmd_opts.medvram: @@ -177,5 +178,6 @@ class InterrogateModels: res += "" self.unload() + shared.state.end() return res -- cgit v1.2.3