From de04573438bc111f137359b8f4998780bf315275 Mon Sep 17 00:00:00 2001 From: w-e-w <40751091+w-e-w@users.noreply.github.com> Date: Thu, 28 Dec 2023 06:22:51 +0900 Subject: create utility truncate_path utli.truncate_path(target_path, base_path) return the target_path relative to base_path if target_path is a sub path of base_path else return the absolute path --- modules/util.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'modules/util.py') diff --git a/modules/util.py b/modules/util.py index 60afc067..4861bcb0 100644 --- a/modules/util.py +++ b/modules/util.py @@ -2,7 +2,7 @@ import os import re from modules import shared -from modules.paths_internal import script_path +from modules.paths_internal import script_path, cwd def natural_sort_key(s, regex=re.compile('([0-9]+)')): @@ -56,3 +56,13 @@ def ldm_print(*args, **kwargs): return print(*args, **kwargs) + + +def truncate_path(target_path, base_path=cwd): + abs_target, abs_base = os.path.abspath(target_path), os.path.abspath(base_path) + try: + if os.path.commonpath([abs_target, abs_base]) == abs_base: + return os.path.relpath(abs_target, abs_base) + except ValueError: + pass + return abs_target -- cgit v1.2.3 From d9034b48a526f0a0c3e8f0dbf7c171bf4f0597fd Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 4 Jan 2024 00:16:58 +0200 Subject: Avoid unnecessary `isfile`/`exists` calls --- modules/cache.py | 17 ++++++++--------- modules/extensions.py | 11 ++++++----- modules/extra_networks.py | 7 ++++--- modules/infotext_utils.py | 4 +++- modules/launch_utils.py | 7 ++++--- modules/postprocessing.py | 7 ++++--- modules/shared_init.py | 4 +++- modules/ui_gradio_extensions.py | 8 +++----- modules/ui_loadsave.py | 5 +++-- modules/util.py | 6 +++--- 10 files changed, 41 insertions(+), 35 deletions(-) (limited to 'modules/util.py') diff --git a/modules/cache.py b/modules/cache.py index 2d37e7b9..a9822a0e 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -62,16 +62,15 @@ def cache(subsection): if cache_data is None: with cache_lock: if cache_data is None: - if not os.path.isfile(cache_filename): + try: + with open(cache_filename, "r", encoding="utf8") as file: + cache_data = json.load(file) + except FileNotFoundError: + cache_data = {} + except Exception: + os.replace(cache_filename, os.path.join(script_path, "tmp", "cache.json")) + print('[ERROR] issue occurred while trying to read cache.json, move current cache to tmp/cache.json and create new cache') cache_data = {} - else: - try: - with open(cache_filename, "r", encoding="utf8") as file: - cache_data = json.load(file) - except Exception: - os.replace(cache_filename, os.path.join(script_path, "tmp", "cache.json")) - print('[ERROR] issue occurred while trying to read cache.json, move current cache to tmp/cache.json and create new cache') - cache_data = {} s = cache_data.get(subsection, {}) cache_data[subsection] = s diff --git a/modules/extensions.py b/modules/extensions.py index 1899cd52..99e7ee60 100644 --- a/modules/extensions.py +++ b/modules/extensions.py @@ -32,11 +32,12 @@ class ExtensionMetadata: self.config = configparser.ConfigParser() filepath = os.path.join(path, self.filename) - if os.path.isfile(filepath): - try: - self.config.read(filepath) - except Exception: - errors.report(f"Error reading {self.filename} for extension {canonical_name}.", exc_info=True) + # `self.config.read()` will quietly swallow OSErrors (which FileNotFoundError is), + # so no need to check whether the file exists beforehand. + try: + self.config.read(filepath) + except Exception: + errors.report(f"Error reading {self.filename} for extension {canonical_name}.", exc_info=True) self.canonical_name = self.config.get("Extension", "Name", fallback=canonical_name) self.canonical_name = canonical_name.lower().strip() diff --git a/modules/extra_networks.py b/modules/extra_networks.py index b9533677..cd030fa3 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -215,9 +215,10 @@ def get_user_metadata(filename): metadata = {} try: - if os.path.isfile(metadata_filename): - with open(metadata_filename, "r", encoding="utf8") as file: - metadata = json.load(file) + with open(metadata_filename, "r", encoding="utf8") as file: + metadata = json.load(file) + except FileNotFoundError: + pass except Exception as e: errors.display(e, f"reading extra network user metadata from {metadata_filename}") diff --git a/modules/infotext_utils.py b/modules/infotext_utils.py index e582ee47..6978a0bf 100644 --- a/modules/infotext_utils.py +++ b/modules/infotext_utils.py @@ -453,9 +453,11 @@ def connect_paste(button, paste_fields, input_comp, override_settings_component, def paste_func(prompt): if not prompt and not shared.cmd_opts.hide_ui_dir_config: filename = os.path.join(data_path, "params.txt") - if os.path.exists(filename): + try: with open(filename, "r", encoding="utf8") as file: prompt = file.read() + except OSError: + pass params = parse_generation_parameters(prompt) script_callbacks.infotext_pasted_callback(prompt, params) diff --git a/modules/launch_utils.py b/modules/launch_utils.py index c2cbd8ce..febd8c24 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -245,9 +245,10 @@ def list_extensions(settings_file): settings = {} try: - if os.path.isfile(settings_file): - with open(settings_file, "r", encoding="utf8") as file: - settings = json.load(file) + with open(settings_file, "r", encoding="utf8") as file: + settings = json.load(file) + except FileNotFoundError: + pass except Exception: errors.report("Could not load settings", exc_info=True) diff --git a/modules/postprocessing.py b/modules/postprocessing.py index 7850328f..7449b0dc 100644 --- a/modules/postprocessing.py +++ b/modules/postprocessing.py @@ -97,11 +97,12 @@ def run_postprocessing(extras_mode, image, image_folder, input_dir, output_dir, if pp.caption: caption_filename = os.path.splitext(fullfn)[0] + ".txt" - if os.path.isfile(caption_filename): + existing_caption = "" + try: with open(caption_filename, encoding="utf8") as file: existing_caption = file.read().strip() - else: - existing_caption = "" + except FileNotFoundError: + pass action = shared.opts.postprocessing_existing_caption_action if action == 'Prepend' and existing_caption: diff --git a/modules/shared_init.py b/modules/shared_init.py index d3fb687e..586be342 100644 --- a/modules/shared_init.py +++ b/modules/shared_init.py @@ -18,8 +18,10 @@ def initialize(): shared.options_templates = shared_options.options_templates shared.opts = options.Options(shared_options.options_templates, shared_options.restricted_opts) shared.restricted_opts = shared_options.restricted_opts - if os.path.exists(shared.config_filename): + try: shared.opts.load(shared.config_filename) + except FileNotFoundError: + pass from modules import devices devices.device, devices.device_interrogate, devices.device_gfpgan, devices.device_esrgan, devices.device_codeformer = \ diff --git a/modules/ui_gradio_extensions.py b/modules/ui_gradio_extensions.py index a86c368e..f5278d22 100644 --- a/modules/ui_gradio_extensions.py +++ b/modules/ui_gradio_extensions.py @@ -35,13 +35,11 @@ def css_html(): return f'' for cssfile in scripts.list_files_with_name("style.css"): - if not os.path.isfile(cssfile): - continue - head += stylesheet(cssfile) - if os.path.exists(os.path.join(data_path, "user.css")): - head += stylesheet(os.path.join(data_path, "user.css")) + user_css = os.path.join(data_path, "user.css") + if os.path.exists(user_css): + head += stylesheet(user_css) return head diff --git a/modules/ui_loadsave.py b/modules/ui_loadsave.py index 693ff75c..2555cdb6 100644 --- a/modules/ui_loadsave.py +++ b/modules/ui_loadsave.py @@ -26,8 +26,9 @@ class UiLoadsave: self.ui_defaults_review = None try: - if os.path.exists(self.filename): - self.ui_settings = self.read_from_file() + self.ui_settings = self.read_from_file() + except FileNotFoundError: + pass except Exception as e: self.error_loading = True errors.display(e, "loading settings") diff --git a/modules/util.py b/modules/util.py index 4861bcb0..d503f267 100644 --- a/modules/util.py +++ b/modules/util.py @@ -21,11 +21,11 @@ def html_path(filename): def html(filename): path = html_path(filename) - if os.path.exists(path): + try: with open(path, encoding="utf8") as file: return file.read() - - return "" + except OSError: + return "" def walk_files(path, allowed_extensions=None): -- cgit v1.2.3 From 420f56c2e85ebbd3f530cf2c7b22022fda13ae13 Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Thu, 4 Jan 2024 02:28:05 +0300 Subject: mass file lister as an attempt to tackle #14507 --- modules/extra_networks.py | 5 ++-- modules/ui_extra_networks.py | 18 ++++++++---- modules/util.py | 70 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 8 deletions(-) (limited to 'modules/util.py') diff --git a/modules/extra_networks.py b/modules/extra_networks.py index b9533677..04249dff 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -206,7 +206,7 @@ def parse_prompts(prompts): return res, extra_data -def get_user_metadata(filename): +def get_user_metadata(filename, lister=None): if filename is None: return {} @@ -215,7 +215,8 @@ def get_user_metadata(filename): metadata = {} try: - if os.path.isfile(metadata_filename): + exists = lister.exists(metadata_filename) if lister else os.path.exists(metadata_filename) + if exists: with open(metadata_filename, "r", encoding="utf8") as file: metadata = json.load(file) except Exception as e: diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index e1c679ec..c06c8664 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -3,7 +3,7 @@ import os.path import urllib.parse from pathlib import Path -from modules import shared, ui_extra_networks_user_metadata, errors, extra_networks +from modules import shared, ui_extra_networks_user_metadata, errors, extra_networks, util from modules.images import read_info_from_image, save_image_with_geninfo import gradio as gr import json @@ -107,6 +107,7 @@ class ExtraNetworksPage: self.allow_negative_prompt = False self.metadata = {} self.items = {} + self.lister = util.MassFileLister() def refresh(self): pass @@ -123,7 +124,7 @@ class ExtraNetworksPage: def link_preview(self, filename): quoted_filename = urllib.parse.quote(filename.replace('\\', '/')) - mtime = os.path.getmtime(filename) + mtime, _ = self.lister.mctime(filename) return f"./sd_extra_networks/thumb?filename={quoted_filename}&mtime={mtime}" def search_terms_from_path(self, filename, possible_directories=None): @@ -137,6 +138,8 @@ class ExtraNetworksPage: return "" def create_html(self, tabname): + self.lister.reset() + items_html = '' self.metadata = {} @@ -282,10 +285,10 @@ class ExtraNetworksPage: List of default keys used for sorting in the UI. """ pth = Path(path) - stat = pth.stat() + mtime, ctime = self.lister.mctime(path) return { - "date_created": int(stat.st_ctime or 0), - "date_modified": int(stat.st_mtime or 0), + "date_created": int(mtime), + "date_modified": int(ctime), "name": pth.name.lower(), "path": str(pth.parent).lower(), } @@ -298,7 +301,7 @@ class ExtraNetworksPage: potential_files = sum([[path + "." + ext, path + ".preview." + ext] for ext in allowed_preview_extensions()], []) for file in potential_files: - if os.path.isfile(file): + if self.lister.exists(file): return self.link_preview(file) return None @@ -308,6 +311,9 @@ class ExtraNetworksPage: Find and read a description file for a given path (without extension). """ for file in [f"{path}.txt", f"{path}.description.txt"]: + if not self.lister.exists(file): + continue + try: with open(file, "r", encoding="utf-8", errors="replace") as f: return f.read() diff --git a/modules/util.py b/modules/util.py index 4861bcb0..c2a27590 100644 --- a/modules/util.py +++ b/modules/util.py @@ -66,3 +66,73 @@ def truncate_path(target_path, base_path=cwd): except ValueError: pass return abs_target + + +class MassFileListerCachedDir: + """A class that caches file metadata for a specific directory.""" + + def __init__(self, dirname): + self.files = None + self.files_cased = None + self.dirname = dirname + + stats = ((x.name, x.stat(follow_symlinks=False)) for x in os.scandir(self.dirname)) + files = [(n, s.st_mtime, s.st_ctime) for n, s in stats] + self.files = {x[0].lower(): x for x in files} + self.files_cased = {x[0]: x for x in files} + + +class MassFileLister: + """A class that provides a way to check for the existence and mtime/ctile of files without doing more than one stat call per file.""" + + def __init__(self): + self.cached_dirs = {} + + def find(self, path): + """ + Find the metadata for a file at the given path. + + Returns: + tuple or None: A tuple of (name, mtime, ctime) if the file exists, or None if it does not. + """ + + dirname, filename = os.path.split(path) + + cached_dir = self.cached_dirs.get(dirname) + if cached_dir is None: + cached_dir = MassFileListerCachedDir(dirname) + self.cached_dirs[dirname] = cached_dir + + stats = cached_dir.files_cased.get(filename) + if stats is not None: + return stats + + stats = cached_dir.files.get(filename.lower()) + if stats is None: + return None + + try: + os_stats = os.stat(path, follow_symlinks=False) + return filename, os_stats.st_mtime, os_stats.st_ctime + except Exception: + return None + + def exists(self, path): + """Check if a file exists at the given path.""" + + return self.find(path) is not None + + def mctime(self, path): + """ + Get the modification and creation times for a file at the given path. + + Returns: + tuple: A tuple of (mtime, ctime) if the file exists, or (0, 0) if it does not. + """ + + stats = self.find(path) + return (0, 0) if stats is None else stats[1:3] + + def reset(self): + """Clear the cache of all directories.""" + self.cached_dirs.clear() -- cgit v1.2.3 From 542611cce4882b9acb885372bfbdf545e699fdfa Mon Sep 17 00:00:00 2001 From: w-e-w <40751091+w-e-w@users.noreply.github.com> Date: Sat, 10 Feb 2024 05:39:01 +0900 Subject: walk_files extensions case insensitive --- modules/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/util.py') diff --git a/modules/util.py b/modules/util.py index ee373e92..8d1aea44 100644 --- a/modules/util.py +++ b/modules/util.py @@ -42,7 +42,7 @@ def walk_files(path, allowed_extensions=None): for filename in sorted(files, key=natural_sort_key): if allowed_extensions is not None: _, ext = os.path.splitext(filename) - if ext not in allowed_extensions: + if ext.lower() not in allowed_extensions: continue if not shared.opts.list_hidden_files and ("/." in root or "\\." in root): -- cgit v1.2.3