Merge branch 'dev' into release_candidate

1 files changed, 44 insertions, 2 deletions

diff --git a/modules/errors.py b/modules/errors.py
index f6b80dbb..5271a9fe 100644
--- a/modules/errors.py
+++ b/modules/errors.py
@@ -1,8 +1,42 @@
 import sys

+import textwrap

 import traceback

 

 

+exception_records = []

+

+

+def record_exception():

+    _, e, tb = sys.exc_info()

+    if e is None:

+        return

+

+    if exception_records and exception_records[-1] == e:

+        return

+

+    exception_records.append((e, tb))

+

+    if len(exception_records) > 5:

+        exception_records.pop(0)

+

+

+def report(message: str, *, exc_info: bool = False) -> None:

+    """

+    Print an error message to stderr, with optional traceback.

+    """

+

+    record_exception()

+

+    for line in message.splitlines():

+        print("***", line, file=sys.stderr)

+    if exc_info:

+        print(textwrap.indent(traceback.format_exc(), "    "), file=sys.stderr)

+        print("---", file=sys.stderr)

+

+

 def print_error_explanation(message):

+    record_exception()

+

     lines = message.strip().split("\n")

     max_len = max([len(x) for x in lines])

 

@@ -12,9 +46,15 @@ def print_error_explanation(message):
     print('=' * max_len, file=sys.stderr)

 

 

-def display(e: Exception, task):

+def display(e: Exception, task, *, full_traceback=False):

+    record_exception()

+

     print(f"{task or 'error'}: {type(e).__name__}", file=sys.stderr)

-    print(traceback.format_exc(), file=sys.stderr)

+    te = traceback.TracebackException.from_exception(e)

+    if full_traceback:

+        # include frames leading up to the try-catch block

+        te.stack = traceback.StackSummary(traceback.extract_stack()[:-2] + te.stack)

+    print(*te.format(), sep="", file=sys.stderr)

 

     message = str(e)

     if "copying a param with shape torch.Size([640, 1024]) from checkpoint, the shape in current model is torch.Size([640, 768])" in message:

@@ -28,6 +68,8 @@ already_displayed = {}
 

 

 def display_once(e: Exception, task):

+    record_exception()

+

     if task in already_displayed:

         return