diff options
| author | C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com> | 2022-10-17 19:18:59 +0000 |
|---|---|---|
| committer | AUTOMATIC1111 <16777216c@gmail.com> | 2022-10-18 08:53:04 +0000 |
| commit | 84823275e896bcc1f7cb4ce098ae3c5d05e17b9a (patch) | |
| tree | 77e89a581d36b5b0e7f50dbbdffa93ecb0883391 /modules/sd_hijack_optimizations.py | |
| parent | 2043c4a231eef838bb15044f502b864b55885037 (diff) | |
| download | stable-diffusion-webui-gfx803-84823275e896bcc1f7cb4ce098ae3c5d05e17b9a.tar.gz stable-diffusion-webui-gfx803-84823275e896bcc1f7cb4ce098ae3c5d05e17b9a.tar.bz2 stable-diffusion-webui-gfx803-84823275e896bcc1f7cb4ce098ae3c5d05e17b9a.zip | |
readd xformers attnblock
Diffstat (limited to 'modules/sd_hijack_optimizations.py')
| -rw-r--r-- | modules/sd_hijack_optimizations.py | 15 |
1 files changed, 15 insertions, 0 deletions
diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py index 60da7459..7ebef3f0 100644 --- a/modules/sd_hijack_optimizations.py +++ b/modules/sd_hijack_optimizations.py @@ -292,3 +292,18 @@ def cross_attention_attnblock_forward(self, x): return h3
+def xformers_attnblock_forward(self, x):
+ try:
+ h_ = x
+ h_ = self.norm(h_)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+ b, c, h, w = q.shape
+ q, k, v = map(lambda t: rearrange(t, 'b c h w -> b (h w) c'), (q, k, v))
+ out = xformers.ops.memory_efficient_attention(q, k, v)
+ out = rearrange(out, 'b (h w) c -> b c h w', h=h)
+ out = self.proj_out(out)
+ return x + out
+ except NotImplementedError:
+ return cross_attention_attnblock_forward(self, x)
|