theluyuan/DiffSynth-Studio
1
0
Fork 0
mirror of https://github.com/modelscope/DiffSynth-Studio.git synced 2026-03-24 01:48:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

support sageattn

Browse Source
This commit is contained in:
philipy1219
2025-03-02 15:09:21 +08:00
parent ceb92ee7aa
commit 852c3d831f
1 changed files with 12 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
diffsynth/models/wan_video_dit.py
View File

@@ -18,6 +18,12 @@ try:
except ModuleNotFoundError: except ModuleNotFoundError:
FLASH_ATTN_2_AVAILABLE = False FLASH_ATTN_2_AVAILABLE = False
try:
from sageattention import sageattn
SAGE_ATTN_AVAILABLE = True
except ModuleNotFoundError:
SAGE_ATTN_AVAILABLE = False
import warnings import warnings
@@ -127,6 +133,12 @@ def flash_attention(
causal=causal, causal=causal,
window_size=window_size, window_size=window_size,
deterministic=deterministic).unflatten(0, (b, lq)) deterministic=deterministic).unflatten(0, (b, lq))
elif SAGE_ATTN_AVAILABLE:
q = q.unsqueeze(0).transpose(1, 2).to(dtype)
k = k.unsqueeze(0).transpose(1, 2).to(dtype)
v = v.unsqueeze(0).transpose(1, 2).to(dtype)
x = sageattn(q, k, v, dropout_p=dropout_p, is_causal=causal)
x = x.transpose(1, 2).contiguous()
else: else:
q = q.unsqueeze(0).transpose(1, 2).to(dtype) q = q.unsqueeze(0).transpose(1, 2).to(dtype)
k = k.unsqueeze(0).transpose(1, 2).to(dtype) k = k.unsqueeze(0).transpose(1, 2).to(dtype)