[fix] fix flash attn

duanjunwen · Nov 14, 2024 · 2154aaf · 2154aaf
1 parent a259651
commit 2154aaf
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/colossalai/shardformer/layer/attn.py b/colossalai/shardformer/layer/attn.py
@@ -644,7 +644,8 @@ def forward(
         max_seqlen_half = max_seqlen // 2
 
         misc_kwargs = {
-            "window_size": (-1, -1),
+            "window_size_left": -1,
+            "window_size_right": -1,
             "alibi_slopes": None,
             "softmax_scale": q.shape[-1] ** -0.5 if softmax_scale is None else softmax_scale,
             "dropout_p": dropout_p,