oops learned value residual mix does not apply to cross attention

lucidrains · Dec 4, 2024 · 3e92969 · 3e92969
1 parent 544c699
commit 3e92969
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 setup(
   name = 'x-transformers',
   packages = find_packages(exclude=['examples']),
-  version = '1.42.23',
+  version = '1.42.24',
   license='MIT',
   description = 'X-Transformers - Pytorch',
   author = 'Phil Wang',

diff --git a/x_transformers/x_transformers.py b/x_transformers/x_transformers.py
@@ -1849,7 +1849,7 @@ def __init__(
                 is_first_self_attn = False
             elif layer_type == 'c':
                 cross_attn_learned_value_residual = learned_value_residual_mix and not is_first_cross_attn
-                layer = Attention(dim, heads = heads, learned_value_residual_mix = cross_attn_learned_value_residual, **{**attn_kwargs, **cross_attn_kwargs})
+                layer = Attention(dim, heads = heads, **{**attn_kwargs, **cross_attn_kwargs})
                 is_first_cross_attn = False
             elif layer_type == 'f':
                 layer = FeedForward(dim, **ff_kwargs)