add finetune fused & add mc2 #8139
Conversation
|
Thanks for your contribution! |
There was a problem hiding this comment.
这两个MC2属于sp的一部分，暂不移动？
| @@ -228,6 +228,11 @@ def scaled_dot_product_attention( | |||
| alibi = alibi.reshape([bsz, num_heads, 1, -1]) | |||
| attention_mask = attention_mask.cast(alibi.dtype) + alibi | |||
| if get_env_device() == "npu": | |||
| if attention_mask is not None: | |||
| attention_mask = attention_mask.astype("bool") | |||
There was a problem hiding this comment.
这个地方看下在外面判断attn_mask一次，不是bool类型才cast, 可以提高性能。看下不在flash_attn里面cast, 在传入Transformer之前判断并cast一次就行？
| @@ -239,6 +244,7 @@ def scaled_dot_product_attention( | |||
| attention_mask is None, | |||
| True, | |||
| False, | |||
| is_casual | |||
There was a problem hiding this comment.
这里建议别判断casual了，SFT/lora容易出错, 直接给False。或者能确定casual能判断正确。
llm/finetune_generation.py
Outdated
|
|
||
| @dataclass | ||
| @add_start_docstrings(ModelArgument.__doc__) | ||
| class SFTModelArguments(ModelArgument): |
There was a problem hiding this comment.
整体挪到arguments.py里面，注意pylint。
| @@ -240,7 +240,7 @@ def scaled_dot_product_attention( | |||
| attention_mask is None, | |||
| True, | |||
| False, | |||
| False, | |||
| False | |||
| if is_casual and alibi is None: | ||
| attention_mask = None | ||
| if get_env_device != "npu": | ||
| is_casual = is_casual_mask(attention_mask) |
There was a problem hiding this comment.
如果这里没问题，可以从这里下发一个变量传到FA的输入中。
| @@ -0,0 +1,550 @@ | |||
| # Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. | |||
Codecov ReportAttention: Patch coverage is
Additional details and impacted files@@ Coverage Diff @@
## develop #8139 +/- ##
===========================================
- Coverage 55.15% 55.10% -0.05%
===========================================
Files 601 602 +1
Lines 91764 91850 +86
===========================================
+ Hits 50611 50613 +2
- Misses 41153 41237 +84 ☔ View full report in Codecov by Sentry. |
| import os | ||
|
|
||
| import paddle | ||
| import paddle_custom_device |
There was a problem hiding this comment.
这个是NPU执行MC2才会引用到的文件
| ScatterOp, | ||
| mark_as_sequence_parallel_parameter, | ||
| ) | ||
|
|
||
| if int(os.getenv("MC2", 0)): |
NINGBENZHE
force-pushed
the
develop
branch
3 times, most recently
from
a356945 to
768f465
Compare
| @@ -102,6 +106,64 @@ class ModelArgument: | |||
| default=None, metadata={"help": "Build-in pretrained model name or the path to local model."} | |||
| ) | |||
| use_flash_attention: bool = field(default=False, metadata={"help": "Whether to use flash attention"}) | |||
| tokenizer_name_or_path: Optional[str] = field( | |||
There was a problem hiding this comment.
后续建议run_pretrain.py也能直接引入arguments的选项
No description provided.