From a37bf42909876c339fe7dbcfd70aa6d84c71863e Mon Sep 17 00:00:00 2001 From: vineethanandh Date: Thu, 18 Apr 2024 13:20:47 +0300 Subject: [PATCH] [SW-12028] - set ds config "reduce scatter" to false - FlanT5 is giving a perf drop with "reduce_scatter" value True in DS config Signed-off-by: vineethanandh --- examples/summarization/ds_flan_t5_z3_config_bf16.json | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/summarization/ds_flan_t5_z3_config_bf16.json b/examples/summarization/ds_flan_t5_z3_config_bf16.json index bb48e2889a..91ebe4c22b 100644 --- a/examples/summarization/ds_flan_t5_z3_config_bf16.json +++ b/examples/summarization/ds_flan_t5_z3_config_bf16.json @@ -27,6 +27,7 @@ "contiguous_gradients": true, "sub_group_size": 1e9, "reduce_bucket_size": 1666777, + "reduce_scatter" : false, "stage3_prefetch_bucket_size": "auto", "stage3_param_persistence_threshold": "auto", "stage3_max_live_parameters": 1e9,