From 68217554e586a62ec7e7eb26942b939fffd6b336 Mon Sep 17 00:00:00 2001 From: Reza Yazdani Date: Mon, 22 Aug 2022 22:23:30 +0500 Subject: [PATCH] fix ds-inference without policy --- deepspeed/module_inject/replace_module.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/deepspeed/module_inject/replace_module.py b/deepspeed/module_inject/replace_module.py index 4ae9e5529d0e..c91e6aff52f2 100755 --- a/deepspeed/module_inject/replace_module.py +++ b/deepspeed/module_inject/replace_module.py @@ -650,11 +650,10 @@ def _replace(child, name, conv_linear_layer): new_bias = torch.empty((weight_shape[0]), device=child.weight.device, dtype=child.weight.dtype) - if z_inference: with deepspeed.zero.GatheredParameters(child.bias, modifier_rank=0): new_bias.data.copy_(child.bias.data) - else: + elif child.bias: new_bias.data.copy_(child.bias.data) return LinearAllreduce(data, child.bias if child.bias is None else \ torch.nn.parameter.Parameter(new_bias.to(torch.cuda.current_device())), mp_group) @@ -701,7 +700,9 @@ def _slice_embedding(child, name, conv_linear_layer): child.weight.shape[1] // mp_size), device=child.weight.device, dtype=child.weight.dtype) - data = mp_replace.copy(new_weight, child.weight.ds_tensor.data) + data = mp_replace.copy(new_weight, + child.weight.ds_tensor.data if hasattr(child.weight, 'ds_tensor') else \ + child.weight.data) new_embedding = nn.Embedding(child.weight.shape[0], child.weight.shape[1] // mp_size) new_embedding.weight.data.copy_(data)