Merge pull request #343 from ooooo-create/accuracy_fused_layer_norm

[Accuracy diff No.91] Fix accuracy diff for paddle.incubate.nn.functional.fused_layer_norm API

Author: cangtianhuang

tester/api_config/6_accuracy_amp/accuracy_amp_gpu_error_dtype_diff.txt

@@ -8947,7 +8947,6 @@ paddle.vision.ops.roi_align(x=Tensor([8, 256, 96, 144],"float16"), boxes=Tensor(
 paddle.vision.ops.roi_align(x=Tensor([8, 256, 96, 148],"float16"), boxes=Tensor([531, 4],"float32"), boxes_num=Tensor([8],"int32"), output_size=7, spatial_scale=0.125, sampling_ratio=0, aligned=True, )
 paddle.incubate.nn.functional.fused_layer_norm(Tensor([101, 64],"float16"), norm_weight=Tensor([64],"float32"), norm_bias=Tensor([64],"float32"), epsilon=1e-05, begin_norm_axis=1, bias=Tensor([64],"float16"), residual=Tensor([101, 64],"float16"), )
 paddle.incubate.nn.functional.fused_layer_norm(Tensor([16, 256],"float16"), Tensor([256],"float32"), Tensor([256],"float32"), 1e-05, begin_norm_axis=1, bias=Tensor([256],"float16"), residual=Tensor([16, 256],"float16"), residual_alpha=0.69204696, )
-paddle.incubate.nn.functional.fused_layer_norm(Tensor([2, 64],"float16"), norm_weight=Tensor([64],"float32"), norm_bias=Tensor([64],"float32"), epsilon=1e-05, begin_norm_axis=1, bias=Tensor([64],"float16"), residual=Tensor([2, 1, 64],"float16"), )
 paddle.incubate.nn.functional.fused_layer_norm(Tensor([58, 64],"float16"), norm_weight=Tensor([64],"float32"), norm_bias=Tensor([64],"float32"), epsilon=1e-05, residual_alpha=1.4142135623730951, begin_norm_axis=1, bias=Tensor([64],"float16"), residual=Tensor([58, 64],"float16"), )
 paddle.incubate.nn.functional.fused_layer_norm(Tensor([59, 64],"float16"), norm_weight=Tensor([64],"float32"), norm_bias=Tensor([64],"float32"), epsilon=1e-05, begin_norm_axis=1, bias=Tensor([64],"float16"), residual=Tensor([59, 64],"float16"), )
 paddle.incubate.nn.functional.fused_layer_norm(Tensor([60, 64],"float16"), norm_weight=Tensor([64],"float32"), norm_bias=Tensor([64],"float32"), epsilon=1e-05, begin_norm_axis=1, bias=Tensor([64],"float16"), residual=Tensor([60, 64],"float16"), )

tester/base_config.yaml

@@ -29,6 +29,7 @@ special_accuracy_atol_rtol:
   paddle.Tensor.logcumsumexp: [1.0, 1.0]
   paddle.incubate.nn.functional.fused_bias_act: [1, 0.01]
   paddle.incubate.nn.functional.fused_rms_norm : [3, 0.5]
+  paddle.incubate.nn.functional.fused_layer_norm: [1, 0.01]
   paddle.lerp : [5, 0.05]
   paddle.nn.functional.upsample: [0.5, 1.5]
   paddle.nn.functional.interpolate: [0.5, 1.5]

tester/paddle_to_torch/rules.py

@@ -2469,8 +2469,8 @@ def fused_layer_norm(x, norm_weight, norm_bias, epsilon, residual_alpha=1.0, beg
         # using banker's rounding
         if quant_round_type == 0:
             x = torch.round(x)
-        else: #  Round to nearest if type != 0
-            x = torch.floor(x + 0.5)
+        else: # round half away from zero
+            x = torch.where(x >= 0, torch.floor(x + 0.5), torch.ceil(x - 0.5))
         x = torch.clamp(x, min=quant_min_bound, max=quant_max_bound).to(torch.int8)
             
     return (x, out_residual, out_mean, out_var)