Move layer norm to phi #40193

phlrain · 2022-03-06T04:08:37Z

PR types

Breaking changes

PR changes

OPs

Describe

move layer norm to phi

… move_layer_norm_to_phi

paddle-bot-old · 2022-03-06T04:08:41Z

Thanks for your contribution!
Please wait for the result of CI firstly. See Paddle CI Manual for details.

… move_layer_norm_to_phi

MingMingShangTian · 2022-03-11T02:16:00Z

paddle/fluid/operators/fused/fused_layernorm_residual_dropout_bias_test.cu

@@ -264,6 +264,7 @@ struct TestFusedLayernormResidualDropoutBias {

 template <typename T>
 static void BaseTest(const bool is_fp16 = false) {
+  std::cerr << "1" << std::endl;


这个调试信息删除

MingMingShangTian · 2022-03-11T02:16:53Z

paddle/fluid/operators/fused/fused_layernorm_residual_dropout_bias_test.cu

@@ -14,6 +14,7 @@ limitations under the License. */

 #include <time.h>

+#include <iostream>


下边的调试信息删除了，这里也要删除

MingMingShangTian · 2022-03-11T02:20:31Z

paddle/phi/kernels/cpu/layer_norm_grad_kernel.cc

+  DenseTensor temp_norm;
+  if (d_scale || d_x) {
+    x_tmp.Resize(matrix_shape);
+    temp.mutable_data<T>(matrix_shape, dev_ctx.GetPlace());


用 dev_ctx.template Alloc() 替换 mutable_data

MingMingShangTian · 2022-03-11T02:20:42Z

paddle/phi/kernels/cpu/layer_norm_grad_kernel.cc

+    x_tmp.Resize(matrix_shape);
+    temp.mutable_data<T>(matrix_shape, dev_ctx.GetPlace());
+
+    temp_norm.mutable_data<T>(matrix_shape, dev_ctx.GetPlace());


MingMingShangTian · 2022-03-11T02:21:14Z

paddle/phi/kernels/cpu/layer_norm_grad_kernel.cc

+  }
+
+  if (d_bias) {
+    d_bias->mutable_data<T>(dev_ctx.GetPlace());


MingMingShangTian · 2022-03-11T02:21:24Z

paddle/phi/kernels/cpu/layer_norm_grad_kernel.cc

+    colwise_sum(dev_ctx, d_y, d_bias);
+  }
+  if (d_scale) {
+    d_scale->mutable_data<T>(dev_ctx.GetPlace());


MingMingShangTian · 2022-03-11T02:22:33Z

paddle/phi/kernels/cpu/layer_norm_grad_kernel.cc

+    auto dx_dim = d_x->dims();
+    DenseTensor temp_vec;
+    temp_vec.mutable_data<T>(vec_shape, dev_ctx.GetPlace());
+


还有这的2处

MingMingShangTian · 2022-03-11T02:24:27Z

paddle/phi/kernels/cpu/layer_norm_kernel.cc

+
+  y->mutable_data<T>(dev_ctx.GetPlace());
+  mean->mutable_data<T>(dev_ctx.GetPlace());
+  var->mutable_data<T>(dev_ctx.GetPlace());


MingMingShangTian · 2022-03-11T02:27:20Z

paddle/phi/kernels/funcs/layer_norm_util.h

+
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/device_context.h"


这个头文件应该可以去掉

MingMingShangTian · 2022-03-11T02:32:19Z

python/paddle/fluid/tests/unittests/test_layer_norm_op.py

@@ -215,6 +215,8 @@ def test_with_place(place,
                                  for name in ['x', 'scale', 'bias', 'y@GRAD']
                              },
                              fetch_list=fetch_list)
+                # print(y)
+                # print(out[0])


调试信息删除

注释删一下

MingMingShangTian · 2022-03-11T02:32:32Z

python/paddle/fluid/tests/unittests/test_layer_norm_op.py

@@ -229,6 +231,7 @@ def test_with_place(place,
                                        "bias_grad")

        places = [core.CPUPlace()]
+        places = []


zyfncg · 2022-03-11T02:44:34Z

paddle/phi/kernels/cpu/layer_norm_grad_kernel.cc

+      // dy_dx
+      phi::funcs::ElementwiseCompute<funcs::MultiplyFunctor<T>, T, T>(
+          dev_ctx, d_y, *scale, /*axis*/ 1, funcs::MultiplyFunctor<T>(), &temp);
+      paddle::framework::TensorCopy(temp, dev_ctx.GetPlace(), dev_ctx, d_x);


可以尝试使用Copy kernel替换TensorCopy

zyfncg · 2022-03-11T02:44:57Z

paddle/phi/kernels/cpu/layer_norm_grad_kernel.cc

+          &temp);
+    } else {
+      // dy_dx
+      paddle::framework::TensorCopy(d_y, dev_ctx.GetPlace(), dev_ctx, d_x);


zyfncg · 2022-03-11T02:48:09Z

paddle/phi/kernels/gpu/layer_norm_grad_kernel.cu

+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/layer_norm_util.h"
+#include "paddle/phi/kernels/layer_norm_grad_kernel.h"


layer_norm_grad_kernel.h放在开头

头文件好像被自动调整了

zyfncg · 2022-03-11T02:48:40Z

paddle/phi/kernels/gpu/layer_norm_grad_kernel.cu

+  auto *var_data = variance.data<U>();
+
+  auto *d_x_data =
+      (d_x == nullptr ? nullptr : d_x->mutable_data<T>(dev_ctx.GetPlace()));


dev_ctx. Alloc

zyfncg · 2022-03-11T02:49:36Z

paddle/phi/kernels/gpu/layer_norm_kernel.cu

+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/layer_norm_util.h"
+#include "paddle/phi/kernels/layer_norm_kernel.h"


layer_norm_kernel.h放在开头

zyfncg · 2022-03-11T02:52:25Z

paddle/phi/kernels/layer_norm_kernel.h

+
+#pragma once
+
+#include "paddle/phi/api/include/tensor.h"


"paddle/phi/api/include/tensor.h" 引用可以去掉

这是因为下面有一个函数使用了stream_t

gpuStream_t在paddle/phi/backends/gpu/gpu_declas.h中，phi core，kernels等目录中不能include api目录下的文件

… move_layer_norm_to_phi

zyfncg · 2022-03-17T02:49:54Z

python/paddle/fluid/tests/unittests/test_layer_norm_op.py

+                # print(y)
+                # print(out[0])


[TODO] 后续删除

XiaoguangHu01

LGTM

* update * fix bugs; test=develop * update; test=develop * fix test compile error; test=develop * fix cpu compile error; test=develop * fix test error; test=develo * fix layer_norm_op plugin error; test=develop * fix error; test=develop * fix test bug; test=develop * update; test=develop * polish code; test=develop * fix bugs; test=develop * remove unused depency; test=develop * polish code; test=develop

phlrain added 3 commits March 3, 2022 14:09

update

6064b49

fix bugs; test=develop

09a40fa

Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

3b26bf0

… move_layer_norm_to_phi

phlrain and others added 11 commits March 6, 2022 04:10

update; test=develop

6f2833d

fix test compile error; test=develop

4b1cb8a

Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

13d4751

… move_layer_norm_to_phi

fix cpu compile error; test=develop

f533350

fix test error; test=develo

521f190

fix layer_norm_op plugin error; test=develop

a1af264

fix error; test=develop

79e3f4b

Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

6ebdc83

… move_layer_norm_to_phi

fix test bug; test=develop

9504426

Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

4fcad84

… move_layer_norm_to_phi

Merge branch 'develop' into move_layer_norm_to_phi

7bbbec0

MingMingShangTian reviewed Mar 11, 2022

View reviewed changes

zyfncg reviewed Mar 11, 2022

View reviewed changes

phlrain added 3 commits March 11, 2022 06:24

Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

334427c

… move_layer_norm_to_phi

Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

88d15dc

… move_layer_norm_to_phi

update; test=develop

09bf412

phlrain added 9 commits March 11, 2022 10:04

Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

63bc6b5

… move_layer_norm_to_phi

polish code; test=develop

7dd3e9d

Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

39eb6ff

… move_layer_norm_to_phi

fix bugs; test=develop

c527480

Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

7b3681a

… move_layer_norm_to_phi

Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

e46bd87

… move_layer_norm_to_phi

remove unused depency; test=develop

72a18c9

Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

fa6c9e6

… move_layer_norm_to_phi

polish code; test=develop

871f90b

zyfncg approved these changes Mar 17, 2022

View reviewed changes

XiaoguangHu01 approved these changes Mar 17, 2022

View reviewed changes

phlrain merged commit 681a686 into PaddlePaddle:develop Mar 17, 2022

		@@ -14,6 +14,7 @@ limitations under the License. */

		#include <time.h>

		#include <iostream>

Move layer norm to phi #40193

Move layer norm to phi #40193

Conversation

phlrain commented Mar 6, 2022

PR types

PR changes

Describe

paddle-bot-old bot commented Mar 6, 2022

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

XiaoguangHu01 left a comment

Choose a reason for hiding this comment