From 11c7419c0ab603c664046f48353d77eb54de026b Mon Sep 17 00:00:00 2001 From: wwbitejotunn Date: Mon, 18 Jul 2022 12:50:11 +0000 Subject: [PATCH 1/9] add yaml and utests of phi softplus add yaml of softplus fix softplus bug in phi --- paddle/phi/api/yaml/legacy_api.yaml | 10 ++++++++++ paddle/phi/api/yaml/legacy_backward.yaml | 11 +++++++++++ .../fluid/tests/unittests/test_activation_op.py | 4 +++- python/paddle/nn/functional/activation.py | 4 +++- 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/paddle/phi/api/yaml/legacy_api.yaml b/paddle/phi/api/yaml/legacy_api.yaml index b9e7361abea7d..378d18156214e 100644 --- a/paddle/phi/api/yaml/legacy_api.yaml +++ b/paddle/phi/api/yaml/legacy_api.yaml @@ -1954,6 +1954,16 @@ func : slice backward : slice_grad +- api : softplus + args : (Tensor x, float beta,float threshold) + output : Tensor + infer_meta : + func : UnchangedInferMeta + param : x + kernel : + func : softplus + backward : softplus_grad + # soft_shrink - api : soft_shrink args : (Tensor x, float lambda) diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml index b4972c68a6477..5e413458c9cfd 100644 --- a/paddle/phi/api/yaml/legacy_backward.yaml +++ b/paddle/phi/api/yaml/legacy_backward.yaml @@ -1870,6 +1870,17 @@ backward : slice_double_grad no_need_buffer : input +- backward_api : softplus_grad + forward : softplus (Tensor x, float beta,float threshold) -> Tensor(out) + args : (Tensor x, Tensor out_grad, float beta, float threshold) + output : Tensor(x_grad) + infer_meta : + func : UnchangedInferMeta + param : [x] + kernel : + func : softplus_grad + inplace : (out_grad -> x_grad) + - backward_api : soft_shrink_grad forward : soft_shrink (Tensor x, float lambda) -> Tensor(out) args : (Tensor x, Tensor out_grad, float lambda) diff --git a/python/paddle/fluid/tests/unittests/test_activation_op.py b/python/paddle/fluid/tests/unittests/test_activation_op.py index 4b817bb22ebf7..37e8827d8145e 100755 --- a/python/paddle/fluid/tests/unittests/test_activation_op.py +++ b/python/paddle/fluid/tests/unittests/test_activation_op.py @@ -2676,6 +2676,7 @@ class TestSoftplus(TestActivation): def setUp(self): self.op_type = "softplus" + self.python_api=paddle.nn.functional.softplus self.init_dtype() beta = 2 @@ -2687,11 +2688,12 @@ def setUp(self): self.inputs = {'X': x} self.attrs = {'beta': beta, "threshold": threshold} self.outputs = {'Out': out} + self.check_eager=True def test_check_grad(self): if self.dtype == np.float16: return - self.check_grad(['X'], 'Out') + self.check_grad(['X'], 'Out',check_eager=True) @unittest.skipIf(not core.is_compiled_with_cuda(), diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py index 7c2dff5247e21..e1dd41763c2b5 100644 --- a/python/paddle/nn/functional/activation.py +++ b/python/paddle/nn/functional/activation.py @@ -1177,7 +1177,9 @@ def softplus(x, beta=1, threshold=20, name=None): x = paddle.to_tensor(np.array([-0.4, -0.2, 0.1, 0.3])) out = F.softplus(x) # [0.513015, 0.598139, 0.744397, 0.854355] """ - if in_dynamic_mode(): + if in_dygraph_mode(): + return _C_ops.final_state_softplus(x, beta, threshold) + elif in_dynamic_mode(): return _C_ops.softplus(x, 'beta', beta, 'threshold', threshold) check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], From cbca3a8dc0d1845d80ee1a0e40d218688304a116 Mon Sep 17 00:00:00 2001 From: wwbitejotunn Date: Tue, 19 Jul 2022 11:23:07 +0000 Subject: [PATCH 2/9] update utests --- python/paddle/fluid/tests/unittests/test_activation_op.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/tests/unittests/test_activation_op.py b/python/paddle/fluid/tests/unittests/test_activation_op.py index 37e8827d8145e..050879730d4a1 100755 --- a/python/paddle/fluid/tests/unittests/test_activation_op.py +++ b/python/paddle/fluid/tests/unittests/test_activation_op.py @@ -2693,7 +2693,9 @@ def setUp(self): def test_check_grad(self): if self.dtype == np.float16: return - self.check_grad(['X'], 'Out',check_eager=True) + if hasattr(self, 'check_eager'): + check_eager = self.check_eager + self.check_grad(['X'], 'Out',check_eager=check_eager) @unittest.skipIf(not core.is_compiled_with_cuda(), From 57b4773155f4aae10caa680bb40ab8ed70df7a8c Mon Sep 17 00:00:00 2001 From: wwbitejotunn Date: Fri, 22 Jul 2022 06:21:57 +0000 Subject: [PATCH 3/9] bug fix --- paddle/phi/api/yaml/legacy_api.yaml | 20 ++++++++-------- paddle/phi/api/yaml/legacy_backward.yaml | 23 ++++++++++--------- .../tests/unittests/test_activation_op.py | 7 +++--- python/paddle/nn/functional/activation.py | 1 + 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/paddle/phi/api/yaml/legacy_api.yaml b/paddle/phi/api/yaml/legacy_api.yaml index 378d18156214e..077f87930ad58 100644 --- a/paddle/phi/api/yaml/legacy_api.yaml +++ b/paddle/phi/api/yaml/legacy_api.yaml @@ -1954,16 +1954,6 @@ func : slice backward : slice_grad -- api : softplus - args : (Tensor x, float beta,float threshold) - output : Tensor - infer_meta : - func : UnchangedInferMeta - param : x - kernel : - func : softplus - backward : softplus_grad - # soft_shrink - api : soft_shrink args : (Tensor x, float lambda) @@ -1985,6 +1975,16 @@ use_gpudnn : true backward : softmax_grad +- api : softplus + args : (Tensor x, float beta,float threshold) + output : Tensor + infer_meta : + func : UnchangedInferMeta + param : [x] + kernel : + func : softplus + backward : softplus_grad + - api : split args : (Tensor x, IntArray num_or_sections, Scalar(int) axis) output : Tensor[] diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml index 5e413458c9cfd..5ed459bbafefd 100644 --- a/paddle/phi/api/yaml/legacy_backward.yaml +++ b/paddle/phi/api/yaml/legacy_backward.yaml @@ -1870,17 +1870,6 @@ backward : slice_double_grad no_need_buffer : input -- backward_api : softplus_grad - forward : softplus (Tensor x, float beta,float threshold) -> Tensor(out) - args : (Tensor x, Tensor out_grad, float beta, float threshold) - output : Tensor(x_grad) - infer_meta : - func : UnchangedInferMeta - param : [x] - kernel : - func : softplus_grad - inplace : (out_grad -> x_grad) - - backward_api : soft_shrink_grad forward : soft_shrink (Tensor x, float lambda) -> Tensor(out) args : (Tensor x, Tensor out_grad, float lambda) @@ -1903,6 +1892,18 @@ func : softmax_grad use_gpudnn : true +# softplus +- backward_api : softplus_grad + forward : softplus (Tensor x, float beta,float threshold) -> Tensor(out) + args : (Tensor x, Tensor out_grad, float beta, float threshold) + output : Tensor(x_grad) + infer_meta : + func : UnchangedInferMeta + param : [x] + kernel : + func : softplus_grad + inplace : (out_grad -> x_grad) + - backward_api : split_grad forward : split (Tensor x, IntArray num_or_sections, Scalar axis) -> Tensor[](out) args : (Tensor[] out_grad, Scalar axis = -1) diff --git a/python/paddle/fluid/tests/unittests/test_activation_op.py b/python/paddle/fluid/tests/unittests/test_activation_op.py index 050879730d4a1..d2cffe0065883 100755 --- a/python/paddle/fluid/tests/unittests/test_activation_op.py +++ b/python/paddle/fluid/tests/unittests/test_activation_op.py @@ -2676,7 +2676,7 @@ class TestSoftplus(TestActivation): def setUp(self): self.op_type = "softplus" - self.python_api=paddle.nn.functional.softplus + self.python_api = paddle.nn.functional.softplus self.init_dtype() beta = 2 @@ -2688,14 +2688,15 @@ def setUp(self): self.inputs = {'X': x} self.attrs = {'beta': beta, "threshold": threshold} self.outputs = {'Out': out} - self.check_eager=True + + self.check_eager = True def test_check_grad(self): if self.dtype == np.float16: return if hasattr(self, 'check_eager'): check_eager = self.check_eager - self.check_grad(['X'], 'Out',check_eager=check_eager) + self.check_grad(['X'], 'Out', check_eager=check_eager) @unittest.skipIf(not core.is_compiled_with_cuda(), diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py index e1dd41763c2b5..526124b3fd916 100644 --- a/python/paddle/nn/functional/activation.py +++ b/python/paddle/nn/functional/activation.py @@ -1177,6 +1177,7 @@ def softplus(x, beta=1, threshold=20, name=None): x = paddle.to_tensor(np.array([-0.4, -0.2, 0.1, 0.3])) out = F.softplus(x) # [0.513015, 0.598139, 0.744397, 0.854355] """ + if in_dygraph_mode(): return _C_ops.final_state_softplus(x, beta, threshold) elif in_dynamic_mode(): From c77c85dcdf59b274b6a1a4a2b71fde7333ad3b8d Mon Sep 17 00:00:00 2001 From: wwbitejotunn Date: Sat, 23 Jul 2022 15:04:19 +0000 Subject: [PATCH 4/9] bug fix for test_layers --- paddle/phi/api/yaml/legacy_api.yaml | 2 +- paddle/phi/api/yaml/legacy_backward.yaml | 2 +- python/paddle/fluid/layers/ops.py | 41 ++++++++++++++++++++--- python/paddle/nn/functional/activation.py | 3 +- 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/paddle/phi/api/yaml/legacy_api.yaml b/paddle/phi/api/yaml/legacy_api.yaml index 6ce7ddb532912..5c953001f43d5 100644 --- a/paddle/phi/api/yaml/legacy_api.yaml +++ b/paddle/phi/api/yaml/legacy_api.yaml @@ -2030,7 +2030,7 @@ backward : softmax_grad - api : softplus - args : (Tensor x, float beta,float threshold) + args : (Tensor x, float beta, float threshold) output : Tensor infer_meta : func : UnchangedInferMeta diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml index 1ae8fb011a3f4..c04f446231720 100644 --- a/paddle/phi/api/yaml/legacy_backward.yaml +++ b/paddle/phi/api/yaml/legacy_backward.yaml @@ -1949,7 +1949,7 @@ # softplus - backward_api : softplus_grad - forward : softplus (Tensor x, float beta,float threshold) -> Tensor(out) + forward : softplus (Tensor x, float beta, float threshold) -> Tensor(out) args : (Tensor x, Tensor out_grad, float beta, float threshold) output : Tensor(x_grad) infer_meta : diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index 84d147bc97c92..133e7816ecc57 100755 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -31,7 +31,6 @@ 'silu', 'logsigmoid', 'tanh_shrink', - 'softplus', 'softsign', 'tanh', ] @@ -494,8 +493,42 @@ """) -add_sample_code( - globals()["softplus"], r""" +__all__ += ['softplus'] + +_softplus_ = generate_layer_fn('softplus') + + +def softplus(x, beta=None, threshold=None, name=None): + check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'softplus') + + locals_val = locals().copy() + kwargs = dict() + for name, val in locals_val.items(): + if val is not None: + kwargs[name] = val + return _softplus_(**kwargs) + + +softplus.__doc__ = r""" + :alias_main: paddle.nn.functional.softplus + :alias: paddle.nn.functional.softplus, paddle.nn.functional.activation.softplus + :old_api: paddle.fluid.layers.softplus + +:strong:`Softplus Activation Operator` + +.. math:: + out = \\frac{1}{beta} * log(1 + e^{beta * x}) + For numerical stability, the implementation reverts to the linear function when: beta * x > threshold. + +Args: + x (Tensor): Input of Softplus operator, an N-D Tensor, with data type float32 and float64 + beta (float, optional): The value of beta for softplus. Default is 1 + threshold (float, optional): The value of threshold for softplus. Default is 20 + name (str, optional): Name for the operation (optional, default is None) + +Returns: + Output of Softplus operator with the same type of x + Examples: .. code-block:: python @@ -507,7 +540,7 @@ print(out) # [0.513015, 0.598139, 0.744397, 0.854355] -""") +""" add_sample_code( globals()["softsign"], r""" diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py index 526124b3fd916..91b07da1b8f7d 100644 --- a/python/paddle/nn/functional/activation.py +++ b/python/paddle/nn/functional/activation.py @@ -1180,7 +1180,8 @@ def softplus(x, beta=1, threshold=20, name=None): if in_dygraph_mode(): return _C_ops.final_state_softplus(x, beta, threshold) - elif in_dynamic_mode(): + + if _in_legacy_dygraph(): return _C_ops.softplus(x, 'beta', beta, 'threshold', threshold) check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], From dd010dd8099cb989561b065252cb44d2ddd30b8b Mon Sep 17 00:00:00 2001 From: wwbitejotunn Date: Sat, 23 Jul 2022 17:02:45 +0000 Subject: [PATCH 5/9] layer api match --- python/paddle/fluid/layers/ops.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index 133e7816ecc57..b41089717b917 100755 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -498,9 +498,8 @@ _softplus_ = generate_layer_fn('softplus') -def softplus(x, beta=None, threshold=None, name=None): +def softplus(x, name=None): check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'softplus') - locals_val = locals().copy() kwargs = dict() for name, val in locals_val.items(): From 0df93cb6ef157280b51425f373ac58000d68a805 Mon Sep 17 00:00:00 2001 From: wwbitejotunn Date: Tue, 26 Jul 2022 08:08:52 +0000 Subject: [PATCH 6/9] match def and doc in ops.py --- python/paddle/fluid/layers/ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index b41089717b917..5f5a3da8cb764 100755 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -498,7 +498,7 @@ _softplus_ = generate_layer_fn('softplus') -def softplus(x, name=None): +def softplus(x, beta: float = 1.0, threshold: float = 20.0, name=None): check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'softplus') locals_val = locals().copy() kwargs = dict() From f45cb2af674ceba76845573127f52f979a5d029b Mon Sep 17 00:00:00 2001 From: wwbitejotunn Date: Tue, 26 Jul 2022 08:44:08 +0000 Subject: [PATCH 7/9] doc polish --- python/paddle/fluid/layers/ops.py | 41 ++++++++++++++----------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index 5f5a3da8cb764..4dda8679da27d 100755 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -70,7 +70,15 @@ 'reciprocal_', ] -__all__ = [] +__all__ = [ + 'softplus', + 'softshrink', + 'hard_shrink', + 'cumsum', + 'thresholded_relu', + 'gelu', + 'erf', +] for _OP in set(__all__): globals()[_OP] = generate_layer_fn(_OP) @@ -493,8 +501,6 @@ """) -__all__ += ['softplus'] - _softplus_ = generate_layer_fn('softplus') @@ -515,18 +521,19 @@ def softplus(x, beta: float = 1.0, threshold: float = 20.0, name=None): :strong:`Softplus Activation Operator` -.. math:: - out = \\frac{1}{beta} * log(1 + e^{beta * x}) - For numerical stability, the implementation reverts to the linear function when: beta * x > threshold. +Equation: + .. math:: + out = \\frac{1}{beta} * log(1 + e^{beta * x}) + For numerical stability, the implementation reverts to the linear function when: beta * x > threshold. Args: - x (Tensor): Input of Softplus operator, an N-D Tensor, with data type float32 and float64 - beta (float, optional): The value of beta for softplus. Default is 1 + x(Tensor): Input of Softplus op, Tensor, dtype: float32 or float64 + beta(float, optional): The value of beta for softplus. Default is 1 threshold (float, optional): The value of threshold for softplus. Default is 20 - name (str, optional): Name for the operation (optional, default is None) + name(str, optional): Name for the operation (optional, default is None) Returns: - Output of Softplus operator with the same type of x + Variable: The output of Softplus op, Tensor, dtype: float32 or float64 Examples: .. code-block:: python @@ -556,8 +563,6 @@ def softplus(x, beta: float = 1.0, threshold: float = 20.0, name=None): """) -__all__ += ['softshrink'] - _softshrink_ = generate_layer_fn('softshrink') @@ -606,8 +611,6 @@ def softshrink(x, alpha=None): result = fluid.layers.softshrink(x=data, alpha=0.3) """ -__all__ += ['hard_shrink'] - _hard_shrink_ = generate_layer_fn('hard_shrink') @@ -632,8 +635,6 @@ def hard_shrink(x, threshold=None): >>> result = fluid.layers.hard_shrink(x=data, threshold=0.3) """ -__all__ += ['cumsum'] - _cum_sum_ = generate_layer_fn('cumsum') @@ -674,8 +675,6 @@ def cumsum(x, axis=None, exclusive=None, reverse=None): result = fluid.layers.cumsum(data, axis=0) """ -__all__ += ['thresholded_relu'] - _thresholded_relu_ = generate_layer_fn('thresholded_relu') @@ -695,7 +694,7 @@ def thresholded_relu(x, threshold=None): thresholded_relu.__doc__ = r""" :alias_main: paddle.nn.functional.thresholded_relu :alias: paddle.nn.functional.thresholded_relu,paddle.nn.functional.activation.thresholded_relu - :old_api: paddle.fluid.layers.thresholded_relu + :old_api: paddle.fluidx .layers.thresholded_relu :strong:`Thresholded ReLU Activation Operator` @@ -764,8 +763,6 @@ def thresholded_relu(x, threshold=None): # [-0. , -0. , 1.0013918 ]], dtype=float32) """ -__all__ += ['gelu'] - _gelu_ = generate_layer_fn('gelu') @@ -849,8 +846,6 @@ def gelu(x, approximate=False): # [ 0.08796856, 0.20387867, 0.2080159 ]], dtype=float32) """ -__all__ += ['erf'] - _erf_ = generate_layer_fn('erf') From b65d6f30d812a44100123995d1b7bb9dbacedb8a Mon Sep 17 00:00:00 2001 From: wwbitejotunn Date: Tue, 26 Jul 2022 09:13:07 +0000 Subject: [PATCH 8/9] fix unwanted modified of thresholded_relu --- python/paddle/fluid/layers/ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index 4dda8679da27d..d73d5e05bd3e3 100755 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -694,7 +694,7 @@ def thresholded_relu(x, threshold=None): thresholded_relu.__doc__ = r""" :alias_main: paddle.nn.functional.thresholded_relu :alias: paddle.nn.functional.thresholded_relu,paddle.nn.functional.activation.thresholded_relu - :old_api: paddle.fluidx .layers.thresholded_relu + :old_api: paddle.fluid.layers.thresholded_relu :strong:`Thresholded ReLU Activation Operator` From 13bc7c8bcb91b57ccb8240ccf444d7069856be1d Mon Sep 17 00:00:00 2001 From: wwbitejotunn Date: Thu, 28 Jul 2022 16:53:23 +0000 Subject: [PATCH 9/9] style imporve --- python/paddle/fluid/layers/ops.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index 9d253a3c5b126..01da331e57b2e 100755 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -471,6 +471,7 @@ _softplus_ = generate_layer_fn('softplus') + def softplus(x, beta: float = 1.0, threshold: float = 20.0, name=None): check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'softplus') locals_val = locals().copy()