diff --git a/src/gluonnlp/layers.py b/src/gluonnlp/layers.py
index a6ea6b181e..c7c8df3540 100644
--- a/src/gluonnlp/layers.py
+++ b/src/gluonnlp/layers.py
@@ -356,7 +356,7 @@ def __init__(self, mode='erf'):
 
     def hybrid_forward(self, F, x):
         if self._mode == 'erf':
-            return F.npx.leaky_relu(x, act_type='gelu')
+            return x * 0.5 * (1.0 + F.npx.erf(x / math.sqrt(2.0)))
         elif self._mode == 'tanh':
             return 0.5 * x\
                    * (1.0 + F.np.tanh(math.sqrt(2.0 / math.pi) * (x + 0.044715 * (x ** 3))))