revert gelu

dmlc · Jul 30, 2020 · 9a7c343 · 9a7c343
1 parent 0425346
commit 9a7c343
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/gluonnlp/layers.py b/src/gluonnlp/layers.py
@@ -356,7 +356,7 @@ def __init__(self, mode='erf'):
 
     def hybrid_forward(self, F, x):
         if self._mode == 'erf':
-            return F.npx.leaky_relu(x, act_type='gelu')
+            return x * 0.5 * (1.0 + F.npx.erf(x / math.sqrt(2.0)))
         elif self._mode == 'tanh':
             return 0.5 * x\
                    * (1.0 + F.np.tanh(math.sqrt(2.0 / math.pi) * (x + 0.044715 * (x ** 3))))