remove Sigmoid from function.py [pr]

tinygrad/function.py

@@ -67,17 +67,6 @@ def forward(self, x:UOp) -> UOp:
 
   def backward(self, grad_output:UOp) -> UOp: return grad_output / (self.ret*2)
 
-# NOTE: the implicit derivative of sigmoid is not stable
-# https://towardsdatascience.com/derivative-of-the-sigmoid-function-536880cf918e
-# TODO: have the backend automatically find this
-class Sigmoid(Function):
-  def forward(self, x:UOp) -> UOp:
-    self.ret = (1 + (x * (-1/math.log(2))).exp2()).reciprocal()
-    return self.ret
-
-  def backward(self, grad_output:UOp) -> UOp:
-    return (self.ret * (1 - self.ret)) * grad_output
-
 class Sign(Function):
   # NOTE: the x*0 is to match torch behavior without function.py
   def forward(self, x:UOp) -> UOp: return x.ne(0).where((x<0).where(x.const_like(-1), x.const_like(1)), x.const_like(0)) + x*0

tinygrad/tensor.py

@@ -2477,6 +2477,7 @@ def exp2(self):
     ```
     """
     return F.Exp.apply(self*math.log(2))
+
   def relu(self):
     """
     Applies the Rectified Linear Unit (ReLU) function element-wise.
@@ -2488,6 +2489,7 @@ def relu(self):
     ```
     """
     return F.Relu.apply(self)
+
   def sigmoid(self):
     """
     Applies the Sigmoid function element-wise.
@@ -2498,7 +2500,8 @@ def sigmoid(self):
     print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).sigmoid().numpy())
     ```
     """
-    return F.Sigmoid.apply(self.cast(least_upper_float(self.dtype)))
+    return 1 / (1 + (-self).exp())
+
   def hardsigmoid(self, alpha:float=1/6, beta:float=0.5):
     """
     Applies the Hardsigmoid function element-wise.