handle non-prime order curves more gracefully

when the order of the curve is not a prime, then point doubling
can return INFINITY, this will cause some negative values not
to be reduced modulo curve p; fix this

Author: tomato42

src/ecdsa/ellipticcurve.py

@@ -728,6 +728,7 @@ def to_affine(self):
             return INFINITY
         self.scale()
         x, y, z = self.__coords
+        assert z == 1
         return Point(self.__curve, x, y, self.__order)
 
     @staticmethod
@@ -887,9 +888,9 @@ def __radd__(self, other):
     def _add(self, X1, Y1, Z1, X2, Y2, Z2, p):
         """add two points, select fastest method."""
         if not Y1 or not Z1:
-            return X2, Y2, Z2
+            return X2 % p, Y2 % p, Z2 % p
         if not Y2 or not Z2:
-            return X1, Y1, Z1
+            return X1 % p, Y1 % p, Z1 % p
         if Z1 == Z2:
             if Z1 == 1:
                 return self._add_with_z_1(X1, Y1, X2, Y2, p)
@@ -1220,7 +1221,12 @@ def leftmost_bit(x):
         # From X9.62 D.3.2:
 
         e3 = 3 * e
-        negative_self = Point(self.__curve, self.__x, -self.__y, self.__order)
+        negative_self = Point(
+            self.__curve,
+            self.__x,
+            (-self.__y) % self.__curve.p(),
+            self.__order,
+        )
         i = leftmost_bit(e3) // 2
         result = self
         # print("Multiplying %s by %d (e3 = %d):" % (self, other, e3))
@@ -1264,6 +1270,9 @@ def double(self):
         x3 = (l * l - 2 * self.__x) % p
         y3 = (l * (self.__x - x3) - self.__y) % p
 
+        if y3 == 0:
+            return INFINITY
+
         return Point(self.__curve, x3, y3)
 
     def x(self):