handle non-prime order curves more gracefully

when the order of the curve is not a prime, then point doubling
can return INFINITY, this will cause some negative values not
to be reduced modulo curve p; fix this

src/ecdsa/ellipticcurve.py

@@ -633,7 +633,7 @@ def __eq__(self, other):
         """
         x1, y1, z1 = self.__coords
         if other is INFINITY:
-            return not y1 or not z1
+            return (not x1 and not y1) or not z1
         if isinstance(other, Point):
             x2, y2, z2 = other.x(), other.y(), 1
         elif isinstance(other, PointJacobi):
@@ -728,6 +728,7 @@ def to_affine(self):
             return INFINITY
         self.scale()
         x, y, z = self.__coords
+        assert z == 1
         return Point(self.__curve, x, y, self.__order)
 
     @staticmethod
@@ -802,7 +803,7 @@ def double(self):
 
         X3, Y3, Z3 = self._double(X1, Y1, Z1, p, a)
 
-        if not Y3 or not Z3:
+        if not Y3 and not X3:
             return INFINITY
         return PointJacobi(self.__curve, X3, Y3, Z3, self.__order)
 
@@ -886,10 +887,10 @@ def __radd__(self, other):
 
     def _add(self, X1, Y1, Z1, X2, Y2, Z2, p):
         """add two points, select fastest method."""
-        if not Y1 or not Z1:
-            return X2, Y2, Z2
-        if not Y2 or not Z2:
-            return X1, Y1, Z1
+        if (not X1 and not Y1) or not Z1:
+            return X2 % p, Y2 % p, Z2 % p
+        if (not X2 and not Y2) or not Z2:
+            return X1 % p, Y1 % p, Z1 % p
         if Z1 == Z2:
             if Z1 == 1:
                 return self._add_with_z_1(X1, Y1, X2, Y2, p)
@@ -917,7 +918,7 @@ def __add__(self, other):
 
         X3, Y3, Z3 = self._add(X1, Y1, Z1, X2, Y2, Z2, p)
 
-        if not Y3 or not Z3:
+        if (not X3 and not Y3) or not Z3:
             return INFINITY
         return PointJacobi(self.__curve, X3, Y3, Z3, self.__order)
 
@@ -972,7 +973,7 @@ def __mul__(self, other):
             elif i > 0:
                 X3, Y3, Z3 = _add(X3, Y3, Z3, X2, Y2, 1, p)
 
-        if not Y3 or not Z3:
+        if (not X3 and not Y3) or not Z3:
             return INFINITY
 
         return PointJacobi(self.__curve, X3, Y3, Z3, self.__order)
@@ -1070,7 +1071,7 @@ def mul_add(self, self_mul, other, other_mul):
                     assert B > 0
                     X3, Y3, Z3 = _add(X3, Y3, Z3, pApB_X, pApB_Y, pApB_Z, p)
 
-        if not Y3 or not Z3:
+        if (not X3 and not Y3) or not Z3:
             return INFINITY
 
         return PointJacobi(self.__curve, X3, Y3, Z3, self.__order)
@@ -1220,7 +1221,12 @@ def leftmost_bit(x):
         # From X9.62 D.3.2:
 
         e3 = 3 * e
-        negative_self = Point(self.__curve, self.__x, -self.__y, self.__order)
+        negative_self = Point(
+            self.__curve,
+            self.__x,
+            (-self.__y) % self.__curve.p(),
+            self.__order,
+        )
         i = leftmost_bit(e3) // 2
         result = self
         # print("Multiplying %s by %d (e3 = %d):" % (self, other, e3))
@@ -1247,7 +1253,6 @@ def __str__(self):
 
     def double(self):
         """Return a new point that is twice the old."""
-
         if self == INFINITY:
             return INFINITY
 
@@ -1261,6 +1266,9 @@ def double(self):
             * numbertheory.inverse_mod(2 * self.__y, p)
         ) % p
 
+        if not l:
+            return INFINITY
+
         x3 = (l * l - 2 * self.__x) % p
         y3 = (l * (self.__x - x3) - self.__y) % p
 

src/ecdsa/test_ellipticcurve.py

@@ -184,6 +184,33 @@ def test_double(self):
         self.assertEqual(p3.x(), x3)
         self.assertEqual(p3.y(), y3)
 
+    def test_double_to_infinity(self):
+        p1 = Point(self.c_23, 11, 20)
+        p2 = p1.double()
+        self.assertEqual((p2.x(), p2.y()), (4, 0))
+        self.assertNotEqual(p2, INFINITY)
+        p3 = p2.double()
+        self.assertEqual(p3, INFINITY)
+        self.assertIs(p3, INFINITY)
+
+    def test_add_self_to_infinity(self):
+        p1 = Point(self.c_23, 11, 20)
+        p2 = p1 + p1
+        self.assertEqual((p2.x(), p2.y()), (4, 0))
+        self.assertNotEqual(p2, INFINITY)
+        p3 = p2 + p2
+        self.assertEqual(p3, INFINITY)
+        self.assertIs(p3, INFINITY)
+
+    def test_mul_to_infinity(self):
+        p1 = Point(self.c_23, 11, 20)
+        p2 = p1 * 2
+        self.assertEqual((p2.x(), p2.y()), (4, 0))
+        self.assertNotEqual(p2, INFINITY)
+        p3 = p2 * 2
+        self.assertEqual(p3, INFINITY)
+        self.assertIs(p3, INFINITY)
+
     def test_multiply(self):
         x1, y1, m, x3, y3 = (3, 10, 2, 7, 12)
         p1 = Point(self.c_23, x1, y1)

src/ecdsa/test_jacobi.py

@@ -641,6 +641,36 @@ def test_add_with_point_at_infinity(self):
 
         self.assertEqual((x, y, z), (2, 3, 1))
 
+    def test_double_to_infinity(self):
+        c_23 = CurveFp(23, 1, 1)
+        p = PointJacobi(c_23, 11, 20, 1)
+        p2 = p.double()
+        self.assertEqual((p2.x(), p2.y()), (4, 0))
+        self.assertNotEqual(p2, INFINITY)
+        p3 = p2.double()
+        self.assertEqual(p3, INFINITY)
+        self.assertIs(p3, INFINITY)
+
+    def test_mul_to_infinity(self):
+        c_23 = CurveFp(23, 1, 1)
+        p = PointJacobi(c_23, 11, 20, 1)
+        p2 = p * 2
+        self.assertEqual((p2.x(), p2.y()), (4, 0))
+        self.assertNotEqual(p2, INFINITY)
+        p3 = p2 * 2
+        self.assertEqual(p3, INFINITY)
+        self.assertIs(p3, INFINITY)
+
+    def test_add_to_infinity(self):
+        c_23 = CurveFp(23, 1, 1)
+        p = PointJacobi(c_23, 11, 20, 1)
+        p2 = p + p
+        self.assertEqual((p2.x(), p2.y()), (4, 0))
+        self.assertNotEqual(p2, INFINITY)
+        p3 = p2 + p2
+        self.assertEqual(p3, INFINITY)
+        self.assertIs(p3, INFINITY)
+
     def test_pickle(self):
         pj = PointJacobi(curve=CurveFp(23, 1, 1, 1), x=2, y=3, z=1, order=1)
         self.assertEqual(pickle.loads(pickle.dumps(pj)), pj)