gh-37172: Fix three polynomial SIGABRT bugs #37169 and #37317
    
Fix #37169.
Fix #37317.

Changes: adding `sig_on()` and `sig_off()` where appropriate.
    
URL: #37172
Reported by: grhkm21
Reviewer(s): Giacomo Pope, grhkm21

Author: Release Manager

src/sage/libs/flint/nmod_poly_linkage.pxi

@@ -582,11 +582,13 @@ cdef inline int celement_gcd(nmod_poly_t res, nmod_poly_t a, nmod_poly_t b, unsi
         nmod_poly_set(res, a)
         return 0
 
-    nmod_poly_gcd(res, a, b)
-    cdef unsigned long leadcoeff = nmod_poly_get_coeff_ui(res, nmod_poly_degree(res))
-    cdef unsigned long modulus = nmod_poly_modulus(res)
-    if n_gcd(modulus,leadcoeff) == 1:
-        nmod_poly_make_monic(res, res)
+    # A check that the leading coefficients are invertible is *not* sufficient
+    try:
+        sig_on()
+        nmod_poly_gcd(res, a, b)
+        sig_off()
+    except RuntimeError:
+        raise ValueError("non-invertible elements encountered during GCD")
 
 cdef inline int celement_xgcd(nmod_poly_t res, nmod_poly_t s, nmod_poly_t t, nmod_poly_t a, nmod_poly_t b, unsigned long n) except -2:
     """

src/sage/libs/ntl/ntl_ZZ_pEX_linkage.pxi

@@ -367,11 +367,11 @@ cdef inline int celement_pow(ZZ_pEX_c* res, ZZ_pEX_c* x, long e, ZZ_pEX_c *modul
             sig_off()
     else:
         if ZZ_pEX_deg(modulus[0]) == 1:
-             ZZ_pEX_rem(y, x[0], modulus[0])
-             sig_on()
-             ZZ_pEX_power(res[0], y, e)
-             sig_off()
-             return 0
+            ZZ_pEX_rem(y, x[0], modulus[0])
+            sig_on()
+            ZZ_pEX_power(res[0], y, e)
+            sig_off()
+            return 0
         ZZ_pEX_Modulus_build(mod, modulus[0])
         if ZZ_pEX_deg(x[0]) < ZZ_pEX_deg(modulus[0]):
             sig_on()

src/sage/rings/polynomial/polynomial_template.pxi

@@ -352,11 +352,35 @@ cdef class Polynomial_template(Polynomial):
             x + 1
             sage: f.gcd(x^2)
             x
+
+        TESTS:
+
+        Ensure non-invertible elements does not crash Sage (:issue:`37317`)::
+
+            sage: R.<x> = Zmod(4)[]
+            sage: f = R(2 * x)
+            sage: f.gcd(f)
+            Traceback (most recent call last):
+            ...
+            ValueError: leading coefficient must be invertible
+
+        ::
+
+            sage: f = x^2 + 3 * x + 1
+            sage: g = x^2 + x + 1
+            sage: f.gcd(g)
+            Traceback (most recent call last):
+            ...
+            ValueError: non-invertible elements encountered during GCD
         """
-        if(celement_is_zero(&self.x, (<Polynomial_template>self)._cparent)):
+        if celement_is_zero(&self.x, (<Polynomial_template>self)._cparent):
             return other
-        if(celement_is_zero(&other.x, (<Polynomial_template>self)._cparent)):
+        if celement_is_zero(&other.x, (<Polynomial_template>self)._cparent):
             return self
+        if celement_equal(&self.x, &other.x, (<Polynomial_template>self)._cparent):
+            # note: gcd(g, g) "canonicalizes" the generator i.e. make polynomials monic
+            # c.f. ring/ring.pyx:445
+            return self.monic()
 
         cdef type T = type(self)
         cdef Polynomial_template r = <Polynomial_template>T.__new__(T)
@@ -569,7 +593,6 @@ cdef class Polynomial_template(Polynomial):
         """
         EXAMPLES::
 
-            sage: P.<x> = GF(2)[]
             sage: P.<x> = GF(2)[]
             sage: x^1000
             x^1000
@@ -583,6 +606,28 @@ cdef class Polynomial_template(Polynomial):
             x^9 + x^8 + x^7 + x^5 + x^3
             sage: pow(f, 2, h)
             x^9 + x^8 + x^7 + x^5 + x^3
+
+        TESTS:
+
+        Ensure modulo `0` and modulo `1` does not crash (:issue:`37169`)::
+
+            sage: R.<x> = GF(2)[]
+            sage: pow(x + 1, 2, R.zero())
+            Traceback (most recent call last):
+            ...
+            ZeroDivisionError: modulus must be nonzero
+            sage: pow(x + 1, 2, R.one())
+            0
+
+        ::
+
+            sage: R.<x> = GF(2^8)[]
+            sage: pow(x + 1, 2, R.zero())
+            Traceback (most recent call last):
+            ...
+            ZeroDivisionError: modulus must be nonzero
+            sage: pow(x + 1, 2, R.one())
+            0
         """
         if not isinstance(self, Polynomial_template):
             raise NotImplementedError("%s^%s not defined."%(ee,self))
@@ -615,6 +660,10 @@ cdef class Polynomial_template(Polynomial):
         else:
             if parent is not (<Polynomial_template>modulus)._parent and parent != (<Polynomial_template>modulus)._parent:
                 modulus = parent.coerce(modulus)
+            if celement_is_zero(&(<Polynomial_template>modulus).x, (<Polynomial_template>self)._cparent):
+                raise ZeroDivisionError("modulus must be nonzero")
+            if celement_is_one(&(<Polynomial_template>modulus).x, (<Polynomial_template>self)._cparent):
+                return parent.zero()
             celement_pow(&r.x, &(<Polynomial_template>self).x, e, &(<Polynomial_template>modulus).x, (<Polynomial_template>self)._cparent)
 
         #assert(r._parent(pari(self)**ee) == r)

src/sage/rings/polynomial/polynomial_zmod_flint.pyx

@@ -532,6 +532,16 @@ cdef class Polynomial_zmod_flint(Polynomial_template):
             (3*x^4 + 2*x^3 + x^2 + 2*x, x^4 + 3*x^3 + x^2 + x)
             sage: (p*d % x^9) == n
             True
+
+        Check that :issue:`37169` is fixed - it does not throw an error::
+
+            sage: R.<x> = Zmod(4)[]
+            sage: _.<z> = R.quotient_ring(x^2 - 1)
+            sage: c = 2 * z + 1
+            sage: c * Zmod(2).zero()
+            Traceback (most recent call last):
+            ...
+            RuntimeError: Aborted
         """
         if n_deg < 0 or d_deg < 0:
             raise ValueError("The degree bounds n_deg and d_deg should be positive.")
@@ -554,7 +564,11 @@ cdef class Polynomial_zmod_flint(Polynomial_template):
         while nmod_poly_length(&t1.x) != 0 and n_deg < nmod_poly_degree(&t1.x):
             q = self._new()
             r1 = self._new()
+
+            sig_on()
             nmod_poly_divrem(&q.x, &r1.x, &s1.x, &t1.x)
+            sig_off()
+
             r0 = s0 - q*t0
             s0 = t0
             s1 = t1
@@ -743,9 +757,12 @@ cdef class Polynomial_zmod_flint(Polynomial_template):
             ...
             ValueError: leading coefficient must be invertible
         """
-        if self.base_ring().characteristic().gcd(
-                self.leading_coefficient().lift()) != 1:
+        cdef unsigned long leadcoeff, modulus
+        leadcoeff = nmod_poly_get_coeff_ui(&self.x, nmod_poly_degree(&self.x))
+        modulus = nmod_poly_modulus(&self.x)
+        if leadcoeff > 1 and n_gcd(modulus, leadcoeff) != 1:
             raise ValueError("leading coefficient must be invertible")
+
         cdef Polynomial_zmod_flint res = self._new()
         nmod_poly_make_monic(&res.x, &self.x)
         return res

src/sage/rings/ring.pyx

@@ -441,7 +441,8 @@ cdef class Ring(ParentWithGens):
             g = gens[0]
             if len(gens) == 1:
                 try:
-                    g = g.gcd(g) # note: we set g = gcd(g, g) to "canonicalize" the generator: make polynomials monic, etc.
+                    # note: we set g = gcd(g, g) to "canonicalize" the generator: make polynomials monic, etc.
+                    g = g.gcd(g)
                 except (AttributeError, NotImplementedError):
                     pass
             else: