gh-111140: Adds PyLong_AsNativeBytes and PyLong_FromNative[Unsigned]Bytes functions

Include/cpython/longobject.h

@@ -4,6 +4,44 @@
 
 PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base);
 
+/* PyLong_AsByteArray: Copy the integer value to a native address.
+   n is the number of bytes available in the buffer.
+   Uses the current build's default endianness.
+   PyLong_AsByteArray ensures the MSB matches the int's sign.
+   PyLong_AsUnsignedByteArray allows the MSB to be set for a
+   positive Python value provided no significant bits are lost
+   (that is, all higher bits are a sign extension of the MSB).
+   Negative values still sign extend to fill the buffer and are
+   guaranteed to have MSB set. Use PyLong_AsByteArray to guarantee
+   that the MSB is set iff the int was negative.
+
+   Returns 0 on success or -1 with an exception set, but if the buffer is
+   not big enough, returns the desired buffer size without setting an
+   exception. Note that the desired size may be larger than strictly
+   necessary to avoid precise calculations. */
+PyAPI_FUNC(int) PyLong_AsByteArray(PyObject* v, void* buffer, size_t n);
+PyAPI_FUNC(int) PyLong_AsUnsignedByteArray(PyObject* v, void* buffer, size_t n);
+PyAPI_FUNC(int) PyLong_AsByteArrayWithOptions(PyObject* v, void* buffer,
+                                              size_t n, int options);
+
+#define PYLONG_ASBYTEARRAY_NATIVE_ENDIAN    0x00
+#define PYLONG_ASBYTEARRAY_LITTLE_ENDIAN    0x01
+#define PYLONG_ASBYTEARRAY_BIG_ENDIAN       0x02
+
+#define PYLONG_ASBYTEARRAY_SIGNED   0x00
+#define PYLONG_ASBYTEARRAY_UNSIGNED 0x04
+
+/* PyLong_FromByteArray: Create an integer value containing the number from
+   a native buffer.
+   n is the number of bytes to read from the buffer.
+   Uses the current build's default endianness, and assumes the value was
+   sign extended to 'n' bytes.
+   PyLong_FromUnsignedByteArray assumes the value was zero extended.
+
+   Returns the int object, or NULL with an exception set. */
+PyAPI_FUNC(PyObject) PyLong_FromByteArray(void* buffer, size_t n);
+PyAPI_FUNC(PyObject) PyLong_FromUnsignedByteArray(void* buffer, size_t n);
+
 PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op);
 PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op);
 
@@ -50,7 +88,7 @@ PyAPI_FUNC(PyObject *) _PyLong_FromByteArray(
 */
 PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v,
     unsigned char* bytes, size_t n,
-    int little_endian, int is_signed);
+    int little_endian, int is_signed, int with_exceptions);
 
 /* For use by the gcd function in mathmodule.c */
 PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *);

Lib/test/test_capi/test_long.py

@@ -1,5 +1,6 @@
 import unittest
 import sys
+import test.support as support
 
 from test.support import import_helper
 
@@ -423,6 +424,98 @@ def test_long_asvoidptr(self):
         self.assertRaises(OverflowError, asvoidptr, -2**1000)
         # CRASHES asvoidptr(NULL)
 
+    def test_long_asbytearray(self):
+        import math
+        from _testcapi import (
+            pylong_asbytearray as asbytearray,
+            pylong_asunsignedbytearray as asunsignedbytearray,
+            pylong_asbytearraywithoptions as asbytearraywithoptions,
+            SIZE_MAX
+        )
+
+        def log2(x):
+            return math.log(x) / math.log(2)
+
+        SIZEOF_SIZE = int(math.ceil(log2(SIZE_MAX + 1)) / 8)
+        MAX_SSIZE = 2 ** (SIZEOF_SIZE * 8 - 1) - 1
+        MAX_USIZE = 2 ** (SIZEOF_SIZE * 8) - 1
+        if support.verbose:
+            print(f"{SIZEOF_SIZE=}\n{MAX_SSIZE=:016X}\n{MAX_USIZE=:016X}")
+
+        for v, expect_u, expect_s in [
+            (0, SIZEOF_SIZE, SIZEOF_SIZE),
+            (512, SIZEOF_SIZE, SIZEOF_SIZE),
+            (-512, SIZEOF_SIZE, SIZEOF_SIZE),
+            (MAX_SSIZE, SIZEOF_SIZE, SIZEOF_SIZE),
+            (MAX_USIZE, SIZEOF_SIZE, SIZEOF_SIZE + 1),
+            (-MAX_SSIZE, SIZEOF_SIZE, SIZEOF_SIZE),
+            (-MAX_USIZE, SIZEOF_SIZE, SIZEOF_SIZE + 1),
+        ]:
+            with self.subTest(f"sizeof-{v:X}"):
+                buffer = bytearray(1)
+                self.assertEqual(expect_u, asunsignedbytearray(v, buffer, 0),
+                    "PyLong_AsUnsignedByteArray(v, NULL, 0)")
+                self.assertEqual(expect_s, asbytearray(v, buffer, 0),
+                    "PyLong_AsByteArray(v, NULL, 0)")
+                self.assertEqual(expect_u, asbytearraywithoptions(v, buffer, 0, 0x05),
+                    "PyLong_AsByteArrayWithOptions(v, NULL, 0, unsigned|little)")
+                self.assertEqual(expect_u, asbytearraywithoptions(v, buffer, 0, 0x06),
+                    "PyLong_AsByteArrayWithOptions(v, NULL, 0, unsigned|big)")
+                self.assertEqual(expect_s, asbytearraywithoptions(v, buffer, 0, 0x01),
+                    "PyLong_AsByteArrayWithOptions(v, NULL, 0, signed|little)")
+                self.assertEqual(expect_s, asbytearraywithoptions(v, buffer, 0, 0x02),
+                    "PyLong_AsByteArrayWithOptions(v, NULL, 0, signed|big)")
+
+        for v, expect_be, signed_fails in [
+            (0, b'\x00', False),
+            (0, b'\x00' * 2, False),
+            (0, b'\x00' * 8, False),
+            (1, b'\x01', False),
+            (1, b'\x00' * 10 + b'\x01', False),
+            (42, b'\x2a', False),
+            (42, b'\x00' * 10 + b'\x2a', False),
+            (-1, b'\xff', False),
+            (-42, b'\xd6', False),
+            (-42, b'\xff' * 10 + b'\xd6', False),
+            # Only unsigned will extract 255 into a single byte
+            (255, b'\xff', True),
+            (255, b'\x00\xff', False),
+            (256, b'\x01\x00', False),
+            (2**63, b'\x80\x00\x00\x00\x00\x00\x00\x00', True),
+            (-2**63, b'\x80\x00\x00\x00\x00\x00\x00\x00', False),
+            (2**63, b'\x00\x80\x00\x00\x00\x00\x00\x00\x00', False),
+            (-2**63, b'\xFF\x80\x00\x00\x00\x00\x00\x00\x00', False),
+        ]:
+            with self.subTest(f"{v:X}-{len(expect_be)}bytes"):
+                n = len(expect_be)
+                buffer = bytearray(n)
+                expect_le = expect_be[::-1]
+
+                self.assertEqual(0, asbytearraywithoptions(v, buffer, n, 0x05),
+                    f"PyLong_AsByteArrayWithOptions(v, buffer, {n}, unsigned+little)")
+                self.assertEqual(expect_le, buffer[:n], "unsigned+little")
+                self.assertEqual(0, asbytearraywithoptions(v, buffer, n, 0x06),
+                    f"PyLong_AsByteArrayWithOptions(v, buffer, {n}, unsigned+big)")
+                self.assertEqual(expect_be, buffer[:n], "unsigned+big")
+
+                if signed_fails:
+                    self.assertEqual(
+                        max(n + 1, SIZEOF_SIZE),
+                        asbytearraywithoptions(v, buffer, n, 0x01),
+                        f"PyLong_AsByteArrayWithOptions(v, buffer, {n}, signed+little)",
+                    )
+                    self.assertEqual(
+                        max(n + 1, SIZEOF_SIZE),
+                        asbytearraywithoptions(v, buffer, n, 0x02),
+                        f"PyLong_AsByteArrayWithOptions(v, buffer, {n}, signed+big)",
+                    )
+                else:
+                    self.assertEqual(0, asbytearraywithoptions(v, buffer, n, 0x01),
+                        f"PyLong_AsByteArrayWithOptions(v, buffer, {n}, signed+little)")
+                    self.assertEqual(expect_le, buffer[:n], "signed+little")
+                    self.assertEqual(0, asbytearraywithoptions(v, buffer, n, 0x02),
+                        f"PyLong_AsByteArrayWithOptions(v, buffer, {n}, signed+big)")
+                    self.assertEqual(expect_be, buffer[:n], "signed+big")
 
 if __name__ == "__main__":
     unittest.main()

Modules/_io/textio.c

@@ -2393,7 +2393,7 @@ textiowrapper_parse_cookie(cookie_type *cookie, PyObject *cookieObj)
         return -1;
 
     if (_PyLong_AsByteArray(cookieLong, buffer, sizeof(buffer),
-                            PY_LITTLE_ENDIAN, 0) < 0) {
+                            PY_LITTLE_ENDIAN, 0, 1) < 0) {
         Py_DECREF(cookieLong);
         return -1;
     }

Modules/_pickle.c

@@ -2162,7 +2162,8 @@ save_long(PicklerObject *self, PyObject *obj)
         pdata = (unsigned char *)PyBytes_AS_STRING(repr);
         i = _PyLong_AsByteArray((PyLongObject *)obj,
                                 pdata, nbytes,
-                                1 /* little endian */ , 1 /* signed */ );
+                                1 /* little endian */ , 1 /* signed */ ,
+                                1 /* with exceptions */);
         if (i < 0)
             goto error;
         /* If the int is negative, this may be a byte more than

Modules/_randommodule.c

@@ -342,7 +342,8 @@ random_seed(RandomObject *self, PyObject *arg)
     res = _PyLong_AsByteArray((PyLongObject *)n,
                               (unsigned char *)key, keyused * 4,
                               PY_LITTLE_ENDIAN,
-                              0); /* unsigned */
+                              0, /* unsigned */
+                              1); /* with exceptions */
     if (res == -1) {
         goto Done;
     }

Modules/_sqlite/util.c

@@ -162,7 +162,7 @@ _pysqlite_long_as_int64(PyObject * py_val)
         sqlite_int64 int64val;
         if (_PyLong_AsByteArray((PyLongObject *)py_val,
                                 (unsigned char *)&int64val, sizeof(int64val),
-                                IS_LITTLE_ENDIAN, 1 /* signed */) >= 0) {
+                                IS_LITTLE_ENDIAN, 1 /* signed */, 0) >= 0) {
             return int64val;
         }
     }

Modules/_struct.c

@@ -1000,9 +1000,10 @@ bp_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
                               (unsigned char *)p,
                               8,
                               0, /* little_endian */
-                              1  /* signed */);
+                              1, /* signed */
+                              0  /* !with_exceptions */);
     Py_DECREF(v);
-    if (res == -1 && PyErr_Occurred()) {
+    if (res < 0) {
         PyErr_Format(state->StructError,
                      "'%c' format requires %lld <= number <= %lld",
                      f->format,
@@ -1024,9 +1025,10 @@ bp_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f
                               (unsigned char *)p,
                               8,
                               0, /* little_endian */
-                              0  /* signed */);
+                              0, /* signed */
+                              0  /* !with_exceptions */);
     Py_DECREF(v);
-    if (res == -1 && PyErr_Occurred()) {
+    if (res < 0) {
         PyErr_Format(state->StructError,
                      "'%c' format requires 0 <= number <= %llu",
                      f->format,
@@ -1260,9 +1262,10 @@ lp_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f)
                               (unsigned char *)p,
                               8,
                               1, /* little_endian */
-                              1  /* signed */);
+                              1, /* signed */
+                              0  /* !with_exceptions */);
     Py_DECREF(v);
-    if (res == -1 && PyErr_Occurred()) {
+    if (res < 0) {
         PyErr_Format(state->StructError,
                      "'%c' format requires %lld <= number <= %lld",
                      f->format,
@@ -1284,9 +1287,10 @@ lp_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f
                               (unsigned char *)p,
                               8,
                               1, /* little_endian */
-                              0  /* signed */);
+                              0, /* signed */
+                              0  /* !with_exceptions */);
     Py_DECREF(v);
-    if (res == -1 && PyErr_Occurred()) {
+    if (res < 0) {
         PyErr_Format(state->StructError,
                      "'%c' format requires 0 <= number <= %llu",
                      f->format,

Modules/_testcapi/long.c

@@ -776,6 +776,78 @@ pylong_asvoidptr(PyObject *module, PyObject *arg)
     return Py_NewRef((PyObject *)value);
 }
 
+static PyObject *
+pylong_asbytearray(PyObject *module, PyObject *args)
+{
+    PyObject *v;
+    Py_buffer buffer;
+    Py_ssize_t n;
+    if (!PyArg_ParseTuple(args, "Ow*n", &v, &buffer, &n)) {
+        return NULL;
+    }
+    if (buffer.readonly) {
+        PyErr_SetString(PyExc_TypeError, "buffer must be writable");
+        PyBuffer_Release(&buffer);
+        return NULL;
+    }
+    if (buffer.len < n) {
+        PyErr_SetString(PyExc_ValueError, "buffer must be at least 'n' bytes");
+        PyBuffer_Release(&buffer);
+        return NULL;
+    }
+    int res = PyLong_AsByteArray(v, buffer.buf, n);
+    PyBuffer_Release(&buffer);
+    return res >= 0 ? PyLong_FromLong(res) : NULL;
+}
+
+static PyObject *
+pylong_asunsignedbytearray(PyObject *module, PyObject *args)
+{
+    PyObject *v;
+    Py_buffer buffer;
+    Py_ssize_t n;
+    if (!PyArg_ParseTuple(args, "Ow*n", &v, &buffer, &n)) {
+        return NULL;
+    }
+    if (buffer.readonly) {
+        PyErr_SetString(PyExc_TypeError, "buffer must be writable");
+        PyBuffer_Release(&buffer);
+        return NULL;
+    }
+    if (buffer.len < n) {
+        PyErr_SetString(PyExc_ValueError, "buffer must be at least 'n' bytes");
+        PyBuffer_Release(&buffer);
+        return NULL;
+    }
+    int res = PyLong_AsUnsignedByteArray(v, buffer.buf, n);
+    PyBuffer_Release(&buffer);
+    return res >= 0 ? PyLong_FromLong(res) : NULL;
+}
+
+static PyObject *
+pylong_asbytearraywithoptions(PyObject *module, PyObject *args)
+{
+    PyObject *v;
+    Py_buffer buffer;
+    Py_ssize_t n, options;
+    if (!PyArg_ParseTuple(args, "Ow*nn", &v, &buffer, &n, &options)) {
+        return NULL;
+    }
+    if (buffer.readonly) {
+        PyErr_SetString(PyExc_TypeError, "buffer must be writable");
+        PyBuffer_Release(&buffer);
+        return NULL;
+    }
+    if (buffer.len < n) {
+        PyErr_SetString(PyExc_ValueError, "buffer must be at least 'n' bytes");
+        PyBuffer_Release(&buffer);
+        return NULL;
+    }
+    int res = PyLong_AsByteArrayWithOptions(v, buffer.buf, n, (int)options);
+    PyBuffer_Release(&buffer);
+    return res >= 0 ? PyLong_FromLong(res) : NULL;
+}
+
 static PyMethodDef test_methods[] = {
     _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF
     _TESTCAPI_TEST_LONG_API_METHODDEF
@@ -804,6 +876,9 @@ static PyMethodDef test_methods[] = {
     {"pylong_as_size_t",            pylong_as_size_t,           METH_O},
     {"pylong_asdouble",             pylong_asdouble,            METH_O},
     {"pylong_asvoidptr",            pylong_asvoidptr,           METH_O},
+    {"pylong_asbytearray",          pylong_asbytearray,         METH_VARARGS},
+    {"pylong_asunsignedbytearray",  pylong_asunsignedbytearray, METH_VARARGS},
+    {"pylong_asbytearraywithoptions", pylong_asbytearraywithoptions, METH_VARARGS},
     {NULL},
 };
 

Modules/_tkinter.c

@@ -926,7 +926,8 @@ AsObj(PyObject *value)
                                 (unsigned char *)(void *)&wideValue,
                                 sizeof(wideValue),
                                 PY_LITTLE_ENDIAN,
-                                /* signed */ 1) == 0) {
+                                /* signed */ 1,
+                                /* with_exceptions */ 1) == 0) {
             return Tcl_NewWideIntObj(wideValue);
         }
         PyErr_Clear();

Modules/cjkcodecs/multibytecodec.c

@@ -973,7 +973,8 @@ _multibytecodec_MultibyteIncrementalEncoder_setstate_impl(MultibyteIncrementalEn
 
     if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes),
                             1 /* little-endian */ ,
-                            0 /* unsigned */ ) < 0) {
+                            0 /* unsigned */ ,
+                            1 /* with_exceptions */) < 0) {
         goto errorexit;
     }
 
@@ -1255,7 +1256,8 @@ _multibytecodec_MultibyteIncrementalDecoder_setstate_impl(MultibyteIncrementalDe
 
     if (_PyLong_AsByteArray(statelong, statebytes, sizeof(statebytes),
                             1 /* little-endian */ ,
-                            0 /* unsigned */ ) < 0) {
+                            0 /* unsigned */ ,
+                            1 /* with_exceptions */) < 0) {
         return NULL;
     }