gh-101101: Unstable C API tier (PEP 689) (GH-101102)

Author: encukou

Doc/c-api/code.rst

@@ -33,28 +33,47 @@ bound into a function.
 
    Return the number of free variables in *co*.
 
-.. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
+.. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
 
    Return a new code object.  If you need a dummy code object to create a frame,
-   use :c:func:`PyCode_NewEmpty` instead.  Calling :c:func:`PyCode_New` directly
-   will bind you to a precise Python version since the definition of the bytecode
-   changes often. The many arguments of this function are inter-dependent in complex
+   use :c:func:`PyCode_NewEmpty` instead.
+
+   Since the definition of the bytecode changes often, calling
+   :c:func:`PyCode_New` directly can bind you to a precise Python version.
+
+   The many arguments of this function are inter-dependent in complex
    ways, meaning that subtle changes to values are likely to result in incorrect
    execution or VM crashes. Use this function only with extreme care.
 
    .. versionchanged:: 3.11
       Added ``exceptiontable`` parameter.
 
-.. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
+   .. index:: single: PyCode_New
+
+   .. versionchanged:: 3.12
+
+      Renamed from ``PyCode_New`` as part of :ref:`unstable-c-api`.
+      The old name is deprecated, but will remain available until the
+      signature changes again.
+
+.. c:function:: PyCodeObject* PyUnstable_Code_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
 
    Similar to :c:func:`PyCode_New`, but with an extra "posonlyargcount" for positional-only arguments.
    The same caveats that apply to ``PyCode_New`` also apply to this function.
 
-   .. versionadded:: 3.8
+   .. index:: single: PyCode_NewWithPosOnlyArgs
+
+   .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs``
 
    .. versionchanged:: 3.11
       Added ``exceptiontable`` parameter.
 
+   .. versionchanged:: 3.12
+
+      Renamed to ``PyUnstable_Code_NewWithPosOnlyArgs``.
+      The old name is deprecated, but will remain available until the
+      signature changes again.
+
 .. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
 
    Return a new empty code object with the specified filename,
@@ -165,3 +184,70 @@ bound into a function.
    :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``.
 
    .. versionadded:: 3.12
+
+
+Extra information
+-----------------
+
+To support low-level extensions to frame evaluation, such as external
+just-in-time compilers, it is possible to attach arbitrary extra data to
+code objects.
+
+These functions are part of the unstable C API tier:
+this functionality is a CPython implementation detail, and the API
+may change without deprecation warnings.
+
+.. c:function:: Py_ssize_t PyUnstable_Eval_RequestCodeExtraIndex(freefunc free)
+
+   Return a new an opaque index value used to adding data to code objects.
+
+   You generally call this function once (per interpreter) and use the result
+   with ``PyCode_GetExtra`` and ``PyCode_SetExtra`` to manipulate
+   data on individual code objects.
+
+   If *free* is not ``NULL``: when a code object is deallocated,
+   *free* will be called on non-``NULL`` data stored under the new index.
+   Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`.
+
+   .. index:: single: _PyEval_RequestCodeExtraIndex
+
+   .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex``
+
+   .. versionchanged:: 3.12
+
+     Renamed to ``PyUnstable_Eval_RequestCodeExtraIndex``.
+     The old private name is deprecated, but will be available until the API
+     changes.
+
+.. c:function:: int PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra)
+
+   Set *extra* to the extra data stored under the given index.
+   Return 0 on success. Set an exception and return -1 on failure.
+
+   If no data was set under the index, set *extra* to ``NULL`` and return
+   0 without setting an exception.
+
+   .. index:: single: _PyCode_GetExtra
+
+   .. versionadded:: 3.6 as ``_PyCode_GetExtra``
+
+   .. versionchanged:: 3.12
+
+     Renamed to ``PyUnstable_Code_GetExtra``.
+     The old private name is deprecated, but will be available until the API
+     changes.
+
+.. c:function:: int PyUnstable_Code_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
+
+   Set the extra data stored under the given index to *extra*.
+   Return 0 on success. Set an exception and return -1 on failure.
+
+   .. index:: single: _PyCode_SetExtra
+
+   .. versionadded:: 3.6 as ``_PyCode_SetExtra``
+
+   .. versionchanged:: 3.12
+
+     Renamed to ``PyUnstable_Code_SetExtra``.
+     The old private name is deprecated, but will be available until the API
+     changes.

Doc/c-api/stable.rst

@@ -6,9 +6,9 @@
 C API Stability
 ***************
 
-Python's C API is covered by the Backwards Compatibility Policy, :pep:`387`.
-While the C API will change with every minor release (e.g. from 3.9 to 3.10),
-most changes will be source-compatible, typically by only adding new API.
+Unless documented otherwise, Python's C API is covered by the Backwards
+Compatibility Policy, :pep:`387`.
+Most changes to it are source-compatible (typically by only adding new API).
 Changing existing API or removing API is only done after a deprecation period
 or to fix serious issues.
 
@@ -18,8 +18,38 @@ way; see :ref:`stable-abi-platform` below).
 So, code compiled for Python 3.10.0 will work on 3.10.8 and vice versa,
 but will need to be compiled separately for 3.9.x and 3.10.x.
 
+There are two tiers of C API with different stability exepectations:
+
+- *Unstable API*, may change in minor versions without a deprecation period.
+  It is marked by the ``PyUnstable`` prefix in names.
+- *Limited API*, is compatible across several minor releases.
+  When :c:macro:`Py_LIMITED_API` is defined, only this subset is exposed
+  from ``Python.h``.
+
+These are discussed in more detail below.
+
 Names prefixed by an underscore, such as ``_Py_InternalState``,
 are private API that can change without notice even in patch releases.
+If you need to use this API, consider reaching out to
+`CPython developers <https://discuss.python.org/c/core-dev/c-api/30>`_
+to discuss adding public API for your use case.
+
+.. _unstable-c-api:
+
+Unstable C API
+==============
+
+.. index:: single: PyUnstable
+
+Any API named with the ``PyUnstable`` prefix exposes CPython implementation
+details, and may change in every minor release (e.g. from 3.9 to 3.10) without
+any deprecation warnings.
+However, it will not change in a bugfix release (e.g. from 3.10.0 to 3.10.1).
+
+It is generally intended for specialized, low-level tools like debuggers.
+
+Projects that use this API are expected to follow
+CPython development and spend extra effort adjusting to changes.
 
 
 Stable Application Binary Interface

Doc/tools/extensions/c_annotations.py

@@ -143,6 +143,22 @@ def add_annotations(self, app, doctree):
                         ' (Only some members are part of the stable ABI.)')
                 node.insert(0, emph_node)
 
+            # Unstable API annotation.
+            if name.startswith('PyUnstable'):
+                warn_node = nodes.admonition(
+                    classes=['unstable-c-api', 'warning'])
+                message = 'This is '
+                emph_node = nodes.emphasis(message, message)
+                ref_node = addnodes.pending_xref(
+                    'Unstable API', refdomain="std",
+                    reftarget='unstable-c-api',
+                    reftype='ref', refexplicit="False")
+                ref_node += nodes.Text('Unstable API')
+                emph_node += ref_node
+                emph_node += nodes.Text('. It may change without warning in minor releases.')
+                warn_node += emph_node
+                node.insert(0, warn_node)
+
             # Return value annotation
             if objtype != 'function':
                 continue

Doc/whatsnew/3.12.rst

@@ -810,6 +810,29 @@ C API Changes
 New Features
 ------------
 
+
+* :pep:`697`: Introduced the :ref:`Unstable C API tier <unstable-c-api>`,
+  intended for low-level tools like debuggers and JIT compilers.
+  This API may change in each minor release of CPython without deprecation
+  warnings.
+  Its contents are marked by the ``PyUnstable_`` prefix in names.
+
+  Code object constructors:
+
+  - ``PyUnstable_Code_New()`` (renamed from ``PyCode_New``)
+  - ``PyUnstable_Code_NewWithPosOnlyArgs()`` (renamed from ``PyCode_NewWithPosOnlyArgs``)
+
+  Extra storage for code objects (:pep:`523`):
+
+  - ``PyUnstable_Eval_RequestCodeExtraIndex()`` (renamed from ``_PyEval_RequestCodeExtraIndex``)
+  - ``PyUnstable_Code_GetExtra()`` (renamed from ``_PyCode_GetExtra``)
+  - ``PyUnstable_Code_SetExtra()`` (renamed from ``_PyCode_SetExtra``)
+
+  The original names will continue to be available until the respective
+  API changes.
+
+  (Contributed by Petr Viktorin in :gh:`101101`.)
+
 * Added the new limited C API function :c:func:`PyType_FromMetaclass`,
   which generalizes the existing :c:func:`PyType_FromModuleAndSpec` using
   an additional metaclass argument.

Include/README.rst

@@ -1,11 +1,13 @@
 The Python C API
 ================
 
-The C API is divided into three sections:
+The C API is divided into these sections:
 
 1. ``Include/``: Limited API
 2. ``Include/cpython/``: CPython implementation details
-3. ``Include/internal/``: The internal API
+3. ``Include/cpython/``, names with the ``PyUnstable_`` prefix: API that can
+   change between minor releases
+4. ``Include/internal/``, and any name with ``_`` prefix: The internal API
 
 Information on changing the C API is available `in the developer guide`_
 

Include/cpython/ceval.h

@@ -22,7 +22,12 @@ PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _P
 PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds);
 PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void);
 
-PyAPI_FUNC(Py_ssize_t) _PyEval_RequestCodeExtraIndex(freefunc);
+PyAPI_FUNC(Py_ssize_t) PyUnstable_Eval_RequestCodeExtraIndex(freefunc);
+// Old name -- remove when this API changes:
+_Py_DEPRECATED_EXTERNALLY(3.12) static inline Py_ssize_t
+_PyEval_RequestCodeExtraIndex(freefunc f) {
+    return PyUnstable_Eval_RequestCodeExtraIndex(f);
+}
 
 PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *);
 PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *);

Include/cpython/code.h

@@ -178,19 +178,40 @@ static inline int PyCode_GetFirstFree(PyCodeObject *op) {
 #define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive)
 #define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT))
 
-/* Public interface */
-PyAPI_FUNC(PyCodeObject *) PyCode_New(
+/* Unstable public interface */
+PyAPI_FUNC(PyCodeObject *) PyUnstable_Code_New(
         int, int, int, int, int, PyObject *, PyObject *,
         PyObject *, PyObject *, PyObject *, PyObject *,
         PyObject *, PyObject *, PyObject *, int, PyObject *,
         PyObject *);
 
-PyAPI_FUNC(PyCodeObject *) PyCode_NewWithPosOnlyArgs(
+PyAPI_FUNC(PyCodeObject *) PyUnstable_Code_NewWithPosOnlyArgs(
         int, int, int, int, int, int, PyObject *, PyObject *,
         PyObject *, PyObject *, PyObject *, PyObject *,
         PyObject *, PyObject *, PyObject *, int, PyObject *,
         PyObject *);
         /* same as struct above */
+// Old names -- remove when this API changes:
+_Py_DEPRECATED_EXTERNALLY(3.12) static inline PyCodeObject *
+PyCode_New(
+        int a, int b, int c, int d, int e, PyObject *f, PyObject *g,
+        PyObject *h, PyObject *i, PyObject *j, PyObject *k,
+        PyObject *l, PyObject *m, PyObject *n, int o, PyObject *p,
+        PyObject *q)
+{
+    return PyUnstable_Code_New(
+        a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q);
+}
+_Py_DEPRECATED_EXTERNALLY(3.12) static inline PyCodeObject *
+PyCode_NewWithPosOnlyArgs(
+        int a, int poac, int b, int c, int d, int e, PyObject *f, PyObject *g,
+        PyObject *h, PyObject *i, PyObject *j, PyObject *k,
+        PyObject *l, PyObject *m, PyObject *n, int o, PyObject *p,
+        PyObject *q)
+{
+    return PyUnstable_Code_NewWithPosOnlyArgs(
+        a, poac, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q);
+}
 
 /* Creates a new empty code object with the specified source location. */
 PyAPI_FUNC(PyCodeObject *)
@@ -269,11 +290,21 @@ PyAPI_FUNC(PyObject*) _PyCode_ConstantKey(PyObject *obj);
 PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts,
                                       PyObject *names, PyObject *lnotab);
 
-
-PyAPI_FUNC(int) _PyCode_GetExtra(PyObject *code, Py_ssize_t index,
-                                 void **extra);
-PyAPI_FUNC(int) _PyCode_SetExtra(PyObject *code, Py_ssize_t index,
-                                 void *extra);
+PyAPI_FUNC(int) PyUnstable_Code_GetExtra(
+    PyObject *code, Py_ssize_t index, void **extra);
+PyAPI_FUNC(int) PyUnstable_Code_SetExtra(
+    PyObject *code, Py_ssize_t index, void *extra);
+// Old names -- remove when this API changes:
+_Py_DEPRECATED_EXTERNALLY(3.12) static inline int
+_PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra)
+{
+    return PyUnstable_Code_GetExtra(code, index, extra);
+}
+_Py_DEPRECATED_EXTERNALLY(3.12) static inline int
+_PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
+{
+    return PyUnstable_Code_SetExtra(code, index, extra);
+}
 
 /* Equivalent to getattr(code, 'co_code') in Python.
    Returns a strong reference to a bytes object. */

Include/pyport.h

@@ -323,6 +323,15 @@ extern "C" {
 #define Py_DEPRECATED(VERSION_UNUSED)
 #endif
 
+// _Py_DEPRECATED_EXTERNALLY(version)
+// Deprecated outside CPython core.
+#ifdef Py_BUILD_CORE
+#define _Py_DEPRECATED_EXTERNALLY(VERSION_UNUSED)
+#else
+#define _Py_DEPRECATED_EXTERNALLY(version) Py_DEPRECATED(version)
+#endif
+
+
 #if defined(__clang__)
 #define _Py_COMP_DIAG_PUSH _Pragma("clang diagnostic push")
 #define _Py_COMP_DIAG_IGNORE_DEPR_DECLS \

Lib/test/test_code.py

@@ -752,15 +752,15 @@ def f():
     py = ctypes.pythonapi
     freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp)
 
-    RequestCodeExtraIndex = py._PyEval_RequestCodeExtraIndex
+    RequestCodeExtraIndex = py.PyUnstable_Eval_RequestCodeExtraIndex
     RequestCodeExtraIndex.argtypes = (freefunc,)
     RequestCodeExtraIndex.restype = ctypes.c_ssize_t
 
-    SetExtra = py._PyCode_SetExtra
+    SetExtra = py.PyUnstable_Code_SetExtra
     SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp)
     SetExtra.restype = ctypes.c_int
 
-    GetExtra = py._PyCode_GetExtra
+    GetExtra = py.PyUnstable_Code_GetExtra
     GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t,
                          ctypes.POINTER(ctypes.c_voidp))
     GetExtra.restype = ctypes.c_int

Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst

@@ -0,0 +1,3 @@
+Introduced the *Unstable C API tier*, marking APi that is allowed to change
+in minor releases without a deprecation period.
+See :pep:`689` for details.

Modules/Setup.stdlib.in

@@ -169,7 +169,7 @@
 @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
 @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
 @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c
-@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c
+@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c
 @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c
 
 # Some testing modules MUST be built as shared libraries.