diff --git a/.gitattributes b/.gitattributes index 671b980ec053be..5b81d2cb3c90e9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -77,6 +77,7 @@ Include/internal/pycore_opcode.h generated Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated Include/internal/pycore_uop_ids.h generated +Include/internal/pycore_uop_metadata.h generated Include/opcode.h generated Include/opcode_ids.h generated Include/token.h generated @@ -96,7 +97,7 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated -Python/tier2_redundancy_eliminator_bytecodes.c.h generated +Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5dbfbbb8ebaf7e..e8eed400d961fc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -38,7 +38,7 @@ Python/ast_opt.c @isidentical Python/bytecodes.c @markshannon @gvanrossum Python/optimizer*.c @markshannon @gvanrossum Python/optimizer_analysis.c @Fidget-Spinner -Python/tier2_redundancy_eliminator_bytecodes.c @Fidget-Spinner +Python/optimizer_bytecodes.c @Fidget-Spinner Lib/test/test_patma.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra Lib/test/test_capi/test_misc.py @markshannon @gvanrossum @@ -247,5 +247,10 @@ Lib/test/test_interpreters/ @ericsnowcurrently /Tools/wasm/ @brettcannon # SBOM +/Misc/externals.spdx.json @sethmlarson /Misc/sbom.spdx.json @sethmlarson /Tools/build/generate_sbom.py @sethmlarson + +# Config Parser +Lib/configparser.py @jaraco +Lib/test/test_configparser.py @jaraco diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 70db2a6250e8da..20d1fad40ecafe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -162,7 +162,7 @@ jobs: - name: Build CPython run: | make -j4 regen-all - make regen-stdlib-module-names + make regen-stdlib-module-names regen-sbom - name: Check for changes run: | git add -u diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 69c7b45376a411..809ac45919fe74 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -5,13 +5,13 @@ on: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' - - 'Python/tier2_redundancy_eliminator_bytecodes.c' + - 'Python/optimizer_bytecodes.c' push: paths: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' - - 'Python/tier2_redundancy_eliminator_bytecodes.c' + - 'Python/optimizer_bytecodes.c' workflow_dispatch: concurrency: @@ -29,6 +29,7 @@ jobs: target: - i686-pc-windows-msvc/msvc - x86_64-pc-windows-msvc/msvc + - aarch64-pc-windows-msvc/msvc - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc @@ -49,6 +50,10 @@ jobs: architecture: x64 runner: windows-latest compiler: msvc + - target: aarch64-pc-windows-msvc/msvc + architecture: ARM64 + runner: windows-latest + compiler: msvc - target: x86_64-apple-darwin/clang architecture: x86_64 runner: macos-13 @@ -70,13 +75,13 @@ jobs: runner: ubuntu-latest compiler: gcc # These fail because of emulation, not because of the JIT: - exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv + exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection - target: aarch64-unknown-linux-gnu/clang architecture: aarch64 runner: ubuntu-latest compiler: clang # These fail because of emulation, not because of the JIT: - exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv + exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection env: CC: ${{ matrix.compiler }} steps: @@ -85,14 +90,21 @@ jobs: with: python-version: '3.11' - - name: Windows - if: runner.os == 'Windows' + - name: Native Windows + if: runner.os == 'Windows' && matrix.architecture != 'ARM64' run: | choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3 - - name: macOS + # No PGO or tests (yet): + - name: Emulated Windows + if: runner.os == 'Windows' && matrix.architecture == 'ARM64' + run: | + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} + ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + + - name: Native macOS if: runner.os == 'macOS' run: | brew install llvm@${{ matrix.llvm }} diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index ba62d9568c6b80..65b73cd791c75c 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -17,6 +17,7 @@ jobs: HOMEBREW_NO_ANALYTICS: 1 HOMEBREW_NO_AUTO_UPDATE: 1 HOMEBREW_NO_INSTALL_CLEANUP: 1 + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 PYTHONSTRICTEXTENSIONBUILD: 1 strategy: fail-fast: false diff --git a/.gitignore b/.gitignore index 6ed7197e3ab626..2194b393aa4821 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,7 @@ Lib/test/data/* /_bootstrap_python /Makefile /Makefile.pre +iOS/Resources/Info.plist Mac/Makefile Mac/PythonLauncher/Info.plist Mac/PythonLauncher/Makefile diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 382cfbff864072..f6fdd7574323c7 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -30,9 +30,13 @@ bound into a function. Return true if *co* is a :ref:`code object `. This function always succeeds. -.. c:function:: int PyCode_GetNumFree(PyCodeObject *co) +.. c:function:: Py_ssize_t PyCode_GetNumFree(PyCodeObject *co) - Return the number of free variables in *co*. + Return the number of free variables in a code object. + +.. c:function:: int PyCode_GetFirstFree(PyCodeObject *co) + + Return the position of the first free variable in a code object. .. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index e6309ae7614d34..ba13fd1b9973e0 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -105,7 +105,7 @@ Printing and clearing parameters help format the warning message; they have the same meaning and values as in :c:func:`PyUnicode_FromFormat`. ``PyErr_WriteUnraisable(obj)`` is roughtly equivalent to - ``PyErr_FormatUnraisable("Exception ignored in: %R, obj)``. + ``PyErr_FormatUnraisable("Exception ignored in: %R", obj)``. If *format* is ``NULL``, only the traceback is printed. .. versionadded:: 3.13 diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index f24282e76a33d1..582f5c7bf05471 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -358,46 +358,86 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Copy the Python integer value to a native *buffer* of size *n_bytes*:: - int value; - Py_ssize_t bytes = PyLong_AsNativeBytes(v, &value, sizeof(value), -1); + int32_t value; + Py_ssize_t bytes = PyLong_AsNativeBits(pylong, &value, sizeof(value), -1); if (bytes < 0) { - // Error occurred + // A Python exception was set with the reason. return NULL; } else if (bytes <= (Py_ssize_t)sizeof(value)) { // Success! } else { - // Overflow occurred, but 'value' contains truncated value + // Overflow occurred, but 'value' contains the truncated + // lowest bits of pylong. } + The above example may look *similar* to + :c:func:`PyLong_As* ` + but instead fills in a specific caller defined type and never raises an + error about of the :class:`int` *pylong*'s value regardless of *n_bytes* + or the returned byte count. + + To get at the entire potentially big Python value, this can be used to + reserve enough space and copy it:: + + // Ask how much space we need. + Py_ssize_t expected = PyLong_AsNativeBits(pylong, NULL, 0, -1); + if (expected < 0) { + // A Python exception was set with the reason. + return NULL; + } + assert(expected != 0); // Impossible per the API definition. + uint8_t *bignum = malloc(expected); + if (!bignum) { + PyErr_SetString(PyExc_MemoryError, "bignum malloc failed."); + return NULL; + } + // Safely get the entire value. + Py_ssize_t bytes = PyLong_AsNativeBits(pylong, bignum, expected, -1); + if (bytes < 0) { // Exception set. + free(bignum); + return NULL; + } + else if (bytes > expected) { // Be safe, should not be possible. + PyErr_SetString(PyExc_RuntimeError, + "Unexpected bignum truncation after a size check."); + free(bignum); + return NULL; + } + // The expected success given the above pre-check. + // ... use bignum ... + free(bignum); + *endianness* may be passed ``-1`` for the native endian that CPython was compiled with, or ``0`` for big endian and ``1`` for little. - Return ``-1`` with an exception raised if *pylong* cannot be interpreted as + Returns ``-1`` with an exception raised if *pylong* cannot be interpreted as an integer. Otherwise, return the size of the buffer required to store the value. If this is equal to or less than *n_bytes*, the entire value was - copied. + copied. ``0`` will never be returned. - Unless an exception is raised, all *n_bytes* of the buffer will be written - with as much of the value as can fit. This allows the caller to ignore all - non-negative results if the intent is to match the typical behavior of a - C-style downcast. No exception is set for this case. + Unless an exception is raised, all *n_bytes* of the buffer will always be + written. In the case of truncation, as many of the lowest bits of the value + as could fit are written. This allows the caller to ignore all non-negative + results if the intent is to match the typical behavior of a C-style + downcast. No exception is set on truncation. - Values are always copied as two's-complement, and sufficient buffer will be + Values are always copied as two's-complement and sufficient buffer will be requested to include a sign bit. For example, this may cause an value that fits into 8 bytes when treated as unsigned to request 9 bytes, even though all eight bytes were copied into the buffer. What has been omitted is the - zero sign bit, which is redundant when the intention is to treat the value as - unsigned. + zero sign bit -- redundant if the caller's intention is to treat the value + as unsigned. - Passing zero to *n_bytes* will return the requested buffer size. + Passing zero to *n_bytes* will return the size of a buffer that would + be large enough to hold the value. This may be larger than technically + necessary, but not unreasonably so. .. note:: - When the value does not fit in the provided buffer, the requested size - returned from the function may be larger than necessary. Passing 0 to this - function is not an accurate way to determine the bit length of a value. + Passing *n_bytes=0* to this function is not an accurate way to determine + the bit length of a value. .. versionadded:: 3.13 diff --git a/Doc/conf.py b/Doc/conf.py index 0e84d866a22f5b..f4c75c5758cb28 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -101,11 +101,13 @@ ('c:func', 'dlopen'), ('c:func', 'exec'), ('c:func', 'fcntl'), + ('c:func', 'flock'), ('c:func', 'fork'), ('c:func', 'free'), ('c:func', 'gettimeofday'), ('c:func', 'gmtime'), ('c:func', 'grantpt'), + ('c:func', 'ioctl'), ('c:func', 'localeconv'), ('c:func', 'localtime'), ('c:func', 'main'), @@ -275,6 +277,7 @@ ('py:attr', '__annotations__'), ('py:meth', '__missing__'), ('py:attr', '__wrapped__'), + ('py:attr', 'decimal.Context.clamp'), ('py:meth', 'index'), # list.index, tuple.index, etc. ] diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 8727332594bda6..ec7c2897594999 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -133,8 +133,6 @@ Python versions are numbered "A.B.C" or "A.B": changes. * *C* is the micro version number -- it is incremented for each bugfix release. -See :pep:`6` for more information about bugfix releases. - Not all releases are bugfix releases. In the run-up to a new feature release, a series of development releases are made, denoted as alpha, beta, or release candidate. Alphas are early releases in which interfaces aren't yet finalized; @@ -157,7 +155,11 @@ unreleased versions, built directly from the CPython development repository. In practice, after a final minor release is made, the version is incremented to the next minor version, which becomes the "a0" version, e.g. "2.4a0". -See also the documentation for :data:`sys.version`, :data:`sys.hexversion`, and +See the `Developer's Guide +`__ +for more information about the development cycle, and +:pep:`387` to learn more about Python's backward compatibility policy. See also +the documentation for :data:`sys.version`, :data:`sys.hexversion`, and :data:`sys.version_info`. diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 75346f2c7618c2..51f9f4a6556e57 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1,8 +1,8 @@ .. _descriptorhowto: -====================== -Descriptor HowTo Guide -====================== +================ +Descriptor Guide +================ :Author: Raymond Hettinger :Contact: @@ -1004,31 +1004,42 @@ here is a pure Python equivalent: if doc is None and fget is not None: doc = fget.__doc__ self.__doc__ = doc - self._name = '' + self._name = None def __set_name__(self, owner, name): self._name = name + @property + def __name__(self): + return self._name if self._name is not None else self.fget.__name__ + + @__name__.setter + def __name__(self, value): + self._name = value + def __get__(self, obj, objtype=None): if obj is None: return self if self.fget is None: raise AttributeError( - f'property {self._name!r} of {type(obj).__name__!r} object has no getter' + f'property {self.__name__!r} of {type(obj).__name__!r} ' + 'object has no getter' ) return self.fget(obj) def __set__(self, obj, value): if self.fset is None: raise AttributeError( - f'property {self._name!r} of {type(obj).__name__!r} object has no setter' + f'property {self.__name__!r} of {type(obj).__name__!r} ' + 'object has no setter' ) self.fset(obj, value) def __delete__(self, obj): if self.fdel is None: raise AttributeError( - f'property {self._name!r} of {type(obj).__name__!r} object has no deleter' + f'property {self.__name__!r} of {type(obj).__name__!r} ' + 'object has no deleter' ) self.fdel(obj) @@ -1192,6 +1203,10 @@ roughly equivalent to: "Emulate method_getattro() in Objects/classobject.c" return getattr(self.__func__, name) + def __get__(self, obj, objtype=None): + "Emulate method_descr_get() in Objects/classobject.c" + return self + To support automatic creation of methods, functions include the :meth:`__get__` method for binding methods during attribute access. This means that functions are non-data descriptors that return bound methods @@ -1214,8 +1229,20 @@ descriptor works in practice: .. testcode:: class D: - def f(self, x): - return x + def f(self): + return self + + class D2: + pass + +.. doctest:: + :hide: + + >>> d = D() + >>> d2 = D2() + >>> d2.f = d.f.__get__(d2, D2) + >>> d2.f() is d + True The function has a :term:`qualified name` attribute to support introspection: diff --git a/Doc/howto/gdb_helpers.rst b/Doc/howto/gdb_helpers.rst new file mode 100644 index 00000000000000..53bbf7ddaa2ab9 --- /dev/null +++ b/Doc/howto/gdb_helpers.rst @@ -0,0 +1,449 @@ +.. _gdb: + +========================================================= +Debugging C API extensions and CPython Internals with GDB +========================================================= + +.. highlight:: none + +This document explains how the Python GDB extension, ``python-gdb.py``, can +be used with the GDB debugger to debug CPython extensions and the +CPython interpreter itself. + +When debugging low-level problems such as crashes or deadlocks, a low-level +debugger, such as GDB, is useful to diagnose and correct the issue. +By default, GDB (or any of its front-ends) doesn't support high-level +information specific to the CPython interpreter. + +The ``python-gdb.py`` extension adds CPython interpreter information to GDB. +The extension helps introspect the stack of currently executing Python functions. +Given a Python object represented by a :c:expr:`PyObject *` pointer, +the extension surfaces the type and value of the object. + +Developers who are working on CPython extensions or tinkering with parts +of CPython that are written in C can use this document to learn how to use the +``python-gdb.py`` extension with GDB. + +.. note:: + + This document assumes that you are familiar with the basics of GDB and the + CPython C API. It consolidates guidance from the + `devguide `_ and the + `Python wiki `_. + + +Prerequisites +============= + +You need to have: + +- GDB 7 or later. (For earlier versions of GDB, see ``Misc/gdbinit`` in the + sources of Python 3.11 or earlier.) +- GDB-compatible debugging information for Python and any extension you are + debugging. +- The ``python-gdb.py`` extension. + +The extension is built with Python, but might be distributed separately or +not at all. Below, we include tips for a few common systems as examples. +Note that even if the instructions match your system, they might be outdated. + + +Setup with Python built from source +----------------------------------- + +When you build CPython from source, debugging information should be available, +and the build should add a ``python-gdb.py`` file to the root directory of +your repository. + +To activate support, you must add the directory containing ``python-gdb.py`` +to GDB's "auto-load-safe-path". +If you haven't done this, recent versions of GDB will print out a warning +with instructions on how to do this. + +.. note:: + + If you do not see instructions for your version of GDB, put this in your + configuration file (``~/.gdbinit`` or ``~/.config/gdb/gdbinit``):: + + add-auto-load-safe-path /path/to/cpython + + You can also add multiple paths, separated by ``:``. + + +Setup for Python from a Linux distro +------------------------------------ + +Most Linux systems provide debug information for the system Python +in a package called ``python-debuginfo``, ``python-dbg`` or similar. +For example: + +- Fedora: + + .. code-block:: shell + + sudo dnf install gdb + sudo dnf debuginfo-install python3 + +- Ubuntu: + + .. code-block:: shell + + sudo apt install gdb python3-dbg + +On several recent Linux systems, GDB can download debugging symbols +automatically using *debuginfod*. +However, this will not install the ``python-gdb.py`` extension; +you generally do need to install the debug info package separately. + + +Using the Debug build and Development mode +========================================== + +For easier debugging, you might want to: + +- Use a :ref:`debug build ` of Python. (When building from source, + use ``configure --with-pydebug``. On Linux distros, install and run a package + like ``python-debug`` or ``python-dbg``, if available.) +- Use the runtime :ref:`development mode ` (``-X dev``). + +Both enable extra assertions and disable some optimizations. +Sometimes this hides the bug you are trying to find, but in most cases they +make the process easier. + + +Using the ``python-gdb`` extension +================================== + +When the extension is loaded, it provides two main features: +pretty printers for Python values, and additional commands. + +Pretty-printers +--------------- + +This is what a GDB backtrace looks like (truncated) when this extension is +enabled:: + + #0 0x000000000041a6b1 in PyObject_Malloc (nbytes=Cannot access memory at address 0x7fffff7fefe8 + ) at Objects/obmalloc.c:748 + #1 0x000000000041b7c0 in _PyObject_DebugMallocApi (id=111 'o', nbytes=24) at Objects/obmalloc.c:1445 + #2 0x000000000041b717 in _PyObject_DebugMalloc (nbytes=24) at Objects/obmalloc.c:1412 + #3 0x000000000044060a in _PyUnicode_New (length=11) at Objects/unicodeobject.c:346 + #4 0x00000000004466aa in PyUnicodeUCS2_DecodeUTF8Stateful (s=0x5c2b8d "__lltrace__", size=11, errors=0x0, consumed= + 0x0) at Objects/unicodeobject.c:2531 + #5 0x0000000000446647 in PyUnicodeUCS2_DecodeUTF8 (s=0x5c2b8d "__lltrace__", size=11, errors=0x0) + at Objects/unicodeobject.c:2495 + #6 0x0000000000440d1b in PyUnicodeUCS2_FromStringAndSize (u=0x5c2b8d "__lltrace__", size=11) + at Objects/unicodeobject.c:551 + #7 0x0000000000440d94 in PyUnicodeUCS2_FromString (u=0x5c2b8d "__lltrace__") at Objects/unicodeobject.c:569 + #8 0x0000000000584abd in PyDict_GetItemString (v= + {'Yuck': , '__builtins__': , '__file__': 'Lib/test/crashers/nasty_eq_vs_dict.py', '__package__': None, 'y': , 'dict': {0: 0, 1: 1, 2: 2, 3: 3}, '__cached__': None, '__name__': '__main__', 'z': , '__doc__': None}, key= + 0x5c2b8d "__lltrace__") at Objects/dictobject.c:2171 + +Notice how the dictionary argument to ``PyDict_GetItemString`` is displayed +as its ``repr()``, rather than an opaque ``PyObject *`` pointer. + +The extension works by supplying a custom printing routine for values of type +``PyObject *``. If you need to access lower-level details of an object, then +cast the value to a pointer of the appropriate type. For example:: + + (gdb) p globals + $1 = {'__builtins__': , '__name__': + '__main__', 'ctypes': , '__doc__': None, + '__package__': None} + + (gdb) p *(PyDictObject*)globals + $2 = {ob_refcnt = 3, ob_type = 0x3dbdf85820, ma_fill = 5, ma_used = 5, + ma_mask = 7, ma_table = 0x63d0f8, ma_lookup = 0x3dbdc7ea70 + , ma_smalltable = {{me_hash = 7065186196740147912, + me_key = '__builtins__', me_value = }, + {me_hash = -368181376027291943, me_key = '__name__', + me_value ='__main__'}, {me_hash = 0, me_key = 0x0, me_value = 0x0}, + {me_hash = 0, me_key = 0x0, me_value = 0x0}, + {me_hash = -9177857982131165996, me_key = 'ctypes', + me_value = }, + {me_hash = -8518757509529533123, me_key = '__doc__', me_value = None}, + {me_hash = 0, me_key = 0x0, me_value = 0x0}, { + me_hash = 6614918939584953775, me_key = '__package__', me_value = None}}} + +Note that the pretty-printers do not actually call ``repr()``. +For basic types, they try to match its result closely. + +An area that can be confusing is that the custom printer for some types look a +lot like GDB's built-in printer for standard types. For example, the +pretty-printer for a Python ``int`` (:c:expr:`PyLongObject *`) +gives a representation that is not distinguishable from one of a +regular machine-level integer:: + + (gdb) p some_machine_integer + $3 = 42 + + (gdb) p some_python_integer + $4 = 42 + +The internal structure can be revealed with a cast to :c:expr:`PyLongObject *`: + + (gdb) p *(PyLongObject*)some_python_integer + $5 = {ob_base = {ob_base = {ob_refcnt = 8, ob_type = 0x3dad39f5e0}, ob_size = 1}, + ob_digit = {42}} + +A similar confusion can arise with the ``str`` type, where the output looks a +lot like gdb's built-in printer for ``char *``:: + + (gdb) p ptr_to_python_str + $6 = '__builtins__' + +The pretty-printer for ``str`` instances defaults to using single-quotes (as +does Python's ``repr`` for strings) whereas the standard printer for ``char *`` +values uses double-quotes and contains a hexadecimal address:: + + (gdb) p ptr_to_char_star + $7 = 0x6d72c0 "hello world" + +Again, the implementation details can be revealed with a cast to +:c:expr:`PyUnicodeObject *`:: + + (gdb) p *(PyUnicodeObject*)$6 + $8 = {ob_base = {ob_refcnt = 33, ob_type = 0x3dad3a95a0}, length = 12, + str = 0x7ffff2128500, hash = 7065186196740147912, state = 1, defenc = 0x0} + +``py-list`` +----------- + + The extension adds a ``py-list`` command, which + lists the Python source code (if any) for the current frame in the selected + thread. The current line is marked with a ">":: + + (gdb) py-list + 901 if options.profile: + 902 options.profile = False + 903 profile_me() + 904 return + 905 + >906 u = UI() + 907 if not u.quit: + 908 try: + 909 gtk.main() + 910 except KeyboardInterrupt: + 911 # properly quit on a keyboard interrupt... + + Use ``py-list START`` to list at a different line number within the Python + source, and ``py-list START,END`` to list a specific range of lines within + the Python source. + +``py-up`` and ``py-down`` +------------------------- + + The ``py-up`` and ``py-down`` commands are analogous to GDB's regular ``up`` + and ``down`` commands, but try to move at the level of CPython frames, rather + than C frames. + + GDB is not always able to read the relevant frame information, depending on + the optimization level with which CPython was compiled. Internally, the + commands look for C frames that are executing the default frame evaluation + function (that is, the core bytecode interpreter loop within CPython) and + look up the value of the related ``PyFrameObject *``. + + They emit the frame number (at the C level) within the thread. + + For example:: + + (gdb) py-up + #37 Frame 0x9420b04, for file /usr/lib/python2.6/site-packages/ + gnome_sudoku/main.py, line 906, in start_game () + u = UI() + (gdb) py-up + #40 Frame 0x948e82c, for file /usr/lib/python2.6/site-packages/ + gnome_sudoku/gnome_sudoku.py, line 22, in start_game(main=) + main.start_game() + (gdb) py-up + Unable to find an older python frame + + so we're at the top of the Python stack. + + The frame numbers correspond to those displayed by GDB's standard + ``backtrace`` command. + The command skips C frames which are not executing Python code. + + Going back down:: + + (gdb) py-down + #37 Frame 0x9420b04, for file /usr/lib/python2.6/site-packages/gnome_sudoku/main.py, line 906, in start_game () + u = UI() + (gdb) py-down + #34 (unable to read python frame information) + (gdb) py-down + #23 (unable to read python frame information) + (gdb) py-down + #19 (unable to read python frame information) + (gdb) py-down + #14 Frame 0x99262ac, for file /usr/lib/python2.6/site-packages/gnome_sudoku/game_selector.py, line 201, in run_swallowed_dialog (self=, puzzle=None, saved_games=[{'gsd.auto_fills': 0, 'tracking': {}, 'trackers': {}, 'notes': [], 'saved_at': 1270084485, 'game': '7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 0 0 0 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5\n7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 1 8 3 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5', 'gsd.impossible_hints': 0, 'timer.__absolute_start_time__': , 'gsd.hints': 0, 'timer.active_time': , 'timer.total_time': }], dialog=, saved_game_model=, sudoku_maker=, main_page=0) at remote 0x98fa6e4>, d=) + gtk.main() + (gdb) py-down + #8 (unable to read python frame information) + (gdb) py-down + Unable to find a newer python frame + + and we're at the bottom of the Python stack. + + Note that in Python 3.12 and newer, the same C stack frame can be used for + multiple Python stack frames. This means that ``py-up`` and ``py-down`` + may move multiple Python frames at once. For example:: + + (gdb) py-up + #6 Frame 0x7ffff7fb62b0, for file /tmp/rec.py, line 5, in recursive_function (n=0) + time.sleep(5) + #6 Frame 0x7ffff7fb6240, for file /tmp/rec.py, line 7, in recursive_function (n=1) + recursive_function(n-1) + #6 Frame 0x7ffff7fb61d0, for file /tmp/rec.py, line 7, in recursive_function (n=2) + recursive_function(n-1) + #6 Frame 0x7ffff7fb6160, for file /tmp/rec.py, line 7, in recursive_function (n=3) + recursive_function(n-1) + #6 Frame 0x7ffff7fb60f0, for file /tmp/rec.py, line 7, in recursive_function (n=4) + recursive_function(n-1) + #6 Frame 0x7ffff7fb6080, for file /tmp/rec.py, line 7, in recursive_function (n=5) + recursive_function(n-1) + #6 Frame 0x7ffff7fb6020, for file /tmp/rec.py, line 9, in () + recursive_function(5) + (gdb) py-up + Unable to find an older python frame + + +``py-bt`` +--------- + + The ``py-bt`` command attempts to display a Python-level backtrace of the + current thread. + + For example:: + + (gdb) py-bt + #8 (unable to read python frame information) + #11 Frame 0x9aead74, for file /usr/lib/python2.6/site-packages/gnome_sudoku/dialog_swallower.py, line 48, in run_dialog (self=, main_page=0) at remote 0x98fa6e4>, d=) + gtk.main() + #14 Frame 0x99262ac, for file /usr/lib/python2.6/site-packages/gnome_sudoku/game_selector.py, line 201, in run_swallowed_dialog (self=, puzzle=None, saved_games=[{'gsd.auto_fills': 0, 'tracking': {}, 'trackers': {}, 'notes': [], 'saved_at': 1270084485, 'game': '7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 0 0 0 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5\n7 8 0 0 0 0 0 5 6 0 0 9 0 8 0 1 0 0 0 4 6 0 0 0 0 7 0 6 5 1 8 3 4 7 9 2 0 0 0 9 0 1 0 0 0 3 9 7 6 0 0 0 1 8 0 6 0 0 0 0 2 8 0 0 0 5 0 4 0 6 0 0 2 1 0 0 0 0 0 4 5', 'gsd.impossible_hints': 0, 'timer.__absolute_start_time__': , 'gsd.hints': 0, 'timer.active_time': , 'timer.total_time': }], dialog=, saved_game_model=, sudoku_maker=) + main.start_game() + + The frame numbers correspond to those displayed by GDB's standard + ``backtrace`` command. + +``py-print`` +------------ + + The ``py-print`` command looks up a Python name and tries to print it. + It looks in locals within the current thread, then globals, then finally + builtins:: + + (gdb) py-print self + local 'self' = , + main_page=0) at remote 0x98fa6e4> + (gdb) py-print __name__ + global '__name__' = 'gnome_sudoku.dialog_swallower' + (gdb) py-print len + builtin 'len' = + (gdb) py-print scarlet_pimpernel + 'scarlet_pimpernel' not found + + If the current C frame corresponds to multiple Python frames, ``py-print`` + only considers the first one. + +``py-locals`` +------------- + + The ``py-locals`` command looks up all Python locals within the current + Python frame in the selected thread, and prints their representations:: + + (gdb) py-locals + self = , + main_page=0) at remote 0x98fa6e4> + d = + + If the current C frame corresponds to multiple Python frames, locals from + all of them will be shown:: + + (gdb) py-locals + Locals for recursive_function + n = 0 + Locals for recursive_function + n = 1 + Locals for recursive_function + n = 2 + Locals for recursive_function + n = 3 + Locals for recursive_function + n = 4 + Locals for recursive_function + n = 5 + Locals for + + +Use with GDB commands +===================== + +The extension commands complement GDB's built-in commands. +For example, you can use a frame numbers shown by ``py-bt`` with the ``frame`` +command to go a specific frame within the selected thread, like this:: + + (gdb) py-bt + (output snipped) + #68 Frame 0xaa4560, for file Lib/test/regrtest.py, line 1548, in () + main() + (gdb) frame 68 + #68 0x00000000004cd1e6 in PyEval_EvalFrameEx (f=Frame 0xaa4560, for file Lib/test/regrtest.py, line 1548, in (), throwflag=0) at Python/ceval.c:2665 + 2665 x = call_function(&sp, oparg); + (gdb) py-list + 1543 # Run the tests in a context manager that temporary changes the CWD to a + 1544 # temporary and writable directory. If it's not possible to create or + 1545 # change the CWD, the original CWD will be used. The original CWD is + 1546 # available from test_support.SAVEDCWD. + 1547 with test_support.temp_cwd(TESTCWD, quiet=True): + >1548 main() + +The ``info threads`` command will give you a list of the threads within the +process, and you can use the ``thread`` command to select a different one:: + + (gdb) info threads + 105 Thread 0x7fffefa18710 (LWP 10260) sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:86 + 104 Thread 0x7fffdf5fe710 (LWP 10259) sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:86 + * 1 Thread 0x7ffff7fe2700 (LWP 10145) 0x00000038e46d73e3 in select () at ../sysdeps/unix/syscall-template.S:82 + +You can use ``thread apply all COMMAND`` or (``t a a COMMAND`` for short) to run +a command on all threads. With ``py-bt``, this lets you see what every +thread is doing at the Python level:: + + (gdb) t a a py-bt + + Thread 105 (Thread 0x7fffefa18710 (LWP 10260)): + #5 Frame 0x7fffd00019d0, for file /home/david/coding/python-svn/Lib/threading.py, line 155, in _acquire_restore (self=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=, _RLock__count=1) at remote 0xd7ff40>, count_owner=(1, 140737213728528), count=1, owner=140737213728528) + self.__block.acquire() + #8 Frame 0x7fffac001640, for file /home/david/coding/python-svn/Lib/threading.py, line 269, in wait (self=<_Condition(_Condition__lock=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=, _RLock__count=1) at remote 0xd7ff40>, acquire=, _is_owned=, _release_save=, release=, _acquire_restore=, _Verbose__verbose=False, _Condition__waiters=[]) at remote 0xd7fd10>, timeout=None, waiter=, saved_state=(1, 140737213728528)) + self._acquire_restore(saved_state) + #12 Frame 0x7fffb8001a10, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 348, in f () + cond.wait() + #16 Frame 0x7fffb8001c40, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 37, in task (tid=140737213728528) + f() + + Thread 104 (Thread 0x7fffdf5fe710 (LWP 10259)): + #5 Frame 0x7fffe4001580, for file /home/david/coding/python-svn/Lib/threading.py, line 155, in _acquire_restore (self=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=, _RLock__count=1) at remote 0xd7ff40>, count_owner=(1, 140736940992272), count=1, owner=140736940992272) + self.__block.acquire() + #8 Frame 0x7fffc8002090, for file /home/david/coding/python-svn/Lib/threading.py, line 269, in wait (self=<_Condition(_Condition__lock=<_RLock(_Verbose__verbose=False, _RLock__owner=140737354016512, _RLock__block=, _RLock__count=1) at remote 0xd7ff40>, acquire=, _is_owned=, _release_save=, release=, _acquire_restore=, _Verbose__verbose=False, _Condition__waiters=[]) at remote 0xd7fd10>, timeout=None, waiter=, saved_state=(1, 140736940992272)) + self._acquire_restore(saved_state) + #12 Frame 0x7fffac001c90, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 348, in f () + cond.wait() + #16 Frame 0x7fffac0011c0, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 37, in task (tid=140736940992272) + f() + + Thread 1 (Thread 0x7ffff7fe2700 (LWP 10145)): + #5 Frame 0xcb5380, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 16, in _wait () + time.sleep(0.01) + #8 Frame 0x7fffd00024a0, for file /home/david/coding/python-svn/Lib/test/lock_tests.py, line 378, in _check_notify (self=, skipped=[], _mirrorOutput=False, testsRun=39, buffer=False, _original_stderr=, _stdout_buffer=, _stderr_buffer=, _moduleSetUpFailed=False, expectedFailures=[], errors=[], _previousTestClass=, unexpectedSuccesses=[], failures=[], shouldStop=False, failfast=False) at remote 0xc185a0>, _threads=(0,), _cleanups=[], _type_equality_funcs={: , : , : , : , >> sorted("This is a test string from Andrew".split(), key=str.lower) + >>> sorted("This is a test string from Andrew".split(), key=str.casefold) ['a', 'Andrew', 'from', 'is', 'string', 'test', 'This'] The value of the *key* parameter should be a function (or other callable) that @@ -97,10 +96,14 @@ The same technique works for objects with named attributes. For example: >>> sorted(student_objects, key=lambda student: student.age) # sort by age [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] -Operator Module Functions -========================= +Objects with named attributes can be made by a regular class as shown +above, or they can be instances of :class:`~dataclasses.dataclass` or +a :term:`named tuple`. -The key-function patterns shown above are very common, so Python provides +Operator Module Functions and Partial Function Evaluation +========================================================= + +The :term:`key function` patterns shown above are very common, so Python provides convenience functions to make accessor functions easier and faster. The :mod:`operator` module has :func:`~operator.itemgetter`, :func:`~operator.attrgetter`, and a :func:`~operator.methodcaller` function. @@ -128,6 +131,24 @@ sort by *grade* then by *age*: >>> sorted(student_objects, key=attrgetter('grade', 'age')) [('john', 'A', 15), ('dave', 'B', 10), ('jane', 'B', 12)] +The :mod:`functools` module provides another helpful tool for making +key-functions. The :func:`~functools.partial` function can reduce the +`arity `_ of a multi-argument +function making it suitable for use as a key-function. + +.. doctest:: + + >>> from functools import partial + >>> from unicodedata import normalize + + >>> names = 'Zoë Åbjørn Núñez Élana Zeke Abe Nubia Eloise'.split() + + >>> sorted(names, key=partial(normalize, 'NFD')) + ['Abe', 'Åbjørn', 'Eloise', 'Élana', 'Nubia', 'Núñez', 'Zeke', 'Zoë'] + + >>> sorted(names, key=partial(normalize, 'NFC')) + ['Abe', 'Eloise', 'Nubia', 'Núñez', 'Zeke', 'Zoë', 'Åbjørn', 'Élana'] + Ascending and Descending ======================== @@ -200,6 +221,8 @@ This idiom is called Decorate-Sort-Undecorate after its three steps: For example, to sort the student data by *grade* using the DSU approach: +.. doctest:: + >>> decorated = [(student.grade, i, student) for i, student in enumerate(student_objects)] >>> decorated.sort() >>> [student for grade, i, student in decorated] # undecorate @@ -282,7 +305,11 @@ Odds and Ends [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] However, note that ``<`` can fall back to using :meth:`~object.__gt__` if - :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__`). + :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__` + for details on the mechanics). To avoid surprises, :pep:`8` + recommends that all six comparison methods be implemented. + The :func:`~functools.total_ordering` decorator is provided to make that + task easier. * Key functions need not depend directly on the objects being sorted. A key function can also access external resources. For instance, if the student grades @@ -295,3 +322,24 @@ Odds and Ends >>> newgrades = {'john': 'F', 'jane':'A', 'dave': 'C'} >>> sorted(students, key=newgrades.__getitem__) ['jane', 'dave', 'john'] + +Partial Sorts +============= + +Some applications require only some of the data to be ordered. The standard +library provides several tools that do less work than a full sort: + +* :func:`min` and :func:`max` return the smallest and largest values, + respectively. These functions make a single pass over the input data and + require almost no auxiliary memory. + +* :func:`heapq.nsmallest` and :func:`heapq.nlargest` return + the *n* smallest and largest values, respectively. These functions + make a single pass over the data keeping only *n* elements in memory + at a time. For values of *n* that are small relative to the number of + inputs, these functions make far fewer comparisons than a full sort. + +* :func:`heapq.heappush` and :func:`heapq.heappop` create and maintain a + partially sorted arrangement of data that keeps the smallest element + at position ``0``. These functions are suitable for implementing + priority queues which are commonly used for task scheduling. diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index c073ea955abaa4..10e2cba50e49b0 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -101,11 +101,11 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: subclass of the ABC. (This class method is called from the :meth:`~class.__subclasscheck__` method of the ABC.) - This method should return ``True``, ``False`` or ``NotImplemented``. If + This method should return ``True``, ``False`` or :data:`NotImplemented`. If it returns ``True``, the *subclass* is considered a subclass of this ABC. If it returns ``False``, the *subclass* is not considered a subclass of this ABC, even if it would normally be one. If it returns - ``NotImplemented``, the subclass check is continued with the usual + :data:`!NotImplemented`, the subclass check is continued with the usual mechanism. .. XXX explain the "usual mechanism" diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 952643a46416d2..eaddd44e2defd7 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -745,7 +745,7 @@ The add_argument() method .. method:: ArgumentParser.add_argument(name or flags..., [action], [nargs], \ [const], [default], [type], [choices], [required], \ - [help], [metavar], [dest]) + [help], [metavar], [dest], [deprecated]) Define how a single command-line argument should be parsed. Each parameter has its own more detailed description below, but in short they are: diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 043badf05ffc12..cdf21db8779fe8 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -279,4 +279,3 @@ Examples:: `NumPy `_ The NumPy package defines another array type. - diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index c943c2f498173e..d10f3f275c0eaf 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -103,20 +103,15 @@ Node classes For example, to create and populate an :class:`ast.UnaryOp` node, you could use :: - node = ast.UnaryOp() - node.op = ast.USub() - node.operand = ast.Constant() - node.operand.value = 5 - node.operand.lineno = 0 - node.operand.col_offset = 0 - node.lineno = 0 - node.col_offset = 0 - - or the more compact :: - node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0), lineno=0, col_offset=0) + If a field that is optional in the grammar is omitted from the constructor, + it defaults to ``None``. If a list field is omitted, it defaults to the empty + list. If any other field is omitted, a :exc:`DeprecationWarning` is raised + and the AST node will not have this field. In Python 3.15, this condition will + raise an error. + .. versionchanged:: 3.8 Class :class:`ast.Constant` is now used for all constants. @@ -140,6 +135,14 @@ Node classes In the meantime, instantiating them will return an instance of a different class. +.. deprecated-removed:: 3.13 3.15 + + Previous versions of Python allowed the creation of AST nodes that were missing + required fields. Similarly, AST node constructors allowed arbitrary keyword + arguments that were set as attributes of the AST node, even if they did not + match any of the fields of the AST node. This behavior is deprecated and will + be removed in Python 3.15. + .. note:: The descriptions of the specific node classes displayed here were initially adapted from the fantastic `Green Tree @@ -2180,14 +2183,17 @@ and classes for traversing abstract syntax trees: modified to correspond to :pep:`484` "signature type comments", e.g. ``(str, int) -> List[str]``. - Also, setting ``feature_version`` to a tuple ``(major, minor)`` - will attempt to parse using that Python version's grammar. - Currently ``major`` must equal to ``3``. For example, setting - ``feature_version=(3, 4)`` will allow the use of ``async`` and - ``await`` as variable names. The lowest supported version is - ``(3, 7)``; the highest is ``sys.version_info[0:2]``. - - If source contains a null character ('\0'), :exc:`ValueError` is raised. + Setting ``feature_version`` to a tuple ``(major, minor)`` will result in + a "best-effort" attempt to parse using that Python version's grammar. + For example, setting ``feature_version=(3, 9)`` will attempt to disallow + parsing of :keyword:`match` statements. + Currently ``major`` must equal to ``3``. The lowest supported version is + ``(3, 7)`` (and this may increase in future Python versions); + the highest is ``sys.version_info[0:2]``. "Best-effort" attempt means there + is no guarantee that the parse (or success of the parse) is the same as + when run on the Python version corresponding to ``feature_version``. + + If source contains a null character (``\0``), :exc:`ValueError` is raised. .. warning:: Note that successfully parsing source code into an AST object doesn't diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index ecd8cdc709af7d..7c08d65f26bc27 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -362,6 +362,11 @@ Datagram Transports This method does not block; it buffers the data and arranges for it to be sent out asynchronously. + .. versionchanged:: 3.13 + This method can be called with an empty bytes object to send a + zero-length datagram. The buffer size calculation used for flow + control is also updated to account for the datagram header. + .. method:: DatagramTransport.abort() Close the transport immediately, without waiting for pending diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 3427da1b43caef..68b1dff20213e1 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -347,7 +347,7 @@ StreamWriter be resumed. When there is nothing to wait for, the :meth:`drain` returns immediately. - .. coroutinemethod:: start_tls(sslcontext, \*, server_hostname=None, \ + .. coroutinemethod:: start_tls(sslcontext, *, server_hostname=None, \ ssl_handshake_timeout=None, ssl_shutdown_timeout=None) Upgrade an existing stream-based connection to TLS. diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index d5b6af8c1928ef..e596893358f3fb 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -244,6 +244,24 @@ The modern interface provides: .. versionadded:: 3.4 +.. function:: z85encode(s) + + Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) + and return the encoded :class:`bytes`. See `Z85 specification + `_ for more information. + + .. versionadded:: 3.13 + + +.. function:: z85decode(s) + + Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and + return the decoded :class:`bytes`. See `Z85 specification + `_ for more information. + + .. versionadded:: 3.13 + + The legacy interface: .. function:: decode(input, output) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 4617624686b1f3..a757f19b99448c 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1132,7 +1132,8 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp875 | | Greek | +-----------------+--------------------------------+--------------------------------+ -| cp932 | 932, ms932, mskanji, ms-kanji | Japanese | +| cp932 | 932, ms932, mskanji, ms-kanji, | Japanese | +| | windows-31j | | +-----------------+--------------------------------+--------------------------------+ | cp949 | 949, ms949, uhc | Korean | +-----------------+--------------------------------+--------------------------------+ @@ -1540,13 +1541,13 @@ This module implements the ANSI codepage (CP_ACP). .. availability:: Windows. -.. versionchanged:: 3.3 - Support any error handler. - .. versionchanged:: 3.2 Before 3.2, the *errors* argument was ignored; ``'replace'`` was always used to encode, and ``'ignore'`` to decode. +.. versionchanged:: 3.3 + Support any error handler. + :mod:`encodings.utf_8_sig` --- UTF-8 codec with BOM signature ------------------------------------------------------------- diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index 401dc9a320c5e0..93a7244f87de6b 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -33,27 +33,27 @@ A small number of constants live in the built-in namespace. They are: the other type; may be returned by the in-place binary special methods (e.g. :meth:`~object.__imul__`, :meth:`~object.__iand__`, etc.) for the same purpose. It should not be evaluated in a boolean context. - ``NotImplemented`` is the sole instance of the :data:`types.NotImplementedType` type. + :data:`!NotImplemented` is the sole instance of the :data:`types.NotImplementedType` type. .. note:: - When a binary (or in-place) method returns ``NotImplemented`` the + When a binary (or in-place) method returns :data:`!NotImplemented` the interpreter will try the reflected operation on the other type (or some other fallback, depending on the operator). If all attempts return - ``NotImplemented``, the interpreter will raise an appropriate exception. - Incorrectly returning ``NotImplemented`` will result in a misleading - error message or the ``NotImplemented`` value being returned to Python code. + :data:`!NotImplemented`, the interpreter will raise an appropriate exception. + Incorrectly returning :data:`!NotImplemented` will result in a misleading + error message or the :data:`!NotImplemented` value being returned to Python code. See :ref:`implementing-the-arithmetic-operations` for examples. .. note:: - ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, + ``NotImplementedError`` and :data:`!NotImplemented` are not interchangeable, even though they have similar names and purposes. See :exc:`NotImplementedError` for details on when to use it. .. versionchanged:: 3.9 - Evaluating ``NotImplemented`` in a boolean context is deprecated. While + Evaluating :data:`!NotImplemented` in a boolean context is deprecated. While it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. It will raise a :exc:`TypeError` in a future version of Python. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 73779547b35a1f..eed18201e3ede0 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -200,7 +200,7 @@ calls). Python objects that can directly be used as parameters in these function calls. ``None`` is passed as a C ``NULL`` pointer, bytes objects and strings are passed as pointer to the memory block that contains their data (:c:expr:`char *` or -:c:expr:`wchar_t *`). Python integers are passed as the platforms default C +:c:expr:`wchar_t *`). Python integers are passed as the platform's default C :c:expr:`int` type, their value is masked to fit into the C type. Before we move on calling functions with other parameter types, we have to learn @@ -1117,10 +1117,6 @@ api:: >>> print(hex(version.value)) 0x30c00a0 -If the interpreter would have been started with :option:`-O`, the sample would -have printed ``c_long(1)``, or ``c_long(2)`` if :option:`-OO` would have been -specified. - An extended example which also demonstrates the use of pointers accesses the :c:data:`PyImport_FrozenModules` pointer exported by Python. @@ -1445,7 +1441,7 @@ function exported by these libraries, and reacquired afterwards. All these classes can be instantiated by calling them with at least one argument, the pathname of the shared library. If you have an existing handle to an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platforms :c:func:`!dlopen` or +parameter, otherwise the underlying platform's :c:func:`!dlopen` or :c:func:`!LoadLibrary` function is used to load the library into the process, and to get a handle to it. @@ -1456,7 +1452,7 @@ configurable. The *use_errno* parameter, when set to true, enables a ctypes mechanism that allows accessing the system :data:`errno` error number in a safe way. -:mod:`ctypes` maintains a thread-local copy of the systems :data:`errno` +:mod:`ctypes` maintains a thread-local copy of the system's :data:`errno` variable; if you call foreign functions created with ``use_errno=True`` then the :data:`errno` value before the function call is swapped with the ctypes private copy, the same happens immediately after the function call. diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 4602132f37f733..1905c9e1ca755d 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1209,6 +1209,9 @@ Supported operations: Naive and aware :class:`!datetime` objects are never equal. + If both comparands are aware, and have the same :attr:`!tzinfo` attribute, + the :attr:`!tzinfo` and :attr:`~.datetime.fold` attributes are ignored and + the base datetimes are compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo` attributes, the comparison acts as comparands were first converted to UTC datetimes except that the implementation never overflows. @@ -1222,6 +1225,9 @@ Supported operations: Order comparison between naive and aware :class:`.datetime` objects raises :exc:`TypeError`. + If both comparands are aware, and have the same :attr:`!tzinfo` attribute, + the :attr:`!tzinfo` and :attr:`~.datetime.fold` attributes are ignored and + the base datetimes are compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo` attributes, the comparison acts as comparands were first converted to UTC datetimes except that the implementation never overflows. @@ -1778,8 +1784,8 @@ Naive and aware :class:`!time` objects are never equal. Order comparison between naive and aware :class:`!time` objects raises :exc:`TypeError`. -If both comparands are aware, and have -the same :attr:`~.time.tzinfo` attribute, the common :attr:`!tzinfo` attribute is +If both comparands are aware, and have the same :attr:`~.time.tzinfo` +attribute, the :attr:`!tzinfo` and :attr:`!fold` attributes are ignored and the base times are compared. If both comparands are aware and have different :attr:`!tzinfo` attributes, the comparands are first adjusted by subtracting their UTC offsets (obtained from ``self.utcoffset()``). diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 30d80ce8d488cc..49bf40aa5a2869 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -170,7 +170,7 @@ Data Types final *enum*, as well as creating the enum members, properly handling duplicates, providing iteration over the enum class, etc. - .. method:: EnumType.__call__(cls, value, names=None, \*, module=None, qualname=None, type=None, start=1, boundary=None) + .. method:: EnumType.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) This method is called in two different ways: @@ -350,7 +350,7 @@ Data Types >>> PowersOfThree.SECOND.value 9 - .. method:: Enum.__init__(self, \*args, \**kwds) + .. method:: Enum.__init__(self, *args, **kwds) By default, does nothing. If multiple values are given in the member assignment, those values become separate arguments to ``__init__``; e.g. @@ -361,7 +361,7 @@ Data Types ``Weekday.__init__()`` would be called as ``Weekday.__init__(self, 1, 'Mon')`` - .. method:: Enum.__init_subclass__(cls, \**kwds) + .. method:: Enum.__init_subclass__(cls, **kwds) A *classmethod* that is used to further configure subsequent subclasses. By default, does nothing. @@ -388,7 +388,7 @@ Data Types >>> Build('deBUG') - .. method:: Enum.__new__(cls, \*args, \**kwds) + .. method:: Enum.__new__(cls, *args, **kwds) By default, doesn't exist. If specified, either in the enum class definition or in a mixin class (such as ``int``), all values given @@ -400,6 +400,9 @@ Data Types results in the call ``int('1a', 16)`` and a value of ``17`` for the member. + ..note:: When writing a custom ``__new__``, do not use ``super().__new__`` -- + call the appropriate ``__new__`` instead. + .. method:: Enum.__repr__(self) Returns the string used for *repr()* calls. By default, returns the diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 88417b40e4aa7f..7879fb015bddfa 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -335,9 +335,9 @@ The following exceptions are the exceptions that are usually raised. .. note:: - ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, + ``NotImplementedError`` and :data:`NotImplemented` are not interchangeable, even though they have similar names and purposes. See - :data:`NotImplemented` for details on when to use it. + :data:`!NotImplemented` for details on when to use it. .. exception:: OSError([arg]) OSError(errno, strerror[, filename[, winerror[, filename2]]]) diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 13ad2dd7da5090..b93d6ac7aab956 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -13,10 +13,10 @@ ---------------- -This module performs file control and I/O control on file descriptors. It is an -interface to the :c:func:`fcntl` and :c:func:`ioctl` Unix routines. For a -complete description of these calls, see :manpage:`fcntl(2)` and -:manpage:`ioctl(2)` Unix manual pages. +This module performs file and I/O control on file descriptors. It is an +interface to the :c:func:`fcntl` and :c:func:`ioctl` Unix routines. +See the :manpage:`fcntl(2)` and :manpage:`ioctl(2)` Unix manual pages +for full details. .. availability:: Unix, not Emscripten, not WASI. @@ -101,7 +101,7 @@ The module defines the following functions: most likely to result in a segmentation violation or a more subtle data corruption. - If the :c:func:`fcntl` fails, an :exc:`OSError` is raised. + If the :c:func:`fcntl` call fails, an :exc:`OSError` is raised. .. audit-event:: fcntl.fcntl fd,cmd,arg fcntl.fcntl @@ -139,7 +139,7 @@ The module defines the following functions: buffer 1024 bytes long which is then passed to :func:`ioctl` and copied back into the supplied buffer. - If the :c:func:`ioctl` fails, an :exc:`OSError` exception is raised. + If the :c:func:`ioctl` call fails, an :exc:`OSError` exception is raised. An example:: @@ -164,7 +164,7 @@ The module defines the following functions: :manpage:`flock(2)` for details. (On some systems, this function is emulated using :c:func:`fcntl`.) - If the :c:func:`flock` fails, an :exc:`OSError` exception is raised. + If the :c:func:`flock` call fails, an :exc:`OSError` exception is raised. .. audit-event:: fcntl.flock fd,operation fcntl.flock @@ -176,17 +176,28 @@ The module defines the following functions: method are accepted as well) of the file to lock or unlock, and *cmd* is one of the following values: - * :const:`LOCK_UN` -- unlock - * :const:`LOCK_SH` -- acquire a shared lock - * :const:`LOCK_EX` -- acquire an exclusive lock + .. data:: LOCK_UN - When *cmd* is :const:`LOCK_SH` or :const:`LOCK_EX`, it can also be - bitwise ORed with :const:`LOCK_NB` to avoid blocking on lock acquisition. - If :const:`LOCK_NB` is used and the lock cannot be acquired, an + Release an existing lock. + + .. data:: LOCK_SH + + Acquire a shared lock. + + .. data:: LOCK_EX + + Acquire an exclusive lock. + + .. data:: LOCK_NB + + Bitwise OR with any of the other three ``LOCK_*`` constants to make + the request non-blocking. + + If :const:`!LOCK_NB` is used and the lock cannot be acquired, an :exc:`OSError` will be raised and the exception will have an *errno* - attribute set to :const:`EACCES` or :const:`EAGAIN` (depending on the + attribute set to :const:`~errno.EACCES` or :const:`~errno.EAGAIN` (depending on the operating system; for portability, check for both values). On at least some - systems, :const:`LOCK_EX` can only be used if the file descriptor refers to a + systems, :const:`!LOCK_EX` can only be used if the file descriptor refers to a file opened for writing. *len* is the number of bytes to lock, *start* is the byte offset at diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index dfe4b7c59fd578..42d20b9c201783 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -70,7 +70,7 @@ The :mod:`filecmp` module defines the following functions: The :class:`dircmp` class ------------------------- -.. class:: dircmp(a, b, ignore=None, hide=None) +.. class:: dircmp(a, b, ignore=None, hide=None, shallow=True) Construct a new directory comparison object, to compare the directories *a* and *b*. *ignore* is a list of names to ignore, and defaults to @@ -78,7 +78,12 @@ The :class:`dircmp` class defaults to ``[os.curdir, os.pardir]``. The :class:`dircmp` class compares files by doing *shallow* comparisons - as described for :func:`filecmp.cmp`. + as described for :func:`filecmp.cmp` by default using the *shallow* + parameter. + + .. versionchanged:: 3.13 + + Added the *shallow* parameter. The :class:`dircmp` class provides the following methods: diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 9abf7974d1936d..8d1aae018ada12 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -232,8 +232,8 @@ FTP objects .. method:: FTP.voidcmd(cmd) Send a simple command string to the server and handle the response. Return - nothing if a response code corresponding to success (codes in the range - 200--299) is received. Raise :exc:`error_reply` otherwise. + the response string if the response code corresponds to success (codes in + the range 200--299). Raise :exc:`error_reply` otherwise. .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.voidcmd diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 27fce5aa0f1a63..e598ef423de497 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -526,9 +526,20 @@ are always available. They are listed here in alphabetical order. .. function:: eval(expression, globals=None, locals=None) - The arguments are a string and optional globals and locals. If provided, - *globals* must be a dictionary. If provided, *locals* can be any mapping - object. + :param expression: + A Python expression. + :type expression: :class:`str` | :ref:`code object ` + + :param globals: + The global namespace (default: ``None``). + :type globals: :class:`dict` | ``None`` + + :param locals: + The local namespace (default: ``None``). + :type locals: :term:`mapping` | ``None`` + + :returns: The result of the evaluated expression. + :raises: Syntax errors are reported as exceptions. The *expression* argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the *globals* and *locals* @@ -545,8 +556,7 @@ are always available. They are listed here in alphabetical order. :term:`nested scopes ` (non-locals) in the enclosing environment. - The return value is the result of - the evaluated expression. Syntax errors are reported as exceptions. Example: + Example: >>> x = 1 >>> eval('x+1') @@ -1569,6 +1579,16 @@ are always available. They are listed here in alphabetical order. If :func:`sys.displayhook` is not accessible, this function will raise :exc:`RuntimeError`. + This class has a custom representation that can be evaluated:: + + class Person: + def __init__(self, name, age): + self.name = name + self.age = age + + def __repr__(self): + return f"Person('{self.name}', {self.age})" + .. function:: reversed(seq) diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 761dd84edee299..0726be4590e399 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -121,7 +121,7 @@ More condensed: Constructors ------------ -.. function:: new(name[, data], \*, usedforsecurity=True) +.. function:: new(name[, data], *, usedforsecurity=True) Is a generic constructor that takes the string *name* of the desired algorithm as its first parameter. It also exists to allow access to the diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 12a6d768437ea5..2fe188be641c2d 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -649,6 +649,11 @@ internal consistency, so you should know what you're doing if you do that. :const:`None`. +.. attribute:: Cookie.domain + + Cookie domain (a string). + + .. attribute:: Cookie.path Cookie path (a string, eg. ``'/acme/rocket_launchers'``). diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index bc59d3d17912fd..886e359bd8cd62 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -520,6 +520,12 @@ the ``--cgi`` option:: :mod:`http.server` command line ``--cgi`` support is being removed because :class:`CGIHTTPRequestHandler` is being removed. +.. warning:: + + :class:`CGIHTTPRequestHandler` and the ``--cgi`` command line option + are not intended for use by untrusted clients and may be vulnerable + to exploitation. Always use within a secure environment. + .. _http.server-security: Security Considerations diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 2402bc5cd3ee2c..d92bb2f8e5cf83 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -265,7 +265,7 @@ ABC hierarchy:: when invalidating the caches of all finders on :data:`sys.meta_path`. .. versionchanged:: 3.4 - Returns ``None`` when called instead of ``NotImplemented``. + Returns ``None`` when called instead of :data:`NotImplemented`. .. class:: PathEntryFinder diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 8a74cadb98a0db..ed8d705da3b0b5 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -665,9 +665,6 @@ function. Accepts a wide range of Python callables, from plain functions and classes to :func:`functools.partial` objects. - If the passed object has a ``__signature__`` attribute, this function - returns it without further computations. - For objects defined in modules using stringized annotations (``from __future__ import annotations``), :func:`signature` will attempt to automatically un-stringize the annotations using @@ -702,6 +699,13 @@ function. Python. For example, in CPython, some built-in functions defined in C provide no metadata about their arguments. + .. impl-detail:: + + If the passed object has a :attr:`!__signature__` attribute, + we may use it to create the signature. + The exact semantics are an implementation detail and are subject to + unannounced changes. Consult the source code for current semantics. + .. class:: Signature(parameters=None, *, return_annotation=Signature.empty) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 338a5f9615aae3..2ee39fd9104df8 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -56,13 +56,13 @@ Iterator Arguments Results :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` :func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` -:func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` -:func:`filterfalse` pred, seq elements of seq where pred(elem) is false ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` +:func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` +:func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` :func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) --> C D E F G`` :func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') --> AB BC CD DE EF FG`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000`` -:func:`takewhile` pred, seq seq[0], seq[1], until pred fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4`` +:func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4`` :func:`tee` it, n it1, it2, ... itn splits one iterator into n :func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D-`` ============================ ============================ ================================================= ============================================================= @@ -90,7 +90,7 @@ Examples Results .. _itertools-functions: -Itertool functions +Itertool Functions ------------------ The following module functions all construct and return iterators. Some provide @@ -688,6 +688,14 @@ loops that truncate the stream. else: break + Note, the element that first fails the predicate condition is + consumed from the input iterator and there is no way to access it. + This could be an issue if an application wants to further consume the + input iterator after takewhile has been run to exhaustion. To work + around this problem, consider using `more-iterools before_and_after() + `_ + instead. + .. function:: tee(iterable, n=2) @@ -770,7 +778,7 @@ The primary purpose of the itertools recipes is educational. The recipes show various ways of thinking about individual tools — for example, that ``chain.from_iterable`` is related to the concept of flattening. The recipes also give ideas about ways that the tools can be combined — for example, how -``compress()`` and ``range()`` can work together. The recipes also show patterns +``starmap()`` and ``repeat()`` can work together. The recipes also show patterns for using itertools with the :mod:`operator` and :mod:`collections` modules as well as with the built-in itertools such as ``map()``, ``filter()``, ``reversed()``, and ``enumerate()``. @@ -851,27 +859,19 @@ which incur interpreter overhead. "Returns the nth item or a default value." return next(islice(iterable, n, None), default) - def quantify(iterable, pred=bool): + def quantify(iterable, predicate=bool): "Given a predicate that returns True or False, count the True results." - return sum(map(pred, iterable)) - - def all_equal(iterable): - "Returns True if all the elements are equal to each other." - g = groupby(iterable) - return next(g, True) and not next(g, False) - - def first_true(iterable, default=False, pred=None): - """Returns the first true value in the iterable. - - If no true value is found, returns *default* - - If *pred* is not None, returns the first item - for which pred(item) is true. + return sum(map(predicate, iterable)) - """ + def first_true(iterable, default=False, predicate=None): + "Returns the first true value or the *default* if there is no true value." # first_true([a,b,c], x) --> a or b or c or x # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x - return next(filter(pred, iterable), default) + return next(filter(predicate, iterable), default) + + def all_equal(iterable, key=None): + "Returns True if all the elements are equal to each other." + return len(take(2, groupby(iterable, key))) <= 1 def unique_everseen(iterable, key=None): "List unique elements, preserving order. Remember all elements ever seen." @@ -957,14 +957,14 @@ which incur interpreter overhead. num_active -= 1 nexts = cycle(islice(nexts, num_active)) - def partition(pred, iterable): + def partition(predicate, iterable): """Partition entries into false entries and true entries. - If *pred* is slow, consider wrapping it with functools.lru_cache(). + If *predicate* is slow, consider wrapping it with functools.lru_cache(). """ # partition(is_odd, range(10)) --> 0 2 4 6 8 and 1 3 5 7 9 t1, t2 = tee(iterable) - return filterfalse(pred, t1), filter(pred, t2) + return filterfalse(predicate, t1), filter(predicate, t2) def subslices(seq): "Return all contiguous non-empty subslices of a sequence." @@ -976,61 +976,16 @@ which incur interpreter overhead. """ Call a function repeatedly until an exception is raised. Converts a call-until-exception interface to an iterator interface. - Like builtins.iter(func, sentinel) but uses an exception instead - of a sentinel to end the loop. - - Priority queue iterator: - iter_except(functools.partial(heappop, h), IndexError) - - Non-blocking dictionary iterator: - iter_except(d.popitem, KeyError) - - Non-blocking deque iterator: - iter_except(d.popleft, IndexError) - - Non-blocking iterator over a producer Queue: - iter_except(q.get_nowait, Queue.Empty) - - Non-blocking set iterator: - iter_except(s.pop, KeyError) - """ + # iter_except(d.popitem, KeyError) --> non-blocking dictionary iterator try: if first is not None: - # For database APIs needing an initial call to db.first() yield first() while True: yield func() except exception: pass - def before_and_after(predicate, it): - """ Variant of takewhile() that allows complete - access to the remainder of the iterator. - - >>> it = iter('ABCdEfGhI') - >>> all_upper, remainder = before_and_after(str.isupper, it) - >>> ''.join(all_upper) - 'ABC' - >>> ''.join(remainder) # takewhile() would lose the 'd' - 'dEfGhI' - - Note that the true iterator must be fully consumed - before the remainder iterator can generate valid results. - """ - it = iter(it) - transition = [] - - def true_iterator(): - for elem in it: - if predicate(elem): - yield elem - else: - transition.append(elem) - return - - return true_iterator(), chain(transition, it) - The following recipes have a more mathematical flavor: @@ -1243,6 +1198,8 @@ The following recipes have a more mathematical flavor: >>> [all_equal(s) for s in ('', 'A', 'AAAA', 'AAAB', 'AAABA')] [True, True, True, False, False] + >>> [all_equal(s, key=str.casefold) for s in ('', 'A', 'AaAa', 'AAAB', 'AAABA')] + [True, True, True, False, False] >>> quantify(range(99), lambda x: x%2==0) 50 @@ -1250,7 +1207,7 @@ The following recipes have a more mathematical flavor: >>> quantify([True, False, False, True, True]) 3 - >>> quantify(range(12), pred=lambda x: x%2==1) + >>> quantify(range(12), predicate=lambda x: x%2==1) 6 >>> a = [[1, 2, 3], [4, 5, 6]] @@ -1543,13 +1500,6 @@ The following recipes have a more mathematical flavor: >>> list(odds) [1, 3, 5, 7, 9] - >>> it = iter('ABCdEfGhI') - >>> all_upper, remainder = before_and_after(str.isupper, it) - >>> ''.join(all_upper) - 'ABC' - >>> ''.join(remainder) - 'dEfGhI' - >>> list(subslices('ABCD')) ['A', 'AB', 'ABC', 'ABCD', 'B', 'BC', 'BCD', 'C', 'CD', 'D'] @@ -1640,6 +1590,32 @@ The following recipes have a more mathematical flavor: result.append(pool[-1-n]) return tuple(result) + def before_and_after(predicate, it): + """ Variant of takewhile() that allows complete + access to the remainder of the iterator. + + >>> it = iter('ABCdEfGhI') + >>> all_upper, remainder = before_and_after(str.isupper, it) + >>> ''.join(all_upper) + 'ABC' + >>> ''.join(remainder) # takewhile() would lose the 'd' + 'dEfGhI' + + Note that the true iterator must be fully consumed + before the remainder iterator can generate valid results. + """ + it = iter(it) + transition = [] + + def true_iterator(): + for elem in it: + if predicate(elem): + yield elem + else: + transition.append(elem) + return + + return true_iterator(), chain(transition, it) .. doctest:: :hide: @@ -1669,3 +1645,10 @@ The following recipes have a more mathematical flavor: >>> combos = list(combinations(iterable, r)) >>> all(nth_combination(iterable, r, i) == comb for i, comb in enumerate(combos)) True + + >>> it = iter('ABCdEfGhI') + >>> all_upper, remainder = before_and_after(str.isupper, it) + >>> ''.join(all_upper) + 'ABC' + >>> ''.join(remainder) + 'dEfGhI' diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 0ce4b697145cb3..c82ff9dc325b4c 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -106,7 +106,7 @@ Extending :class:`JSONEncoder`:: ... if isinstance(obj, complex): ... return [obj.real, obj.imag] ... # Let the base class default method raise the TypeError - ... return json.JSONEncoder.default(self, obj) + ... return super().default(obj) ... >>> json.dumps(2 + 1j, cls=ComplexEncoder) '[2.0, 1.0]' @@ -504,7 +504,7 @@ Encoders and Decoders else: return list(iterable) # Let the base class default method raise the TypeError - return json.JSONEncoder.default(self, o) + return super().default(o) .. method:: encode(o) diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 39eb41ce1f1670..e38b7435498507 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -77,6 +77,27 @@ is the module's name in the Python package namespace. .. class:: Logger + .. attribute:: Logger.name + + This is the logger's name, and is the value that was passed to :func:`getLogger` + to obtain the logger. + + .. note:: This attribute should be treated as read-only. + + .. attribute:: Logger.level + + The threshold of this logger, as set by the :meth:`setLevel` method. + + .. note:: Do not set this attribute directly - always use :meth:`setLevel`, + which has checks for the level passed to it. + + .. attribute:: Logger.parent + + The parent logger of this logger. It may change based on later instantiation + of loggers which are higher up in the namespace hierarchy. + + .. note:: This value should be treated as read-only. + .. attribute:: Logger.propagate If this attribute evaluates to true, events logged to this logger will be @@ -108,6 +129,21 @@ is the module's name in the Python package namespace. scenario is to attach handlers only to the root logger, and to let propagation take care of the rest. + .. attribute:: Logger.handlers + + The list of handlers directly attached to this logger instance. + + .. note:: This attribute should be treated as read-only; it is normally changed via + the :meth:`addHandler` and :meth:`removeHandler` methods, which use locks to ensure + thread-safe operation. + + .. attribute:: Logger.disabled + + This attribute disables handling of any events. It is set to ``False`` in the + initializer, and only changed by logging configuration code. + + .. note:: This attribute should be treated as read-only. + .. method:: Logger.setLevel(level) Sets the threshold for this logger to *level*. Logging messages which are less diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 9caf7230eed0aa..3c850317f60858 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -239,11 +239,11 @@ Number-theoretic and representation functions See also :func:`math.ulp`. + .. versionadded:: 3.9 + .. versionchanged:: 3.12 Added the *steps* argument. - .. versionadded:: 3.9 - .. function:: perm(n, k=None) Return the number of ways to choose *k* items from *n* items @@ -680,11 +680,11 @@ Constants >>> math.isnan(float('nan')) True + .. versionadded:: 3.5 + .. versionchanged:: 3.11 It is now always available. - .. versionadded:: 3.5 - .. impl-detail:: diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index d570d4eb0dae78..0b87de4c61e6aa 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -150,18 +150,18 @@ to start a process. These *start methods* are over Unix pipes such as Linux. -.. versionchanged:: 3.8 - - On macOS, the *spawn* start method is now the default. The *fork* start - method should be considered unsafe as it can lead to crashes of the - subprocess as macOS system libraries may start threads. See :issue:`33725`. - .. versionchanged:: 3.4 *spawn* added on all POSIX platforms, and *forkserver* added for some POSIX platforms. Child processes no longer inherit all of the parents inheritable handles on Windows. +.. versionchanged:: 3.8 + + On macOS, the *spawn* start method is now the default. The *fork* start + method should be considered unsafe as it can lead to crashes of the + subprocess as macOS system libraries may start threads. See :issue:`33725`. + On POSIX using the *spawn* or *forkserver* start methods will also start a *resource tracker* process which tracks the unlinked named system resources (such as named semaphores or @@ -519,7 +519,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the to the process. .. versionchanged:: 3.3 - Added the *daemon* argument. + Added the *daemon* parameter. .. method:: run() @@ -1084,13 +1084,13 @@ Miscellaneous The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` or ``None``. See :ref:`multiprocessing-start-methods`. -.. versionchanged:: 3.8 + .. versionadded:: 3.4 - On macOS, the *spawn* start method is now the default. The *fork* start - method should be considered unsafe as it can lead to crashes of the - subprocess. See :issue:`33725`. + .. versionchanged:: 3.8 - .. versionadded:: 3.4 + On macOS, the *spawn* start method is now the default. The *fork* start + method should be considered unsafe as it can lead to crashes of the + subprocess. See :issue:`33725`. .. function:: set_executable(executable) @@ -1245,8 +1245,7 @@ Connection objects are usually created using Connection objects themselves can now be transferred between processes using :meth:`Connection.send` and :meth:`Connection.recv`. - .. versionadded:: 3.3 - Connection objects now support the context management protocol -- see + Connection objects also now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the connection object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. @@ -2250,11 +2249,11 @@ with the :class:`Pool` class. as CPython does not assure that the finalizer of the pool will be called (see :meth:`object.__del__` for more information). - .. versionadded:: 3.2 - *maxtasksperchild* + .. versionchanged:: 3.2 + Added the *maxtasksperchild* parameter. - .. versionadded:: 3.4 - *context* + .. versionchanged:: 3.4 + Added the *context* parameter. .. versionchanged:: 3.13 *processes* uses :func:`os.process_cpu_count` by default, instead of @@ -2380,7 +2379,7 @@ with the :class:`Pool` class. Wait for the worker processes to exit. One must call :meth:`close` or :meth:`terminate` before using :meth:`join`. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Pool objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the pool object, and :meth:`~contextmanager.__exit__` calls :meth:`terminate`. @@ -2549,7 +2548,7 @@ multiple connections at the same time. The address from which the last accepted connection came. If this is unavailable then it is ``None``. - .. versionadded:: 3.3 + .. versionchanged:: 3.3 Listener objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the listener object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. @@ -2979,7 +2978,7 @@ Beware of replacing :data:`sys.stdin` with a "file like object" The *spawn* and *forkserver* start methods ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are a few extra restriction which don't apply to the *fork* +There are a few extra restrictions which don't apply to the *fork* start method. More picklability diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 17d1a275f04c9b..306bdd94aaca13 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -166,7 +166,7 @@ Complex``. I'll consider ``a + b``: 2. If ``A`` falls back to the boilerplate code, and it were to return a value from :meth:`~object.__add__`, we'd miss the possibility that ``B`` defines a more intelligent :meth:`~object.__radd__`, so the - boilerplate should return :const:`NotImplemented` from + boilerplate should return :data:`NotImplemented` from :meth:`!__add__`. (Or ``A`` may not implement :meth:`!__add__` at all.) 3. Then ``B``'s :meth:`~object.__radd__` gets a chance. If it accepts diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 34bc76b231de92..3ee2b7db1e511b 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -79,7 +79,7 @@ the :mod:`glob` module.) .. function:: commonpath(paths) - Return the longest common sub-path of each pathname in the sequence + Return the longest common sub-path of each pathname in the iterable *paths*. Raise :exc:`ValueError` if *paths* contain both absolute and relative pathnames, the *paths* are on the different drives or if *paths* is empty. Unlike :func:`commonprefix`, this returns a @@ -92,6 +92,9 @@ the :mod:`glob` module.) .. versionchanged:: 3.6 Accepts a sequence of :term:`path-like objects `. + .. versionchanged:: 3.13 + Any iterable can now be passed, rather than just sequences. + .. function:: commonprefix(list) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index f94b6fb3805684..4b461a5d4a2949 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -572,6 +572,9 @@ Pure paths provide the following methods and properties: >>> PurePath('/a/b/c.py').full_match('**/*.py') True + .. seealso:: + :ref:`pathlib-pattern-language` documentation. + As with other methods, case-sensitivity follows platform defaults:: >>> PurePosixPath('b.py').full_match('*.PY') @@ -991,11 +994,6 @@ call fails (for example because the path doesn't exist). [PosixPath('pathlib.py'), PosixPath('setup.py'), PosixPath('test_pathlib.py')] >>> sorted(Path('.').glob('*/*.py')) [PosixPath('docs/conf.py')] - - Patterns are the same as for :mod:`fnmatch`, with the addition of "``**``" - which means "this directory and all subdirectories, recursively". In other - words, it enables recursive globbing:: - >>> sorted(Path('.').glob('**/*.py')) [PosixPath('build/lib/pathlib.py'), PosixPath('docs/conf.py'), @@ -1003,13 +1001,8 @@ call fails (for example because the path doesn't exist). PosixPath('setup.py'), PosixPath('test_pathlib.py')] - .. note:: - Using the "``**``" pattern in large directory trees may consume - an inordinate amount of time. - - .. tip:: - Set *follow_symlinks* to ``True`` or ``False`` to improve performance - of recursive globbing. + .. seealso:: + :ref:`pathlib-pattern-language` documentation. This method calls :meth:`Path.is_dir` on the top-level directory and propagates any :exc:`OSError` exception that is raised. Subsequent @@ -1025,11 +1018,11 @@ call fails (for example because the path doesn't exist). wildcards. Set *follow_symlinks* to ``True`` to always follow symlinks, or ``False`` to treat all symlinks as files. - .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob + .. tip:: + Set *follow_symlinks* to ``True`` or ``False`` to improve performance + of recursive globbing. - .. versionchanged:: 3.11 - Return only directories if *pattern* ends with a pathname components - separator (:data:`~os.sep` or :data:`~os.altsep`). + .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob .. versionchanged:: 3.12 The *case_sensitive* parameter was added. @@ -1038,12 +1031,29 @@ call fails (for example because the path doesn't exist). The *follow_symlinks* parameter was added. .. versionchanged:: 3.13 - Return files and directories if *pattern* ends with "``**``". In - previous versions, only directories were returned. + The *pattern* parameter accepts a :term:`path-like object`. + + +.. method:: Path.rglob(pattern, *, case_sensitive=None, follow_symlinks=None) + + Glob the given relative *pattern* recursively. This is like calling + :func:`Path.glob` with "``**/``" added in front of the *pattern*. + + .. seealso:: + :ref:`pathlib-pattern-language` and :meth:`Path.glob` documentation. + + .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob + + .. versionchanged:: 3.12 + The *case_sensitive* parameter was added. + + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. .. versionchanged:: 3.13 The *pattern* parameter accepts a :term:`path-like object`. + .. method:: Path.group(*, follow_symlinks=True) Return the name of the group owning the file. :exc:`KeyError` is raised @@ -1471,44 +1481,6 @@ call fails (for example because the path doesn't exist). strict mode, and no exception is raised in non-strict mode. In previous versions, :exc:`RuntimeError` is raised no matter the value of *strict*. -.. method:: Path.rglob(pattern, *, case_sensitive=None, follow_symlinks=None) - - Glob the given relative *pattern* recursively. This is like calling - :func:`Path.glob` with "``**/``" added in front of the *pattern*, where - *patterns* are the same as for :mod:`fnmatch`:: - - >>> sorted(Path().rglob("*.py")) - [PosixPath('build/lib/pathlib.py'), - PosixPath('docs/conf.py'), - PosixPath('pathlib.py'), - PosixPath('setup.py'), - PosixPath('test_pathlib.py')] - - By default, or when the *case_sensitive* keyword-only argument is set to - ``None``, this method matches paths using platform-specific casing rules: - typically, case-sensitive on POSIX, and case-insensitive on Windows. - Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. - - By default, or when the *follow_symlinks* keyword-only argument is set to - ``None``, this method follows symlinks except when expanding "``**``" - wildcards. Set *follow_symlinks* to ``True`` to always follow symlinks, or - ``False`` to treat all symlinks as files. - - .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob - - .. versionchanged:: 3.11 - Return only directories if *pattern* ends with a pathname components - separator (:data:`~os.sep` or :data:`~os.altsep`). - - .. versionchanged:: 3.12 - The *case_sensitive* parameter was added. - - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. - - .. versionchanged:: 3.13 - The *pattern* parameter accepts a :term:`path-like object`. - .. method:: Path.rmdir() Remove this directory. The directory must be empty. @@ -1639,6 +1611,81 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.10 The *newline* parameter was added. + +.. _pathlib-pattern-language: + +Pattern language +---------------- + +The following wildcards are supported in patterns for +:meth:`~PurePath.full_match`, :meth:`~Path.glob` and :meth:`~Path.rglob`: + +``**`` (entire segment) + Matches any number of file or directory segments, including zero. +``*`` (entire segment) + Matches one file or directory segment. +``*`` (part of a segment) + Matches any number of non-separator characters, including zero. +``?`` + Matches one non-separator character. +``[seq]`` + Matches one character in *seq*. +``[!seq]`` + Matches one character not in *seq*. + +For a literal match, wrap the meta-characters in brackets. +For example, ``"[?]"`` matches the character ``"?"``. + +The "``**``" wildcard enables recursive globbing. A few examples: + +========================= =========================================== +Pattern Meaning +========================= =========================================== +"``**/*``" Any path with at least one segment. +"``**/*.py``" Any path with a final segment ending "``.py``". +"``assets/**``" Any path starting with "``assets/``". +"``assets/**/*``" Any path starting with "``assets/``", excluding "``assets/``" itself. +========================= =========================================== + +.. note:: + Globbing with the "``**``" wildcard visits every directory in the tree. + Large directory trees may take a long time to search. + +.. versionchanged:: 3.13 + Globbing with a pattern that ends with "``**``" returns both files and + directories. In previous versions, only directories were returned. + +In :meth:`Path.glob` and :meth:`~Path.rglob`, a trailing slash may be added to +the pattern to match only directories. + +.. versionchanged:: 3.11 + Globbing with a pattern that ends with a pathname components separator + (:data:`~os.sep` or :data:`~os.altsep`) returns only directories. + + +Comparison to the :mod:`glob` module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The patterns accepted and results generated by :meth:`Path.glob` and +:meth:`Path.rglob` differ slightly from those by the :mod:`glob` module: + +1. Files beginning with a dot are not special in pathlib. This is + like passing ``include_hidden=True`` to :func:`glob.glob`. +2. "``**``" pattern components are always recursive in pathlib. This is like + passing ``recursive=True`` to :func:`glob.glob`. +3. "``**``" pattern components do not follow symlinks by default in pathlib. + This behaviour has no equivalent in :func:`glob.glob`, but you can pass + ``follow_symlinks=True`` to :meth:`Path.glob` for compatible behaviour. +4. Like all :class:`PurePath` and :class:`Path` objects, the values returned + from :meth:`Path.glob` and :meth:`Path.rglob` don't include trailing + slashes. +5. The values returned from pathlib's ``path.glob()`` and ``path.rglob()`` + include the *path* as a prefix, unlike the results of + ``glob.glob(root_dir=path)``. +6. ``bytes``-based paths and :ref:`paths relative to directory descriptors + ` are not supported by pathlib. + + Correspondence to tools in the :mod:`os` module ----------------------------------------------- diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index acada092afb679..223c27237e4d34 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -377,7 +377,7 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, Special reducer that can be defined in :class:`Pickler` subclasses. This method has priority over any reducer in the :attr:`dispatch_table`. It should conform to the same interface as a :meth:`~object.__reduce__` method, and - can optionally return ``NotImplemented`` to fallback on + can optionally return :data:`NotImplemented` to fallback on :attr:`dispatch_table`-registered reducers to pickle ``obj``. For a detailed example, see :ref:`reducer_override`. @@ -503,7 +503,7 @@ What can be pickled and unpickled? The following types can be pickled: * built-in constants (``None``, ``True``, ``False``, ``Ellipsis``, and - ``NotImplemented``); + :data:`NotImplemented`); * integers, floating-point numbers, complex numbers; @@ -653,8 +653,8 @@ methods: .. note:: - If :meth:`__getstate__` returns a false value, the :meth:`__setstate__` - method will not be called upon unpickling. + If :meth:`__reduce__` returns a state with value ``None`` at pickling, + the :meth:`__setstate__` method will not be called upon unpickling. Refer to the section :ref:`pickle-state` for more information about how to use @@ -905,7 +905,7 @@ functions and classes. For those cases, it is possible to subclass from the :class:`Pickler` class and implement a :meth:`~Pickler.reducer_override` method. This method can return an arbitrary reduction tuple (see :meth:`~object.__reduce__`). It can alternatively return -``NotImplemented`` to fallback to the traditional behavior. +:data:`NotImplemented` to fallback to the traditional behavior. If both the :attr:`~Pickler.dispatch_table` and :meth:`~Pickler.reducer_override` are defined, then diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index e883acd67d6c72..2a2eb098646364 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -31,7 +31,93 @@ Dictionaries are sorted by key before the display is computed. .. versionchanged:: 3.10 Added support for pretty-printing :class:`dataclasses.dataclass`. -The :mod:`pprint` module defines one class: +.. _pprint-functions: + +Functions +--------- + +.. function:: pp(object, *args, sort_dicts=False, **kwargs) + + Prints the formatted representation of *object* followed by a newline. + If *sort_dicts* is false (the default), dictionaries will be displayed with + their keys in insertion order, otherwise the dict keys will be sorted. + *args* and *kwargs* will be passed to :func:`~pprint.pprint` as formatting + parameters. + + .. versionadded:: 3.8 + + +.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ + compact=False, sort_dicts=True, underscore_numbers=False) + + Prints the formatted representation of *object* on *stream*, followed by a + newline. If *stream* is ``None``, :data:`sys.stdout` is used. This may be used + in the interactive interpreter instead of the :func:`print` function for + inspecting values (you can even reassign ``print = pprint.pprint`` for use + within a scope). + + The configuration parameters *stream*, *indent*, *width*, *depth*, + *compact*, *sort_dicts* and *underscore_numbers* are passed to the + :class:`PrettyPrinter` constructor and their meanings are as + described in its documentation above. + + >>> import pprint + >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> stuff.insert(0, stuff) + >>> pprint.pprint(stuff) + [, + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni'] + +.. function:: pformat(object, indent=1, width=80, depth=None, *, \ + compact=False, sort_dicts=True, underscore_numbers=False) + + Return the formatted representation of *object* as a string. *indent*, + *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are + passed to the :class:`PrettyPrinter` constructor as formatting parameters + and their meanings are as described in its documentation above. + + +.. function:: isreadable(object) + + .. index:: pair: built-in function; eval + + Determine if the formatted representation of *object* is "readable", or can be + used to reconstruct the value using :func:`eval`. This always returns ``False`` + for recursive objects. + + >>> pprint.isreadable(stuff) + False + + +.. function:: isrecursive(object) + + Determine if *object* requires a recursive representation. This function is + subject to the same limitations as noted in :func:`saferepr` below and may raise an + :exc:`RecursionError` if it fails to detect a recursive object. + + +.. function:: saferepr(object) + + Return a string representation of *object*, protected against recursion in + some common data structures, namely instances of :class:`dict`, :class:`list` + and :class:`tuple` or subclasses whose ``__repr__`` has not been overridden. If the + representation of object exposes a recursive entry, the recursive reference + will be represented as ````. The + representation is not otherwise formatted. + + >>> pprint.saferepr(stuff) + "[, 'spam', 'eggs', 'lumberjack', 'knights', 'ni']" + +.. _prettyprinter-objects: + +PrettyPrinter Objects +--------------------- + +This module defines one class: .. First the implementation class: @@ -44,9 +130,9 @@ The :mod:`pprint` module defines one class: Construct a :class:`PrettyPrinter` instance. This constructor understands several keyword parameters. - *stream* (default ``sys.stdout``) is a :term:`file-like object` to + *stream* (default :data:`!sys.stdout`) is a :term:`file-like object` to which the output will be written by calling its :meth:`!write` method. - If both *stream* and ``sys.stdout`` are ``None``, then + If both *stream* and :data:`!sys.stdout` are ``None``, then :meth:`~PrettyPrinter.pprint` silently returns. Other values configure the manner in which nesting of complex data @@ -87,7 +173,7 @@ The :mod:`pprint` module defines one class: Added the *underscore_numbers* parameter. .. versionchanged:: 3.11 - No longer attempts to write to ``sys.stdout`` if it is ``None``. + No longer attempts to write to :data:`!sys.stdout` if it is ``None``. >>> import pprint >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] @@ -112,89 +198,6 @@ The :mod:`pprint` module defines one class: >>> pp.pprint(tup) ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) -.. function:: pformat(object, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) - - Return the formatted representation of *object* as a string. *indent*, - *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are - passed to the :class:`PrettyPrinter` constructor as formatting parameters - and their meanings are as described in its documentation above. - - -.. function:: pp(object, *args, sort_dicts=False, **kwargs) - - Prints the formatted representation of *object* followed by a newline. - If *sort_dicts* is false (the default), dictionaries will be displayed with - their keys in insertion order, otherwise the dict keys will be sorted. - *args* and *kwargs* will be passed to :func:`pprint` as formatting - parameters. - - .. versionadded:: 3.8 - - -.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) - - Prints the formatted representation of *object* on *stream*, followed by a - newline. If *stream* is ``None``, ``sys.stdout`` is used. This may be used - in the interactive interpreter instead of the :func:`print` function for - inspecting values (you can even reassign ``print = pprint.pprint`` for use - within a scope). - - The configuration parameters *stream*, *indent*, *width*, *depth*, - *compact*, *sort_dicts* and *underscore_numbers* are passed to the - :class:`PrettyPrinter` constructor and their meanings are as - described in its documentation above. - - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff) - >>> pprint.pprint(stuff) - [, - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] - -.. function:: isreadable(object) - - .. index:: pair: built-in function; eval - - Determine if the formatted representation of *object* is "readable", or can be - used to reconstruct the value using :func:`eval`. This always returns ``False`` - for recursive objects. - - >>> pprint.isreadable(stuff) - False - - -.. function:: isrecursive(object) - - Determine if *object* requires a recursive representation. This function is - subject to the same limitations as noted in :func:`saferepr` below and may raise an - :exc:`RecursionError` if it fails to detect a recursive object. - - -One more support function is also defined: - -.. function:: saferepr(object) - - Return a string representation of *object*, protected against recursion in - some common data structures, namely instances of :class:`dict`, :class:`list` - and :class:`tuple` or subclasses whose ``__repr__`` has not been overridden. If the - representation of object exposes a recursive entry, the recursive reference - will be represented as ````. The - representation is not otherwise formatted. - - >>> pprint.saferepr(stuff) - "[, 'spam', 'eggs', 'lumberjack', 'knights', 'ni']" - - -.. _prettyprinter-objects: - -PrettyPrinter Objects ---------------------- :class:`PrettyPrinter` instances have the following methods: @@ -258,7 +261,7 @@ are converted to strings. The default implementation uses the internals of the Example ------- -To demonstrate several uses of the :func:`pprint` function and its parameters, +To demonstrate several uses of the :func:`~pprint.pprint` function and its parameters, let's fetch information about a project from `PyPI `_:: >>> import json @@ -267,7 +270,7 @@ let's fetch information about a project from `PyPI `_:: >>> with urlopen('https://pypi.org/pypi/sampleproject/json') as resp: ... project_info = json.load(resp)['info'] -In its basic form, :func:`pprint` shows the whole object:: +In its basic form, :func:`~pprint.pprint` shows the whole object:: >>> pprint.pprint(project_info) {'author': 'The Python Packaging Authority', diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index cc059b66fcb84b..3ca802e024bc27 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -299,6 +299,13 @@ functions: Create a :class:`~pstats.Stats` object based on the current profile and print the results to stdout. + The *sort* parameter specifies the sorting order of the displayed + statistics. It accepts a single key or a tuple of keys to enable + multi-level sorting, as in :func:`Stats.sort_stats `. + + .. versionadded:: 3.13 + :meth:`~Profile.print_stats` now accepts a tuple of keys. + .. method:: dump_stats(filename) Write the results of the current profile to *filename*. diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 935e872480efda..c897ec9e47b7ca 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -196,6 +196,37 @@ XMLParser Objects :exc:`ExpatError` to be raised with the :attr:`code` attribute set to ``errors.codes[errors.XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING]``. +.. method:: xmlparser.SetReparseDeferralEnabled(enabled) + + .. warning:: + + Calling ``SetReparseDeferralEnabled(False)`` has security implications, + as detailed below; please make sure to understand these consequences + prior to using the ``SetReparseDeferralEnabled`` method. + + Expat 2.6.0 introduced a security mechanism called "reparse deferral" + where instead of causing denial of service through quadratic runtime + from reparsing large tokens, reparsing of unfinished tokens is now delayed + by default until a sufficient amount of input is reached. + Due to this delay, registered handlers may — depending of the sizing of + input chunks pushed to Expat — no longer be called right after pushing new + input to the parser. Where immediate feedback and taking over responsiblity + of protecting against denial of service from large tokens are both wanted, + calling ``SetReparseDeferralEnabled(False)`` disables reparse deferral + for the current Expat parser instance, temporarily or altogether. + Calling ``SetReparseDeferralEnabled(True)`` allows re-enabling reparse + deferral. + + .. versionadded:: 3.13 + +.. method:: xmlparser.GetReparseDeferralEnabled() + + Returns whether reparse deferral is currently enabled for the given + Expat parser instance. + + .. versionadded:: 3.13 + + :class:`xmlparser` objects have the following attributes: @@ -214,7 +245,8 @@ XMLParser Objects :meth:`CharacterDataHandler` callback whenever possible. This can improve performance substantially since Expat normally breaks character data into chunks at every line ending. This attribute is false by default, and may be changed at - any time. + any time. Note that when it is false, data that does not contain newlines + may be chunked too. .. attribute:: xmlparser.buffer_used @@ -372,7 +404,10 @@ otherwise stated. marked content, and ignorable whitespace. Applications which must distinguish these cases can use the :attr:`StartCdataSectionHandler`, :attr:`EndCdataSectionHandler`, and :attr:`ElementDeclHandler` callbacks to - collect the required information. + collect the required information. Note that the character data may be + chunked even if it is short and so you may receive more than one call to + :meth:`CharacterDataHandler`. Set the :attr:`buffer_text` instance attribute + to ``True`` to avoid that. .. method:: xmlparser.UnparsedEntityDeclHandler(entityName, base, systemId, publicId, notationName) diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 1421fc2e552f0e..f2a6dbf589fd87 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -187,11 +187,12 @@ fully processed by daemon consumer threads. processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`put` into the queue). + ``shutdown(immediate=True)`` calls :meth:`task_done` for each remaining item + in the queue. + Raises a :exc:`ValueError` if called more times than there were items placed in the queue. - Raises :exc:`ShutDown` if the queue has been shut down immediately. - .. method:: Queue.join() @@ -202,8 +203,6 @@ fully processed by daemon consumer threads. indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, :meth:`join` unblocks. - Raises :exc:`ShutDown` if the queue has been shut down immediately. - Example of how to wait for enqueued tasks to be completed:: diff --git a/Doc/library/random.rst b/Doc/library/random.rst index d0ced2416c9578..8fbce18c56f17c 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -301,7 +301,8 @@ be found in any statistics text. ``a <= b`` and ``b <= N <= a`` for ``b < a``. The end-point value ``b`` may or may not be included in the range - depending on floating-point rounding in the equation ``a + (b-a) * random()``. + depending on floating-point rounding in the expression + ``a + (b-a) * random()``. .. function:: triangular(low, high, mode) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index a5bd5c73f2fac7..0336121c2bc631 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1344,7 +1344,8 @@ when there is no match, you can test whether there was a match with a simple Escapes such as ``\n`` are converted to the appropriate characters, and numeric backreferences (``\1``, ``\2``) and named backreferences (``\g<1>``, ``\g``) are replaced by the contents of the - corresponding group. + corresponding group. The backreference ``\g<0>`` will be + replaced by the entire match. .. versionchanged:: 3.5 Unmatched groups are replaced with an empty string. diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 01bac5afd0b9b3..4c980dd97f9394 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -36,7 +36,7 @@ scheduler: Example:: >>> import sched, time - >>> s = sched.scheduler(time.monotonic, time.sleep) + >>> s = sched.scheduler(time.time, time.sleep) >>> def print_time(a='default'): ... print("From print_time", time.time(), a) ... diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index ff8c9a189ab3de..4f07b9f6040d24 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -39,7 +39,7 @@ Directory and files operations .. function:: copyfileobj(fsrc, fdst[, length]) - Copy the contents of the file-like object *fsrc* to the file-like object *fdst*. + Copy the contents of the :term:`file-like object ` *fsrc* to the file-like object *fdst*. The integer *length*, if given, is the buffer size. In particular, a negative *length* value means to copy the data without looping over the source data in chunks; by default the data is read in chunks to avoid uncontrolled memory @@ -52,7 +52,7 @@ Directory and files operations Copy the contents (no metadata) of the file named *src* to a file named *dst* and return *dst* in the most efficient way possible. - *src* and *dst* are path-like objects or path names given as strings. + *src* and *dst* are :term:`path-like objects ` or path names given as strings. *dst* must be the complete target file name; look at :func:`~shutil.copy` for a copy that accepts a target directory path. If *src* and *dst* @@ -94,7 +94,7 @@ Directory and files operations .. function:: copymode(src, dst, *, follow_symlinks=True) Copy the permission bits from *src* to *dst*. The file contents, owner, and - group are unaffected. *src* and *dst* are path-like objects or path names + group are unaffected. *src* and *dst* are :term:`path-like objects ` or path names given as strings. If *follow_symlinks* is false, and both *src* and *dst* are symbolic links, :func:`copymode` will attempt to modify the mode of *dst* itself (rather @@ -113,7 +113,7 @@ Directory and files operations Copy the permission bits, last access time, last modification time, and flags from *src* to *dst*. On Linux, :func:`copystat` also copies the "extended attributes" where possible. The file contents, owner, and - group are unaffected. *src* and *dst* are path-like objects or path + group are unaffected. *src* and *dst* are :term:`path-like objects ` or path names given as strings. If *follow_symlinks* is false, and *src* and *dst* both @@ -274,16 +274,16 @@ Directory and files operations .. audit-event:: shutil.copytree src,dst shutil.copytree - .. versionchanged:: 3.3 - Copy metadata when *symlinks* is false. - Now returns *dst*. - .. versionchanged:: 3.2 Added the *copy_function* argument to be able to provide a custom copy function. Added the *ignore_dangling_symlinks* argument to silence dangling symlinks errors when *symlinks* is false. + .. versionchanged:: 3.3 + Copy metadata when *symlinks* is false. + Now returns *dst*. + .. versionchanged:: 3.8 Platform-specific fast-copy syscalls may be used internally in order to copy the file more efficiently. See diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 4bfb0d8c2cfeac..3a931e25de91e5 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -445,6 +445,11 @@ Constants Added ``IP_PKTINFO``, ``IP_UNBLOCK_SOURCE``, ``IP_BLOCK_SOURCE``, ``IP_ADD_SOURCE_MEMBERSHIP``, ``IP_DROP_SOURCE_MEMBERSHIP``. + .. versionchanged:: 3.13 + Added ``SO_BINDTOIFINDEX``. On Linux this constant can be used in the + same way that ``SO_BINDTODEVICE`` is used, but with the index of a + network interface instead of its name. + .. data:: AF_CAN PF_CAN SOL_CAN_* @@ -1605,8 +1610,9 @@ to sockets. Receive data from the socket. The return value is a bytes object representing the data received. The maximum amount of data to be received at once is specified - by *bufsize*. See the Unix manual page :manpage:`recv(2)` for the meaning of - the optional argument *flags*; it defaults to zero. + by *bufsize*. A returned empty bytes object indicates that the client has disconnected. + See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument + *flags*; it defaults to zero. .. note:: diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index c3a012dd32de61..d91ed87919ddb2 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1810,6 +1810,9 @@ to speed up repeated connections from the same clients. *session*, see :attr:`~SSLSocket.session`. + To wrap an :class:`SSLSocket` in another :class:`SSLSocket`, use + :meth:`SSLContext.wrap_bio`. + .. versionchanged:: 3.5 Always allow a server_hostname to be passed, even if OpenSSL does not have SNI. @@ -1995,7 +1998,7 @@ to speed up repeated connections from the same clients. .. versionchanged:: 3.10 - The flag had no effect with OpenSSL before version 1.1.1k. Python 3.8.9, + The flag had no effect with OpenSSL before version 1.1.1l. Python 3.8.9, 3.9.3, and 3.10 include workarounds for previous versions. .. attribute:: SSLContext.security_level diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 0417b3f38a9807..1785c6bcc212b7 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -76,6 +76,7 @@ or sample. :func:`fmean` Fast, floating point arithmetic mean, with optional weighting. :func:`geometric_mean` Geometric mean of data. :func:`harmonic_mean` Harmonic mean of data. +:func:`kde` Estimate the probability density distribution of the data. :func:`median` Median (middle value) of data. :func:`median_low` Low median of data. :func:`median_high` High median of data. @@ -259,6 +260,54 @@ However, for reading convenience, most of the examples show sorted sequences. .. versionchanged:: 3.10 Added support for *weights*. + +.. function:: kde(data, h, kernel='normal') + + `Kernel Density Estimation (KDE) + `_: + Create a continuous probability density function from discrete samples. + + The basic idea is to smooth the data using `a kernel function + `_. + to help draw inferences about a population from a sample. + + The degree of smoothing is controlled by the scaling parameter *h* + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. + + The *kernel* determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. + + Kernels that give some weight to every sample point include + *normal* or *gauss*, *logistic*, and *sigmoid*. + + Kernels that only give weight to sample points within the bandwidth + include *rectangular* or *uniform*, *triangular*, *parabolic* or + *epanechnikov*, *quartic* or *biweight*, *triweight*, and *cosine*. + + A :exc:`StatisticsError` will be raised if the *data* sequence is empty. + + `Wikipedia has an example + `_ + where we can use :func:`kde` to generate and plot a probability + density function estimated from a small sample: + + .. doctest:: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) + >>> xarr = [i/100 for i in range(-750, 1100)] + >>> yarr = [f_hat(x) for x in xarr] + + The points in ``xarr`` and ``yarr`` can be used to make a PDF plot: + + .. image:: kde_example.png + :alt: Scatter plot of the estimated probability density function. + + .. versionadded:: 3.13 + + .. function:: median(data) Return the median (middle value) of numeric data, using the common "mean of @@ -1095,46 +1144,6 @@ The final prediction goes to the largest posterior. This is known as the 'female' -Kernel density estimation -************************* - -It is possible to estimate a continuous probability density function -from a fixed number of discrete samples. - -The basic idea is to smooth the data using `a kernel function such as a -normal distribution, triangular distribution, or uniform distribution -`_. -The degree of smoothing is controlled by a scaling parameter, ``h``, -which is called the *bandwidth*. - -.. testcode:: - - def kde_normal(sample, h): - "Create a continuous probability density function from a sample." - # Smooth the sample with a normal distribution kernel scaled by h. - kernel_h = NormalDist(0.0, h).pdf - n = len(sample) - def pdf(x): - return sum(kernel_h(x - x_i) for x_i in sample) / n - return pdf - -`Wikipedia has an example -`_ -where we can use the ``kde_normal()`` recipe to generate and plot -a probability density function estimated from a small sample: - -.. doctest:: - - >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - >>> f_hat = kde_normal(sample, h=1.5) - >>> xarr = [i/100 for i in range(-750, 1100)] - >>> yarr = [f_hat(x) for x in xarr] - -The points in ``xarr`` and ``yarr`` can be used to make a PDF plot: - -.. image:: kde_example.png - :alt: Scatter plot of the estimated probability density function. - .. # This modelines must appear within the last ten lines of the file. kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 1a4c12590c1018..c81626a194e640 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5450,10 +5450,10 @@ The NotImplemented Object This object is returned from comparisons and binary operations when they are asked to operate on types they don't support. See :ref:`comparisons` for more -information. There is exactly one ``NotImplemented`` object. -``type(NotImplemented)()`` produces the singleton instance. +information. There is exactly one :data:`NotImplemented` object. +:code:`type(NotImplemented)()` produces the singleton instance. -It is written as ``NotImplemented``. +It is written as :code:`NotImplemented`. .. _typesinternal: diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index e2e6fc542e3e67..3e507c1c7e7c85 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -160,6 +160,21 @@ following table: If the first character is not one of these, ``'@'`` is assumed. +.. note:: + + The number 1023 (``0x3ff`` in hexadecimal) has the following byte representations: + + * ``03 ff`` in big-endian (``>``) + * ``ff 03`` in little-endian (``<``) + + Python example: + + >>> import struct + >>> struct.pack('>h', 1023) + b'\x03\xff' + >>> struct.pack('`. + + (2) When used with the :func:`strptime` function, the ``%p`` directive only affects the output hour field if the ``%I`` directive is used to parse the hour. .. _leap-second: - (2) + (3) The range really is ``0`` to ``61``; value ``60`` is valid in timestamps representing `leap seconds`_ and value ``61`` is supported for historical reasons. - (3) + (4) When used with the :func:`strptime` function, ``%U`` and ``%W`` are only used in calculations when the day of the week and the year are specified. diff --git a/Doc/library/types.rst b/Doc/library/types.rst index c8c981024c1aeb..b856544e44207c 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -188,7 +188,7 @@ Standard names are defined for the following types: .. index:: pair: built-in function; compile - The type for code objects such as returned by :func:`compile`. + The type of :ref:`code objects ` such as returned by :func:`compile`. .. audit-event:: code.__new__ code,filename,name,argcount,posonlyargcount,kwonlyargcount,nlocals,stacksize,flags types.CodeType @@ -196,14 +196,6 @@ Standard names are defined for the following types: required by the initializer. The audit event only occurs for direct instantiation of code objects, and is not raised for normal compilation. - .. method:: CodeType.replace(**kwargs) - - Return a copy of the code object with new values for the specified fields. - - Code objects are also supported by generic function :func:`copy.replace`. - - .. versionadded:: 3.8 - .. data:: CellType The type for cell objects: such objects are used as containers for diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index eca20b94ec8e74..1f25a16f544da8 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -2143,10 +2143,10 @@ to change the default. Methods and their defaults: -* ``__lt__``: ``NotImplemented`` -* ``__gt__``: ``NotImplemented`` -* ``__le__``: ``NotImplemented`` -* ``__ge__``: ``NotImplemented`` +* ``__lt__``: :data:`NotImplemented` +* ``__gt__``: :data:`!NotImplemented` +* ``__le__``: :data:`!NotImplemented` +* ``__ge__``: :data:`!NotImplemented` * ``__int__``: ``1`` * ``__contains__``: ``False`` * ``__len__``: ``0`` @@ -2426,6 +2426,14 @@ passed in. >>> m.mock_calls == [call(1), call(1, 2), ANY] True +:data:`ANY` is not limited to comparisons with call objects and so +can also be used in test assertions:: + + class TestStringMethods(unittest.TestCase): + + def test_split(self): + s = 'hello world' + self.assertEqual(s.split(), ['hello', ANY]) FILTER_DIR diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index aa18873f223a6b..2e7ff345a06234 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -286,15 +286,15 @@ creation according to their needs, the :class:`EnvBuilder` class. the virtual environment. - .. versionchanged:: 3.12 - The attribute ``lib_path`` was added to the context, and the context - object was documented. - .. versionchanged:: 3.11 The *venv* :ref:`sysconfig installation scheme ` is used to construct the paths of the created directories. + .. versionchanged:: 3.12 + The attribute ``lib_path`` was added to the context, and the context + object was documented. + .. method:: create_configuration(context) Creates the ``pyvenv.cfg`` configuration file in the environment. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 75a7915c15240d..19c7af452e2b71 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -166,6 +166,11 @@ data but would still like to have incremental parsing capabilities, take a look at :func:`iterparse`. It can be useful when you're reading a large XML document and don't want to hold it wholly in memory. +Where *immediate* feedback through events is wanted, calling method +:meth:`XMLPullParser.flush` can help reduce delay; +please make sure to study the related security notes. + + Finding interesting elements ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1387,6 +1392,19 @@ XMLParser Objects Feeds data to the parser. *data* is encoded data. + + .. method:: flush() + + Triggers parsing of any previously fed unparsed data, which can be + used to ensure more immediate feedback, in particular with Expat >=2.6.0. + The implementation of :meth:`flush` temporarily disables reparse deferral + with Expat (if currently enabled) and triggers a reparse. + Disabling reparse deferral has security consequences; please see + :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. + + .. versionadded:: 3.13 + + :meth:`XMLParser.feed` calls *target*\'s ``start(tag, attrs_dict)`` method for each opening tag, its ``end(tag)`` method for each closing tag, and data is processed by method ``data(data)``. For further supported callback @@ -1448,6 +1466,17 @@ XMLPullParser Objects Feed the given bytes data to the parser. + .. method:: flush() + + Triggers parsing of any previously fed unparsed data, which can be + used to ensure more immediate feedback, in particular with Expat >=2.6.0. + The implementation of :meth:`flush` temporarily disables reparse deferral + with Expat (if currently enabled) and triggers a reparse. + Disabling reparse deferral has security consequences; please see + :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. + + .. versionadded:: 3.13 + .. method:: close() Signal the parser that the data stream is terminated. Unlike diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index 909022ea4ba6a4..662cc459197e2c 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -68,6 +68,7 @@ quadratic blowup **Vulnerable** (1) **Vulnerable** (1) **Vulnerable* external entity expansion Safe (5) Safe (2) Safe (3) Safe (5) Safe (4) `DTD`_ retrieval Safe (5) Safe Safe Safe (5) Safe decompression bomb Safe Safe Safe Safe **Vulnerable** +large tokens **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) ========================= ================== ================== ================== ================== ================== 1. Expat 2.4.1 and newer is not vulnerable to the "billion laughs" and @@ -81,6 +82,11 @@ decompression bomb Safe Safe Safe 4. :mod:`xmlrpc.client` doesn't expand external entities and omits them. 5. Since Python 3.7.1, external general entities are no longer processed by default. +6. Expat 2.6.0 and newer is not vulnerable to denial of service + through quadratic runtime caused by parsing large tokens. + Items still listed as vulnerable due to + potential reliance on system-provided libraries. Check + :const:`!pyexpat.EXPAT_VERSION`. billion laughs / exponential entity expansion @@ -114,6 +120,13 @@ decompression bomb files. For an attacker it can reduce the amount of transmitted data by three magnitudes or more. +large tokens + Expat needs to re-parse unfinished tokens; without the protection + introduced in Expat 2.6.0, this can lead to quadratic runtime that can + be used to cause denial of service in the application parsing XML. + The issue is known as + `CVE-2023-52425 `_. + The documentation for `defusedxml`_ on PyPI has further information about all known attack vectors with examples and references. diff --git a/Doc/license.rst b/Doc/license.rst index 9fc0ff7161a591..cbe918bd1acfe3 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -1095,3 +1095,35 @@ which is distributed under the MIT license:: LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Global Unbounded Sequences (GUS) +-------------------------------- + +The file :file:`Python/qsbr.c` is adapted from FreeBSD's "Global Unbounded +Sequences" safe memory reclamation scheme in +`subr_smr.c `_. +The file is distributed under the 2-Clause BSD License:: + + Copyright (c) 2019,2020 Jeffrey Roberson + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice unmodified, this list of conditions, and the following + disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 88bc025c7c3fb4..75b656f385d34b 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -34,7 +34,7 @@ represented by objects.) Every object has an identity, a type and a value. An object's *identity* never changes once it has been created; you may think of it as the object's address in -memory. The ':keyword:`is`' operator compares the identity of two objects; the +memory. The :keyword:`is` operator compares the identity of two objects; the :func:`id` function returns an integer representing its identity. .. impl-detail:: @@ -81,7 +81,7 @@ are still reachable. Note that the use of the implementation's tracing or debugging facilities may keep objects alive that would normally be collectable. Also note that catching -an exception with a ':keyword:`try`...\ :keyword:`except`' statement may keep +an exception with a :keyword:`try`...\ :keyword:`except` statement may keep objects alive. Some objects contain references to "external" resources such as open files or @@ -89,8 +89,8 @@ windows. It is understood that these resources are freed when the object is garbage-collected, but since garbage collection is not guaranteed to happen, such objects also provide an explicit way to release the external resource, usually a :meth:`!close` method. Programs are strongly recommended to explicitly -close such objects. The ':keyword:`try`...\ :keyword:`finally`' statement -and the ':keyword:`with`' statement provide convenient ways to do this. +close such objects. The :keyword:`try`...\ :keyword:`finally` statement +and the :keyword:`with` statement provide convenient ways to do this. .. index:: single: container @@ -159,7 +159,7 @@ NotImplemented .. index:: pair: object; NotImplemented This type has a single value. There is a single object with this value. This -object is accessed through the built-in name ``NotImplemented``. Numeric methods +object is accessed through the built-in name :data:`NotImplemented`. Numeric methods and rich comparison methods should return this value if they do not implement the operation for the operands provided. (The interpreter will then try the reflected operation, or some other fallback, depending on the operator.) It @@ -170,7 +170,7 @@ See for more details. .. versionchanged:: 3.9 - Evaluating ``NotImplemented`` in a boolean context is deprecated. While + Evaluating :data:`NotImplemented` in a boolean context is deprecated. While it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. It will raise a :exc:`TypeError` in a future version of Python. @@ -1292,6 +1292,14 @@ Methods on code objects :pep:`626` - Precise line numbers for debugging and other tools. The PEP that introduced the :meth:`!co_lines` method. +.. method:: codeobject.replace(**kwargs) + + Return a copy of the code object with new values for the specified fields. + + Code objects are also supported by the generic function :func:`copy.replace`. + + .. versionadded:: 3.8 + .. _frame-objects: @@ -1779,7 +1787,7 @@ Basic customization ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and ``x>=y`` calls ``x.__ge__(y)``. - A rich comparison method may return the singleton ``NotImplemented`` if it does + A rich comparison method may return the singleton :data:`NotImplemented` if it does not implement the operation for a given pair of arguments. By convention, ``False`` and ``True`` are returned for a successful comparison. However, these methods can return any value, so if the comparison operator is used in a Boolean @@ -1787,10 +1795,10 @@ Basic customization :func:`bool` on the value to determine if the result is true or false. By default, ``object`` implements :meth:`__eq__` by using ``is``, returning - ``NotImplemented`` in the case of a false comparison: + :data:`NotImplemented` in the case of a false comparison: ``True if x is y else NotImplemented``. For :meth:`__ne__`, by default it delegates to :meth:`__eq__` and inverts the result unless it is - ``NotImplemented``. There are no other implied relationships among the + :data:`!NotImplemented`. There are no other implied relationships among the comparison operators or default implementations; for example, the truth of ``(x=`` 0. The return value may also be - :const:`NotImplemented`, which is treated the same as if the + :data:`NotImplemented`, which is treated the same as if the ``__length_hint__`` method didn't exist at all. This method is purely an optimization and is never required for correctness. @@ -2972,7 +2983,7 @@ left undefined. function is to be supported. If one of those methods does not support the operation with the supplied - arguments, it should return ``NotImplemented``. + arguments, it should return :data:`NotImplemented`. .. method:: object.__radd__(self, other) @@ -3002,7 +3013,7 @@ left undefined. types. [#]_ For instance, to evaluate the expression ``x - y``, where *y* is an instance of a class that has an :meth:`__rsub__` method, ``type(y).__rsub__(y, x)`` is called if ``type(x).__sub__(x, y)`` returns - *NotImplemented*. + :data:`NotImplemented`. .. index:: pair: built-in function; pow @@ -3036,10 +3047,12 @@ left undefined. (``+=``, ``-=``, ``*=``, ``@=``, ``/=``, ``//=``, ``%=``, ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods should attempt to do the operation in-place (modifying *self*) and return the result (which could be, - but does not have to be, *self*). If a specific method is not defined, the + but does not have to be, *self*). If a specific method is not defined, or if + that method returns :data:`NotImplemented`, the augmented assignment falls back to the normal methods. For instance, if *x* is an instance of a class with an :meth:`__iadd__` method, ``x += y`` is - equivalent to ``x = x.__iadd__(y)`` . Otherwise, ``x.__add__(y)`` and + equivalent to ``x = x.__iadd__(y)`` . If :meth:`__iadd__` does not exist, or if ``x.__iadd__(y)`` + returns :data:`!NotImplemented`, ``x.__add__(y)`` and ``y.__radd__(x)`` are considered, as with the evaluation of ``x + y``. In certain situations, augmented assignment can result in unexpected errors (see :ref:`faq-augmented-assignment-tuple-error`), but this behavior is in fact @@ -3495,7 +3508,7 @@ An example of an asynchronous context manager class:: the behavior that ``None`` is not callable. .. [#] "Does not support" here means that the class has no such method, or - the method returns ``NotImplemented``. Do not set the method to + the method returns :data:`NotImplemented`. Do not set the method to ``None`` if you want to force fallback to the right operand's reflected method—that will instead have the opposite effect of explicitly *blocking* such fallback. diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 50e0f97a6534af..00b57effd3e1c0 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1539,7 +1539,7 @@ built-in types. ``x == x`` are all false, while ``x != x`` is true. This behavior is compliant with IEEE 754. -* ``None`` and ``NotImplemented`` are singletons. :PEP:`8` advises that +* ``None`` and :data:`NotImplemented` are singletons. :PEP:`8` advises that comparisons for singletons should always be done with ``is`` or ``is not``, never the equality operators. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 33129e898e51d6..5fbc24c6ee65a5 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -29,7 +29,6 @@ Doc/library/email.parser.rst Doc/library/email.policy.rst Doc/library/exceptions.rst Doc/library/faulthandler.rst -Doc/library/fcntl.rst Doc/library/functools.rst Doc/library/http.cookiejar.rst Doc/library/http.server.rst @@ -79,15 +78,11 @@ Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst Doc/tutorial/datastructures.rst Doc/using/windows.rst -Doc/whatsnew/2.0.rst -Doc/whatsnew/2.1.rst Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst Doc/whatsnew/2.6.rst Doc/whatsnew/2.7.rst Doc/whatsnew/3.0.rst -Doc/whatsnew/3.1.rst -Doc/whatsnew/3.2.rst Doc/whatsnew/3.3.rst Doc/whatsnew/3.4.rst Doc/whatsnew/3.5.rst diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 4536ab9486d39c..0f16dae8b1418f 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -405,13 +405,6 @@ indexed and sliced:: >>> squares[-3:] # slicing returns a new list [9, 16, 25] -All slice operations return a new list containing the requested elements. This -means that the following slice returns a -:ref:`shallow copy ` of the list:: - - >>> squares[:] - [1, 4, 9, 16, 25] - Lists also support operations like concatenation:: >>> squares + [36, 49, 64, 81, 100] @@ -435,6 +428,30 @@ the :meth:`!list.append` *method* (we will see more about methods later):: >>> cubes [1, 8, 27, 64, 125, 216, 343] +Simple assignment in Python never copies data. When you assign a list +to a variable, the variable refers to the *existing list*. +Any changes you make to the list through one variable will be seen +through all other variables that refer to it.:: + + >>> rgb = ["Red", "Green", "Blue"] + >>> rgba = rgb + >>> id(rgb) == id(rgba) # they reference the same object + True + >>> rgba.append("Alph") + >>> rgb + ["Red", "Green", "Blue", "Alph"] + +All slice operations return a new list containing the requested elements. This +means that the following slice returns a +:ref:`shallow copy ` of the list:: + + >>> correct_rgba = rgba[:] + >>> correct_rgba[-1] = "Alpha" + >>> correct_rgba + ["Red", "Green", "Blue", "Alpha"] + >>> rgba + ["Red", "Green", "Blue", "Alph"] + Assignment to slices is also possible, and this can even change the size of the list or clear it entirely:: diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index aab9469b44828a..26e355a735deaa 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -2,6 +2,8 @@ Configure Python **************** +.. highlight:: sh + Build Requirements ================== @@ -30,31 +32,31 @@ Features and minimum versions required to build CPython: * Autoconf 2.71 and aclocal 1.16.4 are required to regenerate the :file:`configure` script. -.. versionchanged:: 3.13: - Autoconf 2.71, aclocal 1.16.4 and SQLite 3.15.2 are now required. +.. versionchanged:: 3.1 + Tcl/Tk version 8.3.1 is now required. -.. versionchanged:: 3.11 - C11 compiler, IEEE 754 and NaN support are now required. - On Windows, Visual Studio 2017 or later is required. - Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. +.. versionchanged:: 3.5 + On Windows, Visual Studio 2015 or later is now required. + Tcl/Tk version 8.4 is now required. -.. versionchanged:: 3.10 - OpenSSL 1.1.1 is now required. - Require SQLite 3.7.15. +.. versionchanged:: 3.6 + Selected C99 features are now required, like ```` and ``static + inline`` functions. .. versionchanged:: 3.7 Thread support and OpenSSL 1.0.2 are now required. -.. versionchanged:: 3.6 - Selected C99 features are now required, like ```` and ``static - inline`` functions. +.. versionchanged:: 3.10 + OpenSSL 1.1.1 is now required. + Require SQLite 3.7.15. -.. versionchanged:: 3.5 - On Windows, Visual Studio 2015 or later is now required. - Tcl/Tk version 8.4 is now required. +.. versionchanged:: 3.11 + C11 compiler, IEEE 754 and NaN support are now required. + On Windows, Visual Studio 2017 or later is required. + Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. -.. versionchanged:: 3.1 - Tcl/Tk version 8.3.1 is now required. +.. versionchanged:: 3.13 + Autoconf 2.71, aclocal 1.16.4 and SQLite 3.15.2 are now required. See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform support". @@ -275,7 +277,7 @@ General Options * to/from free lists; * dictionary materialized/dematerialized; * type cache; - * optimization attemps; + * optimization attempts; * optimization traces created/executed; * uops executed. @@ -694,12 +696,12 @@ Debug options :ref:`Statically allocated objects ` are not traced. + .. versionadded:: 3.8 + .. versionchanged:: 3.13 This build is now ABI compatible with release build and :ref:`debug build `. - .. versionadded:: 3.8 - .. option:: --with-assertions Build with C assertions enabled (default is no): ``assert(...);`` and @@ -941,7 +943,9 @@ the version of the cross compiled host Python. An environment variable that points to a file with configure overrides. - Example *config.site* file:: + Example *config.site* file: + + .. code-block:: ini # config.site-aarch64 ac_cv_buggy_getaddrinfo=no @@ -1019,7 +1023,9 @@ C extensions Some C extensions are built as built-in modules, like the ``sys`` module. They are built with the ``Py_BUILD_CORE_BUILTIN`` macro defined. -Built-in modules have no ``__file__`` attribute:: +Built-in modules have no ``__file__`` attribute: + +.. code-block:: pycon >>> import sys >>> sys @@ -1031,7 +1037,9 @@ Built-in modules have no ``__file__`` attribute:: Other C extensions are built as dynamic libraries, like the ``_asyncio`` module. They are built with the ``Py_BUILD_CORE_MODULE`` macro defined. -Example on Linux x86-64:: +Example on Linux x86-64: + +.. code-block:: pycon >>> import _asyncio >>> _asyncio diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index 1cf438b198a9af..354eb1541ceac2 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -14,14 +14,14 @@ used at environment creation time). It also creates an (initially empty) ``Lib\site-packages``). If an existing directory is specified, it will be re-used. +.. versionchanged:: 3.5 + The use of ``venv`` is now recommended for creating virtual environments. + .. deprecated:: 3.6 ``pyvenv`` was the recommended tool for creating virtual environments for Python 3.3 and 3.4, and is :ref:`deprecated in Python 3.6 `. -.. versionchanged:: 3.5 - The use of ``venv`` is now recommended for creating virtual environments. - .. highlight:: none On Windows, invoke the ``venv`` command as follows:: diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 2a0e7b4b06f586..cc4db34b04d900 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -14,8 +14,8 @@ know about when using Python on Microsoft Windows. Unlike most Unix systems and services, Windows does not include a system supported installation of Python. To make Python available, the CPython team -has compiled Windows installers (MSI packages) with every `release -`_ for many years. These installers +has compiled Windows installers with every `release +`_ for many years. These installers are primarily intended to add a per-user installation of Python, with the core interpreter and library being used by a single user. The installer is also able to install for all users of a single machine, and a separate ZIP file is diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index af8171487fbcfa..1a949ec4035807 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -217,13 +217,13 @@ often use the ``codecs.lookup(encoding)`` function, which returns a was consumed. * *stream_reader* is a class that supports decoding input from a stream. - *stream_reader(file_obj)* returns an object that supports the :meth:`read`, - :meth:`readline`, and :meth:`readlines` methods. These methods will all + *stream_reader(file_obj)* returns an object that supports the :meth:`!read`, + :meth:`!readline`, and :meth:`!readlines` methods. These methods will all translate from the given encoding and return Unicode strings. * *stream_writer*, similarly, is a class that supports encoding output to a stream. *stream_writer(file_obj)* returns an object that supports the - :meth:`write` and :meth:`writelines` methods. These methods expect Unicode + :meth:`!write` and :meth:`!writelines` methods. These methods expect Unicode strings, translating them to the given encoding on output. For example, the following code writes a Unicode string into a file, encoding @@ -356,8 +356,8 @@ variable ``a`` by 2, equivalent to the slightly lengthier ``a = a + 2``. The full list of supported assignment operators is ``+=``, ``-=``, ``*=``, ``/=``, ``%=``, ``**=``, ``&=``, ``|=``, ``^=``, ``>>=``, and ``<<=``. Python classes can override the augmented assignment operators by defining methods -named :meth:`__iadd__`, :meth:`__isub__`, etc. For example, the following -:class:`Number` class stores a number and supports using += to create a new +named :meth:`!__iadd__`, :meth:`!__isub__`, etc. For example, the following +:class:`!Number` class stores a number and supports using += to create a new instance with an incremented value. .. The empty groups below prevent conversion to guillemets. @@ -374,7 +374,7 @@ instance with an incremented value. n += 3 print n.value -The :meth:`__iadd__` special method is called with the value of the increment, +The :meth:`!__iadd__` special method is called with the value of the increment, and should return a new instance with an appropriately modified value; this return value is bound as the new value of the variable on the left-hand side. @@ -390,10 +390,10 @@ String Methods ============== Until now string-manipulation functionality was in the :mod:`string` module, -which was usually a front-end for the :mod:`strop` module written in C. The -addition of Unicode posed a difficulty for the :mod:`strop` module, because the +which was usually a front-end for the :mod:`!strop` module written in C. The +addition of Unicode posed a difficulty for the :mod:`!strop` module, because the functions would all need to be rewritten in order to accept either 8-bit or -Unicode strings. For functions such as :func:`string.replace`, which takes 3 +Unicode strings. For functions such as :func:`!string.replace`, which takes 3 string arguments, that means eight possible permutations, and correspondingly complicated code. @@ -416,13 +416,13 @@ The old :mod:`string` module is still around for backwards compatibility, but it mostly acts as a front-end to the new string methods. Two methods which have no parallel in pre-2.0 versions, although they did exist -in JPython for quite some time, are :meth:`startswith` and :meth:`endswith`. +in JPython for quite some time, are :meth:`!startswith` and :meth:`!endswith`. ``s.startswith(t)`` is equivalent to ``s[:len(t)] == t``, while ``s.endswith(t)`` is equivalent to ``s[-len(t):] == t``. -One other method which deserves special mention is :meth:`join`. The -:meth:`join` method of a string receives one parameter, a sequence of strings, -and is equivalent to the :func:`string.join` function from the old :mod:`string` +One other method which deserves special mention is :meth:`!join`. The +:meth:`!join` method of a string receives one parameter, a sequence of strings, +and is equivalent to the :func:`!string.join` function from the old :mod:`string` module, with the arguments reversed. In other words, ``s.join(seq)`` is equivalent to the old ``string.join(seq, s)``. @@ -503,9 +503,9 @@ Minor Language Changes A new syntax makes it more convenient to call a given function with a tuple of arguments and/or a dictionary of keyword arguments. In Python 1.5 and earlier, -you'd use the :func:`apply` built-in function: ``apply(f, args, kw)`` calls the -function :func:`f` with the argument tuple *args* and the keyword arguments in -the dictionary *kw*. :func:`apply` is the same in 2.0, but thanks to a patch +you'd use the :func:`!apply` built-in function: ``apply(f, args, kw)`` calls the +function :func:`!f` with the argument tuple *args* and the keyword arguments in +the dictionary *kw*. :func:`!apply` is the same in 2.0, but thanks to a patch from Greg Ewing, ``f(*args, **kw)`` is a shorter and clearer way to achieve the same effect. This syntax is symmetrical with the syntax for defining functions:: @@ -518,7 +518,7 @@ functions:: The ``print`` statement can now have its output directed to a file-like object by following the ``print`` with ``>> file``, similar to the redirection operator in Unix shells. Previously you'd either have to use the -:meth:`write` method of the file-like object, which lacks the convenience and +:meth:`!write` method of the file-like object, which lacks the convenience and simplicity of ``print``, or you could assign a new value to ``sys.stdout`` and then restore the old value. For sending output to standard error, it's much easier to write this:: @@ -540,7 +540,7 @@ Previously there was no way to implement a class that overrode Python's built-in true if *obj* is present in the sequence *seq*; Python computes this by simply trying every index of the sequence until either *obj* is found or an :exc:`IndexError` is encountered. Moshe Zadka contributed a patch which adds a -:meth:`__contains__` magic method for providing a custom implementation for +:meth:`!__contains__` magic method for providing a custom implementation for :keyword:`!in`. Additionally, new built-in objects written in C can define what :keyword:`!in` means for them via a new slot in the sequence protocol. @@ -562,7 +562,7 @@ the python-dev mailing list for the discussion leading up to this implementation, and some useful relevant links. Note that comparisons can now also raise exceptions. In earlier versions of Python, a comparison operation such as ``cmp(a,b)`` would always produce an answer, even if a user-defined -:meth:`__cmp__` method encountered an error, since the resulting exception would +:meth:`!__cmp__` method encountered an error, since the resulting exception would simply be silently swallowed. .. Starting URL: @@ -607,7 +607,7 @@ seq1, seq2)`` is that :func:`map` pads the sequences with ``None`` if the sequences aren't all of the same length, while :func:`zip` truncates the returned list to the length of the shortest argument sequence. -The :func:`int` and :func:`long` functions now accept an optional "base" +The :func:`int` and :func:`!long` functions now accept an optional "base" parameter when the first argument is a string. ``int('123', 10)`` returns 123, while ``int('123', 16)`` returns 291. ``int(123, 16)`` raises a :exc:`TypeError` exception with the message "can't convert non-string with @@ -620,8 +620,8 @@ would be ``(2, 0, 1, 'beta', 1)``. *level* is a string such as ``"alpha"``, ``"beta"``, or ``"final"`` for a final release. Dictionaries have an odd new method, ``setdefault(key, default)``, which -behaves similarly to the existing :meth:`get` method. However, if the key is -missing, :meth:`setdefault` both returns the value of *default* as :meth:`get` +behaves similarly to the existing :meth:`!get` method. However, if the key is +missing, :meth:`!setdefault` both returns the value of *default* as :meth:`!get` would do, and also inserts it into the dictionary as the value for *key*. Thus, the following lines of code:: @@ -656,7 +656,7 @@ break. The change which will probably break the most code is tightening up the arguments accepted by some methods. Some methods would take multiple arguments and treat them as a tuple, particularly various list methods such as -:meth:`append` and :meth:`insert`. In earlier versions of Python, if ``L`` is +:meth:`!append` and :meth:`!insert`. In earlier versions of Python, if ``L`` is a list, ``L.append( 1,2 )`` appends the tuple ``(1,2)`` to the list. In Python 2.0 this causes a :exc:`TypeError` exception to be raised, with the message: 'append requires exactly 1 argument; 2 given'. The fix is to simply add an @@ -693,7 +693,7 @@ advantage of this fact will break in 2.0. Some work has been done to make integers and long integers a bit more interchangeable. In 1.5.2, large-file support was added for Solaris, to allow -reading files larger than 2 GiB; this made the :meth:`tell` method of file +reading files larger than 2 GiB; this made the :meth:`!tell` method of file objects return a long integer instead of a regular integer. Some code would subtract two file offsets and attempt to use the result to multiply a sequence or slice a string, but this raised a :exc:`TypeError`. In 2.0, long integers @@ -701,7 +701,7 @@ can be used to multiply or slice a sequence, and it'll behave as you'd intuitively expect it to; ``3L * 'abc'`` produces 'abcabcabc', and ``(0,1,2,3)[2L:4L]`` produces (2,3). Long integers can also be used in various contexts where previously only integers were accepted, such as in the -:meth:`seek` method of file objects, and in the formats supported by the ``%`` +:meth:`!seek` method of file objects, and in the formats supported by the ``%`` operator (``%d``, ``%i``, ``%x``, etc.). For example, ``"%d" % 2L**64`` will produce the string ``18446744073709551616``. @@ -715,7 +715,7 @@ digit. Taking the :func:`repr` of a float now uses a different formatting precision than :func:`str`. :func:`repr` uses ``%.17g`` format string for C's -:func:`sprintf`, while :func:`str` uses ``%.12g`` as before. The effect is that +:func:`!sprintf`, while :func:`str` uses ``%.12g`` as before. The effect is that :func:`repr` may occasionally show more decimal places than :func:`str`, for certain numbers. For example, the number 8.1 can't be represented exactly in binary, so ``repr(8.1)`` is ``'8.0999999999999996'``, while str(8.1) is @@ -723,7 +723,7 @@ binary, so ``repr(8.1)`` is ``'8.0999999999999996'``, while str(8.1) is The ``-X`` command-line option, which turned all standard exceptions into strings instead of classes, has been removed; the standard exceptions will now -always be classes. The :mod:`exceptions` module containing the standard +always be classes. The :mod:`!exceptions` module containing the standard exceptions was translated from Python to a built-in C module, written by Barry Warsaw and Fredrik Lundh. @@ -879,11 +879,11 @@ joins the basic set of Python documentation. XML Modules =========== -Python 1.5.2 included a simple XML parser in the form of the :mod:`xmllib` +Python 1.5.2 included a simple XML parser in the form of the :mod:`!xmllib` module, contributed by Sjoerd Mullender. Since 1.5.2's release, two different interfaces for processing XML have become common: SAX2 (version 2 of the Simple API for XML) provides an event-driven interface with some similarities to -:mod:`xmllib`, and the DOM (Document Object Model) provides a tree-based +:mod:`!xmllib`, and the DOM (Document Object Model) provides a tree-based interface, transforming an XML document into a tree of nodes that can be traversed and modified. Python 2.0 includes a SAX2 interface and a stripped-down DOM interface as part of the :mod:`xml` package. Here we will give a brief @@ -898,9 +898,9 @@ SAX2 Support SAX defines an event-driven interface for parsing XML. To use SAX, you must write a SAX handler class. Handler classes inherit from various classes provided by SAX, and override various methods that will then be called by the -XML parser. For example, the :meth:`startElement` and :meth:`endElement` +XML parser. For example, the :meth:`~xml.sax.handler.ContentHandler.startElement` and :meth:`~xml.sax.handler.ContentHandler.endElement` methods are called for every starting and end tag encountered by the parser, the -:meth:`characters` method is called for every chunk of character data, and so +:meth:`~xml.sax.handler.ContentHandler.characters` method is called for every chunk of character data, and so forth. The advantage of the event-driven approach is that the whole document doesn't @@ -940,8 +940,8 @@ DOM Support ----------- The Document Object Model is a tree-based representation for an XML document. A -top-level :class:`Document` instance is the root of the tree, and has a single -child which is the top-level :class:`Element` instance. This :class:`Element` +top-level :class:`!Document` instance is the root of the tree, and has a single +child which is the top-level :class:`!Element` instance. This :class:`!Element` has children nodes representing character data and any sub-elements, which may have further children of their own, and so forth. Using the DOM you can traverse the resulting tree any way you like, access element and attribute @@ -955,18 +955,18 @@ simply writing ````...\ ```` to a file. The DOM implementation included with Python lives in the :mod:`xml.dom.minidom` module. It's a lightweight implementation of the Level 1 DOM with support for -XML namespaces. The :func:`parse` and :func:`parseString` convenience +XML namespaces. The :func:`!parse` and :func:`!parseString` convenience functions are provided for generating a DOM tree:: from xml.dom import minidom doc = minidom.parse('hamlet.xml') -``doc`` is a :class:`Document` instance. :class:`Document`, like all the other -DOM classes such as :class:`Element` and :class:`Text`, is a subclass of the -:class:`Node` base class. All the nodes in a DOM tree therefore support certain -common methods, such as :meth:`toxml` which returns a string containing the XML +``doc`` is a :class:`!Document` instance. :class:`!Document`, like all the other +DOM classes such as :class:`!Element` and :class:`Text`, is a subclass of the +:class:`!Node` base class. All the nodes in a DOM tree therefore support certain +common methods, such as :meth:`!toxml` which returns a string containing the XML representation of the node and its children. Each class also has special -methods of its own; for example, :class:`Element` and :class:`Document` +methods of its own; for example, :class:`!Element` and :class:`!Document` instances have a method to find all child elements with a given tag name. Continuing from the previous 2-line example:: @@ -995,7 +995,7 @@ its children can be easily modified by deleting, adding, or removing nodes:: root.insertBefore( root.childNodes[0], root.childNodes[20] ) Again, I will refer you to the Python documentation for a complete listing of -the different :class:`Node` classes and their various methods. +the different :class:`!Node` classes and their various methods. Relationship to PyXML @@ -1020,7 +1020,7 @@ features in PyXML include: * The xmlproc validating parser, written by Lars Marius Garshol. -* The :mod:`sgmlop` parser accelerator module, written by Fredrik Lundh. +* The :mod:`!sgmlop` parser accelerator module, written by Fredrik Lundh. .. ====================================================================== @@ -1031,7 +1031,7 @@ Module changes Lots of improvements and bugfixes were made to Python's extensive standard library; some of the affected modules include :mod:`readline`, :mod:`ConfigParser `, :mod:`!cgi`, :mod:`calendar`, :mod:`posix`, :mod:`readline`, -:mod:`xmllib`, :mod:`!aifc`, :mod:`!chunk`, :mod:`wave`, :mod:`random`, :mod:`shelve`, +:mod:`!xmllib`, :mod:`!aifc`, :mod:`!chunk`, :mod:`wave`, :mod:`random`, :mod:`shelve`, and :mod:`!nntplib`. Consult the CVS logs for the exact patch-by-patch details. Brian Gallew contributed OpenSSL support for the :mod:`socket` module. OpenSSL @@ -1044,11 +1044,12 @@ were also changed to support ``https://`` URLs, though no one has implemented FTP or SMTP over SSL. The :mod:`httplib ` module has been rewritten by Greg Stein to support HTTP/1.1. + Backward compatibility with the 1.5 version of :mod:`!httplib` is provided, though using HTTP/1.1 features such as pipelining will require rewriting code to use a different set of interfaces. -The :mod:`Tkinter` module now supports Tcl/Tk version 8.1, 8.2, or 8.3, and +The :mod:`!Tkinter` module now supports Tcl/Tk version 8.1, 8.2, or 8.3, and support for the older 7.x versions has been dropped. The Tkinter module now supports displaying Unicode strings in Tk widgets. Also, Fredrik Lundh contributed an optimization which makes operations like ``create_line`` and @@ -1083,11 +1084,11 @@ module. calling :func:`atexit.register` with the function to be called on exit. (Contributed by Skip Montanaro.) -* :mod:`codecs`, :mod:`encodings`, :mod:`unicodedata`: Added as part of the new +* :mod:`codecs`, :mod:`!encodings`, :mod:`unicodedata`: Added as part of the new Unicode support. -* :mod:`filecmp`: Supersedes the old :mod:`cmp`, :mod:`cmpcache` and - :mod:`dircmp` modules, which have now become deprecated. (Contributed by Gordon +* :mod:`filecmp`: Supersedes the old :mod:`!cmp`, :mod:`!cmpcache` and + :mod:`!dircmp` modules, which have now become deprecated. (Contributed by Gordon MacMillan and Moshe Zadka.) * :mod:`gettext`: This module provides internationalization (I18N) and @@ -1105,7 +1106,7 @@ module. be passed to functions that expect ordinary strings, such as the :mod:`re` module. (Contributed by Sam Rushing, with some extensions by A.M. Kuchling.) -* :mod:`pyexpat`: An interface to the Expat XML parser. (Contributed by Paul +* :mod:`!pyexpat`: An interface to the Expat XML parser. (Contributed by Paul Prescod.) * :mod:`robotparser `: Parse a :file:`robots.txt` file, which is used for writing @@ -1117,7 +1118,7 @@ module. * :mod:`tabnanny`: A module/script to check Python source code for ambiguous indentation. (Contributed by Tim Peters.) -* :mod:`UserString`: A base class useful for deriving objects that behave like +* :mod:`!UserString`: A base class useful for deriving objects that behave like strings. * :mod:`webbrowser`: A module that provides a platform independent way to launch @@ -1184,13 +1185,13 @@ Deleted and Deprecated Modules ============================== A few modules have been dropped because they're obsolete, or because there are -now better ways to do the same thing. The :mod:`stdwin` module is gone; it was +now better ways to do the same thing. The :mod:`!stdwin` module is gone; it was for a platform-independent windowing toolkit that's no longer developed. A number of modules have been moved to the :file:`lib-old` subdirectory: -:mod:`cmp`, :mod:`cmpcache`, :mod:`dircmp`, :mod:`dump`, :mod:`find`, -:mod:`grep`, :mod:`packmail`, :mod:`poly`, :mod:`util`, :mod:`whatsound`, -:mod:`zmod`. If you have code which relies on a module that's been moved to +:mod:`!cmp`, :mod:`!cmpcache`, :mod:`!dircmp`, :mod:`!dump`, :mod:`!find`, +:mod:`!grep`, :mod:`!packmail`, :mod:`!poly`, :mod:`!util`, :mod:`!whatsound`, +:mod:`!zmod`. If you have code which relies on a module that's been moved to :file:`lib-old`, you can simply add that directory to ``sys.path`` to get them back, but you're encouraged to update any code that uses these modules. diff --git a/Doc/whatsnew/2.1.rst b/Doc/whatsnew/2.1.rst index 6d2d3cc02b8768..b4002f06e92adc 100644 --- a/Doc/whatsnew/2.1.rst +++ b/Doc/whatsnew/2.1.rst @@ -48,7 +48,7 @@ nested recursive function definition doesn't work:: return g(value-1) + 1 ... -The function :func:`g` will always raise a :exc:`NameError` exception, because +The function :func:`!g` will always raise a :exc:`NameError` exception, because the binding of the name ``g`` isn't in either its local namespace or in the module-level namespace. This isn't much of a problem in practice (how often do you recursively define interior functions like this?), but this also made using @@ -104,7 +104,7 @@ To make the preceding explanation a bit clearer, here's an example:: Line 4 containing the ``exec`` statement is a syntax error, since ``exec`` would define a new local variable named ``x`` whose value should -be accessed by :func:`g`. +be accessed by :func:`!g`. This shouldn't be much of a limitation, since ``exec`` is rarely used in most Python code (and when it is used, it's often a sign of a poor design @@ -161,7 +161,7 @@ PEP 207: Rich Comparisons In earlier versions, Python's support for implementing comparisons on user-defined classes and extension types was quite simple. Classes could implement a -:meth:`__cmp__` method that was given two instances of a class, and could only +:meth:`!__cmp__` method that was given two instances of a class, and could only return 0 if they were equal or +1 or -1 if they weren't; the method couldn't raise an exception or return anything other than a Boolean value. Users of Numeric Python often found this model too weak and restrictive, because in the @@ -175,21 +175,21 @@ In Python 2.1, rich comparisons were added in order to support this need. Python classes can now individually overload each of the ``<``, ``<=``, ``>``, ``>=``, ``==``, and ``!=`` operations. The new magic method names are: -+-----------+----------------+ -| Operation | Method name | -+===========+================+ -| ``<`` | :meth:`__lt__` | -+-----------+----------------+ -| ``<=`` | :meth:`__le__` | -+-----------+----------------+ -| ``>`` | :meth:`__gt__` | -+-----------+----------------+ -| ``>=`` | :meth:`__ge__` | -+-----------+----------------+ -| ``==`` | :meth:`__eq__` | -+-----------+----------------+ -| ``!=`` | :meth:`__ne__` | -+-----------+----------------+ ++-----------+------------------------+ +| Operation | Method name | ++===========+========================+ +| ``<`` | :meth:`~object.__lt__` | ++-----------+------------------------+ +| ``<=`` | :meth:`~object.__le__` | ++-----------+------------------------+ +| ``>`` | :meth:`~object.__gt__` | ++-----------+------------------------+ +| ``>=`` | :meth:`~object.__ge__` | ++-----------+------------------------+ +| ``==`` | :meth:`~object.__eq__` | ++-----------+------------------------+ +| ``!=`` | :meth:`~object.__ne__` | ++-----------+------------------------+ (The magic methods are named after the corresponding Fortran operators ``.LT.``. ``.LE.``, &c. Numeric programmers are almost certainly quite familiar with @@ -208,7 +208,7 @@ The built-in ``cmp(A,B)`` function can use the rich comparison machinery, and now accepts an optional argument specifying which comparison operation to use; this is given as one of the strings ``"<"``, ``"<="``, ``">"``, ``">="``, ``"=="``, or ``"!="``. If called without the optional third argument, -:func:`cmp` will only return -1, 0, or +1 as in previous versions of Python; +:func:`!cmp` will only return -1, 0, or +1 as in previous versions of Python; otherwise it will call the appropriate method and can return any Python object. There are also corresponding changes of interest to C programmers; there's a new @@ -245,7 +245,7 @@ out warnings that you don't want to be displayed. Third-party modules can also use this framework to deprecate old features that they no longer wish to support. -For example, in Python 2.1 the :mod:`regex` module is deprecated, so importing +For example, in Python 2.1 the :mod:`!regex` module is deprecated, so importing it causes a warning to be printed:: >>> import regex @@ -262,7 +262,7 @@ can be used to specify a particular warning category. Filters can be added to disable certain warnings; a regular expression pattern can be applied to the message or to the module name in order to suppress a -warning. For example, you may have a program that uses the :mod:`regex` module +warning. For example, you may have a program that uses the :mod:`!regex` module and not want to spare the time to convert it to use the :mod:`re` module right now. The warning can be suppressed by calling :: @@ -274,7 +274,7 @@ now. The warning can be suppressed by calling :: This adds a filter that will apply only to warnings of the class :class:`DeprecationWarning` triggered in the :mod:`__main__` module, and applies -a regular expression to only match the message about the :mod:`regex` module +a regular expression to only match the message about the :mod:`!regex` module being deprecated, and will cause such warnings to be ignored. Warnings can also be printed only once, printed every time the offending code is executed, or turned into exceptions that will cause the program to stop (unless the @@ -368,7 +368,7 @@ dictionary:: This version works for simple things such as integers, but it has a side effect; the ``_cache`` dictionary holds a reference to the return values, so they'll never be deallocated until the Python process exits and cleans up. This isn't -very noticeable for integers, but if :func:`f` returns an object, or a data +very noticeable for integers, but if :func:`!f` returns an object, or a data structure that takes up a lot of memory, this can be a problem. Weak references provide a way to implement a cache that won't keep objects alive @@ -379,7 +379,7 @@ created by calling ``wr = weakref.ref(obj)``. The object being referred to is returned by calling the weak reference as if it were a function: ``wr()``. It will return the referenced object, or ``None`` if the object no longer exists. -This makes it possible to write a :func:`memoize` function whose cache doesn't +This makes it possible to write a :func:`!memoize` function whose cache doesn't keep objects alive, by storing weak references in the cache. :: _cache = {} @@ -402,7 +402,7 @@ weak references --- an object referenced only by proxy objects is deallocated -- but instead of requiring an explicit call to retrieve the object, the proxy transparently forwards all operations to the object as long as the object still exists. If the object is deallocated, attempting to use a proxy will cause a -:exc:`weakref.ReferenceError` exception to be raised. :: +:exc:`!weakref.ReferenceError` exception to be raised. :: proxy = weakref.proxy(obj) proxy.attr # Equivalent to obj.attr @@ -446,7 +446,7 @@ The dictionary containing attributes can be accessed as the function's :attr:`~object.__dict__`. Unlike the :attr:`~object.__dict__` attribute of class instances, in functions you can actually assign a new dictionary to :attr:`~object.__dict__`, though the new value is restricted to a regular Python dictionary; you *can't* be -tricky and set it to a :class:`UserDict` instance, or any other random object +tricky and set it to a :class:`!UserDict` instance, or any other random object that behaves like a mapping. @@ -584,11 +584,11 @@ available from the Distutils SIG at https://www.python.org/community/sigs/curren New and Improved Modules ======================== -* Ka-Ping Yee contributed two new modules: :mod:`inspect.py`, a module for - getting information about live Python code, and :mod:`pydoc.py`, a module for +* Ka-Ping Yee contributed two new modules: :mod:`!inspect.py`, a module for + getting information about live Python code, and :mod:`!pydoc.py`, a module for interactively converting docstrings to HTML or text. As a bonus, :file:`Tools/scripts/pydoc`, which is now automatically installed, uses - :mod:`pydoc.py` to display documentation given a Python module, package, or + :mod:`!pydoc.py` to display documentation given a Python module, package, or class name. For example, ``pydoc xml.dom`` displays the following:: Python Library Documentation: package xml.dom in xml @@ -617,7 +617,7 @@ New and Improved Modules Kent Beck's Smalltalk testing framework. See https://pyunit.sourceforge.net/ for more information about PyUnit. -* The :mod:`difflib` module contains a class, :class:`SequenceMatcher`, which +* The :mod:`difflib` module contains a class, :class:`~difflib.SequenceMatcher`, which compares two sequences and computes the changes required to transform one sequence into the other. For example, this module can be used to write a tool similar to the Unix :program:`diff` program, and in fact the sample program @@ -633,7 +633,7 @@ New and Improved Modules 2.1 includes an updated version of the :mod:`xml` package. Some of the noteworthy changes include support for Expat 1.2 and later versions, the ability for Expat parsers to handle files in any encoding supported by Python, and - various bugfixes for SAX, DOM, and the :mod:`minidom` module. + various bugfixes for SAX, DOM, and the :mod:`!minidom` module. * Ping also contributed another hook for handling uncaught exceptions. :func:`sys.excepthook` can be set to a callable object. When an exception isn't @@ -643,8 +643,8 @@ New and Improved Modules printing an extended traceback that not only lists the stack frames, but also lists the function arguments and the local variables for each frame. -* Various functions in the :mod:`time` module, such as :func:`asctime` and - :func:`localtime`, require a floating point argument containing the time in +* Various functions in the :mod:`time` module, such as :func:`~time.asctime` and + :func:`~time.localtime`, require a floating point argument containing the time in seconds since the epoch. The most common use of these functions is to work with the current time, so the floating point argument has been made optional; when a value isn't provided, the current time will be used. For example, log file @@ -724,10 +724,10 @@ of the more notable changes are: a discussion in comp.lang.python. A new module and method for file objects was also added, contributed by Jeff - Epler. The new method, :meth:`xreadlines`, is similar to the existing - :func:`xrange` built-in. :func:`xreadlines` returns an opaque sequence object + Epler. The new method, :meth:`!xreadlines`, is similar to the existing + :func:`!xrange` built-in. :func:`!xreadlines` returns an opaque sequence object that only supports being iterated over, reading a line on every iteration but - not reading the entire file into memory as the existing :meth:`readlines` method + not reading the entire file into memory as the existing :meth:`!readlines` method does. You'd use it like this:: for line in sys.stdin.xreadlines(): @@ -737,7 +737,7 @@ of the more notable changes are: For a fuller discussion of the line I/O changes, see the python-dev summary for January 1--15, 2001 at https://mail.python.org/pipermail/python-dev/2001-January/. -* A new method, :meth:`popitem`, was added to dictionaries to enable +* A new method, :meth:`~dict.popitem`, was added to dictionaries to enable destructively iterating through the contents of a dictionary; this can be faster for large dictionaries because there's no need to construct a list containing all the keys or values. ``D.popitem()`` removes a random ``(key, value)`` pair diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 2a42664c02852c..5c99fbc503ba65 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1130,7 +1130,7 @@ changes, or look through the Subversion logs for all the details. (Added by Raymond Hettinger; :issue:`1818`.) Finally, the :class:`~collections.abc.Mapping` abstract base class now - returns :const:`NotImplemented` if a mapping is compared to + returns :data:`NotImplemented` if a mapping is compared to another type that isn't a :class:`Mapping`. (Fixed by Daniel Stutzbach; :issue:`8729`.) diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index b7dd8f2c7bf531..69b273e58385d2 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -81,7 +81,7 @@ Support was also added for third-party tools like `PyYAML ` written by Raymond Hettinger. Since an ordered dictionary remembers its insertion order, it can be used -in conjuction with sorting to make a sorted dictionary:: +in conjunction with sorting to make a sorted dictionary:: >>> # regular unsorted dictionary >>> d = {'banana': 3, 'apple':4, 'pear': 1, 'orange': 2} @@ -174,7 +174,7 @@ Some smaller changes made to the core Python language are: (Contributed by Eric Smith; :issue:`5237`.) -* The :func:`string.maketrans` function is deprecated and is replaced by new +* The :func:`!string.maketrans` function is deprecated and is replaced by new static methods, :meth:`bytes.maketrans` and :meth:`bytearray.maketrans`. This change solves the confusion around which types were supported by the :mod:`string` module. Now, :class:`str`, :class:`bytes`, and @@ -381,16 +381,20 @@ New, Improved, and Deprecated Modules x / 0 In addition, several new assertion methods were added including - :func:`assertSetEqual`, :func:`assertDictEqual`, - :func:`assertDictContainsSubset`, :func:`assertListEqual`, - :func:`assertTupleEqual`, :func:`assertSequenceEqual`, - :func:`assertRaisesRegexp`, :func:`assertIsNone`, - and :func:`assertIsNotNone`. + :meth:`~unittest.TestCase.assertSetEqual`, + :meth:`~unittest.TestCase.assertDictEqual`, + :meth:`!assertDictContainsSubset`, + :meth:`~unittest.TestCase.assertListEqual`, + :meth:`~unittest.TestCase.assertTupleEqual`, + :meth:`~unittest.TestCase.assertSequenceEqual`, + :meth:`assertRaisesRegexp() `, + :meth:`~unittest.TestCase.assertIsNone`, + and :meth:`~unittest.TestCase.assertIsNotNone`. (Contributed by Benjamin Peterson and Antoine Pitrou.) -* The :mod:`io` module has three new constants for the :meth:`seek` - method :data:`SEEK_SET`, :data:`SEEK_CUR`, and :data:`SEEK_END`. +* The :mod:`io` module has three new constants for the :meth:`~io.IOBase.seek` + method: :data:`~os.SEEK_SET`, :data:`~os.SEEK_CUR`, and :data:`~os.SEEK_END`. * The :data:`sys.version_info` tuple is now a named tuple:: diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 9770b12c8af711..e35179a2d8e513 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -828,7 +828,7 @@ Other Language Changes :meth:`~object.__index__` method). (Contributed by Serhiy Storchaka in :issue:`37999`.) -* If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator will +* If :func:`object.__ipow__` returns :data:`NotImplemented`, the operator will correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. (Contributed by Alex Shkop in :issue:`38302`.) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 0c977a2fa8fc86..e468bc67de5f63 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -173,6 +173,17 @@ Other Language Changes (Contributed by Victor Stinner in :gh:`114570`.) +* Allow controlling Expat >=2.6.0 reparse deferral (CVE-2023-52425) + by adding five new methods: + + * :meth:`xml.etree.ElementTree.XMLParser.flush` + * :meth:`xml.etree.ElementTree.XMLPullParser.flush` + * :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` + * :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` + * :meth:`!xml.sax.expatreader.ExpatParser.flush` + + (Contributed by Sebastian Pipping in :gh:`115623`.) + * The :func:`ssl.create_default_context` API now includes :data:`ssl.VERIFY_X509_PARTIAL_CHAIN` and :data:`ssl.VERIFY_X509_STRICT` in its default flags. @@ -219,6 +230,21 @@ array ast --- +* The constructors of node types in the :mod:`ast` module are now stricter + in the arguments they accept, and have more intuitive behaviour when + arguments are omitted. + + If an optional field on an AST node is not included as an argument when + constructing an instance, the field will now be set to ``None``. Similarly, + if a list field is omitted, that field will now be set to an empty list. + (Previously, in both cases, the attribute would be missing on the newly + constructed AST node instance.) + + If other arguments are omitted, a :exc:`DeprecationWarning` is emitted. + This will cause an exception in Python 3.15. Similarly, passing a keyword + argument that does not map to a field on the AST node is now deprecated, + and will raise an exception in Python 3.15. + * :func:`ast.parse` now accepts an optional argument ``optimize`` which is passed on to the :func:`compile` built-in. This makes it possible to obtain an optimized ``AST``. @@ -231,6 +257,20 @@ asyncio the Unix socket when the server is closed. (Contributed by Pierre Ossman in :gh:`111246`.) +* :meth:`asyncio.DatagramTransport.sendto` will now send zero-length + datagrams if called with an empty bytes object. The transport flow + control also now accounts for the datagram header when calculating + the buffer size. + (Contributed by Jamie Phan in :gh:`115199`.) + +base64 +--- + +* Add :func:`base64.z85encode` and :func:`base64.z85decode` functions which allow encoding + and decoding z85 data. + See `Z85 specification `_ for more information. + (Contributed by Matan Perelman in :gh:`75299`.) + copy ---- @@ -320,6 +360,14 @@ ipaddress * Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which returns the IPv4-mapped IPv6 address. (Contributed by Charles Machalow in :gh:`109466`.) +itertools +--------- + +* Added a ``strict`` option to :func:`itertools.batched`. + This raises a :exc:`ValueError` if the final batch is shorter + than the specified batch size. + (Contributed by Raymond Hettinger in :gh:`113202`.) + marshal ------- @@ -467,6 +515,14 @@ sqlite3 for filtering database objects to dump. (Contributed by Mariusz Felisiak in :gh:`91602`.) +statistics +---------- + +* Add :func:`statistics.kde` for kernel density estimation. + This makes it possible to estimate a continuous probability density function + from a fixed number of discrete samples. + (Contributed by Raymond Hettinger in :gh:`115863`.) + subprocess ---------- @@ -748,6 +804,9 @@ Deprecated coroutine. (Contributed by Irit Katriel in :gh:`81137`.) +* The undocumented and unused ``tarfile`` attribute of :class:`tarfile.TarFile` + is deprecated and scheduled for removal in Python 3.16. + Pending Removal in Python 3.14 ------------------------------ diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 4f70d902243d4d..52474517f5facc 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -344,8 +344,8 @@ aspects that are visible to the programmer: * The :mod:`importlib.abc` module has been updated with new :term:`abstract base classes ` for loading bytecode files. The obsolete - ABCs, :class:`~importlib.abc.PyLoader` and - :class:`~importlib.abc.PyPycLoader`, have been deprecated (instructions on how + ABCs, :class:`!PyLoader` and + :class:`!PyPycLoader`, have been deprecated (instructions on how to stay Python 3.1 compatible are included with the documentation). .. seealso:: @@ -401,7 +401,7 @@ The *native strings* are always of type :class:`str` but are restricted to code points between *U+0000* through *U+00FF* which are translatable to bytes using *Latin-1* encoding. These strings are used for the keys and values in the environment dictionary and for response headers and statuses in the -:func:`start_response` function. They must follow :rfc:`2616` with respect to +:func:`!start_response` function. They must follow :rfc:`2616` with respect to encoding. That is, they must either be *ISO-8859-1* characters or use :rfc:`2047` MIME encoding. @@ -415,8 +415,8 @@ points: encoded in utf-8 was using ``h.encode('utf-8')`` now needs to convert from bytes to native strings using ``h.encode('utf-8').decode('latin-1')``. -* Values yielded by an application or sent using the :meth:`write` method - must be byte strings. The :func:`start_response` function and environ +* Values yielded by an application or sent using the :meth:`!write` method + must be byte strings. The :func:`!start_response` function and environ must use native strings. The two cannot be mixed. For server implementers writing CGI-to-WSGI pathways or other CGI-style @@ -499,7 +499,7 @@ Some smaller changes made to the core Python language are: * The :func:`hasattr` function works by calling :func:`getattr` and detecting whether an exception is raised. This technique allows it to detect methods - created dynamically by :meth:`__getattr__` or :meth:`__getattribute__` which + created dynamically by :meth:`~object.__getattr__` or :meth:`~object.__getattribute__` which would otherwise be absent from the class dictionary. Formerly, *hasattr* would catch any exception, possibly masking genuine errors. Now, *hasattr* has been tightened to only catch :exc:`AttributeError` and let other @@ -620,7 +620,7 @@ Some smaller changes made to the core Python language are: * :class:`range` objects now support *index* and *count* methods. This is part of an effort to make more objects fully implement the - :class:`collections.Sequence` :term:`abstract base class`. As a result, the + :class:`collections.Sequence ` :term:`abstract base class`. As a result, the language will have a more uniform API. In addition, :class:`range` objects now support slicing and negative indices, even with values larger than :data:`sys.maxsize`. This makes *range* more interoperable with lists:: @@ -720,7 +720,7 @@ format. elementtree ----------- -The :mod:`xml.etree.ElementTree` package and its :mod:`xml.etree.cElementTree` +The :mod:`xml.etree.ElementTree` package and its :mod:`!xml.etree.cElementTree` counterpart have been updated to version 1.3. Several new and useful functions and methods have been added: @@ -1008,13 +1008,13 @@ datetime and time after 1900. The new supported year range is from 1000 to 9999 inclusive. * Whenever a two-digit year is used in a time tuple, the interpretation has been - governed by :data:`time.accept2dyear`. The default is ``True`` which means that + governed by :data:`!time.accept2dyear`. The default is ``True`` which means that for a two-digit year, the century is guessed according to the POSIX rules governing the ``%y`` strptime format. Starting with Py3.2, use of the century guessing heuristic will emit a :exc:`DeprecationWarning`. Instead, it is recommended that - :data:`time.accept2dyear` be set to ``False`` so that large date ranges + :data:`!time.accept2dyear` be set to ``False`` so that large date ranges can be used without guesswork:: >>> import time, warnings @@ -1032,7 +1032,7 @@ datetime and time 'Fri Jan 1 12:34:56 11' Several functions now have significantly expanded date ranges. When - :data:`time.accept2dyear` is false, the :func:`time.asctime` function will + :data:`!time.accept2dyear` is false, the :func:`time.asctime` function will accept any year that fits in a C int, while the :func:`time.mktime` and :func:`time.strftime` functions will accept the full range supported by the corresponding operating system functions. @@ -1148,15 +1148,15 @@ for slice notation are well-suited to in-place editing:: reprlib ------- -When writing a :meth:`__repr__` method for a custom container, it is easy to +When writing a :meth:`~object.__repr__` method for a custom container, it is easy to forget to handle the case where a member refers back to the container itself. Python's builtin objects such as :class:`list` and :class:`set` handle self-reference by displaying "..." in the recursive part of the representation string. -To help write such :meth:`__repr__` methods, the :mod:`reprlib` module has a new +To help write such :meth:`~object.__repr__` methods, the :mod:`reprlib` module has a new decorator, :func:`~reprlib.recursive_repr`, for detecting recursive calls to -:meth:`__repr__` and substituting a placeholder string instead:: +:meth:`!__repr__` and substituting a placeholder string instead:: >>> class MyList(list): ... @recursive_repr() @@ -1308,7 +1308,7 @@ used for the imaginary part of a number: >>> sys.hash_info # doctest: +SKIP sys.hash_info(width=64, modulus=2305843009213693951, inf=314159, nan=0, imag=1000003) -An early decision to limit the inter-operability of various numeric types has +An early decision to limit the interoperability of various numeric types has been relaxed. It is still unsupported (and ill-advised) to have implicit mixing in arithmetic expressions such as ``Decimal('1.1') + float('1.1')`` because the latter loses information in the process of constructing the binary @@ -1336,7 +1336,7 @@ Decimal('1.100000000000000088817841970012523233890533447265625') Fraction(2476979795053773, 2251799813685248) Another useful change for the :mod:`decimal` module is that the -:attr:`Context.clamp` attribute is now public. This is useful in creating +:attr:`Context.clamp ` attribute is now public. This is useful in creating contexts that correspond to the decimal interchange formats specified in IEEE 754 (see :issue:`8540`). @@ -1428,7 +1428,7 @@ before compressing and decompressing: Aides and Brian Curtin in :issue:`9962`, :issue:`1675951`, :issue:`7471` and :issue:`2846`.) -Also, the :class:`zipfile.ZipExtFile` class was reworked internally to represent +Also, the :class:`zipfile.ZipExtFile ` class was reworked internally to represent files stored inside an archive. The new implementation is significantly faster and can be wrapped in an :class:`io.BufferedReader` object for more speedups. It also solves an issue where interleaved calls to *read* and *readline* gave the @@ -1596,7 +1596,7 @@ sqlite3 The :mod:`sqlite3` module was updated to pysqlite version 2.6.0. It has two new capabilities. -* The :attr:`sqlite3.Connection.in_transit` attribute is true if there is an +* The :attr:`!sqlite3.Connection.in_transit` attribute is true if there is an active transaction for uncommitted changes. * The :meth:`sqlite3.Connection.enable_load_extension` and @@ -1643,11 +1643,11 @@ for secure (encrypted, authenticated) internet connections: other options. It includes a :meth:`~ssl.SSLContext.wrap_socket` for creating an SSL socket from an SSL context. -* A new function, :func:`ssl.match_hostname`, supports server identity +* A new function, :func:`!ssl.match_hostname`, supports server identity verification for higher-level protocols by implementing the rules of HTTPS (from :rfc:`2818`) which are also suitable for other protocols. -* The :func:`ssl.wrap_socket` constructor function now takes a *ciphers* +* The :func:`ssl.wrap_socket() ` constructor function now takes a *ciphers* argument. The *ciphers* string lists the allowed encryption algorithms using the format described in the `OpenSSL documentation `__. @@ -1759,7 +1759,7 @@ names. (Contributed by Michael Foord.) * Experimentation at the interactive prompt is now easier because the - :class:`unittest.case.TestCase` class can now be instantiated without + :class:`unittest.TestCase` class can now be instantiated without arguments: >>> from unittest import TestCase @@ -1797,7 +1797,7 @@ names. * In addition, the method names in the module have undergone a number of clean-ups. For example, :meth:`~unittest.TestCase.assertRegex` is the new name for - :meth:`~unittest.TestCase.assertRegexpMatches` which was misnamed because the + :meth:`!assertRegexpMatches` which was misnamed because the test uses :func:`re.search`, not :func:`re.match`. Other methods using regular expressions are now named using short form "Regex" in preference to "Regexp" -- this matches the names used in other unittest implementations, @@ -1812,11 +1812,11 @@ names. =============================== ============================== Old Name Preferred Name =============================== ============================== - :meth:`assert_` :meth:`.assertTrue` - :meth:`assertEquals` :meth:`.assertEqual` - :meth:`assertNotEquals` :meth:`.assertNotEqual` - :meth:`assertAlmostEquals` :meth:`.assertAlmostEqual` - :meth:`assertNotAlmostEquals` :meth:`.assertNotAlmostEqual` + :meth:`!assert_` :meth:`.assertTrue` + :meth:`!assertEquals` :meth:`.assertEqual` + :meth:`!assertNotEquals` :meth:`.assertNotEqual` + :meth:`!assertAlmostEquals` :meth:`.assertAlmostEqual` + :meth:`!assertNotAlmostEquals` :meth:`.assertNotAlmostEqual` =============================== ============================== Likewise, the ``TestCase.fail*`` methods deprecated in Python 3.1 are expected @@ -1824,7 +1824,7 @@ names. (Contributed by Ezio Melotti; :issue:`9424`.) -* The :meth:`~unittest.TestCase.assertDictContainsSubset` method was deprecated +* The :meth:`!assertDictContainsSubset` method was deprecated because it was misimplemented with the arguments in the wrong order. This created hard-to-debug optical illusions where tests like ``TestCase().assertDictContainsSubset({'a':1, 'b':2}, {'a':1})`` would fail. @@ -1997,7 +1997,7 @@ under-the-hood. dbm --- -All database modules now support the :meth:`get` and :meth:`setdefault` methods. +All database modules now support the :meth:`!get` and :meth:`!setdefault` methods. (Suggested by Ray Allen in :issue:`9523`.) @@ -2118,7 +2118,7 @@ The :mod:`pdb` debugger module gained a number of usability improvements: :file:`.pdbrc` script file. * A :file:`.pdbrc` script file can contain ``continue`` and ``next`` commands that continue debugging. -* The :class:`Pdb` class constructor now accepts a *nosigint* argument. +* The :class:`~pdb.Pdb` class constructor now accepts a *nosigint* argument. * New commands: ``l(list)``, ``ll(long list)`` and ``source`` for listing source code. * New commands: ``display`` and ``undisplay`` for showing or hiding @@ -2394,11 +2394,11 @@ A number of small performance enhancements have been added: (Contributed by Antoine Pitrou; :issue:`3001`.) -* The fast-search algorithm in stringlib is now used by the :meth:`split`, - :meth:`rsplit`, :meth:`splitlines` and :meth:`replace` methods on +* The fast-search algorithm in stringlib is now used by the :meth:`~str.split`, + :meth:`~str.rsplit`, :meth:`~str.splitlines` and :meth:`~str.replace` methods on :class:`bytes`, :class:`bytearray` and :class:`str` objects. Likewise, the - algorithm is also used by :meth:`rfind`, :meth:`rindex`, :meth:`rsplit` and - :meth:`rpartition`. + algorithm is also used by :meth:`~str.rfind`, :meth:`~str.rindex`, :meth:`~str.rsplit` and + :meth:`~str.rpartition`. (Patch by Florent Xicluna in :issue:`7622` and :issue:`7462`.) @@ -2410,8 +2410,8 @@ A number of small performance enhancements have been added: There were several other minor optimizations. Set differencing now runs faster when one operand is much larger than the other (patch by Andress Bennetts in -:issue:`8685`). The :meth:`array.repeat` method has a faster implementation -(:issue:`1569291` by Alexander Belopolsky). The :class:`BaseHTTPRequestHandler` +:issue:`8685`). The :meth:`!array.repeat` method has a faster implementation +(:issue:`1569291` by Alexander Belopolsky). The :class:`~http.server.BaseHTTPRequestHandler` has more efficient buffering (:issue:`3709` by Andrew Schaaf). The :func:`operator.attrgetter` function has been sped-up (:issue:`10160` by Christos Georgiou). And :class:`~configparser.ConfigParser` loads multi-line arguments a bit @@ -2562,11 +2562,11 @@ Changes to Python's build process and to the C API include: (Suggested by Raymond Hettinger and implemented by Benjamin Peterson; :issue:`9778`.) -* A new macro :c:macro:`Py_VA_COPY` copies the state of the variable argument +* A new macro :c:macro:`!Py_VA_COPY` copies the state of the variable argument list. It is equivalent to C99 *va_copy* but available on all Python platforms (:issue:`2443`). -* A new C API function :c:func:`PySys_SetArgvEx` allows an embedded interpreter +* A new C API function :c:func:`!PySys_SetArgvEx` allows an embedded interpreter to set :data:`sys.argv` without also modifying :data:`sys.path` (:issue:`5753`). @@ -2650,8 +2650,9 @@ require changes to your code: * :class:`bytearray` objects can no longer be used as filenames; instead, they should be converted to :class:`bytes`. -* The :meth:`array.tostring` and :meth:`array.fromstring` have been renamed to - :meth:`array.tobytes` and :meth:`array.frombytes` for clarity. The old names +* The :meth:`!array.tostring` and :meth:`!array.fromstring` have been renamed to + :meth:`array.tobytes() ` and + :meth:`array.frombytes() ` for clarity. The old names have been deprecated. (See :issue:`8990`.) * ``PyArg_Parse*()`` functions: @@ -2664,7 +2665,7 @@ require changes to your code: instead; the new type has a well-defined interface for passing typing safety information and a less complicated signature for calling a destructor. -* The :func:`sys.setfilesystemencoding` function was removed because +* The :func:`!sys.setfilesystemencoding` function was removed because it had a flawed design. * The :func:`random.seed` function and method now salt string seeds with an @@ -2672,7 +2673,7 @@ require changes to your code: reproduce Python 3.1 sequences, set the *version* argument to *1*, ``random.seed(s, version=1)``. -* The previously deprecated :func:`string.maketrans` function has been removed +* The previously deprecated :func:`!string.maketrans` function has been removed in favor of the static methods :meth:`bytes.maketrans` and :meth:`bytearray.maketrans`. This change solves the confusion around which types were supported by the :mod:`string` module. Now, :class:`str`, diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index e07eda985d9bad..3dd400c3771ed2 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -872,7 +872,7 @@ multiple implementations of an operation that allows it to work with PEP written and implemented by Łukasz Langa. :func:`~functools.total_ordering` now supports a return value of -:const:`NotImplemented` from the underlying comparison function. (Contributed +:data:`NotImplemented` from the underlying comparison function. (Contributed by Katie Miller in :issue:`10042`.) A pure-python version of the :func:`~functools.partial` function is now in the diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index b24c13813be220..49d926b0edcd0f 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -1126,7 +1126,7 @@ Changes in the Python API ``logging.getLogger(__name__)`` in some top-level module called ``'root.py'``. (Contributed by Vinay Sajip in :issue:`37742`.) -* Division handling of :class:`~pathlib.PurePath` now returns ``NotImplemented`` +* Division handling of :class:`~pathlib.PurePath` now returns :data:`NotImplemented` instead of raising a :exc:`TypeError` when passed something other than an instance of ``str`` or :class:`~pathlib.PurePath`. This allows creating compatible classes that don't inherit from those mentioned types. diff --git a/Grammar/python.gram b/Grammar/python.gram index 174b4dbb6f7842..797c195a0a91ba 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -393,7 +393,7 @@ for_stmt[stmt_ty]: with_stmt[stmt_ty]: | invalid_with_stmt_indent | 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' tc=[TYPE_COMMENT] b=block { - CHECK_VERSION(stmt_ty, 9, "Parenthesized context managers are", _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) } + _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | 'async' 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block { diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index f5ccbb704e8bb8..f037c7bb90deda 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -62,21 +62,32 @@ typedef long stwodigits; /* signed variant of twodigits */ #define PyLong_MASK ((digit)(PyLong_BASE - 1)) /* Long integer representation. + + Long integers are made up of a number of 30- or 15-bit digits, depending on + the platform. The number of digits (ndigits) is stored in the high bits of + the lv_tag field (lvtag >> _PyLong_NON_SIZE_BITS). + The absolute value of a number is equal to - SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) - Negative numbers are represented with ob_size < 0; - zero is represented by ob_size == 0. - In a normalized number, ob_digit[abs(ob_size)-1] (the most significant + SUM(for i=0 through ndigits-1) ob_digit[i] * 2**(PyLong_SHIFT*i) + + The sign of the value is stored in the lower 2 bits of lv_tag. + + - 0: Positive + - 1: Zero + - 2: Negative + + The third lowest bit of lv_tag is reserved for an immortality flag, but is + not currently used. + + In a normalized number, ob_digit[ndigits-1] (the most significant digit) is never zero. Also, in all cases, for all valid i, - 0 <= ob_digit[i] <= MASK. + 0 <= ob_digit[i] <= PyLong_MASK. + The allocation function takes care of allocating extra memory - so that ob_digit[0] ... ob_digit[abs(ob_size)-1] are actually available. + so that ob_digit[0] ... ob_digit[ndigits-1] are actually available. We always allocate memory for at least one digit, so accessing ob_digit[0] - is always safe. However, in the case ob_size == 0, the contents of + is always safe. However, in the case ndigits == 0, the contents of ob_digit[0] may be undefined. - - CAUTION: Generic code manipulating subtypes of PyVarObject has to - aware that ints abuse ob_size's sign bit. */ typedef struct _PyLongValue { diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index 58a30aeea6ac64..e0c2ce286f13ce 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -85,3 +85,20 @@ PyAPI_FUNC(PyObject **) PyObject_GET_WEAKREFS_LISTPTR(PyObject *op); PyAPI_FUNC(PyObject *) PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *, size_t); + + +/* Visit all live GC-capable objects, similar to gc.get_objects(None). The + * supplied callback is called on every such object with the void* arg set + * to the supplied arg. Returning 0 from the callback ends iteration, returning + * 1 allows iteration to continue. Returning any other value may result in + * undefined behaviour. + * + * If new objects are (de)allocated by the callback it is undefined if they + * will be visited. + + * Garbage collection is disabled during operation. Explicitly running a + * collection in the callback may lead to undefined behaviour e.g. visiting the + * same objects multiple times or not at all. + */ +typedef int (*gcvisitobjects_t)(PyObject*, void*); +PyAPI_FUNC(void) PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void* arg); diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index f710ca76b2ba24..6d7b8bc3c1433a 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -33,16 +33,28 @@ typedef struct { typedef struct { uint16_t opcode; uint16_t oparg; - uint32_t target; + union { + uint32_t target; + uint32_t exit_index; + }; uint64_t operand; // A cache entry } _PyUOpInstruction; +typedef struct _exit_data { + uint32_t target; + int16_t temperature; + const struct _PyExecutorObject *executor; +} _PyExitData; + typedef struct _PyExecutorObject { PyObject_VAR_HEAD + const _PyUOpInstruction *trace; _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ - void *jit_code; + uint32_t exit_count; + uint32_t code_size; size_t jit_size; - _PyUOpInstruction trace[1]; + void *jit_code; + _PyExitData exits[1]; } _PyExecutorObject; typedef struct _PyOptimizerObject _PyOptimizerObject; @@ -59,6 +71,7 @@ typedef struct _PyOptimizerObject { /* These thresholds are treated as signed so do not exceed INT16_MAX * Use INT16_MAX to indicate that the optimizer should never be called */ uint16_t resume_threshold; + uint16_t side_threshold; uint16_t backedge_threshold; /* Data needed by the optimizer goes here, but is opaque to the VM */ } _PyOptimizerObject; @@ -73,22 +86,19 @@ PyAPI_FUNC(int) PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *in _PyOptimizerObject *_Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject* optimizer); -PyAPI_FUNC(void) PyUnstable_SetOptimizer(_PyOptimizerObject* optimizer); +PyAPI_FUNC(int) PyUnstable_SetOptimizer(_PyOptimizerObject* optimizer); PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void); PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset); -int -_PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer); - -void _Py_ExecutorInit(_PyExecutorObject *, _PyBloomFilter *); +void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); void _Py_ExecutorClear(_PyExecutorObject *); void _Py_BloomFilter_Init(_PyBloomFilter *); void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); -PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj); -extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp); +PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); +extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); /* For testing */ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index e10d48285367cf..c3e132d3877ca5 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -360,6 +360,8 @@ _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj); static inline void * _Py_atomic_load_ptr_relaxed(const void *obj); +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj); // --- _Py_atomic_store ------------------------------------------------------ // Atomically performs `*obj = value` (sequential consistency) @@ -452,6 +454,10 @@ _Py_atomic_store_ptr_relaxed(void *obj, void *value); static inline void _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value); +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value); + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -463,15 +469,27 @@ _Py_atomic_load_ptr_acquire(const void *obj); static inline void _Py_atomic_store_ptr_release(void *obj, void *value); +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); + static inline void _Py_atomic_store_int_release(int *obj, int value); static inline int _Py_atomic_load_int_acquire(const int *obj); +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value); + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj); + static inline uint32_t _Py_atomic_load_uint32_acquire(const uint32_t *obj); +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); + // --- _Py_atomic_fence ------------------------------------------------------ diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 4095e1873d8b07..0b40f81bd8736d 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -358,6 +358,10 @@ static inline void * _Py_atomic_load_ptr_relaxed(const void *obj) { return (void *)__atomic_load_n((const void **)obj, __ATOMIC_RELAXED); } +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + // --- _Py_atomic_store ------------------------------------------------------ @@ -476,6 +480,11 @@ static inline void _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELAXED); } +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -491,14 +500,30 @@ static inline void _Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline int _Py_atomic_load_int_acquire(const int *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + static inline uint32_t _Py_atomic_load_uint32_acquire(const uint32_t *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index b5c1ec94112562..3205e253b28546 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -712,6 +712,12 @@ _Py_atomic_load_ptr_relaxed(const void *obj) return *(void * volatile *)obj; } +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ + return *(volatile unsigned long long *)obj; +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -886,6 +892,14 @@ _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) *(volatile Py_ssize_t *)obj = value; } +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ + *(volatile unsigned long long *)obj = value; +} + + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ static inline void * @@ -925,6 +939,18 @@ _Py_atomic_store_int_release(int *obj, int value) #endif } +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(Py_ssize_t volatile *)obj = value; +#elif defined(_M_ARM64) + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_ssize_release" +#endif +} + static inline int _Py_atomic_load_int_acquire(const int *obj) { @@ -938,18 +964,56 @@ _Py_atomic_load_int_acquire(const int *obj) #endif } +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint64_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_uint64_release" +#endif +} + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uint64_t volatile *)obj; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint64_acquire" +#endif +} + static inline uint32_t _Py_atomic_load_uint32_acquire(const uint32_t *obj) { #if defined(_M_X64) || defined(_M_IX86) return *(uint32_t volatile *)obj; #elif defined(_M_ARM64) - return (int)__ldar32((uint32_t volatile *)obj); + return (uint32_t)__ldar32((uint32_t volatile *)obj); #else # error "no implementation of _Py_atomic_load_uint32_acquire" #endif } +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(Py_ssize_t volatile *)obj; +#elif defined(_M_ARM64) + return (Py_ssize_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_ssize_acquire" +#endif +} + // --- _Py_atomic_fence ------------------------------------------------------ static inline void diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 6c934a2c5e7b64..f3970a45df24f9 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -619,6 +619,14 @@ _Py_atomic_load_ptr_relaxed(const void *obj) memory_order_relaxed); } +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned long long)*)obj, + memory_order_relaxed); +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -835,6 +843,15 @@ _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) memory_order_relaxed); } +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned long long)*)obj, value, + memory_order_relaxed); +} + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -862,6 +879,14 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value, + memory_order_release); +} + static inline int _Py_atomic_load_int_acquire(const int *obj) { @@ -870,12 +895,34 @@ _Py_atomic_load_int_acquire(const int *obj) memory_order_acquire); } +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint64_t)*)obj, value, + memory_order_release); +} + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint64_t)*)obj, + memory_order_acquire); +} + static inline uint32_t _Py_atomic_load_uint32_acquire(const uint32_t *obj) { _Py_USING_STD; return atomic_load_explicit((const _Atomic(uint32_t)*)obj, - memory_order_acquire); +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj, } diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 9bc8758e72bd8f..ac7ff83748dbfc 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -68,6 +68,11 @@ struct _ts { PyThreadState *next; PyInterpreterState *interp; + /* The global instrumentation version in high bits, plus flags indicating + when to break out of the interpreter loop in lower bits. See details in + pycore_ceval.h. */ + uintptr_t eval_breaker; + struct { /* Has been initialized to a safe state. @@ -212,6 +217,8 @@ struct _ts { /* The thread's exception stack entry. (Always the last entry.) */ _PyErr_StackItem exc_state; + PyObject *previous_executor; + }; #ifdef Py_DEBUG diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index db9aaedec950e4..887fbbedf88502 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -115,6 +115,7 @@ typedef struct _optimization_stats { uint64_t inner_loop; uint64_t recursive_call; uint64_t low_confidence; + uint64_t executors_invalidated; UOpStats opcode[512]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index b8cae24507fc5e..ed93e45062c2db 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -312,6 +312,7 @@ typedef struct mi_page_s { uint8_t is_committed : 1; // `true` if the page virtual memory is committed uint8_t is_zero_init : 1; // `true` if the page was initially zero initialized uint8_t tag : 4; // tag from the owning heap + uint8_t debug_offset; // number of bytes to preserve when filling freed or uninitialized memory // layout like this to optimize access in `mi_malloc` and `mi_free` uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` @@ -553,6 +554,7 @@ struct mi_heap_s { mi_heap_t* next; // list of heaps per thread bool no_reclaim; // `true` if this heap should not reclaim abandoned pages uint8_t tag; // custom identifier for this heap + uint8_t debug_offset; // number of bytes to preserve when filling freed or uninitialized memory }; diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index b158fc9ff5ebc1..6eab2ba1daedf8 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -42,7 +42,7 @@ PyAPI_FUNC(int) _PyEval_MakePendingCalls(PyThreadState *); extern void _Py_FinishPendingCalls(PyThreadState *tstate); extern void _PyEval_InitState(PyInterpreterState *); -extern void _PyEval_SignalReceived(PyInterpreterState *interp); +extern void _PyEval_SignalReceived(void); // bitwise flags: #define _Py_PENDING_MAINTHREADONLY 1 @@ -55,7 +55,6 @@ PyAPI_FUNC(int) _PyEval_AddPendingCall( void *arg, int flags); -extern void _PyEval_SignalAsyncExc(PyInterpreterState *interp); #ifdef HAVE_FORK extern PyStatus _PyEval_ReInitThreads(PyThreadState *tstate); #endif @@ -182,58 +181,65 @@ extern PyObject* _Py_MakeCoro(PyFunctionObject *func); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ -extern int _Py_HandlePending(PyThreadState *tstate); +PyAPI_FUNC(int) _Py_HandlePending(PyThreadState *tstate); extern PyObject * _PyEval_GetFrameLocals(void); -extern const binaryfunc _PyEval_BinaryOps[]; -int _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right); -int _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right); -int _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest); -void _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg); -void _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); -void _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); -void _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); -PyObject *_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); -PyObject *_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); -int _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp); -void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); - - -#define _PY_GIL_DROP_REQUEST_BIT 0 -#define _PY_SIGNALS_PENDING_BIT 1 -#define _PY_CALLS_TO_DO_BIT 2 -#define _PY_ASYNC_EXCEPTION_BIT 3 -#define _PY_GC_SCHEDULED_BIT 4 -#define _PY_EVAL_PLEASE_STOP_BIT 5 -#define _PY_EVAL_EXPLICIT_MERGE_BIT 6 +typedef PyObject *(*conversion_func)(PyObject *); + +PyAPI_DATA(const binaryfunc) _PyEval_BinaryOps[]; +PyAPI_DATA(const conversion_func) _PyEval_ConversionFuncs[]; + +PyAPI_FUNC(int) _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right); +PyAPI_FUNC(int) _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right); +PyAPI_FUNC(int) _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest); +PyAPI_FUNC(void) _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg); +PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); +PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); +PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); +PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); +PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); +PyAPI_FUNC(int) _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp); +PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); + + +/* Bits that can be set in PyThreadState.eval_breaker */ +#define _PY_GIL_DROP_REQUEST_BIT (1U << 0) +#define _PY_SIGNALS_PENDING_BIT (1U << 1) +#define _PY_CALLS_TO_DO_BIT (1U << 2) +#define _PY_ASYNC_EXCEPTION_BIT (1U << 3) +#define _PY_GC_SCHEDULED_BIT (1U << 4) +#define _PY_EVAL_PLEASE_STOP_BIT (1U << 5) +#define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6) /* Reserve a few bits for future use */ #define _PY_EVAL_EVENTS_BITS 8 #define _PY_EVAL_EVENTS_MASK ((1 << _PY_EVAL_EVENTS_BITS)-1) static inline void -_Py_set_eval_breaker_bit(PyInterpreterState *interp, uint32_t bit, uint32_t set) +_Py_set_eval_breaker_bit(PyThreadState *tstate, uintptr_t bit) { - assert(set == 0 || set == 1); - uintptr_t to_set = set << bit; - uintptr_t mask = ((uintptr_t)1) << bit; - uintptr_t old = _Py_atomic_load_uintptr(&interp->ceval.eval_breaker); - if ((old & mask) == to_set) { - return; - } - uintptr_t new; - do { - new = (old & ~mask) | to_set; - } while (!_Py_atomic_compare_exchange_uintptr(&interp->ceval.eval_breaker, &old, new)); + _Py_atomic_or_uintptr(&tstate->eval_breaker, bit); +} + +static inline void +_Py_unset_eval_breaker_bit(PyThreadState *tstate, uintptr_t bit) +{ + _Py_atomic_and_uintptr(&tstate->eval_breaker, ~bit); } -static inline bool -_Py_eval_breaker_bit_is_set(PyInterpreterState *interp, int32_t bit) +static inline int +_Py_eval_breaker_bit_is_set(PyThreadState *tstate, uintptr_t bit) { - return _Py_atomic_load_uintptr_relaxed(&interp->ceval.eval_breaker) & (((uintptr_t)1) << bit); + uintptr_t b = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + return (b & bit) != 0; } +// Free-threaded builds use these functions to set or unset a bit on all +// threads in the given interpreter. +void _Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); +void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 28738980eb49be..b453328f15649e 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -78,13 +78,10 @@ struct _ceval_runtime_state { struct _ceval_state { - /* This single variable consolidates all requests to break out of - * the fast path in the eval loop. - * It is by far the hottest field in this struct and - * should be placed at the beginning. */ - uintptr_t eval_breaker; - /* Avoid false sharing */ - int64_t padding[7]; + /* This variable holds the global instrumentation version. When a thread is + running, this value is overlaid onto PyThreadState.eval_breaker so that + changes in the instrumentation version will trigger the eval breaker. */ + uintptr_t instrumentation_version; int recursion_limit; struct _gil_runtime_state *gil; int own_gil; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 85536162132072..a4e64825743ada 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -245,7 +245,12 @@ extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); /** API for executors */ extern void _PyCode_Clear_Executors(PyCodeObject *code); +#ifdef Py_GIL_DISABLED +// gh-115999 tracks progress on addressing this. +#define ENABLE_SPECIALIZATION 0 +#else #define ENABLE_SPECIALIZATION 1 +#endif /* Specialization functions */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index e5ef9a8607a83b..cd171a4384db5d 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -52,7 +52,7 @@ PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); of a key wins, if override is 2, a KeyError with conflicting key as argument is raised. */ -extern int _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); +PyAPI_FUNC(int) _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); extern void _PyDict_DebugMallocStats(FILE *out); @@ -100,10 +100,10 @@ extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t has extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); -extern PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); +PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); /* Consumes references to key and value */ -extern int _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); +PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value); extern int _PyDict_Pop_KnownHash( @@ -136,6 +136,11 @@ struct _dictkeysobject { /* Kind of keys */ uint8_t dk_kind; +#ifdef Py_GIL_DISABLED + /* Lock used to protect shared keys */ + PyMutex dk_mutex; +#endif + /* Version number -- Reset to 0 by any modification to keys */ uint32_t dk_version; @@ -145,6 +150,7 @@ struct _dictkeysobject { /* Number of used entries in dk_entries. */ Py_ssize_t dk_nentries; + /* Actual hash table of dk_size entries. It holds indices in dk_entries, or DKIX_EMPTY(-1) or DKIX_DUMMY(-2). @@ -205,6 +211,9 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1) #define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1) +#define DICT_VALUES_SIZE(values) ((uint8_t *)values)[-1] +#define DICT_VALUES_USED_SIZE(values) ((uint8_t *)values)[-2] + #ifdef Py_GIL_DISABLED #define DICT_NEXT_VERSION(INTERP) \ (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT) @@ -238,8 +247,8 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, } extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); -extern bool _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); -extern PyObject *_PyDict_FromItems( +PyAPI_FUNC(bool) _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); +PyAPI_FUNC(PyObject *)_PyDict_FromItems( PyObject *const *keys, Py_ssize_t keys_offset, PyObject *const *values, Py_ssize_t values_offset, Py_ssize_t length); @@ -250,7 +259,7 @@ _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) assert(ix < SHARED_KEYS_MAX_SIZE); uint8_t *size_ptr = ((uint8_t *)values)-2; int size = *size_ptr; - assert(size+2 < ((uint8_t *)values)[-1]); + assert(size+2 < DICT_VALUES_SIZE(values)); size++; size_ptr[-size] = (uint8_t)ix; *size_ptr = size; diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 3767df5506d43f..f984df695696c3 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -34,7 +34,7 @@ struct _Py_float_runtime_state { -void _PyFloat_ExactDealloc(PyObject *op); +PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyObject *op); extern void _PyFloat_DebugMallocStats(FILE* out); diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h index 9900ce98037060..e684e084b8bef8 100644 --- a/Include/internal/pycore_freelist.h +++ b/Include/internal/pycore_freelist.h @@ -105,11 +105,15 @@ struct _Py_async_gen_freelist { fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend are short-living objects that are instantiated for every __anext__() call. */ - struct _PyAsyncGenWrappedValue* value_freelist[_PyAsyncGen_MAXFREELIST]; - int value_numfree; + struct _PyAsyncGenWrappedValue* items[_PyAsyncGen_MAXFREELIST]; + int numfree; +#endif +}; - struct PyAsyncGenASend* asend_freelist[_PyAsyncGen_MAXFREELIST]; - int asend_numfree; +struct _Py_async_gen_asend_freelist { +#ifdef WITH_FREELISTS + struct PyAsyncGenASend* items[_PyAsyncGen_MAXFREELIST]; + int numfree; #endif }; @@ -129,6 +133,7 @@ struct _Py_object_freelists { struct _Py_slice_freelist slices; struct _Py_context_freelist contexts; struct _Py_async_gen_freelist async_gens; + struct _Py_async_gen_asend_freelist async_gen_asends; struct _Py_object_stack_freelist object_stacks; }; diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 3f3da8a44b77e4..dad6a89af77dec 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -29,7 +29,7 @@ struct _py_func_state { extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr); extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); -extern void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); +PyAPI_FUNC(void) _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version); extern PyObject *_Py_set_function_type_params( diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 582a16bf5218ce..40414a868518bb 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -260,6 +260,13 @@ struct _gc_runtime_state { Py_ssize_t long_lived_pending; }; +#ifdef Py_GIL_DISABLED +struct _gc_thread_state { + /* Thread-local allocation count. */ + Py_ssize_t alloc_count; +}; +#endif + extern void _PyGC_InitState(struct _gc_runtime_state *); @@ -279,7 +286,7 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); -extern void _Py_ScheduleGC(PyInterpreterState *interp); +extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); #ifdef __cplusplus diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index b2aa017598409f..9463c822ad8669 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -10,7 +10,7 @@ extern "C" { #include "pycore_freelist.h" -extern PyObject *_PyGen_yf(PyGenObject *); +PyAPI_FUNC(PyObject *)_PyGen_yf(PyGenObject *); extern void _PyGen_Finalize(PyObject *self); // Export for '_asyncio' shared extension @@ -19,7 +19,7 @@ PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); -extern PyObject *_PyCoro_GetAwaitableIter(PyObject *o); +PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o); extern PyObject *_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); extern PyTypeObject _PyCoroWrapper_Type; diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 3253b5271a9b7c..e8b62bd0dcd369 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -753,6 +753,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_check_retval_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_dealloc_warn)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_feature_version)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_field_types)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fields_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8780f7ef949165..b41f5c8952021e 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -242,6 +242,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_check_retval_) STRUCT_FOR_ID(_dealloc_warn) STRUCT_FOR_ID(_feature_version) + STRUCT_FOR_ID(_field_types) STRUCT_FOR_ID(_fields_) STRUCT_FOR_ID(_finalizing) STRUCT_FOR_ID(_find_and_load) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index c84f87a831bf38..eb8a9a0db46c22 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -11,7 +11,6 @@ extern "C" { #include "pycore_lock.h" // PyMutex #include "pycore_hashtable.h" // _Py_hashtable_t -#include "pycore_time.h" // _PyTime_t extern int _PyImport_IsInitialized(PyInterpreterState *); @@ -103,7 +102,7 @@ struct _import_state { /* diagnostic info in PyImport_ImportModuleLevelObject() */ struct { int import_level; - _PyTime_t accumulated; + PyTime_t accumulated; int header; } find_and_load; }; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index c07447183d6209..6a00aafea73779 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -30,6 +30,7 @@ extern "C" { #include "pycore_mimalloc.h" // struct _mimalloc_interp_state #include "pycore_object_state.h" // struct _py_object_state #include "pycore_obmalloc.h" // struct _obmalloc_state +#include "pycore_qsbr.h" // struct _qsbr_state #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tuple.h" // struct _Py_tuple_state #include "pycore_typeobject.h" // struct types_state @@ -197,6 +198,7 @@ struct _is { struct _warnings_runtime_state warnings; struct atexit_state atexit; struct _stoptheworld_state stoptheworld; + struct _qsbr_shared qsbr; #if defined(Py_GIL_DISABLED) struct _mimalloc_interp_state mimalloc; @@ -229,16 +231,20 @@ struct _is { struct _Py_dict_state dict_state; struct _Py_exc_state exc_state; + struct _Py_mem_interp_free_queue mem_free_queue; struct ast_state ast; struct types_state types; struct callable_cache callable_cache; _PyOptimizerObject *optimizer; _PyExecutorObject *executor_list_head; - /* These values are shifted and offset to speed up check in JUMP_BACKWARD */ + + /* These two values are shifted and offset to speed up check in JUMP_BACKWARD */ uint32_t optimizer_resume_threshold; uint32_t optimizer_backedge_threshold; + uint16_t optimizer_side_threshold; + uint32_t next_func_version; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 3a8dd95cff8e5d..8fa88ea3f74caa 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -44,7 +44,7 @@ typedef struct { const char *name; } intrinsic_func2_info; -extern const intrinsic_func1_info _PyIntrinsics_UnaryFunctions[]; -extern const intrinsic_func2_info _PyIntrinsics_BinaryFunctions[]; +PyAPI_DATA(const intrinsic_func1_info) _PyIntrinsics_UnaryFunctions[]; +PyAPI_DATA(const intrinsic_func2_info) _PyIntrinsics_BinaryFunctions[]; #endif // !Py_INTERNAL_INTRINSIC_H diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 0b71eb6f758ac6..17bd23f0752be2 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -13,7 +13,7 @@ extern "C" { typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate); -int _PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t length); +int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); #endif // _Py_JIT diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 50dc13c4da4487..2a82912e41d557 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -10,12 +10,12 @@ extern "C" { #include "pycore_freelist.h" // _PyFreeListState -extern PyObject* _PyList_Extend(PyListObject *, PyObject *); +PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) -extern int +PyAPI_FUNC(int) _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem); // In free-threaded build: self should be locked by the caller, if it should be thread-safe. @@ -54,7 +54,7 @@ typedef struct { PyListObject *it_seq; /* Set to NULL when iterator is exhausted */ } _PyListIterObject; -extern PyObject *_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n); +PyAPI_FUNC(PyObject *)_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n); #ifdef __cplusplus } diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index 674a1d170fec10..f648be496ea4af 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -13,8 +13,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_time.h" // _PyTime_t - // A mutex that occupies one byte. The lock can be zero initialized. // @@ -28,6 +26,9 @@ extern "C" { // Typical initialization: // PyMutex m = (PyMutex){0}; // +// Or initialize as global variables: +// static PyMutex m; +// // Typical usage: // PyMutex_Lock(&m); // ... @@ -113,7 +114,7 @@ typedef enum _PyLockFlags { // Lock a mutex with an optional timeout and additional options. See // _PyLockFlags for details. extern PyLockStatus -_PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout_ns, _PyLockFlags flags); +_PyMutex_LockTimed(PyMutex *m, PyTime_t timeout_ns, _PyLockFlags flags); // Lock a mutex with aditional options. See _PyLockFlags for details. static inline void @@ -135,6 +136,10 @@ typedef struct { uint8_t v; } PyEvent; +// Check if the event is set without blocking. Returns 1 if the event is set or +// 0 otherwise. +PyAPI_FUNC(int) _PyEvent_IsSet(PyEvent *evt); + // Set the event and notify any waiting threads. // Export for '_testinternalcapi' shared extension PyAPI_FUNC(void) _PyEvent_Notify(PyEvent *evt); @@ -146,8 +151,17 @@ PyAPI_FUNC(void) PyEvent_Wait(PyEvent *evt); // Wait for the event to be set, or until the timeout expires. If the event is // already set, then this returns immediately. Returns 1 if the event was set, // and 0 if the timeout expired or thread was interrupted. -PyAPI_FUNC(int) PyEvent_WaitTimed(PyEvent *evt, _PyTime_t timeout_ns); +PyAPI_FUNC(int) PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns); + +// A one-time event notification with reference counting. +typedef struct _PyEventRc { + PyEvent event; + Py_ssize_t refcount; +} _PyEventRc; +_PyEventRc *_PyEventRc_New(void); +void _PyEventRc_Incref(_PyEventRc *erc); +void _PyEventRc_Decref(_PyEventRc *erc); // _PyRawMutex implements a word-sized mutex that that does not depend on the // parking lot API, and therefore can be used in the parking lot diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index ec27df9e416c58..f04f66d053bab9 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -121,9 +121,9 @@ PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, size_t); // Export for 'math' shared extension PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, size_t); -extern PyObject* _PyLong_Add(PyLongObject *left, PyLongObject *right); -extern PyObject* _PyLong_Multiply(PyLongObject *left, PyLongObject *right); -extern PyObject* _PyLong_Subtract(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(PyObject*) _PyLong_Subtract(PyLongObject *left, PyLongObject *right); // Export for 'binascii' shared extension. PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 34a83ea228e8b1..9809f5f2e0271a 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -73,7 +73,7 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); .ob_size = size \ } -extern void _Py_NO_RETURN _Py_FatalRefcountErrorFunc( +PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( const char *func, const char *message); @@ -684,7 +684,7 @@ PyAPI_FUNC(PyObject*) _PyObject_LookupSpecial(PyObject *, PyObject *); extern int _PyObject_IsAbstract(PyObject *); -extern int _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +PyAPI_FUNC(int) _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); extern PyObject* _PyObject_NextNotImplemented(PyObject *); // Pickle support. diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 6b60a6fbffdc5e..ab34366ab1066c 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -519,7 +519,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL_ALLOC_AND_ENTER_INIT: return 1; case CALL_BOUND_METHOD_EXACT_ARGS: - return ((0) ? 1 : 0); + return 0; case CALL_BUILTIN_CLASS: return 1; case CALL_BUILTIN_FAST: @@ -551,7 +551,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL_METHOD_DESCRIPTOR_O: return 1; case CALL_PY_EXACT_ARGS: - return ((0) ? 1 : 0); + return 0; case CALL_PY_WITH_DEFAULTS: return 1; case CALL_STR_1: @@ -697,23 +697,23 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case LOAD_ATTR_CLASS: return 1 + (oparg & 1); case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_INSTANCE_VALUE: return 1 + (oparg & 1); case LOAD_ATTR_METHOD_LAZY_DICT: - return 1 + ((1) ? 1 : 0); + return 2; case LOAD_ATTR_METHOD_NO_DICT: - return 1 + ((1) ? 1 : 0); + return 2; case LOAD_ATTR_METHOD_WITH_VALUES: - return 1 + ((1) ? 1 : 0); + return 2; case LOAD_ATTR_MODULE: return 1 + (oparg & 1); case LOAD_ATTR_NONDESCRIPTOR_NO_DICT: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_PROPERTY: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_ATTR_SLOT: return 1 + (oparg & 1); case LOAD_ATTR_WITH_HINT: @@ -749,7 +749,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case LOAD_SUPER_ATTR: return 1 + (oparg & 1); case LOAD_SUPER_ATTR_ATTR: - return 1 + ((0) ? 1 : 0); + return 1; case LOAD_SUPER_ATTR_METHOD: return 2; case MAKE_CELL: @@ -909,8 +909,10 @@ enum InstructionFormat { #define HAS_DEOPT_FLAG (128) #define HAS_ERROR_FLAG (256) #define HAS_ESCAPES_FLAG (512) -#define HAS_PURE_FLAG (1024) -#define HAS_PASSTHROUGH_FLAG (2048) +#define HAS_EXIT_FLAG (1024) +#define HAS_PURE_FLAG (2048) +#define HAS_PASSTHROUGH_FLAG (4096) +#define HAS_OPARG_AND_1_FLAG (8192) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -921,8 +923,10 @@ enum InstructionFormat { #define OPCODE_HAS_DEOPT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_DEOPT_FLAG)) #define OPCODE_HAS_ERROR(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_FLAG)) #define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) +#define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG)) #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) +#define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -945,14 +949,14 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -995,9 +999,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, @@ -1060,16 +1064,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1118,8 +1122,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1133,12 +1137,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, @@ -1217,9 +1221,9 @@ _PyOpcode_macro_expansion[256] = { [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, 0, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, 0, 0 } } }, [COMPARE_OP] = { .nuops = 1, .uops = { { _COMPARE_OP, 0, 0 } } }, - [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { _COMPARE_OP_FLOAT, 0, 0 } } }, - [COMPARE_OP_INT] = { .nuops = 1, .uops = { { _COMPARE_OP_INT, 0, 0 } } }, - [COMPARE_OP_STR] = { .nuops = 1, .uops = { { _COMPARE_OP_STR, 0, 0 } } }, + [COMPARE_OP_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _COMPARE_OP_FLOAT, 0, 0 } } }, + [COMPARE_OP_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _COMPARE_OP_INT, 0, 0 } } }, + [COMPARE_OP_STR] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _COMPARE_OP_STR, 0, 0 } } }, [CONTAINS_OP] = { .nuops = 1, .uops = { { _CONTAINS_OP, 0, 0 } } }, [CONVERT_VALUE] = { .nuops = 1, .uops = { { _CONVERT_VALUE, 0, 0 } } }, [COPY] = { .nuops = 1, .uops = { { _COPY, 0, 0 } } }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index eee71c700d4904..d32e6c0174f680 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_uop_ids.h" +#include // This is the length of the trace we project initially. #define UOP_MAX_TRACE_LENGTH 512 @@ -25,6 +26,93 @@ extern PyTypeObject _PyDefaultOptimizer_Type; extern PyTypeObject _PyUOpExecutor_Type; extern PyTypeObject _PyUOpOptimizer_Type; +/* Symbols */ +/* See explanation in optimizer_symbols.c */ + +struct _Py_UopsSymbol { + int flags; // 0 bits: Top; 2 or more bits: Bottom + PyTypeObject *typ; // Borrowed reference + PyObject *const_val; // Owned reference (!) +}; + +// Holds locals, stack, locals, stack ... co_consts (in that order) +#define MAX_ABSTRACT_INTERP_SIZE 4096 + +#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) + +// Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) +#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) + +typedef struct _Py_UopsSymbol _Py_UopsSymbol; + +struct _Py_UOpsAbstractFrame { + // Max stacklen + int stack_len; + int locals_len; + + _Py_UopsSymbol **stack_pointer; + _Py_UopsSymbol **stack; + _Py_UopsSymbol **locals; +}; + +typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; + +typedef struct ty_arena { + int ty_curr_number; + int ty_max_number; + _Py_UopsSymbol arena[TY_ARENA_SIZE]; +} ty_arena; + +struct _Py_UOpsContext { + PyObject_HEAD + // The current "executing" frame. + _Py_UOpsAbstractFrame *frame; + _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; + int curr_frame_depth; + + // Arena for the symbolic types. + ty_arena t_arena; + + _Py_UopsSymbol **n_consumed; + _Py_UopsSymbol **limit; + _Py_UopsSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; +}; + +typedef struct _Py_UOpsContext _Py_UOpsContext; + +extern bool _Py_uop_sym_is_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_is_not_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_is_const(_Py_UopsSymbol *sym); +extern PyObject *_Py_uop_sym_get_const(_Py_UopsSymbol *sym); +extern _Py_UopsSymbol *_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx); +extern _Py_UopsSymbol *_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx); +extern _Py_UopsSymbol *_Py_uop_sym_new_type( + _Py_UOpsContext *ctx, PyTypeObject *typ); +extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val); +extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); +extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_set_non_null(_Py_UopsSymbol *sym); +extern bool _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val); +extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); + + +extern int _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); +extern void _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx); + +extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( + _Py_UOpsContext *ctx, + PyCodeObject *co, + _Py_UopsSymbol **localsplus_start, + int n_locals_already_filled, + int curr_stackentries); +extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); + +PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); + +PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer, _PyExecutorObject **exec_ptr); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_parking_lot.h b/Include/internal/pycore_parking_lot.h index f444da730055e8..8c9260e2636fbc 100644 --- a/Include/internal/pycore_parking_lot.h +++ b/Include/internal/pycore_parking_lot.h @@ -18,8 +18,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_time.h" // _PyTime_t - enum { // The thread was unparked by another thread. @@ -61,7 +59,7 @@ enum { // } PyAPI_FUNC(int) _PyParkingLot_Park(const void *address, const void *expected, - size_t address_size, _PyTime_t timeout_ns, + size_t address_size, PyTime_t timeout_ns, void *park_arg, int detach); // Callback for _PyParkingLot_Unpark: diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index cbbe90e009c8d2..e441600d54e1aa 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -23,11 +23,17 @@ extern "C" { #define FT_ATOMIC_LOAD_SSIZE(value) _Py_atomic_load_ssize(&value) #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) \ _Py_atomic_load_ssize_relaxed(&value) +#define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ + _Py_atomic_store_ptr_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \ + _Py_atomic_store_ptr_release(&value, new_value) #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ _Py_atomic_store_ssize_relaxed(&value, new_value) #else #define FT_ATOMIC_LOAD_SSIZE(value) value #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value +#define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value #endif diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 0f16fb894d17e1..910335fd2cf33b 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -95,7 +95,7 @@ extern void _PyErr_Fetch( extern PyObject* _PyErr_GetRaisedException(PyThreadState *tstate); -extern int _PyErr_ExceptionMatches( +PyAPI_FUNC(int) _PyErr_ExceptionMatches( PyThreadState *tstate, PyObject *exc); @@ -114,18 +114,18 @@ extern void _PyErr_SetObject( extern void _PyErr_ChainStackItem(void); -extern void _PyErr_Clear(PyThreadState *tstate); +PyAPI_FUNC(void) _PyErr_Clear(PyThreadState *tstate); extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); -extern void _PyErr_SetString( +PyAPI_FUNC(void) _PyErr_SetString( PyThreadState *tstate, PyObject *exception, const char *string); -extern PyObject* _PyErr_Format( +PyAPI_FUNC(PyObject*) _PyErr_Format( PyThreadState *tstate, PyObject *exception, const char *format, diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 1a72d07b50b738..1aea91abc5d69f 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -1,6 +1,7 @@ #ifndef Py_INTERNAL_PYMEM_H #define Py_INTERNAL_PYMEM_H +#include "pycore_llist.h" // struct llist_node #include "pycore_lock.h" // PyMutex #ifdef __cplusplus @@ -48,6 +49,11 @@ struct _pymem_allocators { PyObjectArenaAllocator obj_arena; }; +struct _Py_mem_interp_free_queue { + int has_work; // true if the queue is not empty + PyMutex mutex; // protects the queue + struct llist_node head; // queue of _mem_work_chunk items +}; /* Set the memory allocator of the specified domain to the default. Save the old allocator into *old_alloc if it's non-NULL. @@ -110,6 +116,19 @@ extern int _PyMem_SetupAllocators(PyMemAllocatorName allocator); /* Is the debug allocator enabled? */ extern int _PyMem_DebugEnabled(void); +// Enqueue a pointer to be freed possibly after some delay. +extern void _PyMem_FreeDelayed(void *ptr); + +// Periodically process delayed free requests. +extern void _PyMem_ProcessDelayed(PyThreadState *tstate); + +// Abandon all thread-local delayed free requests and push them to the +// interpreter's queue. +extern void _PyMem_AbandonDelayed(PyThreadState *tstate); + +// On interpreter shutdown, frees all delayed free requests. +extern void _PyMem_FiniDelayed(PyInterpreterState *interp); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pymem_init.h b/Include/internal/pycore_pymem_init.h index 96c49ed7338d6d..c593edc86d9952 100644 --- a/Include/internal/pycore_pymem_init.h +++ b/Include/internal/pycore_pymem_init.h @@ -92,6 +92,11 @@ extern void _PyMem_ArenaFree(void *, void *, size_t); { NULL, _PyMem_ArenaAlloc, _PyMem_ArenaFree } +#define _Py_mem_free_queue_INIT(queue) \ + { \ + .head = LLIST_INIT(queue.head), \ + } + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index 265299d7574838..d2e7cc2a206ced 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -46,7 +46,8 @@ extern "C" { #if defined(HAVE_PTHREAD_STUBS) -#include // bool +#include "cpython/pthread_stubs.h" // PTHREAD_KEYS_MAX +#include // bool // pthread_key struct py_stub_tls_entry { @@ -96,7 +97,7 @@ extern void _PyThread_AfterFork(struct _pythread_runtime_state *state); // unset: -1 seconds, in nanoseconds -#define PyThread_UNSET_TIMEOUT ((_PyTime_t)(-1 * 1000 * 1000 * 1000)) +#define PyThread_UNSET_TIMEOUT ((PyTime_t)(-1 * 1000 * 1000 * 1000)) // Exported for the _xxinterpchannels module. PyAPI_FUNC(int) PyThread_ParseTimeoutArg( diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h new file mode 100644 index 00000000000000..475f00deedc226 --- /dev/null +++ b/Include/internal/pycore_qsbr.h @@ -0,0 +1,139 @@ +// The QSBR APIs (quiescent state-based reclamation) provide a mechanism for +// the free-threaded build to safely reclaim memory when there may be +// concurrent accesses. +// +// Many operations in the free-threaded build are protected by locks. However, +// in some cases, we want to allow reads to happen concurrently with updates. +// In this case, we need to delay freeing ("reclaiming") any memory that may be +// concurrently accessed by a reader. The QSBR APIs provide a way to do this. +#ifndef Py_INTERNAL_QSBR_H +#define Py_INTERNAL_QSBR_H + +#include +#include +#include "pycore_lock.h" // PyMutex + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// The shared write sequence is always odd and incremented by two. Detached +// threads are indicated by a read sequence of zero. This avoids collisions +// between the offline state and any valid sequence number even if the +// sequences numbers wrap around. +#define QSBR_OFFLINE 0 +#define QSBR_INITIAL 1 +#define QSBR_INCR 2 + +struct _qsbr_shared; +struct _PyThreadStateImpl; // forward declare to avoid circular dependency + +// Per-thread state +struct _qsbr_thread_state { + // Last observed write sequence (or 0 if detached) + uint64_t seq; + + // Shared (per-interpreter) QSBR state + struct _qsbr_shared *shared; + + // Thread state (or NULL) + PyThreadState *tstate; + + // Used to defer advancing write sequence a fixed number of times + int deferrals; + + // Is this thread state allocated? + bool allocated; + struct _qsbr_thread_state *freelist_next; +}; + +// Padding to avoid false sharing +struct _qsbr_pad { + struct _qsbr_thread_state qsbr; + char __padding[64 - sizeof(struct _qsbr_thread_state)]; +}; + +// Per-interpreter state +struct _qsbr_shared { + // Write sequence: always odd, incremented by two + uint64_t wr_seq; + + // Minimum observed read sequence of all QSBR thread states + uint64_t rd_seq; + + // Array of QSBR thread states. + struct _qsbr_pad *array; + Py_ssize_t size; + + // Freelist of unused _qsbr_thread_states (protected by mutex) + PyMutex mutex; + struct _qsbr_thread_state *freelist; +}; + +static inline uint64_t +_Py_qsbr_shared_current(struct _qsbr_shared *shared) +{ + return _Py_atomic_load_uint64_acquire(&shared->wr_seq); +} + +// Reports a quiescent state: the caller no longer holds any pointer to shared +// data not protected by locks or reference counts. +static inline void +_Py_qsbr_quiescent_state(struct _qsbr_thread_state *qsbr) +{ + uint64_t seq = _Py_qsbr_shared_current(qsbr->shared); + _Py_atomic_store_uint64_release(&qsbr->seq, seq); +} + +// Advance the write sequence and return the new goal. This should be called +// after data is removed. The returned goal is used with `_Py_qsbr_poll()` to +// determine when it is safe to reclaim (free) the memory. +extern uint64_t +_Py_qsbr_advance(struct _qsbr_shared *shared); + +// Batches requests to advance the write sequence. This advances the write +// sequence every N calls, which reduces overhead but increases time to +// reclamation. Returns the new goal. +extern uint64_t +_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr); + +// Have the read sequences advanced to the given goal? If this returns true, +// it safe to reclaim any memory tagged with the goal (or earlier goal). +extern bool +_Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal); + +// Called when thread attaches to interpreter +extern void +_Py_qsbr_attach(struct _qsbr_thread_state *qsbr); + +// Called when thread detaches from interpreter +extern void +_Py_qsbr_detach(struct _qsbr_thread_state *qsbr); + +// Reserves (allocates) a QSBR state and returns its index. +extern Py_ssize_t +_Py_qsbr_reserve(PyInterpreterState *interp); + +// Associates a PyThreadState with the QSBR state at the given index +extern void +_Py_qsbr_register(struct _PyThreadStateImpl *tstate, + PyInterpreterState *interp, Py_ssize_t index); + +// Disassociates a PyThreadState from the QSBR state and frees the QSBR state. +extern void +_Py_qsbr_unregister(struct _PyThreadStateImpl *tstate); + +extern void +_Py_qsbr_fini(PyInterpreterState *interp); + +extern void +_Py_qsbr_after_fork(struct _PyThreadStateImpl *tstate); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_QSBR_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 7c705d1224f915..dc6f6f100f7a92 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -55,74 +55,81 @@ typedef struct _Py_DebugOffsets { uint64_t version; // Runtime state offset; struct _runtime_state { - off_t finalizing; - off_t interpreters_head; + uint64_t finalizing; + uint64_t interpreters_head; } runtime_state; // Interpreter state offset; struct _interpreter_state { - off_t next; - off_t threads_head; - off_t gc; - off_t imports_modules; - off_t sysdict; - off_t builtins; - off_t ceval_gil; - off_t gil_runtime_state_locked; - off_t gil_runtime_state_holder; + uint64_t next; + uint64_t threads_head; + uint64_t gc; + uint64_t imports_modules; + uint64_t sysdict; + uint64_t builtins; + uint64_t ceval_gil; + uint64_t gil_runtime_state_locked; + uint64_t gil_runtime_state_holder; } interpreter_state; // Thread state offset; struct _thread_state{ - off_t prev; - off_t next; - off_t interp; - off_t current_frame; - off_t thread_id; - off_t native_thread_id; + uint64_t prev; + uint64_t next; + uint64_t interp; + uint64_t current_frame; + uint64_t thread_id; + uint64_t native_thread_id; } thread_state; // InterpreterFrame offset; struct _interpreter_frame { - off_t previous; - off_t executable; - off_t instr_ptr; - off_t localsplus; - off_t owner; + uint64_t previous; + uint64_t executable; + uint64_t instr_ptr; + uint64_t localsplus; + uint64_t owner; } interpreter_frame; // CFrame offset; struct _cframe { - off_t current_frame; - off_t previous; + uint64_t current_frame; + uint64_t previous; } cframe; // Code object offset; struct _code_object { - off_t filename; - off_t name; - off_t linetable; - off_t firstlineno; - off_t argcount; - off_t localsplusnames; - off_t localspluskinds; - off_t co_code_adaptive; + uint64_t filename; + uint64_t name; + uint64_t linetable; + uint64_t firstlineno; + uint64_t argcount; + uint64_t localsplusnames; + uint64_t localspluskinds; + uint64_t co_code_adaptive; } code_object; // PyObject offset; struct _pyobject { - off_t ob_type; + uint64_t ob_type; } pyobject; // PyTypeObject object offset; struct _type_object { - off_t tp_name; + uint64_t tp_name; } type_object; // PyTuple object offset; struct _tuple_object { - off_t ob_item; + uint64_t ob_item; } tuple_object; + + // Unicode object offset; + struct _unicode_object { + uint64_t state; + uint64_t length; + size_t asciiobject_size; + } unicode_object; } _Py_DebugOffsets; /* Full Python runtime state */ @@ -191,7 +198,10 @@ typedef struct pyruntimestate { int64_t next_id; } interpreters; + /* Platform-specific identifier and PyThreadState, respectively, for the + main thread in the main interpreter. */ unsigned long main_thread; + PyThreadState *main_tstate; /* ---------- IMPORTANT --------------------------- The fields above this line are declared as early as diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 7a05c105d7bf12..cc47b9a82e2879 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -17,6 +17,7 @@ extern "C" { #include "pycore_pyhash.h" // pyhash_state_INIT #include "pycore_pymem_init.h" // _pymem_allocators_standard_INIT #include "pycore_pythread.h" // _pythread_RUNTIME_INIT +#include "pycore_qsbr.h" // QSBR_INITIAL #include "pycore_runtime_init_generated.h" // _Py_bytes_characters_INIT #include "pycore_signal.h" // _signals_RUNTIME_INIT #include "pycore_tracemalloc.h" // _tracemalloc_runtime_state_INIT @@ -82,6 +83,11 @@ extern PyTypeObject _PyExc_MemoryError; .tuple_object = { \ .ob_item = offsetof(PyTupleObject, ob_item), \ }, \ + .unicode_object = { \ + .state = offsetof(PyUnicodeObject, _base._base.state), \ + .length = offsetof(PyUnicodeObject, _base._base.length), \ + .asciiobject_size = sizeof(PyASCIIObject), \ + }, \ }, \ .allocators = { \ .standard = _pymem_allocators_standard_INIT(runtime), \ @@ -169,8 +175,13 @@ extern PyTypeObject _PyExc_MemoryError; { .threshold = 10, }, \ }, \ }, \ + .qsbr = { \ + .wr_seq = QSBR_INITIAL, \ + .rd_seq = QSBR_INITIAL, \ + }, \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ .dict_state = _dict_state_INIT, \ + .mem_free_queue = _Py_mem_free_queue_INIT(INTERP.mem_free_queue), \ .func_state = { \ .next_version = 1, \ }, \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index a9d514856dab1f..016eae02a2103d 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -751,6 +751,7 @@ extern "C" { INIT_ID(_check_retval_), \ INIT_ID(_dealloc_warn), \ INIT_ID(_feature_version), \ + INIT_ID(_field_types), \ INIT_ID(_fields_), \ INIT_ID(_finalizing), \ INIT_ID(_find_and_load), \ diff --git a/Include/internal/pycore_semaphore.h b/Include/internal/pycore_semaphore.h index 4c37df7b39a48a..ffcc6d80344d6e 100644 --- a/Include/internal/pycore_semaphore.h +++ b/Include/internal/pycore_semaphore.h @@ -8,7 +8,6 @@ #endif #include "pycore_pythread.h" // _POSIX_SEMAPHORES -#include "pycore_time.h" // _PyTime_t #ifdef MS_WINDOWS # define WIN32_LEAN_AND_MEAN @@ -48,7 +47,7 @@ typedef struct _PySemaphore { // If `detach` is true, then the thread will detach/release the GIL while // sleeping. PyAPI_FUNC(int) -_PySemaphore_Wait(_PySemaphore *sema, _PyTime_t timeout_ns, int detach); +_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout_ns, int detach); // Wakes up a single thread waiting on sema. Note that _PySemaphore_Wakeup() // can be called before _PySemaphore_Wait(). diff --git a/Include/internal/pycore_sliceobject.h b/Include/internal/pycore_sliceobject.h index 89086f67683a2f..ba8b1f1cb27dee 100644 --- a/Include/internal/pycore_sliceobject.h +++ b/Include/internal/pycore_sliceobject.h @@ -11,7 +11,7 @@ extern "C" { /* runtime lifecycle */ -extern PyObject * +PyAPI_FUNC(PyObject *) _PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop); #ifdef __cplusplus diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 1aad6ccea69ae3..40b28e0ba221ae 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -62,7 +62,6 @@ extern "C" { struct timeval; #endif -typedef PyTime_t _PyTime_t; #define _SIZEOF_PYTIME_T 8 typedef enum { @@ -130,69 +129,64 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimespec( // Create a timestamp from a number of seconds. // Export for '_socket' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_FromSeconds(int seconds); +PyAPI_FUNC(PyTime_t) _PyTime_FromSeconds(int seconds); // Create a timestamp from a number of seconds in double. // Export for '_socket' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round); +PyAPI_FUNC(PyTime_t) _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round); // Macro to create a timestamp from a number of seconds, no integer overflow. // Only use the macro for small values, prefer _PyTime_FromSeconds(). #define _PYTIME_FROMSECONDS(seconds) \ - ((_PyTime_t)(seconds) * (1000 * 1000 * 1000)) - -// Create a timestamp from a number of nanoseconds. -// Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(_PyTime_t ns); + ((PyTime_t)(seconds) * (1000 * 1000 * 1000)) // Create a timestamp from a number of microseconds. // Clamp to [PyTime_MIN; PyTime_MAX] on overflow. -extern _PyTime_t _PyTime_FromMicrosecondsClamp(_PyTime_t us); +extern PyTime_t _PyTime_FromMicrosecondsClamp(PyTime_t us); -// Create a timestamp from nanoseconds (Python int). +// Create a timestamp from a Python int object (number of nanoseconds). // Export for '_lsprof' shared extension. -PyAPI_FUNC(int) _PyTime_FromNanosecondsObject(_PyTime_t *t, +PyAPI_FUNC(int) _PyTime_FromLong(PyTime_t *t, PyObject *obj); // Convert a number of seconds (Python float or int) to a timestamp. // Raise an exception and return -1 on error, return 0 on success. // Export for '_socket' shared extension. -PyAPI_FUNC(int) _PyTime_FromSecondsObject(_PyTime_t *t, +PyAPI_FUNC(int) _PyTime_FromSecondsObject(PyTime_t *t, PyObject *obj, _PyTime_round_t round); // Convert a number of milliseconds (Python float or int, 10^-3) to a timestamp. // Raise an exception and return -1 on error, return 0 on success. // Export for 'select' shared extension. -PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(_PyTime_t *t, +PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(PyTime_t *t, PyObject *obj, _PyTime_round_t round); // Convert timestamp to a number of milliseconds (10^-3 seconds). // Export for '_ssl' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_AsMilliseconds(_PyTime_t t, +PyAPI_FUNC(PyTime_t) _PyTime_AsMilliseconds(PyTime_t t, _PyTime_round_t round); // Convert timestamp to a number of microseconds (10^-6 seconds). // Export for '_queue' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_AsMicroseconds(_PyTime_t t, +PyAPI_FUNC(PyTime_t) _PyTime_AsMicroseconds(PyTime_t t, _PyTime_round_t round); #ifdef MS_WINDOWS // Convert timestamp to a number of 100 nanoseconds (10^-7 seconds). -extern _PyTime_t _PyTime_As100Nanoseconds(_PyTime_t t, +extern PyTime_t _PyTime_As100Nanoseconds(PyTime_t t, _PyTime_round_t round); #endif -// Convert timestamp to a number of nanoseconds (10^-9 seconds) as a Python int -// object. +// Convert a timestamp (number of nanoseconds) as a Python int object. // Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(PyObject*) _PyTime_AsNanosecondsObject(_PyTime_t t); +PyAPI_FUNC(PyObject*) _PyTime_AsLong(PyTime_t t); #ifndef MS_WINDOWS // Create a timestamp from a timeval structure. // Raise an exception and return -1 on overflow, return 0 on success. -extern int _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); +extern int _PyTime_FromTimeval(PyTime_t *tp, struct timeval *tv); #endif // Convert a timestamp to a timeval structure (microsecond resolution). @@ -200,14 +194,14 @@ extern int _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); // Raise an exception and return -1 if the conversion overflowed, // return 0 on success. // Export for 'select' shared extension. -PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, +PyAPI_FUNC(int) _PyTime_AsTimeval(PyTime_t t, struct timeval *tv, _PyTime_round_t round); // Similar to _PyTime_AsTimeval() but don't raise an exception on overflow. -// On overflow, clamp tv_sec to _PyTime_t min/max. +// On overflow, clamp tv_sec to PyTime_t min/max. // Export for 'select' shared extension. -PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, +PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(PyTime_t t, struct timeval *tv, _PyTime_round_t round); @@ -219,7 +213,7 @@ PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, // return 0 on success. // Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( - _PyTime_t t, + PyTime_t t, time_t *secs, int *us, _PyTime_round_t round); @@ -227,23 +221,23 @@ PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) // Create a timestamp from a timespec structure. // Raise an exception and return -1 on overflow, return 0 on success. -extern int _PyTime_FromTimespec(_PyTime_t *tp, const struct timespec *ts); +extern int _PyTime_FromTimespec(PyTime_t *tp, const struct timespec *ts); // Convert a timestamp to a timespec structure (nanosecond resolution). // tv_nsec is always positive. // Raise an exception and return -1 on error, return 0 on success. // Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); +PyAPI_FUNC(int) _PyTime_AsTimespec(PyTime_t t, struct timespec *ts); // Similar to _PyTime_AsTimespec() but don't raise an exception on overflow. -// On overflow, clamp tv_sec to _PyTime_t min/max. +// On overflow, clamp tv_sec to PyTime_t min/max. // Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts); +PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(PyTime_t t, struct timespec *ts); #endif // Compute t1 + t2. Clamp to [PyTime_MIN; PyTime_MAX] on overflow. -extern _PyTime_t _PyTime_Add(_PyTime_t t1, _PyTime_t t2); +extern PyTime_t _PyTime_Add(PyTime_t t1, PyTime_t t2); // Structure used by time.get_clock_info() typedef struct { @@ -253,37 +247,28 @@ typedef struct { double resolution; } _Py_clock_info_t; -// Get the current time from the system clock. -// -// If the internal clock fails, silently ignore the error and return 0. -// On integer overflow, silently ignore the overflow and clamp the clock to -// [_PyTime_MIN; _PyTime_MAX]. +// Similar to PyTime_Time() but silently ignore the error and return 0 if the +// internal clock fails. // -// Use _PyTime_GetSystemClockWithInfo or the public PyTime_Time() to check +// Use _PyTime_TimeWithInfo() or the public PyTime_Time() to check // for failure. // Export for '_random' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); +PyAPI_FUNC(PyTime_t) _PyTime_TimeUnchecked(void); // Get the current time from the system clock. // On success, set *t and *info (if not NULL), and return 0. // On error, raise an exception and return -1. -extern int _PyTime_GetSystemClockWithInfo( - _PyTime_t *t, +extern int _PyTime_TimeWithInfo( + PyTime_t *t, _Py_clock_info_t *info); -// Get the time of a monotonic clock, i.e. a clock that cannot go backwards. -// The clock is not affected by system clock updates. The reference point of -// the returned value is undefined, so that only the difference between the -// results of consecutive calls is valid. +// Similar to PyTime_Monotonic() but silently ignore the error and return 0 if +// the internal clock fails. // -// If the internal clock fails, silently ignore the error and return 0. -// On integer overflow, silently ignore the overflow and clamp the clock to -// [_PyTime_MIN; _PyTime_MAX]. -// -// Use _PyTime_GetMonotonicClockWithInfo or the public PyTime_Monotonic() +// Use _PyTime_MonotonicWithInfo() or the public PyTime_Monotonic() // to check for failure. // Export for '_random' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); +PyAPI_FUNC(PyTime_t) _PyTime_MonotonicUnchecked(void); // Get the time of a monotonic clock, i.e. a clock that cannot go backwards. // The clock is not affected by system clock updates. The reference point of @@ -294,8 +279,8 @@ PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); // // Return 0 on success, raise an exception and return -1 on error. // Export for '_testsinglephase' shared extension. -PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo( - _PyTime_t *t, +PyAPI_FUNC(int) _PyTime_MonotonicWithInfo( + PyTime_t *t, _Py_clock_info_t *info); @@ -309,17 +294,13 @@ PyAPI_FUNC(int) _PyTime_localtime(time_t t, struct tm *tm); // Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); -// Get the performance counter: clock with the highest available resolution to -// measure a short duration. -// -// If the internal clock fails, silently ignore the error and return 0. -// On integer overflow, silently ignore the overflow and clamp the clock to -// [_PyTime_MIN; _PyTime_MAX]. +// Similar to PyTime_PerfCounter() but silently ignore the error and return 0 +// if the internal clock fails. // -// Use _PyTime_GetPerfCounterWithInfo() or the public PyTime_PerfCounter -// to check for failure. +// Use _PyTime_PerfCounterWithInfo() or the public PyTime_PerfCounter() to +// check for failure. // Export for '_lsprof' shared extension. -PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); +PyAPI_FUNC(PyTime_t) _PyTime_PerfCounterUnchecked(void); // Get the performance counter: clock with the highest available resolution to @@ -328,34 +309,29 @@ PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); // Fill info (if set) with information of the function used to get the time. // // Return 0 on success, raise an exception and return -1 on error. -extern int _PyTime_GetPerfCounterWithInfo( - _PyTime_t *t, +extern int _PyTime_PerfCounterWithInfo( + PyTime_t *t, _Py_clock_info_t *info); -// Alias for backward compatibility -#define _PyTime_MIN PyTime_MIN -#define _PyTime_MAX PyTime_MAX -#define _PyTime_AsSecondsDouble PyTime_AsSecondsDouble - // --- _PyDeadline ----------------------------------------------------------- // Create a deadline. -// Pseudo code: _PyTime_GetMonotonicClock() + timeout. +// Pseudo code: _PyTime_MonotonicUnchecked() + timeout. // Export for '_ssl' shared extension. -PyAPI_FUNC(_PyTime_t) _PyDeadline_Init(_PyTime_t timeout); +PyAPI_FUNC(PyTime_t) _PyDeadline_Init(PyTime_t timeout); // Get remaining time from a deadline. -// Pseudo code: deadline - _PyTime_GetMonotonicClock(). +// Pseudo code: deadline - _PyTime_MonotonicUnchecked(). // Export for '_ssl' shared extension. -PyAPI_FUNC(_PyTime_t) _PyDeadline_Get(_PyTime_t deadline); +PyAPI_FUNC(PyTime_t) _PyDeadline_Get(PyTime_t deadline); // --- _PyTimeFraction ------------------------------------------------------- typedef struct { - _PyTime_t numer; - _PyTime_t denom; + PyTime_t numer; + PyTime_t denom; } _PyTimeFraction; // Set a fraction. @@ -363,13 +339,13 @@ typedef struct { // Return -1 if the fraction is invalid. extern int _PyTimeFraction_Set( _PyTimeFraction *frac, - _PyTime_t numer, - _PyTime_t denom); + PyTime_t numer, + PyTime_t denom); // Compute ticks * frac.numer / frac.denom. -// Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -extern _PyTime_t _PyTimeFraction_Mul( - _PyTime_t ticks, +// Clamp to [PyTime_MIN; PyTime_MAX] on overflow. +extern PyTime_t _PyTimeFraction_Mul( + PyTime_t ticks, const _PyTimeFraction *frac); // Compute a clock resolution: frac.numer / frac.denom / 1e9. diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 97aa85a659fa7b..e268e6fbbb087b 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -8,9 +8,10 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_brc.h" // struct _brc_thread_state #include "pycore_freelist.h" // struct _Py_freelist_state #include "pycore_mimalloc.h" // struct _mimalloc_thread_state -#include "pycore_brc.h" // struct _brc_thread_state +#include "pycore_qsbr.h" // struct qsbr static inline void @@ -27,7 +28,11 @@ typedef struct _PyThreadStateImpl { // semi-public fields are in PyThreadState. PyThreadState base; + struct _qsbr_thread_state *qsbr; // only used by free-threaded build + struct llist_node mem_free_queue; // delayed free queue + #ifdef Py_GIL_DISABLED + struct _gc_thread_state gc; struct _mimalloc_thread_state mimalloc; struct _Py_object_freelists freelists; struct _brc_thread_state brc; diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index 4605f355ccbc38..14a9e42c3a324c 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -21,7 +21,7 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t); -extern PyObject *_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); +PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); typedef struct { PyObject_HEAD diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 9134ab45cd0039..c214111fed6f97 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -147,7 +147,7 @@ extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); extern PyTypeObject _PyBufferWrapper_Type; -extern PyObject* _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, +PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 7ee540154b23d8..fea5ceea0954f4 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -31,7 +31,7 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency( PyObject *op, int check_content); -extern void _PyUnicode_ExactDealloc(PyObject *op); +PyAPI_FUNC(void) _PyUnicode_ExactDealloc(PyObject *op); extern Py_ssize_t _PyUnicode_InternedSize(void); // Get a copy of a Unicode string. @@ -202,7 +202,7 @@ PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( /* --- Methods & Slots ---------------------------------------------------- */ -extern PyObject* _PyUnicode_JoinArray( +PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray( PyObject *separator, PyObject *const *items, Py_ssize_t seqlen diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index f3b064e2a2cb25..64c4cf8c077056 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -567,6 +567,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(_feature_version); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(_field_types); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(_fields_); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Include/internal/pycore_unionobject.h b/Include/internal/pycore_unionobject.h index 87264635b6e1cf..6ece7134cdeca0 100644 --- a/Include/internal/pycore_unionobject.h +++ b/Include/internal/pycore_unionobject.h @@ -8,9 +8,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -extern PyTypeObject _PyUnion_Type; +// For extensions created by test_peg_generator +PyAPI_DATA(PyTypeObject) _PyUnion_Type; +PyAPI_FUNC(PyObject *) _Py_union_type_or(PyObject *, PyObject *); + #define _PyUnion_Check(op) Py_IS_TYPE((op), &_PyUnion_Type) -extern PyObject *_Py_union_type_or(PyObject *, PyObject *); #define _PyGenericAlias_Check(op) PyObject_TypeCheck((op), &Py_GenericAliasType) extern PyObject *_Py_subs_parameters(PyObject *, PyObject *, PyObject *, PyObject *); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 9bb537d355055d..8f71eab44d914d 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -11,234 +11,264 @@ extern "C" { #define _EXIT_TRACE 300 #define _SET_IP 301 -#define _NOP NOP -#define _RESUME_CHECK RESUME_CHECK -#define _INSTRUMENTED_RESUME INSTRUMENTED_RESUME -#define _LOAD_FAST_CHECK LOAD_FAST_CHECK -#define _LOAD_FAST LOAD_FAST -#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST -#define _LOAD_CONST LOAD_CONST -#define _STORE_FAST STORE_FAST -#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST -#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST -#define _POP_TOP POP_TOP -#define _PUSH_NULL PUSH_NULL -#define _END_SEND END_SEND -#define _UNARY_NEGATIVE UNARY_NEGATIVE -#define _UNARY_NOT UNARY_NOT -#define _TO_BOOL 302 -#define _TO_BOOL_BOOL TO_BOOL_BOOL -#define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST TO_BOOL_LIST -#define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR TO_BOOL_STR -#define _TO_BOOL_ALWAYS_TRUE TO_BOOL_ALWAYS_TRUE -#define _UNARY_INVERT UNARY_INVERT -#define _GUARD_BOTH_INT 303 -#define _BINARY_OP_MULTIPLY_INT 304 -#define _BINARY_OP_ADD_INT 305 -#define _BINARY_OP_SUBTRACT_INT 306 -#define _GUARD_BOTH_FLOAT 307 -#define _BINARY_OP_MULTIPLY_FLOAT 308 -#define _BINARY_OP_ADD_FLOAT 309 -#define _BINARY_OP_SUBTRACT_FLOAT 310 -#define _GUARD_BOTH_UNICODE 311 -#define _BINARY_OP_ADD_UNICODE 312 -#define _BINARY_SUBSCR 313 +#define _BEFORE_ASYNC_WITH BEFORE_ASYNC_WITH +#define _BEFORE_WITH BEFORE_WITH +#define _BINARY_OP 302 +#define _BINARY_OP_ADD_FLOAT 303 +#define _BINARY_OP_ADD_INT 304 +#define _BINARY_OP_ADD_UNICODE 305 +#define _BINARY_OP_MULTIPLY_FLOAT 306 +#define _BINARY_OP_MULTIPLY_INT 307 +#define _BINARY_OP_SUBTRACT_FLOAT 308 +#define _BINARY_OP_SUBTRACT_INT 309 #define _BINARY_SLICE BINARY_SLICE -#define _STORE_SLICE STORE_SLICE +#define _BINARY_SUBSCR 310 +#define _BINARY_SUBSCR_DICT BINARY_SUBSCR_DICT +#define _BINARY_SUBSCR_GETITEM BINARY_SUBSCR_GETITEM #define _BINARY_SUBSCR_LIST_INT BINARY_SUBSCR_LIST_INT #define _BINARY_SUBSCR_STR_INT BINARY_SUBSCR_STR_INT #define _BINARY_SUBSCR_TUPLE_INT BINARY_SUBSCR_TUPLE_INT -#define _BINARY_SUBSCR_DICT BINARY_SUBSCR_DICT -#define _BINARY_SUBSCR_GETITEM BINARY_SUBSCR_GETITEM -#define _LIST_APPEND LIST_APPEND -#define _SET_ADD SET_ADD -#define _STORE_SUBSCR 314 -#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT -#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT -#define _DELETE_SUBSCR DELETE_SUBSCR +#define _BUILD_CONST_KEY_MAP BUILD_CONST_KEY_MAP +#define _BUILD_LIST BUILD_LIST +#define _BUILD_MAP BUILD_MAP +#define _BUILD_SET BUILD_SET +#define _BUILD_SLICE BUILD_SLICE +#define _BUILD_STRING BUILD_STRING +#define _BUILD_TUPLE BUILD_TUPLE +#define _CALL 311 +#define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT +#define _CALL_BUILTIN_CLASS CALL_BUILTIN_CLASS +#define _CALL_BUILTIN_FAST CALL_BUILTIN_FAST +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS CALL_BUILTIN_FAST_WITH_KEYWORDS +#define _CALL_BUILTIN_O CALL_BUILTIN_O +#define _CALL_FUNCTION_EX CALL_FUNCTION_EX #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _POP_FRAME 315 -#define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE -#define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST +#define _CALL_ISINSTANCE CALL_ISINSTANCE +#define _CALL_KW CALL_KW +#define _CALL_LEN CALL_LEN +#define _CALL_METHOD_DESCRIPTOR_FAST CALL_METHOD_DESCRIPTOR_FAST +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS +#define _CALL_METHOD_DESCRIPTOR_NOARGS CALL_METHOD_DESCRIPTOR_NOARGS +#define _CALL_METHOD_DESCRIPTOR_O CALL_METHOD_DESCRIPTOR_O +#define _CALL_PY_WITH_DEFAULTS CALL_PY_WITH_DEFAULTS +#define _CALL_STR_1 CALL_STR_1 +#define _CALL_TUPLE_1 CALL_TUPLE_1 +#define _CALL_TYPE_1 CALL_TYPE_1 +#define _CHECK_ATTR_CLASS 312 +#define _CHECK_ATTR_METHOD_LAZY_DICT 313 +#define _CHECK_ATTR_MODULE 314 +#define _CHECK_ATTR_WITH_HINT 315 +#define _CHECK_BUILTINS 316 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 317 +#define _CHECK_EG_MATCH CHECK_EG_MATCH +#define _CHECK_EXC_MATCH CHECK_EXC_MATCH +#define _CHECK_FUNCTION_EXACT_ARGS 318 +#define _CHECK_GLOBALS 319 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 320 +#define _CHECK_PEP_523 321 +#define _CHECK_STACK_SPACE 322 +#define _CHECK_VALIDITY 323 +#define _CHECK_VALIDITY_AND_SET_IP 324 +#define _COLD_EXIT 325 +#define _COMPARE_OP 326 +#define _COMPARE_OP_FLOAT 327 +#define _COMPARE_OP_INT 328 +#define _COMPARE_OP_STR 329 +#define _CONTAINS_OP CONTAINS_OP +#define _CONVERT_VALUE CONVERT_VALUE +#define _COPY COPY +#define _COPY_FREE_VARS COPY_FREE_VARS +#define _DELETE_ATTR DELETE_ATTR +#define _DELETE_DEREF DELETE_DEREF +#define _DELETE_FAST DELETE_FAST +#define _DELETE_GLOBAL DELETE_GLOBAL +#define _DELETE_NAME DELETE_NAME +#define _DELETE_SUBSCR DELETE_SUBSCR +#define _DICT_MERGE DICT_MERGE +#define _DICT_UPDATE DICT_UPDATE +#define _END_SEND END_SEND +#define _EXIT_INIT_CHECK EXIT_INIT_CHECK +#define _FATAL_ERROR 330 +#define _FORMAT_SIMPLE FORMAT_SIMPLE +#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC +#define _FOR_ITER 331 +#define _FOR_ITER_GEN FOR_ITER_GEN +#define _FOR_ITER_TIER_TWO 332 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE -#define _SEND 316 -#define _SEND_GEN SEND_GEN +#define _GET_ITER GET_ITER +#define _GET_LEN GET_LEN +#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER +#define _GUARD_BOTH_FLOAT 333 +#define _GUARD_BOTH_INT 334 +#define _GUARD_BOTH_UNICODE 335 +#define _GUARD_BUILTINS_VERSION 336 +#define _GUARD_DORV_VALUES 337 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 338 +#define _GUARD_GLOBALS_VERSION 339 +#define _GUARD_IS_FALSE_POP 340 +#define _GUARD_IS_NONE_POP 341 +#define _GUARD_IS_NOT_NONE_POP 342 +#define _GUARD_IS_TRUE_POP 343 +#define _GUARD_KEYS_VERSION 344 +#define _GUARD_NOT_EXHAUSTED_LIST 345 +#define _GUARD_NOT_EXHAUSTED_RANGE 346 +#define _GUARD_NOT_EXHAUSTED_TUPLE 347 +#define _GUARD_TYPE_VERSION 348 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 349 +#define _INIT_CALL_PY_EXACT_ARGS 350 +#define _INIT_CALL_PY_EXACT_ARGS_0 351 +#define _INIT_CALL_PY_EXACT_ARGS_1 352 +#define _INIT_CALL_PY_EXACT_ARGS_2 353 +#define _INIT_CALL_PY_EXACT_ARGS_3 354 +#define _INIT_CALL_PY_EXACT_ARGS_4 355 +#define _INSTRUMENTED_CALL INSTRUMENTED_CALL +#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX +#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW +#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER +#define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION +#define _INSTRUMENTED_JUMP_BACKWARD INSTRUMENTED_JUMP_BACKWARD +#define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD +#define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR +#define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE +#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE +#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE +#define _INSTRUMENTED_RESUME INSTRUMENTED_RESUME +#define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST +#define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _POP_EXCEPT POP_EXCEPT +#define _INTERNAL_INCREMENT_OPT_COUNTER 356 +#define _IS_NONE 357 +#define _IS_OP IS_OP +#define _ITER_CHECK_LIST 358 +#define _ITER_CHECK_RANGE 359 +#define _ITER_CHECK_TUPLE 360 +#define _ITER_JUMP_LIST 361 +#define _ITER_JUMP_RANGE 362 +#define _ITER_JUMP_TUPLE 363 +#define _ITER_NEXT_LIST 364 +#define _ITER_NEXT_RANGE 365 +#define _ITER_NEXT_TUPLE 366 +#define _JUMP_TO_TOP 367 +#define _LIST_APPEND LIST_APPEND +#define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR +#define _LOAD_ATTR 368 +#define _LOAD_ATTR_CLASS 369 +#define _LOAD_ATTR_CLASS_0 370 +#define _LOAD_ATTR_CLASS_1 371 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN +#define _LOAD_ATTR_INSTANCE_VALUE 372 +#define _LOAD_ATTR_INSTANCE_VALUE_0 373 +#define _LOAD_ATTR_INSTANCE_VALUE_1 374 +#define _LOAD_ATTR_METHOD_LAZY_DICT 375 +#define _LOAD_ATTR_METHOD_NO_DICT 376 +#define _LOAD_ATTR_METHOD_WITH_VALUES 377 +#define _LOAD_ATTR_MODULE 378 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 379 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 380 +#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY +#define _LOAD_ATTR_SLOT 381 +#define _LOAD_ATTR_SLOT_0 382 +#define _LOAD_ATTR_SLOT_1 383 +#define _LOAD_ATTR_WITH_HINT 384 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _STORE_NAME STORE_NAME -#define _DELETE_NAME DELETE_NAME -#define _UNPACK_SEQUENCE 317 -#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE -#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE -#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST -#define _UNPACK_EX UNPACK_EX -#define _STORE_ATTR 318 -#define _DELETE_ATTR DELETE_ATTR -#define _STORE_GLOBAL STORE_GLOBAL -#define _DELETE_GLOBAL DELETE_GLOBAL -#define _LOAD_LOCALS LOAD_LOCALS +#define _LOAD_CONST LOAD_CONST +#define _LOAD_CONST_INLINE 385 +#define _LOAD_CONST_INLINE_BORROW 386 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 387 +#define _LOAD_CONST_INLINE_WITH_NULL 388 +#define _LOAD_DEREF LOAD_DEREF +#define _LOAD_FAST 389 +#define _LOAD_FAST_0 390 +#define _LOAD_FAST_1 391 +#define _LOAD_FAST_2 392 +#define _LOAD_FAST_3 393 +#define _LOAD_FAST_4 394 +#define _LOAD_FAST_5 395 +#define _LOAD_FAST_6 396 +#define _LOAD_FAST_7 397 +#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR +#define _LOAD_FAST_CHECK LOAD_FAST_CHECK +#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST +#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS +#define _LOAD_GLOBAL 398 +#define _LOAD_GLOBAL_BUILTINS 399 +#define _LOAD_GLOBAL_MODULE 400 +#define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_GLOBAL 319 -#define _GUARD_GLOBALS_VERSION 320 -#define _GUARD_BUILTINS_VERSION 321 -#define _LOAD_GLOBAL_MODULE 322 -#define _LOAD_GLOBAL_BUILTINS 323 -#define _DELETE_FAST DELETE_FAST -#define _MAKE_CELL MAKE_CELL -#define _DELETE_DEREF DELETE_DEREF -#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF -#define _LOAD_DEREF LOAD_DEREF -#define _STORE_DEREF STORE_DEREF -#define _COPY_FREE_VARS COPY_FREE_VARS -#define _BUILD_STRING BUILD_STRING -#define _BUILD_TUPLE BUILD_TUPLE -#define _BUILD_LIST BUILD_LIST -#define _LIST_EXTEND LIST_EXTEND -#define _SET_UPDATE SET_UPDATE -#define _BUILD_SET BUILD_SET -#define _BUILD_MAP BUILD_MAP -#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS -#define _BUILD_CONST_KEY_MAP BUILD_CONST_KEY_MAP -#define _DICT_UPDATE DICT_UPDATE -#define _DICT_MERGE DICT_MERGE -#define _MAP_ADD MAP_ADD -#define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _LOAD_ATTR 324 -#define _GUARD_TYPE_VERSION 325 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 326 -#define _LOAD_ATTR_INSTANCE_VALUE 327 -#define _CHECK_ATTR_MODULE 328 -#define _LOAD_ATTR_MODULE 329 -#define _CHECK_ATTR_WITH_HINT 330 -#define _LOAD_ATTR_WITH_HINT 331 -#define _LOAD_ATTR_SLOT 332 -#define _CHECK_ATTR_CLASS 333 -#define _LOAD_ATTR_CLASS 334 -#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _GUARD_DORV_VALUES 335 -#define _STORE_ATTR_INSTANCE_VALUE 336 -#define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT -#define _STORE_ATTR_SLOT 337 -#define _COMPARE_OP 338 -#define _COMPARE_OP_FLOAT COMPARE_OP_FLOAT -#define _COMPARE_OP_INT COMPARE_OP_INT -#define _COMPARE_OP_STR COMPARE_OP_STR -#define _IS_OP IS_OP -#define _CONTAINS_OP CONTAINS_OP -#define _CHECK_EG_MATCH CHECK_EG_MATCH -#define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _JUMP_BACKWARD JUMP_BACKWARD -#define _POP_JUMP_IF_FALSE 339 -#define _POP_JUMP_IF_TRUE 340 -#define _IS_NONE 341 -#define _GET_LEN GET_LEN +#define _MAKE_CELL MAKE_CELL +#define _MAKE_FUNCTION MAKE_FUNCTION +#define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS +#define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MATCH_KEYS MATCH_KEYS -#define _GET_ITER GET_ITER -#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _FOR_ITER 342 -#define _FOR_ITER_TIER_TWO 343 -#define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER -#define _ITER_CHECK_LIST 344 -#define _ITER_JUMP_LIST 345 -#define _GUARD_NOT_EXHAUSTED_LIST 346 -#define _ITER_NEXT_LIST 347 -#define _ITER_CHECK_TUPLE 348 -#define _ITER_JUMP_TUPLE 349 -#define _GUARD_NOT_EXHAUSTED_TUPLE 350 -#define _ITER_NEXT_TUPLE 351 -#define _ITER_CHECK_RANGE 352 -#define _ITER_JUMP_RANGE 353 -#define _GUARD_NOT_EXHAUSTED_RANGE 354 -#define _ITER_NEXT_RANGE 355 -#define _FOR_ITER_GEN FOR_ITER_GEN -#define _BEFORE_ASYNC_WITH BEFORE_ASYNC_WITH -#define _BEFORE_WITH BEFORE_WITH -#define _WITH_EXCEPT_START WITH_EXCEPT_START +#define _NOP NOP +#define _POP_EXCEPT POP_EXCEPT +#define _POP_FRAME 401 +#define _POP_JUMP_IF_FALSE 402 +#define _POP_JUMP_IF_TRUE 403 +#define _POP_TOP POP_TOP +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 404 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 356 -#define _GUARD_KEYS_VERSION 357 -#define _LOAD_ATTR_METHOD_WITH_VALUES 358 -#define _LOAD_ATTR_METHOD_NO_DICT 359 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 360 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 361 -#define _CHECK_ATTR_METHOD_LAZY_DICT 362 -#define _LOAD_ATTR_METHOD_LAZY_DICT 363 -#define _INSTRUMENTED_CALL INSTRUMENTED_CALL -#define _CALL 364 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 365 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 366 -#define _CHECK_PEP_523 367 -#define _CHECK_FUNCTION_EXACT_ARGS 368 -#define _CHECK_STACK_SPACE 369 -#define _INIT_CALL_PY_EXACT_ARGS 370 -#define _PUSH_FRAME 371 -#define _CALL_PY_WITH_DEFAULTS CALL_PY_WITH_DEFAULTS -#define _CALL_TYPE_1 CALL_TYPE_1 -#define _CALL_STR_1 CALL_STR_1 -#define _CALL_TUPLE_1 CALL_TUPLE_1 -#define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT -#define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _CALL_BUILTIN_CLASS CALL_BUILTIN_CLASS -#define _CALL_BUILTIN_O CALL_BUILTIN_O -#define _CALL_BUILTIN_FAST CALL_BUILTIN_FAST -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS CALL_BUILTIN_FAST_WITH_KEYWORDS -#define _CALL_LEN CALL_LEN -#define _CALL_ISINSTANCE CALL_ISINSTANCE -#define _CALL_METHOD_DESCRIPTOR_O CALL_METHOD_DESCRIPTOR_O -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS -#define _CALL_METHOD_DESCRIPTOR_NOARGS CALL_METHOD_DESCRIPTOR_NOARGS -#define _CALL_METHOD_DESCRIPTOR_FAST CALL_METHOD_DESCRIPTOR_FAST -#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW -#define _CALL_KW CALL_KW -#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX -#define _CALL_FUNCTION_EX CALL_FUNCTION_EX -#define _MAKE_FUNCTION MAKE_FUNCTION +#define _PUSH_FRAME 405 +#define _PUSH_NULL PUSH_NULL +#define _RESUME_CHECK RESUME_CHECK +#define _SAVE_RETURN_OFFSET 406 +#define _SEND 407 +#define _SEND_GEN SEND_GEN +#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS +#define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE -#define _BUILD_SLICE BUILD_SLICE -#define _CONVERT_VALUE CONVERT_VALUE -#define _FORMAT_SIMPLE FORMAT_SIMPLE -#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _COPY COPY -#define _BINARY_OP 372 +#define _SET_UPDATE SET_UPDATE +#define _START_EXECUTOR 408 +#define _STORE_ATTR 409 +#define _STORE_ATTR_INSTANCE_VALUE 410 +#define _STORE_ATTR_SLOT 411 +#define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT +#define _STORE_DEREF STORE_DEREF +#define _STORE_FAST 412 +#define _STORE_FAST_0 413 +#define _STORE_FAST_1 414 +#define _STORE_FAST_2 415 +#define _STORE_FAST_3 416 +#define _STORE_FAST_4 417 +#define _STORE_FAST_5 418 +#define _STORE_FAST_6 419 +#define _STORE_FAST_7 420 +#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST +#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST +#define _STORE_GLOBAL STORE_GLOBAL +#define _STORE_NAME STORE_NAME +#define _STORE_SLICE STORE_SLICE +#define _STORE_SUBSCR 421 +#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT +#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION -#define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD -#define _INSTRUMENTED_JUMP_BACKWARD INSTRUMENTED_JUMP_BACKWARD -#define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE -#define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE -#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE -#define _GUARD_IS_TRUE_POP 373 -#define _GUARD_IS_FALSE_POP 374 -#define _GUARD_IS_NONE_POP 375 -#define _GUARD_IS_NOT_NONE_POP 376 -#define _JUMP_TO_TOP 377 -#define _SAVE_RETURN_OFFSET 378 -#define _CHECK_VALIDITY 379 -#define _LOAD_CONST_INLINE 380 -#define _LOAD_CONST_INLINE_BORROW 381 -#define _LOAD_CONST_INLINE_WITH_NULL 382 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 383 -#define _CHECK_GLOBALS 384 -#define _CHECK_BUILTINS 385 -#define _INTERNAL_INCREMENT_OPT_COUNTER 386 -#define _CHECK_VALIDITY_AND_SET_IP 387 -#define MAX_UOP_ID 387 +#define _TO_BOOL 422 +#define _TO_BOOL_ALWAYS_TRUE TO_BOOL_ALWAYS_TRUE +#define _TO_BOOL_BOOL TO_BOOL_BOOL +#define _TO_BOOL_INT TO_BOOL_INT +#define _TO_BOOL_LIST TO_BOOL_LIST +#define _TO_BOOL_NONE TO_BOOL_NONE +#define _TO_BOOL_STR TO_BOOL_STR +#define _UNARY_INVERT UNARY_INVERT +#define _UNARY_NEGATIVE UNARY_NEGATIVE +#define _UNARY_NOT UNARY_NOT +#define _UNPACK_EX UNPACK_EX +#define _UNPACK_SEQUENCE 423 +#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST +#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE +#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE +#define _WITH_EXCEPT_START WITH_EXCEPT_START +#define MAX_UOP_ID 423 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 163a0320aa2298..7f921a6cd3f4c8 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -12,6 +12,7 @@ extern "C" { #include #include "pycore_uop_ids.h" extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; +extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1]; extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; #ifdef NEED_OPCODE_METADATA @@ -19,10 +20,26 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, + [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG, + [_STORE_FAST_0] = HAS_LOCAL_FLAG, + [_STORE_FAST_1] = HAS_LOCAL_FLAG, + [_STORE_FAST_2] = HAS_LOCAL_FLAG, + [_STORE_FAST_3] = HAS_LOCAL_FLAG, + [_STORE_FAST_4] = HAS_LOCAL_FLAG, + [_STORE_FAST_5] = HAS_LOCAL_FLAG, + [_STORE_FAST_6] = HAS_LOCAL_FLAG, + [_STORE_FAST_7] = HAS_LOCAL_FLAG, [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, @@ -32,22 +49,22 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_TO_BOOL_INT] = HAS_DEOPT_FLAG, - [_TO_BOOL_LIST] = HAS_DEOPT_FLAG, - [_TO_BOOL_NONE] = HAS_DEOPT_FLAG, - [_TO_BOOL_STR] = HAS_DEOPT_FLAG, - [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG, + [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_TO_BOOL_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_LIST] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_NONE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_STR] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, - [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -112,23 +129,29 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT_1] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, [_CHECK_ATTR_CLASS] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG, + [_LOAD_ATTR_CLASS_0] = 0, + [_LOAD_ATTR_CLASS_1] = 0, + [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, [_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, [_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_COMPARE_OP_STR] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_COMPARE_OP_STR] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_IS_OP] = HAS_ARG_FLAG, [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EG_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -168,6 +191,11 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_PEP_523] = HAS_DEOPT_FLAG, [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_CHECK_STACK_SPACE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_0] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_1] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_PUSH_FRAME] = HAS_ESCAPES_FLAG, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, @@ -193,25 +221,35 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG, - [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG, - [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG, - [_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG, + [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG, [_SET_IP] = 0, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, - [_EXIT_TRACE] = HAS_DEOPT_FLAG, + [_EXIT_TRACE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_WITH_NULL] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, [_CHECK_GLOBALS] = HAS_DEOPT_FLAG, [_CHECK_BUILTINS] = HAS_DEOPT_FLAG, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, + [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_START_EXECUTOR] = 0, + [_FATAL_ERROR] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, }; +const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { + [_LOAD_FAST] = 8, + [_STORE_FAST] = 8, + [_INIT_CALL_PY_EXACT_ARGS] = 5, +}; + const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BEFORE_ASYNC_WITH] = "_BEFORE_ASYNC_WITH", [_BEFORE_WITH] = "_BEFORE_WITH", @@ -266,6 +304,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", [_CHECK_VALIDITY_AND_SET_IP] = "_CHECK_VALIDITY_AND_SET_IP", + [_COLD_EXIT] = "_COLD_EXIT", [_COMPARE_OP] = "_COMPARE_OP", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", [_COMPARE_OP_INT] = "_COMPARE_OP_INT", @@ -285,6 +324,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_END_SEND] = "_END_SEND", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", + [_FATAL_ERROR] = "_FATAL_ERROR", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", @@ -312,6 +352,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_INIT_CALL_PY_EXACT_ARGS_0] = "_INIT_CALL_PY_EXACT_ARGS_0", + [_INIT_CALL_PY_EXACT_ARGS_1] = "_INIT_CALL_PY_EXACT_ARGS_1", + [_INIT_CALL_PY_EXACT_ARGS_2] = "_INIT_CALL_PY_EXACT_ARGS_2", + [_INIT_CALL_PY_EXACT_ARGS_3] = "_INIT_CALL_PY_EXACT_ARGS_3", + [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", [_INTERNAL_INCREMENT_OPT_COUNTER] = "_INTERNAL_INCREMENT_OPT_COUNTER", [_IS_NONE] = "_IS_NONE", [_IS_OP] = "_IS_OP", @@ -327,7 +372,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_ASSERTION_ERROR] = "_LOAD_ASSERTION_ERROR", [_LOAD_ATTR] = "_LOAD_ATTR", [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", + [_LOAD_ATTR_CLASS_0] = "_LOAD_ATTR_CLASS_0", + [_LOAD_ATTR_CLASS_1] = "_LOAD_ATTR_CLASS_1", [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_LOAD_ATTR_INSTANCE_VALUE_0] = "_LOAD_ATTR_INSTANCE_VALUE_0", + [_LOAD_ATTR_INSTANCE_VALUE_1] = "_LOAD_ATTR_INSTANCE_VALUE_1", [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", @@ -335,6 +384,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_LOAD_ATTR_SLOT_0] = "_LOAD_ATTR_SLOT_0", + [_LOAD_ATTR_SLOT_1] = "_LOAD_ATTR_SLOT_1", [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", [_LOAD_CONST] = "_LOAD_CONST", @@ -344,6 +395,14 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_CONST_INLINE_WITH_NULL] = "_LOAD_CONST_INLINE_WITH_NULL", [_LOAD_DEREF] = "_LOAD_DEREF", [_LOAD_FAST] = "_LOAD_FAST", + [_LOAD_FAST_0] = "_LOAD_FAST_0", + [_LOAD_FAST_1] = "_LOAD_FAST_1", + [_LOAD_FAST_2] = "_LOAD_FAST_2", + [_LOAD_FAST_3] = "_LOAD_FAST_3", + [_LOAD_FAST_4] = "_LOAD_FAST_4", + [_LOAD_FAST_5] = "_LOAD_FAST_5", + [_LOAD_FAST_6] = "_LOAD_FAST_6", + [_LOAD_FAST_7] = "_LOAD_FAST_7", [_LOAD_FAST_AND_CLEAR] = "_LOAD_FAST_AND_CLEAR", [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", [_LOAD_FAST_LOAD_FAST] = "_LOAD_FAST_LOAD_FAST", @@ -367,6 +426,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_POP_EXCEPT] = "_POP_EXCEPT", [_POP_FRAME] = "_POP_FRAME", [_POP_TOP] = "_POP_TOP", + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", [_PUSH_FRAME] = "_PUSH_FRAME", [_PUSH_NULL] = "_PUSH_NULL", @@ -377,11 +437,20 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", [_SET_IP] = "_SET_IP", [_SET_UPDATE] = "_SET_UPDATE", + [_START_EXECUTOR] = "_START_EXECUTOR", [_STORE_ATTR] = "_STORE_ATTR", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", [_STORE_DEREF] = "_STORE_DEREF", [_STORE_FAST] = "_STORE_FAST", + [_STORE_FAST_0] = "_STORE_FAST_0", + [_STORE_FAST_1] = "_STORE_FAST_1", + [_STORE_FAST_2] = "_STORE_FAST_2", + [_STORE_FAST_3] = "_STORE_FAST_3", + [_STORE_FAST_4] = "_STORE_FAST_4", + [_STORE_FAST_5] = "_STORE_FAST_5", + [_STORE_FAST_6] = "_STORE_FAST_6", + [_STORE_FAST_7] = "_STORE_FAST_7", [_STORE_FAST_LOAD_FAST] = "_STORE_FAST_LOAD_FAST", [_STORE_FAST_STORE_FAST] = "_STORE_FAST_STORE_FAST", [_STORE_GLOBAL] = "_STORE_GLOBAL", diff --git a/Include/methodobject.h b/Include/methodobject.h index 452f891a7aba83..39272815b127f4 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -31,7 +31,7 @@ typedef PyObject *(*PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, // Note that the underscore-prefixed names were documented in public docs; // people may be using them. typedef PyCFunctionFast _PyCFunctionFast; -typedef PyCFunctionWithKeywords _PyCFunctionWithKeywords; +typedef PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords; // Cast an function to the PyCFunction type to use it with PyMethodDef. // diff --git a/Include/objimpl.h b/Include/objimpl.h index ff5fa7a8c1d3d8..56472a72e42d34 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -153,25 +153,6 @@ PyAPI_FUNC(int) PyGC_Enable(void); PyAPI_FUNC(int) PyGC_Disable(void); PyAPI_FUNC(int) PyGC_IsEnabled(void); - -#if !defined(Py_LIMITED_API) -/* Visit all live GC-capable objects, similar to gc.get_objects(None). The - * supplied callback is called on every such object with the void* arg set - * to the supplied arg. Returning 0 from the callback ends iteration, returning - * 1 allows iteration to continue. Returning any other value may result in - * undefined behaviour. - * - * If new objects are (de)allocated by the callback it is undefined if they - * will be visited. - - * Garbage collection is disabled during operation. Explicitly running a - * collection in the callback may lead to undefined behaviour e.g. visiting the - * same objects multiple times or not at all. - */ -typedef int (*gcvisitobjects_t)(PyObject*, void*); -PyAPI_FUNC(void) PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void* arg); -#endif - /* Test if a type has a GC head */ #define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) diff --git a/Include/pyexpat.h b/Include/pyexpat.h index 07020b5dc964cb..e0cff33b53227a 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -3,7 +3,7 @@ /* note: you must import expat.h before importing this module! */ -#define PyExpat_CAPI_MAGIC "pyexpat.expat_CAPI 1.1" +#define PyExpat_CAPI_MAGIC "pyexpat.expat_CAPI 1.2" #define PyExpat_CAPSULE_NAME "pyexpat.expat_CAPI" struct PyExpat_CAPI @@ -48,8 +48,10 @@ struct PyExpat_CAPI enum XML_Status (*SetEncoding)(XML_Parser parser, const XML_Char *encoding); int (*DefaultUnknownEncodingHandler)( void *encodingHandlerData, const XML_Char *name, XML_Encoding *info); - /* might be none for expat < 2.1.0 */ + /* might be NULL for expat < 2.1.0 */ int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt); + /* might be NULL for expat < 2.6.0 */ + XML_Bool (*SetReparseDeferralEnabled)(XML_Parser parser, XML_Bool enabled); /* always add new stuff to the end! */ }; diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 8a0d0dc4b1a0b8..a3fede699218a1 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -1197,7 +1197,8 @@ def _readinto(self, buf, read1): return written def tell(self): - return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos + # GH-95782: Keep return value non-negative + return max(_BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos, 0) def seek(self, pos, whence=0): if whence not in valid_seek_flags: diff --git a/Lib/argparse.py b/Lib/argparse.py index 04ee3b19aca755..0dbdd67a82f391 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -223,7 +223,8 @@ def format_help(self): # add the heading if the section was non-empty if self.heading is not SUPPRESS and self.heading is not None: current_indent = self.formatter._current_indent - heading = '%*s%s:\n' % (current_indent, '', self.heading) + heading_text = _('%(heading)s:') % dict(heading=self.heading) + heading = '%*s%s\n' % (current_indent, '', heading_text) else: heading = '' @@ -413,6 +414,8 @@ def _format_actions_usage(self, actions, groups): suppressed_actions_count += 1 exposed_actions_count = group_action_count - suppressed_actions_count + if not exposed_actions_count: + continue if not group.required: if start in inserts: @@ -698,14 +701,6 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter): """ def _get_help_string(self, action): - """ - Add the default value to the option help message. - - ArgumentDefaultsHelpFormatter and BooleanOptionalAction when it isn't - already present. This code will do that, detecting cornercases to - prevent duplicates or cases where it wouldn't make sense to the end - user. - """ help = action.help if help is None: help = '' @@ -714,7 +709,7 @@ def _get_help_string(self, action): if action.default is not SUPPRESS: defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] if action.option_strings or action.nargs in defaulting_nargs: - help += ' (default: %(default)s)' + help += _(' (default: %(default)s)') return help @@ -1165,8 +1160,10 @@ def __init__(self, version=None, dest=SUPPRESS, default=SUPPRESS, - help="show program's version number and exit", + help=None, deprecated=False): + if help is None: + help = _("show program's version number and exit") super(_VersionAction, self).__init__( option_strings=option_strings, dest=dest, @@ -2033,7 +2030,7 @@ def consume_optional(start_index): # get the optional identified at this index option_tuple = option_string_indices[start_index] - action, option_string, explicit_arg = option_tuple + action, option_string, sep, explicit_arg = option_tuple # identify additional optionals in the same arg string # (e.g. -xyz is the same as -x -y -z if no args are required) @@ -2060,18 +2057,27 @@ def consume_optional(start_index): and option_string[1] not in chars and explicit_arg != '' ): + if sep or explicit_arg[0] in chars: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) action_tuples.append((action, [], option_string)) char = option_string[0] option_string = char + explicit_arg[0] - new_explicit_arg = explicit_arg[1:] or None optionals_map = self._option_string_actions if option_string in optionals_map: action = optionals_map[option_string] - explicit_arg = new_explicit_arg + explicit_arg = explicit_arg[1:] + if not explicit_arg: + sep = explicit_arg = None + elif explicit_arg[0] == '=': + sep = '=' + explicit_arg = explicit_arg[1:] + else: + sep = '' else: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - + extras.append(char + explicit_arg) + stop = start_index + 1 + break # if the action expect exactly one argument, we've # successfully matched the option; exit the loop elif arg_count == 1: @@ -2147,10 +2153,11 @@ def consume_positionals(start_index): while start_index <= max_option_string_index: # consume any Positionals preceding the next option - next_option_string_index = min([ - index - for index in option_string_indices - if index >= start_index]) + next_option_string_index = start_index + while next_option_string_index <= max_option_string_index: + if next_option_string_index in option_string_indices: + break + next_option_string_index += 1 if start_index != next_option_string_index: positionals_end_index = consume_positionals(start_index) @@ -2299,18 +2306,17 @@ def _parse_optional(self, arg_string): # if the option string is present in the parser, return the action if arg_string in self._option_string_actions: action = self._option_string_actions[arg_string] - return action, arg_string, None + return action, arg_string, None, None # if it's just a single character, it was meant to be positional if len(arg_string) == 1: return None # if the option string before the "=" is present, return the action - if '=' in arg_string: - option_string, explicit_arg = arg_string.split('=', 1) - if option_string in self._option_string_actions: - action = self._option_string_actions[option_string] - return action, option_string, explicit_arg + option_string, sep, explicit_arg = arg_string.partition('=') + if sep and option_string in self._option_string_actions: + action = self._option_string_actions[option_string] + return action, option_string, sep, explicit_arg # search through all possible prefixes of the option string # and all actions in the parser for possible interpretations @@ -2319,7 +2325,7 @@ def _parse_optional(self, arg_string): # if multiple actions match, the option string was ambiguous if len(option_tuples) > 1: options = ', '.join([option_string - for action, option_string, explicit_arg in option_tuples]) + for action, option_string, sep, explicit_arg in option_tuples]) args = {'option': arg_string, 'matches': options} msg = _('ambiguous option: %(option)s could match %(matches)s') self.error(msg % args) @@ -2343,7 +2349,7 @@ def _parse_optional(self, arg_string): # it was meant to be an optional but there is no such option # in this parser (though it might be a valid option in a subparser) - return None, arg_string, None + return None, arg_string, None, None def _get_option_tuples(self, option_string): result = [] @@ -2353,15 +2359,13 @@ def _get_option_tuples(self, option_string): chars = self.prefix_chars if option_string[0] in chars and option_string[1] in chars: if self.allow_abbrev: - if '=' in option_string: - option_prefix, explicit_arg = option_string.split('=', 1) - else: - option_prefix = option_string - explicit_arg = None + option_prefix, sep, explicit_arg = option_string.partition('=') + if not sep: + sep = explicit_arg = None for option_string in self._option_string_actions: if option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg + tup = action, option_string, sep, explicit_arg result.append(tup) # single character options can be concatenated with their arguments @@ -2369,18 +2373,17 @@ def _get_option_tuples(self, option_string): # separate elif option_string[0] in chars and option_string[1] not in chars: option_prefix = option_string - explicit_arg = None short_option_prefix = option_string[:2] short_explicit_arg = option_string[2:] for option_string in self._option_string_actions: if option_string == short_option_prefix: action = self._option_string_actions[option_string] - tup = action, option_string, short_explicit_arg + tup = action, option_string, '', short_explicit_arg result.append(tup) elif option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg + tup = action, option_string, None, None result.append(tup) # shouldn't ever get here diff --git a/Lib/ast.py b/Lib/ast.py index 43703a8325cc5e..b8c4ce6f919e6b 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1269,14 +1269,18 @@ def visit_JoinedStr(self, node): quote_type = quote_types[0] self.write(f"{quote_type}{value}{quote_type}") - def _write_fstring_inner(self, node, escape_newlines=False): + def _write_fstring_inner(self, node, is_format_spec=False): if isinstance(node, JoinedStr): # for both the f-string itself, and format_spec for value in node.values: - self._write_fstring_inner(value, escape_newlines=escape_newlines) + self._write_fstring_inner(value, is_format_spec=is_format_spec) elif isinstance(node, Constant) and isinstance(node.value, str): value = node.value.replace("{", "{{").replace("}", "}}") - if escape_newlines: + + if is_format_spec: + value = value.replace("\\", "\\\\") + value = value.replace("'", "\\'") + value = value.replace('"', '\\"') value = value.replace("\n", "\\n") self.write(value) elif isinstance(node, FormattedValue): @@ -1300,10 +1304,7 @@ def unparse_inner(inner): self.write(f"!{chr(node.conversion)}") if node.format_spec: self.write(":") - self._write_fstring_inner( - node.format_spec, - escape_newlines=True - ) + self._write_fstring_inner(node.format_spec, is_format_spec=True) def visit_Name(self, node): self.write(node.id) diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 18bb87a5bc4ffd..cbc1d7c93ef76f 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -3,6 +3,7 @@ import code import concurrent.futures import inspect +import site import sys import threading import types @@ -109,6 +110,21 @@ def run(self): except ImportError: pass + interactive_hook = getattr(sys, "__interactivehook__", None) + + if interactive_hook is not None: + interactive_hook() + + if interactive_hook is site.register_readline: + # Fix the completer function to use the interactive console locals + try: + import rlcompleter + except: + pass + else: + completer = rlcompleter.Completer(console.locals) + readline.set_completer(completer.complete) + repl_thread = REPLThread() repl_thread.daemon = True repl_thread.start() diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index aadc4f478f8b56..6c5cf28e7c59d4 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -45,6 +45,7 @@ from . import sslproto from . import staggered from . import tasks +from . import timeouts from . import transports from . import trsock from .log import logger @@ -598,23 +599,24 @@ async def shutdown_default_executor(self, timeout=None): thread = threading.Thread(target=self._do_shutdown, args=(future,)) thread.start() try: - await future - finally: - thread.join(timeout) - - if thread.is_alive(): + async with timeouts.timeout(timeout): + await future + except TimeoutError: warnings.warn("The executor did not finishing joining " - f"its threads within {timeout} seconds.", - RuntimeWarning, stacklevel=2) + f"its threads within {timeout} seconds.", + RuntimeWarning, stacklevel=2) self._default_executor.shutdown(wait=False) + else: + thread.join() def _do_shutdown(self, future): try: self._default_executor.shutdown(wait=True) if not self.is_closed(): - self.call_soon_threadsafe(future.set_result, None) + self.call_soon_threadsafe(futures._set_result_unless_cancelled, + future, None) except Exception as ex: - if not self.is_closed(): + if not self.is_closed() and not future.cancelled(): self.call_soon_threadsafe(future.set_exception, ex) def _check_running(self): diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 072a99fee123c3..680749325025db 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -54,7 +54,8 @@ def _repr_info(self): info.append('cancelled') if self._callback is not None: info.append(format_helpers._format_callback_source( - self._callback, self._args)) + self._callback, self._args, + debug=self._loop.get_debug())) if self._source_traceback: frame = self._source_traceback[-1] info.append(f'created at {frame[0]}:{frame[1]}') @@ -90,7 +91,8 @@ def _run(self): raise except BaseException as exc: cb = format_helpers._format_callback_source( - self._callback, self._args) + self._callback, self._args, + debug=self._loop.get_debug()) msg = f'Exception in callback {cb}' context = { 'message': msg, diff --git a/Lib/asyncio/format_helpers.py b/Lib/asyncio/format_helpers.py index 27d11fd4fa9553..93737b7708a67b 100644 --- a/Lib/asyncio/format_helpers.py +++ b/Lib/asyncio/format_helpers.py @@ -19,19 +19,26 @@ def _get_function_source(func): return None -def _format_callback_source(func, args): - func_repr = _format_callback(func, args, None) +def _format_callback_source(func, args, *, debug=False): + func_repr = _format_callback(func, args, None, debug=debug) source = _get_function_source(func) if source: func_repr += f' at {source[0]}:{source[1]}' return func_repr -def _format_args_and_kwargs(args, kwargs): +def _format_args_and_kwargs(args, kwargs, *, debug=False): """Format function arguments and keyword arguments. Special case for a single parameter: ('hello',) is formatted as ('hello'). + + Note that this function only returns argument details when + debug=True is specified, as arguments may contain sensitive + information. """ + if not debug: + return '()' + # use reprlib to limit the length of the output items = [] if args: @@ -41,10 +48,11 @@ def _format_args_and_kwargs(args, kwargs): return '({})'.format(', '.join(items)) -def _format_callback(func, args, kwargs, suffix=''): +def _format_callback(func, args, kwargs, *, debug=False, suffix=''): if isinstance(func, functools.partial): - suffix = _format_args_and_kwargs(args, kwargs) + suffix - return _format_callback(func.func, func.args, func.keywords, suffix) + suffix = _format_args_and_kwargs(args, kwargs, debug=debug) + suffix + return _format_callback(func.func, func.args, func.keywords, + debug=debug, suffix=suffix) if hasattr(func, '__qualname__') and func.__qualname__: func_repr = func.__qualname__ @@ -53,7 +61,7 @@ def _format_callback(func, args, kwargs, suffix=''): else: func_repr = repr(func) - func_repr += _format_args_and_kwargs(args, kwargs) + func_repr += _format_args_and_kwargs(args, kwargs, debug=debug) if suffix: func_repr += suffix return func_repr diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 1e2a730cf368a9..a512db6367b20a 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -487,9 +487,6 @@ def sendto(self, data, addr=None): raise TypeError('data argument must be bytes-like object (%r)', type(data)) - if not data: - return - if self._address is not None and addr not in (None, self._address): raise ValueError( f'Invalid address: must be None or {self._address}') @@ -502,7 +499,7 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + self._buffer_size += len(data) + 8 # include header bytes if self._write_fut is None: # No current write operations are active, kick one off diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 10fbdd76e93f79..8e888d26ea0737 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -1241,8 +1241,6 @@ def sendto(self, data, addr=None): if not isinstance(data, (bytes, bytearray, memoryview)): raise TypeError(f'data argument must be a bytes-like object, ' f'not {type(data).__name__!r}') - if not data: - return if self._address: if addr not in (None, self._address): @@ -1278,7 +1276,7 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + self._buffer_size += len(data) + 8 # include header bytes self._maybe_pause_protocol() def _sendto_ready(self): diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index df58b7a799a5ad..3fe52dbac25c91 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -201,7 +201,6 @@ def __init__(self, stream_reader, client_connected_cb=None, loop=None): # is established. self._strong_reader = stream_reader self._reject_connection = False - self._stream_writer = None self._task = None self._transport = None self._client_connected_cb = client_connected_cb @@ -214,10 +213,8 @@ def _stream_reader(self): return None return self._stream_reader_wr() - def _replace_writer(self, writer): + def _replace_transport(self, transport): loop = self._loop - transport = writer.transport - self._stream_writer = writer self._transport = transport self._over_ssl = transport.get_extra_info('sslcontext') is not None @@ -239,11 +236,8 @@ def connection_made(self, transport): reader.set_transport(transport) self._over_ssl = transport.get_extra_info('sslcontext') is not None if self._client_connected_cb is not None: - self._stream_writer = StreamWriter(transport, self, - reader, - self._loop) - res = self._client_connected_cb(reader, - self._stream_writer) + writer = StreamWriter(transport, self, reader, self._loop) + res = self._client_connected_cb(reader, writer) if coroutines.iscoroutine(res): def callback(task): if task.cancelled(): @@ -405,7 +399,7 @@ async def start_tls(self, sslcontext, *, ssl_handshake_timeout=ssl_handshake_timeout, ssl_shutdown_timeout=ssl_shutdown_timeout) self._transport = new_transport - protocol._replace_writer(self) + protocol._replace_transport(new_transport) def __del__(self, warnings=warnings): if not self._transport.is_closing(): diff --git a/Lib/asyncio/transports.py b/Lib/asyncio/transports.py index 30fd41d49af71f..34c7ad44ffd8ab 100644 --- a/Lib/asyncio/transports.py +++ b/Lib/asyncio/transports.py @@ -181,6 +181,8 @@ def sendto(self, data, addr=None): to be sent out asynchronously. addr is target socket address. If addr is None use target address pointed on transport creation. + If data is an empty bytes object a zero-length datagram will be + sent. """ raise NotImplementedError diff --git a/Lib/base64.py b/Lib/base64.py index e3e983b3064fe7..25164d1a1df4fc 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -18,7 +18,7 @@ 'b64encode', 'b64decode', 'b32encode', 'b32decode', 'b32hexencode', 'b32hexdecode', 'b16encode', 'b16decode', # Base85 and Ascii85 encodings - 'b85encode', 'b85decode', 'a85encode', 'a85decode', + 'b85encode', 'b85decode', 'a85encode', 'a85decode', 'z85encode', 'z85decode', # Standard Base64 encoding 'standard_b64encode', 'standard_b64decode', # Some common Base64 alternatives. As referenced by RFC 3458, see thread @@ -497,6 +497,33 @@ def b85decode(b): result = result[:-padding] return result +_z85alphabet = (b'0123456789abcdefghijklmnopqrstuvwxyz' + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#') +# Translating b85 valid but z85 invalid chars to b'\x00' is required +# to prevent them from being decoded as b85 valid chars. +_z85_b85_decode_diff = b';_`|~' +_z85_decode_translation = bytes.maketrans( + _z85alphabet + _z85_b85_decode_diff, + _b85alphabet + b'\x00' * len(_z85_b85_decode_diff) +) +_z85_encode_translation = bytes.maketrans(_b85alphabet, _z85alphabet) + +def z85encode(s): + """Encode bytes-like object b in z85 format and return a bytes object.""" + return b85encode(s).translate(_z85_encode_translation) + +def z85decode(s): + """Decode the z85-encoded bytes-like object or ASCII string b + + The result is returned as a bytes object. + """ + s = _bytes_from_decode_data(s) + s = s.translate(_z85_decode_translation) + try: + return b85decode(s) + except ValueError as e: + raise ValueError(e.args[0].replace('base85', 'z85')) from None + # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it # though. The files should be opened in binary mode. diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 135a12c3965c00..9c132372dc4ee0 100755 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -41,7 +41,9 @@ class Profile(_lsprof.Profiler): def print_stats(self, sort=-1): import pstats - pstats.Stats(self).strip_dirs().sort_stats(sort).print_stats() + if not isinstance(sort, tuple): + sort = (sort,) + pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats() def dump_stats(self, file): import marshal diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 141142a57dcb3e..d54ee05b15f5bd 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -468,6 +468,8 @@ def LoadLibrary(self, name): if _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) +elif hasattr(_sys, "getandroidapilevel"): + pythonapi = PyDLL("libpython%d.%d.so" % _sys.version_info[:2]) elif _sys.platform == "cygwin": pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) else: diff --git a/Lib/dis.py b/Lib/dis.py index f05ea1a24f45a7..d146bcbb5097ef 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -505,6 +505,20 @@ def __init__(self, co_consts=None, names=None, varname_from_oparg=None, labels_m self.varname_from_oparg = varname_from_oparg self.labels_map = labels_map or {} + def offset_from_jump_arg(self, op, arg, offset): + deop = _deoptop(op) + if deop in hasjabs: + return arg * 2 + elif deop in hasjrel: + signed_arg = -arg if _is_backward_jump(deop) else arg + argval = offset + 2 + signed_arg*2 + caches = _get_cache_size(_all_opname[deop]) + argval += 2 * caches + if deop == ENTER_EXECUTOR: + argval += 2 + return argval + return None + def get_label_for_offset(self, offset): return self.labels_map.get(offset, None) @@ -536,17 +550,11 @@ def get_argval_argrepr(self, op, arg, offset): argrepr = f"{argrepr} + NULL|self" else: argval, argrepr = _get_name_info(arg, get_name) - elif deop in hasjabs: - argval = arg*2 - argrepr = f"to L{self.labels_map[argval]}" - elif deop in hasjrel: - signed_arg = -arg if _is_backward_jump(deop) else arg - argval = offset + 2 + signed_arg*2 - caches = _get_cache_size(_all_opname[deop]) - argval += 2 * caches - if deop == ENTER_EXECUTOR: - argval += 2 - argrepr = f"to L{self.labels_map[argval]}" + elif deop in hasjump or deop in hasexc: + argval = self.offset_from_jump_arg(op, arg, offset) + lbl = self.get_label_for_offset(argval) + assert lbl is not None + argrepr = f"to L{lbl}" elif deop in (LOAD_FAST_LOAD_FAST, STORE_FAST_LOAD_FAST, STORE_FAST_STORE_FAST): arg1 = arg >> 4 arg2 = arg & 15 diff --git a/Lib/doctest.py b/Lib/doctest.py index 1969777b667787..6049423b5147a5 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -2225,13 +2225,13 @@ def __init__(self, test, optionflags=0, setUp=None, tearDown=None, unittest.TestCase.__init__(self) self._dt_optionflags = optionflags self._dt_checker = checker - self._dt_globs = test.globs.copy() self._dt_test = test self._dt_setUp = setUp self._dt_tearDown = tearDown def setUp(self): test = self._dt_test + self._dt_globs = test.globs.copy() if self._dt_setUp is not None: self._dt_setUp(test) diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 5b653f66c18554..e4a342d446f6a3 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -949,6 +949,7 @@ class _InvalidEwError(errors.HeaderParseError): # up other parse trees. Maybe should have tests for that, too. DOT = ValueTerminal('.', 'dot') ListSeparator = ValueTerminal(',', 'list-separator') +ListSeparator.as_ew_allowed = False RouteComponentMarker = ValueTerminal('@', 'route-component-marker') # @@ -2022,7 +2023,7 @@ def get_address_list(value): address_list.defects.append(errors.InvalidHeaderDefect( "invalid address in address-list")) if value: # Must be a , at this point. - address_list.append(ValueTerminal(',', 'list-separator')) + address_list.append(ListSeparator) value = value[1:] return address_list, value diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index d85afd6d5cf704..6a5ca046b5eb6c 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -209,6 +209,7 @@ 'ms932' : 'cp932', 'mskanji' : 'cp932', 'ms_kanji' : 'cp932', + 'windows_31j' : 'cp932', # cp949 codec '949' : 'cp949', diff --git a/Lib/enum.py b/Lib/enum.py index 98a8966f5eb159..22963cca4466f2 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -279,9 +279,10 @@ def __set_name__(self, enum_class, member_name): enum_member._sort_order_ = len(enum_class._member_names_) if Flag is not None and issubclass(enum_class, Flag): - enum_class._flag_mask_ |= value - if _is_single_bit(value): - enum_class._singles_mask_ |= value + if isinstance(value, int): + enum_class._flag_mask_ |= value + if _is_single_bit(value): + enum_class._singles_mask_ |= value enum_class._all_bits_ = 2 ** ((enum_class._flag_mask_).bit_length()) - 1 # If another member with the same value was already defined, the @@ -309,6 +310,7 @@ def __set_name__(self, enum_class, member_name): elif ( Flag is not None and issubclass(enum_class, Flag) + and isinstance(value, int) and _is_single_bit(value) ): # no other instances found, record this member in _member_names_ @@ -545,7 +547,10 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_inverted_'] = None try: exc = None + classdict['_%s__in_progress' % cls] = True enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) + classdict['_%s__in_progress' % cls] = False + delattr(enum_class, '_%s__in_progress' % cls) except Exception as e: # since 3.12 the line "Error calling __set_name__ on '_proto_member' instance ..." # is tacked on to the error instead of raising a RuntimeError @@ -1153,6 +1158,8 @@ def __new__(cls, value): # still not found -- verify that members exist, in-case somebody got here mistakenly # (such as via super when trying to override __new__) if not cls._member_map_: + if getattr(cls, '_%s__in_progress' % cls.__name__, False): + raise TypeError('do not use `super().__new__; call the appropriate __new__ directly') from None raise TypeError("%r has no members defined" % cls) # # still not found -- try _missing_ hook @@ -1558,37 +1565,50 @@ def __str__(self): def __bool__(self): return bool(self._value_) + def _get_value(self, flag): + if isinstance(flag, self.__class__): + return flag._value_ + elif self._member_type_ is not object and isinstance(flag, self._member_type_): + return flag + return NotImplemented + def __or__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with |") value = self._value_ - return self.__class__(value | other) + return self.__class__(value | other_value) def __and__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with &") value = self._value_ - return self.__class__(value & other) + return self.__class__(value & other_value) def __xor__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with ^") value = self._value_ - return self.__class__(value ^ other) + return self.__class__(value ^ other_value) def __invert__(self): + if self._get_value(self) is None: + raise TypeError(f"'{self}' cannot be inverted") + if self._inverted_ is None: if self._boundary_ in (EJECT, KEEP): self._inverted_ = self.__class__(~self._value_) diff --git a/Lib/filecmp.py b/Lib/filecmp.py index 30bd900fa805aa..6ffc71fc059a80 100644 --- a/Lib/filecmp.py +++ b/Lib/filecmp.py @@ -88,12 +88,15 @@ def _do_cmp(f1, f2): class dircmp: """A class that manages the comparison of 2 directories. - dircmp(a, b, ignore=None, hide=None) + dircmp(a, b, ignore=None, hide=None, shallow=True) A and B are directories. IGNORE is a list of names to ignore, defaults to DEFAULT_IGNORES. HIDE is a list of names to hide, defaults to [os.curdir, os.pardir]. + SHALLOW specifies whether to just check the stat signature (do not read + the files). + defaults to True. High level usage: x = dircmp(dir1, dir2) @@ -121,7 +124,7 @@ class dircmp: in common_dirs. """ - def __init__(self, a, b, ignore=None, hide=None): # Initialize + def __init__(self, a, b, ignore=None, hide=None, shallow=True): # Initialize self.left = a self.right = b if hide is None: @@ -132,6 +135,7 @@ def __init__(self, a, b, ignore=None, hide=None): # Initialize self.ignore = DEFAULT_IGNORES else: self.ignore = ignore + self.shallow = shallow def phase0(self): # Compare everything except common subdirectories self.left_list = _filter(os.listdir(self.left), @@ -184,7 +188,7 @@ def phase2(self): # Distinguish files, directories, funnies self.common_funny.append(x) def phase3(self): # Find out differences between common files - xx = cmpfiles(self.left, self.right, self.common_files) + xx = cmpfiles(self.left, self.right, self.common_files, self.shallow) self.same_files, self.diff_files, self.funny_files = xx def phase4(self): # Find out differences between common subdirectories @@ -196,7 +200,8 @@ def phase4(self): # Find out differences between common subdirectories for x in self.common_dirs: a_x = os.path.join(self.left, x) b_x = os.path.join(self.right, x) - self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide) + self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide, + self.shallow) def phase4_closure(self): # Recursively call phase4() on subdirectories self.phase4() diff --git a/Lib/functools.py b/Lib/functools.py index ee4197b386178d..601cb8e7c0b74b 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -303,13 +303,13 @@ def __call__(self, /, *args, **keywords): @recursive_repr() def __repr__(self): - qualname = type(self).__qualname__ + cls = type(self) + qualname = cls.__qualname__ + module = cls.__module__ args = [repr(self.func)] args.extend(repr(x) for x in self.args) args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items()) - if type(self).__module__ == "functools": - return f"functools.{qualname}({', '.join(args)})" - return f"{qualname}({', '.join(args)})" + return f"{module}.{qualname}({', '.join(args)})" def __reduce__(self): return type(self), (self.func,), (self.func, self.args, @@ -918,7 +918,6 @@ def wrapper(*args, **kw): if not args: raise TypeError(f'{funcname} requires at least ' '1 positional argument') - return dispatch(args[0].__class__)(*args, **kw) funcname = getattr(func, '__name__', 'singledispatch function') @@ -968,7 +967,11 @@ def __get__(self, obj, cls=None): return _method dispatch = self.dispatcher.dispatch + funcname = getattr(self.func, '__name__', 'singledispatchmethod method') def _method(*args, **kwargs): + if not args: + raise TypeError(f'{funcname} requires at least ' + '1 positional argument') return dispatch(args[0].__class__).__get__(obj, cls)(*args, **kwargs) _method.__isabstractmethod__ = self.__isabstractmethod__ diff --git a/Lib/http/client.py b/Lib/http/client.py index 5eebfccafbca59..a353716a8506e6 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -936,17 +936,23 @@ def _get_hostport(self, host, port): host = host[:i] else: port = self.default_port - if host and host[0] == '[' and host[-1] == ']': - host = host[1:-1] + if host and host[0] == '[' and host[-1] == ']': + host = host[1:-1] return (host, port) def set_debuglevel(self, level): self.debuglevel = level + def _wrap_ipv6(self, ip): + if b':' in ip and ip[0] != b'['[0]: + return b"[" + ip + b"]" + return ip + def _tunnel(self): connect = b"CONNECT %s:%d %s\r\n" % ( - self._tunnel_host.encode("idna"), self._tunnel_port, + self._wrap_ipv6(self._tunnel_host.encode("idna")), + self._tunnel_port, self._http_vsn_str.encode("ascii")) headers = [connect] for header, value in self._tunnel_headers.items(): @@ -1221,9 +1227,8 @@ def putrequest(self, method, url, skip_host=False, # As per RFC 273, IPv6 address should be wrapped with [] # when used as Host header - + host_enc = self._wrap_ipv6(host_enc) if ":" in host: - host_enc = b'[' + host_enc + b']' host_enc = _strip_ipv6_iface(host_enc) if port == self.default_port: diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 8ee8eba64367a5..7bfa0932500d81 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -1044,7 +1044,9 @@ def open_recent_file(fn_closure=file_name): def saved_change_hook(self): short = self.short_title() long = self.long_title() - if short and long: + if short and long and not macosx.isCocoaTk(): + # Don't use both values on macOS because + # that doesn't match platform conventions. title = short + " - " + long + _py_version elif short: title = short @@ -1059,6 +1061,13 @@ def saved_change_hook(self): self.top.wm_title(title) self.top.wm_iconname(icon) + if macosx.isCocoaTk(): + # Add a proxy icon to the window title + self.top.wm_attributes("-titlepath", long) + + # Maintain the modification status for the window + self.top.wm_attributes("-modified", not self.get_saved()) + def get_saved(self): return self.undo.get_saved() diff --git a/Lib/importlib/resources/simple.py b/Lib/importlib/resources/simple.py index 7770c922c84fab..96f117fec62c10 100644 --- a/Lib/importlib/resources/simple.py +++ b/Lib/importlib/resources/simple.py @@ -88,7 +88,7 @@ def is_dir(self): def open(self, mode='r', *args, **kwargs): stream = self.parent.reader.open_binary(self.name) if 'b' not in mode: - stream = io.TextIOWrapper(*args, **kwargs) + stream = io.TextIOWrapper(stream, *args, **kwargs) return stream def joinpath(self, name): diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 3ad71d31c2f438..ff4f12fb1af70c 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -13,6 +13,7 @@ import _imp import sys +import threading import types @@ -171,36 +172,54 @@ class _LazyModule(types.ModuleType): def __getattribute__(self, attr): """Trigger the load of the module and return the attribute.""" - # All module metadata must be garnered from __spec__ in order to avoid - # using mutated values. - # Stop triggering this method. - self.__class__ = types.ModuleType - # Get the original name to make sure no object substitution occurred - # in sys.modules. - original_name = self.__spec__.name - # Figure out exactly what attributes were mutated between the creation - # of the module and now. - attrs_then = self.__spec__.loader_state['__dict__'] - attrs_now = self.__dict__ - attrs_updated = {} - for key, value in attrs_now.items(): - # Code that set the attribute may have kept a reference to the - # assigned object, making identity more important than equality. - if key not in attrs_then: - attrs_updated[key] = value - elif id(attrs_now[key]) != id(attrs_then[key]): - attrs_updated[key] = value - self.__spec__.loader.exec_module(self) - # If exec_module() was used directly there is no guarantee the module - # object was put into sys.modules. - if original_name in sys.modules: - if id(self) != id(sys.modules[original_name]): - raise ValueError(f"module object for {original_name!r} " - "substituted in sys.modules during a lazy " - "load") - # Update after loading since that's what would happen in an eager - # loading situation. - self.__dict__.update(attrs_updated) + __spec__ = object.__getattribute__(self, '__spec__') + loader_state = __spec__.loader_state + with loader_state['lock']: + # Only the first thread to get the lock should trigger the load + # and reset the module's class. The rest can now getattr(). + if object.__getattribute__(self, '__class__') is _LazyModule: + # The first thread comes here multiple times as it descends the + # call stack. The first time, it sets is_loading and triggers + # exec_module(), which will access module.__dict__, module.__name__, + # and/or module.__spec__, reentering this method. These accesses + # need to be allowed to proceed without triggering the load again. + if loader_state['is_loading'] and attr.startswith('__') and attr.endswith('__'): + return object.__getattribute__(self, attr) + loader_state['is_loading'] = True + + __dict__ = object.__getattribute__(self, '__dict__') + + # All module metadata must be gathered from __spec__ in order to avoid + # using mutated values. + # Get the original name to make sure no object substitution occurred + # in sys.modules. + original_name = __spec__.name + # Figure out exactly what attributes were mutated between the creation + # of the module and now. + attrs_then = loader_state['__dict__'] + attrs_now = __dict__ + attrs_updated = {} + for key, value in attrs_now.items(): + # Code that set an attribute may have kept a reference to the + # assigned object, making identity more important than equality. + if key not in attrs_then: + attrs_updated[key] = value + elif id(attrs_now[key]) != id(attrs_then[key]): + attrs_updated[key] = value + __spec__.loader.exec_module(self) + # If exec_module() was used directly there is no guarantee the module + # object was put into sys.modules. + if original_name in sys.modules: + if id(self) != id(sys.modules[original_name]): + raise ValueError(f"module object for {original_name!r} " + "substituted in sys.modules during a lazy " + "load") + # Update after loading since that's what would happen in an eager + # loading situation. + __dict__.update(attrs_updated) + # Finally, stop triggering this method. + self.__class__ = types.ModuleType + return getattr(self, attr) def __delattr__(self, attr): @@ -244,5 +263,7 @@ def exec_module(self, module): loader_state = {} loader_state['__dict__'] = module.__dict__.copy() loader_state['__class__'] = module.__class__ + loader_state['lock'] = threading.RLock() + loader_state['is_loading'] = False module.__spec__.loader_state = loader_state module.__class__ = _LazyModule diff --git a/Lib/inspect.py b/Lib/inspect.py index 450093a8b4c1ee..8a2b2c96e993b5 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -762,18 +762,14 @@ def unwrap(func, *, stop=None): :exc:`ValueError` is raised if a cycle is encountered. """ - if stop is None: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') - else: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') and not stop(f) f = func # remember the original func for error reporting # Memoise by id to tolerate non-hashable objects, but store objects to # ensure they aren't destroyed, which would allow their IDs to be reused. memo = {id(f): f} recursion_limit = sys.getrecursionlimit() - while _is_wrapper(func): + while not isinstance(func, type) and hasattr(func, '__wrapped__'): + if stop is not None and stop(func): + break func = func.__wrapped__ id_func = id(func) if (id_func in memo) or (len(memo) >= recursion_limit): @@ -834,9 +830,8 @@ def _finddoc(obj): cls = self.__class__ # Should be tested before isdatadescriptor(). elif isinstance(obj, property): - func = obj.fget - name = func.__name__ - cls = _findclass(func) + name = obj.__name__ + cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None elif ismethoddescriptor(obj) or isdatadescriptor(obj): @@ -2044,15 +2039,17 @@ def _signature_get_user_defined_method(cls, method_name): named ``method_name`` and returns it only if it is a pure python function. """ - try: - meth = getattr(cls, method_name) - except AttributeError: - return + if method_name == '__new__': + meth = getattr(cls, method_name, None) else: - if not isinstance(meth, _NonUserDefinedCallables): - # Once '__signature__' will be added to 'C'-level - # callables, this check won't be necessary - return meth + meth = getattr_static(cls, method_name, None) + if meth is None or isinstance(meth, _NonUserDefinedCallables): + # Once '__signature__' will be added to 'C'-level + # callables, this check won't be necessary + return None + if method_name != '__new__': + meth = _descriptor_get(meth, cls) + return meth def _signature_get_partial(wrapped_sig, partial, extra_args=()): @@ -2497,6 +2494,15 @@ def _signature_from_function(cls, func, skip_bound_arg=True, __validate_parameters__=is_duck_function) +def _descriptor_get(descriptor, obj): + if isclass(descriptor): + return descriptor + get = getattr(type(descriptor), '__get__', _sentinel) + if get is _sentinel: + return descriptor + return get(descriptor, obj, type(obj)) + + def _signature_from_callable(obj, *, follow_wrapper_chains=True, skip_bound_arg=True, @@ -2605,7 +2611,6 @@ def _signature_from_callable(obj, *, wrapped_sig = _get_signature_of(obj.func) return _signature_get_partial(wrapped_sig, obj) - sig = None if isinstance(obj, type): # obj is a class or a metaclass @@ -2613,88 +2618,65 @@ def _signature_from_callable(obj, *, # in its metaclass call = _signature_get_user_defined_method(type(obj), '__call__') if call is not None: - sig = _get_signature_of(call) - else: - factory_method = None - new = _signature_get_user_defined_method(obj, '__new__') - init = _signature_get_user_defined_method(obj, '__init__') - - # Go through the MRO and see if any class has user-defined - # pure Python __new__ or __init__ method - for base in obj.__mro__: - # Now we check if the 'obj' class has an own '__new__' method - if new is not None and '__new__' in base.__dict__: - factory_method = new - break - # or an own '__init__' method - elif init is not None and '__init__' in base.__dict__: - factory_method = init - break + return _get_signature_of(call) - if factory_method is not None: - sig = _get_signature_of(factory_method) - - if sig is None: - # At this point we know, that `obj` is a class, with no user- - # defined '__init__', '__new__', or class-level '__call__' - - for base in obj.__mro__[:-1]: - # Since '__text_signature__' is implemented as a - # descriptor that extracts text signature from the - # class docstring, if 'obj' is derived from a builtin - # class, its own '__text_signature__' may be 'None'. - # Therefore, we go through the MRO (except the last - # class in there, which is 'object') to find the first - # class with non-empty text signature. - try: - text_sig = base.__text_signature__ - except AttributeError: - pass - else: - if text_sig: - # If 'base' class has a __text_signature__ attribute: - # return a signature based on it - return _signature_fromstr(sigcls, base, text_sig) - - # No '__text_signature__' was found for the 'obj' class. - # Last option is to check if its '__init__' is - # object.__init__ or type.__init__. - if type not in obj.__mro__: - # We have a class (not metaclass), but no user-defined - # __init__ or __new__ for it - if (obj.__init__ is object.__init__ and - obj.__new__ is object.__new__): - # Return a signature of 'object' builtin. - return sigcls.from_callable(object) - else: - raise ValueError( - 'no signature found for builtin type {!r}'.format(obj)) + new = _signature_get_user_defined_method(obj, '__new__') + init = _signature_get_user_defined_method(obj, '__init__') - elif not isinstance(obj, _NonUserDefinedCallables): - # An object with __call__ - # We also check that the 'obj' is not an instance of - # types.WrapperDescriptorType or types.MethodWrapperType to avoid - # infinite recursion (and even potential segfault) - call = _signature_get_user_defined_method(type(obj), '__call__') - if call is not None: + # Go through the MRO and see if any class has user-defined + # pure Python __new__ or __init__ method + for base in obj.__mro__: + # Now we check if the 'obj' class has an own '__new__' method + if new is not None and '__new__' in base.__dict__: + sig = _get_signature_of(new) + if skip_bound_arg: + sig = _signature_bound_method(sig) + return sig + # or an own '__init__' method + elif init is not None and '__init__' in base.__dict__: + return _get_signature_of(init) + + # At this point we know, that `obj` is a class, with no user- + # defined '__init__', '__new__', or class-level '__call__' + + for base in obj.__mro__[:-1]: + # Since '__text_signature__' is implemented as a + # descriptor that extracts text signature from the + # class docstring, if 'obj' is derived from a builtin + # class, its own '__text_signature__' may be 'None'. + # Therefore, we go through the MRO (except the last + # class in there, which is 'object') to find the first + # class with non-empty text signature. try: - sig = _get_signature_of(call) - except ValueError as ex: - msg = 'no signature found for {!r}'.format(obj) - raise ValueError(msg) from ex - - if sig is not None: - # For classes and objects we skip the first parameter of their - # __call__, __new__, or __init__ methods - if skip_bound_arg: - return _signature_bound_method(sig) - else: - return sig + text_sig = base.__text_signature__ + except AttributeError: + pass + else: + if text_sig: + # If 'base' class has a __text_signature__ attribute: + # return a signature based on it + return _signature_fromstr(sigcls, base, text_sig) + + # No '__text_signature__' was found for the 'obj' class. + # Last option is to check if its '__init__' is + # object.__init__ or type.__init__. + if type not in obj.__mro__: + # We have a class (not metaclass), but no user-defined + # __init__ or __new__ for it + if (obj.__init__ is object.__init__ and + obj.__new__ is object.__new__): + # Return a signature of 'object' builtin. + return sigcls.from_callable(object) + else: + raise ValueError( + 'no signature found for builtin type {!r}'.format(obj)) - if isinstance(obj, types.BuiltinFunctionType): - # Raise a nicer error message for builtins - msg = 'no signature found for builtin function {!r}'.format(obj) - raise ValueError(msg) + else: + # An object with __call__ + call = getattr_static(type(obj), '__call__', None) + if call is not None: + call = _descriptor_get(call, obj) + return _get_signature_of(call) raise ValueError('callable {!r} is not supported by signature'.format(obj)) diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 45f547741885a8..597849eca0524a 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -174,7 +174,7 @@ def default(self, o): else: return list(iterable) # Let the base class default method raise the TypeError - return JSONEncoder.default(self, o) + return super().default(o) """ raise TypeError(f'Object of type {o.__class__.__name__} ' diff --git a/Lib/linecache.py b/Lib/linecache.py index 329a14053458b7..04c8f45a6c60ca 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -166,13 +166,11 @@ def lazycache(filename, module_globals): return False # Try for a __loader__, if available if module_globals and '__name__' in module_globals: - name = module_globals['__name__'] - if (loader := module_globals.get('__loader__')) is None: - if spec := module_globals.get('__spec__'): - try: - loader = spec.loader - except AttributeError: - pass + spec = module_globals.get('__spec__') + name = getattr(spec, 'name', None) or module_globals['__name__'] + loader = getattr(spec, 'loader', None) + if loader is None: + loader = module_globals.get('__loader__') get_source = getattr(loader, 'get_source', None) if name and get_source: diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index e7f1322e4ba3d9..cdb014bca04b6e 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -232,19 +232,19 @@ def __init__(self, filename, when='h', interval=1, backupCount=0, if self.when == 'S': self.interval = 1 # one second self.suffix = "%Y-%m-%d_%H-%M-%S" - self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(\.\w+)?$" + extMatch = r"(? (plen + 1) and not fileName[plen+1].isdigit()): - continue - - if fileName[:plen] == prefix: - suffix = fileName[plen:] - # See bpo-45628: The date/time suffix could be anywhere in the - # filename - parts = suffix.split('.') - for part in parts: - if self.extMatch.match(part): + if self.namer is None: + prefix = baseName + '.' + plen = len(prefix) + for fileName in fileNames: + if fileName[:plen] == prefix: + suffix = fileName[plen:] + if self.extMatch.fullmatch(suffix): + result.append(os.path.join(dirName, fileName)) + else: + for fileName in fileNames: + # Our files could be just about anything after custom naming, + # but they should contain the datetime suffix. + # Try to find the datetime suffix in the file name and verify + # that the file name can be generated by this handler. + m = self.extMatch.search(fileName) + while m: + dfn = self.namer(self.baseFilename + "." + m[0]) + if os.path.basename(dfn) == fileName: result.append(os.path.join(dirName, fileName)) break + m = self.extMatch.search(fileName, m.start() + 1) + if len(result) < self.backupCount: result = [] else: @@ -410,17 +413,14 @@ def doRollover(self): then we have to get a list of matching filenames, sort them and remove the one with the oldest suffix. """ - if self.stream: - self.stream.close() - self.stream = None # get the time that this sequence started at and make it a TimeTuple currentTime = int(time.time()) - dstNow = time.localtime(currentTime)[-1] t = self.rolloverAt - self.interval if self.utc: timeTuple = time.gmtime(t) else: timeTuple = time.localtime(t) + dstNow = time.localtime(currentTime)[-1] dstThen = timeTuple[-1] if dstNow != dstThen: if dstNow: @@ -431,26 +431,19 @@ def doRollover(self): dfn = self.rotation_filename(self.baseFilename + "." + time.strftime(self.suffix, timeTuple)) if os.path.exists(dfn): - os.remove(dfn) + # Already rolled over. + return + + if self.stream: + self.stream.close() + self.stream = None self.rotate(self.baseFilename, dfn) if self.backupCount > 0: for s in self.getFilesToDelete(): os.remove(s) if not self.delay: self.stream = self._open() - newRolloverAt = self.computeRollover(currentTime) - while newRolloverAt <= currentTime: - newRolloverAt = newRolloverAt + self.interval - #If DST changes and midnight or weekly rollover, adjust for this. - if (self.when == 'MIDNIGHT' or self.when.startswith('W')) and not self.utc: - dstAtRollover = time.localtime(newRolloverAt)[-1] - if dstNow != dstAtRollover: - if not dstNow: # DST kicks in before next rollover, so we need to deduct an hour - addend = -3600 - else: # DST bows out before next rollover, so we need to add an hour - addend = 3600 - newRolloverAt += addend - self.rolloverAt = newRolloverAt + self.rolloverAt = self.computeRollover(currentTime) class WatchedFileHandler(logging.FileHandler): """ diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 58d697fdecacc0..b7e1e132172d02 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -476,8 +476,9 @@ def accept(self): ''' if self._listener is None: raise OSError('listener is closed') + c = self._listener.accept() - if self._authkey: + if self._authkey is not None: deliver_challenge(c, self._authkey) answer_challenge(c, self._authkey) return c diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 852ae87b276861..925f043900004e 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -20,8 +20,6 @@ from queue import Empty, Full -import _multiprocessing - from . import connection from . import context _ForkingPickler = context.reduction.ForkingPickler diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 8e41f461cc934e..20ddd9c50e3d88 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -29,8 +29,12 @@ _HAVE_SIGMASK = hasattr(signal, 'pthread_sigmask') _IGNORED_SIGNALS = (signal.SIGINT, signal.SIGTERM) +def cleanup_noop(name): + raise RuntimeError('noop should never be registered or cleaned up') + _CLEANUP_FUNCS = { - 'noop': lambda: None, + 'noop': cleanup_noop, + 'dummy': lambda name: None, # Dummy resource used in tests } if os.name == 'posix': @@ -61,6 +65,7 @@ def __init__(self): self._lock = threading.RLock() self._fd = None self._pid = None + self._exitcode = None def _reentrant_call_error(self): # gh-109629: this happens if an explicit call to the ResourceTracker @@ -84,9 +89,16 @@ def _stop(self): os.close(self._fd) self._fd = None - os.waitpid(self._pid, 0) + _, status = os.waitpid(self._pid, 0) + self._pid = None + try: + self._exitcode = os.waitstatus_to_exitcode(status) + except ValueError: + # os.waitstatus_to_exitcode may raise an exception for invalid values + self._exitcode = None + def getfd(self): self.ensure_running() return self._fd @@ -119,6 +131,7 @@ def ensure_running(self): pass self._fd = None self._pid = None + self._exitcode = None warnings.warn('resource_tracker: process died unexpectedly, ' 'relaunching. Some resources might leak.') @@ -221,6 +234,8 @@ def main(fd): pass cache = {rtype: set() for rtype in _CLEANUP_FUNCS.keys()} + exit_code = 0 + try: # keep track of registered/unregistered resources with open(fd, 'rb') as f: @@ -242,6 +257,7 @@ def main(fd): else: raise RuntimeError('unrecognized command %r' % cmd) except Exception: + exit_code = 3 try: sys.excepthook(*sys.exc_info()) except: @@ -251,10 +267,17 @@ def main(fd): for rtype, rtype_cache in cache.items(): if rtype_cache: try: - warnings.warn( - f'resource_tracker: There appear to be {len(rtype_cache)} ' - f'leaked {rtype} objects to clean up at shutdown: {rtype_cache}' - ) + exit_code = 1 + if rtype == 'dummy': + # The test 'dummy' resource is expected to leak. + # We skip the warning (and *only* the warning) for it. + pass + else: + warnings.warn( + f'resource_tracker: There appear to be ' + f'{len(rtype_cache)} leaked {rtype} objects to ' + f'clean up at shutdown: {rtype_cache}' + ) except Exception: pass for name in rtype_cache: @@ -265,6 +288,9 @@ def main(fd): try: _CLEANUP_FUNCS[rtype](name) except Exception as e: + exit_code = 2 warnings.warn('resource_tracker: %r: %s' % (name, e)) finally: pass + + sys.exit(exit_code) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 27c6b4e367a050..645d62a0f0699a 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -313,7 +313,14 @@ def with_name(self, name): def with_stem(self, stem): """Return a new path with the stem changed.""" - return self.with_name(stem + self.suffix) + suffix = self.suffix + if not suffix: + return self.with_name(stem) + elif not stem: + # If the suffix is non-empty, we can't make the stem empty. + raise ValueError(f"{self!r} has a non-empty suffix") + else: + return self.with_name(stem + suffix) def with_suffix(self, suffix): """Return a new path with the file suffix changed. If the path @@ -324,6 +331,7 @@ def with_suffix(self, suffix): if not suffix: return self.with_name(stem) elif not stem: + # If the stem is empty, we can't make the suffix non-empty. raise ValueError(f"{self!r} has an empty name") elif suffix.startswith('.') and len(suffix) > 1: return self.with_name(stem + suffix) @@ -781,7 +789,7 @@ def _make_child_direntry(self, entry): def _make_child_relpath(self, name): return self.joinpath(name) - def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None): + def glob(self, pattern, *, case_sensitive=None, follow_symlinks=True): """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ @@ -838,7 +846,7 @@ def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None): paths = _select_children(paths, bool(stack), follow_symlinks, match) return paths - def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=None): + def rglob(self, pattern, *, case_sensitive=None, follow_symlinks=True): """Recursively yield all existing files (of any kind, including directories) matching the given relative pattern, anywhere in this subtree. diff --git a/Lib/pdb.py b/Lib/pdb.py index 0754e8b628cf57..519c1ccd5640a1 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -188,6 +188,7 @@ def namespace(self): __name__='__main__', __file__=self, __builtins__=__builtins__, + __spec__=None, ) @property diff --git a/Lib/posixpath.py b/Lib/posixpath.py index e4f155e41a3221..33943b4403636a 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -546,10 +546,11 @@ def relpath(path, start=None): def commonpath(paths): """Given a sequence of path names, returns the longest common sub-path.""" + paths = tuple(map(os.fspath, paths)) + if not paths: raise ValueError('commonpath() arg is an empty sequence') - paths = tuple(map(os.fspath, paths)) if isinstance(paths[0], bytes): sep = b'/' curdir = b'.' diff --git a/Lib/profile.py b/Lib/profile.py index 4b82523b03d64b..f2f8c2f21333e0 100755 --- a/Lib/profile.py +++ b/Lib/profile.py @@ -387,8 +387,9 @@ def simulate_cmd_complete(self): def print_stats(self, sort=-1): import pstats - pstats.Stats(self).strip_dirs().sort_stats(sort). \ - print_stats() + if not isinstance(sort, tuple): + sort = (sort,) + pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats() def dump_stats(self, file): with open(file, 'wb') as f: diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 6d145abda9d4ab..407c0205c7ab66 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -127,9 +127,8 @@ def _finddoc(obj): cls = self.__class__ # Should be tested before isdatadescriptor(). elif isinstance(obj, property): - func = obj.fget - name = func.__name__ - cls = _findclass(func) + name = obj.__name__ + cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None elif inspect.ismethoddescriptor(obj) or inspect.isdatadescriptor(obj): @@ -856,9 +855,9 @@ def docmodule(self, object, name=None, mod=None, *ignored): cdict[key] = cdict[base] = modname + '.html#' + key funcs, fdict = [], {} for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) fdict[key] = '#-' + key @@ -1144,7 +1143,8 @@ def docroutine(self, object, name=None, mod=None, # XXX lambda's won't usually have func_annotations['return'] # since the syntax doesn't support but it is possible. # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + if not object.__annotations__: + argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' @@ -1299,9 +1299,9 @@ def docmodule(self, object, name=None, mod=None, *ignored): classes.append((key, value)) funcs = [] for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) data = [] @@ -1586,7 +1586,8 @@ def docroutine(self, object, name=None, mod=None, cl=None, homecls=None): # XXX lambda's won't usually have func_annotations['return'] # since the syntax doesn't support but it is possible. # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + if not object.__annotations__: + argspec = argspec[1:-1] if not argspec: argspec = '(...)' decl = asyncqualifier + title + argspec + note @@ -1684,8 +1685,17 @@ def plain(text): def pipepager(text, cmd): """Page through text by feeding it to another program.""" import subprocess + env = os.environ.copy() + prompt_string = ( + ' ' + '?ltline %lt?L/%L.' + ':byte %bB?s/%s.' + '.' + '?e (END):?pB %pB\\%..' + ' (press h for help or q to quit)') + env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - errors='backslashreplace') + errors='backslashreplace', env=env) try: with proc.stdin as pipe: try: diff --git a/Lib/queue.py b/Lib/queue.py index 467ff4fcecb134..18fad8df08e469 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -72,10 +72,11 @@ def task_done(self): have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). + shutdown(immediate=True) calls task_done() for each remaining item in + the queue. + Raises a ValueError if called more times than there were items placed in the queue. - - Raises ShutDown if the queue has been shut down immediately. ''' with self.all_tasks_done: unfinished = self.unfinished_tasks - 1 @@ -93,8 +94,6 @@ def join(self): to indicate the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, join() unblocks. - - Raises ShutDown if the queue has been shut down immediately. ''' with self.all_tasks_done: while self.unfinished_tasks: @@ -227,18 +226,17 @@ def get_nowait(self): return self.get(block=False) def shutdown(self, immediate=False): - '''Shut-down the queue, making queue gets and puts raise. + '''Shut-down the queue, making queue gets and puts raise ShutDown. By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. All blocked callers of put() will be unblocked, and also get() - and join() if 'immediate'. The ShutDown exception is raised. + and join() if 'immediate'. ''' with self.mutex: self.is_shutdown = True if immediate: - n_items = self._qsize() while self._qsize(): self._get() if self.unfinished_tasks > 0: diff --git a/Lib/site.py b/Lib/site.py index 0631f3f6115ec0..2aee63e24ca52b 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -460,60 +460,64 @@ def gethistoryfile(): def enablerlcompleter(): """Enable default readline configuration on interactive prompts, by registering a sys.__interactivehook__. + """ + sys.__interactivehook__ = register_readline + + +def register_readline(): + """Configure readline completion on interactive prompts. If the readline module can be imported, the hook will set the Tab key as completion key and register ~/.python_history as history file. This can be overridden in the sitecustomize or usercustomize module, or in a PYTHONSTARTUP file. """ - def register_readline(): - import atexit - try: - import readline - import rlcompleter - except ImportError: - return - - # Reading the initialization (config) file may not be enough to set a - # completion key, so we set one first and then read the file. - if readline.backend == 'editline': - readline.parse_and_bind('bind ^I rl_complete') - else: - readline.parse_and_bind('tab: complete') + import atexit + try: + import readline + import rlcompleter + except ImportError: + return + # Reading the initialization (config) file may not be enough to set a + # completion key, so we set one first and then read the file. + if readline.backend == 'editline': + readline.parse_and_bind('bind ^I rl_complete') + else: + readline.parse_and_bind('tab: complete') + + try: + readline.read_init_file() + except OSError: + # An OSError here could have many causes, but the most likely one + # is that there's no .inputrc file (or .editrc file in the case of + # Mac OS X + libedit) in the expected location. In that case, we + # want to ignore the exception. + pass + + if readline.get_current_history_length() == 0: + # If no history was loaded, default to .python_history, + # or PYTHON_HISTORY. + # The guard is necessary to avoid doubling history size at + # each interpreter exit when readline was already configured + # through a PYTHONSTARTUP hook, see: + # http://bugs.python.org/issue5845#msg198636 + history = gethistoryfile() try: - readline.read_init_file() + readline.read_history_file(history) except OSError: - # An OSError here could have many causes, but the most likely one - # is that there's no .inputrc file (or .editrc file in the case of - # Mac OS X + libedit) in the expected location. In that case, we - # want to ignore the exception. pass - if readline.get_current_history_length() == 0: - # If no history was loaded, default to .python_history, - # or PYTHON_HISTORY. - # The guard is necessary to avoid doubling history size at - # each interpreter exit when readline was already configured - # through a PYTHONSTARTUP hook, see: - # http://bugs.python.org/issue5845#msg198636 - history = gethistoryfile() + def write_history(): try: - readline.read_history_file(history) - except OSError: + readline.write_history_file(history) + except (FileNotFoundError, PermissionError): + # home directory does not exist or is not writable + # https://bugs.python.org/issue19891 pass - def write_history(): - try: - readline.write_history_file(history) - except OSError: - # bpo-19891, bpo-41193: Home directory does not exist - # or is not writable, or the filesystem is read-only. - pass + atexit.register(write_history) - atexit.register(write_history) - - sys.__interactivehook__ = register_readline def venv(known_paths): global PREFIXES, ENABLE_USER_SITE diff --git a/Lib/statistics.py b/Lib/statistics.py index 83aaedb04515e0..7924123c05b8c3 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -112,6 +112,7 @@ 'fmean', 'geometric_mean', 'harmonic_mean', + 'kde', 'linear_regression', 'mean', 'median', @@ -137,7 +138,7 @@ from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod -from math import isfinite, isinf +from math import isfinite, isinf, pi, cos, cosh from functools import reduce from operator import itemgetter from collections import Counter, namedtuple, defaultdict @@ -802,6 +803,171 @@ def multimode(data): return [value for value, count in counts.items() if count == maxcount] +def kde(data, h, kernel='normal'): + """Kernel Density Estimation: Create a continuous probability + density function from discrete samples. + + The basic idea is to smooth the data using a kernel function + to help draw inferences about a population from a sample. + + The degree of smoothing is controlled by the scaling parameter h + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. + + The kernel determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. + + Kernels that give some weight to every sample point: + + normal or gauss + logistic + sigmoid + + Kernels that only give weight to sample points within + the bandwidth: + + rectangular or uniform + triangular + parabolic or epanechnikov + quartic or biweight + triweight + cosine + + A StatisticsError will be raised if the data sequence is empty. + + Example + ------- + + Given a sample of six data points, construct a continuous + function that estimates the underlying probability density: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) + + Compute the area under the curve: + + >>> sum(f_hat(x) for x in range(-20, 20)) + 1.0 + + Plot the estimated probability density function at + evenly spaced points from -6 to 10: + + >>> for x in range(-6, 11): + ... density = f_hat(x) + ... plot = ' ' * int(density * 400) + 'x' + ... print(f'{x:2}: {density:.3f} {plot}') + ... + -6: 0.002 x + -5: 0.009 x + -4: 0.031 x + -3: 0.070 x + -2: 0.111 x + -1: 0.125 x + 0: 0.110 x + 1: 0.086 x + 2: 0.068 x + 3: 0.059 x + 4: 0.066 x + 5: 0.082 x + 6: 0.082 x + 7: 0.058 x + 8: 0.028 x + 9: 0.009 x + 10: 0.002 x + + References + ---------- + + Kernel density estimation and its application: + https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf + + Kernel functions in common use: + https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use + + Interactive graphical demonstration and exploration: + https://demonstrations.wolfram.com/KernelDensityEstimation/ + + """ + + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + match kernel: + + case 'normal' | 'gauss': + c = 1 / sqrt(2 * pi) + K = lambda t: c * exp(-1/2 * t * t) + support = None + + case 'logistic': + # 1.0 / (exp(t) + 2.0 + exp(-t)) + K = lambda t: 1/2 / (1.0 + cosh(t)) + support = None + + case 'sigmoid': + # (2/pi) / (exp(t) + exp(-t)) + c = 1 / pi + K = lambda t: c / cosh(t) + support = None + + case 'rectangular' | 'uniform': + K = lambda t: 1/2 + support = 1.0 + + case 'triangular': + K = lambda t: 1.0 - abs(t) + support = 1.0 + + case 'parabolic' | 'epanechnikov': + K = lambda t: 3/4 * (1.0 - t * t) + support = 1.0 + + case 'quartic' | 'biweight': + K = lambda t: 15/16 * (1.0 - t * t) ** 2 + support = 1.0 + + case 'triweight': + K = lambda t: 35/32 * (1.0 - t * t) ** 3 + support = 1.0 + + case 'cosine': + c1 = pi / 4 + c2 = pi / 2 + K = lambda t: c1 * cos(c2 * t) + support = 1.0 + + case _: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + + if support is None: + + def pdf(x): + return sum(K((x - x_i) / h) for x_i in data) / (n * h) + + else: + + sample = sorted(data) + bandwidth = h * support + + def pdf(x): + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum(K((x - x_i) / h) for x_i in supported) / (n * h) + + pdf.__doc__ = f'PDF estimate with {kernel=!r} and {h=!r}' + + return pdf + + # Notes on methods for computing quantiles # ---------------------------------------- # diff --git a/Lib/tarfile.py b/Lib/tarfile.py index f4dd0fdab4a3e4..6f315a6408f185 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -872,7 +872,7 @@ class TarInfo(object): pax_headers = ('A dictionary containing key-value pairs of an ' 'associated pax extended header.'), sparse = 'Sparse member information.', - tarfile = None, + _tarfile = None, _sparse_structs = None, _link_target = None, ) @@ -901,6 +901,24 @@ def __init__(self, name=""): self.sparse = None # sparse member information self.pax_headers = {} # pax header information + @property + def tarfile(self): + import warnings + warnings.warn( + 'The undocumented "tarfile" attribute of TarInfo objects ' + + 'is deprecated and will be removed in Python 3.16', + DeprecationWarning, stacklevel=2) + return self._tarfile + + @tarfile.setter + def tarfile(self, tarfile): + import warnings + warnings.warn( + 'The undocumented "tarfile" attribute of TarInfo objects ' + + 'is deprecated and will be removed in Python 3.16', + DeprecationWarning, stacklevel=2) + self._tarfile = tarfile + @property def path(self): 'In pax headers, "name" is called "path".' @@ -2030,7 +2048,7 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): # Now, fill the TarInfo object with # information specific for the file. tarinfo = self.tarinfo() - tarinfo.tarfile = self # Not needed + tarinfo._tarfile = self # To be removed in 3.16. # Use os.stat or os.lstat, depending on if symlinks shall be resolved. if fileobj is None: diff --git a/Lib/test/_test_monitoring_shutdown.py b/Lib/test/_test_monitoring_shutdown.py new file mode 100644 index 00000000000000..3d0fbec029dc09 --- /dev/null +++ b/Lib/test/_test_monitoring_shutdown.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +# gh-115832: An object destructor running during the final GC of interpreter +# shutdown triggered an infinite loop in the instrumentation code. + +import sys + +class CallableCycle: + def __init__(self): + self._cycle = self + + def __del__(self): + pass + + def __call__(self, code, instruction_offset): + pass + +def tracefunc(frame, event, arg): + pass + +def main(): + tool_id = sys.monitoring.PROFILER_ID + event_id = sys.monitoring.events.PY_START + + sys.monitoring.use_tool_id(tool_id, "test profiler") + sys.monitoring.set_events(tool_id, event_id) + sys.monitoring.register_callback(tool_id, event_id, CallableCycle()) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 94ce85cac754ae..058537bab5af26 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3504,6 +3504,25 @@ def test_context(self): if self.TYPE == 'processes': self.assertRaises(OSError, l.accept) + def test_empty_authkey(self): + # bpo-43952: allow empty bytes as authkey + def handler(*args): + raise RuntimeError('Connection took too long...') + + def run(addr, authkey): + client = self.connection.Client(addr, authkey=authkey) + client.send(1729) + + key = b"" + + with self.connection.Listener(authkey=key) as listener: + threading.Thread(target=run, args=(listener.address, key)).start() + with listener.accept() as d: + self.assertEqual(d.recv(), 1729) + + if self.TYPE == 'processes': + self.assertRaises(OSError, listener.accept) + @unittest.skipUnless(util.abstract_sockets_supported, "test needs abstract socket support") def test_abstract_socket(self): @@ -3971,6 +3990,21 @@ def _new_shm_name(self, prefix): # test_multiprocessing_spawn, etc) in parallel. return prefix + str(os.getpid()) + def test_shared_memory_name_with_embedded_null(self): + name_tsmb = self._new_shm_name('test01_null') + sms = shared_memory.SharedMemory(name_tsmb, create=True, size=512) + self.addCleanup(sms.unlink) + with self.assertRaises(ValueError): + shared_memory.SharedMemory(name_tsmb + '\0a', create=False, size=512) + if shared_memory._USE_POSIX: + orig_name = sms._name + try: + sms._name = orig_name + '\0a' + with self.assertRaises(ValueError): + sms.unlink() + finally: + sms._name = orig_name + def test_shared_memory_basics(self): name_tsmb = self._new_shm_name('test01_tsmb') sms = shared_memory.SharedMemory(name_tsmb, create=True, size=512) @@ -4105,7 +4139,7 @@ def test_shared_memory_recreate(self): self.addCleanup(shm2.unlink) self.assertEqual(shm2._name, names[1]) - def test_invalid_shared_memory_cration(self): + def test_invalid_shared_memory_creation(self): # Test creating a shared memory segment with negative size with self.assertRaises(ValueError): sms_invalid = shared_memory.SharedMemory(create=True, size=-1) @@ -5609,8 +5643,9 @@ def create_and_register_resource(rtype): ''' for rtype in resource_tracker._CLEANUP_FUNCS: with self.subTest(rtype=rtype): - if rtype == "noop": + if rtype in ("noop", "dummy"): # Artefact resource type used by the resource_tracker + # or tests continue r, w = os.pipe() p = subprocess.Popen([sys.executable, @@ -5730,6 +5765,38 @@ def test_too_long_name_resource(self): with self.assertRaises(ValueError): resource_tracker.register(too_long_name_resource, rtype) + def _test_resource_tracker_leak_resources(self, cleanup): + # We use a separate instance for testing, since the main global + # _resource_tracker may be used to watch test infrastructure. + from multiprocessing.resource_tracker import ResourceTracker + tracker = ResourceTracker() + tracker.ensure_running() + self.assertTrue(tracker._check_alive()) + + self.assertIsNone(tracker._exitcode) + tracker.register('somename', 'dummy') + if cleanup: + tracker.unregister('somename', 'dummy') + expected_exit_code = 0 + else: + expected_exit_code = 1 + + self.assertTrue(tracker._check_alive()) + self.assertIsNone(tracker._exitcode) + tracker._stop() + self.assertEqual(tracker._exitcode, expected_exit_code) + + def test_resource_tracker_exit_code(self): + """ + Test the exit code of the resource tracker. + + If no leaked resources were found, exit code should be 0, otherwise 1 + """ + for cleanup in [True, False]: + with self.subTest(cleanup=cleanup): + self._test_resource_tracker_leak_resources( + cleanup=cleanup, + ) class TestSimpleQueue(unittest.TestCase): diff --git a/Lib/test/bisect_cmd.py b/Lib/test/bisect_cmd.py index 5cb804bd469dc3..aee2e8ac120852 100755 --- a/Lib/test/bisect_cmd.py +++ b/Lib/test/bisect_cmd.py @@ -51,6 +51,7 @@ def python_cmd(): cmd = [sys.executable] cmd.extend(subprocess._args_from_interpreter_flags()) cmd.extend(subprocess._optim_args_from_interpreter_flags()) + cmd.extend(('-X', 'faulthandler')) return cmd @@ -77,9 +78,13 @@ def run_tests(args, tests, huntrleaks=None): write_tests(tmp, tests) cmd = python_cmd() - cmd.extend(['-m', 'test', '--matchfile', tmp]) + cmd.extend(['-u', '-m', 'test', '--matchfile', tmp]) cmd.extend(args.test_args) print("+ %s" % format_shell_args(cmd)) + + sys.stdout.flush() + sys.stderr.flush() + proc = subprocess.run(cmd) return proc.returncode finally: @@ -137,8 +142,8 @@ def main(): ntest = max(ntest // 2, 1) subtests = random.sample(tests, ntest) - print("[+] Iteration %s: run %s tests/%s" - % (iteration, len(subtests), len(tests))) + print(f"[+] Iteration {iteration}/{args.max_iter}: " + f"run {len(subtests)} tests/{len(tests)}") print() exitcode = run_tests(args, subtests) @@ -170,10 +175,10 @@ def main(): if len(tests) <= args.max_tests: print("Bisection completed in %s iterations and %s" % (iteration, datetime.timedelta(seconds=dt))) - sys.exit(1) else: print("Bisection failed after %s iterations and %s" % (iteration, datetime.timedelta(seconds=dt))) + sys.exit(1) if __name__ == "__main__": diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 168f6f73f6186f..58ffc0ad4ab88b 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5004,12 +5004,16 @@ Test_property_set_impl(TestObj *self, PyObject *value); static int Test_property_set(TestObj *self, PyObject *value, void *Py_UNUSED(context)) { - return Test_property_set_impl(self, value); + int return_value; + + return_value = Test_property_set_impl(self, value); + + return return_value; } static int Test_property_set_impl(TestObj *self, PyObject *value) -/*[clinic end generated code: output=9797cd03c5204ddb input=3bc3f46a23c83a88]*/ +/*[clinic end generated code: output=d51023f17c4ac3a1 input=3bc3f46a23c83a88]*/ /*[clinic input] output push @@ -5327,52 +5331,6 @@ Test__pyarg_parsestackandkeywords_impl(TestObj *self, PyTypeObject *cls, /*[clinic end generated code: output=4fda8a7f2547137c input=fc72ef4b4cfafabc]*/ -/*[clinic input] -Test.__init__ -> long -Test overriding the __init__ return converter -[clinic start generated code]*/ - -PyDoc_STRVAR(Test___init____doc__, -"Test()\n" -"--\n" -"\n" -"Test overriding the __init__ return converter"); - -static long -Test___init___impl(TestObj *self); - -static int -Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) -{ - int return_value = -1; - PyTypeObject *base_tp = TestType; - long _return_value; - - if ((Py_IS_TYPE(self, base_tp) || - Py_TYPE(self)->tp_new == base_tp->tp_new) && - !_PyArg_NoPositional("Test", args)) { - goto exit; - } - if ((Py_IS_TYPE(self, base_tp) || - Py_TYPE(self)->tp_new == base_tp->tp_new) && - !_PyArg_NoKeywords("Test", kwargs)) { - goto exit; - } - _return_value = Test___init___impl((TestObj *)self); - if ((_return_value == -1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromLong(_return_value); - -exit: - return return_value; -} - -static long -Test___init___impl(TestObj *self) -/*[clinic end generated code: output=daf6ee12c4e443fb input=311af0dc7f17e8e9]*/ - - /*[clinic input] fn_with_default_binop_expr arg: object(c_default='CONST_A + CONST_B') = a+b diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 0053bce4292f64..608b12bb6f2a38 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -347,6 +347,8 @@ def _create_parser(): help='override the working directory for the test run') group.add_argument('--cleanup', action='store_true', help='remove old test_python_* directories') + group.add_argument('--bisect', action='store_true', + help='if some tests fail, run test.bisect_cmd on them') group.add_argument('--dont-add-python-opts', dest='_add_python_opts', action='store_false', help="internal option, don't use it") diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index b24c1b9205450b..6fbe3d00981ddd 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -7,8 +7,7 @@ import time import trace -from test import support -from test.support import os_helper, MS_WINDOWS +from test.support import os_helper, MS_WINDOWS, flush_std_streams from .cmdline import _parse_args, Namespace from .findtests import findtests, split_test_packages, list_cases @@ -73,6 +72,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.want_cleanup: bool = ns.cleanup self.want_rerun: bool = ns.rerun self.want_run_leaks: bool = ns.runleaks + self.want_bisect: bool = ns.bisect self.ci_mode: bool = (ns.fast_ci or ns.slow_ci) self.want_add_python_opts: bool = (_add_python_opts @@ -273,6 +273,55 @@ def rerun_failed_tests(self, runtests: RunTests): self.display_result(rerun_runtests) + def _run_bisect(self, runtests: RunTests, test: str, progress: str) -> bool: + print() + title = f"Bisect {test}" + if progress: + title = f"{title} ({progress})" + print(title) + print("#" * len(title)) + print() + + cmd = runtests.create_python_cmd() + cmd.extend([ + "-u", "-m", "test.bisect_cmd", + # Limit to 25 iterations (instead of 100) to not abuse CI resources + "--max-iter", "25", + "-v", + # runtests.match_tests is not used (yet) for bisect_cmd -i arg + ]) + cmd.extend(runtests.bisect_cmd_args()) + cmd.append(test) + print("+", shlex.join(cmd), flush=True) + + flush_std_streams() + + import subprocess + proc = subprocess.run(cmd, timeout=runtests.timeout) + exitcode = proc.returncode + + title = f"{title}: exit code {exitcode}" + print(title) + print("#" * len(title)) + print(flush=True) + + if exitcode: + print(f"Bisect failed with exit code {exitcode}") + return False + + return True + + def run_bisect(self, runtests: RunTests) -> None: + tests, _ = self.results.prepare_rerun(clear=False) + + for index, name in enumerate(tests, 1): + if len(tests) > 1: + progress = f"{index}/{len(tests)}" + else: + progress = "" + if not self._run_bisect(runtests, name, progress): + return + def display_result(self, runtests): # If running the test suite for PGO then no one cares about results. if runtests.pgo: @@ -466,7 +515,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: setup_process() - if self.hunt_refleak and not self.num_workers: + if (runtests.hunt_refleak is not None) and (not self.num_workers): # gh-109739: WindowsLoadTracker thread interfers with refleak check use_load_tracker = False else: @@ -486,6 +535,9 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: if self.want_rerun and self.results.need_rerun(): self.rerun_failed_tests(runtests) + + if self.want_bisect and self.results.need_rerun(): + self.run_bisect(runtests) finally: if use_load_tracker: self.logger.stop_load_tracker() diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 71a70af6882d16..f582c0d3e7ff13 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -88,9 +88,12 @@ def get_pooled_int(value): rc_before = alloc_before = fd_before = interned_before = 0 if not quiet: - print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, - flush=True) + print("beginning", repcount, "repetitions. Showing number of leaks " + "(. for 0 or less, X for 10 or more)", + file=sys.stderr) + numbers = ("1234567890"*(repcount//10 + 1))[:repcount] + numbers = numbers[:warmups] + ':' + numbers[warmups:] + print(numbers, file=sys.stderr, flush=True) results = None dash_R_cleanup(fs, ps, pic, zdc, abcs) @@ -116,13 +119,27 @@ def get_pooled_int(value): rc_after = gettotalrefcount() - interned_after * 2 fd_after = fd_count() - if not quiet: - print('.', end='', file=sys.stderr, flush=True) - rc_deltas[i] = get_pooled_int(rc_after - rc_before) alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before) fd_deltas[i] = get_pooled_int(fd_after - fd_before) + if not quiet: + # use max, not sum, so total_leaks is one of the pooled ints + total_leaks = max(rc_deltas[i], alloc_deltas[i], fd_deltas[i]) + if total_leaks <= 0: + symbol = '.' + elif total_leaks < 10: + symbol = ( + '.', '1', '2', '3', '4', '5', '6', '7', '8', '9', + )[total_leaks] + else: + symbol = 'X' + if i == warmups: + print(' ', end='', file=sys.stderr, flush=True) + print(symbol, end='', file=sys.stderr, flush=True) + del total_leaks + del symbol + alloc_before = alloc_after rc_before = rc_after fd_before = fd_after @@ -158,14 +175,20 @@ def check_fd_deltas(deltas): ]: # ignore warmup runs deltas = deltas[warmups:] - if checker(deltas): + failing = checker(deltas) + suspicious = any(deltas) + if failing or suspicious: msg = '%s leaked %s %s, sum=%s' % ( test_name, deltas, item_name, sum(deltas)) - print(msg, file=sys.stderr, flush=True) - with open(filename, "a", encoding="utf-8") as refrep: - print(msg, file=refrep) - refrep.flush() - failed = True + print(msg, end='', file=sys.stderr) + if failing: + print(file=sys.stderr, flush=True) + with open(filename, "a", encoding="utf-8") as refrep: + print(msg, file=refrep) + refrep.flush() + failed = True + else: + print(' (this is fine)', file=sys.stderr, flush=True) return (failed, results) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index a41ea8aba028c3..85c82052eae19b 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -138,7 +138,7 @@ def get_coverage_results(self) -> trace.CoverageResults: def need_rerun(self): return bool(self.rerun_results) - def prepare_rerun(self) -> tuple[TestTuple, FilterDict]: + def prepare_rerun(self, *, clear: bool = True) -> tuple[TestTuple, FilterDict]: tests: TestList = [] match_tests_dict = {} for result in self.rerun_results: @@ -149,11 +149,12 @@ def prepare_rerun(self) -> tuple[TestTuple, FilterDict]: if match_tests: match_tests_dict[result.test_name] = match_tests - # Clear previously failed tests - self.rerun_bad.extend(self.bad) - self.bad.clear() - self.env_changed.clear() - self.rerun_results.clear() + if clear: + # Clear previously failed tests + self.rerun_bad.extend(self.bad) + self.bad.clear() + self.env_changed.clear() + self.rerun_results.clear() return (tuple(tests), match_tests_dict) diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index edd72276320e41..8e9779524c56a2 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -2,7 +2,9 @@ import dataclasses import json import os +import shlex import subprocess +import sys from typing import Any from test import support @@ -67,6 +69,11 @@ class HuntRefleak: runs: int filename: StrPath + def bisect_cmd_args(self) -> list[str]: + # Ignore filename since it can contain colon (":"), + # and usually it's not used. Use the default filename. + return ["-R", f"{self.warmups}:{self.runs}:"] + @dataclasses.dataclass(slots=True, frozen=True) class RunTests: @@ -137,6 +144,49 @@ def json_file_use_stdout(self) -> bool: or support.is_wasi ) + def create_python_cmd(self) -> list[str]: + python_opts = support.args_from_interpreter_flags() + if self.python_cmd is not None: + executable = self.python_cmd + # Remove -E option, since --python=COMMAND can set PYTHON + # environment variables, such as PYTHONPATH, in the worker + # process. + python_opts = [opt for opt in python_opts if opt != "-E"] + else: + executable = (sys.executable,) + cmd = [*executable, *python_opts] + if '-u' not in python_opts: + cmd.append('-u') # Unbuffered stdout and stderr + if self.coverage: + cmd.append("-Xpresite=test.cov") + return cmd + + def bisect_cmd_args(self) -> list[str]: + args = [] + if self.fail_fast: + args.append("--failfast") + if self.fail_env_changed: + args.append("--fail-env-changed") + if self.timeout: + args.append(f"--timeout={self.timeout}") + if self.hunt_refleak is not None: + args.extend(self.hunt_refleak.bisect_cmd_args()) + if self.test_dir: + args.extend(("--testdir", self.test_dir)) + if self.memory_limit: + args.extend(("--memlimit", self.memory_limit)) + if self.gc_threshold: + args.append(f"--threshold={self.gc_threshold}") + if self.use_resources: + args.extend(("-u", ','.join(self.use_resources))) + if self.python_cmd: + cmd = shlex.join(self.python_cmd) + args.extend(("--python", cmd)) + if self.randomize: + args.append(f"--randomize") + args.append(f"--randseed={self.random_seed}") + return args + @dataclasses.dataclass(slots=True, frozen=True) class WorkerRunTests(RunTests): diff --git a/Lib/test/libregrtest/win_utils.py b/Lib/test/libregrtest/win_utils.py index 5736cdfd3c7292..b51fde0af57d1f 100644 --- a/Lib/test/libregrtest/win_utils.py +++ b/Lib/test/libregrtest/win_utils.py @@ -24,6 +24,10 @@ class WindowsLoadTracker(): """ def __init__(self): + # make __del__ not fail if pre-flight test fails + self._running = None + self._stopped = None + # Pre-flight test for access to the performance data; # `PermissionError` will be raised if not allowed winreg.QueryInfoKey(winreg.HKEY_PERFORMANCE_DATA) diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 7a6d33d4499943..f8b8e45eca3276 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -3,7 +3,6 @@ import os from typing import Any, NoReturn -from test import support from test.support import os_helper, Py_DEBUG from .setup import setup_process, setup_test_dir @@ -19,23 +18,10 @@ def create_worker_process(runtests: WorkerRunTests, output_fd: int, tmp_dir: StrPath | None = None) -> subprocess.Popen: - python_cmd = runtests.python_cmd worker_json = runtests.as_json() - python_opts = support.args_from_interpreter_flags() - if python_cmd is not None: - executable = python_cmd - # Remove -E option, since --python=COMMAND can set PYTHON environment - # variables, such as PYTHONPATH, in the worker process. - python_opts = [opt for opt in python_opts if opt != "-E"] - else: - executable = (sys.executable,) - if runtests.coverage: - python_opts.append("-Xpresite=test.cov") - cmd = [*executable, *python_opts, - '-u', # Unbuffered stdout and stderr - '-m', 'test.libregrtest.worker', - worker_json] + cmd = runtests.create_python_cmd() + cmd.extend(['-m', 'test.libregrtest.worker', worker_json]) env = dict(os.environ) if tmp_dir is not None: diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index d9ab21d4941cdb..26118e14bb97e0 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -562,3 +562,8 @@ def test_exhausted_iterator(self): self.assertEqual(list(exhit), []) self.assertEqual(list(empit), [9]) self.assertEqual(a, self.type2test([1, 2, 3, 9])) + + # gh-115733: Crash when iterating over exhausted iterator + exhit = iter(self.type2test([1, 2, 3])) + for _ in exhit: + next(exhit, 1) diff --git a/Lib/test/sortperf.py b/Lib/test/sortperf.py deleted file mode 100644 index 14a9d827ed57c5..00000000000000 --- a/Lib/test/sortperf.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Sort performance test. - -See main() for command line syntax. -See tabulate() for output format. - -""" - -import sys -import time -import random -import marshal -import tempfile -import os - -td = tempfile.gettempdir() - -def randfloats(n): - """Return a list of n random floats in [0, 1).""" - # Generating floats is expensive, so this writes them out to a file in - # a temp directory. If the file already exists, it just reads them - # back in and shuffles them a bit. - fn = os.path.join(td, "rr%06d" % n) - try: - fp = open(fn, "rb") - except OSError: - r = random.random - result = [r() for i in range(n)] - try: - try: - fp = open(fn, "wb") - marshal.dump(result, fp) - fp.close() - fp = None - finally: - if fp: - try: - os.unlink(fn) - except OSError: - pass - except OSError as msg: - print("can't write", fn, ":", msg) - else: - result = marshal.load(fp) - fp.close() - # Shuffle it a bit... - for i in range(10): - i = random.randrange(n) - temp = result[:i] - del result[:i] - temp.reverse() - result.extend(temp) - del temp - assert len(result) == n - return result - -def flush(): - sys.stdout.flush() - -def doit(L): - t0 = time.perf_counter() - L.sort() - t1 = time.perf_counter() - print("%6.2f" % (t1-t0), end=' ') - flush() - -def tabulate(r): - r"""Tabulate sort speed for lists of various sizes. - - The sizes are 2**i for i in r (the argument, a list). - - The output displays i, 2**i, and the time to sort arrays of 2**i - floating point numbers with the following properties: - - *sort: random data - \sort: descending data - /sort: ascending data - 3sort: ascending, then 3 random exchanges - +sort: ascending, then 10 random at the end - %sort: ascending, then randomly replace 1% of the elements w/ random values - ~sort: many duplicates - =sort: all equal - !sort: worst case scenario - - """ - cases = tuple([ch + "sort" for ch in r"*\/3+%~=!"]) - fmt = ("%2s %7s" + " %6s"*len(cases)) - print(fmt % (("i", "2**i") + cases)) - for i in r: - n = 1 << i - L = randfloats(n) - print("%2d %7d" % (i, n), end=' ') - flush() - doit(L) # *sort - L.reverse() - doit(L) # \sort - doit(L) # /sort - - # Do 3 random exchanges. - for dummy in range(3): - i1 = random.randrange(n) - i2 = random.randrange(n) - L[i1], L[i2] = L[i2], L[i1] - doit(L) # 3sort - - # Replace the last 10 with random floats. - if n >= 10: - L[-10:] = [random.random() for dummy in range(10)] - doit(L) # +sort - - # Replace 1% of the elements at random. - for dummy in range(n // 100): - L[random.randrange(n)] = random.random() - doit(L) # %sort - - # Arrange for lots of duplicates. - if n > 4: - del L[4:] - L = L * (n // 4) - # Force the elements to be distinct objects, else timings can be - # artificially low. - L = list(map(lambda x: --x, L)) - doit(L) # ~sort - del L - - # All equal. Again, force the elements to be distinct objects. - L = list(map(abs, [-0.5] * n)) - doit(L) # =sort - del L - - # This one looks like [3, 2, 1, 0, 0, 1, 2, 3]. It was a bad case - # for an older implementation of quicksort, which used the median - # of the first, last and middle elements as the pivot. - half = n // 2 - L = list(range(half - 1, -1, -1)) - L.extend(range(half)) - # Force to float, so that the timings are comparable. This is - # significantly faster if we leave them as ints. - L = list(map(float, L)) - doit(L) # !sort - print() - -def main(): - """Main program when invoked as a script. - - One argument: tabulate a single row. - Two arguments: tabulate a range (inclusive). - Extra arguments are used to seed the random generator. - - """ - # default range (inclusive) - k1 = 15 - k2 = 20 - if sys.argv[1:]: - # one argument: single point - k1 = k2 = int(sys.argv[1]) - if sys.argv[2:]: - # two arguments: specify range - k2 = int(sys.argv[2]) - if sys.argv[3:]: - # derive random seed from remaining arguments - x = 1 - for a in sys.argv[3:]: - x = 69069 * x + hash(a) - random.seed(x) - r = range(k1, k2+1) # include the end point - tabulate(r) - -if __name__ == '__main__': - main() diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 1d03ec0f5bd12b..14e3766a34377b 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -532,24 +532,33 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" -# Apple mobile platforms (iOS/tvOS/watchOS) are POSIX-like but do not -# have subprocess or fork support. is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"} is_apple = is_apple_mobile or sys.platform == "darwin" has_fork_support = hasattr(os, "fork") and not ( + # WASM and Apple mobile platforms do not support subprocesses. is_emscripten or is_wasi or is_apple_mobile + + # Although Android supports fork, it's unsafe to call it from Python because + # all Android apps are multi-threaded. + or is_android ) def requires_fork(): return unittest.skipUnless(has_fork_support, "requires working os.fork()") has_subprocess_support = not ( + # WASM and Apple mobile platforms do not support subprocesses. is_emscripten or is_wasi or is_apple_mobile + + # Although Android supports subproceses, they're almost never useful in + # practice (see PEP 738). And most of the tests that use them are calling + # sys.executable, which won't work when Python is embedded in an Android app. + or is_android ) def requires_subprocess(): @@ -2381,6 +2390,46 @@ def sleeping_retry(timeout, err_msg=None, /, delay = min(delay * 2, max_delay) +class CPUStopwatch: + """Context manager to roughly time a CPU-bound operation. + + Disables GC. Uses CPU time if it can (i.e. excludes sleeps & time of + other processes). + + N.B.: + - This *includes* time spent in other threads. + - Some systems only have a coarse resolution; check + stopwatch.clock_info.rseolution if. + + Usage: + + with ProcessStopwatch() as stopwatch: + ... + elapsed = stopwatch.seconds + resolution = stopwatch.clock_info.resolution + """ + def __enter__(self): + get_time = time.process_time + clock_info = time.get_clock_info('process_time') + if get_time() <= 0: # some platforms like WASM lack process_time() + get_time = time.monotonic + clock_info = time.get_clock_info('monotonic') + self.context = disable_gc() + self.context.__enter__() + self.get_time = get_time + self.clock_info = clock_info + self.start_time = get_time() + return self + + def __exit__(self, *exc): + try: + end_time = self.get_time() + finally: + result = self.context.__exit__(*exc) + self.seconds = end_time - self.start_time + return result + + @contextlib.contextmanager def adjust_int_max_str_digits(max_digits): """Temporarily change the integer string conversion length limit.""" diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index 15a908e9663593..d02ffbae1113c0 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -6,7 +6,7 @@ # aliases: from _xxsubinterpreters import ( - InterpreterError, InterpreterNotFoundError, + InterpreterError, InterpreterNotFoundError, NotShareableError, is_shareable, ) @@ -14,7 +14,8 @@ __all__ = [ 'get_current', 'get_main', 'create', 'list_all', 'is_shareable', 'Interpreter', - 'InterpreterError', 'InterpreterNotFoundError', 'ExecFailure', + 'InterpreterError', 'InterpreterNotFoundError', 'ExecutionFailed', + 'NotShareableError', 'create_queue', 'Queue', 'QueueEmpty', 'QueueFull', ] @@ -42,7 +43,11 @@ def __getattr__(name): {formatted} """.strip() -class ExecFailure(RuntimeError): +class ExecutionFailed(RuntimeError): + """An unhandled exception happened during execution. + + This is raised from Interpreter.exec() and Interpreter.call(). + """ def __init__(self, excinfo): msg = excinfo.formatted @@ -157,7 +162,7 @@ def prepare_main(self, ns=None, /, **kwargs): ns = dict(ns, **kwargs) if ns is not None else kwargs _interpreters.set___main___attrs(self._id, ns) - def exec_sync(self, code, /): + def exec(self, code, /): """Run the given source code in the interpreter. This is essentially the same as calling the builtin "exec" @@ -166,10 +171,10 @@ def exec_sync(self, code, /): There is no return value. - If the code raises an unhandled exception then an ExecFailure - is raised, which summarizes the unhandled exception. The actual - exception is discarded because objects cannot be shared between - interpreters. + If the code raises an unhandled exception then an ExecutionFailed + exception is raised, which summarizes the unhandled exception. + The actual exception is discarded because objects cannot be + shared between interpreters. This blocks the current Python thread until done. During that time, the previous interpreter is allowed to run @@ -177,11 +182,35 @@ def exec_sync(self, code, /): """ excinfo = _interpreters.exec(self._id, code) if excinfo is not None: - raise ExecFailure(excinfo) + raise ExecutionFailed(excinfo) + + def call(self, callable, /): + """Call the object in the interpreter with given args/kwargs. + + Only functions that take no arguments and have no closure + are supported. - def run(self, code, /): + The return value is discarded. + + If the callable raises an exception then the error display + (including full traceback) is send back between the interpreters + and an ExecutionFailed exception is raised, much like what + happens with Interpreter.exec(). + """ + # XXX Support args and kwargs. + # XXX Support arbitrary callables. + # XXX Support returning the return value (e.g. via pickle). + excinfo = _interpreters.call(self._id, callable) + if excinfo is not None: + raise ExecutionFailed(excinfo) + + def call_in_thread(self, callable, /): + """Return a new thread that calls the object in the interpreter. + + The return value and any raised exception are discarded. + """ def task(): - self.exec_sync(code) + self.call(callable) t = threading.Thread(target=task) t.start() return t diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py index aead0c40ca9667..f9978f0bec5a62 100644 --- a/Lib/test/support/interpreters/queues.py +++ b/Lib/test/support/interpreters/queues.py @@ -1,5 +1,6 @@ """Cross-interpreter Queues High Level Module.""" +import pickle import queue import time import weakref @@ -31,20 +32,26 @@ class QueueFull(_queues.QueueFull, queue.Full): """ -def create(maxsize=0): +_SHARED_ONLY = 0 +_PICKLED = 1 + +def create(maxsize=0, *, syncobj=False): """Return a new cross-interpreter queue. The queue may be used to pass data safely between interpreters. + + "syncobj" sets the default for Queue.put() + and Queue.put_nowait(). """ - qid = _queues.create(maxsize) - return Queue(qid) + fmt = _SHARED_ONLY if syncobj else _PICKLED + qid = _queues.create(maxsize, fmt) + return Queue(qid, _fmt=fmt) def list_all(): """Return a list of all open queues.""" - return [Queue(qid) - for qid in _queues.list_all()] - + return [Queue(qid, _fmt=fmt) + for qid, fmt in _queues.list_all()] _known_queues = weakref.WeakValueDictionary() @@ -52,17 +59,20 @@ def list_all(): class Queue: """A cross-interpreter queue.""" - def __new__(cls, id, /): + def __new__(cls, id, /, *, _fmt=None): # There is only one instance for any given ID. if isinstance(id, int): id = int(id) else: raise TypeError(f'id must be an int, got {id!r}') + if _fmt is None: + _fmt = _queues.get_default_fmt(id) try: self = _known_queues[id] except KeyError: self = super().__new__(cls) self._id = id + self._fmt = _fmt _known_queues[id] = self _queues.bind(id) return self @@ -105,20 +115,50 @@ def qsize(self): return _queues.get_count(self._id) def put(self, obj, timeout=None, *, + syncobj=None, _delay=10 / 1000, # 10 milliseconds ): """Add the object to the queue. This blocks while the queue is full. + + If "syncobj" is None (the default) then it uses the + queue's default, set with create_queue().. + + If "syncobj" is false then all objects are supported, + at the expense of worse performance. + + If "syncobj" is true then the object must be "shareable". + Examples of "shareable" objects include the builtin singletons, + str, and memoryview. One benefit is that such objects are + passed through the queue efficiently. + + The key difference, though, is conceptual: the corresponding + object returned from Queue.get() will be strictly equivalent + to the given obj. In other words, the two objects will be + effectively indistinguishable from each other, even if the + object is mutable. The received object may actually be the + same object, or a copy (immutable values only), or a proxy. + Regardless, the received object should be treated as though + the original has been shared directly, whether or not it + actually is. That's a slightly different and stronger promise + than just (initial) equality, which is all "syncobj=False" + can promise. """ + if syncobj is None: + fmt = self._fmt + else: + fmt = _SHARED_ONLY if syncobj else _PICKLED if timeout is not None: timeout = int(timeout) if timeout < 0: raise ValueError(f'timeout value must be non-negative') end = time.time() + timeout + if fmt is _PICKLED: + obj = pickle.dumps(obj) while True: try: - _queues.put(self._id, obj) + _queues.put(self._id, obj, fmt) except _queues.QueueFull as exc: if timeout is not None and time.time() >= end: exc.__class__ = QueueFull @@ -127,9 +167,15 @@ def put(self, obj, timeout=None, *, else: break - def put_nowait(self, obj): + def put_nowait(self, obj, *, syncobj=None): + if syncobj is None: + fmt = self._fmt + else: + fmt = _SHARED_ONLY if syncobj else _PICKLED + if fmt is _PICKLED: + obj = pickle.dumps(obj) try: - return _queues.put(self._id, obj) + _queues.put(self._id, obj, fmt) except _queues.QueueFull as exc: exc.__class__ = QueueFull raise # re-raise @@ -148,12 +194,18 @@ def get(self, timeout=None, *, end = time.time() + timeout while True: try: - return _queues.get(self._id) + obj, fmt = _queues.get(self._id) except _queues.QueueEmpty as exc: if timeout is not None and time.time() >= end: exc.__class__ = QueueEmpty raise # re-raise time.sleep(_delay) + else: + break + if fmt == _PICKLED: + obj = pickle.loads(obj) + else: + assert fmt == _SHARED_ONLY return obj def get_nowait(self): @@ -163,10 +215,15 @@ def get_nowait(self): is the same as get(). """ try: - return _queues.get(self._id) + obj, fmt = _queues.get(self._id) except _queues.QueueEmpty as exc: exc.__class__ = QueueEmpty raise # re-raise + if fmt == _PICKLED: + obj = pickle.loads(obj) + else: + assert fmt == _SHARED_ONLY + return obj _queues._register_queue_type(Queue) diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 22787e32b5f3ab..ffa5fc52e9b2bd 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -198,6 +198,23 @@ def skip_unless_symlink(test): return test if ok else unittest.skip(msg)(test) +_can_hardlink = None + +def can_hardlink(): + global _can_hardlink + if _can_hardlink is None: + # Android blocks hard links using SELinux + # (https://stackoverflow.com/q/32365690). + _can_hardlink = hasattr(os, "link") and not support.is_android + return _can_hardlink + + +def skip_unless_hardlink(test): + ok = can_hardlink() + msg = "requires hardlink support" + return test if ok else unittest.skip(msg)(test) + + _can_xattr = None diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index c87cde4b3d1fab..19dcbb207e914a 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -5,11 +5,6 @@ import sys import types -try: - import _multiprocessing -except ModuleNotFoundError: - _multiprocessing = None - if support.check_sanitizer(address=True, memory=True): SKIP_MODULES = frozenset(( @@ -36,17 +31,6 @@ class FailedImport(RuntimeError): class AllTest(unittest.TestCase): - def setUp(self): - # concurrent.futures uses a __getattr__ hook. Its __all__ triggers - # import of a submodule, which fails when _multiprocessing is not - # available. - if _multiprocessing is None: - sys.modules["_multiprocessing"] = types.ModuleType("_multiprocessing") - - def tearDown(self): - if _multiprocessing is None: - sys.modules.pop("_multiprocessing") - def check_all(self, modname): names = {} with warnings_helper.check_warnings( diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 86d6e81a71642b..617b1721f3dbb1 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2274,6 +2274,34 @@ def test_parse_known_args(self): (NS(foo=False, bar=0.5, w=7, x='b'), ['-W', '-X', 'Y', 'Z']), ) + def test_parse_known_args_with_single_dash_option(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('-k', '--known', action='count', default=0) + parser.add_argument('-n', '--new', action='count', default=0) + self.assertEqual(parser.parse_known_args(['-k', '-u']), + (NS(known=1, new=0), ['-u'])) + self.assertEqual(parser.parse_known_args(['-u', '-k']), + (NS(known=1, new=0), ['-u'])) + self.assertEqual(parser.parse_known_args(['-ku']), + (NS(known=1, new=0), ['-u'])) + self.assertArgumentParserError(parser.parse_known_args, ['-k=u']) + self.assertEqual(parser.parse_known_args(['-uk']), + (NS(known=0, new=0), ['-uk'])) + self.assertEqual(parser.parse_known_args(['-u=k']), + (NS(known=0, new=0), ['-u=k'])) + self.assertEqual(parser.parse_known_args(['-kunknown']), + (NS(known=1, new=0), ['-unknown'])) + self.assertArgumentParserError(parser.parse_known_args, ['-k=unknown']) + self.assertEqual(parser.parse_known_args(['-ku=nknown']), + (NS(known=1, new=0), ['-u=nknown'])) + self.assertEqual(parser.parse_known_args(['-knew']), + (NS(known=1, new=1), ['-ew'])) + self.assertArgumentParserError(parser.parse_known_args, ['-kn=ew']) + self.assertArgumentParserError(parser.parse_known_args, ['-k-new']) + self.assertArgumentParserError(parser.parse_known_args, ['-kn-ew']) + self.assertEqual(parser.parse_known_args(['-kne-w']), + (NS(known=1, new=1), ['-e-w'])) + def test_dest(self): parser = ErrorRaisingArgumentParser() parser.add_argument('--foo', action='store_true') @@ -2836,6 +2864,27 @@ def test_help(self): ''' self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): + self.maxDiff = None + parser = ErrorRaisingArgumentParser(prog='PROG') + commands = parser.add_subparsers(title="commands", dest="command") + cmd_foo = commands.add_parser("foo") + group = cmd_foo.add_mutually_exclusive_group() + group.add_argument('--verbose', action='store_true', help=argparse.SUPPRESS) + group.add_argument('--quiet', action='store_true', help=argparse.SUPPRESS) + longopt = '--' + 'long'*32 + longmeta = 'LONG'*32 + cmd_foo.add_argument(longopt) + expected = f'''\ + usage: PROG foo [-h] + [{longopt} {longmeta}] + + options: + -h, --help show this help message and exit + {longopt} {longmeta} + ''' + self.assertEqual(cmd_foo.format_help(), textwrap.dedent(expected)) + def test_empty_group(self): # See issue 26952 parser = argparse.ArgumentParser() diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 3789ac22e3899c..7cecf319e3638f 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -525,17 +525,38 @@ def test_field_attr_existence(self): if name == 'Index': continue if self._is_ast_node(name, item): - x = item() + x = self._construct_ast_class(item) if isinstance(x, ast.AST): self.assertIs(type(x._fields), tuple) + def _construct_ast_class(self, cls): + kwargs = {} + for name, typ in cls.__annotations__.items(): + if typ is str: + kwargs[name] = 'capybara' + elif typ is int: + kwargs[name] = 42 + elif typ is object: + kwargs[name] = b'capybara' + elif isinstance(typ, type) and issubclass(typ, ast.AST): + kwargs[name] = self._construct_ast_class(typ) + return cls(**kwargs) + def test_arguments(self): x = ast.arguments() self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs', 'kw_defaults', 'kwarg', 'defaults')) - - with self.assertRaises(AttributeError): - x.args + self.assertEqual(x.__annotations__, { + 'posonlyargs': list[ast.arg], + 'args': list[ast.arg], + 'vararg': ast.arg | None, + 'kwonlyargs': list[ast.arg], + 'kw_defaults': list[ast.expr], + 'kwarg': ast.arg | None, + 'defaults': list[ast.expr], + }) + + self.assertEqual(x.args, []) self.assertIsNone(x.vararg) x = ast.arguments(*range(1, 8)) @@ -551,7 +572,7 @@ def test_field_attr_writable_deprecated(self): self.assertEqual(x._fields, 666) def test_field_attr_writable(self): - x = ast.Constant() + x = ast.Constant(1) # We can assign to _fields x._fields = 666 self.assertEqual(x._fields, 666) @@ -611,15 +632,22 @@ def test_classattrs_deprecated(self): self.assertEqual([str(w.message) for w in wlog], [ 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + "Constant.__init__ missing 1 required positional argument: 'value'. This will become " + 'an error in Python 3.15.', 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + "Constant.__init__ missing 1 required positional argument: 'value'. This will become " + 'an error in Python 3.15.', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', + "Constant.__init__ got an unexpected keyword argument 'foo'. Support for " + 'arbitrary keyword arguments is deprecated and will be removed in Python ' + '3.15.', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', @@ -636,7 +664,8 @@ def test_classattrs_deprecated(self): ]) def test_classattrs(self): - x = ast.Constant() + with self.assertWarns(DeprecationWarning): + x = ast.Constant() self.assertEqual(x._fields, ('value', 'kind')) with self.assertRaises(AttributeError): @@ -651,7 +680,7 @@ def test_classattrs(self): with self.assertRaises(AttributeError): x.foobar - x = ast.Constant(lineno=2) + x = ast.Constant(lineno=2, value=3) self.assertEqual(x.lineno, 2) x = ast.Constant(42, lineno=0) @@ -662,8 +691,9 @@ def test_classattrs(self): self.assertRaises(TypeError, ast.Constant, 1, None, 2) self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0) - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') + # Arbitrary keyword arguments are supported (but deprecated) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): ast.Constant(1, value=2) @@ -815,11 +845,11 @@ def test_isinstance(self): assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes) assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant) assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) - assertNumDeprecated(self.assertNotIsInstance, Constant(), Num) - assertStrDeprecated(self.assertNotIsInstance, Constant(), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant(), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(), Ellipsis) + assertNumDeprecated(self.assertNotIsInstance, Constant(None), Num) + assertStrDeprecated(self.assertNotIsInstance, Constant(None), Str) + assertBytesDeprecated(self.assertNotIsInstance, Constant(None), Bytes) + assertNameConstantDeprecated(self.assertNotIsInstance, Constant(1), NameConstant) + assertEllipsisDeprecated(self.assertNotIsInstance, Constant(None), Ellipsis) class S(str): pass with assertStrDeprecated(): @@ -888,8 +918,9 @@ def test_module(self): self.assertEqual(x.body, body) def test_nodeclasses(self): - # Zero arguments constructor explicitly allowed - x = ast.BinOp() + # Zero arguments constructor explicitly allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.BinOp() self.assertEqual(x._fields, ('left', 'op', 'right')) # Random attribute allowed too @@ -927,8 +958,9 @@ def test_nodeclasses(self): self.assertEqual(x.right, 3) self.assertEqual(x.lineno, 0) - # Random kwargs also allowed - x = ast.BinOp(1, 2, 3, foobarbaz=42) + # Random kwargs also allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.BinOp(1, 2, 3, foobarbaz=42) self.assertEqual(x.foobarbaz, 42) def test_no_fields(self): @@ -941,8 +973,9 @@ def test_pickling(self): for protocol in range(pickle.HIGHEST_PROTOCOL + 1): for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests): - ast2 = pickle.loads(pickle.dumps(ast, protocol)) - self.assertEqual(to_tuple(ast2), to_tuple(ast)) + with self.subTest(ast=ast, protocol=protocol): + ast2 = pickle.loads(pickle.dumps(ast, protocol)) + self.assertEqual(to_tuple(ast2), to_tuple(ast)) def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) @@ -1045,19 +1078,15 @@ def test_positional_only_feature_version(self): with self.assertRaises(SyntaxError): ast.parse('lambda x=1, /: ...', feature_version=(3, 7)) - def test_parenthesized_with_feature_version(self): - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 10)) - # While advertised as a feature in Python 3.10, this was allowed starting 3.9 - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 9)) - with self.assertRaises(SyntaxError): - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 8)) - ast.parse('with CtxManager() as example: ...', feature_version=(3, 8)) - def test_assignment_expression_feature_version(self): ast.parse('(x := 0)', feature_version=(3, 8)) with self.assertRaises(SyntaxError): ast.parse('(x := 0)', feature_version=(3, 7)) + def test_conditional_context_managers_parse_with_low_feature_version(self): + # regression test for gh-115881 + ast.parse('with (x() if y else z()): ...', feature_version=(3, 8)) + def test_exception_groups_feature_version(self): code = dedent(''' try: ... @@ -1314,8 +1343,9 @@ def test_copy_location(self): 'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, ' 'col_offset=0, end_lineno=1, end_col_offset=5))' ) - src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1) - new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None)) + func = ast.Name('spam', ast.Load()) + src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1, func=func) + new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None, func=func)) self.assertIsNone(new.end_lineno) self.assertIsNone(new.end_col_offset) self.assertEqual(new.lineno, 1) @@ -1574,15 +1604,15 @@ def test_level_as_none(self): self.assertIn('sleep', ns) def test_recursion_direct(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) + e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = e with self.assertRaises(RecursionError): with support.infinite_recursion(): compile(ast.Expression(e), "", "eval") def test_recursion_indirect(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) - f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) + e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) + f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = f f.operand = e with self.assertRaises(RecursionError): @@ -2870,6 +2900,23 @@ def visit_Call(self, node: ast.Call): self.assertASTTransformation(PrintToLog, code, expected) +class ASTConstructorTests(unittest.TestCase): + """Test the autogenerated constructors for AST nodes.""" + + def test_FunctionDef(self): + args = ast.arguments() + self.assertEqual(args.args, []) + self.assertEqual(args.posonlyargs, []) + with self.assertWarnsRegex(DeprecationWarning, + r"FunctionDef\.__init__ missing 1 required positional argument: 'name'"): + node = ast.FunctionDef(args=args) + self.assertFalse(hasattr(node, "name")) + self.assertEqual(node.decorator_list, []) + node = ast.FunctionDef(name='foo', args=args) + self.assertEqual(node.name, 'foo') + self.assertEqual(node.decorator_list, []) + + @support.cpython_only class ModuleStateTests(unittest.TestCase): # bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 82071edb252570..4cd872d3a5b2d8 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -231,6 +231,22 @@ def test_set_default_executor_error(self): self.assertIsNone(self.loop._default_executor) + def test_shutdown_default_executor_timeout(self): + class DummyExecutor(concurrent.futures.ThreadPoolExecutor): + def shutdown(self, wait=True, *, cancel_futures=False): + if wait: + time.sleep(0.1) + + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + executor = DummyExecutor() + self.loop.set_default_executor(executor) + + with self.assertWarnsRegex(RuntimeWarning, + "The executor did not finishing joining"): + self.loop.run_until_complete( + self.loop.shutdown_default_executor(timeout=0.01)) + def test_call_soon(self): def cb(): pass diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index c92c88bd5b2429..5b9c871e1d1b5a 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2250,7 +2250,7 @@ def test_handle_repr(self): h = asyncio.Handle(noop, (1, 2), self.loop) filename, lineno = test_utils.get_function_source(noop) self.assertEqual(repr(h), - '' + '' % (filename, lineno)) # cancelled handle @@ -2268,14 +2268,14 @@ def test_handle_repr(self): # partial function cb = functools.partial(noop, 1, 2) h = asyncio.Handle(cb, (3,), self.loop) - regex = (r'^$' + regex = (r'^$' % (re.escape(filename), lineno)) self.assertRegex(repr(h), regex) # partial function with keyword args cb = functools.partial(noop, x=1) h = asyncio.Handle(cb, (2, 3), self.loop) - regex = (r'^$' + regex = (r'^$' % (re.escape(filename), lineno)) self.assertRegex(repr(h), regex) @@ -2316,6 +2316,24 @@ def test_handle_repr_debug(self): '' % (filename, lineno, create_filename, create_lineno)) + # partial function + cb = functools.partial(noop, 1, 2) + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(cb, (3,), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno, + re.escape(create_filename), create_lineno)) + self.assertRegex(repr(h), regex) + + # partial function with keyword args + cb = functools.partial(noop, x=1) + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(cb, (2, 3), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno, + re.escape(create_filename), create_lineno)) + self.assertRegex(repr(h), regex) + def test_handle_source_traceback(self): loop = asyncio.get_event_loop_policy().new_event_loop() loop.set_debug(True) diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index c42856e578b8cc..fcaa2f6ade2b76 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -585,11 +585,10 @@ def test_sendto_memoryview(self): def test_sendto_no_data(self): transport = self.datagram_transport() - transport._buffer.append((b'data', ('0.0.0.0', 12345))) - transport.sendto(b'', ()) - self.assertFalse(self.sock.sendto.called) - self.assertEqual( - [(b'data', ('0.0.0.0', 12345))], list(transport._buffer)) + transport.sendto(b'', ('0.0.0.0', 1234)) + self.assertTrue(self.proactor.sendto.called) + self.proactor.sendto.assert_called_with( + self.sock, b'', addr=('0.0.0.0', 1234)) def test_sendto_buffer(self): transport = self.datagram_transport() @@ -628,6 +627,19 @@ def test_sendto_buffer_memoryview(self): list(transport._buffer)) self.assertIsInstance(transport._buffer[1][0], bytes) + def test_sendto_buffer_nodata(self): + data2 = b'' + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport._write_fut = object() + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.proactor.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + @mock.patch('asyncio.proactor_events.logger') def test_sendto_exception(self, m_log): data = b'data' diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index c22b780b5edcb8..aaeda33dd0c677 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -1280,11 +1280,10 @@ def test_sendto_memoryview(self): def test_sendto_no_data(self): transport = self.datagram_transport() - transport._buffer.append((b'data', ('0.0.0.0', 12345))) - transport.sendto(b'', ()) - self.assertFalse(self.sock.sendto.called) + transport.sendto(b'', ('0.0.0.0', 1234)) + self.assertTrue(self.sock.sendto.called) self.assertEqual( - [(b'data', ('0.0.0.0', 12345))], list(transport._buffer)) + self.sock.sendto.call_args[0], (b'', ('0.0.0.0', 1234))) def test_sendto_buffer(self): transport = self.datagram_transport() @@ -1320,6 +1319,18 @@ def test_sendto_buffer_memoryview(self): list(transport._buffer)) self.assertIsInstance(transport._buffer[1][0], bytes) + def test_sendto_buffer_nodata(self): + data2 = b'' + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.sock.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + def test_sendto_tryagain(self): data = b'data' diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 210990593adfa9..bf123ebf9bd158 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -1130,6 +1130,31 @@ async def inner(httpd): self.assertEqual(messages, []) + def test_unclosed_server_resource_warnings(self): + async def inner(rd, wr): + fut.set_result(True) + with self.assertWarns(ResourceWarning) as cm: + del wr + gc.collect() + self.assertEqual(len(cm.warnings), 1) + self.assertTrue(str(cm.warnings[0].message).startswith("unclosed iWS5d]J*6CRx17-skh9337x""" + b"""ar.{NbQB=+c[cR@eg&FcfFLssg=mfIi5%2YjuU>)kTv.7l}6Nnnj=AD""" + b"""oIFnTp/ga?r8($2sxO*itWpVyu$0IOwmYv=xLzi%y&a6dAb/]tBAI+J""" + b"""CZjQZE0{D[FpSr8GOteoH(41EJe-&}x#)cTlf[Bu8v].4}L}1:^-""" + b"""@qDP""", + b"""abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ""" + b"""0123456789!@#0^&*();:<>,. []{}""": + b"""vpA.SwObN*x>?B1zeKohADlbxB-}$ND3R+ylQTvjm[uizoh55PpF:[^""" + b"""q=D:$s6eQefFLssg=mfIi5@cEbqrBJdKV-ciY]OSe*aw7DWL""", + b'no padding..': b'zF{UpvpS[.zF7NO', + b'zero compression\x00\x00\x00\x00': b'Ds.bnay/tbAb]JhB7]Mg00000', + b'zero compression\x00\x00\x00': b'Ds.bnay/tbAb]JhB7]Mg0000', + b"""Boundary:\x00\x00\x00\x00""": b"""lt}0:wmoI7iSGcW00""", + b'Space compr: ': b'q/DePwGUG3ze:IRarR^H', + b'\xff': b'@@', + b'\xff'*2: b'%nJ', + b'\xff'*3: b'%nS9', + b'\xff'*4: b'%nSc0', + } + + for data, res in tests.items(): + eq(base64.z85encode(data), res) + + self.check_other_types(base64.z85encode, b"www.python.org", + b'CxXl-AcVLsz/dgCA+t') + def test_a85decode(self): eq = self.assertEqual @@ -626,6 +660,41 @@ def test_b85decode(self): self.check_other_types(base64.b85decode, b'cXxL#aCvlSZ*DGca%T', b"www.python.org") + def test_z85decode(self): + eq = self.assertEqual + + tests = { + b'': b'', + b'CxXl-AcVLsz/dgCA+t': b'www.python.org', + b"""009c61o!#m2NH?C3>iWS5d]J*6CRx17-skh9337x""" + b"""ar.{NbQB=+c[cR@eg&FcfFLssg=mfIi5%2YjuU>)kTv.7l}6Nnnj=AD""" + b"""oIFnTp/ga?r8($2sxO*itWpVyu$0IOwmYv=xLzi%y&a6dAb/]tBAI+J""" + b"""CZjQZE0{D[FpSr8GOteoH(41EJe-&}x#)cTlf[Bu8v].4}L}1:^-""" + b"""@qDP""": bytes(range(255)), + b"""vpA.SwObN*x>?B1zeKohADlbxB-}$ND3R+ylQTvjm[uizoh55PpF:[^""" + b"""q=D:$s6eQefFLssg=mfIi5@cEbqrBJdKV-ciY]OSe*aw7DWL""": + b"""abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ""" + b"""0123456789!@#0^&*();:<>,. []{}""", + b'zF{UpvpS[.zF7NO': b'no padding..', + b'Ds.bnay/tbAb]JhB7]Mg00000': b'zero compression\x00\x00\x00\x00', + b'Ds.bnay/tbAb]JhB7]Mg0000': b'zero compression\x00\x00\x00', + b"""lt}0:wmoI7iSGcW00""": b"""Boundary:\x00\x00\x00\x00""", + b'q/DePwGUG3ze:IRarR^H': b'Space compr: ', + b'@@': b'\xff', + b'%nJ': b'\xff'*2, + b'%nS9': b'\xff'*3, + b'%nSc0': b'\xff'*4, + } + + for data, res in tests.items(): + eq(base64.z85decode(data), res) + eq(base64.z85decode(data.decode("ascii")), res) + + self.check_other_types(base64.z85decode, b'CxXl-AcVLsz/dgCA+t', + b'www.python.org') + def test_a85_padding(self): eq = self.assertEqual @@ -707,6 +776,21 @@ def test_b85decode_errors(self): self.assertRaises(ValueError, base64.b85decode, b'|NsC') self.assertRaises(ValueError, base64.b85decode, b'|NsC1') + def test_z85decode_errors(self): + illegal = list(range(33)) + \ + list(b'"\',;_`|\\~') + \ + list(range(128, 256)) + for c in illegal: + with self.assertRaises(ValueError, msg=bytes([c])): + base64.z85decode(b'0000' + bytes([c])) + + # b'\xff\xff\xff\xff' encodes to b'%nSc0', the following will overflow: + self.assertRaises(ValueError, base64.z85decode, b'%') + self.assertRaises(ValueError, base64.z85decode, b'%n') + self.assertRaises(ValueError, base64.z85decode, b'%nS') + self.assertRaises(ValueError, base64.z85decode, b'%nSc') + self.assertRaises(ValueError, base64.z85decode, b'%nSc1') + def test_decode_nonascii_str(self): decode_funcs = (base64.b64decode, base64.standard_b64decode, @@ -714,7 +798,8 @@ def test_decode_nonascii_str(self): base64.b32decode, base64.b16decode, base64.b85decode, - base64.a85decode) + base64.a85decode, + base64.z85decode) for f in decode_funcs: self.assertRaises(ValueError, f, 'with non-ascii \xcb') diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index a3804a945f2e3a..71bb1e75c6affd 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1599,6 +1599,13 @@ def test_extend(self): a = bytearray(b'') a.extend([Indexable(ord('a'))]) self.assertEqual(a, b'a') + a = bytearray(b'abc') + self.assertRaisesRegex(TypeError, # Override for string. + "expected iterable of integers; got: 'str'", + a.extend, 'def') + self.assertRaisesRegex(TypeError, # But not for others. + "can't extend bytearray with float", + a.extend, 1.0) def test_remove(self): b = bytearray(b'hello') diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 1f0b9adc3698b4..772f0eacce28f5 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -3,19 +3,19 @@ import array import unittest +import io from io import BytesIO, DEFAULT_BUFFER_SIZE import os import pickle import glob import tempfile -import pathlib import random import shutil import subprocess import threading from test.support import import_helper from test.support import threading_helper -from test.support.os_helper import unlink +from test.support.os_helper import unlink, FakePath import _compression import sys @@ -537,12 +537,136 @@ def testMultiStreamOrdering(self): with BZ2File(self.filename) as bz2f: self.assertEqual(bz2f.read(), data1 + data2) + def testOpenFilename(self): + with BZ2File(self.filename, "wb") as f: + f.write(b'content') + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with BZ2File(self.filename, "ab") as f: + f.write(b'appendix') + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with BZ2File(self.filename, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.fileno() + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def testOpenFileWithName(self): + with open(self.filename, 'wb') as raw: + with BZ2File(raw, 'wb') as f: + f.write(b'content') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with open(self.filename, 'ab') as raw: + with BZ2File(raw, 'ab') as f: + f.write(b'appendix') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with open(self.filename, 'rb') as raw: + with BZ2File(raw, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.fileno() + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def testOpenFileWithoutName(self): + bio = BytesIO() + with BZ2File(bio, 'wb') as f: + f.write(b'content') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertRaises(ValueError, f.fileno) + + with BZ2File(bio, 'ab') as f: + f.write(b'appendix') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertRaises(ValueError, f.fileno) + + bio.seek(0) + with BZ2File(bio, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertRaises(io.UnsupportedOperation, f.fileno) + with self.assertRaises(ValueError): + f.fileno() + + def testOpenFileWithIntName(self): + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with open(fd, 'wb') as raw: + with BZ2File(raw, 'wb') as f: + f.write(b'content') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertRaises(ValueError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND) + with open(fd, 'ab') as raw: + with BZ2File(raw, 'ab') as f: + f.write(b'appendix') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertRaises(ValueError, f.fileno) + + fd = os.open(self.filename, os.O_RDONLY) + with open(fd, 'rb') as raw: + with BZ2File(raw, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.fileno(), raw.fileno()) + with self.assertRaises(ValueError): + f.fileno() + def testOpenBytesFilename(self): str_filename = self.filename - try: - bytes_filename = str_filename.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(str_filename) with BZ2File(bytes_filename, "wb") as f: f.write(self.DATA) with BZ2File(bytes_filename, "rb") as f: @@ -552,7 +676,7 @@ def testOpenBytesFilename(self): self.assertEqual(f.read(), self.DATA) def testOpenPathLikeFilename(self): - filename = pathlib.Path(self.filename) + filename = FakePath(self.filename) with BZ2File(filename, "wb") as f: f.write(self.DATA) with BZ2File(filename, "rb") as f: diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index fc82cbfa66ea7a..9f5ee507a8eb85 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -438,7 +438,12 @@ def test_long_asnativebytes(self): if support.verbose: print(f"SIZEOF_SIZE={SZ}\n{MAX_SSIZE=:016X}\n{MAX_USIZE=:016X}") - # These tests check that the requested buffer size is correct + # These tests check that the requested buffer size is correct. + # This matches our current implementation: We only specify that the + # return value is a size *sufficient* to hold the result when queried + # using n_bytes=0. If our implementation changes, feel free to update + # the expectations here -- or loosen them to be range checks. + # (i.e. 0 *could* be stored in 1 byte and 512 in 2) for v, expect in [ (0, SZ), (512, SZ), @@ -453,12 +458,25 @@ def test_long_asnativebytes(self): (-(2**256-1), 33), ]: with self.subTest(f"sizeof-{v:X}"): - buffer = bytearray(1) + buffer = bytearray(b"\x5a") self.assertEqual(expect, asnativebytes(v, buffer, 0, -1), - "PyLong_AsNativeBytes(v, NULL, 0, -1)") + "PyLong_AsNativeBytes(v, , 0, -1)") + self.assertEqual(buffer, b"\x5a", + "buffer overwritten when it should not have been") # Also check via the __index__ path self.assertEqual(expect, asnativebytes(Index(v), buffer, 0, -1), - "PyLong_AsNativeBytes(Index(v), NULL, 0, -1)") + "PyLong_AsNativeBytes(Index(v), , 0, -1)") + self.assertEqual(buffer, b"\x5a", + "buffer overwritten when it should not have been") + + # Test that we populate n=2 bytes but do not overwrite more. + buffer = bytearray(b"\x99"*3) + self.assertEqual(2, asnativebytes(4, buffer, 2, 0), # BE + "PyLong_AsNativeBytes(v, <3 byte buffer>, 2, 0) // BE") + self.assertEqual(buffer, b"\x00\x04\x99") + self.assertEqual(2, asnativebytes(4, buffer, 2, 1), # LE + "PyLong_AsNativeBytes(v, <3 byte buffer>, 2, 1) // LE") + self.assertEqual(buffer, b"\x04\x00\x99") # We request as many bytes as `expect_be` contains, and always check # the result (both big and little endian). We check the return value @@ -510,7 +528,9 @@ def test_long_asnativebytes(self): ]: with self.subTest(f"{v:X}-{len(expect_be)}bytes"): n = len(expect_be) - buffer = bytearray(n) + # Fill the buffer with dummy data to ensure all bytes + # are overwritten. + buffer = bytearray(b"\xa5"*n) expect_le = expect_be[::-1] self.assertEqual(expect_n, asnativebytes(v, buffer, n, 0), diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 1a8ed3441fa855..a0a19225b79433 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -4,10 +4,11 @@ import textwrap import unittest import gc +import os import _testinternalcapi -from test.support import script_helper +from test.support import script_helper, requires_specialization @contextlib.contextmanager @@ -30,18 +31,19 @@ def clear_executors(func): func.__code__ = func.__code__.replace() +@requires_specialization class TestOptimizerAPI(unittest.TestCase): - def test_get_counter_optimizer_dealloc(self): + def test_new_counter_optimizer_dealloc(self): # See gh-108727 def f(): - _testinternalcapi.get_counter_optimizer() + _testinternalcapi.new_counter_optimizer() f() def test_get_set_optimizer(self): old = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.get_counter_optimizer() + opt = _testinternalcapi.new_counter_optimizer() try: _testinternalcapi.set_optimizer(opt) self.assertEqual(_testinternalcapi.get_optimizer(), opt) @@ -62,7 +64,7 @@ def loop(): loop = ns['loop'] for repeat in range(5): - opt = _testinternalcapi.get_counter_optimizer() + opt = _testinternalcapi.new_counter_optimizer() with temporary_optimizer(opt): self.assertEqual(opt.get_count(), 0) with clear_executors(loop): @@ -90,7 +92,7 @@ def long_loop(): """), ns, ns) long_loop = ns['long_loop'] - opt = _testinternalcapi.get_counter_optimizer() + opt = _testinternalcapi.new_counter_optimizer() with temporary_optimizer(opt): self.assertEqual(opt.get_count(), 0) long_loop() @@ -102,7 +104,7 @@ def testfunc(x): while i < x: i += 1 - opt = _testinternalcapi.get_counter_optimizer() + opt = _testinternalcapi.new_counter_optimizer() with temporary_optimizer(opt): testfunc(1000) code, replace_code = testfunc.__code__, testfunc.__code__.replace() @@ -123,11 +125,21 @@ def get_first_executor(func): return None +def iter_opnames(ex): + for item in ex: + yield item[0] + + +def get_opnames(ex): + return set(iter_opnames(ex)) + + +@requires_specialization class TestExecutorInvalidation(unittest.TestCase): def setUp(self): self.old = _testinternalcapi.get_optimizer() - self.opt = _testinternalcapi.get_counter_optimizer() + self.opt = _testinternalcapi.new_counter_optimizer() _testinternalcapi.set_optimizer(self.opt) def tearDown(self): @@ -176,7 +188,7 @@ def f(): pass """), ns, ns) f = ns['f'] - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): f() exe = get_first_executor(f) @@ -189,7 +201,7 @@ def test_sys__clear_internal_caches(self): def f(): for _ in range(1000): pass - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): f() exe = get_first_executor(f) @@ -200,6 +212,9 @@ def f(): exe = get_first_executor(f) self.assertIsNone(exe) + +@requires_specialization +@unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUops(unittest.TestCase): def test_basic_loop(self): @@ -208,15 +223,15 @@ def testfunc(x): while i < x: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(1000) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_SET_IP", uops) - self.assertIn("_LOAD_FAST", uops) + self.assertIn("_LOAD_FAST_0", uops) def test_extended_arg(self): "Check EXTENDED_ARG handling in superblock creation" @@ -255,7 +270,7 @@ def many_vars(): """), ns, ns) many_vars = ns["many_vars"] - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): ex = get_first_executor(many_vars) self.assertIsNone(ex) @@ -263,7 +278,8 @@ def many_vars(): ex = get_first_executor(many_vars) self.assertIsNotNone(ex) - self.assertIn(("_LOAD_FAST", 259, 0), list(ex)) + self.assertTrue(any((opcode, oparg, operand) == ("_LOAD_FAST", 259, 0) + for opcode, oparg, _, operand in list(ex))) def test_unspecialized_unpack(self): # An example of an unspecialized opcode @@ -277,14 +293,14 @@ def testfunc(x): while i < x: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_UNPACK_SEQUENCE", uops) def test_pop_jump_if_false(self): @@ -293,13 +309,13 @@ def testfunc(n): while i < n: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_IS_TRUE_POP", uops) def test_pop_jump_if_none(self): @@ -308,13 +324,13 @@ def testfunc(a): if x is None: x = 0 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(range(20)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_IS_NOT_NONE_POP", uops) def test_pop_jump_if_not_none(self): @@ -324,13 +340,13 @@ def testfunc(a): if x is not None: x = 0 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(range(20)) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_IS_NONE_POP", uops) def test_pop_jump_if_true(self): @@ -339,13 +355,13 @@ def testfunc(n): while not i >= n: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_IS_FALSE_POP", uops) def test_jump_backward(self): @@ -354,13 +370,13 @@ def testfunc(n): while i < n: i += 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_JUMP_TO_TOP", uops) def test_jump_forward(self): @@ -374,13 +390,13 @@ def testfunc(n): a += 1 return a - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) # Since there is no JUMP_FORWARD instruction, # look for indirect evidence: the += operator self.assertIn("_BINARY_OP_ADD_INT", uops) @@ -392,7 +408,7 @@ def testfunc(n): total += i return total - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): total = testfunc(20) self.assertEqual(total, 190) @@ -401,7 +417,7 @@ def testfunc(n): self.assertIsNotNone(ex) # for i, (opname, oparg) in enumerate(ex): # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_NOT_EXHAUSTED_RANGE", uops) # Verification that the jump goes past END_FOR # is done by manual inspection of the output @@ -413,7 +429,7 @@ def testfunc(a): total += i return total - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): a = list(range(20)) total = testfunc(a) @@ -423,7 +439,7 @@ def testfunc(a): self.assertIsNotNone(ex) # for i, (opname, oparg) in enumerate(ex): # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_NOT_EXHAUSTED_LIST", uops) # Verification that the jump goes past END_FOR # is done by manual inspection of the output @@ -435,7 +451,7 @@ def testfunc(a): total += i return total - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): a = tuple(range(20)) total = testfunc(a) @@ -445,7 +461,7 @@ def testfunc(a): self.assertIsNotNone(ex) # for i, (opname, oparg) in enumerate(ex): # print(f"{i:4d}: {opname:<20s} {oparg:3d}") - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_NOT_EXHAUSTED_TUPLE", uops) # Verification that the jump goes past END_FOR # is done by manual inspection of the output @@ -455,7 +471,7 @@ def testfunc(it): for x in it: pass - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): a = [1, 2, 3] it = iter(a) @@ -471,13 +487,13 @@ def dummy(x): for i in range(n): dummy(i) - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_PUSH_FRAME", uops) self.assertIn("_BINARY_OP_ADD_INT", uops) @@ -489,13 +505,13 @@ def testfunc(n): else: i = 1 - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): testfunc(20) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_IS_FALSE_POP", uops) def test_for_iter_tier_two(self): @@ -517,7 +533,7 @@ def testfunc(n, m): x += 1000*i + j return x - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): x = testfunc(10, 10) @@ -525,7 +541,7 @@ def testfunc(n, m): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_FOR_ITER_TIER_TWO", uops) def test_confidence_score(self): @@ -546,40 +562,47 @@ def testfunc(n): bits += 1 return bits - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): x = testfunc(20) self.assertEqual(x, 40) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) - ops = [opname for opname, _, _ in ex] - count = ops.count("_GUARD_IS_TRUE_POP") - # Because Each 'if' halves the score, the second branch is - # too much already. - self.assertEqual(count, 1) + ops = list(iter_opnames(ex)) + #Since branch is 50/50 the trace could go either way. + count = ops.count("_GUARD_IS_TRUE_POP") + ops.count("_GUARD_IS_FALSE_POP") + self.assertLessEqual(count, 2) + +@requires_specialization +@unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUopsOptimization(unittest.TestCase): + def _run_with_optimizer(self, testfunc, arg): + res = None + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + res = testfunc(arg) + + ex = get_first_executor(testfunc) + return res, ex + + def test_int_type_propagation(self): def testfunc(loops): num = 0 - while num < loops: + for i in range(loops): x = num + num a = x + 1 num += 1 return a - opt = _testinternalcapi.get_uop_optimizer() - res = None - with temporary_optimizer(opt): - res = testfunc(32) - - ex = get_first_executor(testfunc) + res, ex = self._run_with_optimizer(testfunc, 32) self.assertIsNotNone(ex) self.assertEqual(res, 63) - binop_count = [opname for opname, _, _ in ex if opname == "_BINARY_OP_ADD_INT"] - guard_both_int_count = [opname for opname, _, _ in ex if opname == "_GUARD_BOTH_INT"] + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] self.assertGreaterEqual(len(binop_count), 3) self.assertLessEqual(len(guard_both_int_count), 1) @@ -588,13 +611,13 @@ def double(x): return x + x def testfunc(loops): num = 0 - while num < loops: + for i in range(loops): x = num + num a = double(x) num += 1 return a - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() res = None with temporary_optimizer(opt): res = testfunc(32) @@ -602,8 +625,8 @@ def testfunc(loops): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) self.assertEqual(res, 124) - binop_count = [opname for opname, _, _ in ex if opname == "_BINARY_OP_ADD_INT"] - guard_both_int_count = [opname for opname, _, _ in ex if opname == "_GUARD_BOTH_INT"] + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] self.assertGreaterEqual(len(binop_count), 3) self.assertLessEqual(len(guard_both_int_count), 1) @@ -612,13 +635,13 @@ def double(x): return x + x def testfunc(loops): num = 0 - while num < loops: + for i in range(loops): a = double(num) x = a + a num += 1 return x - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() res = None with temporary_optimizer(opt): res = testfunc(32) @@ -626,8 +649,8 @@ def testfunc(loops): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) self.assertEqual(res, 124) - binop_count = [opname for opname, _, _ in ex if opname == "_BINARY_OP_ADD_INT"] - guard_both_int_count = [opname for opname, _, _ in ex if opname == "_GUARD_BOTH_INT"] + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] + guard_both_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] self.assertGreaterEqual(len(binop_count), 3) self.assertLessEqual(len(guard_both_int_count), 1) @@ -642,14 +665,9 @@ def testfunc(loops): num += 1 return a - opt = _testinternalcapi.get_uop_optimizer() - res = None - with temporary_optimizer(opt): - res = testfunc(64) - - ex = get_first_executor(testfunc) + res, ex = self._run_with_optimizer(testfunc, 64) self.assertIsNotNone(ex) - binop_count = [opname for opname, _, _ in ex if opname == "_BINARY_OP_ADD_INT"] + binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] self.assertGreaterEqual(len(binop_count), 3) def test_call_py_exact_args(self): @@ -659,13 +677,9 @@ def dummy(x): for i in range(n): dummy(i) - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(32) - - ex = get_first_executor(testfunc) + res, ex = self._run_with_optimizer(testfunc, 32) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_PUSH_FRAME", uops) self.assertIn("_BINARY_OP_ADD_INT", uops) self.assertNotIn("_CHECK_PEP_523", uops) @@ -677,14 +691,10 @@ def testfunc(n): x = i + i return x - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - res = testfunc(32) - - ex = get_first_executor(testfunc) + res, ex = self._run_with_optimizer(testfunc, 32) self.assertEqual(res, 62) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertNotIn("_GUARD_BOTH_INT", uops) def test_int_value_numbering(self): @@ -699,16 +709,12 @@ def testfunc(n): res = x + z + a + b return res - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - res = testfunc(32) - - ex = get_first_executor(testfunc) + res, ex = self._run_with_optimizer(testfunc, 32) self.assertEqual(res, 4) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertIn("_GUARD_BOTH_INT", uops) - guard_count = [opname for opname, _, _ in ex if opname == "_GUARD_BOTH_INT"] + guard_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] self.assertEqual(len(guard_count), 1) def test_comprehension(self): @@ -716,13 +722,10 @@ def testfunc(n): for _ in range(n): return [i for i in range(n)] - opt = _testinternalcapi.get_uop_optimizer() - with temporary_optimizer(opt): - testfunc(32) - - ex = get_first_executor(testfunc) + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, list(range(32))) self.assertIsNotNone(ex) - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) self.assertNotIn("_BINARY_OP_ADD_INT", uops) def test_call_py_exact_args_disappearing(self): @@ -733,7 +736,7 @@ def testfunc(n): for i in range(n): dummy(i) - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() # Trigger specialization testfunc(8) with temporary_optimizer(opt): @@ -768,23 +771,187 @@ def get_first_executor(func): pass return None + def get_opnames(ex): + return {item[0] for item in ex} + def testfunc(n): for i in range(n): x = range(i) return x - opt = _testinternalcapi.get_uop_optimizer() + opt = _testinternalcapi.new_uop_optimizer() _testinternalcapi.set_optimizer(opt) testfunc(64) ex = get_first_executor(testfunc) assert ex is not None - uops = {opname for opname, _, _ in ex} + uops = get_opnames(ex) assert "_LOAD_GLOBAL_BUILTINS" not in uops assert "_LOAD_CONST_INLINE_BORROW_WITH_NULL" in uops """)) self.assertEqual(result[0].rc, 0, result) + def test_float_add_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a + 0.25 + a = a + 0.25 + a = a + 0.25 + a = a + 0.25 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertAlmostEqual(res, 33.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_ADD_FLOAT", uops) + + def test_float_subtract_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a - 0.25 + a = a - 0.25 + a = a - 0.25 + a = a - 0.25 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertAlmostEqual(res, -31.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_SUBTRACT_FLOAT", uops) + + def test_float_multiply_constant_propagation(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + a = a * 1.0 + a = a * 1.0 + a = a * 1.0 + a = a * 1.0 + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertAlmostEqual(res, 1.0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + # TODO gh-115506: this assertion may change after propagating constants. + # We'll also need to verify that propagation actually occurs. + self.assertIn("_BINARY_OP_MULTIPLY_FLOAT", uops) + + def test_add_unicode_propagation(self): + def testfunc(n): + a = "" + for _ in range(n): + a + a + a + a + a + a + a + a + return a + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertEqual(res, "") + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_unicode_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_UNICODE"] + self.assertLessEqual(len(guard_both_unicode_count), 1) + self.assertIn("_BINARY_OP_ADD_UNICODE", uops) + + def test_compare_op_type_propagation_float(self): + def testfunc(n): + a = 1.0 + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_FLOAT"] + self.assertLessEqual(len(guard_both_float_count), 1) + self.assertIn("_COMPARE_OP_FLOAT", uops) + + def test_compare_op_type_propagation_int(self): + def testfunc(n): + a = 1 + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertLessEqual(len(guard_both_float_count), 1) + self.assertIn("_COMPARE_OP_INT", uops) + + def test_compare_op_type_propagation_unicode(self): + def testfunc(n): + a = "" + for _ in range(n): + x = a == a + x = a == a + x = a == a + x = a == a + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_both_float_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_UNICODE"] + self.assertLessEqual(len(guard_both_float_count), 1) + self.assertIn("_COMPARE_OP_STR", uops) + + def test_type_inconsistency(self): + ns = {} + src = textwrap.dedent(""" + def testfunc(n): + for i in range(n): + x = _test_global + _test_global + """) + exec(src, ns, ns) + testfunc = ns['testfunc'] + ns['_test_global'] = 0 + _, ex = self._run_with_optimizer(testfunc, 16) + self.assertIsNone(ex) + ns['_test_global'] = 1 + _, ex = self._run_with_optimizer(testfunc, 16) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_BOTH_INT", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) + # Try again, but between the runs, set the global to a float. + # This should result in no executor the second time. + ns = {} + exec(src, ns, ns) + testfunc = ns['testfunc'] + ns['_test_global'] = 0 + _, ex = self._run_with_optimizer(testfunc, 16) + self.assertIsNone(ex) + ns['_test_global'] = 3.14 + _, ex = self._run_with_optimizer(testfunc, 16) + self.assertIsNone(ex) if __name__ == "__main__": diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index f5e9b11ad1cc8a..02a293e04b2182 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2146,6 +2146,14 @@ class Foo "" "" expected_error = err_template.format(invalid_kind) self.expect_failure(block, expected_error, lineno=3) + def test_init_cannot_define_a_return_type(self): + block = """ + class Foo "" "" + Foo.__init__ -> long + """ + expected_error = "__init__ methods cannot define a return type" + self.expect_failure(block, expected_error, lineno=1) + def test_invalid_getset(self): annotations = ["@getter", "@setter"] for annotation in annotations: @@ -2167,7 +2175,7 @@ class Foo "" "" obj: int / """ - expected_error = f"{annotation} method cannot define parameters" + expected_error = f"{annotation} methods cannot define parameters" self.expect_failure(block, expected_error) def test_setter_docstring(self): @@ -2647,7 +2655,6 @@ def test_cli_converters(self): bool() double() float() - init() int() long() Py_ssize_t() @@ -3937,7 +3944,7 @@ def test_Function_and_Parameter_reprs(self): cls=None, c_basename=None, full_name='foofoo', - return_converter=clinic.init_return_converter(), + return_converter=clinic.int_return_converter(), kind=clinic.FunctionKind.METHOD_INIT, coexist=False ) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 4d8647be6a14f1..d3e69bfedccd07 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -527,11 +527,11 @@ def test_compile_ast(self): self.assertRaises(TypeError, compile, co1, '', 'eval') # raise exception when node type is no start node - self.assertRaises(TypeError, compile, _ast.If(), '', 'exec') + self.assertRaises(TypeError, compile, _ast.If(test=_ast.Name(id='x', ctx=_ast.Load())), '', 'exec') # raise exception when node has invalid children ast = _ast.Module() - ast.body = [_ast.BoolOp()] + ast.body = [_ast.BoolOp(op=_ast.Or())] self.assertRaises(TypeError, compile, ast, '', 'exec') def test_compile_invalid_typealias(self): @@ -749,7 +749,7 @@ def f(): return "unused" self.assertEqual(f.__code__.co_consts, - ("docstring", "used")) + (f.__doc__, "used")) @support.cpython_only def test_remove_unused_consts_no_docstring(self): @@ -794,7 +794,7 @@ def test_strip_unused_None(self): def f1(): "docstring" return 42 - self.assertEqual(f1.__code__.co_consts, ("docstring", 42)) + self.assertEqual(f1.__code__.co_consts, (f1.__doc__, 42)) # This is a regression test for a CPython specific peephole optimizer # implementation bug present in a few releases. It's assertion verifies @@ -1047,6 +1047,8 @@ def no_code2(): for func in (no_code1, no_code2): with self.subTest(func=func): + if func is no_code1 and no_code1.__doc__ is None: + continue code = func.__code__ [(start, end, line)] = code.co_lines() self.assertEqual(start, 0) @@ -1524,6 +1526,7 @@ def test_multiline_boolean_expression(self): self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', line=4, end_line=4, column=8, end_column=13, occurrence=2) + @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") def test_multiline_assert(self): snippet = textwrap.dedent("""\ assert (a > 0 and diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 0ec6013dc11e3e..14c2af19e3eb28 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -977,7 +977,7 @@ class CommandLineTestsNoSourceEpoch(CommandLineTestsBase, -@unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') +@os_helper.skip_unless_hardlink class HardlinkDedupTestsBase: # Test hardlink_dupes parameter of compileall.compile_dir() diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 5696433e529d0a..ab9f04dd63af20 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -1,3 +1,6 @@ +import dis +import io +import textwrap import types from test.support.bytecode_helper import AssemblerTestCase @@ -22,11 +25,13 @@ def complete_metadata(self, metadata, filename="myfile.py"): metadata.setdefault('filename', filename) return metadata - def assemble_test(self, insts, metadata, expected): + def insts_to_code_object(self, insts, metadata): metadata = self.complete_metadata(metadata) insts = self.complete_insts_info(insts) + return self.get_code_object(metadata['filename'], insts, metadata) - co = self.get_code_object(metadata['filename'], insts, metadata) + def assemble_test(self, insts, metadata, expected): + co = self.insts_to_code_object(insts, metadata) self.assertIsInstance(co, types.CodeType) expected_metadata = {} @@ -108,3 +113,35 @@ def inner(): expected = {(0,): 0, (1,): 1, (2,): 0, (120,): 0, (121,): 1} self.assemble_test(instructions, metadata, expected) + + + def test_exception_table(self): + metadata = { + 'filename' : 'exc.py', + 'name' : 'exc', + 'consts' : {2 : 0}, + } + + # code for "try: pass\n except: pass" + insts = [ + ('RESUME', 0), + ('SETUP_FINALLY', 3), + ('RETURN_CONST', 0), + ('SETUP_CLEANUP', 8), + ('PUSH_EXC_INFO', 0), + ('POP_TOP', 0), + ('POP_EXCEPT', 0), + ('RETURN_CONST', 0), + ('COPY', 3), + ('POP_EXCEPT', 0), + ('RERAISE', 1), + ] + co = self.insts_to_code_object(insts, metadata) + output = io.StringIO() + dis.dis(co, file=output) + exc_table = textwrap.dedent(""" + ExceptionTable: + L1 to L2 -> L2 [0] + L2 to L3 -> L3 [1] lasti + """) + self.assertTrue(output.getvalue().endswith(exc_table)) diff --git a/Lib/test/test_concurrent_futures/__init__.py b/Lib/test/test_concurrent_futures/__init__.py index 430fa93aa456a2..b38bd38d338f0e 100644 --- a/Lib/test/test_concurrent_futures/__init__.py +++ b/Lib/test/test_concurrent_futures/__init__.py @@ -1,10 +1,12 @@ import os.path import unittest from test import support -from test.support import import_helper +from test.support import threading_helper + + +# Adjust if we ever have a platform with processes but not threads. +threading_helper.requires_working_threading(module=True) -# Skip tests if _multiprocessing wasn't built. -import_helper.import_module('_multiprocessing') if support.check_sanitizer(address=True, memory=True): # gh-90791: Skip the test because it is too slow when Python is built diff --git a/Lib/test/test_concurrent_futures/test_init.py b/Lib/test/test_concurrent_futures/test_init.py index ce01e0ff0f287a..113a4d1c54be03 100644 --- a/Lib/test/test_concurrent_futures/test_init.py +++ b/Lib/test/test_concurrent_futures/test_init.py @@ -3,7 +3,10 @@ import queue import time import unittest +import sys from concurrent.futures._base import BrokenExecutor +from concurrent.futures.process import _check_system_limits + from logging.handlers import QueueHandler from test import support @@ -109,6 +112,36 @@ def _assert_logged(self, msg): create_executor_tests(globals(), FailingInitializerMixin) +@unittest.skipIf(sys.platform == "win32", "Resource Tracker doesn't run on Windows") +class FailingInitializerResourcesTest(unittest.TestCase): + """ + Source: https://github.com/python/cpython/issues/104090 + """ + + def _test(self, test_class): + try: + _check_system_limits() + except NotImplementedError: + self.skipTest("ProcessPoolExecutor unavailable on this system") + + runner = unittest.TextTestRunner() + runner.run(test_class('test_initializer')) + + # GH-104090: + # Stop resource tracker manually now, so we can verify there are not leaked resources by checking + # the process exit code + from multiprocessing.resource_tracker import _resource_tracker + _resource_tracker._stop() + + self.assertEqual(_resource_tracker._exitcode, 0) + + def test_spawn(self): + self._test(ProcessPoolSpawnFailingInitializerTest) + + def test_forkserver(self): + self._test(ProcessPoolForkserverFailingInitializerTest) + + def setUpModule(): setup_module() diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index dc48bec796b87f..3e855031913042 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -136,6 +136,12 @@ def strip_mixin(name): def setup_module(): - unittest.addModuleCleanup(multiprocessing.util._cleanup_tests) + try: + _check_system_limits() + except NotImplementedError: + pass + else: + unittest.addModuleCleanup(multiprocessing.util._cleanup_tests) + thread_info = threading_helper.threading_setup() unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 21a4cb586ff665..d74ab7e016f78c 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -64,8 +64,7 @@ def _test_arg_valid(self, ctor, arg): ctor(arg, delimiter='\t', skipinitialspace=True) ctor(arg, escapechar='\t', skipinitialspace=True) ctor(arg, quotechar='\t', skipinitialspace=True) - self.assertRaises(ValueError, ctor, arg, - delimiter=' ', skipinitialspace=True) + ctor(arg, delimiter=' ', skipinitialspace=True) self.assertRaises(ValueError, ctor, arg, escapechar=' ', skipinitialspace=True) self.assertRaises(ValueError, ctor, arg, @@ -192,9 +191,6 @@ def _write_error_test(self, exc, fields, **kwargs): def test_write_arg_valid(self): self._write_error_test(csv.Error, None) - self._write_test((), '') - self._write_test([None], '""') - self._write_error_test(csv.Error, [None], quoting = csv.QUOTE_NONE) # Check that exceptions are passed up the chain self._write_error_test(OSError, BadIterable()) class BadList: @@ -208,7 +204,6 @@ class BadItem: def __str__(self): raise OSError self._write_error_test(OSError, [BadItem()]) - def test_write_bigfield(self): # This exercises the buffer realloc functionality bigstring = 'X' * 50000 @@ -270,9 +265,11 @@ def test_write_lineterminator(self): writer = csv.writer(sio, lineterminator=lineterminator) writer.writerow(['a', 'b']) writer.writerow([1, 2]) + writer.writerow(['\r', '\n']) self.assertEqual(sio.getvalue(), f'a,b{lineterminator}' - f'1,2{lineterminator}') + f'1,2{lineterminator}' + f'"\r","\n"{lineterminator}') def test_write_iterable(self): self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"') @@ -315,6 +312,49 @@ def test_writerows_with_none(self): fileobj.seek(0) self.assertEqual(fileobj.read(), 'a\r\n""\r\n') + + def test_write_empty_fields(self): + self._write_test((), '') + self._write_test([''], '""') + self._write_error_test(csv.Error, [''], quoting=csv.QUOTE_NONE) + self._write_test([''], '""', quoting=csv.QUOTE_STRINGS) + self._write_test([''], '""', quoting=csv.QUOTE_NOTNULL) + self._write_test([None], '""') + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_NONE) + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_STRINGS) + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_NOTNULL) + self._write_test(['', ''], ',') + self._write_test([None, None], ',') + + def test_write_empty_fields_space_delimiter(self): + self._write_test([''], '""', delimiter=' ', skipinitialspace=False) + self._write_test([''], '""', delimiter=' ', skipinitialspace=True) + self._write_test([None], '""', delimiter=' ', skipinitialspace=False) + self._write_test([None], '""', delimiter=' ', skipinitialspace=True) + + self._write_test(['', ''], ' ', delimiter=' ', skipinitialspace=False) + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=True) + self._write_test([None, None], ' ', delimiter=' ', skipinitialspace=False) + self._write_test([None, None], '"" ""', delimiter=' ', skipinitialspace=True) + + self._write_test(['', ''], ' ', delimiter=' ', skipinitialspace=False, + quoting=csv.QUOTE_NONE) + self._write_error_test(csv.Error, ['', ''], + delimiter=' ', skipinitialspace=True, + quoting=csv.QUOTE_NONE) + for quoting in csv.QUOTE_STRINGS, csv.QUOTE_NOTNULL: + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=False, + quoting=quoting) + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=True, + quoting=quoting) + + for quoting in csv.QUOTE_NONE, csv.QUOTE_STRINGS, csv.QUOTE_NOTNULL: + self._write_test([None, None], ' ', delimiter=' ', skipinitialspace=False, + quoting=quoting) + self._write_error_test(csv.Error, [None, None], + delimiter=' ', skipinitialspace=True, + quoting=quoting) + def test_writerows_errors(self): with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj) @@ -429,6 +469,14 @@ def test_read_skipinitialspace(self): [[None, None, None]], skipinitialspace=True, quoting=csv.QUOTE_STRINGS) + def test_read_space_delimiter(self): + self._read_test(['a b', ' a ', ' ', ''], + [['a', '', '', 'b'], ['', '', 'a', '', ''], ['', '', ''], []], + delimiter=' ', skipinitialspace=False) + self._read_test(['a b', ' a ', ' ', ''], + [['a', 'b'], ['a', ''], [''], []], + delimiter=' ', skipinitialspace=True) + def test_read_bigfield(self): # This exercises the buffer realloc functionality and field size # limits. @@ -461,22 +509,44 @@ def test_read_linenum(self): self.assertEqual(r.line_num, 3) def test_roundtrip_quoteed_newlines(self): - with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: - writer = csv.writer(fileobj) - rows = [['a\nb','b'],['c','x\r\nd']] - writer.writerows(rows) - fileobj.seek(0) - for i, row in enumerate(csv.reader(fileobj)): - self.assertEqual(row, rows[i]) + rows = [ + ['\na', 'b\nc', 'd\n'], + ['\re', 'f\rg', 'h\r'], + ['\r\ni', 'j\r\nk', 'l\r\n'], + ['\n\rm', 'n\n\ro', 'p\n\r'], + ['\r\rq', 'r\r\rs', 't\r\r'], + ['\n\nu', 'v\n\nw', 'x\n\n'], + ] + for lineterminator in '\r\n', '\n', '\r': + with self.subTest(lineterminator=lineterminator): + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: + writer = csv.writer(fileobj, lineterminator=lineterminator) + writer.writerows(rows) + fileobj.seek(0) + for i, row in enumerate(csv.reader(fileobj)): + self.assertEqual(row, rows[i]) def test_roundtrip_escaped_unquoted_newlines(self): - with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: - writer = csv.writer(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\") - rows = [['a\nb','b'],['c','x\r\nd']] - writer.writerows(rows) - fileobj.seek(0) - for i, row in enumerate(csv.reader(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\")): - self.assertEqual(row,rows[i]) + rows = [ + ['\na', 'b\nc', 'd\n'], + ['\re', 'f\rg', 'h\r'], + ['\r\ni', 'j\r\nk', 'l\r\n'], + ['\n\rm', 'n\n\ro', 'p\n\r'], + ['\r\rq', 'r\r\rs', 't\r\r'], + ['\n\nu', 'v\n\nw', 'x\n\n'], + ] + for lineterminator in '\r\n', '\n', '\r': + with self.subTest(lineterminator=lineterminator): + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: + writer = csv.writer(fileobj, lineterminator=lineterminator, + quoting=csv.QUOTE_NONE, escapechar="\\") + writer.writerows(rows) + fileobj.seek(0) + for i, row in enumerate(csv.reader(fileobj, + quoting=csv.QUOTE_NONE, + escapechar="\\")): + self.assertEqual(row, rows[i]) + class TestDialectRegistry(unittest.TestCase): def test_registry_badargs(self): @@ -555,10 +625,10 @@ class space(csv.excel): escapechar = "\\" with TemporaryFile("w+", encoding="utf-8") as fileobj: - fileobj.write("abc def\nc1ccccc1 benzene\n") + fileobj.write("abc def\nc1ccccc1 benzene\n") fileobj.seek(0) reader = csv.reader(fileobj, dialect=space()) - self.assertEqual(next(reader), ["abc", "def"]) + self.assertEqual(next(reader), ["abc", "", "", "def"]) self.assertEqual(next(reader), ["c1ccccc1", "benzene"]) def compare_dialect_123(self, expected, *writeargs, **kwwriteargs): @@ -1164,8 +1234,9 @@ class mydialect(csv.Dialect): self.assertRaises(csv.Error, create_invalid, field_name, 5) self.assertRaises(ValueError, create_invalid, field_name, "\n") self.assertRaises(ValueError, create_invalid, field_name, "\r") - self.assertRaises(ValueError, create_invalid, field_name, " ", - skipinitialspace=True) + if field_name != "delimiter": + self.assertRaises(ValueError, create_invalid, field_name, " ", + skipinitialspace=True) class TestSniffer(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py index 19f4158c0ac846..64f92ffdca6a3f 100644 --- a/Lib/test/test_ctypes/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -148,9 +148,10 @@ def callback(a, b): print(f"a={a}, b={b}, c={c}") return c dll = cdll[_ctypes_test.__file__] - # With no fix for i38748, the next line will raise OSError and cause the test to fail. - self.assertEqual(dll._test_i38748_runCallback(callback, 5, 10), 15) - + with support.captured_stdout() as out: + # With no fix for i38748, the next line will raise OSError and cause the test to fail. + self.assertEqual(dll._test_i38748_runCallback(callback, 5, 10), 15) + self.assertEqual(out.getvalue(), "a=5, b=10, c=15\n") if hasattr(ctypes, 'WINFUNCTYPE'): class StdcallCallbacks(Callbacks): diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index 59d7f51935f3cd..b218e9e7720c79 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -59,11 +59,15 @@ def test_load_version(self): self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) def test_find(self): + found = False for name in ("c", "m"): lib = find_library(name) if lib: + found = True cdll.LoadLibrary(lib) CDLL(lib) + if not found: + self.skipTest("Could not find c and m libraries") @unittest.skipUnless(os.name == "nt", 'test specific to Windows') diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index a5917da346dded..a93cb509b651c5 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1986,6 +1986,22 @@ def f(opcode, oparg, offset, *init_args): self.assertEqual(f(opcode.opmap["BINARY_OP"], 3, *args), (3, '<<')) self.assertEqual(f(opcode.opmap["CALL_INTRINSIC_1"], 2, *args), (2, 'INTRINSIC_IMPORT_STAR')) + def test_custom_arg_resolver(self): + class MyArgResolver(dis.ArgResolver): + def offset_from_jump_arg(self, op, arg, offset): + return arg + 1 + + def get_label_for_offset(self, offset): + return 2 * offset + + def f(opcode, oparg, offset, *init_args): + arg_resolver = MyArgResolver(*init_args) + return arg_resolver.get_argval_argrepr(opcode, oparg, offset) + offset = 42 + self.assertEqual(f(opcode.opmap["JUMP_BACKWARD"], 1, offset), (2, 'to L4')) + self.assertEqual(f(opcode.opmap["SETUP_FINALLY"], 2, offset), (3, 'to L6')) + + def get_instructions(self, code): return dis._get_instructions_bytes(code) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index bdb0e55f21069f..f7e80749c456f8 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -2985,6 +2985,11 @@ def test_address_list_with_unicode_names_in_quotes(self): '=?utf-8?q?H=C3=BCbsch?= Kaktus ,\n' ' =?utf-8?q?bei=C3=9Ft_bei=C3=9Ft?= \n') + def test_address_list_with_list_separator_after_fold(self): + to = '0123456789' * 8 + '@foo, ä ' + self._test(parser.get_address_list(to)[0], + '0123456789' * 8 + '@foo,\n =?utf-8?q?=C3=A4?= \n') + # XXX Need tests with comments on various sides of a unicode token, # and with unicode tokens in the comments. Spaces inside the quotes # currently don't do the right thing. diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 61060f3dc29fd4..27f8bbaf952afc 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -447,7 +447,7 @@ def spam(cls): def test_bad_new_super(self): with self.assertRaisesRegex( TypeError, - 'has no members defined', + 'do not use .super...__new__;', ): class BadSuper(self.enum_type): def __new__(cls, value): @@ -1048,6 +1048,22 @@ class TestPlainEnumFunction(_EnumTests, _PlainOutputTests, unittest.TestCase): class TestPlainFlagClass(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): enum_type = Flag + def test_none_member(self): + class FlagWithNoneMember(Flag): + A = 1 + E = None + + self.assertEqual(FlagWithNoneMember.A.value, 1) + self.assertIs(FlagWithNoneMember.E.value, None) + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with |"): + FlagWithNoneMember.A | FlagWithNoneMember.E + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with &"): + FlagWithNoneMember.E & FlagWithNoneMember.A + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be combined with other flags with \^"): + FlagWithNoneMember.A ^ FlagWithNoneMember.E + with self.assertRaisesRegex(TypeError, r"'FlagWithNoneMember.E' cannot be inverted"): + ~FlagWithNoneMember.E + class TestPlainFlagFunction(_EnumTests, _PlainOutputTests, _FlagTests, unittest.TestCase): enum_type = Flag @@ -3393,6 +3409,17 @@ def __new__(cls, int_value, *value_aliases): self.assertIs(Types(2), Types.NetList) self.assertIs(Types('nl'), Types.NetList) + def test_no_members(self): + with self.assertRaisesRegex( + TypeError, + 'has no members', + ): + Enum(7) + with self.assertRaisesRegex( + TypeError, + 'has no members', + ): + Flag(7) class TestOrder(unittest.TestCase): "test usage of the `_order_` attribute" diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index c7e76414ff0715..c5eff8ad8ccca1 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -301,6 +301,7 @@ def baz(): { 6 0="""''', 5, 13) + check('b"fooжжж"'.encode(), 1, 1, 1, 10) # Errors thrown by symtable.c check('x = [(yield i) for i in range(3)]', 1, 7) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py new file mode 100644 index 00000000000000..86c07de507e39c --- /dev/null +++ b/Lib/test/test_external_inspection.py @@ -0,0 +1,84 @@ +import unittest +import os +import textwrap +import importlib +import sys +from test.support import os_helper, SHORT_TIMEOUT +from test.support.script_helper import make_script + +import subprocess + +PROCESS_VM_READV_SUPPORTED = False + +try: + from _testexternalinspection import PROCESS_VM_READV_SUPPORTED + from _testexternalinspection import get_stack_trace +except ImportError: + unittest.skip("Test only runs when _testexternalinspection is available") + +def _make_test_script(script_dir, script_basename, source): + to_return = make_script(script_dir, script_basename, source) + importlib.invalidate_caches() + return to_return + +class TestGetStackTrace(unittest.TestCase): + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import time, sys, os + def bar(): + for x in range(100): + if x == 50: + baz() + def baz(): + foo() + + def foo(): + fifo = sys.argv[1] + with open(sys.argv[1], "w") as fifo: + fifo.write("ready") + time.sleep(1000) + + bar() + """) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_stack_trace(p.pid) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + + expected_stack_trace = [ + 'foo', + 'baz', + 'bar', + '' + ] + self.assertEqual(stack_trace, expected_stack_trace) + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_self_trace(self): + stack_trace = get_stack_trace(os.getpid()) + self.assertEqual(stack_trace[0], "test_self_trace") + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 6d734d052454d3..8b4ed4a9e3a4fe 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -129,7 +129,8 @@ def test_fcntl_bad_file_overflow(self): fcntl.fcntl(BadFile(INT_MIN - 1), fcntl.F_SETFL, os.O_NONBLOCK) @unittest.skipIf( - platform.machine().startswith('arm') and platform.system() == 'Linux', + any(platform.machine().startswith(name) for name in {"arm", "aarch"}) + and platform.system() in {"Linux", "Android"}, "ARM Linux returns EINVAL for F_NOTIFY DN_MULTISHOT") def test_fcntl_64_bit(self): # Issue #1309352: fcntl shouldn't fail when the third arg fits in a diff --git a/Lib/test/test_filecmp.py b/Lib/test/test_filecmp.py index 9b5ac12bccc58f..b5df71678264a8 100644 --- a/Lib/test/test_filecmp.py +++ b/Lib/test/test_filecmp.py @@ -8,11 +8,24 @@ from test.support import os_helper +def _create_file_shallow_equal(template_path, new_path): + """create a file with the same size and mtime but different content.""" + shutil.copy2(template_path, new_path) + with open(new_path, 'r+b') as f: + next_char = bytearray(f.read(1)) + next_char[0] = (next_char[0] + 1) % 256 + f.seek(0) + f.write(next_char) + shutil.copystat(template_path, new_path) + assert os.stat(new_path).st_size == os.stat(template_path).st_size + assert os.stat(new_path).st_mtime == os.stat(template_path).st_mtime + class FileCompareTestCase(unittest.TestCase): def setUp(self): self.name = os_helper.TESTFN self.name_same = os_helper.TESTFN + '-same' self.name_diff = os_helper.TESTFN + '-diff' + self.name_same_shallow = os_helper.TESTFN + '-same-shallow' data = 'Contents of file go here.\n' for name in [self.name, self.name_same, self.name_diff]: with open(name, 'w', encoding="utf-8") as output: @@ -20,12 +33,19 @@ def setUp(self): with open(self.name_diff, 'a+', encoding="utf-8") as output: output.write('An extra line.\n') + + for name in [self.name_same, self.name_diff]: + shutil.copystat(self.name, name) + + _create_file_shallow_equal(self.name, self.name_same_shallow) + self.dir = tempfile.gettempdir() def tearDown(self): os.unlink(self.name) os.unlink(self.name_same) os.unlink(self.name_diff) + os.unlink(self.name_same_shallow) def test_matching(self): self.assertTrue(filecmp.cmp(self.name, self.name), @@ -36,12 +56,17 @@ def test_matching(self): "Comparing file to identical file fails") self.assertTrue(filecmp.cmp(self.name, self.name_same, shallow=False), "Comparing file to identical file fails") + self.assertTrue(filecmp.cmp(self.name, self.name_same_shallow), + "Shallow identical files should be considered equal") def test_different(self): self.assertFalse(filecmp.cmp(self.name, self.name_diff), "Mismatched files compare as equal") self.assertFalse(filecmp.cmp(self.name, self.dir), "File and directory compare as equal") + self.assertFalse(filecmp.cmp(self.name, self.name_same_shallow, + shallow=False), + "Mismatched file to shallow identical file compares as equal") def test_cache_clear(self): first_compare = filecmp.cmp(self.name, self.name_same, shallow=False) @@ -56,6 +81,8 @@ def setUp(self): self.dir = os.path.join(tmpdir, 'dir') self.dir_same = os.path.join(tmpdir, 'dir-same') self.dir_diff = os.path.join(tmpdir, 'dir-diff') + self.dir_diff_file = os.path.join(tmpdir, 'dir-diff-file') + self.dir_same_shallow = os.path.join(tmpdir, 'dir-same-shallow') # Another dir is created under dir_same, but it has a name from the # ignored list so it should not affect testing results. @@ -63,7 +90,17 @@ def setUp(self): self.caseinsensitive = os.path.normcase('A') == os.path.normcase('a') data = 'Contents of file go here.\n' - for dir in (self.dir, self.dir_same, self.dir_diff, self.dir_ignored): + + shutil.rmtree(self.dir, True) + os.mkdir(self.dir) + subdir_path = os.path.join(self.dir, 'subdir') + os.mkdir(subdir_path) + dir_file_path = os.path.join(self.dir, "file") + with open(dir_file_path, 'w', encoding="utf-8") as output: + output.write(data) + + for dir in (self.dir_same, self.dir_same_shallow, + self.dir_diff, self.dir_diff_file): shutil.rmtree(dir, True) os.mkdir(dir) subdir_path = os.path.join(dir, 'subdir') @@ -72,14 +109,25 @@ def setUp(self): fn = 'FiLe' # Verify case-insensitive comparison else: fn = 'file' - with open(os.path.join(dir, fn), 'w', encoding="utf-8") as output: - output.write(data) + + file_path = os.path.join(dir, fn) + + if dir is self.dir_same_shallow: + _create_file_shallow_equal(dir_file_path, file_path) + else: + shutil.copy2(dir_file_path, file_path) with open(os.path.join(self.dir_diff, 'file2'), 'w', encoding="utf-8") as output: output.write('An extra file.\n') + # Add different file2 with respect to dir_diff + with open(os.path.join(self.dir_diff_file, 'file2'), 'w', encoding="utf-8") as output: + output.write('Different contents.\n') + + def tearDown(self): - for dir in (self.dir, self.dir_same, self.dir_diff): + for dir in (self.dir, self.dir_same, self.dir_diff, + self.dir_same_shallow, self.dir_diff_file): shutil.rmtree(dir) def test_default_ignores(self): @@ -102,11 +150,7 @@ def test_cmpfiles(self): shallow=False), "Comparing directory to same fails") - # Add different file2 - with open(os.path.join(self.dir, 'file2'), 'w', encoding="utf-8") as output: - output.write('Different contents.\n') - - self.assertFalse(filecmp.cmpfiles(self.dir, self.dir_same, + self.assertFalse(filecmp.cmpfiles(self.dir, self.dir_diff_file, ['file', 'file2']) == (['file'], ['file2'], []), "Comparing mismatched directories fails") @@ -116,11 +160,22 @@ def _assert_lists(self, actual, expected): """Assert that two lists are equal, up to ordering.""" self.assertEqual(sorted(actual), sorted(expected)) + def test_dircmp_identical_directories(self): + self._assert_dircmp_identical_directories() + self._assert_dircmp_identical_directories(shallow=False) - def test_dircmp(self): + def test_dircmp_different_file(self): + self._assert_dircmp_different_file() + self._assert_dircmp_different_file(shallow=False) + + def test_dircmp_different_directories(self): + self._assert_dircmp_different_directories() + self._assert_dircmp_different_directories(shallow=False) + + def _assert_dircmp_identical_directories(self, **options): # Check attributes for comparison of two identical directories left_dir, right_dir = self.dir, self.dir_same - d = filecmp.dircmp(left_dir, right_dir) + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) if self.caseinsensitive: @@ -142,9 +197,10 @@ def test_dircmp(self): ] self._assert_report(d.report, expected_report) + def _assert_dircmp_different_directories(self, **options): # Check attributes for comparison of two different directories (right) left_dir, right_dir = self.dir, self.dir_diff - d = filecmp.dircmp(left_dir, right_dir) + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) self._assert_lists(d.left_list, ['file', 'subdir']) @@ -164,12 +220,8 @@ def test_dircmp(self): self._assert_report(d.report, expected_report) # Check attributes for comparison of two different directories (left) - left_dir, right_dir = self.dir, self.dir_diff - shutil.move( - os.path.join(self.dir_diff, 'file2'), - os.path.join(self.dir, 'file2') - ) - d = filecmp.dircmp(left_dir, right_dir) + left_dir, right_dir = self.dir_diff, self.dir + d = filecmp.dircmp(left_dir, right_dir, **options) self.assertEqual(d.left, left_dir) self.assertEqual(d.right, right_dir) self._assert_lists(d.left_list, ['file', 'file2', 'subdir']) @@ -180,27 +232,51 @@ def test_dircmp(self): self.assertEqual(d.same_files, ['file']) self.assertEqual(d.diff_files, []) expected_report = [ - "diff {} {}".format(self.dir, self.dir_diff), - "Only in {} : ['file2']".format(self.dir), + "diff {} {}".format(self.dir_diff, self.dir), + "Only in {} : ['file2']".format(self.dir_diff), "Identical files : ['file']", "Common subdirectories : ['subdir']", ] self._assert_report(d.report, expected_report) - # Add different file2 - with open(os.path.join(self.dir_diff, 'file2'), 'w', encoding="utf-8") as output: - output.write('Different contents.\n') - d = filecmp.dircmp(self.dir, self.dir_diff) + + def _assert_dircmp_different_file(self, **options): + # A different file2 + d = filecmp.dircmp(self.dir_diff, self.dir_diff_file, **options) self.assertEqual(d.same_files, ['file']) self.assertEqual(d.diff_files, ['file2']) expected_report = [ - "diff {} {}".format(self.dir, self.dir_diff), + "diff {} {}".format(self.dir_diff, self.dir_diff_file), "Identical files : ['file']", "Differing files : ['file2']", "Common subdirectories : ['subdir']", ] self._assert_report(d.report, expected_report) + def test_dircmp_no_shallow_different_file(self): + # A non shallow different file2 + d = filecmp.dircmp(self.dir, self.dir_same_shallow, shallow=False) + self.assertEqual(d.same_files, []) + self.assertEqual(d.diff_files, ['file']) + expected_report = [ + "diff {} {}".format(self.dir, self.dir_same_shallow), + "Differing files : ['file']", + "Common subdirectories : ['subdir']", + ] + self._assert_report(d.report, expected_report) + + def test_dircmp_shallow_same_file(self): + # A non shallow different file2 + d = filecmp.dircmp(self.dir, self.dir_same_shallow) + self.assertEqual(d.same_files, ['file']) + self.assertEqual(d.diff_files, []) + expected_report = [ + "diff {} {}".format(self.dir, self.dir_same_shallow), + "Identical files : ['file']", + "Common subdirectories : ['subdir']", + ] + self._assert_report(d.report, expected_report) + def test_dircmp_subdirs_type(self): """Check that dircmp.subdirs respects subclassing.""" class MyDirCmp(filecmp.dircmp): diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index baed03d92b9e56..f8812c281c2deb 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -13,7 +13,7 @@ _testcapi = None from test import support -from test.support import threading_helper +from test.support import threading_helper, Py_GIL_DISABLED from test.support.script_helper import assert_python_ok @@ -294,6 +294,7 @@ def gen(): assert_python_ok("-c", code) @support.cpython_only + @unittest.skipIf(Py_GIL_DISABLED, "test requires precise GC scheduling") def test_sneaky_frame_object(self): def trace(frame, event, arg): @@ -330,6 +331,7 @@ def f(): # on the *very next* allocation: gc.collect() gc.set_threshold(1, 0, 0) + sys._clear_internal_caches() # Okay, so here's the nightmare scenario: # - We're tracing the resumption of a generator, which creates a new # frame object. diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 81115e9db888cf..bed0e6d973b8da 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -543,8 +543,8 @@ def test_set_pasv(self): self.assertFalse(self.client.passiveserver) def test_voidcmd(self): - self.client.voidcmd('echo 200') - self.client.voidcmd('echo 299') + self.assertEqual(self.client.voidcmd('echo 200'), '200') + self.assertEqual(self.client.voidcmd('echo 299'), '299') self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 199') self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 300') diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 7c66b906d308ba..4eb322644fc541 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -31,10 +31,6 @@ decimal = import_helper.import_fresh_module('decimal', fresh=['_decimal']) -_partial_types = [py_functools.partial] -if c_functools: - _partial_types.append(c_functools.partial) - @contextlib.contextmanager def replaced_module(name, replacement): @@ -207,10 +203,7 @@ def test_repr(self): kwargs = {'a': object(), 'b': object()} kwargs_reprs = ['a={a!r}, b={b!r}'.format_map(kwargs), 'b={b!r}, a={a!r}'.format_map(kwargs)] - if self.partial in _partial_types: - name = 'functools.partial' - else: - name = self.partial.__name__ + name = f"{self.partial.__module__}.{self.partial.__qualname__}" f = self.partial(capture) self.assertEqual(f'{name}({capture!r})', repr(f)) @@ -229,10 +222,7 @@ def test_repr(self): for kwargs_repr in kwargs_reprs]) def test_recursive_repr(self): - if self.partial in _partial_types: - name = 'functools.partial' - else: - name = self.partial.__name__ + name = f"{self.partial.__module__}.{self.partial.__qualname__}" f = self.partial(capture) f.__setstate__((f, (), {}, {})) @@ -2867,11 +2857,26 @@ def _(arg: typing.Union[int, typing.Iterable[str]]): def test_invalid_positional_argument(self): @functools.singledispatch - def f(*args): + def f(*args, **kwargs): pass msg = 'f requires at least 1 positional argument' with self.assertRaisesRegex(TypeError, msg): f() + msg = 'f requires at least 1 positional argument' + with self.assertRaisesRegex(TypeError, msg): + f(a=1) + + def test_invalid_positional_argument_singledispatchmethod(self): + class A: + @functools.singledispatchmethod + def t(self, *args, **kwargs): + pass + msg = 't requires at least 1 positional argument' + with self.assertRaisesRegex(TypeError, msg): + A().t() + msg = 't requires at least 1 positional argument' + with self.assertRaisesRegex(TypeError, msg): + A().t(a=1) def test_union(self): @functools.singledispatch diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index b01f344cb14a1a..dd09643788d62f 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -363,6 +363,7 @@ def __del__(self): # To minimize variations, though, we first store the get_count() results # and check them at the end. @refcount_test + @unittest.skipIf(Py_GIL_DISABLED, 'needs precise allocation counts') def test_get_count(self): gc.collect() a, b, c = gc.get_count() diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index a7ad6c7320b4ee..32c2c2fca05c4e 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -33,7 +33,7 @@ def skip_if_different_mount_drives(): import parser from stack import Stack import tier1_generator - import tier2_abstract_generator + import optimizer_generator def handle_stderr(): @@ -350,12 +350,15 @@ def test_cache_effect(self): output = """ TARGET(OP) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(OP); PyObject *value; value = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; uint32_t extra = read_u32(&this_instr[2].cache); + (void)extra; stack_pointer += -1; DISPATCH(); } @@ -399,6 +402,7 @@ def test_macro_instruction(self): INSTRUCTION_STATS(OP); PREDICTED(OP); _Py_CODEUNIT *this_instr = next_instr - 6; + (void)this_instr; PyObject *right; PyObject *left; PyObject *arg2; @@ -408,6 +412,7 @@ def test_macro_instruction(self): left = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; op1(left, right); } /* Skip 2 cache entries */ @@ -415,6 +420,7 @@ def test_macro_instruction(self): arg2 = stack_pointer[-3]; { uint32_t extra = read_u32(&this_instr[4].cache); + (void)extra; res = op2(arg2, left, right); } stack_pointer[-3] = res; @@ -424,6 +430,7 @@ def test_macro_instruction(self): TARGET(OP1) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(OP1); PyObject *right; @@ -431,6 +438,7 @@ def test_macro_instruction(self): right = stack_pointer[-1]; left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; op1(left, right); DISPATCH(); } @@ -794,6 +802,17 @@ def test_annotated_op(self): self.run_cases_test(input, output) + def test_deopt_and_exit(self): + input = """ + pure op(OP, (arg1 -- out)) { + DEOPT_IF(1); + EXIT_IF(1); + } + """ + output = "" + with self.assertRaises(Exception): + self.run_cases_test(input, output) + class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: super().setUp() @@ -830,7 +849,7 @@ def run_cases_test(self, input: str, input2: str, expected: str): temp_input.flush() with handle_stderr(): - tier2_abstract_generator.generate_tier2_abstract_from_files( + optimizer_generator.generate_tier2_abstract_from_files( [self.temp_input_filename, self.temp_input2_filename], self.temp_output_filename ) @@ -879,8 +898,8 @@ def test_overridden_abstract_args(self): """ output = """ case OP: { - _Py_UOpsSymType *arg1; - _Py_UOpsSymType *out; + _Py_UopsSymbol *arg1; + _Py_UopsSymbol *out; arg1 = stack_pointer[-1]; eggs(); stack_pointer[-1] = out; @@ -888,7 +907,7 @@ def test_overridden_abstract_args(self): } case OP2: { - _Py_UOpsSymType *out; + _Py_UopsSymbol *out; out = sym_new_unknown(ctx); if (out == NULL) goto out_of_space; stack_pointer[-1] = out; @@ -913,7 +932,7 @@ def test_no_overridden_case(self): """ output = """ case OP: { - _Py_UOpsSymType *out; + _Py_UopsSymbol *out; out = sym_new_unknown(ctx); if (out == NULL) goto out_of_space; stack_pointer[-1] = out; @@ -921,8 +940,8 @@ def test_no_overridden_case(self): } case OP2: { - _Py_UOpsSymType *arg1; - _Py_UOpsSymType *out; + _Py_UopsSymbol *arg1; + _Py_UopsSymbol *out; arg1 = stack_pointer[-1]; stack_pointer[-1] = out; break; diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index 128f933787a3f6..d220c7d06e50c9 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -5,7 +5,6 @@ import functools import io import os -import pathlib import struct import sys import unittest @@ -79,16 +78,18 @@ def test_write(self): f.close() def test_write_read_with_pathlike_file(self): - filename = pathlib.Path(self.filename) + filename = os_helper.FakePath(self.filename) with gzip.GzipFile(filename, 'w') as f: f.write(data1 * 50) self.assertIsInstance(f.name, str) + self.assertEqual(f.name, self.filename) with gzip.GzipFile(filename, 'a') as f: f.write(data1) with gzip.GzipFile(filename) as f: d = f.read() self.assertEqual(d, data1 * 51) self.assertIsInstance(f.name, str) + self.assertEqual(f.name, self.filename) # The following test_write_xy methods test that write accepts # the corresponding bytes-like object type as input @@ -472,13 +473,118 @@ def test_textio_readlines(self): with io.TextIOWrapper(f, encoding="ascii") as t: self.assertEqual(t.readlines(), lines) + def test_fileobj_with_name(self): + with open(self.filename, "xb") as raw: + with gzip.GzipFile(fileobj=raw, mode="x") as f: + f.write(b'one') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "wb") as raw: + with gzip.GzipFile(fileobj=raw, mode="w") as f: + f.write(b'two') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "ab") as raw: + with gzip.GzipFile(fileobj=raw, mode="a") as f: + f.write(b'three') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + with open(self.filename, "rb") as raw: + with gzip.GzipFile(fileobj=raw, mode="r") as f: + self.assertEqual(f.read(), b'twothree') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, raw.name) + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + def test_fileobj_from_fdopen(self): # Issue #13781: Opening a GzipFile for writing fails when using a # fileobj created with os.fdopen(). - fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT) - with os.fdopen(fd, "wb") as f: - with gzip.GzipFile(fileobj=f, mode="w") as g: - pass + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + with os.fdopen(fd, "xb") as raw: + with gzip.GzipFile(fileobj=raw, mode="x") as f: + f.write(b'one') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with os.fdopen(fd, "wb") as raw: + with gzip.GzipFile(fileobj=raw, mode="w") as f: + f.write(b'two') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND) + with os.fdopen(fd, "ab") as raw: + with gzip.GzipFile(fileobj=raw, mode="a") as f: + f.write(b'three') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + + fd = os.open(self.filename, os.O_RDONLY) + with os.fdopen(fd, "rb") as raw: + with gzip.GzipFile(fileobj=raw, mode="r") as f: + self.assertEqual(f.read(), b'twothree') + self.assertEqual(f.name, '') + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) def test_fileobj_mode(self): gzip.GzipFile(self.filename, "wb").close() @@ -508,17 +614,69 @@ def test_fileobj_mode(self): def test_bytes_filename(self): str_filename = self.filename - try: - bytes_filename = str_filename.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(str_filename) with gzip.GzipFile(bytes_filename, "wb") as f: f.write(data1 * 50) + self.assertEqual(f.name, bytes_filename) with gzip.GzipFile(bytes_filename, "rb") as f: self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, bytes_filename) # Sanity check that we are actually operating on the right file. with gzip.GzipFile(str_filename, "rb") as f: self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, str_filename) + + def test_fileobj_without_name(self): + bio = io.BytesIO() + with gzip.GzipFile(fileobj=bio, mode='wb') as f: + f.write(data1 * 50) + self.assertEqual(f.name, '') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.WRITE) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), True) + + bio.seek(0) + with gzip.GzipFile(fileobj=bio, mode='rb') as f: + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, '') + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertEqual(f.name, '') + self.assertRaises(AttributeError, f.fileno) + self.assertEqual(f.mode, gzip.READ) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + + def test_fileobj_and_filename(self): + filename2 = self.filename + 'new' + with (open(self.filename, 'wb') as fileobj, + gzip.GzipFile(fileobj=fileobj, filename=filename2, mode='wb') as f): + f.write(data1 * 50) + self.assertEqual(f.name, filename2) + with (open(self.filename, 'rb') as fileobj, + gzip.GzipFile(fileobj=fileobj, filename=filename2, mode='rb') as f): + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, filename2) + # Sanity check that we are actually operating on the right file. + with gzip.GzipFile(self.filename, 'rb') as f: + self.assertEqual(f.read(), data1 * 50) + self.assertEqual(f.name, self.filename) def test_decompress_limited(self): """Decompressed data buffering should be limited""" @@ -707,13 +865,16 @@ def test_binary_modes(self): self.assertEqual(file_data, uncompressed) def test_pathlike_file(self): - filename = pathlib.Path(self.filename) + filename = os_helper.FakePath(self.filename) with gzip.open(filename, "wb") as f: f.write(data1 * 50) + self.assertEqual(f.name, self.filename) with gzip.open(filename, "ab") as f: f.write(data1) + self.assertEqual(f.name, self.filename) with gzip.open(filename) as f: self.assertEqual(f.read(), data1 * 51) + self.assertEqual(f.name, self.filename) def test_implicit_binary_modes(self): # Test implicit binary modes (no "b" or "t" in mode string). diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index a39a2c45ebc2e2..1502fba9f3e8b8 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -479,6 +479,14 @@ def test_exercise_all_methods(self): self.fail("Exception raised during normal usage of HMAC class.") +class UpdateTestCase(unittest.TestCase): + @hashlib_helper.requires_hashdigest('sha256') + def test_with_str_update(self): + with self.assertRaises(TypeError): + h = hmac.new(b"key", digestmod='sha256') + h.update("invalid update") + + class CopyTestCase(unittest.TestCase): @hashlib_helper.requires_hashdigest('sha256') diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 089bf5be40a0e2..6e63a8872d9c6e 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2408,6 +2408,22 @@ def test_connect_put_request(self): self.assertIn(b'PUT / HTTP/1.1\r\nHost: %(host)s\r\n' % d, self.conn.sock.data) + def test_connect_put_request_ipv6(self): + self.conn.set_tunnel('[1:2:3::4]', 1234) + self.conn.request('PUT', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT [1:2:3::4]:1234', self.conn.sock.data) + self.assertIn(b'Host: [1:2:3::4]:1234', self.conn.sock.data) + + def test_connect_put_request_ipv6_port(self): + self.conn.set_tunnel('[1:2:3::4]:1234') + self.conn.request('PUT', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT [1:2:3::4]:1234', self.conn.sock.data) + self.assertIn(b'Host: [1:2:3::4]:1234', self.conn.sock.data) + def test_tunnel_debuglog(self): expected_header = 'X-Dummy: 1' response_text = 'HTTP/1.0 200 OK\r\n{}\r\n\r\n'.format(expected_header) diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py index cc993f333e355a..38ab21907b58d9 100644 --- a/Lib/test/test_importlib/test_lazy.py +++ b/Lib/test/test_importlib/test_lazy.py @@ -2,9 +2,12 @@ from importlib import abc from importlib import util import sys +import time +import threading import types import unittest +from test.support import threading_helper from test.test_importlib import util as test_util @@ -40,6 +43,7 @@ class TestingImporter(abc.MetaPathFinder, abc.Loader): module_name = 'lazy_loader_test' mutated_name = 'changed' loaded = None + load_count = 0 source_code = 'attr = 42; __name__ = {!r}'.format(mutated_name) def find_spec(self, name, path, target=None): @@ -48,8 +52,10 @@ def find_spec(self, name, path, target=None): return util.spec_from_loader(name, util.LazyLoader(self)) def exec_module(self, module): + time.sleep(0.01) # Simulate a slow load. exec(self.source_code, module.__dict__) self.loaded = module + self.load_count += 1 class LazyLoaderTests(unittest.TestCase): @@ -59,8 +65,9 @@ def test_init(self): # Classes that don't define exec_module() trigger TypeError. util.LazyLoader(object) - def new_module(self, source_code=None): - loader = TestingImporter() + def new_module(self, source_code=None, loader=None): + if loader is None: + loader = TestingImporter() if source_code is not None: loader.source_code = source_code spec = util.spec_from_loader(TestingImporter.module_name, @@ -140,6 +147,37 @@ def test_module_already_in_sys(self): # Force the load; just care that no exception is raised. module.__name__ + @threading_helper.requires_working_threading() + def test_module_load_race(self): + with test_util.uncache(TestingImporter.module_name): + loader = TestingImporter() + module = self.new_module(loader=loader) + self.assertEqual(loader.load_count, 0) + + class RaisingThread(threading.Thread): + exc = None + def run(self): + try: + super().run() + except Exception as exc: + self.exc = exc + + def access_module(): + return module.attr + + threads = [] + for _ in range(2): + threads.append(thread := RaisingThread(target=access_module)) + thread.start() + + # Races could cause errors + for thread in threads: + thread.join() + self.assertIsNone(thread.exc) + + # Or multiple load attempts + self.assertEqual(loader.load_count, 1) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_inspect/inspect_fodder.py b/Lib/test/test_inspect/inspect_fodder.py index 60ba7aa78394e8..febd54c86fe1d1 100644 --- a/Lib/test/test_inspect/inspect_fodder.py +++ b/Lib/test/test_inspect/inspect_fodder.py @@ -68,9 +68,9 @@ class FesteringGob(MalodorousPervert, ParrotDroppings): def abuse(self, a, b, c): pass - @property - def contradiction(self): + def _getter(self): pass + contradiction = property(_getter) async def lobbest(grenade): pass diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index c5a6de5993fad4..52cf68b93b85fa 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -2928,9 +2928,12 @@ def p(name): return signature.parameters[name].default # This doesn't work now. # (We don't have a valid signature for "type" in 3.4) + class ThisWorksNow: + __call__ = type + # TODO: Support type. + self.assertEqual(ThisWorksNow()(1), int) + self.assertEqual(ThisWorksNow()('A', (), {}).__name__, 'A') with self.assertRaisesRegex(ValueError, "no signature found"): - class ThisWorksNow: - __call__ = type test_callable(ThisWorksNow()) # Regression test for issue #20786 @@ -3137,6 +3140,10 @@ def m1d(*args, **kwargs): int)) def test_signature_on_classmethod(self): + self.assertEqual(self.signature(classmethod), + ((('function', ..., ..., "positional_only"),), + ...)) + class Test: @classmethod def foo(cls, arg1, *, arg2=1): @@ -3155,6 +3162,10 @@ def foo(cls, arg1, *, arg2=1): ...)) def test_signature_on_staticmethod(self): + self.assertEqual(self.signature(staticmethod), + ((('function', ..., ..., "positional_only"),), + ...)) + class Test: @staticmethod def foo(cls, *, arg): @@ -3513,6 +3524,98 @@ def __init__(self, b): ((('a', ..., ..., "positional_or_keyword"),), ...)) + with self.subTest('classmethod'): + class CM(type): + @classmethod + def __call__(cls, a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class CM(type): + @staticmethod + def __call__(a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + return a + class CM(type): + __call__ = A().call + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class CM(type): + __call__ = functools.partial(lambda x, a: (x, a), 2) + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class CM(type): + __call__ = functools.partialmethod(lambda self, x, a: (x, a), 2) + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class CM(type): + __call__ = ':'.join + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(['a', 'bc']), 'a:bc') + # BUG: Returns '' + with self.assertRaises(AssertionError): + self.assertEqual(self.signature(C), self.signature(''.join)) + + with self.subTest('MethodWrapperType'): + class CM(type): + __call__ = (2).__pow__ + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(3), 8) + self.assertEqual(C(3, 7), 1) + # BUG: Returns '' + with self.assertRaises(AssertionError): + self.assertEqual(self.signature(C), self.signature((0).__pow__)) + class CM(type): def __new__(mcls, name, bases, dct, *, foo=1): return super().__new__(mcls, name, bases, dct) @@ -3574,6 +3677,169 @@ def __init__(self, b): ('bar', 2, ..., "keyword_only")), ...)) + def test_signature_on_class_with_init(self): + class C: + def __init__(self, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + def __init__(cls, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __init__(b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + pass + class C: + __init__ = A().call + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __init__ = functools.partial(lambda x, a: None, 2) + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + def _init(self, x, a): + self.a = (x, a) + __init__ = functools.partialmethod(_init, 2) + + self.assertEqual(C(1).a, (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + def test_signature_on_class_with_new(self): + with self.subTest('FunctionType'): + class C: + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + def __new__(cls, cls2, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, cls, a): + return a + class C: + __new__ = A().call + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __new__ = functools.partial(lambda x, cls, a: (x, a), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __new__ = functools.partialmethod(lambda cls, x, a: (x, a), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class C: + __new__ = str.__subclasscheck__ + + self.assertEqual(C(), False) + # TODO: Support BuiltinMethodType + # self.assertEqual(self.signature(C), ((), ...)) + self.assertRaises(ValueError, self.signature, C) + + with self.subTest('MethodWrapperType'): + class C: + __new__ = type.__or__.__get__(int, type) + + self.assertEqual(C(), C | int) + # TODO: Support MethodWrapperType + # self.assertEqual(self.signature(C), ((), ...)) + self.assertRaises(ValueError, self.signature, C) + + # TODO: Test ClassMethodDescriptorType + + with self.subTest('MethodDescriptorType'): + class C: + __new__ = type.__dict__['__subclasscheck__'] + + self.assertEqual(C(C), True) + self.assertEqual(self.signature(C), self.signature(C.__subclasscheck__)) + + with self.subTest('WrapperDescriptorType'): + class C: + __new__ = type.__or__ + + self.assertEqual(C(int), C | int) + # TODO: Support WrapperDescriptorType + # self.assertEqual(self.signature(C), self.signature(C.__or__)) + self.assertRaises(ValueError, self.signature, C) + def test_signature_on_subclass(self): class A: def __new__(cls, a=1, *args, **kwargs): @@ -3627,8 +3893,11 @@ class D(C): pass # Test meta-classes without user-defined __init__ or __new__ class C(type): pass class D(C): pass + self.assertEqual(C('A', (), {}).__name__, 'A') + # TODO: Support type. with self.assertRaisesRegex(ValueError, "callable.*is not supported"): self.assertEqual(inspect.signature(C), None) + self.assertEqual(D('A', (), {}).__name__, 'A') with self.assertRaisesRegex(ValueError, "callable.*is not supported"): self.assertEqual(inspect.signature(D), None) @@ -3678,16 +3947,117 @@ class Bar(Spam, Foo): ((('a', ..., ..., "positional_or_keyword"),), ...)) - class Wrapped: - pass - Wrapped.__wrapped__ = lambda a: None - self.assertEqual(self.signature(Wrapped), + with self.subTest('classmethod'): + class C: + @classmethod + def __call__(cls, a): + pass + + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __call__(a): + pass + + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + return a + class C: + __call__ = A().call + + self.assertEqual(C()(1), 1) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __call__ = functools.partial(lambda x, a: (x, a), 2) + + self.assertEqual(C()(1), (2, 1)) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __call__ = functools.partialmethod(lambda self, x, a: (x, a), 2) + + self.assertEqual(C()(1), (2, 1)) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class C: + __call__ = ':'.join + + self.assertEqual(C()(['a', 'bc']), 'a:bc') + self.assertEqual(self.signature(C()), self.signature(''.join)) + + with self.subTest('MethodWrapperType'): + class C: + __call__ = (2).__pow__ + + self.assertEqual(C()(3), 8) + self.assertEqual(self.signature(C()), self.signature((0).__pow__)) + + with self.subTest('ClassMethodDescriptorType'): + class C(dict): + __call__ = dict.__dict__['fromkeys'] + + res = C()([1, 2], 3) + self.assertEqual(res, {1: 3, 2: 3}) + self.assertEqual(type(res), C) + self.assertEqual(self.signature(C()), self.signature(dict.fromkeys)) + + with self.subTest('MethodDescriptorType'): + class C(str): + __call__ = str.join + + self.assertEqual(C(':')(['a', 'bc']), 'a:bc') + self.assertEqual(self.signature(C()), self.signature(''.join)) + + with self.subTest('WrapperDescriptorType'): + class C(int): + __call__ = int.__pow__ + + self.assertEqual(C(2)(3), 8) + self.assertEqual(self.signature(C()), self.signature((0).__pow__)) + + with self.subTest('MemberDescriptorType'): + class C: + __slots__ = '__call__' + c = C() + c.__call__ = lambda a: a + self.assertEqual(c(1), 1) + self.assertEqual(self.signature(c), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + def test_signature_on_wrapper(self): + class Wrapper: + def __call__(self, b): + pass + wrapper = Wrapper() + wrapper.__wrapped__ = lambda a: None + self.assertEqual(self.signature(wrapper), ((('a', ..., ..., "positional_or_keyword"),), ...)) # wrapper loop: - Wrapped.__wrapped__ = Wrapped + wrapper = Wrapper() + wrapper.__wrapped__ = wrapper with self.assertRaisesRegex(ValueError, 'wrapper loop'): - self.signature(Wrapped) + self.signature(wrapper) def test_signature_on_lambdas(self): self.assertEqual(self.signature((lambda a=10: a)), @@ -4999,6 +5369,14 @@ def test_recursion_limit(self): with self.assertRaisesRegex(ValueError, 'wrapper loop'): inspect.unwrap(obj) + def test_wrapped_descriptor(self): + self.assertIs(inspect.unwrap(NTimesUnwrappable), NTimesUnwrappable) + self.assertIs(inspect.unwrap(staticmethod), staticmethod) + self.assertIs(inspect.unwrap(classmethod), classmethod) + self.assertIs(inspect.unwrap(staticmethod(classmethod)), classmethod) + self.assertIs(inspect.unwrap(classmethod(staticmethod)), staticmethod) + + class TestMain(unittest.TestCase): def test_only_source(self): module = importlib.import_module('unittest') diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 0bf55facad9fed..47fc50a0e20349 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -664,84 +664,78 @@ def test_denial_of_service_prevented_int_to_str(self): """Regression test: ensure we fail before performing O(N**2) work.""" maxdigits = sys.get_int_max_str_digits() assert maxdigits < 50_000, maxdigits # A test prerequisite. - get_time = time.process_time - if get_time() <= 0: # some platforms like WASM lack process_time() - get_time = time.monotonic huge_int = int(f'0x{"c"*65_000}', base=16) # 78268 decimal digits. digits = 78_268 - with support.adjust_int_max_str_digits(digits): - start = get_time() + with ( + support.adjust_int_max_str_digits(digits), + support.CPUStopwatch() as sw_convert): huge_decimal = str(huge_int) - seconds_to_convert = get_time() - start self.assertEqual(len(huge_decimal), digits) # Ensuring that we chose a slow enough conversion to measure. # It takes 0.1 seconds on a Zen based cloud VM in an opt build. # Some OSes have a low res 1/64s timer, skip if hard to measure. - if seconds_to_convert < 1/64: + if sw_convert.seconds < sw_convert.clock_info.resolution * 2: raise unittest.SkipTest('"slow" conversion took only ' - f'{seconds_to_convert} seconds.') + f'{sw_convert.seconds} seconds.') # We test with the limit almost at the size needed to check performance. # The performant limit check is slightly fuzzy, give it a some room. with support.adjust_int_max_str_digits(int(.995 * digits)): - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_huge): str(huge_int) - seconds_to_fail_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) # Now we test that a conversion that would take 30x as long also fails # in a similarly fast fashion. extra_huge_int = int(f'0x{"c"*500_000}', base=16) # 602060 digits. - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_extra_huge): # If not limited, 8 seconds said Zen based cloud VM. str(extra_huge_int) - seconds_to_fail_extra_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLess(seconds_to_fail_extra_huge, seconds_to_convert/2) + self.assertLess(sw_fail_extra_huge.seconds, sw_convert.seconds/2) def test_denial_of_service_prevented_str_to_int(self): """Regression test: ensure we fail before performing O(N**2) work.""" maxdigits = sys.get_int_max_str_digits() assert maxdigits < 100_000, maxdigits # A test prerequisite. - get_time = time.process_time - if get_time() <= 0: # some platforms like WASM lack process_time() - get_time = time.monotonic digits = 133700 huge = '8'*digits - with support.adjust_int_max_str_digits(digits): - start = get_time() + with ( + support.adjust_int_max_str_digits(digits), + support.CPUStopwatch() as sw_convert): int(huge) - seconds_to_convert = get_time() - start # Ensuring that we chose a slow enough conversion to measure. # It takes 0.1 seconds on a Zen based cloud VM in an opt build. # Some OSes have a low res 1/64s timer, skip if hard to measure. - if seconds_to_convert < 1/64: + if sw_convert.seconds < sw_convert.clock_info.resolution * 2: raise unittest.SkipTest('"slow" conversion took only ' - f'{seconds_to_convert} seconds.') + f'{sw_convert.seconds} seconds.') with support.adjust_int_max_str_digits(digits - 1): - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_huge): int(huge) - seconds_to_fail_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) # Now we test that a conversion that would take 30x as long also fails # in a similarly fast fashion. extra_huge = '7'*1_200_000 - with self.assertRaises(ValueError) as err: - start = get_time() + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_extra_huge): # If not limited, 8 seconds in the Zen based cloud VM. int(extra_huge) - seconds_to_fail_extra_huge = get_time() - start self.assertIn('conversion', str(err.exception)) - self.assertLessEqual(seconds_to_fail_extra_huge, seconds_to_convert/2) + self.assertLessEqual(sw_fail_extra_huge.seconds, sw_convert.seconds/2) def test_power_of_two_bases_unlimited(self): """The limit does not apply to power of 2 bases.""" diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index aefd326977095f..363143fa810f35 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -280,7 +280,7 @@ def test_subinterpreter(self): def test_finished(self): r, w = self.pipe() interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os os.write({w}, b'x') """) @@ -312,7 +312,7 @@ def test_with_only_background_threads(self): FINISHED = b'F' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading @@ -326,7 +326,7 @@ def task(): self.assertFalse(interp.is_running()) os.write(w_thread, DONE) - interp.exec_sync('t.join()') + interp.exec('t.join()') self.assertEqual(os.read(r_interp, 1), FINISHED) @@ -393,7 +393,7 @@ def test_from_sibling(self): interp2 = interpreters.create() self.assertEqual(set(interpreters.list_all()), {main, interp1, interp2}) - interp1.exec_sync(dedent(f""" + interp1.exec(dedent(f""" from test.support import interpreters interp2 = interpreters.Interpreter({interp2.id}) interp2.close() @@ -427,7 +427,7 @@ def test_subthreads_still_running(self): FINISHED = b'F' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading import time @@ -503,27 +503,27 @@ def test_not_shareable(self): interp.prepare_main(spam={'spam': 'eggs', 'foo': 'bar'}) # Make sure neither was actually bound. - with self.assertRaises(interpreters.ExecFailure): - interp.exec_sync('print(foo)') - with self.assertRaises(interpreters.ExecFailure): - interp.exec_sync('print(spam)') + with self.assertRaises(interpreters.ExecutionFailed): + interp.exec('print(foo)') + with self.assertRaises(interpreters.ExecutionFailed): + interp.exec('print(spam)') -class TestInterpreterExecSync(TestBase): +class TestInterpreterExec(TestBase): def test_success(self): interp = interpreters.create() script, file = _captured_script('print("it worked!", end="")') with file: - interp.exec_sync(script) + interp.exec(script) out = file.read() self.assertEqual(out, 'it worked!') def test_failure(self): interp = interpreters.create() - with self.assertRaises(interpreters.ExecFailure): - interp.exec_sync('raise Exception') + with self.assertRaises(interpreters.ExecutionFailed): + interp.exec('raise Exception') def test_display_preserved_exception(self): tempdir = self.temp_dir() @@ -542,21 +542,21 @@ def script(): spam.eggs() interp = interpreters.create() - interp.exec_sync(script) + interp.exec(script) """) stdout, stderr = self.assert_python_failure(scriptfile) self.maxDiff = None - interpmod_line, = (l for l in stderr.splitlines() if ' exec_sync' in l) - # File "{interpreters.__file__}", line 179, in exec_sync + interpmod_line, = (l for l in stderr.splitlines() if ' exec' in l) + # File "{interpreters.__file__}", line 179, in exec self.assertEqual(stderr, dedent(f"""\ Traceback (most recent call last): File "{scriptfile}", line 9, in - interp.exec_sync(script) - ~~~~~~~~~~~~~~~~^^^^^^^^ + interp.exec(script) + ~~~~~~~~~~~^^^^^^^^ {interpmod_line.strip()} - raise ExecFailure(excinfo) - test.support.interpreters.ExecFailure: RuntimeError: uh-oh! + raise ExecutionFailed(excinfo) + test.support.interpreters.ExecutionFailed: RuntimeError: uh-oh! Uncaught in the interpreter: @@ -578,7 +578,7 @@ def test_in_thread(self): script, file = _captured_script('print("it worked!", end="")') with file: def f(): - interp.exec_sync(script) + interp.exec(script) t = threading.Thread(target=f) t.start() @@ -604,7 +604,7 @@ def test_fork(self): with open('{file.name}', 'w', encoding='utf-8') as out: out.write('{expected}') """) - interp.exec_sync(script) + interp.exec(script) file.seek(0) content = file.read() @@ -615,17 +615,17 @@ def test_already_running(self): interp = interpreters.create() with _running(interp): with self.assertRaises(RuntimeError): - interp.exec_sync('print("spam")') + interp.exec('print("spam")') def test_bad_script(self): interp = interpreters.create() with self.assertRaises(TypeError): - interp.exec_sync(10) + interp.exec(10) def test_bytes_for_script(self): interp = interpreters.create() with self.assertRaises(TypeError): - interp.exec_sync(b'print("spam")') + interp.exec(b'print("spam")') def test_with_background_threads_still_running(self): r_interp, w_interp = self.pipe() @@ -636,7 +636,7 @@ def test_with_background_threads_still_running(self): FINISHED = b'F' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading @@ -648,46 +648,229 @@ def task(): t.start() os.write({w_interp}, {RAN!r}) """) - interp.exec_sync(f"""if True: + interp.exec(f"""if True: os.write({w_interp}, {RAN!r}) """) os.write(w_thread, DONE) - interp.exec_sync('t.join()') + interp.exec('t.join()') self.assertEqual(os.read(r_interp, 1), RAN) self.assertEqual(os.read(r_interp, 1), RAN) self.assertEqual(os.read(r_interp, 1), FINISHED) # test_xxsubinterpreters covers the remaining - # Interpreter.exec_sync() behavior. + # Interpreter.exec() behavior. -class TestInterpreterRun(TestBase): - - def test_success(self): - interp = interpreters.create() - script, file = _captured_script('print("it worked!", end="")') - with file: - t = interp.run(script) +def call_func_noop(): + pass + + +def call_func_return_shareable(): + return (1, None) + + +def call_func_return_not_shareable(): + return [1, 2, 3] + + +def call_func_failure(): + raise Exception('spam!') + + +def call_func_ident(value): + return value + + +def get_call_func_closure(value): + def call_func_closure(): + return value + return call_func_closure + + +class Spam: + + @staticmethod + def noop(): + pass + + @classmethod + def from_values(cls, *values): + return cls(values) + + def __init__(self, value): + self.value = value + + def __call__(self, *args, **kwargs): + return (self.value, args, kwargs) + + def __eq__(self, other): + if not isinstance(other, Spam): + return NotImplemented + return self.value == other.value + + def run(self, *args, **kwargs): + return (self.value, args, kwargs) + + +def call_func_complex(op, /, value=None, *args, exc=None, **kwargs): + if exc is not None: + raise exc + if op == '': + raise ValueError('missing op') + elif op == 'ident': + if args or kwargs: + raise Exception((args, kwargs)) + return value + elif op == 'full-ident': + return (value, args, kwargs) + elif op == 'globals': + if value is not None or args or kwargs: + raise Exception((value, args, kwargs)) + return __name__ + elif op == 'interpid': + if value is not None or args or kwargs: + raise Exception((value, args, kwargs)) + return interpreters.get_current().id + elif op == 'closure': + if args or kwargs: + raise Exception((args, kwargs)) + return get_call_func_closure(value) + elif op == 'custom': + if args or kwargs: + raise Exception((args, kwargs)) + return Spam(value) + elif op == 'custom-inner': + if args or kwargs: + raise Exception((args, kwargs)) + class Eggs(Spam): + pass + return Eggs(value) + elif not isinstance(op, str): + raise TypeError(op) + else: + raise NotImplementedError(op) + + +class TestInterpreterCall(TestBase): + + # signature + # - blank + # - args + # - kwargs + # - args, kwargs + # return + # - nothing (None) + # - simple + # - closure + # - custom + # ops: + # - do nothing + # - fail + # - echo + # - do complex, relative to interpreter + # scope + # - global func + # - local closure + # - returned closure + # - callable type instance + # - type + # - classmethod + # - staticmethod + # - instance method + # exception + # - builtin + # - custom + # - preserves info (e.g. SyntaxError) + # - matching error display + + def test_call(self): + interp = interpreters.create() + + for i, (callable, args, kwargs) in enumerate([ + (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), + (Spam.noop, (), {}), + ]): + with self.subTest(f'success case #{i+1}'): + res = interp.call(callable) + self.assertIs(res, None) + + for i, (callable, args, kwargs) in enumerate([ + (call_func_ident, ('spamspamspam',), {}), + (get_call_func_closure, (42,), {}), + (get_call_func_closure(42), (), {}), + (Spam.from_values, (), {}), + (Spam.from_values, (1, 2, 3), {}), + (Spam, ('???'), {}), + (Spam(101), (), {}), + (Spam(10101).run, (), {}), + (call_func_complex, ('ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), + (call_func_complex, ('globals',), {}), + (call_func_complex, ('interpid',), {}), + (call_func_complex, ('closure',), {'value': '~~~'}), + (call_func_complex, ('custom', 'spam!'), {}), + (call_func_complex, ('custom-inner', 'eggs!'), {}), + (call_func_complex, ('???',), {'exc': ValueError('spam')}), + ]): + with self.subTest(f'invalid case #{i+1}'): + with self.assertRaises(Exception): + if args or kwargs: + raise Exception((args, kwargs)) + interp.call(callable) + + with self.assertRaises(interpreters.ExecutionFailed): + interp.call(call_func_failure) + + def test_call_in_thread(self): + interp = interpreters.create() + + for i, (callable, args, kwargs) in enumerate([ + (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), + (Spam.noop, (), {}), + ]): + with self.subTest(f'success case #{i+1}'): + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(callable) + t.join() + self.assertIsNone(ctx.caught) + + for i, (callable, args, kwargs) in enumerate([ + (call_func_ident, ('spamspamspam',), {}), + (get_call_func_closure, (42,), {}), + (get_call_func_closure(42), (), {}), + (Spam.from_values, (), {}), + (Spam.from_values, (1, 2, 3), {}), + (Spam, ('???'), {}), + (Spam(101), (), {}), + (Spam(10101).run, (), {}), + (call_func_complex, ('ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam'), {}), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), + (call_func_complex, ('globals',), {}), + (call_func_complex, ('interpid',), {}), + (call_func_complex, ('closure',), {'value': '~~~'}), + (call_func_complex, ('custom', 'spam!'), {}), + (call_func_complex, ('custom-inner', 'eggs!'), {}), + (call_func_complex, ('???',), {'exc': ValueError('spam')}), + ]): + with self.subTest(f'invalid case #{i+1}'): + if args or kwargs: + continue + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(callable) + t.join() + self.assertIsNotNone(ctx.caught) + + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(call_func_failure) t.join() - out = file.read() - - self.assertEqual(out, 'it worked!') - - def test_failure(self): - caught = False - def excepthook(args): - nonlocal caught - caught = True - threading.excepthook = excepthook - try: - interp = interpreters.create() - t = interp.run('raise Exception') - t.join() - - self.assertTrue(caught) - except BaseException: - threading.excepthook = threading.__excepthook__ + self.assertIsNotNone(ctx.caught) class TestIsShareable(TestBase): diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 3c3e18832d4168..07e503837bcf75 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -120,7 +120,7 @@ def test_send_recv_main(self): def test_send_recv_same_interpreter(self): interp = interpreters.create() - interp.exec_sync(dedent(""" + interp.exec(dedent(""" from test.support.interpreters import channels r, s = channels.create() orig = b'spam' @@ -193,7 +193,7 @@ def test_send_recv_nowait_main_with_default(self): def test_send_recv_nowait_same_interpreter(self): interp = interpreters.create() - interp.exec_sync(dedent(""" + interp.exec(dedent(""" from test.support.interpreters import channels r, s = channels.create() orig = b'spam' diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py index c2917d839904f9..becf003e2e5f20 100644 --- a/Lib/test/test_interpreters/test_lifecycle.py +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -124,7 +124,7 @@ def test_sys_path_0(self): orig = sys.path[0] interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import json import sys print(json.dumps({{ @@ -164,6 +164,7 @@ def test_sys_path_0(self): class FinalizationTests(TestBase): + @support.requires_subprocess() def test_gh_109793(self): # Make sure finalization finishes and the correct error code # is reported, even when subinterpreters get cleaned up at the end. diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 2a8ca99c1f6e3f..905ef035d43feb 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -51,20 +51,20 @@ def test_shareable(self): queue1 = queues.create() interp = interpreters.create() - interp.exec_sync(dedent(f""" + interp.exec(dedent(f""" from test.support.interpreters import queues queue1 = queues.Queue({queue1.id}) """)); with self.subTest('same interpreter'): queue2 = queues.create() - queue1.put(queue2) + queue1.put(queue2, syncobj=True) queue3 = queue1.get() self.assertIs(queue3, queue2) with self.subTest('from current interpreter'): queue4 = queues.create() - queue1.put(queue4) + queue1.put(queue4, syncobj=True) out = _run_output(interp, dedent(""" queue4 = queue1.get() print(queue4.id) @@ -75,7 +75,7 @@ def test_shareable(self): with self.subTest('from subinterpreter'): out = _run_output(interp, dedent(""" queue5 = queues.create() - queue1.put(queue5) + queue1.put(queue5, syncobj=True) print(queue5.id) """)) qid = int(out) @@ -118,7 +118,7 @@ class TestQueueOps(TestBase): def test_empty(self): queue = queues.create() before = queue.empty() - queue.put(None) + queue.put(None, syncobj=True) during = queue.empty() queue.get() after = queue.empty() @@ -133,7 +133,7 @@ def test_full(self): queue = queues.create(3) for _ in range(3): actual.append(queue.full()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.full()) for _ in range(3): queue.get() @@ -147,16 +147,16 @@ def test_qsize(self): queue = queues.create() for _ in range(3): actual.append(queue.qsize()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.qsize()) queue.get() actual.append(queue.qsize()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.qsize()) for _ in range(3): queue.get() actual.append(queue.qsize()) - queue.put(None) + queue.put(None, syncobj=True) actual.append(queue.qsize()) queue.get() actual.append(queue.qsize()) @@ -165,30 +165,91 @@ def test_qsize(self): def test_put_get_main(self): expected = list(range(20)) - queue = queues.create() - for i in range(20): - queue.put(i) - actual = [queue.get() for _ in range(20)] + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create() + for i in range(20): + queue.put(i, **kwds) + actual = [queue.get() for _ in range(20)] - self.assertEqual(actual, expected) + self.assertEqual(actual, expected) def test_put_timeout(self): - queue = queues.create(2) - queue.put(None) - queue.put(None) - with self.assertRaises(queues.QueueFull): - queue.put(None, timeout=0.1) - queue.get() - queue.put(None) + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create(2) + queue.put(None, **kwds) + queue.put(None, **kwds) + with self.assertRaises(queues.QueueFull): + queue.put(None, timeout=0.1, **kwds) + queue.get() + queue.put(None, **kwds) def test_put_nowait(self): - queue = queues.create(2) - queue.put_nowait(None) - queue.put_nowait(None) - with self.assertRaises(queues.QueueFull): - queue.put_nowait(None) - queue.get() - queue.put_nowait(None) + for syncobj in (True, False): + kwds = dict(syncobj=syncobj) + with self.subTest(f'syncobj={syncobj}'): + queue = queues.create(2) + queue.put_nowait(None, **kwds) + queue.put_nowait(None, **kwds) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None, **kwds) + queue.get() + queue.put_nowait(None, **kwds) + + def test_put_syncobj(self): + for obj in [ + None, + True, + 10, + 'spam', + b'spam', + (0, 'a'), + ]: + with self.subTest(repr(obj)): + queue = queues.create() + + queue.put(obj, syncobj=True) + obj2 = queue.get() + self.assertEqual(obj2, obj) + + queue.put(obj, syncobj=True) + obj2 = queue.get_nowait() + self.assertEqual(obj2, obj) + + for obj in [ + [1, 2, 3], + {'a': 13, 'b': 17}, + ]: + with self.subTest(repr(obj)): + queue = queues.create() + with self.assertRaises(interpreters.NotShareableError): + queue.put(obj, syncobj=True) + + def test_put_not_syncobj(self): + for obj in [ + None, + True, + 10, + 'spam', + b'spam', + (0, 'a'), + # not shareable + [1, 2, 3], + {'a': 13, 'b': 17}, + ]: + with self.subTest(repr(obj)): + queue = queues.create() + + queue.put(obj, syncobj=False) + obj2 = queue.get() + self.assertEqual(obj2, obj) + + queue.put(obj, syncobj=False) + obj2 = queue.get_nowait() + self.assertEqual(obj2, obj) def test_get_timeout(self): queue = queues.create() @@ -200,17 +261,54 @@ def test_get_nowait(self): with self.assertRaises(queues.QueueEmpty): queue.get_nowait() + def test_put_get_default_syncobj(self): + expected = list(range(20)) + queue = queues.create(syncobj=True) + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + get = getattr(queue, methname) + for i in range(20): + queue.put(i) + actual = [get() for _ in range(20)] + self.assertEqual(actual, expected) + + obj = [1, 2, 3] # lists are not shareable + with self.assertRaises(interpreters.NotShareableError): + queue.put(obj) + + def test_put_get_default_not_syncobj(self): + expected = list(range(20)) + queue = queues.create(syncobj=False) + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + get = getattr(queue, methname) + + for i in range(20): + queue.put(i) + actual = [get() for _ in range(20)] + self.assertEqual(actual, expected) + + obj = [1, 2, 3] # lists are not shareable + queue.put(obj) + obj2 = get() + self.assertEqual(obj, obj2) + self.assertIsNot(obj, obj2) + def test_put_get_same_interpreter(self): interp = interpreters.create() - interp.exec_sync(dedent(""" + interp.exec(dedent(""" from test.support.interpreters import queues queue = queues.create() - orig = b'spam' - queue.put(orig) - obj = queue.get() - assert obj == orig, 'expected: obj == orig' - assert obj is not orig, 'expected: obj is not orig' """)) + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + interp.exec(dedent(f""" + orig = b'spam' + queue.put(orig, syncobj=True) + obj = queue.{methname}() + assert obj == orig, 'expected: obj == orig' + assert obj is not orig, 'expected: obj is not orig' + """)) def test_put_get_different_interpreters(self): interp = interpreters.create() @@ -218,34 +316,37 @@ def test_put_get_different_interpreters(self): queue2 = queues.create() self.assertEqual(len(queues.list_all()), 2) - obj1 = b'spam' - queue1.put(obj1) - - out = _run_output( - interp, - dedent(f""" - from test.support.interpreters import queues - queue1 = queues.Queue({queue1.id}) - queue2 = queues.Queue({queue2.id}) - assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' - obj = queue1.get() - assert queue1.qsize() == 0, 'expected: queue1.qsize() == 0' - assert obj == b'spam', 'expected: obj == obj1' - # When going to another interpreter we get a copy. - assert id(obj) != {id(obj1)}, 'expected: obj is not obj1' - obj2 = b'eggs' - print(id(obj2)) - assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0' - queue2.put(obj2) - assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1' - """)) - self.assertEqual(len(queues.list_all()), 2) - self.assertEqual(queue1.qsize(), 0) - self.assertEqual(queue2.qsize(), 1) - - obj2 = queue2.get() - self.assertEqual(obj2, b'eggs') - self.assertNotEqual(id(obj2), int(out)) + for methname in ('get', 'get_nowait'): + with self.subTest(f'{methname}()'): + obj1 = b'spam' + queue1.put(obj1, syncobj=True) + + out = _run_output( + interp, + dedent(f""" + from test.support.interpreters import queues + queue1 = queues.Queue({queue1.id}) + queue2 = queues.Queue({queue2.id}) + assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' + obj = queue1.{methname}() + assert queue1.qsize() == 0, 'expected: queue1.qsize() == 0' + assert obj == b'spam', 'expected: obj == obj1' + # When going to another interpreter we get a copy. + assert id(obj) != {id(obj1)}, 'expected: obj is not obj1' + obj2 = b'eggs' + print(id(obj2)) + assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0' + queue2.put(obj2, syncobj=True) + assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1' + """)) + self.assertEqual(len(queues.list_all()), 2) + self.assertEqual(queue1.qsize(), 0) + self.assertEqual(queue2.qsize(), 1) + + get = getattr(queue2, methname) + obj2 = get() + self.assertEqual(obj2, b'eggs') + self.assertNotEqual(id(obj2), int(out)) def test_put_cleared_with_subinterpreter(self): interp = interpreters.create() @@ -258,8 +359,8 @@ def test_put_cleared_with_subinterpreter(self): queue = queues.Queue({queue.id}) obj1 = b'spam' obj2 = b'eggs' - queue.put(obj1) - queue.put(obj2) + queue.put(obj1, syncobj=True) + queue.put(obj2, syncobj=True) """)) self.assertEqual(queue.qsize(), 2) @@ -281,12 +382,12 @@ def f(): break except queues.QueueEmpty: continue - queue2.put(obj) + queue2.put(obj, syncobj=True) t = threading.Thread(target=f) t.start() orig = b'spam' - queue1.put(orig) + queue1.put(orig, syncobj=True) obj = queue2.get() t.join() diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 3a37ed09dd8943..973d05d4f96dcb 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -4,8 +4,9 @@ import subprocess import sys import tempfile -import threading from textwrap import dedent +import threading +import types import unittest from test import support @@ -41,7 +42,7 @@ def _run_output(interp, request, init=None): with rpipe: if init: interp.prepare_main(init) - interp.exec_sync(script) + interp.exec(script) return rpipe.read() @@ -49,7 +50,7 @@ def _run_output(interp, request, init=None): def _running(interp): r, w = os.pipe() def run(): - interp.exec_sync(dedent(f""" + interp.exec(dedent(f""" # wait for "signal" with open({r}) as rpipe: rpipe.read() @@ -84,6 +85,18 @@ def temp_dir(self): self.addCleanup(lambda: os_helper.rmtree(tempdir)) return tempdir + @contextlib.contextmanager + def captured_thread_exception(self): + ctx = types.SimpleNamespace(caught=None) + def excepthook(args): + ctx.caught = args + orig_excepthook = threading.excepthook + threading.excepthook = excepthook + try: + yield ctx + finally: + threading.excepthook = orig_excepthook + def make_script(self, filename, dirname=None, text=None): if text: text = dedent(text) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index cc387afa391909..5491c0575dbd3f 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -257,6 +257,27 @@ class PyMockUnseekableIO(MockUnseekableIO, pyio.BytesIO): UnsupportedOperation = pyio.UnsupportedOperation +class MockCharPseudoDevFileIO(MockFileIO): + # GH-95782 + # ftruncate() does not work on these special files (and CPython then raises + # appropriate exceptions), so truncate() does not have to be accounted for + # here. + def __init__(self, data): + super().__init__(data) + + def seek(self, *args): + return 0 + + def tell(self, *args): + return 0 + +class CMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, io.BytesIO): + pass + +class PyMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, pyio.BytesIO): + pass + + class MockNonBlockWriterIO: def __init__(self): @@ -1648,6 +1669,30 @@ def test_truncate_on_read_only(self): self.assertRaises(self.UnsupportedOperation, bufio.truncate) self.assertRaises(self.UnsupportedOperation, bufio.truncate, 0) + def test_tell_character_device_file(self): + # GH-95782 + # For the (former) bug in BufferedIO to manifest, the wrapped IO obj + # must be able to produce at least 2 bytes. + raw = self.MockCharPseudoDevFileIO(b"12") + buf = self.tp(raw) + self.assertEqual(buf.tell(), 0) + self.assertEqual(buf.read(1), b"1") + self.assertEqual(buf.tell(), 0) + + def test_seek_character_device_file(self): + raw = self.MockCharPseudoDevFileIO(b"12") + buf = self.tp(raw) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + self.assertEqual(buf.seek(1, io.SEEK_SET), 0) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + self.assertEqual(buf.read(1), b"1") + + # In the C implementation, tell() sets the BufferedIO's abs_pos to 0, + # which means that the next seek() could return a negative offset if it + # does not sanity-check: + self.assertEqual(buf.tell(), 0) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + class CBufferedReaderTest(BufferedReaderTest, SizeofTest): tp = io.BufferedReader @@ -4880,7 +4925,7 @@ def load_tests(loader, tests, pattern): # classes in the __dict__ of each test. mocks = (MockRawIO, MisbehavedRawIO, MockFileIO, CloseFailureIO, MockNonBlockWriterIO, MockUnseekableIO, MockRawIOWithoutRead, - SlowFlushRawIO) + SlowFlushRawIO, MockCharPseudoDevFileIO) all_members = io.__all__ c_io_ns = {name : getattr(io, name) for name in all_members} py_io_ns = {name : getattr(pyio, name) for name in all_members} diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 72dd40136cfdb2..e42df3d9496bc8 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -5,6 +5,7 @@ import os.path import tempfile import tokenize +from importlib.machinery import ModuleSpec from test import support from test.support import os_helper @@ -97,6 +98,16 @@ class BadUnicode_WithDeclaration(GetLineTestsBadData, unittest.TestCase): file_byte_string = b'# coding=utf-8\n\x80abc' +class FakeLoader: + def get_source(self, fullname): + return f'source for {fullname}' + + +class NoSourceLoader: + def get_source(self, fullname): + return None + + class LineCacheTests(unittest.TestCase): def test_getline(self): @@ -238,6 +249,33 @@ def raise_memoryerror(*args, **kwargs): self.assertEqual(lines3, []) self.assertEqual(linecache.getlines(FILENAME), lines) + def test_loader(self): + filename = 'scheme://path' + + for loader in (None, object(), NoSourceLoader()): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': loader} + self.assertEqual(linecache.getlines(filename, module_globals), []) + + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader()} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + + for spec in (None, object(), ModuleSpec('', FakeLoader())): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader(), + '__spec__': spec} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + + linecache.clearcache() + spec = ModuleSpec('x.y.z', FakeLoader()) + module_globals = {'__name__': 'a.b.c', '__loader__': spec.loader, + '__spec__': spec} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for x.y.z\n']) + class LineCacheInvalidationTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index cf09bad4c9187b..d71385bf2c78d7 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -6080,6 +6080,73 @@ def test_rollover(self): print(tf.read()) self.assertTrue(found, msg=msg) + def test_rollover_at_midnight(self): + atTime = datetime.datetime.now().time() + fmt = logging.Formatter('%(asctime)s %(message)s') + for i in range(3): + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', atTime=atTime) + fh.setFormatter(fmt) + r2 = logging.makeLogRecord({'msg': f'testing1 {i}'}) + fh.emit(r2) + fh.close() + self.assertLogFile(self.fn) + with open(self.fn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing1 {i}', line) + + os.utime(self.fn, (time.time() - 1,)*2) + for i in range(2): + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', atTime=atTime) + fh.setFormatter(fmt) + r2 = logging.makeLogRecord({'msg': f'testing2 {i}'}) + fh.emit(r2) + fh.close() + rolloverDate = datetime.datetime.now() - datetime.timedelta(days=1) + otherfn = f'{self.fn}.{rolloverDate:%Y-%m-%d}' + self.assertLogFile(otherfn) + with open(self.fn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing2 {i}', line) + with open(otherfn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing1 {i}', line) + + def test_rollover_at_weekday(self): + now = datetime.datetime.now() + atTime = now.time() + fmt = logging.Formatter('%(asctime)s %(message)s') + for i in range(3): + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when=f'W{now.weekday()}', atTime=atTime) + fh.setFormatter(fmt) + r2 = logging.makeLogRecord({'msg': f'testing1 {i}'}) + fh.emit(r2) + fh.close() + self.assertLogFile(self.fn) + with open(self.fn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing1 {i}', line) + + os.utime(self.fn, (time.time() - 1,)*2) + for i in range(2): + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when=f'W{now.weekday()}', atTime=atTime) + fh.setFormatter(fmt) + r2 = logging.makeLogRecord({'msg': f'testing2 {i}'}) + fh.emit(r2) + fh.close() + rolloverDate = datetime.datetime.now() - datetime.timedelta(days=7) + otherfn = f'{self.fn}.{rolloverDate:%Y-%m-%d}' + self.assertLogFile(otherfn) + with open(self.fn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing2 {i}', line) + with open(otherfn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing1 {i}', line) + def test_invalid(self): assertRaises = self.assertRaises assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, @@ -6089,22 +6156,47 @@ def test_invalid(self): assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, self.fn, 'W7', encoding="utf-8", delay=True) + # TODO: Test for utc=False. def test_compute_rollover_daily_attime(self): currentTime = 0 + rh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', + utc=True, atTime=None) + try: + actual = rh.computeRollover(currentTime) + self.assertEqual(actual, currentTime + 24 * 60 * 60) + + actual = rh.computeRollover(currentTime + 24 * 60 * 60 - 1) + self.assertEqual(actual, currentTime + 24 * 60 * 60) + + actual = rh.computeRollover(currentTime + 24 * 60 * 60) + self.assertEqual(actual, currentTime + 48 * 60 * 60) + + actual = rh.computeRollover(currentTime + 25 * 60 * 60) + self.assertEqual(actual, currentTime + 48 * 60 * 60) + finally: + rh.close() + atTime = datetime.time(12, 0, 0) rh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when='MIDNIGHT', interval=1, backupCount=0, + self.fn, encoding="utf-8", when='MIDNIGHT', utc=True, atTime=atTime) try: actual = rh.computeRollover(currentTime) self.assertEqual(actual, currentTime + 12 * 60 * 60) + actual = rh.computeRollover(currentTime + 12 * 60 * 60 - 1) + self.assertEqual(actual, currentTime + 12 * 60 * 60) + + actual = rh.computeRollover(currentTime + 12 * 60 * 60) + self.assertEqual(actual, currentTime + 36 * 60 * 60) + actual = rh.computeRollover(currentTime + 13 * 60 * 60) self.assertEqual(actual, currentTime + 36 * 60 * 60) finally: rh.close() - #@unittest.skipIf(True, 'Temporarily skipped while failures investigated.') + # TODO: Test for utc=False. def test_compute_rollover_weekly_attime(self): currentTime = int(time.time()) today = currentTime - currentTime % 86400 @@ -6129,14 +6221,28 @@ def test_compute_rollover_weekly_attime(self): expected += 12 * 60 * 60 # Add in adjustment for today expected += today + actual = rh.computeRollover(today) if actual != expected: print('failed in timezone: %d' % time.timezone) print('local vars: %s' % locals()) self.assertEqual(actual, expected) + + actual = rh.computeRollover(today + 12 * 60 * 60 - 1) + if actual != expected: + print('failed in timezone: %d' % time.timezone) + print('local vars: %s' % locals()) + self.assertEqual(actual, expected) + if day == wday: # goes into following week expected += 7 * 24 * 60 * 60 + actual = rh.computeRollover(today + 12 * 60 * 60) + if actual != expected: + print('failed in timezone: %d' % time.timezone) + print('local vars: %s' % locals()) + self.assertEqual(actual, expected) + actual = rh.computeRollover(today + 13 * 60 * 60) if actual != expected: print('failed in timezone: %d' % time.timezone) @@ -6154,7 +6260,7 @@ def test_compute_files_to_delete(self): for i in range(10): times.append(dt.strftime('%Y-%m-%d_%H-%M-%S')) dt += datetime.timedelta(seconds=5) - prefixes = ('a.b', 'a.b.c', 'd.e', 'd.e.f') + prefixes = ('a.b', 'a.b.c', 'd.e', 'd.e.f', 'g') files = [] rotators = [] for prefix in prefixes: @@ -6167,10 +6273,22 @@ def test_compute_files_to_delete(self): if prefix.startswith('a.b'): for t in times: files.append('%s.log.%s' % (prefix, t)) - else: - rotator.namer = lambda name: name.replace('.log', '') + '.log' + elif prefix.startswith('d.e'): + def namer(filename): + dirname, basename = os.path.split(filename) + basename = basename.replace('.log', '') + '.log' + return os.path.join(dirname, basename) + rotator.namer = namer for t in times: files.append('%s.%s.log' % (prefix, t)) + elif prefix == 'g': + def namer(filename): + dirname, basename = os.path.split(filename) + basename = 'g' + basename[6:] + '.oldlog' + return os.path.join(dirname, basename) + rotator.namer = namer + for t in times: + files.append('g%s.oldlog' % t) # Create empty files for fn in files: p = os.path.join(wd, fn) @@ -6180,19 +6298,301 @@ def test_compute_files_to_delete(self): for i, prefix in enumerate(prefixes): rotator = rotators[i] candidates = rotator.getFilesToDelete() - self.assertEqual(len(candidates), 3) + self.assertEqual(len(candidates), 3, candidates) if prefix.startswith('a.b'): p = '%s.log.' % prefix for c in candidates: d, fn = os.path.split(c) self.assertTrue(fn.startswith(p)) - else: + elif prefix.startswith('d.e'): for c in candidates: d, fn = os.path.split(c) - self.assertTrue(fn.endswith('.log')) + self.assertTrue(fn.endswith('.log'), fn) self.assertTrue(fn.startswith(prefix + '.') and fn[len(prefix) + 2].isdigit()) + elif prefix == 'g': + for c in candidates: + d, fn = os.path.split(c) + self.assertTrue(fn.endswith('.oldlog')) + self.assertTrue(fn.startswith('g') and fn[1].isdigit()) + + def test_compute_files_to_delete_same_filename_different_extensions(self): + # See GH-93205 for background + wd = pathlib.Path(tempfile.mkdtemp(prefix='test_logging_')) + self.addCleanup(shutil.rmtree, wd) + times = [] + dt = datetime.datetime.now() + n_files = 10 + for _ in range(n_files): + times.append(dt.strftime('%Y-%m-%d_%H-%M-%S')) + dt += datetime.timedelta(seconds=5) + prefixes = ('a.log', 'a.log.b') + files = [] + rotators = [] + for i, prefix in enumerate(prefixes): + backupCount = i+1 + rotator = logging.handlers.TimedRotatingFileHandler(wd / prefix, when='s', + interval=5, + backupCount=backupCount, + delay=True) + rotators.append(rotator) + for t in times: + files.append('%s.%s' % (prefix, t)) + for t in times: + files.append('a.log.%s.c' % t) + # Create empty files + for f in files: + (wd / f).touch() + # Now the checks that only the correct files are offered up for deletion + for i, prefix in enumerate(prefixes): + backupCount = i+1 + rotator = rotators[i] + candidates = rotator.getFilesToDelete() + self.assertEqual(len(candidates), n_files - backupCount, candidates) + matcher = re.compile(r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\Z") + for c in candidates: + d, fn = os.path.split(c) + self.assertTrue(fn.startswith(prefix+'.')) + suffix = fn[(len(prefix)+1):] + self.assertRegex(suffix, matcher) + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_MIDNIGHT_local(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False) + + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 3, 12, 0, 0)) + + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 5, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(12, 0, 0)) + test(DT(2012, 3, 10, 11, 59, 59), DT(2012, 3, 10, 12, 0)) + test(DT(2012, 3, 10, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 10, 13, 0), DT(2012, 3, 11, 12, 0)) + + test(DT(2012, 11, 3, 11, 59, 59), DT(2012, 11, 3, 12, 0)) + test(DT(2012, 11, 3, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 11, 3, 13, 0), DT(2012, 11, 4, 12, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(2, 0, 0)) + + test(DT(2012, 3, 10, 1, 59, 59), DT(2012, 3, 10, 2, 0)) + # 2:00:00 is the same as 3:00:00 at 2012-3-11. + test(DT(2012, 3, 10, 2, 0), DT(2012, 3, 11, 3, 0)) + test(DT(2012, 3, 10, 3, 0), DT(2012, 3, 11, 3, 0)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 0)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 12, 2, 0)) + test(DT(2012, 3, 11, 4, 0), DT(2012, 3, 12, 2, 0)) + + test(DT(2012, 11, 3, 1, 59, 59), DT(2012, 11, 3, 2, 0)) + test(DT(2012, 11, 3, 2, 0), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 3, 3, 0), DT(2012, 11, 4, 2, 0)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 5, 2, 0)) + test(DT(2012, 11, 4, 3, 0), DT(2012, 11, 5, 2, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(2, 30, 0)) + + test(DT(2012, 3, 10, 2, 29, 59), DT(2012, 3, 10, 2, 30)) + # No time 2:30:00 at 2012-3-11. + test(DT(2012, 3, 10, 2, 30), DT(2012, 3, 11, 3, 30)) + test(DT(2012, 3, 10, 3, 0), DT(2012, 3, 11, 3, 30)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 12, 2, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 12, 2, 30)) + + test(DT(2012, 11, 3, 2, 29, 59), DT(2012, 11, 3, 2, 30)) + test(DT(2012, 11, 3, 2, 30), DT(2012, 11, 4, 2, 30)) + test(DT(2012, 11, 3, 3, 0), DT(2012, 11, 4, 2, 30)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(1, 30, 0)) + + test(DT(2012, 3, 11, 1, 29, 59), DT(2012, 3, 11, 1, 30)) + test(DT(2012, 3, 11, 1, 30), DT(2012, 3, 12, 1, 30)) + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 12, 1, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 12, 1, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 12, 1, 30)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 29, 59), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 30), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 5, 1, 30)) + # It is weird, but the rollover date jumps back from 2012-11-5 + # to 2012-11-4. + test(DT(2012, 11, 4, 1, 0, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 29, 59, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 30, fold=1), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 2, 30), DT(2012, 11, 5, 1, 30)) + + fh.close() + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_W6_local(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False) + + test(DT(2012, 3, 4, 23, 59, 59), DT(2012, 3, 5, 0, 0)) + test(DT(2012, 3, 5, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 5, 1, 0), DT(2012, 3, 12, 0, 0)) + + test(DT(2012, 10, 28, 23, 59, 59), DT(2012, 10, 29, 0, 0)) + test(DT(2012, 10, 29, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 10, 29, 1, 0), DT(2012, 11, 5, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(0, 0, 0)) + + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 3, 18, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 3, 18, 0, 0)) + + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 11, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 11, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 3, 4, 11, 59, 59), DT(2012, 3, 4, 12, 0)) + test(DT(2012, 3, 4, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 4, 13, 0), DT(2012, 3, 11, 12, 0)) + + test(DT(2012, 10, 28, 11, 59, 59), DT(2012, 10, 28, 12, 0)) + test(DT(2012, 10, 28, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 10, 28, 13, 0), DT(2012, 11, 4, 12, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(2, 0, 0)) + + test(DT(2012, 3, 4, 1, 59, 59), DT(2012, 3, 4, 2, 0)) + # 2:00:00 is the same as 3:00:00 at 2012-3-11. + test(DT(2012, 3, 4, 2, 0), DT(2012, 3, 11, 3, 0)) + test(DT(2012, 3, 4, 3, 0), DT(2012, 3, 11, 3, 0)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 0)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 18, 2, 0)) + test(DT(2012, 3, 11, 4, 0), DT(2012, 3, 18, 2, 0)) + + test(DT(2012, 10, 28, 1, 59, 59), DT(2012, 10, 28, 2, 0)) + test(DT(2012, 10, 28, 2, 0), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 10, 28, 3, 0), DT(2012, 11, 4, 2, 0)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 11, 2, 0)) + test(DT(2012, 11, 4, 3, 0), DT(2012, 11, 11, 2, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(2, 30, 0)) + + test(DT(2012, 3, 4, 2, 29, 59), DT(2012, 3, 4, 2, 30)) + # No time 2:30:00 at 2012-3-11. + test(DT(2012, 3, 4, 2, 30), DT(2012, 3, 11, 3, 30)) + test(DT(2012, 3, 4, 3, 0), DT(2012, 3, 11, 3, 30)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 18, 2, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 18, 2, 30)) + + test(DT(2012, 10, 28, 2, 29, 59), DT(2012, 10, 28, 2, 30)) + test(DT(2012, 10, 28, 2, 30), DT(2012, 11, 4, 2, 30)) + test(DT(2012, 10, 28, 3, 0), DT(2012, 11, 4, 2, 30)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(1, 30, 0)) + + test(DT(2012, 3, 11, 1, 29, 59), DT(2012, 3, 11, 1, 30)) + test(DT(2012, 3, 11, 1, 30), DT(2012, 3, 18, 1, 30)) + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 18, 1, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 18, 1, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 18, 1, 30)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 29, 59), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 30), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 11, 1, 30)) + # It is weird, but the rollover date jumps back from 2012-11-11 + # to 2012-11-4. + test(DT(2012, 11, 4, 1, 0, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 29, 59, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 30, fold=1), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 2, 30), DT(2012, 11, 11, 1, 30)) + + fh.close() def secs(**kw): return datetime.timedelta(**kw) // datetime.timedelta(seconds=1) diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index 65e6488c5d7b10..db290e139327e0 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -2,7 +2,6 @@ import array from io import BytesIO, UnsupportedOperation, DEFAULT_BUFFER_SIZE import os -import pathlib import pickle import random import sys @@ -12,7 +11,7 @@ from test.support import _4G, bigmemtest from test.support.import_helper import import_module from test.support.os_helper import ( - TESTFN, unlink + TESTFN, unlink, FakePath ) lzma = import_module("lzma") @@ -548,7 +547,7 @@ def test_init(self): pass def test_init_with_PathLike_filename(self): - filename = pathlib.Path(TESTFN) + filename = FakePath(TESTFN) with TempFile(filename, COMPRESSED_XZ): with LZMAFile(filename) as f: self.assertEqual(f.read(), INPUT) @@ -585,11 +584,10 @@ def test_init_with_x_mode(self): self.addCleanup(unlink, TESTFN) for mode in ("x", "xb"): unlink(TESTFN) - with LZMAFile(TESTFN, mode): + with LZMAFile(TESTFN, mode) as f: pass with self.assertRaises(FileExistsError): - with LZMAFile(TESTFN, mode): - pass + LZMAFile(TESTFN, mode) def test_init_bad_mode(self): with self.assertRaises(ValueError): @@ -867,17 +865,59 @@ def test_read_from_file(self): with LZMAFile(TESTFN) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) def test_read_from_file_with_bytes_filename(self): - try: - bytes_filename = TESTFN.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(TESTFN) with TempFile(TESTFN, COMPRESSED_XZ): with LZMAFile(bytes_filename) as f: self.assertEqual(f.read(), INPUT) self.assertEqual(f.read(), b"") + def test_read_from_fileobj(self): + with TempFile(TESTFN, COMPRESSED_XZ): + with open(TESTFN, 'rb') as raw: + with LZMAFile(raw) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def test_read_from_fileobj_with_int_name(self): + with TempFile(TESTFN, COMPRESSED_XZ): + fd = os.open(TESTFN, os.O_RDONLY) + with open(fd, 'rb') as raw: + with LZMAFile(raw) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + def test_read_incomplete(self): with LZMAFile(BytesIO(COMPRESSED_XZ[:128])) as f: self.assertRaises(EOFError, f.read) @@ -1051,6 +1091,17 @@ def test_write_to_file(self): try: with LZMAFile(TESTFN, "w") as f: f.write(INPUT) + self.assertIsInstance(f.fileno(), int) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + expected = lzma.compress(INPUT) with open(TESTFN, "rb") as f: self.assertEqual(f.read(), expected) @@ -1058,10 +1109,7 @@ def test_write_to_file(self): unlink(TESTFN) def test_write_to_file_with_bytes_filename(self): - try: - bytes_filename = TESTFN.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(TESTFN) try: with LZMAFile(bytes_filename, "w") as f: f.write(INPUT) @@ -1071,6 +1119,51 @@ def test_write_to_file_with_bytes_filename(self): finally: unlink(TESTFN) + def test_write_to_fileobj(self): + try: + with open(TESTFN, "wb") as raw: + with LZMAFile(raw, "w") as f: + f.write(INPUT) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_to_fileobj_with_int_name(self): + try: + fd = os.open(TESTFN, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with open(fd, 'wb') as raw: + with LZMAFile(raw, "w") as f: + f.write(INPUT) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + def test_write_append_to_file(self): part1 = INPUT[:1024] part2 = INPUT[1024:1536] @@ -1276,7 +1369,7 @@ def test_filename(self): self.assertEqual(f.read(), INPUT * 2) def test_with_pathlike_filename(self): - filename = pathlib.Path(TESTFN) + filename = FakePath(TESTFN) with TempFile(filename): with lzma.open(filename, "wb") as f: f.write(INPUT) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 60b6326bfbad5e..1e77eb6a2eea4c 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -9,6 +9,8 @@ import types import unittest import asyncio +from test import support +from test.support import requires_specialization, script_helper PAIR = (0,1) @@ -35,6 +37,9 @@ def g1(): TEST_TOOL2 = 3 TEST_TOOL3 = 4 +def nth_line(func, offset): + return func.__code__.co_firstlineno + offset + class MonitoringBasicTest(unittest.TestCase): def test_has_objects(self): @@ -529,8 +534,8 @@ def test_lines_single(self): f1() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - start = LineMonitoringTest.test_lines_single.__code__.co_firstlineno - self.assertEqual(events, [start+7, 16, start+8]) + start = nth_line(LineMonitoringTest.test_lines_single, 0) + self.assertEqual(events, [start+7, nth_line(f1, 1), start+8]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) @@ -547,8 +552,13 @@ def test_lines_loop(self): floop() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - start = LineMonitoringTest.test_lines_loop.__code__.co_firstlineno - self.assertEqual(events, [start+7, 23, 24, 23, 24, 23, start+8]) + start = nth_line(LineMonitoringTest.test_lines_loop, 0) + floop_1 = nth_line(floop, 1) + floop_2 = nth_line(floop, 2) + self.assertEqual( + events, + [start+7, floop_1, floop_2, floop_1, floop_2, floop_1, start+8] + ) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) @@ -569,8 +579,8 @@ def test_lines_two(self): sys.monitoring.set_events(TEST_TOOL, 0); sys.monitoring.set_events(TEST_TOOL2, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) - start = LineMonitoringTest.test_lines_two.__code__.co_firstlineno - expected = [start+10, 16, start+11] + start = nth_line(LineMonitoringTest.test_lines_two, 0) + expected = [start+10, nth_line(f1, 1), start+11] self.assertEqual(events, expected) self.assertEqual(events2, expected) finally: @@ -807,6 +817,9 @@ def func1(): self.check_events(func1, [("raise", KeyError)]) + # gh-116090: This test doesn't really require specialization, but running + # it without specialization exposes a monitoring bug. + @requires_specialization def test_implicit_stop_iteration(self): def gen(): @@ -955,6 +968,7 @@ def func(): ) self.assertEqual(events[0], ("throw", IndexError)) + @requires_specialization def test_no_unwind_for_shim_frame(self): class B: @@ -1799,7 +1813,7 @@ class TestOptimizer(MonitoringTestBase, unittest.TestCase): def setUp(self): import _testinternalcapi self.old_opt = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.get_counter_optimizer() + opt = _testinternalcapi.new_counter_optimizer() _testinternalcapi.set_optimizer(opt) super(TestOptimizer, self).setUp() @@ -1845,3 +1859,12 @@ def test_func(recorder): sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) sys.monitoring.set_events(TEST_TOOL, 0) self.assertGreater(len(events), 250) + +class TestMonitoringAtShutdown(unittest.TestCase): + + def test_monitoring_live_at_shutdown(self): + # gh-115832: An object destructor running during the final GC of + # interpreter shutdown triggered an infinite loop in the + # instrumentation code. + script = support.findfile("_test_monitoring_shutdown.py") + script_helper.run_test_script(script) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 7b2fa844827ae9..f2017bdffcf968 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -298,6 +298,16 @@ def test_named_expression_invalid_set_comprehension_iterable_expression(self): with self.assertRaisesRegex(SyntaxError, msg): exec(f"lambda: {code}", {}) # Function scope + def test_named_expression_invalid_mangled_class_variables(self): + code = """class Foo: + def bar(self): + [[(__x:=2) for _ in range(2)] for __x in range(2)] + """ + + with self.assertRaisesRegex(SyntaxError, + "assignment expression cannot rebind comprehension iteration variable '__x'"): + exec(code, {}, {}) + class NamedExpressionAssignmentTest(unittest.TestCase): @@ -674,6 +684,18 @@ def test_named_expression_scope_in_genexp(self): for idx, elem in enumerate(genexp): self.assertEqual(elem, b[idx] + a) + def test_named_expression_scope_mangled_names(self): + class Foo: + def f(self_): + global __x1 + __x1 = 0 + [_Foo__x1 := 1 for a in [2]] + self.assertEqual(__x1, 1) + [__x1 := 2 for a in [3]] + self.assertEqual(__x1, 2) + + Foo().f() + self.assertEqual(_Foo__x1, 2) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 2b2783d57be8f4..5fb4b815c95d07 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -4,7 +4,7 @@ import threading import types import unittest -from test.support import threading_helper, check_impl_detail +from test.support import threading_helper, check_impl_detail, requires_specialization # Skip this module on other interpreters, it is cpython specific: if check_impl_detail(cpython=False): @@ -506,6 +506,7 @@ def f(x, y): @threading_helper.requires_working_threading() +@requires_specialization class TestRacesDoNotCrash(unittest.TestCase): # Careful with these. Bigger numbers have a higher chance of catching bugs, # but you can also burn through a *ton* of type/dict/function versions: @@ -1021,6 +1022,7 @@ def write(items): class C: pass +@requires_specialization class TestInstanceDict(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_optimizer.py b/Lib/test/test_optimizer.py index dfea8be3c6956f..899a4507317334 100644 --- a/Lib/test/test_optimizer.py +++ b/Lib/test/test_optimizer.py @@ -77,5 +77,12 @@ def func(x=0): _testinternalcapi.get_rare_event_counters()["func_modification"] ) + +class TestOptimizerSymbols(unittest.TestCase): + + def test_optimizer_symbols(self): + _testinternalcapi.uop_symbols_test() + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 2c8823ae47c726..3b2d5fccc30f3c 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1592,6 +1592,9 @@ def test_yields_correct_dir_fd(self): @unittest.skipIf( support.is_emscripten, "Cannot dup stdout on Emscripten" ) + @unittest.skipIf( + support.is_android, "dup return value is unpredictable on Android" + ) def test_fd_leak(self): # Since we're opening a lot of FDs, we must be careful to avoid leaks: # we both check that calling fwalk() a large number of times doesn't @@ -2492,8 +2495,10 @@ def test_listdir(self): # test listdir without arguments current_directory = os.getcwd() try: - os.chdir(os.sep) - self.assertEqual(set(os.listdir()), set(os.listdir(os.sep))) + # The root directory is not readable on Android, so use a directory + # we created ourselves. + os.chdir(self.dir) + self.assertEqual(set(os.listdir()), expected) finally: os.chdir(current_directory) @@ -3506,22 +3511,22 @@ class ProgramPriorityTests(unittest.TestCase): """Tests for os.getpriority() and os.setpriority().""" def test_set_get_priority(self): - base = os.getpriority(os.PRIO_PROCESS, os.getpid()) - os.setpriority(os.PRIO_PROCESS, os.getpid(), base + 1) - try: - new_prio = os.getpriority(os.PRIO_PROCESS, os.getpid()) - if base >= 19 and new_prio <= 19: - raise unittest.SkipTest("unable to reliably test setpriority " - "at current nice level of %s" % base) - else: - self.assertEqual(new_prio, base + 1) - finally: - try: - os.setpriority(os.PRIO_PROCESS, os.getpid(), base) - except OSError as err: - if err.errno != errno.EACCES: - raise + code = f"""if 1: + import os + os.setpriority(os.PRIO_PROCESS, os.getpid(), {base} + 1) + print(os.getpriority(os.PRIO_PROCESS, os.getpid())) + """ + + # Subprocess inherits the current process' priority. + _, out, _ = assert_python_ok("-c", code) + new_prio = int(out) + # nice value cap is 19 for linux and 20 for FreeBSD + if base >= 19 and new_prio <= base: + raise unittest.SkipTest("unable to reliably test setpriority " + "at current nice level of %s" % base) + else: + self.assertEqual(new_prio, base + 1) @unittest.skipUnless(hasattr(os, 'sendfile'), "test needs os.sendfile()") @@ -4838,7 +4843,7 @@ def check_entry(self, entry, name, is_dir, is_file, is_symlink): os.name == 'nt') def test_attributes(self): - link = hasattr(os, 'link') + link = os_helper.can_hardlink() symlink = os_helper.can_symlink() dirname = os.path.join(self.path, "dir") diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index c0dcf314da4bfc..7e44ae61a5eba7 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -796,7 +796,7 @@ def test_rmdir(self): self.assertFileNotFound(p.stat) self.assertFileNotFound(p.unlink) - @unittest.skipUnless(hasattr(os, "link"), "os.link() is not present") + @os_helper.skip_unless_hardlink def test_hardlink_to(self): P = self.cls(self.base) target = P / 'fileA' diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 1d30deca8f7a1b..fb467a015a80d2 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -957,6 +957,8 @@ def test_with_stem_empty(self): self.assertEqual(P('/').with_stem('d'), P('/d')) self.assertEqual(P('a/b').with_stem(''), P('a/')) self.assertEqual(P('a/b').with_stem('.'), P('a/.')) + self.assertRaises(ValueError, P('foo.gz').with_stem, '') + self.assertRaises(ValueError, P('/a/b/foo.gz').with_stem, '') def test_with_stem_seps(self): P = self.cls @@ -1842,51 +1844,46 @@ def _check(path, glob, expected): _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) _check(p, "*/dirD/**/", ["dirC/dirD/"]) - def test_rglob_common(self): - def _check(glob, expected): - self.assertEqual(set(glob), {P(self.base, q) for q in expected}) + def test_rglob_follow_symlinks_none(self): + def _check(path, glob, expected): + actual = set(path.rglob(glob, follow_symlinks=None)) + self.assertEqual(actual, { P(self.base, q) for q in expected }) P = self.cls p = P(self.base) it = p.rglob("fileA") self.assertIsInstance(it, collections.abc.Iterator) - _check(it, ["fileA"]) - _check(p.rglob("fileB"), ["dirB/fileB"]) - _check(p.rglob("**/fileB"), ["dirB/fileB"]) - _check(p.rglob("*/fileA"), []) - if not self.can_symlink: - _check(p.rglob("*/fileB"), ["dirB/fileB"]) - else: - _check(p.rglob("*/fileB"), ["dirB/fileB", "dirB/linkD/fileB", - "linkB/fileB", "dirA/linkC/fileB"]) - _check(p.rglob("file*"), ["fileA", "dirB/fileB", - "dirC/fileC", "dirC/dirD/fileD"]) - if not self.can_symlink: - _check(p.rglob("*/"), [ - "dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/", - ]) - else: - _check(p.rglob("*/"), [ + _check(p, "fileA", ["fileA"]) + _check(p, "fileB", ["dirB/fileB"]) + _check(p, "**/fileB", ["dirB/fileB"]) + _check(p, "*/fileA", []) + + if self.can_symlink: + _check(p, "*/fileB", ["dirB/fileB", "dirB/linkD/fileB", + "linkB/fileB", "dirA/linkC/fileB"]) + _check(p, "*/", [ "dirA/", "dirA/linkC/", "dirB/", "dirB/linkD/", "dirC/", - "dirC/dirD/", "dirE/", "linkB/", - ]) - _check(p.rglob(""), ["", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) + "dirC/dirD/", "dirE/", "linkB/"]) + else: + _check(p, "*/fileB", ["dirB/fileB"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) + _check(p, "file*", ["fileA", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "", ["", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) p = P(self.base, "dirC") - _check(p.rglob("*"), ["dirC/fileC", "dirC/novel.txt", + _check(p, "*", ["dirC/fileC", "dirC/novel.txt", "dirC/dirD", "dirC/dirD/fileD"]) - _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("dir*/**"), ["dirC/dirD/", "dirC/dirD/fileD"]) - _check(p.rglob("dir*/**/"), ["dirC/dirD/"]) - _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) - _check(p.rglob("*/"), ["dirC/dirD/"]) - _check(p.rglob(""), ["dirC/", "dirC/dirD/"]) - _check(p.rglob("**"), [ - "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt"]) - _check(p.rglob("**/"), ["dirC/", "dirC/dirD/"]) + _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "**/file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "dir*/**", ["dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "dir*/**/", ["dirC/dirD/"]) + _check(p, "*/*", ["dirC/dirD/fileD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) + _check(p, "**", ["dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt"]) + _check(p, "**/", ["dirC/", "dirC/dirD/"]) # gh-91616, a re module regression - _check(p.rglob("*.txt"), ["dirC/novel.txt"]) - _check(p.rglob("*.*"), ["dirC/novel.txt"]) + _check(p, "*.txt", ["dirC/novel.txt"]) + _check(p, "*.*", ["dirC/novel.txt"]) @needs_posix def test_rglob_posix(self): @@ -1967,7 +1964,7 @@ def test_rglob_symlink_loop(self): # Don't get fooled by symlink loops (Issue #26012). P = self.cls p = P(self.base) - given = set(p.rglob('*')) + given = set(p.rglob('*', follow_symlinks=None)) expect = {'brokenLink', 'dirA', 'dirA/linkC', 'dirB', 'dirB/fileB', 'dirB/linkD', diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 2b0795cdad707e..3dd275caf43200 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2695,6 +2695,18 @@ def bœr(): ('bœr', 2), ) + def test_spec(self): + # Test that __main__.__spec__ is set to None when running a script + script = """ + import __main__ + print(__main__.__spec__) + """ + + commands = "continue" + + stdout, _ = self.run_pdb_script(script, commands) + self.assertIn('None', stdout) + def test_find_function_first_executable_line(self): code = textwrap.dedent("""\ def foo(): pass diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index a45f620e18dc1d..2706d5eb6d9830 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1270,9 +1270,10 @@ def test_get_and_set_scheduler_and_param(self): self.assertIn(mine, possible_schedulers) try: parent = posix.sched_getscheduler(os.getppid()) - except OSError as e: - if e.errno != errno.EPERM: - raise + except PermissionError: + # POSIX specifies EPERM, but Android returns EACCES. Both errno + # values are mapped to PermissionError. + pass else: self.assertIn(parent, possible_schedulers) self.assertRaises(OSError, posix.sched_getscheduler, -1) @@ -1287,9 +1288,8 @@ def test_get_and_set_scheduler_and_param(self): try: posix.sched_setscheduler(0, mine, param) posix.sched_setparam(0, param) - except OSError as e: - if e.errno != errno.EPERM: - raise + except PermissionError: + pass self.assertRaises(OSError, posix.sched_setparam, -1, param) self.assertRaises(OSError, posix.sched_setscheduler, -1, mine, param) diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 86ce1b1d41ba61..cbb7c4c52d9697 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -703,7 +703,9 @@ def check_error(exc, paths): self.assertRaises(exc, posixpath.commonpath, [os.fsencode(p) for p in paths]) + self.assertRaises(TypeError, posixpath.commonpath, None) self.assertRaises(ValueError, posixpath.commonpath, []) + self.assertRaises(ValueError, posixpath.commonpath, iter([])) check_error(ValueError, ['/usr', 'usr']) check_error(ValueError, ['usr', '/usr']) diff --git a/Lib/test/test_profile.py b/Lib/test/test_profile.py index a1dfc9abbb8ef7..0f16b92334999c 100644 --- a/Lib/test/test_profile.py +++ b/Lib/test/test_profile.py @@ -7,7 +7,7 @@ from difflib import unified_diff from io import StringIO from test.support.os_helper import TESTFN, unlink, temp_dir, change_cwd -from contextlib import contextmanager +from contextlib import contextmanager, redirect_stdout import profile from test.profilee import testfunc, timer @@ -92,6 +92,11 @@ def test_run(self): self.profilermodule.run("int('1')", filename=TESTFN) self.assertTrue(os.path.exists(TESTFN)) + def test_run_with_sort_by_values(self): + with redirect_stdout(StringIO()) as f: + self.profilermodule.run("int('1')", sort=('tottime', 'stdname')) + self.assertIn("Ordered by: internal time, standard name", f.getvalue()) + def test_runctx(self): with silent(): self.profilermodule.runctx("testfunc()", globals(), locals()) diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py index 8ace9fd17ab96e..408e64f53142db 100644 --- a/Lib/test/test_property.py +++ b/Lib/test/test_property.py @@ -183,6 +183,77 @@ def test_refleaks_in___init__(self): fake_prop.__init__('fget', 'fset', 'fdel', 'doc') self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + @support.refcount_test + def test_gh_115618(self): + # Py_XDECREF() was improperly called for None argument + # in property methods. + gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') + prop = property() + refs_before = gettotalrefcount() + for i in range(100): + prop = prop.getter(None) + self.assertIsNone(prop.fget) + for i in range(100): + prop = prop.setter(None) + self.assertIsNone(prop.fset) + for i in range(100): + prop = prop.deleter(None) + self.assertIsNone(prop.fdel) + self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + + def test_property_name(self): + def getter(self): + return 42 + + def setter(self, value): + pass + + class A: + @property + def foo(self): + return 1 + + @foo.setter + def oof(self, value): + pass + + bar = property(getter) + baz = property(None, setter) + + self.assertEqual(A.foo.__name__, 'foo') + self.assertEqual(A.oof.__name__, 'oof') + self.assertEqual(A.bar.__name__, 'bar') + self.assertEqual(A.baz.__name__, 'baz') + + A.quux = property(getter) + self.assertEqual(A.quux.__name__, 'getter') + A.quux.__name__ = 'myquux' + self.assertEqual(A.quux.__name__, 'myquux') + self.assertEqual(A.bar.__name__, 'bar') # not affected + A.quux.__name__ = None + self.assertIsNone(A.quux.__name__) + + with self.assertRaisesRegex( + AttributeError, "'property' object has no attribute '__name__'" + ): + property(None, setter).__name__ + + with self.assertRaisesRegex( + AttributeError, "'property' object has no attribute '__name__'" + ): + property(1).__name__ + + class Err: + def __getattr__(self, attr): + raise RuntimeError('fail') + + p = property(Err()) + with self.assertRaisesRegex(RuntimeError, 'fail'): + p.__name__ + + p.__name__ = 'not_fail' + self.assertEqual(p.__name__, 'not_fail') + def test_property_set_name_incorrect_args(self): p = property() diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index 3f2bac0155fd9e..dee94533c74549 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,7 +1,6 @@ -import sys import unittest from test.support import ( - is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose + is_android, is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -9,9 +8,8 @@ # Skip these tests if termios is not available import_module('termios') -# Skip tests on WASM platforms, plus iOS/tvOS/watchOS -if is_apple_mobile or is_emscripten or is_wasi: - raise unittest.SkipTest(f"pty tests not required on {sys.platform}") +if is_android or is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest("pty is not available on this platform") import errno import os diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index c4e6551f605782..64387296e84621 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -227,7 +227,8 @@ class PyCompileCLITestCase(unittest.TestCase): def setUp(self): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') - self.cache_path = importlib.util.cache_from_source(self.source_path) + self.cache_path = importlib.util.cache_from_source(self.source_path, + optimization='' if __debug__ else 1) with open(self.source_path, 'w') as file: file.write('x = 123\n') @@ -250,6 +251,7 @@ def pycompilecmd_failure(self, *args): return script_helper.assert_python_failure('-m', 'py_compile', *args) def test_stdin(self): + self.assertFalse(os.path.exists(self.cache_path)) result = self.pycompilecmd('-', input=self.source_path) self.assertEqual(result.returncode, 0) self.assertEqual(result.stdout, b'') diff --git a/Lib/test/test_pydoc/pydocfodder.py b/Lib/test/test_pydoc/pydocfodder.py index 27037e048db819..3cc2d5bd57fe5b 100644 --- a/Lib/test/test_pydoc/pydocfodder.py +++ b/Lib/test/test_pydoc/pydocfodder.py @@ -81,6 +81,8 @@ def B_classmethod(cls, x): A_method_ref = A().A_method A_method_alias = A.A_method B_method_alias = B_method + count = list.count # same name + list_count = list.count __repr__ = object.__repr__ # same name object_repr = object.__repr__ get = {}.get # same name @@ -180,5 +182,7 @@ def __call__(self, inst): B_method2 = B.B_method count = list.count # same name list_count = list.count +__repr__ = object.__repr__ # same name +object_repr = object.__repr__ get = {}.get # same name dict_get = {}.get diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 0dd24e6d347364..9d40234ed01697 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -693,6 +693,30 @@ def test_help_output_redirect(self): finally: pydoc.getpager = getpager_old + def test_lambda_with_return_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"return": int} + with captured_output('stdout') as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a, b, c) -> int", helptext) + + def test_lambda_without_return_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"a": int, "b": int, "c": int} + with captured_output('stdout') as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a: int, b: int, c: int)", helptext) + + def test_lambda_with_return_and_params_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"a": int, "b": int, "c": int, "return": int} + with captured_output('stdout') as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a: int, b: int, c: int) -> int", helptext) + def test_namedtuple_fields(self): Person = namedtuple('Person', ['nickname', 'firstname']) with captured_stdout() as help_io: @@ -1138,6 +1162,17 @@ def test_importfile(self): self.assertEqual(loaded_pydoc.__spec__, pydoc.__spec__) +class Rect: + @property + def area(self): + '''Area of the rect''' + return self.w * self.h + + +class Square(Rect): + area = property(lambda self: self.side**2) + + class TestDescriptions(unittest.TestCase): def test_module(self): @@ -1526,13 +1561,13 @@ def test_namedtuple_field_descriptor(self): @requires_docstrings def test_property(self): - class Rect: - @property - def area(self): - '''Area of the rect''' - return self.w * self.h - self.assertEqual(self._get_summary_lines(Rect.area), """\ +area + Area of the rect +""") + # inherits the docstring from Rect.area + self.assertEqual(self._get_summary_lines(Square.area), """\ +area Area of the rect """) self.assertIn(""" @@ -1651,6 +1686,8 @@ def test_text_doc_routines_in_class(self, cls=pydocfodder.B): self.assertIn(' | global_func(x, y) from test.test_pydoc.pydocfodder', lines) self.assertIn(' | global_func_alias = global_func(x, y)', lines) self.assertIn(' | global_func2_alias = global_func2(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn(' | count(self, value, /) from builtins.list', lines) + self.assertIn(' | list_count = count(self, value, /)', lines) self.assertIn(' | __repr__(self, /) from builtins.object', lines) self.assertIn(' | object_repr = __repr__(self, /)', lines) @@ -1679,6 +1716,8 @@ def test_html_doc_routines_in_class(self, cls=pydocfodder.B): self.assertIn('global_func(x, y) from test.test_pydoc.pydocfodder', lines) self.assertIn('global_func_alias = global_func(x, y)', lines) self.assertIn('global_func2_alias = global_func2(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn('count(self, value, /) from builtins.list', lines) + self.assertIn('list_count = count(self, value, /)', lines) self.assertIn('__repr__(self, /) from builtins.object', lines) self.assertIn('object_repr = __repr__(self, /)', lines) @@ -1722,6 +1761,10 @@ def test_text_doc_routines_in_module(self): # unbound methods self.assertIn(' B_method(self)', lines) self.assertIn(' B_method2 = B_method(self)', lines) + self.assertIn(' count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(self, /) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(self, /) unbound builtins.object method', lines) def test_html_doc_routines_in_module(self): doc = pydoc.HTMLDoc() @@ -1747,6 +1790,10 @@ def test_html_doc_routines_in_module(self): # unbound methods self.assertIn(' B_method(self)', lines) self.assertIn(' B_method2 = B_method(self)', lines) + self.assertIn(' count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(self, /) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(self, /) unbound builtins.object method', lines) @unittest.skipIf( diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index d941a1a8f9ebc6..1d56ccd71cf962 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -755,5 +755,59 @@ def resolve_entity(context, base, system_id, public_id): self.assertEqual(handler_call_args, [("bar", "baz")]) +class ReparseDeferralTest(unittest.TestCase): + def test_getter_setter_round_trip(self): + parser = expat.ParserCreate() + enabled = (expat.version_info >= (2, 6, 0)) + + self.assertIs(parser.GetReparseDeferralEnabled(), enabled) + parser.SetReparseDeferralEnabled(False) + self.assertIs(parser.GetReparseDeferralEnabled(), False) + parser.SetReparseDeferralEnabled(True) + self.assertIs(parser.GetReparseDeferralEnabled(), enabled) + + def test_reparse_deferral_enabled(self): + if expat.version_info < (2, 6, 0): + self.skipTest(f'Expat {expat.version_info} does not ' + 'support reparse deferral') + + started = [] + + def start_element(name, _): + started.append(name) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + self.assertTrue(parser.GetReparseDeferralEnabled()) + + for chunk in (b''): + parser.Parse(chunk, False) + + # The key test: Have handlers already fired? Expecting: no. + self.assertEqual(started, []) + + parser.Parse(b'', True) + + self.assertEqual(started, ['doc']) + + def test_reparse_deferral_disabled(self): + started = [] + + def start_element(name, _): + started.append(name) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + if expat.version_info >= (2, 6, 0): + parser.SetReparseDeferralEnabled(False) + self.assertFalse(parser.GetReparseDeferralEnabled()) + + for chunk in (b''): + parser.Parse(chunk, False) + + # The key test: Have handlers already fired? Expecting: yes. + self.assertEqual(started, ['doc']) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 993a7d6e264a1f..b1ac22c28cf7c1 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1,7 +1,7 @@ from test.support import (gc_collect, bigmemtest, _2G, cpython_only, captured_stdout, check_disallow_instantiation, is_emscripten, is_wasi, - warnings_helper, SHORT_TIMEOUT) + warnings_helper, SHORT_TIMEOUT, CPUStopwatch) import locale import re import string @@ -2284,17 +2284,16 @@ def test_bug_40736(self): def test_search_anchor_at_beginning(self): s = 'x'*10**7 - start = time.perf_counter() - for p in r'\Ay', r'^y': - self.assertIsNone(re.search(p, s)) - self.assertEqual(re.split(p, s), [s]) - self.assertEqual(re.findall(p, s), []) - self.assertEqual(list(re.finditer(p, s)), []) - self.assertEqual(re.sub(p, '', s), s) - t = time.perf_counter() - start + with CPUStopwatch() as stopwatch: + for p in r'\Ay', r'^y': + self.assertIsNone(re.search(p, s)) + self.assertEqual(re.split(p, s), [s]) + self.assertEqual(re.findall(p, s), []) + self.assertEqual(list(re.finditer(p, s)), []) + self.assertEqual(re.sub(p, '', s), s) # Without optimization it takes 1 second on my computer. # With optimization -- 0.0003 seconds. - self.assertLess(t, 0.1) + self.assertLess(stopwatch.seconds, 0.1) def test_possessive_quantifiers(self): """Test Possessive Quantifiers diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 89562fa5eac62c..7e1eaa7d6a515e 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -399,7 +399,7 @@ def test_unknown_option(self): self.checkError(['--unknown-option'], 'unrecognized arguments: --unknown-option') - def check_ci_mode(self, args, use_resources, rerun=True): + def create_regrtest(self, args): ns = cmdline._parse_args(args) # Check Regrtest attributes which are more reliable than Namespace @@ -411,6 +411,10 @@ def check_ci_mode(self, args, use_resources, rerun=True): regrtest = main.Regrtest(ns) + return regrtest + + def check_ci_mode(self, args, use_resources, rerun=True): + regrtest = self.create_regrtest(args) self.assertEqual(regrtest.num_workers, -1) self.assertEqual(regrtest.want_rerun, rerun) self.assertTrue(regrtest.randomize) @@ -455,6 +459,11 @@ def test_dont_add_python_opts(self): ns = cmdline._parse_args(args) self.assertFalse(ns._add_python_opts) + def test_bisect(self): + args = ['--bisect'] + regrtest = self.create_regrtest(args) + self.assertTrue(regrtest.want_bisect) + @dataclasses.dataclass(slots=True) class Rerun: @@ -1162,8 +1171,8 @@ def check_leak(self, code, what, *, run_workers=False): stderr=subprocess.STDOUT) self.check_executed_tests(output, [test], failed=test, stats=1) - line = 'beginning 6 repetitions\n123456\n......\n' - self.check_line(output, re.escape(line)) + line = r'beginning 6 repetitions. .*\n123:456\n[.0-9X]{3} 111\n' + self.check_line(output, line) line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what) self.assertIn(line2, output) @@ -1192,6 +1201,47 @@ def test_huntrleaks(self): def test_huntrleaks_mp(self): self.check_huntrleaks(run_workers=True) + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') + def test_huntrleaks_bisect(self): + # test --huntrleaks --bisect + code = textwrap.dedent(""" + import unittest + + GLOBAL_LIST = [] + + class RefLeakTest(unittest.TestCase): + def test1(self): + pass + + def test2(self): + pass + + def test3(self): + GLOBAL_LIST.append(object()) + + def test4(self): + pass + """) + + test = self.create_test('huntrleaks', code=code) + + filename = 'reflog.txt' + self.addCleanup(os_helper.unlink, filename) + cmd = ['--huntrleaks', '3:3:', '--bisect', test] + output = self.run_tests(*cmd, + exitcode=EXITCODE_BAD_TEST, + stderr=subprocess.STDOUT) + + self.assertIn(f"Bisect {test}", output) + self.assertIn(f"Bisect {test}: exit code 0", output) + + # test3 is the one which leaks + self.assertIn("Bisection completed in", output) + self.assertIn( + "Tests (1):\n" + f"* {test}.RefLeakTest.test3\n", + output) + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index eda4e6a46df437..97e96668f85c8a 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -19,6 +19,7 @@ from io import BytesIO, StringIO import codecs import os.path +import pyexpat import shutil import sys from urllib.error import URLError @@ -1214,6 +1215,56 @@ def test_expat_incremental_reset(self): self.assertEqual(result.getvalue(), start + b"text") + def test_flush_reparse_deferral_enabled(self): + if pyexpat.version_info < (2, 6, 0): + self.skipTest(f'Expat {pyexpat.version_info} does not support reparse deferral') + + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + for chunk in (""): + parser.feed(chunk) + + self.assertEqual(result.getvalue(), start) # i.e. no elements started + self.assertTrue(parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assertTrue(parser._parser.GetReparseDeferralEnabled()) + self.assertEqual(result.getvalue(), start + b"") + + parser.feed("") + parser.close() + + self.assertEqual(result.getvalue(), start + b"") + + def test_flush_reparse_deferral_disabled(self): + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + for chunk in (""): + parser.feed(chunk) + + if pyexpat.version_info >= (2, 6, 0): + parser._parser.SetReparseDeferralEnabled(False) + + self.assertEqual(result.getvalue(), start) # i.e. no elements started + self.assertFalse(parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assertFalse(parser._parser.GetReparseDeferralEnabled()) + self.assertEqual(result.getvalue(), start + b"") + + parser.feed("") + parser.close() + + self.assertEqual(result.getvalue(), start + b"") + # ===== Locator support def test_expat_locator_noinfo(self): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 17964234992062..b936e9ae91daca 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -46,6 +46,7 @@ VSOCKPORT = 1234 AIX = platform.system() == "AIX" +WSL = "microsoft-standard-WSL" in platform.release() try: import _socket @@ -510,6 +511,7 @@ def clientTearDown(self): ThreadableTest.clientTearDown(self) @unittest.skipIf(fcntl is None, "need fcntl") +@unittest.skipIf(WSL, 'VSOCK does not work on Microsoft WSL') @unittest.skipUnless(HAVE_SOCKET_VSOCK, 'VSOCK sockets required for this test.') @unittest.skipUnless(get_cid() != 2, @@ -526,6 +528,7 @@ def setUp(self): self.serv.bind((socket.VMADDR_CID_ANY, VSOCKPORT)) self.serv.listen() self.serverExplicitReady() + self.serv.settimeout(support.LOOPBACK_TIMEOUT) self.conn, self.connaddr = self.serv.accept() self.addCleanup(self.conn.close) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index bf2c254c9ee7d9..1cf41638a7f01a 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2353,6 +2353,66 @@ def test_mixed_int_and_float(self): self.assertAlmostEqual(actual_mean, expected_mean, places=5) +class TestKDE(unittest.TestCase): + + def test_kde(self): + kde = statistics.kde + StatisticsError = statistics.StatisticsError + + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # The approximate integral of a PDF should be close to 1.0 + + def integrate(func, low, high, steps=10_000): + "Numeric approximation of a definite function integral." + dx = (high - low) / steps + midpoints = (low + (i + 1/2) * dx for i in range(steps)) + return sum(map(func, midpoints)) * dx + + for kernel in kernels: + with self.subTest(kernel=kernel): + f_hat = kde(sample, h=1.5, kernel=kernel) + area = integrate(f_hat, -20, 20) + self.assertAlmostEqual(area, 1.0, places=4) + + # Check error cases + + with self.assertRaises(StatisticsError): + kde([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde(sample, h=0.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde(sample, h=0.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde(sample, h=1.0, kernel='bogus') # Invalid kernel + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + f_hat = kde(sample, h, kernel) + self.assertEqual(f_hat.__name__, 'pdf') + self.assertIn(kernel, f_hat.__doc__) + self.assertIn(str(h), f_hat.__doc__) + + # Test closed interval for the support boundaries. + # In particular, 'uniform' should non-zero at the boundaries. + + f_hat = kde([0], 1.0, 'uniform') + self.assertEqual(f_hat(-1.0), 1/2) + self.assertEqual(f_hat(1.0), 1/2) + + class TestQuantiles(unittest.TestCase): def test_specific_cases(self): diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 71671a5a984256..38dcabd84d8170 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -729,7 +729,7 @@ def test_subinterp_intern_dynamically_allocated(self): self.assertIs(t, s) interp = interpreters.create() - interp.exec_sync(textwrap.dedent(f''' + interp.exec(textwrap.dedent(f''' import sys t = sys.intern({s!r}) assert id(t) != {id(s)}, (id(t), {id(s)}) @@ -744,7 +744,7 @@ def test_subinterp_intern_statically_allocated(self): t = sys.intern(s) interp = interpreters.create() - interp.exec_sync(textwrap.dedent(f''' + interp.exec(textwrap.dedent(f''' import sys t = sys.intern({s!r}) assert id(t) == {id(t)}, (id(t), {id(t)}) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 51f070e96047a6..a047780fdd7e17 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -507,14 +507,32 @@ def test_length_zero_header(self): with tarfile.open(support.findfile('recursion.tar', subdir='archivetestdata')): pass - def test_extractfile_name(self): + def test_extractfile_attrs(self): # gh-74468: TarFile.name must name a file, not a parent archive. file = self.tar.getmember('ustar/regtype') with self.tar.extractfile(file) as fobj: self.assertEqual(fobj.name, 'ustar/regtype') + self.assertRaises(AttributeError, fobj.fileno) + self.assertIs(fobj.readable(), True) + self.assertIs(fobj.writable(), False) + if self.is_stream: + self.assertRaises(AttributeError, fobj.seekable) + else: + self.assertIs(fobj.seekable(), True) + self.assertIs(fobj.closed, False) + self.assertIs(fobj.closed, True) + self.assertEqual(fobj.name, 'ustar/regtype') + self.assertRaises(AttributeError, fobj.fileno) + self.assertIs(fobj.readable(), True) + self.assertIs(fobj.writable(), False) + if self.is_stream: + self.assertRaises(AttributeError, fobj.seekable) + else: + self.assertIs(fobj.seekable(), True) class MiscReadTestBase(CommonReadTest): + is_stream = False def requires_name_attribute(self): pass @@ -807,6 +825,7 @@ def requires_name_attribute(self): class StreamReadTest(CommonReadTest, unittest.TestCase): prefix="r|" + is_stream = True def test_read_through(self): # Issue #11224: A poorly designed _FileInFile.read() method diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index 931cb4b797e0b2..83235230d5c112 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -189,8 +189,8 @@ def task(): with threading_helper.wait_threads_exit(): handle = thread.start_joinable_thread(task) handle.join() - with self.assertRaisesRegex(ValueError, "not joinable"): - handle.join() + # Subsequent join() calls should succeed + handle.join() def test_joinable_not_joined(self): handle_destroyed = thread.allocate_lock() @@ -233,58 +233,61 @@ def task(): with self.assertRaisesRegex(RuntimeError, "Cannot join current thread"): raise errors[0] - def test_detach_from_self(self): - errors = [] - handles = [] - start_joinable_thread_returned = thread.allocate_lock() - start_joinable_thread_returned.acquire() - thread_detached = thread.allocate_lock() - thread_detached.acquire() + def test_join_then_self_join(self): + # make sure we can't deadlock in the following scenario with + # threads t0 and t1 (see comment in `ThreadHandle_join()` for more + # details): + # + # - t0 joins t1 + # - t1 self joins + def make_lock(): + lock = thread.allocate_lock() + lock.acquire() + return lock + + error = None + self_joiner_handle = None + self_joiner_started = make_lock() + self_joiner_barrier = make_lock() + def self_joiner(): + nonlocal error + + self_joiner_started.release() + self_joiner_barrier.acquire() - def task(): - start_joinable_thread_returned.acquire() try: - handles[0].detach() + self_joiner_handle.join() except Exception as e: - errors.append(e) - finally: - thread_detached.release() + error = e + + joiner_started = make_lock() + def joiner(): + joiner_started.release() + self_joiner_handle.join() with threading_helper.wait_threads_exit(): - handle = thread.start_joinable_thread(task) - handles.append(handle) - start_joinable_thread_returned.release() - thread_detached.acquire() - with self.assertRaisesRegex(ValueError, "not joinable"): - handle.join() + self_joiner_handle = thread.start_joinable_thread(self_joiner) + # Wait for the self-joining thread to start + self_joiner_started.acquire() - assert len(errors) == 0 + # Start the thread that joins the self-joiner + joiner_handle = thread.start_joinable_thread(joiner) - def test_detach_then_join(self): - lock = thread.allocate_lock() - lock.acquire() + # Wait for the joiner to start + joiner_started.acquire() - def task(): - lock.acquire() + # Not great, but I don't think there's a deterministic way to make + # sure that the self-joining thread has been joined. + time.sleep(0.1) - with threading_helper.wait_threads_exit(): - handle = thread.start_joinable_thread(task) - # detach() returns even though the thread is blocked on lock - handle.detach() - # join() then cannot be called anymore - with self.assertRaisesRegex(ValueError, "not joinable"): - handle.join() - lock.release() - - def test_join_then_detach(self): - def task(): - pass + # Unblock the self-joiner + self_joiner_barrier.release() - with threading_helper.wait_threads_exit(): - handle = thread.start_joinable_thread(task) - handle.join() - with self.assertRaisesRegex(ValueError, "not joinable"): - handle.detach() + self_joiner_handle.join() + joiner_handle.join() + + with self.assertRaisesRegex(RuntimeError, "Cannot join current thread"): + raise error class Barrier: diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 1ab223b81e939e..3b5c37c948c8c3 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1478,7 +1478,7 @@ def test_threads_join_with_no_main(self): DONE = b'D' interp = interpreters.create() - interp.exec_sync(f"""if True: + interp.exec(f"""if True: import os import threading import time diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 21e8637a7ca905..4428e8cea1964c 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1877,6 +1877,43 @@ def test_roundtrip(self): " print('Can not import' # comment2\n)" "else: print('Loaded')\n") + self.check_roundtrip("f'\\N{EXCLAMATION MARK}'") + self.check_roundtrip(r"f'\\N{SNAKE}'") + self.check_roundtrip(r"f'\\N{{SNAKE}}'") + self.check_roundtrip(r"f'\N{SNAKE}'") + self.check_roundtrip(r"f'\\\N{SNAKE}'") + self.check_roundtrip(r"f'\\\\\N{SNAKE}'") + self.check_roundtrip(r"f'\\\\\\\N{SNAKE}'") + + self.check_roundtrip(r"f'\\N{1}'") + self.check_roundtrip(r"f'\\\\N{2}'") + self.check_roundtrip(r"f'\\\\\\N{3}'") + self.check_roundtrip(r"f'\\\\\\\\N{4}'") + + self.check_roundtrip(r"f'\\N{{'") + self.check_roundtrip(r"f'\\\\N{{'") + self.check_roundtrip(r"f'\\\\\\N{{'") + self.check_roundtrip(r"f'\\\\\\\\N{{'") + cases = [ + """ +if 1: + "foo" +"bar" +""", + """ +if 1: + ("foo" + "bar") +""", + """ +if 1: + "foo" + "bar" +""" ] + for case in cases: + self.check_roundtrip(case) + + def test_continuation(self): # Balancing continuation self.check_roundtrip("a = (3,4, \n" @@ -1911,9 +1948,6 @@ def test_random_files(self): tempdir = os.path.dirname(__file__) or os.curdir testfiles = glob.glob(os.path.join(glob.escape(tempdir), "test*.py")) - # TODO: Remove this once we can untokenize PEP 701 syntax - testfiles.remove(os.path.join(tempdir, "test_fstring.py")) - if not support.is_resource_enabled("cpu"): testfiles = random.sample(testfiles, 10) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 58572c6f4d3157..e90e315c808361 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -2,7 +2,7 @@ import unittest import dis from test import support -from test.support import import_helper +from test.support import import_helper, requires_specialization try: from sys import _clear_type_cache except ImportError: @@ -94,6 +94,7 @@ class C: @support.cpython_only +@requires_specialization class TypeCacheWithSpecializationTests(unittest.TestCase): def tearDown(self): _clear_type_cache() diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index 5a911da56f8f8a..ee8939f62d082c 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -309,7 +309,7 @@ def test_withstmt(self): self.assertEqual(tree.body[0].type_comment, None) def test_parenthesized_withstmt(self): - for tree in self.parse_all(parenthesized_withstmt, minver=9): + for tree in self.parse_all(parenthesized_withstmt): self.assertEqual(tree.body[0].type_comment, "int") self.assertEqual(tree.body[1].type_comment, "int") tree = self.classic_parse(parenthesized_withstmt) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 1acb2a4d81adf3..16985122bc0219 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -713,6 +713,26 @@ def test_hash(self): self.assertEqual(hash(int | str), hash(str | int)) self.assertEqual(hash(int | str), hash(typing.Union[int, str])) + def test_union_of_unhashable(self): + class UnhashableMeta(type): + __hash__ = None + + class A(metaclass=UnhashableMeta): ... + class B(metaclass=UnhashableMeta): ... + + self.assertEqual((A | B).__args__, (A, B)) + union1 = A | B + with self.assertRaises(TypeError): + hash(union1) + + union2 = int | B + with self.assertRaises(TypeError): + hash(union2) + + union3 = A | int + with self.assertRaises(TypeError): + hash(union3) + def test_instancecheck_and_subclasscheck(self): for x in (int | str, typing.Union[int, str]): with self.subTest(x=x): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 176623171c9888..dbbbe63e95ad46 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2,10 +2,11 @@ import collections import collections.abc from collections import defaultdict -from functools import lru_cache, wraps +from functools import lru_cache, wraps, reduce import gc import inspect import itertools +import operator import pickle import re import sys @@ -1769,6 +1770,26 @@ def test_union_union(self): v = Union[u, Employee] self.assertEqual(v, Union[int, float, Employee]) + def test_union_of_unhashable(self): + class UnhashableMeta(type): + __hash__ = None + + class A(metaclass=UnhashableMeta): ... + class B(metaclass=UnhashableMeta): ... + + self.assertEqual(Union[A, B].__args__, (A, B)) + union1 = Union[A, B] + with self.assertRaises(TypeError): + hash(union1) + + union2 = Union[int, B] + with self.assertRaises(TypeError): + hash(union2) + + union3 = Union[A, int] + with self.assertRaises(TypeError): + hash(union3) + def test_repr(self): self.assertEqual(repr(Union), 'typing.Union') u = Union[Employee, int] @@ -5506,10 +5527,8 @@ def some(self): self.assertFalse(hasattr(WithOverride.some, "__override__")) def test_multiple_decorators(self): - import functools - def with_wraps(f): # similar to `lru_cache` definition - @functools.wraps(f) + @wraps(f) def wrapper(*args, **kwargs): return f(*args, **kwargs) return wrapper @@ -8524,6 +8543,76 @@ def test_flatten(self): self.assertEqual(A.__metadata__, (4, 5)) self.assertEqual(A.__origin__, int) + def test_deduplicate_from_union(self): + # Regular: + self.assertEqual(get_args(Annotated[int, 1] | int), + (Annotated[int, 1], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], int]), + (Annotated[int, 1], int)) + self.assertEqual(get_args(Annotated[int, 1] | Annotated[int, 2] | int), + (Annotated[int, 1], Annotated[int, 2], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], Annotated[int, 2], int]), + (Annotated[int, 1], Annotated[int, 2], int)) + self.assertEqual(get_args(Annotated[int, 1] | Annotated[str, 1] | int), + (Annotated[int, 1], Annotated[str, 1], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], Annotated[str, 1], int]), + (Annotated[int, 1], Annotated[str, 1], int)) + + # Duplicates: + self.assertEqual(Annotated[int, 1] | Annotated[int, 1] | int, + Annotated[int, 1] | int) + self.assertEqual(Union[Annotated[int, 1], Annotated[int, 1], int], + Union[Annotated[int, 1], int]) + + # Unhashable metadata: + self.assertEqual(get_args(str | Annotated[int, {}] | Annotated[int, set()] | int), + (str, Annotated[int, {}], Annotated[int, set()], int)) + self.assertEqual(get_args(Union[str, Annotated[int, {}], Annotated[int, set()], int]), + (str, Annotated[int, {}], Annotated[int, set()], int)) + self.assertEqual(get_args(str | Annotated[int, {}] | Annotated[str, {}] | int), + (str, Annotated[int, {}], Annotated[str, {}], int)) + self.assertEqual(get_args(Union[str, Annotated[int, {}], Annotated[str, {}], int]), + (str, Annotated[int, {}], Annotated[str, {}], int)) + + self.assertEqual(get_args(Annotated[int, 1] | str | Annotated[str, {}] | int), + (Annotated[int, 1], str, Annotated[str, {}], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], str, Annotated[str, {}], int]), + (Annotated[int, 1], str, Annotated[str, {}], int)) + + import dataclasses + @dataclasses.dataclass + class ValueRange: + lo: int + hi: int + v = ValueRange(1, 2) + self.assertEqual(get_args(Annotated[int, v] | None), + (Annotated[int, v], types.NoneType)) + self.assertEqual(get_args(Union[Annotated[int, v], None]), + (Annotated[int, v], types.NoneType)) + self.assertEqual(get_args(Optional[Annotated[int, v]]), + (Annotated[int, v], types.NoneType)) + + # Unhashable metadata duplicated: + self.assertEqual(Annotated[int, {}] | Annotated[int, {}] | int, + Annotated[int, {}] | int) + self.assertEqual(Annotated[int, {}] | Annotated[int, {}] | int, + int | Annotated[int, {}]) + self.assertEqual(Union[Annotated[int, {}], Annotated[int, {}], int], + Union[Annotated[int, {}], int]) + self.assertEqual(Union[Annotated[int, {}], Annotated[int, {}], int], + Union[int, Annotated[int, {}]]) + + def test_order_in_union(self): + expr1 = Annotated[int, 1] | str | Annotated[str, {}] | int + for args in itertools.permutations(get_args(expr1)): + with self.subTest(args=args): + self.assertEqual(expr1, reduce(operator.or_, args)) + + expr2 = Union[Annotated[int, 1], str, Annotated[str, {}], int] + for args in itertools.permutations(get_args(expr2)): + with self.subTest(args=args): + self.assertEqual(expr2, Union[args]) + def test_specialize(self): L = Annotated[List[T], "my decoration"] LI = Annotated[List[int], "my decoration"] @@ -8544,6 +8633,16 @@ def test_hash_eq(self): {Annotated[int, 4, 5], Annotated[int, 4, 5], Annotated[T, 4, 5]}, {Annotated[int, 4, 5], Annotated[T, 4, 5]} ) + # Unhashable `metadata` raises `TypeError`: + a1 = Annotated[int, []] + with self.assertRaises(TypeError): + hash(a1) + + class A: + __hash__ = None + a2 = Annotated[int, A()] + with self.assertRaises(TypeError): + hash(a2) def test_instantiate(self): class C: diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 6f698a8d891815..bb15f64c59dbd1 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -370,13 +370,13 @@ def test_slices(self): self.check_ast_roundtrip("a[i:j, k]") def test_invalid_raise(self): - self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X"))) + self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X", ctx=ast.Load()))) def test_invalid_fstring_value(self): self.check_invalid( ast.JoinedStr( values=[ - ast.Name(id="test"), + ast.Name(id="test", ctx=ast.Load()), ast.Constant(value="test") ] ) @@ -649,6 +649,30 @@ def test_multiquote_joined_string(self): self.check_ast_roundtrip("""f'''""\"''\\'{"\\n\\"'"}''' """) self.check_ast_roundtrip("""f'''""\"''\\'{""\"\\n\\"'''""\" '''\\n'''}''' """) + def test_backslash_in_format_spec(self): + import re + msg = re.escape("invalid escape sequence '\\ '") + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\ }" """) + self.check_ast_roundtrip("""f"{x:\\n}" """) + + self.check_ast_roundtrip("""f"{x:\\\\ }" """) + + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\\\\\ }" """) + self.check_ast_roundtrip("""f"{x:\\\\\\n}" """) + + self.check_ast_roundtrip("""f"{x:\\\\\\\\ }" """) + + def test_quote_in_format_spec(self): + self.check_ast_roundtrip("""f"{x:'}" """) + self.check_ast_roundtrip("""f"{x:\\'}" """) + self.check_ast_roundtrip("""f"{x:\\\\'}" """) + + self.check_ast_roundtrip("""f'\\'{x:"}' """) + self.check_ast_roundtrip("""f'\\'{x:\\"}' """) + self.check_ast_roundtrip("""f'\\'{x:\\\\"}' """) + class ManualASTCreationTestCase(unittest.TestCase): """Test that AST nodes created without a type_params field unparse correctly.""" @@ -694,7 +718,7 @@ def test_function_with_type_params_and_bound(self): body=[ast.Pass()], decorator_list=[], returns=None, - type_params=[ast.TypeVar("T", bound=ast.Name("int"))], + type_params=[ast.TypeVar("T", bound=ast.Name("int", ctx=ast.Load()))], ) ast.fix_missing_locations(node) self.assertEqual(ast.unparse(node), "def f[T: int]():\n pass") diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index fa528a675892b5..739c15df13de21 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -15,10 +15,11 @@ import subprocess import urllib.request -# The proxy bypass method imported below has logic specific to the OSX -# proxy config data structure but is testable on all platforms. +# The proxy bypass method imported below has logic specific to the +# corresponding system but is testable on all platforms. from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler, HTTPPasswordMgrWithPriorAuth, _parse_proxy, + _proxy_bypass_winreg_override, _proxy_bypass_macosx_sysconf, AbstractDigestAuthHandler) from urllib.parse import urlparse @@ -1485,6 +1486,30 @@ def test_proxy_https_proxy_authorization(self): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") + @unittest.skipUnless(os.name == "nt", "only relevant for Windows") + def test_winreg_proxy_bypass(self): + proxy_override = "www.example.com;*.example.net; 192.168.0.1" + proxy_bypass = _proxy_bypass_winreg_override + for host in ("www.example.com", "www.example.net", "192.168.0.1"): + self.assertTrue(proxy_bypass(host, proxy_override), + "expected bypass of %s to be true" % host) + + for host in ("example.com", "www.example.org", "example.net", + "192.168.0.2"): + self.assertFalse(proxy_bypass(host, proxy_override), + "expected bypass of %s to be False" % host) + + # check intranet address bypass + proxy_override = "example.com; " + self.assertTrue(proxy_bypass("example.com", proxy_override), + "expected bypass of %s to be true" % host) + self.assertFalse(proxy_bypass("example.net", proxy_override), + "expected bypass of %s to be False" % host) + for host in ("test", "localhost"): + self.assertTrue(proxy_bypass(host, proxy_override), + "expect to bypass intranet address '%s'" + % host) + @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index ba31beb81e80b0..f60c662d322e38 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -19,8 +19,9 @@ import tempfile from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, - requires_subprocess, is_apple_mobile, is_emscripten, - is_wasi, requires_venv_with_pip, TEST_HOME_DIR, + requires_subprocess, is_android, is_apple_mobile, + is_emscripten, is_wasi, + requires_venv_with_pip, TEST_HOME_DIR, requires_resource, copy_python_src_ignore) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest @@ -39,10 +40,8 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') -# Skip tests on WASM platforms, plus iOS/tvOS/watchOS -if is_apple_mobile or is_emscripten or is_wasi: - raise unittest.SkipTest(f"venv tests not required on {sys.platform}") - +if is_android or is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest("venv is not available on this platform") @requires_subprocess() def check_output(cmd, encoding=None): diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index c535d631bb646f..14df482ba6c207 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -121,10 +121,6 @@ """ -fails_with_expat_2_6_0 = (unittest.expectedFailure - if pyexpat.version_info >= (2, 6, 0) else - lambda test: test) - def checkwarnings(*filters, quiet=False): def decorator(test): def newtest(*args, **kwargs): @@ -1462,12 +1458,14 @@ def test_attlist_default(self): class XMLPullParserTest(unittest.TestCase): - def _feed(self, parser, data, chunk_size=None): + def _feed(self, parser, data, chunk_size=None, flush=False): if chunk_size is None: parser.feed(data) else: for i in range(0, len(data), chunk_size): parser.feed(data[i:i+chunk_size]) + if flush: + parser.flush() def assert_events(self, parser, expected, max_events=None): self.assertEqual( @@ -1485,34 +1483,32 @@ def assert_event_tags(self, parser, expected, max_events=None): self.assertEqual([(action, elem.tag) for action, elem in events], expected) - def test_simple_xml(self, chunk_size=None): + def test_simple_xml(self, chunk_size=None, flush=False): parser = ET.XMLPullParser() self.assert_event_tags(parser, []) - self._feed(parser, "\n", chunk_size) + self._feed(parser, "\n", chunk_size, flush) self.assert_event_tags(parser, []) self._feed(parser, "\n text\n", chunk_size) + self._feed(parser, ">\n", chunk_size, flush) self.assert_event_tags(parser, [('end', 'element')]) - self._feed(parser, "texttail\n", chunk_size) - self._feed(parser, "\n", chunk_size) + self._feed(parser, "texttail\n", chunk_size, flush) + self._feed(parser, "\n", chunk_size, flush) self.assert_event_tags(parser, [ ('end', 'element'), ('end', 'empty-element'), ]) - self._feed(parser, "\n", chunk_size) + self._feed(parser, "\n", chunk_size, flush) self.assert_event_tags(parser, [('end', 'root')]) self.assertIsNone(parser.close()) - @fails_with_expat_2_6_0 def test_simple_xml_chunk_1(self): - self.test_simple_xml(chunk_size=1) + self.test_simple_xml(chunk_size=1, flush=True) - @fails_with_expat_2_6_0 def test_simple_xml_chunk_5(self): - self.test_simple_xml(chunk_size=5) + self.test_simple_xml(chunk_size=5, flush=True) def test_simple_xml_chunk_22(self): self.test_simple_xml(chunk_size=22) @@ -1711,6 +1707,57 @@ def test_unknown_event(self): with self.assertRaises(ValueError): ET.XMLPullParser(events=('start', 'end', 'bogus')) + def test_flush_reparse_deferral_enabled(self): + if pyexpat.version_info < (2, 6, 0): + self.skipTest(f'Expat {pyexpat.version_info} does not ' + 'support reparse deferral') + + parser = ET.XMLPullParser(events=('start', 'end')) + + for chunk in (""): + parser.feed(chunk) + + self.assert_event_tags(parser, []) # i.e. no elements started + if ET is pyET: + self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assert_event_tags(parser, [('start', 'doc')]) + if ET is pyET: + self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.feed("") + parser.close() + + self.assert_event_tags(parser, [('end', 'doc')]) + + def test_flush_reparse_deferral_disabled(self): + parser = ET.XMLPullParser(events=('start', 'end')) + + for chunk in (""): + parser.feed(chunk) + + if pyexpat.version_info >= (2, 6, 0): + if not ET is pyET: + self.skipTest(f'XMLParser.(Get|Set)ReparseDeferralEnabled ' + 'methods not available in C') + parser._parser._parser.SetReparseDeferralEnabled(False) + + self.assert_event_tags(parser, []) # i.e. no elements started + if ET is pyET: + self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assert_event_tags(parser, [('start', 'doc')]) + if ET is pyET: + self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.feed("") + parser.close() + + self.assert_event_tags(parser, [('end', 'doc')]) # # xinclude tests (samples from appendix C of the xinclude specification) diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 087fa8d65cc336..7d89f753448dd7 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -4,7 +4,6 @@ import io import itertools import os -import pathlib import posixpath import struct import subprocess @@ -25,7 +24,7 @@ captured_stdout, captured_stderr, requires_subprocess ) from test.support.os_helper import ( - TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count + TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count, FakePath ) @@ -160,7 +159,7 @@ def test_open(self): self.zip_open_test(f, self.compression) def test_open_with_pathlike(self): - path = pathlib.Path(TESTFN2) + path = FakePath(TESTFN2) self.zip_open_test(path, self.compression) with zipfile.ZipFile(path, "r", self.compression) as zipfp: self.assertIsInstance(zipfp.filename, str) @@ -447,6 +446,27 @@ def write(self, data): self.assertEqual(zipfp.read('file1'), b'data1') self.assertEqual(zipfp.read('file2'), b'data2') + def test_zipextfile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + zipfp.writestr(fname, "bogus") + + with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: + with zipfp.open(fname) as fid: + self.assertEqual(fid.name, fname) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'r') + self.assertIs(fid.readable(), True) + self.assertIs(fid.writable(), False) + self.assertIs(fid.seekable(), True) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'r') + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertRaises(ValueError, fid.readable) + self.assertIs(fid.writable(), False) + self.assertRaises(ValueError, fid.seekable) def tearDown(self): unlink(TESTFN) @@ -578,17 +598,16 @@ def test_write_default_name(self): def test_io_on_closed_zipextfile(self): fname = "somefile.txt" - with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: zipfp.writestr(fname, "bogus") with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: with zipfp.open(fname) as fid: fid.close() + self.assertIs(fid.closed, True) self.assertRaises(ValueError, fid.read) self.assertRaises(ValueError, fid.seek, 0) self.assertRaises(ValueError, fid.tell) - self.assertRaises(ValueError, fid.readable) - self.assertRaises(ValueError, fid.seekable) def test_write_to_readonly(self): """Check that trying to call write() on a readonly ZipFile object @@ -1285,6 +1304,21 @@ def test_issue44439(self): self.assertEqual(data.write(q), LENGTH) self.assertEqual(zip.getinfo('data').file_size, LENGTH) + def test_zipwritefile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: + with zipfp.open(fname, 'w') as fid: + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + class StoredWriterTests(AbstractWriterTests, unittest.TestCase): compression = zipfile.ZIP_STORED @@ -1487,7 +1521,7 @@ def test_write_pathlike(self): fp.write("print(42)\n") with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - zipfp.writepy(pathlib.Path(TESTFN2) / "mod1.py") + zipfp.writepy(FakePath(os.path.join(TESTFN2, "mod1.py"))) names = zipfp.namelist() self.assertCompiledIn('mod1.py', names) finally: @@ -1545,7 +1579,7 @@ def test_extract_with_target(self): def test_extract_with_target_pathlike(self): with temp_dir() as extdir: - self._test_extract_with_target(pathlib.Path(extdir)) + self._test_extract_with_target(FakePath(extdir)) def test_extract_all(self): with temp_cwd(): @@ -1580,7 +1614,7 @@ def test_extract_all_with_target(self): def test_extract_all_with_target_pathlike(self): with temp_dir() as extdir: - self._test_extract_all_with_target(pathlib.Path(extdir)) + self._test_extract_all_with_target(FakePath(extdir)) def check_file(self, filename, content): self.assertTrue(os.path.isfile(filename)) @@ -1893,7 +1927,7 @@ def test_is_zip_erroneous_file(self): fp.write("this is not a legal zip file\n") self.assertFalse(zipfile.is_zipfile(TESTFN)) # - passing a path-like object - self.assertFalse(zipfile.is_zipfile(pathlib.Path(TESTFN))) + self.assertFalse(zipfile.is_zipfile(FakePath(TESTFN))) # - passing a file object with open(TESTFN, "rb") as fp: self.assertFalse(zipfile.is_zipfile(fp)) @@ -3013,7 +3047,7 @@ def test_from_file(self): self.assertEqual(zi.file_size, os.path.getsize(__file__)) def test_from_file_pathlike(self): - zi = zipfile.ZipInfo.from_file(pathlib.Path(__file__)) + zi = zipfile.ZipInfo.from_file(FakePath(__file__)) self.assertEqual(posixpath.basename(zi.filename), 'test_core.py') self.assertFalse(zi.is_dir()) self.assertEqual(zi.file_size, os.path.getsize(__file__)) diff --git a/Lib/threading.py b/Lib/threading.py index b6ff00acadd58f..ec89550d6b022e 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -931,7 +931,6 @@ class is implemented. if _HAVE_THREAD_NATIVE_ID: self._native_id = None self._tstate_lock = None - self._join_lock = None self._handle = None self._started = Event() self._is_stopped = False @@ -956,14 +955,11 @@ def _after_fork(self, new_ident=None): if self._tstate_lock is not None: self._tstate_lock._at_fork_reinit() self._tstate_lock.acquire() - if self._join_lock is not None: - self._join_lock._at_fork_reinit() else: # This thread isn't alive after fork: it doesn't have a tstate # anymore. self._is_stopped = True self._tstate_lock = None - self._join_lock = None self._handle = None def __repr__(self): @@ -996,8 +992,6 @@ def start(self): if self._started.is_set(): raise RuntimeError("threads can only be started once") - self._join_lock = _allocate_lock() - with _active_limbo_lock: _limbo[self] = self try: @@ -1167,17 +1161,9 @@ def join(self, timeout=None): self._join_os_thread() def _join_os_thread(self): - join_lock = self._join_lock - if join_lock is None: - return - with join_lock: - # Calling join() multiple times would raise an exception - # in one of the callers. - if self._handle is not None: - self._handle.join() - self._handle = None - # No need to keep this around - self._join_lock = None + # self._handle may be cleared post-fork + if self._handle is not None: + self._handle.join() def _wait_for_tstate_lock(self, block=True, timeout=-1): # Issue #18808: wait for the thread state to be gone. @@ -1478,6 +1464,10 @@ def __init__(self): with _active_limbo_lock: _active[self._ident] = self + def _join_os_thread(self): + # No ThreadHandle for main thread + pass + # Helper thread-local instance to detect when a _DummyThread # is collected. Not a part of the public API. diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 0ab1893d42f72f..7f418bb7a1b37f 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -168,6 +168,7 @@ def __init__(self): self.tokens = [] self.prev_row = 1 self.prev_col = 0 + self.prev_type = None self.encoding = None def add_whitespace(self, start): @@ -183,6 +184,29 @@ def add_whitespace(self, start): if col_offset: self.tokens.append(" " * col_offset) + def escape_brackets(self, token): + characters = [] + consume_until_next_bracket = False + for character in token: + if character == "}": + if consume_until_next_bracket: + consume_until_next_bracket = False + else: + characters.append(character) + if character == "{": + n_backslashes = sum( + 1 for char in _itertools.takewhile( + "\\".__eq__, + characters[-2::-1] + ) + ) + if n_backslashes % 2 == 0: + characters.append(character) + else: + consume_until_next_bracket = True + characters.append(character) + return "".join(characters) + def untokenize(self, iterable): it = iter(iterable) indents = [] @@ -214,11 +238,13 @@ def untokenize(self, iterable): startline = False elif tok_type == FSTRING_MIDDLE: if '{' in token or '}' in token: + token = self.escape_brackets(token) + last_line = token.splitlines()[-1] end_line, end_col = end - end = (end_line, end_col + token.count('{') + token.count('}')) - token = re.sub('{', '{{', token) - token = re.sub('}', '}}', token) - + extra_chars = last_line.count("{{") + last_line.count("}}") + end = (end_line, end_col + extra_chars) + elif tok_type in (STRING, FSTRING_START) and self.prev_type in (STRING, FSTRING_END): + self.tokens.append(" ") self.add_whitespace(start) self.tokens.append(token) @@ -226,6 +252,7 @@ def untokenize(self, iterable): if tok_type in (NEWLINE, NL): self.prev_row += 1 self.prev_col = 0 + self.prev_type = tok_type return "".join(self.tokens) def compat(self, token, iterable): @@ -233,6 +260,7 @@ def compat(self, token, iterable): toks_append = self.tokens.append startline = token[0] in (NEWLINE, NL) prevstring = False + in_fstring = 0 for tok in _itertools.chain([token], iterable): toknum, tokval = tok[:2] @@ -251,6 +279,10 @@ def compat(self, token, iterable): else: prevstring = False + if toknum == FSTRING_START: + in_fstring += 1 + elif toknum == FSTRING_END: + in_fstring -= 1 if toknum == INDENT: indents.append(tokval) continue @@ -263,11 +295,18 @@ def compat(self, token, iterable): toks_append(indents[-1]) startline = False elif toknum == FSTRING_MIDDLE: - if '{' in tokval or '}' in tokval: - tokval = re.sub('{', '{{', tokval) - tokval = re.sub('}', '}}', tokval) + tokval = self.escape_brackets(tokval) + + # Insert a space between two consecutive brackets if we are in an f-string + if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring: + tokval = ' ' + tokval + + # Insert a space between two consecutive f-strings + if toknum in (STRING, FSTRING_START) and self.prev_type in (STRING, FSTRING_END): + self.tokens.append(" ") toks_append(tokval) + self.prev_type = toknum def untokenize(iterable): diff --git a/Lib/typing.py b/Lib/typing.py index 914ddeaf504cd0..2cb4a714ea3a85 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -308,19 +308,33 @@ def _unpack_args(args): newargs.append(arg) return newargs -def _deduplicate(params): +def _deduplicate(params, *, unhashable_fallback=False): # Weed out strict duplicates, preserving the first of each occurrence. - all_params = set(params) - if len(all_params) < len(params): - new_params = [] - for t in params: - if t in all_params: - new_params.append(t) - all_params.remove(t) - params = new_params - assert not all_params, all_params - return params - + try: + return dict.fromkeys(params) + except TypeError: + if not unhashable_fallback: + raise + # Happens for cases like `Annotated[dict, {'x': IntValidator()}]` + return _deduplicate_unhashable(params) + +def _deduplicate_unhashable(unhashable_params): + new_unhashable = [] + for t in unhashable_params: + if t not in new_unhashable: + new_unhashable.append(t) + return new_unhashable + +def _compare_args_orderless(first_args, second_args): + first_unhashable = _deduplicate_unhashable(first_args) + second_unhashable = _deduplicate_unhashable(second_args) + t = list(second_unhashable) + try: + for elem in first_unhashable: + t.remove(elem) + except ValueError: + return False + return not t def _remove_dups_flatten(parameters): """Internal helper for Union creation and substitution. @@ -335,7 +349,7 @@ def _remove_dups_flatten(parameters): else: params.append(p) - return tuple(_deduplicate(params)) + return tuple(_deduplicate(params, unhashable_fallback=True)) def _flatten_literal_params(parameters): @@ -1555,7 +1569,10 @@ def copy_with(self, params): def __eq__(self, other): if not isinstance(other, (_UnionGenericAlias, types.UnionType)): return NotImplemented - return set(self.__args__) == set(other.__args__) + try: # fast path + return set(self.__args__) == set(other.__args__) + except TypeError: # not hashable, slow path + return _compare_args_orderless(self.__args__, other.__args__) def __hash__(self): return hash(frozenset(self.__args__)) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index bca594420f6d9d..d22af6618d80f1 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -2563,6 +2563,7 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings): } """ from fnmatch import fnmatch + from ipaddress import AddressValueError, IPv4Address hostonly, port = _splitport(host) @@ -2579,20 +2580,17 @@ def ip2num(ipAddr): return True hostIP = None + try: + hostIP = int(IPv4Address(hostonly)) + except AddressValueError: + pass for value in proxy_settings.get('exceptions', ()): # Items in the list are strings like these: *.local, 169.254/16 if not value: continue m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value) - if m is not None: - if hostIP is None: - try: - hostIP = socket.gethostbyname(hostonly) - hostIP = ip2num(hostIP) - except OSError: - continue - + if m is not None and hostIP is not None: base = ip2num(m.group(1)) mask = m.group(2) if mask is None: @@ -2615,6 +2613,31 @@ def ip2num(ipAddr): return False +# Same as _proxy_bypass_macosx_sysconf, testable on all platforms +def _proxy_bypass_winreg_override(host, override): + """Return True if the host should bypass the proxy server. + + The proxy override list is obtained from the Windows + Internet settings proxy override registry value. + + An example of a proxy override value is: + "www.example.com;*.example.net; 192.168.0.1" + """ + from fnmatch import fnmatch + + host, _ = _splitport(host) + proxy_override = override.split(';') + for test in proxy_override: + test = test.strip() + # "" should bypass the proxy server for all intranet addresses + if test == '': + if '.' not in host: + return True + elif fnmatch(host, test): + return True + return False + + if sys.platform == 'darwin': from _scproxy import _get_proxy_settings, _get_proxies @@ -2713,7 +2736,7 @@ def proxy_bypass_registry(host): import winreg except ImportError: # Std modules, so should be around - but you never know! - return 0 + return False try: internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') @@ -2723,40 +2746,10 @@ def proxy_bypass_registry(host): 'ProxyOverride')[0]) # ^^^^ Returned as Unicode but problems if not converted to ASCII except OSError: - return 0 + return False if not proxyEnable or not proxyOverride: - return 0 - # try to make a host list from name and IP address. - rawHost, port = _splitport(host) - host = [rawHost] - try: - addr = socket.gethostbyname(rawHost) - if addr != rawHost: - host.append(addr) - except OSError: - pass - try: - fqdn = socket.getfqdn(rawHost) - if fqdn != rawHost: - host.append(fqdn) - except OSError: - pass - # make a check value list from the registry entry: replace the - # '' string by the localhost entry and the corresponding - # canonical entry. - proxyOverride = proxyOverride.split(';') - # now check if we match one of the registry values. - for test in proxyOverride: - if test == '': - if '.' not in rawHost: - return 1 - test = test.replace(".", r"\.") # mask dots - test = test.replace("*", r".*") # change glob sequence - test = test.replace("?", r".") # change glob char - for val in host: - if re.match(test, val, re.I): - return 1 - return 0 + return False + return _proxy_bypass_winreg_override(host, proxyOverride) def proxy_bypass(host): """Return True, if host should be bypassed. diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index 636e8ca459d109..0424c53b7ccaf9 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -418,12 +418,18 @@ def register_X_browsers(): if shutil.which("gio"): register("gio", None, BackgroundBrowser(["gio", "open", "--", "%s"])) - # Equivalent of gio open before 2015 - if "GNOME_DESKTOP_SESSION_ID" in os.environ and shutil.which("gvfs-open"): + xdg_desktop = os.getenv("XDG_CURRENT_DESKTOP", "").split(":") + + # The default GNOME3 browser + if (("GNOME" in xdg_desktop or + "GNOME_DESKTOP_SESSION_ID" in os.environ) and + shutil.which("gvfs-open")): register("gvfs-open", None, BackgroundBrowser("gvfs-open")) # The default KDE browser - if "KDE_FULL_SESSION" in os.environ and shutil.which("kfmclient"): + if (("KDE" in xdg_desktop or + "KDE_FULL_SESSION" in os.environ) and + shutil.which("kfmclient")): register("kfmclient", Konqueror, Konqueror("kfmclient")) # Common symbolic link for the default X11 browser diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index a37fead41b750e..9e15d34d22aa6c 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -1320,6 +1320,11 @@ def read_events(self): else: yield event + def flush(self): + if self._parser is None: + raise ValueError("flush() called after end of stream") + self._parser.flush() + def XML(text, parser=None): """Parse XML document from string constant. @@ -1726,6 +1731,15 @@ def close(self): del self.parser, self._parser del self.target, self._target + def flush(self): + was_enabled = self.parser.GetReparseDeferralEnabled() + try: + self.parser.SetReparseDeferralEnabled(False) + self.parser.Parse(b"", False) + except self._error as v: + self._raiseerror(v) + finally: + self.parser.SetReparseDeferralEnabled(was_enabled) # -------------------------------------------------------------------- # C14N 2.0 diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py index b9ad52692db8dd..ba3c1e98517429 100644 --- a/Lib/xml/sax/expatreader.py +++ b/Lib/xml/sax/expatreader.py @@ -214,6 +214,20 @@ def feed(self, data, isFinal=False): # FIXME: when to invoke error()? self._err_handler.fatalError(exc) + def flush(self): + if self._parser is None: + return + + was_enabled = self._parser.GetReparseDeferralEnabled() + try: + self._parser.SetReparseDeferralEnabled(False) + self._parser.Parse(b"", False) + except expat.error as e: + exc = SAXParseException(expat.ErrorString(e.code), e, self) + self._err_handler.fatalError(exc) + finally: + self._parser.SetReparseDeferralEnabled(was_enabled) + def _close_source(self): source = self._source try: diff --git a/Makefile.pre.in b/Makefile.pre.in index 8252e6631c5af5..ee65ecd918ce2a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -41,7 +41,7 @@ AR= @AR@ READELF= @READELF@ SOABI= @SOABI@ LDVERSION= @LDVERSION@ -LIBPYTHON= @LIBPYTHON@ +MODULE_LDFLAGS=@MODULE_LDFLAGS@ GITVERSION= @GITVERSION@ GITTAG= @GITTAG@ GITBRANCH= @GITBRANCH@ @@ -446,6 +446,7 @@ PYTHON_OBJS= \ Python/object_stack.o \ Python/optimizer.o \ Python/optimizer_analysis.o \ + Python/optimizer_symbols.o \ Python/parking_lot.o \ Python/pathconfig.o \ Python/preconfig.o \ @@ -458,6 +459,7 @@ PYTHON_OBJS= \ Python/pystate.o \ Python/pythonrun.o \ Python/pytime.o \ + Python/qsbr.o \ Python/bootstrap_hash.o \ Python/specialize.o \ Python/structmember.o \ @@ -857,18 +859,16 @@ $(LIBRARY): $(LIBRARY_OBJS) $(AR) $(ARFLAGS) $@ $(LIBRARY_OBJS) libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS) - if test $(INSTSONAME) != $(LDLIBRARY); then \ - $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM) + if test $(INSTSONAME) != $@; then \ $(LN) -f $(INSTSONAME) $@; \ - else \ - $(BLDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ fi libpython3.so: libpython$(LDVERSION).so $(BLDSHARED) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^ libpython$(LDVERSION).dylib: $(LIBRARY_OBJS) - $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(PYTHONFRAMEWORKINSTALLNAMEPREFIX)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \ libpython$(VERSION).sl: $(LIBRARY_OBJS) @@ -911,6 +911,21 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ $(LN) -fsn Versions/Current/$(PYTHONFRAMEWORK) $(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK) $(LN) -fsn Versions/Current/Resources $(PYTHONFRAMEWORKDIR)/Resources +# This rule is for iOS, which requires an annoyingly just slighly different +# format for frameworks to macOS. It *doesn't* use a versioned framework, and +# the Info.plist must be in the root of the framework. +$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK): \ + $(LIBRARY) \ + $(RESSRCDIR)/Info.plist + $(INSTALL) -d -m $(DIRMODE) $(PYTHONFRAMEWORKDIR) + $(CC) -o $(LDLIBRARY) $(PY_CORE_LDFLAGS) -dynamiclib \ + -all_load $(LIBRARY) \ + -install_name $(PYTHONFRAMEWORKINSTALLNAMEPREFIX)/$(PYTHONFRAMEWORK) \ + -compatibility_version $(VERSION) \ + -current_version $(VERSION) \ + -framework CoreFoundation $(LIBS); + $(INSTALL_DATA) $(RESSRCDIR)/Info.plist $(PYTHONFRAMEWORKDIR)/Info.plist + # This rule builds the Cygwin Python DLL and import library if configured # for a shared core library; otherwise, this rule is a noop. $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) @@ -1162,6 +1177,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_pystats.h \ $(srcdir)/Include/internal/pycore_pythonrun.h \ $(srcdir)/Include/internal/pycore_pythread.h \ + $(srcdir)/Include/internal/pycore_qsbr.h \ $(srcdir)/Include/internal/pycore_range.h \ $(srcdir)/Include/internal/pycore_runtime.h \ $(srcdir)/Include/internal/pycore_runtime_init.h \ @@ -1625,10 +1641,10 @@ regen-unicodedata: regen-all: regen-cases regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ - regen-test-levenshtein regen-global-objects regen-sbom regen-jit + regen-test-levenshtein regen-global-objects regen-jit @echo @echo "Note: make regen-stdlib-module-names, make regen-limited-abi, " - @echo "make regen-configure and make regen-unicodedata should be run manually" + @echo "make regen-configure, make regen-sbom, and make regen-unicodedata should be run manually" ############################################################################ # Special rules for object files @@ -1872,9 +1888,9 @@ regen-cases: -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \ -o $(srcdir)/Python/executor_cases.c.h.new $(srcdir)/Python/bytecodes.c - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_abstract_generator.py \ - -o $(srcdir)/Python/tier2_redundancy_eliminator_cases.c.h.new \ - $(srcdir)/Python/tier2_redundancy_eliminator_bytecodes.c \ + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/optimizer_generator.py \ + -o $(srcdir)/Python/optimizer_cases.c.h.new \ + $(srcdir)/Python/optimizer_bytecodes.c \ $(srcdir)/Python/bytecodes.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_metadata_generator.py \ -o $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(srcdir)/Python/bytecodes.c @@ -1887,7 +1903,7 @@ regen-cases: $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_metadata.h $(srcdir)/Include/internal/pycore_uop_metadata.h.new $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new - $(UPDATE_FILE) $(srcdir)/Python/tier2_redundancy_eliminator_cases.c.h $(srcdir)/Python/tier2_redundancy_eliminator_cases.c.h.new + $(UPDATE_FILE) $(srcdir)/Python/optimizer_cases.c.h $(srcdir)/Python/optimizer_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Lib/_opcode_metadata.py $(srcdir)/Lib/_opcode_metadata.py.new Python/compile.o: $(srcdir)/Include/internal/pycore_opcode_metadata.h @@ -1910,7 +1926,7 @@ Python/optimizer.o: \ Python/optimizer_analysis.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h \ $(srcdir)/Include/internal/pycore_optimizer.h \ - $(srcdir)/Python/tier2_redundancy_eliminator_cases.c.h + $(srcdir)/Python/optimizer_cases.c.h Python/frozen.o: $(FROZEN_FILES_OUT) @@ -2605,10 +2621,11 @@ frameworkinstall: install # only have to cater for the structural bits of the framework. .PHONY: frameworkinstallframework -frameworkinstallframework: frameworkinstallstructure install frameworkinstallmaclib +frameworkinstallframework: @FRAMEWORKINSTALLFIRST@ install frameworkinstallmaclib -.PHONY: frameworkinstallstructure -frameworkinstallstructure: $(LDLIBRARY) +# macOS uses a versioned frameworks structure that includes a full install +.PHONY: frameworkinstallversionedstructure +frameworkinstallversionedstructure: $(LDLIBRARY) @if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ echo Not configured with --enable-framework; \ exit 1; \ @@ -2629,6 +2646,27 @@ frameworkinstallstructure: $(LDLIBRARY) $(LN) -fsn Versions/Current/Resources $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Resources $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY) +# iOS/tvOS/watchOS uses a non-versioned framework with Info.plist in the +# framework root, no .lproj data, and only stub compilation assistance binaries +.PHONY: frameworkinstallunversionedstructure +frameworkinstallunversionedstructure: $(LDLIBRARY) + @if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ + echo Not configured with --enable-framework; \ + exit 1; \ + else true; \ + fi + if test -d $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include; then \ + echo "Clearing stale header symlink directory"; \ + rm -rf $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include; \ + fi + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR) + sed 's/%VERSION%/'"`$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import platform; print(platform.python_version())'`"'/g' < $(RESSRCDIR)/Info.plist > $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Info.plist + $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY) + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(BINDIR) + for file in $(srcdir)/$(RESSRCDIR)/bin/* ; do \ + $(INSTALL) -m $(EXEMODE) $$file $(DESTDIR)$(BINDIR); \ + done + # This installs Mac/Lib into the framework # Install a number of symlinks to keep software that expects a normal unix # install (which includes python-config) happy. @@ -2669,6 +2707,19 @@ frameworkaltinstallunixtools: frameworkinstallextras: cd Mac && $(MAKE) installextras DESTDIR="$(DESTDIR)" +# On iOS, bin/lib can't live inside the framework; include needs to be called +# "Headers", but *must* be in the framework, and *not* include the `python3.X` +# subdirectory. The install has put these folders in the same folder as +# Python.framework; Move the headers to their final framework-compatible home. +.PHONY: frameworkinstallmobileheaders +frameworkinstallmobileheaders: + if test -d $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers; then \ + echo "Removing old framework headers"; \ + rm -rf $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers; \ + fi + mv "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(VERSION)" "$(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers" + $(LN) -fs "../$(PYTHONFRAMEWORKDIR)/Headers" "$(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include/python$(VERSION)" + # Build the toplevel Makefile Makefile.pre: $(srcdir)/Makefile.pre.in config.status CONFIG_FILES=Makefile.pre CONFIG_HEADERS= ./config.status @@ -2915,7 +2966,7 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h # force rebuild when header file or module build flavor (static/shared) is changed MODULE_DEPS_STATIC=Modules/config.c -MODULE_DEPS_SHARED=$(MODULE_DEPS_STATIC) $(EXPORTSYMS) +MODULE_DEPS_SHARED=@MODULE_DEPS_SHARED@ MODULE__CURSES_DEPS=$(srcdir)/Include/py_curses.h MODULE__CURSES_PANEL_DEPS=$(srcdir)/Include/py_curses.h diff --git a/Misc/ACKS b/Misc/ACKS index 8a80e02ecba26a..f01c7a70a65dc5 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -756,6 +756,7 @@ Raymond Hettinger Lisa Hewus Fresh Kevan Heydon Wouter van Heyst +Derek Higgins Kelsey Hightower Jason Hildebrand Ryan Hileman diff --git a/Misc/NEWS.d/3.10.0a6.rst b/Misc/NEWS.d/3.10.0a6.rst index c379b968c9885b..bad3528084897b 100644 --- a/Misc/NEWS.d/3.10.0a6.rst +++ b/Misc/NEWS.d/3.10.0a6.rst @@ -158,7 +158,7 @@ tests that are unrelated to :class:`ProcessPoolExecutor` on those platforms. .. nonce: hsCNgX .. section: Core and Builtins -If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator +If :func:`object.__ipow__` returns :data:`NotImplemented`, the operator will correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index e8d4a02a11e0f9..754e782dfe661b 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -660,7 +660,7 @@ Karabas. .. section: Core and Builtins Parameter substitution of the union type with wrong types now raises -``TypeError`` instead of returning ``NotImplemented``. +``TypeError`` instead of returning :data:`NotImplemented`. .. diff --git a/Misc/NEWS.d/next/Build/2024-02-13-14-52-59.gh-issue-114099.zjXsQr.rst b/Misc/NEWS.d/next/Build/2024-02-13-14-52-59.gh-issue-114099.zjXsQr.rst new file mode 100644 index 00000000000000..e2858bd71d28cb --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-13-14-52-59.gh-issue-114099.zjXsQr.rst @@ -0,0 +1,2 @@ +Makefile targets were added to support compiling an iOS-compatible framework +build. diff --git a/Misc/NEWS.d/next/Build/2024-02-21-11-58-30.gh-issue-115737.dpNl2T.rst b/Misc/NEWS.d/next/Build/2024-02-21-11-58-30.gh-issue-115737.dpNl2T.rst new file mode 100644 index 00000000000000..112f65258dd84b --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-21-11-58-30.gh-issue-115737.dpNl2T.rst @@ -0,0 +1,2 @@ +The install name for libPython is now correctly set for non-framework macOS +builds. diff --git a/Misc/NEWS.d/next/Build/2024-02-21-18-22-49.gh-issue-111225.Z8C3av.rst b/Misc/NEWS.d/next/Build/2024-02-21-18-22-49.gh-issue-111225.Z8C3av.rst new file mode 100644 index 00000000000000..8cdeba46ba2313 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-21-18-22-49.gh-issue-111225.Z8C3av.rst @@ -0,0 +1 @@ +Link extension modules against libpython on Android. diff --git a/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst b/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst new file mode 100644 index 00000000000000..5492a804255838 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-24-12-50-43.gh-issue-115350.naQA6y.rst @@ -0,0 +1 @@ +Fix building ctypes module with -DWIN32_LEAN_AND_MEAN defined diff --git a/Misc/NEWS.d/next/Build/2024-02-26-14-54-58.gh-issue-71052.XvFay1.rst b/Misc/NEWS.d/next/Build/2024-02-26-14-54-58.gh-issue-71052.XvFay1.rst new file mode 100644 index 00000000000000..bda91335814936 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-26-14-54-58.gh-issue-71052.XvFay1.rst @@ -0,0 +1 @@ +Fix several Android build issues diff --git a/Misc/NEWS.d/next/Build/2024-02-29-15-12-31.gh-issue-116117.eENkQK.rst b/Misc/NEWS.d/next/Build/2024-02-29-15-12-31.gh-issue-116117.eENkQK.rst new file mode 100644 index 00000000000000..22477b343c06f0 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-02-29-15-12-31.gh-issue-116117.eENkQK.rst @@ -0,0 +1,2 @@ +Backport ``libb2``'s PR #42 to fix compiling CPython on 32-bit Windows +with ``clang-cl``. diff --git a/Misc/NEWS.d/next/C API/2024-02-16-15-56-53.gh-issue-114626.ie2esA.rst b/Misc/NEWS.d/next/C API/2024-02-16-15-56-53.gh-issue-114626.ie2esA.rst new file mode 100644 index 00000000000000..763f4cee6d3f0b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-02-16-15-56-53.gh-issue-114626.ie2esA.rst @@ -0,0 +1,4 @@ +Add again ``_PyCFunctionFastWithKeywords`` name, removed in Python 3.13 +alpha 4 by mistake. Keep the old private ``_PyCFunctionFastWithKeywords`` +name (Python 3.7) as an alias to the new public name +``PyCFunctionFastWithKeywords`` (Python 3.13a4). Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-04-16-51-56.gh-issue-96497.HTBuIL.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-04-16-51-56.gh-issue-96497.HTBuIL.rst new file mode 100644 index 00000000000000..6881dde2e6cf44 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-09-04-16-51-56.gh-issue-96497.HTBuIL.rst @@ -0,0 +1,2 @@ +Fix incorrect resolution of mangled class variables used in assignment +expressions in comprehensions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-11-36-50.gh-issue-101860.CKCMbC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-11-36-50.gh-issue-101860.CKCMbC.rst new file mode 100644 index 00000000000000..5a274353466973 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-11-36-50.gh-issue-101860.CKCMbC.rst @@ -0,0 +1 @@ +Expose ``__name__`` attribute on property. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst new file mode 100644 index 00000000000000..9338f662374c9a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst @@ -0,0 +1,8 @@ +Improve the constructors for :mod:`ast` nodes. Arguments of list types now +default to an empty list if omitted, and optional fields default to ``None``. +AST nodes now have an +``__annotations__`` attribute with the expected types of their attributes. +Passing unrecognized extra arguments to AST nodes is deprecated and will +become an error in Python 3.15. Omitting a required argument to an AST node +is deprecated and will become an error in Python 3.15. Patch by Jelle +Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-16-15-02-47.gh-issue-104090.oMjNa9.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-16-15-02-47.gh-issue-104090.oMjNa9.rst new file mode 100644 index 00000000000000..e581d291d047ae --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-16-15-02-47.gh-issue-104090.oMjNa9.rst @@ -0,0 +1,2 @@ +The multiprocessing resource tracker now exits with non-zero status code if a resource +leak was detected. It still exits with status code 0 otherwise. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-08-16-01-18.gh-issue-115154.ji96FV.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-08-16-01-18.gh-issue-115154.ji96FV.rst new file mode 100644 index 00000000000000..045596bfcdca43 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-08-16-01-18.gh-issue-115154.ji96FV.rst @@ -0,0 +1,2 @@ +Fix a bug that was causing the :func:`tokenize.untokenize` function to +handle unicode named literals incorrectly. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst new file mode 100644 index 00000000000000..6d919134bf4d9c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-09-18-59-22.gh-issue-112175.qglugr.rst @@ -0,0 +1 @@ +Every ``PyThreadState`` now has its own ``eval_breaker``, allowing specific threads to be interrupted. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-12-23-29-17.gh-issue-115323.3t6687.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-12-23-29-17.gh-issue-115323.3t6687.rst new file mode 100644 index 00000000000000..171855608fbc6a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-12-23-29-17.gh-issue-115323.3t6687.rst @@ -0,0 +1,2 @@ +Make error message more meaningful for when :meth:`bytearray.extend` is +called with a :class:`str` object. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-20-12-46-20.gh-issue-115700.KLJ5r4.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-20-12-46-20.gh-issue-115700.KLJ5r4.rst new file mode 100644 index 00000000000000..5b7b8e410b5063 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-20-12-46-20.gh-issue-115700.KLJ5r4.rst @@ -0,0 +1 @@ +The regen-cases build stage now works on Windows. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-20-18-49-02.gh-issue-115733.51Zb85.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-20-18-49-02.gh-issue-115733.51Zb85.rst new file mode 100644 index 00000000000000..5cbb292065b5da --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-20-18-49-02.gh-issue-115733.51Zb85.rst @@ -0,0 +1 @@ +Fix crash when calling ``next()`` on exhausted list iterators. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-22-11-33-20.gh-issue-115778.jksd1D.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-22-11-33-20.gh-issue-115778.jksd1D.rst new file mode 100644 index 00000000000000..023f6aa3026733 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-22-11-33-20.gh-issue-115778.jksd1D.rst @@ -0,0 +1 @@ +Add ``tierN`` annotation for instruction definition in interpreter DSL. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst new file mode 100644 index 00000000000000..8cda4c9343d4d7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-22-16-17-53.gh-issue-115823.c1TreJ.rst @@ -0,0 +1,3 @@ +Properly calculate error ranges in the parser when raising +:exc:`SyntaxError` exceptions caused by invalid byte sequences. Patch by +Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-04-10-19-51.gh-issue-116296.gvtxyU.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-04-10-19-51.gh-issue-116296.gvtxyU.rst new file mode 100644 index 00000000000000..0781e9282205d1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-04-10-19-51.gh-issue-116296.gvtxyU.rst @@ -0,0 +1 @@ +Fix possible refleak in :meth:`!object.__reduce__` internal error handling. diff --git a/Misc/NEWS.d/next/Documentation/2024-02-14-20-17-04.gh-issue-115399.fb9a0R.rst b/Misc/NEWS.d/next/Documentation/2024-02-14-20-17-04.gh-issue-115399.fb9a0R.rst new file mode 100644 index 00000000000000..587aea802168bd --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-02-14-20-17-04.gh-issue-115399.fb9a0R.rst @@ -0,0 +1 @@ +Document CVE-2023-52425 of Expat <2.6.0 under "XML vulnerabilities". diff --git a/Misc/NEWS.d/next/IDLE/2023-12-09-11-04-26.gh-issue-88516.SIIvfs.rst b/Misc/NEWS.d/next/IDLE/2023-12-09-11-04-26.gh-issue-88516.SIIvfs.rst new file mode 100644 index 00000000000000..b6dea5029bf353 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2023-12-09-11-04-26.gh-issue-88516.SIIvfs.rst @@ -0,0 +1,2 @@ +On macOS show a proxy icon in the title bar of editor windows to match +platform behaviour. diff --git a/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst b/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst new file mode 100644 index 00000000000000..2a663ac7940dcb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-04-06-23-50-59.bpo-33775.0yhMDc.rst @@ -0,0 +1 @@ +Add 'default' and 'version' help text for localization in argparse. diff --git a/Misc/NEWS.d/next/Library/2020-05-29-18-08-54.bpo-40818.Ij8ffq.rst b/Misc/NEWS.d/next/Library/2020-05-29-18-08-54.bpo-40818.Ij8ffq.rst new file mode 100644 index 00000000000000..27f6a6daa52f53 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-29-18-08-54.bpo-40818.Ij8ffq.rst @@ -0,0 +1,3 @@ +The asyncio REPL now runs :data:`sys.__interactivehook__` on startup. The +default implementation of :data:`sys.__interactivehook__` provides +auto-completion to the asyncio REPL. Patch contributed by Rémi Lapeyre. diff --git a/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst b/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst new file mode 100644 index 00000000000000..76568d407449f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst @@ -0,0 +1,3 @@ +Failing to pass arguments properly to :func:`functools.singledispatchmethod` +now throws a TypeError instead of hitting an index out of bounds +internally. diff --git a/Misc/NEWS.d/next/Library/2020-12-15-22-30-49.bpo-42125.UGyseY.rst b/Misc/NEWS.d/next/Library/2020-12-15-22-30-49.bpo-42125.UGyseY.rst new file mode 100644 index 00000000000000..49d4462e257702 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-12-15-22-30-49.bpo-42125.UGyseY.rst @@ -0,0 +1,2 @@ +linecache: get module name from ``__spec__`` if available. This allows getting +source code for the ``__main__`` module when a custom loader is used. diff --git a/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst b/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst new file mode 100644 index 00000000000000..e164619e44a301 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-05-03-11-04-12.bpo-43952.Me7fJe.rst @@ -0,0 +1,2 @@ +Fix :meth:`multiprocessing.connection.Listener.accept()` to accept empty bytes +as authkey. Not accepting empty bytes as key causes it to hang indefinitely. diff --git a/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst b/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst new file mode 100644 index 00000000000000..ecdb26cdd6edd6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-24-20-47-37.bpo-44865.c3BhZS.rst @@ -0,0 +1 @@ +Add missing call to localization function in :mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2022-01-14-10-50-17.bpo-31116.0bduV9.rst b/Misc/NEWS.d/next/Library/2022-01-14-10-50-17.bpo-31116.0bduV9.rst new file mode 100644 index 00000000000000..d77a96b442bcbb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-01-14-10-50-17.bpo-31116.0bduV9.rst @@ -0,0 +1 @@ +Add Z85 encoding to ``base64``. diff --git a/Misc/NEWS.d/next/Library/2022-05-25-17-49-04.gh-issue-93205.DjhFVR.rst b/Misc/NEWS.d/next/Library/2022-05-25-17-49-04.gh-issue-93205.DjhFVR.rst new file mode 100644 index 00000000000000..4a280b93d93347 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-25-17-49-04.gh-issue-93205.DjhFVR.rst @@ -0,0 +1 @@ +Fixed a bug in :class:`logging.handlers.TimedRotatingFileHandler` where multiple rotating handler instances pointing to files with the same name but different extensions would conflict and not delete the correct files. diff --git a/Misc/NEWS.d/next/Library/2022-08-26-15-50-53.gh-issue-96310.0NssDh.rst b/Misc/NEWS.d/next/Library/2022-08-26-15-50-53.gh-issue-96310.0NssDh.rst new file mode 100644 index 00000000000000..f8efb0002e104a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-08-26-15-50-53.gh-issue-96310.0NssDh.rst @@ -0,0 +1,2 @@ +Fix a traceback in :mod:`argparse` when all options in a mutually exclusive +group are suppressed. diff --git a/Misc/NEWS.d/next/Library/2022-11-22-23-17-43.gh-issue-95782.an_and.rst b/Misc/NEWS.d/next/Library/2022-11-22-23-17-43.gh-issue-95782.an_and.rst new file mode 100644 index 00000000000000..123c3944aa3a3a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-22-23-17-43.gh-issue-95782.an_and.rst @@ -0,0 +1,4 @@ +Fix :func:`io.BufferedReader.tell`, :func:`io.BufferedReader.seek`, +:func:`_pyio.BufferedReader.tell`, :func:`io.BufferedRandom.tell`, +:func:`io.BufferedRandom.seek` and :func:`_pyio.BufferedRandom.tell` +being able to return negative offsets. diff --git a/Misc/NEWS.d/next/Library/2023-01-09-14-08-02.gh-issue-100884.DcmdLl.rst b/Misc/NEWS.d/next/Library/2023-01-09-14-08-02.gh-issue-100884.DcmdLl.rst new file mode 100644 index 00000000000000..2a388178810835 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-09-14-08-02.gh-issue-100884.DcmdLl.rst @@ -0,0 +1,2 @@ +email: fix misfolding of comma in address-lists over multiple lines in +combination with unicode encoding. diff --git a/Misc/NEWS.d/next/Library/2023-01-12-14-16-01.gh-issue-100985.GT5Fvd.rst b/Misc/NEWS.d/next/Library/2023-01-12-14-16-01.gh-issue-100985.GT5Fvd.rst new file mode 100644 index 00000000000000..8d8693a5edb3d4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-12-14-16-01.gh-issue-100985.GT5Fvd.rst @@ -0,0 +1,2 @@ +Update HTTPSConnection to consistently wrap IPv6 Addresses when using a +proxy. diff --git a/Misc/NEWS.d/next/Library/2023-02-14-17-19-59.gh-issue-72249.fv35wU.rst b/Misc/NEWS.d/next/Library/2023-02-14-17-19-59.gh-issue-72249.fv35wU.rst new file mode 100644 index 00000000000000..10cc5a4e7528dd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-14-17-19-59.gh-issue-72249.fv35wU.rst @@ -0,0 +1,2 @@ +:func:`functools.partial`s of :func:`repr` has been improved to include the +:term:`module` name. Patched by Furkan Onder and Anilyka Barry. diff --git a/Misc/NEWS.d/next/Library/2023-03-03-09-05-42.gh-issue-102389.ucmo0_.rst b/Misc/NEWS.d/next/Library/2023-03-03-09-05-42.gh-issue-102389.ucmo0_.rst new file mode 100644 index 00000000000000..8c11567d79ba7b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-03-09-05-42.gh-issue-102389.ucmo0_.rst @@ -0,0 +1 @@ +Add ``windows_31j`` to aliases for ``cp932`` codec diff --git a/Misc/NEWS.d/next/Library/2023-04-02-21-20-35.gh-issue-60346.7mjgua.rst b/Misc/NEWS.d/next/Library/2023-04-02-21-20-35.gh-issue-60346.7mjgua.rst new file mode 100644 index 00000000000000..c15bd6ed11d17f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-02-21-20-35.gh-issue-60346.7mjgua.rst @@ -0,0 +1 @@ +Fix ArgumentParser inconsistent with parse_known_args. diff --git a/Misc/NEWS.d/next/Library/2023-05-01-22-28-57.gh-issue-104061.vxfBXf.rst b/Misc/NEWS.d/next/Library/2023-05-01-22-28-57.gh-issue-104061.vxfBXf.rst new file mode 100644 index 00000000000000..e15a811f904352 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-01-22-28-57.gh-issue-104061.vxfBXf.rst @@ -0,0 +1 @@ +Add :data:`socket.SO_BINDTOIFINDEX` constant. diff --git a/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst b/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst new file mode 100644 index 00000000000000..b0cdf44f7b9e39 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst @@ -0,0 +1 @@ +:meth:`Profile.print_stats` has been improved to accept multiple sort arguments. Patched by Chiu-Hsiang Hsu and Furkan Onder. diff --git a/Misc/NEWS.d/next/Library/2023-07-12-14-52-04.gh-issue-57141.L2k8Xb.rst b/Misc/NEWS.d/next/Library/2023-07-12-14-52-04.gh-issue-57141.L2k8Xb.rst new file mode 100644 index 00000000000000..b8a1236ec3edb0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-12-14-52-04.gh-issue-57141.L2k8Xb.rst @@ -0,0 +1,3 @@ +Add option for *non-shallow* comparisons to :class:`filecmp.dircmp` like +:func:`filecmp.cmp`. Original patch by Steven Ward. Enhanced by +Tobias Rautenkranz diff --git a/Misc/NEWS.d/next/Library/2023-08-02-01-17-32.gh-issue-107155.Mj1K9L.rst b/Misc/NEWS.d/next/Library/2023-08-02-01-17-32.gh-issue-107155.Mj1K9L.rst new file mode 100644 index 00000000000000..8362dc0fcfaa74 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-02-01-17-32.gh-issue-107155.Mj1K9L.rst @@ -0,0 +1,3 @@ +Fix incorrect output of ``help(x)`` where ``x`` is a :keyword:`lambda` +function, which has an ``__annotations__`` dictionary attribute with a +``"return"`` key. diff --git a/Misc/NEWS.d/next/Library/2023-11-07-10-22-06.gh-issue-111775.IoVxfX.rst b/Misc/NEWS.d/next/Library/2023-11-07-10-22-06.gh-issue-111775.IoVxfX.rst new file mode 100644 index 00000000000000..2a3bdd640ea67d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-07-10-22-06.gh-issue-111775.IoVxfX.rst @@ -0,0 +1,2 @@ +Fix :meth:`importlib.resources.simple.ResourceHandle.open` for text mode, +added missed ``stream`` argument. diff --git a/Misc/NEWS.d/next/Library/2023-11-20-16-15-44.gh-issue-112281.gH4EVk.rst b/Misc/NEWS.d/next/Library/2023-11-20-16-15-44.gh-issue-112281.gH4EVk.rst new file mode 100644 index 00000000000000..01f6689bb471cd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-20-16-15-44.gh-issue-112281.gH4EVk.rst @@ -0,0 +1,2 @@ +Allow creating :ref:`union of types` for +:class:`typing.Annotated` with unhashable metadata. diff --git a/Misc/NEWS.d/next/Library/2024-01-26-16-42-31.gh-issue-114610.S18Vuz.rst b/Misc/NEWS.d/next/Library/2024-01-26-16-42-31.gh-issue-114610.S18Vuz.rst new file mode 100644 index 00000000000000..519aede72aaf29 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-26-16-42-31.gh-issue-114610.S18Vuz.rst @@ -0,0 +1,4 @@ +Fix bug where :meth:`pathlib.PurePath.with_stem` converted a non-empty path +suffix to a stem when given an empty *stem* argument. It now raises +:exc:`ValueError`, just like :meth:`pathlib.PurePath.with_suffix` does when +called on a path with an empty stem, given a non-empty *suffix* argument. diff --git a/Misc/NEWS.d/next/Library/2024-01-29-13-46-41.gh-issue-114709.SQ998l.rst b/Misc/NEWS.d/next/Library/2024-01-29-13-46-41.gh-issue-114709.SQ998l.rst new file mode 100644 index 00000000000000..ca0d7902c73d1c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-29-13-46-41.gh-issue-114709.SQ998l.rst @@ -0,0 +1,5 @@ +:func:`posixpath.commonpath()` now raises a :exc:`ValueError` exception when +passed an empty iterable. Previously, :exc:`IndexError` was raised. + +:func:`posixpath.commonpath()` now raises a :exc:`TypeError` exception when +passed ``None``. Previously, :exc:`ValueError` was raised. diff --git a/Misc/NEWS.d/next/Library/2024-01-30-23-28-29.gh-issue-114763.BRjKkg.rst b/Misc/NEWS.d/next/Library/2024-01-30-23-28-29.gh-issue-114763.BRjKkg.rst new file mode 100644 index 00000000000000..e8bdb83dde61fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-30-23-28-29.gh-issue-114763.BRjKkg.rst @@ -0,0 +1,3 @@ +Protect modules loaded with :class:`importlib.util.LazyLoader` from race +conditions when multiple threads try to access attributes before the loading +is complete. diff --git a/Misc/NEWS.d/next/Library/2024-02-09-12-22-47.gh-issue-113812.wOraaG.rst b/Misc/NEWS.d/next/Library/2024-02-09-12-22-47.gh-issue-113812.wOraaG.rst new file mode 100644 index 00000000000000..7ef7bc891cd885 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-09-12-22-47.gh-issue-113812.wOraaG.rst @@ -0,0 +1,3 @@ +:meth:`DatagramTransport.sendto` will now send zero-length datagrams if +called with an empty bytes object. The transport flow control also now +accounts for the datagram header when calculating the buffer size. diff --git a/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst b/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst new file mode 100644 index 00000000000000..e6ca3cc525d74a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-09-19-41-48.gh-issue-115197.20wkWH.rst @@ -0,0 +1,2 @@ +``urllib.request`` no longer resolves the hostname before checking it +against the system's proxy bypass list on macOS and Windows. diff --git a/Misc/NEWS.d/next/Library/2024-02-10-17-18-49.gh-issue-115256.41Fy9P.rst b/Misc/NEWS.d/next/Library/2024-02-10-17-18-49.gh-issue-115256.41Fy9P.rst new file mode 100644 index 00000000000000..8cde053d862298 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-10-17-18-49.gh-issue-115256.41Fy9P.rst @@ -0,0 +1,5 @@ +Added DeprecationWarning when accessing the tarfile attribute of TarInfo +objects. The attribute is never used internally and is only attached to +TarInfos when the tarfile is opened in write-mode, not read-mode. The +attribute creates an unnecessary reference cycle which may cause +corruption when not closing the handle after writing a tarfile. diff --git a/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst b/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst new file mode 100644 index 00000000000000..2da43a43b78798 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-11-20-12-39.gh-issue-113942.i72sMJ.rst @@ -0,0 +1,2 @@ +:mod:`pydoc` no longer skips global functions implemented as builtin methods, +such as :class:`~type.MethodDescriptorType` and :class:`~type.WrapperDescriptorType`. diff --git a/Misc/NEWS.d/next/Library/2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst b/Misc/NEWS.d/next/Library/2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst new file mode 100644 index 00000000000000..47701396c81737 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-12-11-42-48.gh-issue-103092.sGMKr0.rst @@ -0,0 +1 @@ +Isolate :mod:`_lsprof` (apply :pep:`687`). diff --git a/Misc/NEWS.d/next/Library/2024-02-15-19-11-49.gh-issue-101293.898b8l.rst b/Misc/NEWS.d/next/Library/2024-02-15-19-11-49.gh-issue-101293.898b8l.rst new file mode 100644 index 00000000000000..98365d2edbc4b5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-15-19-11-49.gh-issue-101293.898b8l.rst @@ -0,0 +1,4 @@ +Support callables with the ``__call__()`` method and types with +``__new__()`` and ``__init__()`` methods set to class methods, static +methods, bound methods, partial functions, and other types of methods and +descriptors in :meth:`inspect.Signature.from_callable`. diff --git a/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst b/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst new file mode 100644 index 00000000000000..32af2bd24e54f2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-15-23-42-54.gh-issue-112006.4wxcK-.rst @@ -0,0 +1,3 @@ +Fix :func:`inspect.unwrap` for types with the ``__wrapper__`` data +descriptor. Fix :meth:`inspect.Signature.from_callable` for builtins +:func:`classmethod` and :func:`staticmethod`. diff --git a/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst b/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst new file mode 100644 index 00000000000000..32916ede4dee35 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst @@ -0,0 +1,2 @@ +Refactor :class:`dis.ArgResolver` to make it possible to subclass and change +the way jump args are interpreted. diff --git a/Misc/NEWS.d/next/Library/2024-02-17-18-47-12.gh-issue-115618.napiNp.rst b/Misc/NEWS.d/next/Library/2024-02-17-18-47-12.gh-issue-115618.napiNp.rst new file mode 100644 index 00000000000000..cb4b147d5dc663 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-17-18-47-12.gh-issue-115618.napiNp.rst @@ -0,0 +1,3 @@ +Fix improper decreasing the reference count for ``None`` argument in +:class:`property` methods :meth:`~property.getter`, :meth:`~property.setter` +and :meth:`~property.deleter`. diff --git a/Misc/NEWS.d/next/Library/2024-02-18-12-18-12.gh-issue-111358.9yJUMD.rst b/Misc/NEWS.d/next/Library/2024-02-18-12-18-12.gh-issue-111358.9yJUMD.rst new file mode 100644 index 00000000000000..2e895f8f181ce7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-18-12-18-12.gh-issue-111358.9yJUMD.rst @@ -0,0 +1,2 @@ +Fix a bug in :meth:`asyncio.BaseEventLoop.shutdown_default_executor` to +ensure the timeout passed to the coroutine behaves as expected. diff --git a/Misc/NEWS.d/next/Library/2024-02-19-15-52-30.gh-issue-114914.M5-1d8.rst b/Misc/NEWS.d/next/Library/2024-02-19-15-52-30.gh-issue-114914.M5-1d8.rst new file mode 100644 index 00000000000000..f7d392c8bceea3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-19-15-52-30.gh-issue-114914.M5-1d8.rst @@ -0,0 +1,2 @@ +Fix an issue where an abandoned :class:`StreamWriter` would not be garbage +collected. diff --git a/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst b/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst new file mode 100644 index 00000000000000..4f97b2d6085ba0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-19-16-53-48.gh-issue-112997.sYBXRZ.rst @@ -0,0 +1,2 @@ +Stop logging potentially sensitive callback arguments in :mod:`asyncio` +unless debug mode is active. diff --git a/Misc/NEWS.d/next/Library/2024-02-20-07-38-15.gh-issue-112364.EX7uGI.rst b/Misc/NEWS.d/next/Library/2024-02-20-07-38-15.gh-issue-112364.EX7uGI.rst new file mode 100644 index 00000000000000..6af71e60ec2a8e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-20-07-38-15.gh-issue-112364.EX7uGI.rst @@ -0,0 +1 @@ +Fixed :func:`ast.unparse` to handle format_spec with ``"``, ``'`` or ``\\``. Patched by Frank Hoffmann. diff --git a/Misc/NEWS.d/next/Library/2024-02-20-16-42-54.gh-issue-115712.EXVMXw.rst b/Misc/NEWS.d/next/Library/2024-02-20-16-42-54.gh-issue-115712.EXVMXw.rst new file mode 100644 index 00000000000000..8b19064dba779d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-20-16-42-54.gh-issue-115712.EXVMXw.rst @@ -0,0 +1,4 @@ +Restore support of space delimiter with ``skipinitialspace=True`` in +:mod:`csv`. :func:`csv.writer()` now quotes empty fields if delimiter is a +space and skipinitialspace is true and raises exception if quoting is not +possible. diff --git a/Misc/NEWS.d/next/Library/2024-02-20-22-02-34.gh-issue-67044.QF9_Ru.rst b/Misc/NEWS.d/next/Library/2024-02-20-22-02-34.gh-issue-67044.QF9_Ru.rst new file mode 100644 index 00000000000000..095e69b6cadab6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-20-22-02-34.gh-issue-67044.QF9_Ru.rst @@ -0,0 +1,2 @@ +:func:`csv.writer` now always quotes or escapes ``'\r'`` and ``'\n'``, +regardless of *lineterminator* value. diff --git a/Misc/NEWS.d/next/Library/2024-02-22-11-29-27.gh-issue-115809.9H1DhB.rst b/Misc/NEWS.d/next/Library/2024-02-22-11-29-27.gh-issue-115809.9H1DhB.rst new file mode 100644 index 00000000000000..2a47efbae5c84d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-22-11-29-27.gh-issue-115809.9H1DhB.rst @@ -0,0 +1,4 @@ +Improve algorithm for computing which rolled-over log files to delete in +:class:`logging.TimedRotatingFileHandler`. It is now reliable for handlers +without ``namer`` and with arbitrary deterministic ``namer`` that leaves the +datetime part in the file name unmodified. diff --git a/Misc/NEWS.d/next/Library/2024-02-22-12-10-18.gh-issue-115714.P2JsU1.rst b/Misc/NEWS.d/next/Library/2024-02-22-12-10-18.gh-issue-115714.P2JsU1.rst new file mode 100644 index 00000000000000..fb626344c87fdb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-22-12-10-18.gh-issue-115714.P2JsU1.rst @@ -0,0 +1,4 @@ +On WASI, the :mod:`time` module no longer get process time using ``times()`` +or ``CLOCK_PROCESS_CPUTIME_ID``, system API is that is unreliable and is +likely to be removed from WASI. The affected clock functions fall back to +calling ``clock()``. diff --git a/Misc/NEWS.d/next/Library/2024-02-23-11-08-31.gh-issue-115532.zVd3gK.rst b/Misc/NEWS.d/next/Library/2024-02-23-11-08-31.gh-issue-115532.zVd3gK.rst new file mode 100644 index 00000000000000..fb36c0b2a4fabd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-23-11-08-31.gh-issue-115532.zVd3gK.rst @@ -0,0 +1 @@ +Add kernel density estimation to the statistics module. diff --git a/Misc/NEWS.d/next/Library/2024-02-24-18-48-14.gh-issue-115886.rgM6AF.rst b/Misc/NEWS.d/next/Library/2024-02-24-18-48-14.gh-issue-115886.rgM6AF.rst new file mode 100644 index 00000000000000..9688f713d5ba7b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-24-18-48-14.gh-issue-115886.rgM6AF.rst @@ -0,0 +1,2 @@ +Fix silent truncation of the name with an embedded null character in +:class:`multiprocessing.shared_memory.SharedMemory`. diff --git a/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst b/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst new file mode 100644 index 00000000000000..99bccb265ff80c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-25-19-20-05.gh-issue-115881.ro_Kuw.rst @@ -0,0 +1,4 @@ +Fix issue where :func:`ast.parse` would incorrectly flag conditional context +managers (such as ``with (x() if y else z()): ...``) as invalid syntax if +``feature_version=(3, 8)`` was passed. This reverts changes to the +grammar made as part of gh-94949. diff --git a/Misc/NEWS.d/next/Library/2024-02-27-20-11-29.gh-issue-85644.3rgcBm.rst b/Misc/NEWS.d/next/Library/2024-02-27-20-11-29.gh-issue-85644.3rgcBm.rst new file mode 100644 index 00000000000000..818f7046229878 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-27-20-11-29.gh-issue-85644.3rgcBm.rst @@ -0,0 +1,2 @@ +Use the ``XDG_CURRENT_DESKTOP`` environment variable in :mod:`webbrowser` to check desktop. +Prefer it to the deprecated ``GNOME_DESKTOP_SESSION_ID`` for GNOME detection. diff --git a/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst b/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst new file mode 100644 index 00000000000000..7512a09a37cd46 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-28-12-14-31.gh-issue-115821.YO2vKA.rst @@ -0,0 +1,2 @@ +[Enum] Improve error message when calling super().__new__() in custom +__new__. diff --git a/Misc/NEWS.d/next/Library/2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst b/Misc/NEWS.d/next/Library/2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst new file mode 100644 index 00000000000000..7bc6ced120a7be --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-28-17-04-28.gh-issue-65824.gG8KR1.rst @@ -0,0 +1 @@ +Improve the ``less`` prompt in :mod:`pydoc`. diff --git a/Misc/NEWS.d/next/Library/2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst b/Misc/NEWS.d/next/Library/2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst new file mode 100644 index 00000000000000..844340583cd456 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-29-20-06-06.gh-issue-87115.FVMiOR.rst @@ -0,0 +1 @@ +Set ``__main__.__spec__`` to ``None`` when running a script with :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2024-03-01-11-57-32.gh-issue-88352.bZ68rw.rst b/Misc/NEWS.d/next/Library/2024-03-01-11-57-32.gh-issue-88352.bZ68rw.rst new file mode 100644 index 00000000000000..8ad4ff7cb52414 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-01-11-57-32.gh-issue-88352.bZ68rw.rst @@ -0,0 +1,6 @@ +Fix the computation of the next rollover time in the +:class:`logging.TimedRotatingFileHandler` handler. :meth:`!computeRollover` +now always returns a timestamp larger than the specified time and works +correctly during the DST change. :meth:`!doRollover` no longer overwrite the +already rolled over file, saving from data loss when run at midnight or +during repeated time at the DST change. diff --git a/Misc/NEWS.d/next/Library/2024-03-01-14-22-08.gh-issue-115978.r2ePTo.rst b/Misc/NEWS.d/next/Library/2024-03-01-14-22-08.gh-issue-115978.r2ePTo.rst new file mode 100644 index 00000000000000..2adac31ac6c21d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-01-14-22-08.gh-issue-115978.r2ePTo.rst @@ -0,0 +1,4 @@ +Disable preadv(), readv(), pwritev(), and writev() on WASI. + +Under wasmtime for WASI 0.2, these functions don't pass test_posix +(https://github.com/bytecodealliance/wasmtime/issues/7830). diff --git a/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst b/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst new file mode 100644 index 00000000000000..a40fcd35ef99ae --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-02-18-03-14-40.gh-issue-115398.tzvxH8.rst @@ -0,0 +1,8 @@ +Allow controlling Expat >=2.6.0 reparse deferral (CVE-2023-52425) by adding +five new methods: + +* :meth:`xml.etree.ElementTree.XMLParser.flush` +* :meth:`xml.etree.ElementTree.XMLPullParser.flush` +* :meth:`xml.parsers.expat.xmlparser.GetReparseDeferralEnabled` +* :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` +* :meth:`xml.sax.expatreader.ExpatParser.flush` diff --git a/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst b/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst new file mode 100644 index 00000000000000..c2811b133d9314 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst @@ -0,0 +1,2 @@ +On Windows, commas passed in arguments to ``Tools\buildbot\test.bat`` and +``PCbuild\\rt.bat`` are now properly handled. diff --git a/Misc/NEWS.d/next/Tests/2024-02-17-08-25-01.gh-issue-115596.RGPCrR.rst b/Misc/NEWS.d/next/Tests/2024-02-17-08-25-01.gh-issue-115596.RGPCrR.rst new file mode 100644 index 00000000000000..2bcb8b9ac6bcd4 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-17-08-25-01.gh-issue-115596.RGPCrR.rst @@ -0,0 +1,2 @@ +Fix ``ProgramPriorityTests`` in ``test_os`` permanently changing the process +priority. diff --git a/Misc/NEWS.d/next/Tests/2024-02-18-14-20-52.gh-issue-115122.3rGNo9.rst b/Misc/NEWS.d/next/Tests/2024-02-18-14-20-52.gh-issue-115122.3rGNo9.rst new file mode 100644 index 00000000000000..e187a40a40516b --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-18-14-20-52.gh-issue-115122.3rGNo9.rst @@ -0,0 +1,2 @@ +Add ``--bisect`` option to regrtest test runner: run failed tests with +``test.bisect_cmd`` to identify failing tests. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst b/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst new file mode 100644 index 00000000000000..a03ee11d974251 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-20-15-47-41.gh-issue-115720.w8i8UG.rst @@ -0,0 +1,2 @@ +Leak tests (``-R``, ``--huntrleaks``) now show a summary of the number of +leaks found in each iteration. diff --git a/Misc/NEWS.d/next/Tests/2024-02-22-00-17-06.gh-issue-115796.d4hpKy.rst b/Misc/NEWS.d/next/Tests/2024-02-22-00-17-06.gh-issue-115796.d4hpKy.rst new file mode 100644 index 00000000000000..a40be74f73908e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-22-00-17-06.gh-issue-115796.d4hpKy.rst @@ -0,0 +1,2 @@ +Make '_testinternalcapi.assemble_code_object' construct the exception table +for the code object. diff --git a/Misc/NEWS.d/next/Tests/2024-02-25-15-58-28.gh-issue-71052.lxBjqY.rst b/Misc/NEWS.d/next/Tests/2024-02-25-15-58-28.gh-issue-71052.lxBjqY.rst new file mode 100644 index 00000000000000..8bac68b5aea78e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-25-15-58-28.gh-issue-71052.lxBjqY.rst @@ -0,0 +1,2 @@ +Enable ``test_concurrent_futures`` on platforms that support threading but not +multiprocessing. diff --git a/Misc/NEWS.d/next/Tests/2024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst b/Misc/NEWS.d/next/Tests/2024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst new file mode 100644 index 00000000000000..9d3467ca7e7d40 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-02-25-16-28-26.gh-issue-71052.lSb9EC.rst @@ -0,0 +1 @@ +Add test exclusions to support running the test suite on Android. diff --git a/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst b/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst new file mode 100644 index 00000000000000..48a09da7822915 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2021-09-05-02-47-48.bpo-45101.60Zqmt.rst @@ -0,0 +1 @@ +Add consistency in usage message IO between 2 versions of python-config. diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-02-12-19-28-08.gh-issue-100176.Kzs4Zw.rst b/Misc/NEWS.d/next/Tools-Demos/2023-02-12-19-28-08.gh-issue-100176.Kzs4Zw.rst new file mode 100644 index 00000000000000..1a9fc76d93f297 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-02-12-19-28-08.gh-issue-100176.Kzs4Zw.rst @@ -0,0 +1 @@ +Remove outdated Tools/{io,cc,string}bench diff --git a/Misc/NEWS.d/next/Windows/2024-02-15-23-16-31.gh-issue-115543.otrWnw.rst b/Misc/NEWS.d/next/Windows/2024-02-15-23-16-31.gh-issue-115543.otrWnw.rst new file mode 100644 index 00000000000000..ebd15c83b83491 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-15-23-16-31.gh-issue-115543.otrWnw.rst @@ -0,0 +1,3 @@ +:ref:`launcher` can now detect Python 3.13 when installed from the Microsoft +Store, and will install Python 3.12 by default when +:envvar:`PYLAUNCHER_ALLOW_INSTALL` is set. diff --git a/Misc/NEWS.d/next/Windows/2024-02-21-23-48-59.gh-issue-115554.02mpQC.rst b/Misc/NEWS.d/next/Windows/2024-02-21-23-48-59.gh-issue-115554.02mpQC.rst new file mode 100644 index 00000000000000..b3c078b578205e --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-21-23-48-59.gh-issue-115554.02mpQC.rst @@ -0,0 +1,6 @@ +The installer now has more strict rules about updating the :ref:`launcher`. +In general, most users only have a single launcher installed and will see no +difference. When multiple launchers have been installed, the option to +install the launcher is disabled until all but one have been removed. +Downgrading the launcher (which was never allowed) is now more obviously +blocked. diff --git a/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst b/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst new file mode 100644 index 00000000000000..f2e82bf6a3e028 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-23-11-43-43.gh-issue-115582.sk1XPi.rst @@ -0,0 +1,3 @@ +Building extensions intended for free-threaded builds of CPython now require +compiling with ``/DPy_GIL_DISABLED`` manually when using a regular install. This +is expected to change in future releases. diff --git a/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst b/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst new file mode 100644 index 00000000000000..a55e5b1c7b566d --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-27-23-21-55.gh-issue-116012.B9_IwM.rst @@ -0,0 +1 @@ +Ensure the value of ``GetLastError()`` is preserved across GIL operations. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json new file mode 100644 index 00000000000000..2acfccbb004d6b --- /dev/null +++ b/Misc/externals.spdx.json @@ -0,0 +1,174 @@ +{ + "SPDXID": "SPDXRef-DOCUMENT", + "packages": [ + { + "SPDXID": "SPDXRef-PACKAGE-bzip2", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "ab8d1b0cc087c20d4c32c0e4fcf7d0c733a95da12cedc6d63b3f0a9af07427e2" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/bzip2-1.0.8.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:bzip:bzip2:1.0.8:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "bzip2", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0.8" + }, + { + "SPDXID": "SPDXRef-PACKAGE-libffi", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "9d802681adfea27d84cae0487a785fb9caa925bdad44c401b364c59ab2b8edda" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/libffi-3.4.4.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:libffi_project:libffi:3.4.4:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "libffi", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.4.4" + }, + { + "SPDXID": "SPDXRef-PACKAGE-openssl", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "e6a77c273ebb284fedd8ea19b081fce74a9455936ffd47215f7c24713e2614b2" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.13.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.13:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "openssl", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.0.13" + }, + { + "SPDXID": "SPDXRef-PACKAGE-sqlite", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "6f0364a27375435a34137b138ca4fedef8d23eec6493ca1dfff33bfc0c34fda4" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.45.1.0.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.45.1.0:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "sqlite", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.45.1.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-tcl-core", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "1d3f2015e49e269cf681373d433cd54d88d5ef7443fe87f5f50f5fcfe9003e73" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.13.1.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.13.1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "tcl-core", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "8.6.13.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-tk", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "6056203b8a6aaf6ea89d90a7b55dc7f407e55c093f731a98fd830a712a3c81d3" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.13.1.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.13.1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "tk", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "8.6.13.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-xz", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a15c168e39e87d750c3dc766edc7f19bdda57dacf01e509678467eace91ad282" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/xz-5.2.5.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:xz_project:xz:5.2.5:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "xz", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "5.2.5" + }, + { + "SPDXID": "SPDXRef-PACKAGE-zlib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "e3f3fb32564952006eb18b091ca8464740e5eca29d328cfb0b2da22768e0b638" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-1.3.1.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:zlib:zlib:1.3.1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "zlib", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.3.1" + } + ], + "spdxVersion": "SPDX-2.3" +} \ No newline at end of file diff --git a/Misc/platform_triplet.c b/Misc/platform_triplet.c index 3307260544e8a6..0b912e332510a6 100644 --- a/Misc/platform_triplet.c +++ b/Misc/platform_triplet.c @@ -233,7 +233,22 @@ PLATFORM_TRIPLET=i386-gnu # error unknown platform triplet # endif #elif defined(__APPLE__) +# include "TargetConditionals.h" +# if TARGET_OS_IOS +# if TARGET_OS_SIMULATOR +# if __x86_64__ +PLATFORM_TRIPLET=x86_64-iphonesimulator +# else +PLATFORM_TRIPLET=arm64-iphonesimulator +# endif +# else +PLATFORM_TRIPLET=arm64-iphoneos +# endif +# elif TARGET_OS_OSX PLATFORM_TRIPLET=darwin +# else +# error unknown Apple platform +# endif #elif defined(__VXWORKS__) PLATFORM_TRIPLET=vxworks #elif defined(__wasm32__) diff --git a/Misc/python-config.in b/Misc/python-config.in index 81c3316e334a48..dd5d161ab2286f 100644 --- a/Misc/python-config.in +++ b/Misc/python-config.in @@ -13,7 +13,8 @@ valid_opts = ['prefix', 'exec-prefix', 'includes', 'libs', 'cflags', def exit_with_usage(code=1): print("Usage: {0} [{1}]".format( - sys.argv[0], '|'.join('--'+opt for opt in valid_opts)), file=sys.stderr) + sys.argv[0], '|'.join('--'+opt for opt in valid_opts)), + file=sys.stdout if code == 0 else sys.stderr) sys.exit(code) try: diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index 2602fe24c0402e..eb02223ddcd2c3 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -4,7 +4,12 @@ exit_with_usage () { - echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + local USAGE="Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + if [[ "$1" -eq 0 ]]; then + echo "$USAGE" + else + echo "$USAGE" >&2 + fi exit $1 } diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index e28eaea81d6aae..7e9aa6dd82e619 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -132,11 +132,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "baa44fe4581895d42e8d5e83d8ce6a69b1c34dbe" + "checksumValue": "f50c899172acd93fc539007bfb43315b83d407e4" }, { "algorithm": "SHA256", - "checksumValue": "33a7b9ac8bf4571e23272cdf644c6f9808bd44c66b149e3c41ab3870d1888609" + "checksumValue": "d571b8258cfaa067a20adef553e5fcedd6671ca4a8841483496de031bd904567" } ], "fileName": "Modules/expat/pyexpatns.h" @@ -678,11 +678,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "6fa074693aa7305018dfa8db48010a8ef1050ad4" + "checksumValue": "f935d64cc633c38e09fc2d89281c95edfbc1fb05" }, { "algorithm": "SHA256", - "checksumValue": "c8c6dd861ac193d4a0e836242ff44900f83423f86d2c2940c8c4c1e41fbd5812" + "checksumValue": "b932aa273b2504606a48895a50ff08c883f7a68a7e4aced5daa909c43348605a" } ], "fileName": "Modules/_blake2/impl/blake2b.c" @@ -762,11 +762,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "d2691353fa54ac6ffcd7c0a294984dc9d7968ef7" + "checksumValue": "13ac5bb93578a7ee8f815b4e247e82c849992bbe" }, { "algorithm": "SHA256", - "checksumValue": "cfd7948c9fd50e9f9c62f8a93b20a254d1d510a862d1092af4f187b7c1a859a3" + "checksumValue": "25ec5dd5c79f916307358059fe9f633781f27df1c0e0962c4fcccdda1feb93a7" } ], "fileName": "Modules/_blake2/impl/blake2s.c" @@ -1224,11 +1224,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "12402bcf7f0161adb83f78163f41cc10a5e5de5f" + "checksumValue": "9dcb50e3f9c3245972731be5da0b28e7583198d9" }, { "algorithm": "SHA256", - "checksumValue": "cba044c76b6bc3ae6cfa49df1121cad7552140157b9e61e11cbb6580cc5d74cf" + "checksumValue": "7cac49fef5e9d952ec9390bf81c54d83f1b5da32fdf76091c2f0770ed943b7fe" } ], "fileName": "Modules/_decimal/libmpdec/io.c" diff --git a/Modules/Setup b/Modules/Setup index 8ad9a5aebbfcaa..cd1cf24c25d406 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -285,6 +285,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_testcapi _testcapimodule.c #_testimportmultiple _testimportmultiple.c #_testmultiphase _testmultiphase.c +#_testexternalinspection _testexternalinspection.c #_testsinglephase _testsinglephase.c # --- diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index e98775a4808765..73b082691a3fd4 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -171,6 +171,7 @@ @MODULE__TESTIMPORTMULTIPLE_TRUE@_testimportmultiple _testimportmultiple.c @MODULE__TESTMULTIPHASE_TRUE@_testmultiphase _testmultiphase.c @MODULE__TESTMULTIPHASE_TRUE@_testsinglephase _testsinglephase.c +@MODULE__TESTEXTERNALINSPECTION_TRUE@_testexternalinspection _testexternalinspection.c @MODULE__CTYPES_TEST_TRUE@_ctypes_test _ctypes/_ctypes_test.c # Limited API template modules; must be built as shared modules. diff --git a/Modules/_blake2/impl/blake2b.c b/Modules/_blake2/impl/blake2b.c index c1068e8640546a..cef22838917d9d 100644 --- a/Modules/_blake2/impl/blake2b.c +++ b/Modules/_blake2/impl/blake2b.c @@ -27,7 +27,7 @@ #if defined(HAVE_SSE2) #include // MSVC only defines _mm_set_epi64x for x86_64... -#if defined(_MSC_VER) && !defined(_M_X64) +#if defined(_MSC_VER) && !defined(_M_X64) && !defined(__clang__) static inline __m128i _mm_set_epi64x( const uint64_t u1, const uint64_t u0 ) { return _mm_set_epi32( u1 >> 32, u1, u0 >> 32, u0 ); diff --git a/Modules/_blake2/impl/blake2s.c b/Modules/_blake2/impl/blake2s.c index 47514685b8f30b..e7f63fd274f212 100644 --- a/Modules/_blake2/impl/blake2s.c +++ b/Modules/_blake2/impl/blake2s.c @@ -27,7 +27,7 @@ #if defined(HAVE_SSE2) #include // MSVC only defines _mm_set_epi64x for x86_64... -#if defined(_MSC_VER) && !defined(_M_X64) +#if defined(_MSC_VER) && !defined(_M_X64) && !defined(__clang__) static inline __m128i _mm_set_epi64x( const uint64_t u1, const uint64_t u0 ) { return _mm_set_epi32( u1 >> 32, u1, u0 >> 32, u0 ); diff --git a/Modules/_csv.c b/Modules/_csv.c index 3aa648b8e9cec4..660c5455af764e 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -332,9 +332,9 @@ dialect_check_quoting(int quoting) } static int -dialect_check_char(const char *name, Py_UCS4 c, DialectObj *dialect) +dialect_check_char(const char *name, Py_UCS4 c, DialectObj *dialect, bool allowspace) { - if (c == '\r' || c == '\n' || (dialect->skipinitialspace && c == ' ')) { + if (c == '\r' || c == '\n' || (c == ' ' && !allowspace)) { PyErr_Format(PyExc_ValueError, "bad %s value", name); return -1; } @@ -535,9 +535,11 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_TypeError, "lineterminator must be set"); goto err; } - if (dialect_check_char("delimiter", self->delimiter, self) || - dialect_check_char("escapechar", self->escapechar, self) || - dialect_check_char("quotechar", self->quotechar, self) || + if (dialect_check_char("delimiter", self->delimiter, self, true) || + dialect_check_char("escapechar", self->escapechar, self, + !self->skipinitialspace) || + dialect_check_char("quotechar", self->quotechar, self, + !self->skipinitialspace) || dialect_check_chars("delimiter", "escapechar", self->delimiter, self->escapechar) || dialect_check_chars("delimiter", "quotechar", @@ -1150,6 +1152,8 @@ join_append_data(WriterObj *self, int field_kind, const void *field_data, if (c == dialect->delimiter || c == dialect->escapechar || c == dialect->quotechar || + c == '\n' || + c == '\r' || PyUnicode_FindChar( dialect->lineterminator, c, 0, PyUnicode_GET_LENGTH(dialect->lineterminator), 1) >= 0) { @@ -1221,6 +1225,7 @@ join_check_rec_size(WriterObj *self, Py_ssize_t rec_len) static int join_append(WriterObj *self, PyObject *field, int quoted) { + DialectObj *dialect = self->dialect; int field_kind = -1; const void *field_data = NULL; Py_ssize_t field_len = 0; @@ -1231,6 +1236,19 @@ join_append(WriterObj *self, PyObject *field, int quoted) field_data = PyUnicode_DATA(field); field_len = PyUnicode_GET_LENGTH(field); } + if (!field_len && dialect->delimiter == ' ' && dialect->skipinitialspace) { + if (dialect->quoting == QUOTE_NONE || + (field == NULL && + (dialect->quoting == QUOTE_STRINGS || + dialect->quoting == QUOTE_NOTNULL))) + { + PyErr_Format(self->error_obj, + "empty field must be quoted if delimiter is a space " + "and skipinitialspace is true"); + return 0; + } + quoted = 1; + } rec_len = join_append_data(self, field_kind, field_data, field_len, "ed, 0); if (rec_len < 0) @@ -1282,6 +1300,7 @@ csv_writerow(WriterObj *self, PyObject *seq) { DialectObj *dialect = self->dialect; PyObject *iter, *field, *line, *result; + bool null_field = false; iter = PyObject_GetIter(seq); if (iter == NULL) { @@ -1318,11 +1337,12 @@ csv_writerow(WriterObj *self, PyObject *seq) break; } + null_field = (field == Py_None); if (PyUnicode_Check(field)) { append_ok = join_append(self, field, quoted); Py_DECREF(field); } - else if (field == Py_None) { + else if (null_field) { append_ok = join_append(self, NULL, quoted); Py_DECREF(field); } @@ -1348,7 +1368,11 @@ csv_writerow(WriterObj *self, PyObject *seq) return NULL; if (self->num_fields > 0 && self->rec_len == 0) { - if (dialect->quoting == QUOTE_NONE) { + if (dialect->quoting == QUOTE_NONE || + (null_field && + (dialect->quoting == QUOTE_STRINGS || + dialect->quoting == QUOTE_NOTNULL))) + { PyErr_Format(self->error_obj, "single empty field record must be quoted"); return NULL; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 1989723f6f3dbb..02f48a9ed55843 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -32,6 +32,10 @@ #endif #endif +#ifdef MS_WIN32 +#include // for IUnknown interface +#endif + typedef struct { PyTypeObject *DictRemover_Type; PyTypeObject *PyCArg_Type; diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 014ccdd3f6effe..a626bda2ea9be9 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -14,6 +14,8 @@ #include "Python.h" #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyObject_Init() +#include "pycore_time.h" // _PyTime_ObjectToTime_t() + #include "datetime.h" @@ -5131,7 +5133,11 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, static PyObject * datetime_best_possible(PyObject *cls, TM_FUNC f, PyObject *tzinfo) { - _PyTime_t ts = _PyTime_GetSystemClock(); + PyTime_t ts; + if (PyTime_Time(&ts) < 0) { + return NULL; + } + time_t secs; int us; diff --git a/Modules/_decimal/libmpdec/io.c b/Modules/_decimal/libmpdec/io.c index e7bd6aee170056..4e95b8964c8e5d 100644 --- a/Modules/_decimal/libmpdec/io.c +++ b/Modules/_decimal/libmpdec/io.c @@ -48,6 +48,7 @@ #if defined(__GNUC__) && !defined(__INTEL_COMPILER) && __GNUC__ >= 7 #pragma GCC diagnostic ignored "-Wimplicit-fallthrough" #pragma GCC diagnostic ignored "-Wmisleading-indentation" + #pragma GCC diagnostic ignored "-Warray-bounds" #endif diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 54451081211654..edd2f88a4881c3 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -3894,6 +3894,40 @@ _elementtree_XMLParser_close_impl(XMLParserObject *self) } } +/*[clinic input] +_elementtree.XMLParser.flush + +[clinic start generated code]*/ + +static PyObject * +_elementtree_XMLParser_flush_impl(XMLParserObject *self) +/*[clinic end generated code: output=42fdb8795ca24509 input=effbecdb28715949]*/ +{ + if (!_check_xmlparser(self)) { + return NULL; + } + + elementtreestate *st = self->state; + + if (EXPAT(st, SetReparseDeferralEnabled) == NULL) { + Py_RETURN_NONE; + } + + // NOTE: The Expat parser in the C implementation of ElementTree is not + // exposed to the outside; as a result we known that reparse deferral + // is currently enabled, or we would not even have access to function + // XML_SetReparseDeferralEnabled in the first place (which we checked + // for, a few lines up). + + EXPAT(st, SetReparseDeferralEnabled)(self->parser, XML_FALSE); + + PyObject *res = expat_parse(st, self, "", 0, XML_FALSE); + + EXPAT(st, SetReparseDeferralEnabled)(self->parser, XML_TRUE); + + return res; +} + /*[clinic input] _elementtree.XMLParser.feed @@ -4288,6 +4322,7 @@ static PyType_Spec treebuilder_spec = { static PyMethodDef xmlparser_methods[] = { _ELEMENTTREE_XMLPARSER_FEED_METHODDEF _ELEMENTTREE_XMLPARSER_CLOSE_METHODDEF + _ELEMENTTREE_XMLPARSER_FLUSH_METHODDEF _ELEMENTTREE_XMLPARSER__PARSE_WHOLE_METHODDEF _ELEMENTTREE_XMLPARSER__SETEVENTS_METHODDEF {NULL, NULL} diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 9ab847165dc097..d2212d40550a3c 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -365,6 +365,8 @@ partial_repr(partialobject *pto) { PyObject *result = NULL; PyObject *arglist; + PyObject *mod; + PyObject *name; Py_ssize_t i, n; PyObject *key, *value; int status; @@ -399,13 +401,28 @@ partial_repr(partialobject *pto) if (arglist == NULL) goto done; } - result = PyUnicode_FromFormat("%s(%R%U)", Py_TYPE(pto)->tp_name, - pto->fn, arglist); + + mod = _PyType_GetModuleName(Py_TYPE(pto)); + if (mod == NULL) { + goto error; + } + name = PyType_GetQualName(Py_TYPE(pto)); + if (name == NULL) { + Py_DECREF(mod); + goto error; + } + result = PyUnicode_FromFormat("%S.%S(%R%U)", mod, name, pto->fn, arglist); + Py_DECREF(mod); + Py_DECREF(name); Py_DECREF(arglist); done: Py_ReprLeave((PyObject *)pto); return result; + error: + Py_DECREF(arglist); + Py_ReprLeave((PyObject *)pto); + return NULL; } /* Pickle strategy: diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 8ebe9ec7095586..b3450eeaf99401 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1325,7 +1325,11 @@ _io__Buffered_tell_impl(buffered *self) if (pos == -1) return NULL; pos -= RAW_OFFSET(self); - /* TODO: sanity check (pos >= 0) */ + + // GH-95782 + if (pos < 0) + pos = 0; + return PyLong_FromOff_t(pos); } @@ -1395,6 +1399,11 @@ _io__Buffered_seek_impl(buffered *self, PyObject *targetobj, int whence) offset = target; if (offset >= -self->pos && offset <= avail) { self->pos += offset; + + // GH-95782 + if (current - avail + offset < 0) + return PyLong_FromOff_t(0); + return PyLong_FromOff_t(current - avail + offset); } } diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 8f09204097529f..a76c3dea555783 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -6,6 +6,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_SetProfile() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_time.h" // _PyTime_FromLong() #include "rotatingtree.h" @@ -17,8 +18,8 @@ struct _ProfilerEntry; /* represents a function called from another function */ typedef struct _ProfilerSubEntry { rotating_node_t header; - _PyTime_t tt; - _PyTime_t it; + PyTime_t tt; + PyTime_t it; long callcount; long recursivecallcount; long recursionLevel; @@ -28,8 +29,8 @@ typedef struct _ProfilerSubEntry { typedef struct _ProfilerEntry { rotating_node_t header; PyObject *userObj; /* PyCodeObject, or a descriptive str for builtins */ - _PyTime_t tt; /* total time in this entry */ - _PyTime_t it; /* inline time in this entry (not in subcalls) */ + PyTime_t tt; /* total time in this entry */ + PyTime_t it; /* inline time in this entry (not in subcalls) */ long callcount; /* how many times this was called */ long recursivecallcount; /* how many times called recursively */ long recursionLevel; @@ -37,8 +38,8 @@ typedef struct _ProfilerEntry { } ProfilerEntry; typedef struct _ProfilerContext { - _PyTime_t t0; - _PyTime_t subt; + PyTime_t t0; + PyTime_t subt; struct _ProfilerContext *previous; ProfilerEntry *ctxEntry; } ProfilerContext; @@ -84,7 +85,7 @@ _lsprof_get_state(PyObject *module) /*** External Timers ***/ -static _PyTime_t CallExternalTimer(ProfilerObject *pObj) +static PyTime_t CallExternalTimer(ProfilerObject *pObj) { PyObject *o = _PyObject_CallNoArgs(pObj->externalTimer); if (o == NULL) { @@ -92,16 +93,16 @@ static _PyTime_t CallExternalTimer(ProfilerObject *pObj) return 0; } - _PyTime_t result; + PyTime_t result; int err; if (pObj->externalTimerUnit > 0.0) { /* interpret the result as an integer that will be scaled in profiler_getstats() */ - err = _PyTime_FromNanosecondsObject(&result, o); + err = _PyTime_FromLong(&result, o); } else { /* interpret the result as a double measured in seconds. - As the profiler works with _PyTime_t internally + As the profiler works with PyTime_t internally we convert it to a large integer */ err = _PyTime_FromSecondsObject(&result, o, _PyTime_ROUND_FLOOR); } @@ -113,14 +114,14 @@ static _PyTime_t CallExternalTimer(ProfilerObject *pObj) return result; } -static inline _PyTime_t +static inline PyTime_t call_timer(ProfilerObject *pObj) { if (pObj->externalTimer != NULL) { return CallExternalTimer(pObj); } else { - return _PyTime_GetPerfCounter(); + return _PyTime_PerfCounterUnchecked(); } } @@ -311,8 +312,8 @@ initContext(ProfilerObject *pObj, ProfilerContext *self, ProfilerEntry *entry) static void Stop(ProfilerObject *pObj, ProfilerContext *self, ProfilerEntry *entry) { - _PyTime_t tt = call_timer(pObj) - self->t0; - _PyTime_t it = tt - self->subt; + PyTime_t tt = call_timer(pObj) - self->t0; + PyTime_t it = tt - self->subt; if (self->previous) self->previous->subt += tt; pObj->currentProfilerContext = self->previous; @@ -557,7 +558,7 @@ _lsprof_Profiler_getstats_impl(ProfilerObject *self, PyTypeObject *cls) return NULL; } if (!self->externalTimer || self->externalTimerUnit == 0.0) { - _PyTime_t onesec = _PyTime_FromSeconds(1); + PyTime_t onesec = _PyTime_FromSeconds(1); collect.factor = (double)1 / onesec; } else { @@ -1004,9 +1005,7 @@ _lsprof_exec(PyObject *module) static PyModuleDef_Slot _lsprofslots[] = { {Py_mod_exec, _lsprof_exec}, - // XXX gh-103092: fix isolation. - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, - //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 2e6d8eb68c0243..1f6ab718a36984 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -181,7 +181,7 @@ static PyMethodDef module_methods[] = { _MULTIPROCESSING_RECV_METHODDEF _MULTIPROCESSING_SEND_METHODDEF #endif -#if !defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__) +#if !defined(POSIX_SEMAPHORES_NOT_ENABLED) _MULTIPROCESSING_SEM_UNLINK_METHODDEF #endif {NULL} diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index 425ce10075c156..4ab15fa6573665 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -11,6 +11,7 @@ posixshmem - A Python extension that provides shm_open() and shm_unlink() #include +#include // strlen() #include // EINTR #ifdef HAVE_SYS_MMAN_H # include // shm_open(), shm_unlink() @@ -48,10 +49,15 @@ _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, { int fd; int async_err = 0; - const char *name = PyUnicode_AsUTF8AndSize(path, NULL); + Py_ssize_t name_size; + const char *name = PyUnicode_AsUTF8AndSize(path, &name_size); if (name == NULL) { return -1; } + if (strlen(name) != (size_t)name_size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return -1; + } do { Py_BEGIN_ALLOW_THREADS fd = shm_open(name, flags, mode); @@ -87,10 +93,15 @@ _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path) { int rv; int async_err = 0; - const char *name = PyUnicode_AsUTF8AndSize(path, NULL); + Py_ssize_t name_size; + const char *name = PyUnicode_AsUTF8AndSize(path, &name_size); if (name == NULL) { return NULL; } + if (strlen(name) != (size_t)name_size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return NULL; + } do { Py_BEGIN_ALLOW_THREADS rv = shm_unlink(name); diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 18b24855c52ad6..5db9b645849fcd 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -6,7 +6,7 @@ #include "pycore_ceval.h" // Py_MakePendingCalls() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_parking_lot.h" -#include "pycore_time.h" // _PyTime_t +#include "pycore_time.h" // _PyTime_FromSecondsObject() #include #include // offsetof() @@ -372,13 +372,13 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, int block, PyObject *timeout_obj) /*[clinic end generated code: output=5c2cca914cd1e55b input=f7836c65e5839c51]*/ { - _PyTime_t endtime = 0; + PyTime_t endtime = 0; // XXX Use PyThread_ParseTimeoutArg(). if (block != 0 && !Py_IsNone(timeout_obj)) { /* With timeout */ - _PyTime_t timeout; + PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_CEILING) < 0) { return NULL; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 4463157d62248d..56b891dfe0f85f 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -259,13 +259,15 @@ random_seed_urandom(RandomObject *self) return 0; } -static void +static int random_seed_time_pid(RandomObject *self) { PyTime_t now; - uint32_t key[5]; + if (PyTime_Time(&now) < 0) { + return -1; + } - now = _PyTime_GetSystemClock(); + uint32_t key[5]; key[0] = (uint32_t)(now & 0xffffffffU); key[1] = (uint32_t)(now >> 32); @@ -277,11 +279,14 @@ random_seed_time_pid(RandomObject *self) key[2] = 0; #endif - now = _PyTime_GetMonotonicClock(); + if (PyTime_Monotonic(&now) < 0) { + return -1; + } key[3] = (uint32_t)(now & 0xffffffffU); key[4] = (uint32_t)(now >> 32); init_by_array(self, key, Py_ARRAY_LENGTH(key)); + return 0; } static int @@ -299,7 +304,9 @@ random_seed(RandomObject *self, PyObject *arg) /* Reading system entropy failed, fall back on the worst entropy: use the current time and process identifier. */ - random_seed_time_pid(self); + if (random_seed_time_pid(self) < 0) { + return -1; + } } return 0; } diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 950ee3663080e1..d00f407b569fb6 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -28,6 +28,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _PyIsSelectable_fd() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_time.h" // _PyDeadline_Init() #include "pycore_weakref.h" // _PyWeakref_GET_REF() /* Include symbols from _socket module */ @@ -369,7 +370,7 @@ class _ssl.SSLSession "PySSLSession *" "get_state_type(type)->PySSLSession_Type" #include "clinic/_ssl.c.h" -static int PySSL_select(PySocketSockObject *s, int writing, _PyTime_t timeout); +static int PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout); static int PySSL_set_owner(PySSLSocket *, PyObject *, void *); static int PySSL_set_session(PySSLSocket *, PyObject *, void *); @@ -963,7 +964,7 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) _PySSLError err; int sockstate, nonblocking; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (sock) { @@ -2273,12 +2274,12 @@ PySSL_dealloc(PySSLSocket *self) */ static int -PySSL_select(PySocketSockObject *s, int writing, _PyTime_t timeout) +PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) { int rc; #ifdef HAVE_POLL struct pollfd pollfd; - _PyTime_t ms; + PyTime_t ms; #else int nfds; fd_set fds; @@ -2357,7 +2358,7 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) _PySSLError err; int nonblocking; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (sock != NULL) { @@ -2495,7 +2496,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, _PySSLError err; int nonblocking; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (!group_right_1 && len < 0) { @@ -2627,7 +2628,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) int sockstate, nonblocking, ret; int zeros = 0; PySocketSockObject *sock = GET_SOCKET(self); - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; int has_timeout; if (sock != NULL) { diff --git a/Modules/_testcapi/time.c b/Modules/_testcapi/time.c index 57eb9135d30029..68f082bf3f3d88 100644 --- a/Modules/_testcapi/time.c +++ b/Modules/_testcapi/time.c @@ -49,9 +49,11 @@ static PyObject* test_pytime_monotonic(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) { PyTime_t t; - if (PyTime_Monotonic(&t) < 0) { + int res = PyTime_Monotonic(&t); + if (res < 0) { return NULL; } + assert(res == 0); return pytime_as_float(t); } @@ -60,9 +62,11 @@ static PyObject* test_pytime_perf_counter(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) { PyTime_t t; - if (PyTime_PerfCounter(&t) < 0) { + int res = PyTime_PerfCounter(&t); + if (res < 0) { return NULL; } + assert(res == 0); return pytime_as_float(t); } @@ -71,10 +75,11 @@ static PyObject* test_pytime_time(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) { PyTime_t t; - if (PyTime_Time(&t) < 0) { - printf("ERR! %d\n", (int)t); + int res = PyTime_Time(&t); + if (res < 0) { return NULL; } + assert(res == 0); return pytime_as_float(t); } diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c new file mode 100644 index 00000000000000..4929a7bf5a984e --- /dev/null +++ b/Modules/_testexternalinspection.c @@ -0,0 +1,629 @@ +#define _GNU_SOURCE + +#ifdef __linux__ +# include +# include +# if INTPTR_MAX == INT64_MAX +# define Elf_Ehdr Elf64_Ehdr +# define Elf_Shdr Elf64_Shdr +# define Elf_Phdr Elf64_Phdr +# else +# define Elf_Ehdr Elf32_Ehdr +# define Elf_Shdr Elf32_Shdr +# define Elf_Phdr Elf32_Phdr +# endif +# include +#endif + +#ifdef __APPLE__ +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif +#include "Python.h" +#include + +#ifndef HAVE_PROCESS_VM_READV +# define HAVE_PROCESS_VM_READV 0 +#endif + +#ifdef __APPLE__ +static void* +analyze_macho64(mach_port_t proc_ref, void* base, void* map) +{ + struct mach_header_64* hdr = (struct mach_header_64*)map; + int ncmds = hdr->ncmds; + + int cmd_cnt = 0; + struct segment_command_64* cmd = map + sizeof(struct mach_header_64); + + mach_vm_size_t size = 0; + mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); + mach_vm_address_t address = (mach_vm_address_t)base; + vm_region_basic_info_data_64_t region_info; + mach_port_t object_name; + + for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) { + while (cmd->filesize != size) { + address += size; + if (mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)®ion_info, // cppcheck-suppress [uninitvar] + &count, + &object_name) + != KERN_SUCCESS) + { + PyErr_SetString(PyExc_RuntimeError, "Cannot get any more VM maps.\n"); + return NULL; + } + } + base = (void*)address - cmd->vmaddr; + + int nsects = cmd->nsects; + struct section_64* sec = + (struct section_64*)((void*)cmd + sizeof(struct segment_command_64)); + for (int j = 0; j < nsects; j++) { + if (strcmp(sec[j].sectname, "PyRuntime") == 0) { + return base + sec[j].addr; + } + } + cmd_cnt++; + } + + cmd = (struct segment_command_64*)((void*)cmd + cmd->cmdsize); + } + return NULL; +} + +static void* +analyze_macho(char* path, void* base, mach_vm_size_t size, mach_port_t proc_ref) +{ + int fd = open(path, O_RDONLY); + if (fd == -1) { + PyErr_Format(PyExc_RuntimeError, "Cannot open binary %s\n", path); + return NULL; + } + + struct stat fs; + if (fstat(fd, &fs) == -1) { + PyErr_Format(PyExc_RuntimeError, "Cannot get size of binary %s\n", path); + close(fd); + return NULL; + } + + void* map = mmap(0, fs.st_size, PROT_READ, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) { + PyErr_Format(PyExc_RuntimeError, "Cannot map binary %s\n", path); + close(fd); + return NULL; + } + + void* result = NULL; + + struct mach_header_64* hdr = (struct mach_header_64*)map; + switch (hdr->magic) { + case MH_MAGIC: + case MH_CIGAM: + case FAT_MAGIC: + case FAT_CIGAM: + PyErr_SetString(PyExc_RuntimeError, "32-bit Mach-O binaries are not supported"); + break; + case MH_MAGIC_64: + case MH_CIGAM_64: + result = analyze_macho64(proc_ref, base, map); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "Unknown Mach-O magic"); + break; + } + + munmap(map, fs.st_size); + if (close(fd) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + } + return result; +} + +static mach_port_t +pid_to_task(pid_t pid) +{ + mach_port_t task; + kern_return_t result; + + result = task_for_pid(mach_task_self(), pid, &task); + if (result != KERN_SUCCESS) { + PyErr_Format(PyExc_PermissionError, "Cannot get task for PID %d", pid); + return 0; + } + return task; +} + +static void* +get_py_runtime_macos(pid_t pid) +{ + mach_vm_address_t address = 0; + mach_vm_size_t size = 0; + mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); + vm_region_basic_info_data_64_t region_info; + mach_port_t object_name; + + mach_port_t proc_ref = pid_to_task(pid); + if (proc_ref == 0) { + PyErr_SetString(PyExc_PermissionError, "Cannot get task for PID"); + return NULL; + } + + int match_found = 0; + char map_filename[MAXPATHLEN + 1]; + void* result_address = NULL; + while (mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)®ion_info, + &count, + &object_name) + == KERN_SUCCESS) + { + int path_len = proc_regionfilename(pid, address, map_filename, MAXPATHLEN); + if (path_len == 0) { + address += size; + continue; + } + + char* filename = strrchr(map_filename, '/'); + if (filename != NULL) { + filename++; // Move past the '/' + } else { + filename = map_filename; // No path, use the whole string + } + + // Check if the filename starts with "python" or "libpython" + if (!match_found && strncmp(filename, "python", 6) == 0) { + match_found = 1; + result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); + } + if (strncmp(filename, "libpython", 9) == 0) { + match_found = 1; + result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); + break; + } + + address += size; + } + return result_address; +} +#endif + +#ifdef __linux__ +void* +find_python_map_start_address(pid_t pid, char* result_filename) +{ + char maps_file_path[64]; + sprintf(maps_file_path, "/proc/%d/maps", pid); + + FILE* maps_file = fopen(maps_file_path, "r"); + if (maps_file == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + int match_found = 0; + + char line[256]; + char map_filename[PATH_MAX]; + void* result_address = 0; + while (fgets(line, sizeof(line), maps_file) != NULL) { + unsigned long start_address = 0; + sscanf(line, "%lx-%*x %*s %*s %*s %*s %s", &start_address, map_filename); + char* filename = strrchr(map_filename, '/'); + if (filename != NULL) { + filename++; // Move past the '/' + } else { + filename = map_filename; // No path, use the whole string + } + + // Check if the filename starts with "python" or "libpython" + if (!match_found && strncmp(filename, "python", 6) == 0) { + match_found = 1; + result_address = (void*)start_address; + strcpy(result_filename, map_filename); + } + if (strncmp(filename, "libpython", 9) == 0) { + match_found = 1; + result_address = (void*)start_address; + strcpy(result_filename, map_filename); + break; + } + } + + fclose(maps_file); + + if (!match_found) { + map_filename[0] = '\0'; + } + + return result_address; +} + +void* +get_py_runtime_linux(pid_t pid) +{ + char elf_file[256]; + void* start_address = (void*)find_python_map_start_address(pid, elf_file); + + if (start_address == 0) { + PyErr_SetString(PyExc_RuntimeError, "No memory map associated with python or libpython found"); + return NULL; + } + + void* result = NULL; + void* file_memory = NULL; + + int fd = open(elf_file, O_RDONLY); + if (fd < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + struct stat file_stats; + if (fstat(fd, &file_stats) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + file_memory = mmap(NULL, file_stats.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (file_memory == MAP_FAILED) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + + Elf_Ehdr* elf_header = (Elf_Ehdr*)file_memory; + + Elf_Shdr* section_header_table = (Elf_Shdr*)(file_memory + elf_header->e_shoff); + + Elf_Shdr* shstrtab_section = §ion_header_table[elf_header->e_shstrndx]; + char* shstrtab = (char*)(file_memory + shstrtab_section->sh_offset); + + Elf_Shdr* py_runtime_section = NULL; + for (int i = 0; i < elf_header->e_shnum; i++) { + if (strcmp(".PyRuntime", shstrtab + section_header_table[i].sh_name) == 0) { + py_runtime_section = §ion_header_table[i]; + break; + } + } + + Elf_Phdr* program_header_table = (Elf_Phdr*)(file_memory + elf_header->e_phoff); + // Find the first PT_LOAD segment + Elf_Phdr* first_load_segment = NULL; + for (int i = 0; i < elf_header->e_phnum; i++) { + if (program_header_table[i].p_type == PT_LOAD) { + first_load_segment = &program_header_table[i]; + break; + } + } + + if (py_runtime_section != NULL && first_load_segment != NULL) { + uintptr_t elf_load_addr = first_load_segment->p_vaddr + - (first_load_segment->p_vaddr % first_load_segment->p_align); + result = start_address + py_runtime_section->sh_addr - elf_load_addr; + } + +exit: + if (close(fd) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + } + if (file_memory != NULL) { + munmap(file_memory, file_stats.st_size); + } + return result; +} +#endif + +ssize_t +read_memory(pid_t pid, void* remote_address, size_t len, void* dst) +{ + ssize_t total_bytes_read = 0; +#if defined(__linux__) && HAVE_PROCESS_VM_READV + struct iovec local[1]; + struct iovec remote[1]; + ssize_t result = 0; + ssize_t read = 0; + + do { + local[0].iov_base = dst + result; + local[0].iov_len = len - result; + remote[0].iov_base = (void*)(remote_address + result); + remote[0].iov_len = len - result; + + read = process_vm_readv(pid, local, 1, remote, 1, 0); + if (read < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + result += read; + } while ((size_t)read != local[0].iov_len); + total_bytes_read = result; +#elif defined(__APPLE__) + ssize_t result = -1; + kern_return_t kr = mach_vm_read_overwrite( + pid_to_task(pid), + (mach_vm_address_t)remote_address, + len, + (mach_vm_address_t)dst, + (mach_vm_size_t*)&result); + + if (kr != KERN_SUCCESS) { + switch (kr) { + case KERN_PROTECTION_FAILURE: + PyErr_SetString(PyExc_PermissionError, "Not enough permissions to read memory"); + break; + case KERN_INVALID_ARGUMENT: + PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_read_overwrite"); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "Unknown error reading memory"); + } + return -1; + } + total_bytes_read = len; +#else + return -1; +#endif + return total_bytes_read; +} + +int +read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, void* address, char* buffer, Py_ssize_t size) +{ + Py_ssize_t len; + ssize_t bytes_read = + read_memory(pid, address + debug_offsets->unicode_object.length, sizeof(Py_ssize_t), &len); + if (bytes_read == -1) { + return -1; + } + if (len >= size) { + PyErr_SetString(PyExc_RuntimeError, "Buffer too small"); + return -1; + } + size_t offset = debug_offsets->unicode_object.asciiobject_size; + bytes_read = read_memory(pid, address + offset, len, buffer); + if (bytes_read == -1) { + return -1; + } + buffer[len] = '\0'; + return 0; +} + +void* +get_py_runtime(pid_t pid) +{ +#if defined(__linux__) + return get_py_runtime_linux(pid); +#elif defined(__APPLE__) + return get_py_runtime_macos(pid); +#else + return NULL; +#endif +} + +static int +parse_code_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + void* address, + void** previous_frame) +{ + void* address_of_function_name; + read_memory( + pid, + (void*)(address + offsets->code_object.name), + sizeof(void*), + &address_of_function_name); + + if (address_of_function_name == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No function name found"); + return -1; + } + + char function_name[256]; + if (read_string(pid, offsets, address_of_function_name, function_name, sizeof(function_name)) != 0) { + return -1; + } + + PyObject* py_function_name = PyUnicode_FromString(function_name); + if (py_function_name == NULL) { + return -1; + } + + if (PyList_Append(result, py_function_name) == -1) { + Py_DECREF(py_function_name); + return -1; + } + Py_DECREF(py_function_name); + + return 0; +} + +static int +parse_frame_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + void* address, + void** previous_frame) +{ + ssize_t bytes_read = read_memory( + pid, + (void*)(address + offsets->interpreter_frame.previous), + sizeof(void*), + previous_frame); + if (bytes_read == -1) { + return -1; + } + + char owner; + bytes_read = + read_memory(pid, (void*)(address + offsets->interpreter_frame.owner), sizeof(char), &owner); + if (bytes_read < 0) { + return -1; + } + + if (owner == FRAME_OWNED_BY_CSTACK) { + return 0; + } + + void* address_of_code_object; + bytes_read = read_memory( + pid, + (void*)(address + offsets->interpreter_frame.executable), + sizeof(void*), + &address_of_code_object); + if (bytes_read == -1) { + return -1; + } + + if (address_of_code_object == NULL) { + return 0; + } + return parse_code_object(pid, result, offsets, address_of_code_object, previous_frame); +} + +static PyObject* +get_stack_trace(PyObject* self, PyObject* args) +{ +#if (!defined(__linux__) && !defined(__APPLE__)) || (defined(__linux__) && !HAVE_PROCESS_VM_READV) + PyErr_SetString(PyExc_RuntimeError, "get_stack_trace is not supported on this platform"); + return NULL; +#endif + int pid; + + if (!PyArg_ParseTuple(args, "i", &pid)) { + return NULL; + } + + void* runtime_start_address = get_py_runtime(pid); + if (runtime_start_address == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "Failed to get .PyRuntime address"); + } + return NULL; + } + size_t size = sizeof(struct _Py_DebugOffsets); + struct _Py_DebugOffsets local_debug_offsets; + + ssize_t bytes_read = read_memory(pid, runtime_start_address, size, &local_debug_offsets); + if (bytes_read == -1) { + return NULL; + } + off_t thread_state_list_head = local_debug_offsets.runtime_state.interpreters_head; + + void* address_of_interpreter_state; + bytes_read = read_memory( + pid, + (void*)(runtime_start_address + thread_state_list_head), + sizeof(void*), + &address_of_interpreter_state); + if (bytes_read == -1) { + return NULL; + } + + if (address_of_interpreter_state == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + return NULL; + } + + void* address_of_thread; + bytes_read = read_memory( + pid, + (void*)(address_of_interpreter_state + local_debug_offsets.interpreter_state.threads_head), + sizeof(void*), + &address_of_thread); + if (bytes_read == -1) { + return NULL; + } + + PyObject* result = PyList_New(0); + if (result == NULL) { + return NULL; + } + + // No Python frames are available for us (can happen at tear-down). + if (address_of_thread != NULL) { + void* address_of_current_frame; + (void)read_memory( + pid, + (void*)(address_of_thread + local_debug_offsets.thread_state.current_frame), + sizeof(void*), + &address_of_current_frame); + while (address_of_current_frame != NULL) { + if (parse_frame_object( + pid, + result, + &local_debug_offsets, + address_of_current_frame, + &address_of_current_frame) + < 0) + { + Py_DECREF(result); + return NULL; + } + } + } + + return result; +} + +static PyMethodDef methods[] = { + {"get_stack_trace", get_stack_trace, METH_VARARGS, "Get the Python stack from a given PID"}, + {NULL, NULL, 0, NULL}, +}; + +static struct PyModuleDef module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_testexternalinspection", + .m_size = -1, + .m_methods = methods, +}; + +PyMODINIT_FUNC +PyInit__testexternalinspection(void) +{ + PyObject* mod = PyModule_Create(&module); + int rc = PyModule_AddIntConstant(mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); + if (rc < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 3834f00009cea4..db8817418950b9 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -24,6 +24,7 @@ #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() #include "pycore_object.h" // _PyObject_IsFreed() +#include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -960,13 +961,13 @@ iframe_getlasti(PyObject *self, PyObject *frame) } static PyObject * -get_counter_optimizer(PyObject *self, PyObject *arg) +new_counter_optimizer(PyObject *self, PyObject *arg) { return PyUnstable_Optimizer_NewCounter(); } static PyObject * -get_uop_optimizer(PyObject *self, PyObject *arg) +new_uop_optimizer(PyObject *self, PyObject *arg) { return PyUnstable_Optimizer_NewUOpOptimizer(); } @@ -977,7 +978,9 @@ set_optimizer(PyObject *self, PyObject *opt) if (opt == Py_None) { opt = NULL; } - PyUnstable_SetOptimizer((_PyOptimizerObject*)opt); + if (PyUnstable_SetOptimizer((_PyOptimizerObject*)opt) < 0) { + return NULL; + } Py_RETURN_NONE; } @@ -1033,7 +1036,7 @@ static PyObject * invalidate_executors(PyObject *self, PyObject *obj) { PyInterpreterState *interp = PyInterpreterState_Get(); - _Py_Executors_InvalidateDependency(interp, obj); + _Py_Executors_InvalidateDependency(interp, obj, 1); Py_RETURN_NONE; } @@ -1675,7 +1678,6 @@ get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored)) } #endif - static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1709,8 +1711,8 @@ static PyMethodDef module_functions[] = { {"get_optimizer", get_optimizer, METH_NOARGS, NULL}, {"set_optimizer", set_optimizer, METH_O, NULL}, {"get_executor", _PyCFunction_CAST(get_executor), METH_FASTCALL, NULL}, - {"get_counter_optimizer", get_counter_optimizer, METH_NOARGS, NULL}, - {"get_uop_optimizer", get_uop_optimizer, METH_NOARGS, NULL}, + {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL}, + {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL}, {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), @@ -1745,6 +1747,7 @@ static PyMethodDef module_functions[] = { #ifdef Py_GIL_DISABLED {"py_thread_id", get_py_thread_id, METH_NOARGS}, #endif + {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi/pytime.c b/Modules/_testinternalcapi/pytime.c index f0f758ea032df8..2b0a205d158a96 100644 --- a/Modules/_testinternalcapi/pytime.c +++ b/Modules/_testinternalcapi/pytime.c @@ -2,10 +2,10 @@ #include "parts.h" -#include "pycore_time.h" +#include "pycore_time.h" // _PyTime_FromSeconds() #ifdef MS_WINDOWS -# include // struct timeval +# include // struct timeval #endif @@ -16,8 +16,8 @@ test_pytime_fromseconds(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "i", &seconds)) { return NULL; } - _PyTime_t ts = _PyTime_FromSeconds(seconds); - return _PyTime_AsNanosecondsObject(ts); + PyTime_t ts = _PyTime_FromSeconds(seconds); + return _PyTime_AsLong(ts); } static int @@ -45,11 +45,11 @@ test_pytime_fromsecondsobject(PyObject *self, PyObject *args) if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t ts; + PyTime_t ts; if (_PyTime_FromSecondsObject(&ts, obj, round) == -1) { return NULL; } - return _PyTime_AsNanosecondsObject(ts); + return _PyTime_AsLong(ts); } static PyObject * @@ -63,8 +63,8 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timeval tv; @@ -90,8 +90,8 @@ test_PyTime_AsTimeval_clamp(PyObject *self, PyObject *args) if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timeval tv; @@ -112,8 +112,8 @@ test_PyTime_AsTimespec(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timespec ts; @@ -130,8 +130,8 @@ test_PyTime_AsTimespec_clamp(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O", &obj)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } struct timespec ts; @@ -148,16 +148,15 @@ test_PyTime_AsMilliseconds(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t ms = _PyTime_AsMilliseconds(t, round); - _PyTime_t ns = _PyTime_FromNanoseconds(ms); - return _PyTime_AsNanosecondsObject(ns); + PyTime_t ms = _PyTime_AsMilliseconds(t, round); + return _PyTime_AsLong(ms); } static PyObject * @@ -168,16 +167,15 @@ test_PyTime_AsMicroseconds(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "Oi", &obj, &round)) { return NULL; } - _PyTime_t t; - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + PyTime_t t; + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } if (check_time_rounding(round) < 0) { return NULL; } - _PyTime_t us = _PyTime_AsMicroseconds(t, round); - _PyTime_t ns = _PyTime_FromNanoseconds(us); - return _PyTime_AsNanosecondsObject(ns); + PyTime_t us = _PyTime_AsMicroseconds(t, round); + return _PyTime_AsLong(us); } static PyObject * diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 83081f73a72f64..1c5048170e9f2e 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -1,8 +1,9 @@ // C Extension module to test pycore_lock.h API #include "parts.h" - #include "pycore_lock.h" +#include "pycore_time.h" // _PyTime_MonotonicUnchecked() + #include "clinic/test_lock.c.h" #ifdef MS_WINDOWS @@ -289,7 +290,7 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, goto exit; } - _PyTime_t start = _PyTime_GetMonotonicClock(); + PyTime_t start = _PyTime_MonotonicUnchecked(); for (Py_ssize_t i = 0; i < num_threads; i++) { thread_data[i].bench_data = &bench_data; @@ -306,7 +307,7 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, } Py_ssize_t total_iters = bench_data.total_iters; - _PyTime_t end = _PyTime_GetMonotonicClock(); + PyTime_t end = _PyTime_MonotonicUnchecked(); // Return the total number of acquisitions and the number of acquisitions // for each thread. diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index c42a15a0eff494..092673a9ea43e1 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -8,11 +8,10 @@ //#include #include "Python.h" #include "pycore_namespace.h" // _PyNamespace_New() -#include "pycore_time.h" // _PyTime_t typedef struct { - _PyTime_t initialized; + PyTime_t initialized; PyObject *error; PyObject *int_const; PyObject *str_const; @@ -67,17 +66,17 @@ clear_state(module_state *state) } static int -_set_initialized(_PyTime_t *initialized) +_set_initialized(PyTime_t *initialized) { /* We go strictly monotonic to ensure each time is unique. */ - _PyTime_t prev; - if (_PyTime_GetMonotonicClockWithInfo(&prev, NULL) != 0) { + PyTime_t prev; + if (PyTime_Monotonic(&prev) != 0) { return -1; } /* We do a busy sleep since the interval should be super short. */ - _PyTime_t t; + PyTime_t t; do { - if (_PyTime_GetMonotonicClockWithInfo(&t, NULL) != 0) { + if (PyTime_Monotonic(&t) != 0) { return -1; } } while (t == prev); @@ -136,7 +135,7 @@ init_module(PyObject *module, module_state *state) return -1; } - double d = _PyTime_AsSecondsDouble(state->initialized); + double d = PyTime_AsSecondsDouble(state->initialized); if (PyModule_Add(module, "_module_initialized", PyFloat_FromDouble(d)) < 0) { return -1; } @@ -157,7 +156,7 @@ common_state_initialized(PyObject *self, PyObject *Py_UNUSED(ignored)) if (state == NULL) { Py_RETURN_NONE; } - double d = _PyTime_AsSecondsDouble(state->initialized); + double d = PyTime_AsSecondsDouble(state->initialized); return PyFloat_FromDouble(d); } diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index da6a8bc7b120fe..3a8f77d6dfbbc6 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1,16 +1,18 @@ - /* Thread module */ /* Interface to Sjoerd's portable C thread library */ #include "Python.h" #include "pycore_interp.h" // _PyInterpreterState.threads.count +#include "pycore_lock.h" #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_SetCurrent() #include "pycore_sysmodule.h" // _PySys_GetAttr() +#include "pycore_time.h" // _PyTime_FromSeconds() #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#include #include // offsetof() #ifdef HAVE_SIGNAL_H # include // SIGINT @@ -42,24 +44,76 @@ get_thread_state(PyObject *module) // _ThreadHandle type +// Handles transition from RUNNING to one of JOINED, DETACHED, or INVALID (post +// fork). +typedef enum { + THREAD_HANDLE_RUNNING = 1, + THREAD_HANDLE_JOINED = 2, + THREAD_HANDLE_DETACHED = 3, + THREAD_HANDLE_INVALID = 4, +} ThreadHandleState; + +// A handle around an OS thread. +// +// The OS thread is either joined or detached after the handle is destroyed. +// +// Joining the handle is idempotent; the underlying OS thread is joined or +// detached only once. Concurrent join operations are serialized until it is +// their turn to execute or an earlier operation completes successfully. Once a +// join has completed successfully all future joins complete immediately. typedef struct { PyObject_HEAD struct llist_node node; // linked list node (see _pythread_runtime_state) + + // The `ident` and `handle` fields are immutable once the object is visible + // to threads other than its creator, thus they do not need to be accessed + // atomically. PyThread_ident_t ident; PyThread_handle_t handle; - char joinable; + + // Holds a value from the `ThreadHandleState` enum. + int state; + + // Set immediately before `thread_run` returns to indicate that the OS + // thread is about to exit. This is used to avoid false positives when + // detecting self-join attempts. See the comment in `ThreadHandle_join()` + // for a more detailed explanation. + _PyEventRc *thread_is_exiting; + + // Serializes calls to `join`. + _PyOnceFlag once; } ThreadHandleObject; +static inline int +get_thread_handle_state(ThreadHandleObject *handle) +{ + return _Py_atomic_load_int(&handle->state); +} + +static inline void +set_thread_handle_state(ThreadHandleObject *handle, ThreadHandleState state) +{ + _Py_atomic_store_int(&handle->state, state); +} + static ThreadHandleObject* new_thread_handle(thread_module_state* state) { + _PyEventRc *event = _PyEventRc_New(); + if (event == NULL) { + PyErr_NoMemory(); + return NULL; + } ThreadHandleObject* self = PyObject_New(ThreadHandleObject, state->thread_handle_type); if (self == NULL) { + _PyEventRc_Decref(event); return NULL; } self->ident = 0; self->handle = 0; - self->joinable = 0; + self->thread_is_exiting = event; + self->once = (_PyOnceFlag){0}; + self->state = THREAD_HANDLE_INVALID; HEAD_LOCK(&_PyRuntime); llist_insert_tail(&_PyRuntime.threads.handles, &self->node); @@ -80,13 +134,21 @@ ThreadHandle_dealloc(ThreadHandleObject *self) } HEAD_UNLOCK(&_PyRuntime); - if (self->joinable) { - int ret = PyThread_detach_thread(self->handle); - if (ret) { + // It's safe to access state non-atomically: + // 1. This is the destructor; nothing else holds a reference. + // 2. The refcount going to zero is a "synchronizes-with" event; + // all changes from other threads are visible. + if (self->state == THREAD_HANDLE_RUNNING) { + // This is typically short so no need to release the GIL + if (PyThread_detach_thread(self->handle)) { PyErr_SetString(ThreadError, "Failed detaching thread"); PyErr_WriteUnraisable(tp); } + else { + self->state = THREAD_HANDLE_DETACHED; + } } + _PyEventRc_Decref(self->thread_is_exiting); PyObject_Free(self); Py_DECREF(tp); } @@ -107,8 +169,9 @@ _PyThread_AfterFork(struct _pythread_runtime_state *state) continue; } - // Disallow calls to detach() and join() as they could crash. - hobj->joinable = 0; + // Disallow calls to join() as they could crash. We are the only + // thread; it's safe to set this without an atomic. + hobj->state = THREAD_HANDLE_INVALID; llist_remove(node); } } @@ -126,48 +189,54 @@ ThreadHandle_get_ident(ThreadHandleObject *self, void *ignored) return PyLong_FromUnsignedLongLong(self->ident); } - -static PyObject * -ThreadHandle_detach(ThreadHandleObject *self, void* ignored) +static int +join_thread(ThreadHandleObject *handle) { - if (!self->joinable) { - PyErr_SetString(PyExc_ValueError, - "the thread is not joinable and thus cannot be detached"); - return NULL; - } - self->joinable = 0; - // This is typically short so no need to release the GIL - int ret = PyThread_detach_thread(self->handle); - if (ret) { - PyErr_SetString(ThreadError, "Failed detaching thread"); - return NULL; + assert(get_thread_handle_state(handle) == THREAD_HANDLE_RUNNING); + + int err; + Py_BEGIN_ALLOW_THREADS + err = PyThread_join_thread(handle->handle); + Py_END_ALLOW_THREADS + if (err) { + PyErr_SetString(ThreadError, "Failed joining thread"); + return -1; } - Py_RETURN_NONE; + set_thread_handle_state(handle, THREAD_HANDLE_JOINED); + return 0; } static PyObject * ThreadHandle_join(ThreadHandleObject *self, void* ignored) { - if (!self->joinable) { - PyErr_SetString(PyExc_ValueError, "the thread is not joinable"); + if (get_thread_handle_state(self) == THREAD_HANDLE_INVALID) { + PyErr_SetString(PyExc_ValueError, + "the handle is invalid and thus cannot be joined"); return NULL; } - if (self->ident == PyThread_get_thread_ident_ex()) { + + // We want to perform this check outside of the `_PyOnceFlag` to prevent + // deadlock in the scenario where another thread joins us and we then + // attempt to join ourselves. However, it's not safe to check thread + // identity once the handle's os thread has finished. We may end up reusing + // the identity stored in the handle and erroneously think we are + // attempting to join ourselves. + // + // To work around this, we set `thread_is_exiting` immediately before + // `thread_run` returns. We can be sure that we are not attempting to join + // ourselves if the handle's thread is about to exit. + if (!_PyEvent_IsSet(&self->thread_is_exiting->event) && + self->ident == PyThread_get_thread_ident_ex()) { // PyThread_join_thread() would deadlock or error out. PyErr_SetString(ThreadError, "Cannot join current thread"); return NULL; } - // Before actually joining, we must first mark the thread as non-joinable, - // as joining several times simultaneously or sequentially is undefined behavior. - self->joinable = 0; - int ret; - Py_BEGIN_ALLOW_THREADS - ret = PyThread_join_thread(self->handle); - Py_END_ALLOW_THREADS - if (ret) { - PyErr_SetString(ThreadError, "Failed joining thread"); + + if (_PyOnceFlag_CallOnce(&self->once, (_Py_once_fn_t *)join_thread, + self) == -1) { return NULL; } + assert(get_thread_handle_state(self) == THREAD_HANDLE_JOINED); Py_RETURN_NONE; } @@ -178,7 +247,6 @@ static PyGetSetDef ThreadHandle_getsetlist[] = { static PyMethodDef ThreadHandle_methods[] = { - {"detach", (PyCFunction)ThreadHandle_detach, METH_NOARGS}, {"join", (PyCFunction)ThreadHandle_join, METH_NOARGS}, {0, 0} }; @@ -234,14 +302,14 @@ lock_dealloc(lockobject *self) } static inline PyLockStatus -acquire_timed(PyThread_type_lock lock, _PyTime_t timeout) +acquire_timed(PyThread_type_lock lock, PyTime_t timeout) { return PyThread_acquire_lock_timed_with_retries(lock, timeout); } static int lock_acquire_parse_args(PyObject *args, PyObject *kwds, - _PyTime_t *timeout) + PyTime_t *timeout) { char *kwlist[] = {"blocking", "timeout", NULL}; int blocking = 1; @@ -252,7 +320,7 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, // XXX Use PyThread_ParseTimeoutArg(). - const _PyTime_t unset_timeout = _PyTime_FromSeconds(-1); + const PyTime_t unset_timeout = _PyTime_FromSeconds(-1); *timeout = unset_timeout; if (timeout_obj @@ -273,7 +341,7 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, if (!blocking) *timeout = 0; else if (*timeout != unset_timeout) { - _PyTime_t microseconds; + PyTime_t microseconds; microseconds = _PyTime_AsMicroseconds(*timeout, _PyTime_ROUND_TIMEOUT); if (microseconds > PY_TIMEOUT_MAX) { @@ -288,7 +356,7 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, static PyObject * lock_PyThread_acquire_lock(lockobject *self, PyObject *args, PyObject *kwds) { - _PyTime_t timeout; + PyTime_t timeout; if (lock_acquire_parse_args(args, kwds, &timeout) < 0) return NULL; @@ -489,10 +557,18 @@ rlock_dealloc(rlockobject *self) Py_DECREF(tp); } +static bool +rlock_is_owned_by(rlockobject *self, PyThread_ident_t tid) +{ + PyThread_ident_t owner_tid = + _Py_atomic_load_ullong_relaxed(&self->rlock_owner); + return owner_tid == tid && self->rlock_count > 0; +} + static PyObject * rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds) { - _PyTime_t timeout; + PyTime_t timeout; PyThread_ident_t tid; PyLockStatus r = PY_LOCK_ACQUIRED; @@ -500,7 +576,7 @@ rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds) return NULL; tid = PyThread_get_thread_ident_ex(); - if (self->rlock_count > 0 && tid == self->rlock_owner) { + if (rlock_is_owned_by(self, tid)) { unsigned long count = self->rlock_count + 1; if (count <= self->rlock_count) { PyErr_SetString(PyExc_OverflowError, @@ -513,7 +589,7 @@ rlock_acquire(rlockobject *self, PyObject *args, PyObject *kwds) r = acquire_timed(self->rlock_lock, timeout); if (r == PY_LOCK_ACQUIRED) { assert(self->rlock_count == 0); - self->rlock_owner = tid; + _Py_atomic_store_ullong_relaxed(&self->rlock_owner, tid); self->rlock_count = 1; } else if (r == PY_LOCK_INTR) { @@ -544,13 +620,13 @@ rlock_release(rlockobject *self, PyObject *Py_UNUSED(ignored)) { PyThread_ident_t tid = PyThread_get_thread_ident_ex(); - if (self->rlock_count == 0 || self->rlock_owner != tid) { + if (!rlock_is_owned_by(self, tid)) { PyErr_SetString(PyExc_RuntimeError, "cannot release un-acquired lock"); return NULL; } if (--self->rlock_count == 0) { - self->rlock_owner = 0; + _Py_atomic_store_ullong_relaxed(&self->rlock_owner, 0); PyThread_release_lock(self->rlock_lock); } Py_RETURN_NONE; @@ -589,7 +665,7 @@ rlock_acquire_restore(rlockobject *self, PyObject *args) return NULL; } assert(self->rlock_count == 0); - self->rlock_owner = owner; + _Py_atomic_store_ullong_relaxed(&self->rlock_owner, owner); self->rlock_count = count; Py_RETURN_NONE; } @@ -614,7 +690,7 @@ rlock_release_save(rlockobject *self, PyObject *Py_UNUSED(ignored)) owner = self->rlock_owner; count = self->rlock_count; self->rlock_count = 0; - self->rlock_owner = 0; + _Py_atomic_store_ullong_relaxed(&self->rlock_owner, 0); PyThread_release_lock(self->rlock_lock); return Py_BuildValue("k" Py_PARSE_THREAD_IDENT_T, count, owner); } @@ -628,8 +704,9 @@ static PyObject * rlock_recursion_count(rlockobject *self, PyObject *Py_UNUSED(ignored)) { PyThread_ident_t tid = PyThread_get_thread_ident_ex(); - return PyLong_FromUnsignedLong( - self->rlock_owner == tid ? self->rlock_count : 0UL); + PyThread_ident_t owner = + _Py_atomic_load_ullong_relaxed(&self->rlock_owner); + return PyLong_FromUnsignedLong(owner == tid ? self->rlock_count : 0UL); } PyDoc_STRVAR(rlock_recursion_count_doc, @@ -642,7 +719,7 @@ rlock_is_owned(rlockobject *self, PyObject *Py_UNUSED(ignored)) { PyThread_ident_t tid = PyThread_get_thread_ident_ex(); - if (self->rlock_count > 0 && self->rlock_owner == tid) { + if (rlock_is_owned_by(self, tid)) { Py_RETURN_TRUE; } Py_RETURN_FALSE; @@ -676,10 +753,12 @@ rlock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * rlock_repr(rlockobject *self) { + PyThread_ident_t owner = + _Py_atomic_load_ullong_relaxed(&self->rlock_owner); return PyUnicode_FromFormat( "<%s %s object owner=%" PY_FORMAT_THREAD_IDENT_T " count=%lu at %p>", self->rlock_count ? "locked" : "unlocked", - Py_TYPE(self)->tp_name, self->rlock_owner, + Py_TYPE(self)->tp_name, owner, self->rlock_count, self); } @@ -1197,11 +1276,15 @@ _localdummy_destroyed(PyObject *localweakref, PyObject *dummyweakref) /* Module functions */ +// bootstate is used to "bootstrap" new threads. Any arguments needed by +// `thread_run()`, which can only take a single argument due to platform +// limitations, are contained in bootstate. struct bootstate { PyThreadState *tstate; PyObject *func; PyObject *args; PyObject *kwargs; + _PyEventRc *thread_is_exiting; }; @@ -1213,6 +1296,9 @@ thread_bootstate_free(struct bootstate *boot, int decref) Py_DECREF(boot->args); Py_XDECREF(boot->kwargs); } + if (boot->thread_is_exiting != NULL) { + _PyEventRc_Decref(boot->thread_is_exiting); + } PyMem_RawFree(boot); } @@ -1223,6 +1309,10 @@ thread_run(void *boot_raw) struct bootstate *boot = (struct bootstate *) boot_raw; PyThreadState *tstate = boot->tstate; + // `thread_is_exiting` needs to be set after bootstate has been freed + _PyEventRc *thread_is_exiting = boot->thread_is_exiting; + boot->thread_is_exiting = NULL; + // gh-108987: If _thread.start_new_thread() is called before or while // Python is being finalized, thread_run() can called *after*. // _PyRuntimeState_SetFinalizing() is called. At this point, all Python @@ -1267,6 +1357,11 @@ thread_run(void *boot_raw) _PyThreadState_DeleteCurrent(tstate); exit: + if (thread_is_exiting != NULL) { + _PyEvent_Notify(&thread_is_exiting->event); + _PyEventRc_Decref(thread_is_exiting); + } + // bpo-44434: Don't call explicitly PyThread_exit_thread(). On Linux with // the glibc, pthread_exit() can abort the whole process if dlopen() fails // to open the libgcc_s.so library (ex: EMFILE error). @@ -1295,7 +1390,8 @@ static int do_start_new_thread(thread_module_state* state, PyObject *func, PyObject* args, PyObject* kwargs, int joinable, - PyThread_ident_t* ident, PyThread_handle_t* handle) + PyThread_ident_t* ident, PyThread_handle_t* handle, + _PyEventRc *thread_is_exiting) { PyInterpreterState *interp = _PyInterpreterState_GET(); if (!_PyInterpreterState_HasFeature(interp, Py_RTFLAGS_THREADS)) { @@ -1328,6 +1424,10 @@ do_start_new_thread(thread_module_state* state, boot->func = Py_NewRef(func); boot->args = Py_NewRef(args); boot->kwargs = Py_XNewRef(kwargs); + boot->thread_is_exiting = thread_is_exiting; + if (thread_is_exiting != NULL) { + _PyEventRc_Incref(thread_is_exiting); + } int err; if (joinable) { @@ -1379,7 +1479,7 @@ thread_PyThread_start_new_thread(PyObject *module, PyObject *fargs) PyThread_ident_t ident = 0; PyThread_handle_t handle; if (do_start_new_thread(state, func, args, kwargs, /*joinable=*/ 0, - &ident, &handle)) { + &ident, &handle, NULL)) { return NULL; } return PyLong_FromUnsignedLongLong(ident); @@ -1423,13 +1523,13 @@ thread_PyThread_start_joinable_thread(PyObject *module, PyObject *func) return NULL; } if (do_start_new_thread(state, func, args, /*kwargs=*/ NULL, /*joinable=*/ 1, - &hobj->ident, &hobj->handle)) { + &hobj->ident, &hobj->handle, hobj->thread_is_exiting)) { Py_DECREF(args); Py_DECREF(hobj); return NULL; } + set_thread_handle_state(hobj, THREAD_HANDLE_RUNNING); Py_DECREF(args); - hobj->joinable = 1; return (PyObject*) hobj; } @@ -1935,7 +2035,7 @@ thread_module_exec(PyObject *module) // TIMEOUT_MAX double timeout_max = (double)PY_TIMEOUT_MAX * 1e-6; - double time_max = _PyTime_AsSecondsDouble(_PyTime_MAX); + double time_max = PyTime_AsSecondsDouble(PyTime_MAX); timeout_max = Py_MIN(timeout_max, time_max); // Round towards minus infinity timeout_max = floor(timeout_max); diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index 7d8c67f49fefb8..e35d1699cfea89 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -294,6 +294,8 @@ handle_queue_error(int err, PyObject *mod, int64_t qid) case ERR_QUEUES_ALLOC: PyErr_NoMemory(); break; + case -1: + return -1; default: state = get_module_state(mod); assert(state->QueueError != NULL); @@ -320,14 +322,17 @@ struct _queueitem; typedef struct _queueitem { _PyCrossInterpreterData *data; + int fmt; struct _queueitem *next; } _queueitem; static void -_queueitem_init(_queueitem *item, _PyCrossInterpreterData *data) +_queueitem_init(_queueitem *item, + _PyCrossInterpreterData *data, int fmt) { *item = (_queueitem){ .data = data, + .fmt = fmt, }; } @@ -344,14 +349,14 @@ _queueitem_clear(_queueitem *item) } static _queueitem * -_queueitem_new(_PyCrossInterpreterData *data) +_queueitem_new(_PyCrossInterpreterData *data, int fmt) { _queueitem *item = GLOBAL_MALLOC(_queueitem); if (item == NULL) { PyErr_NoMemory(); return NULL; } - _queueitem_init(item, data); + _queueitem_init(item, data, fmt); return item; } @@ -373,9 +378,11 @@ _queueitem_free_all(_queueitem *item) } static void -_queueitem_popped(_queueitem *item, _PyCrossInterpreterData **p_data) +_queueitem_popped(_queueitem *item, + _PyCrossInterpreterData **p_data, int *p_fmt) { *p_data = item->data; + *p_fmt = item->fmt; // We clear them here, so they won't be released in _queueitem_clear(). item->data = NULL; _queueitem_free(item); @@ -393,10 +400,11 @@ typedef struct _queue { _queueitem *first; _queueitem *last; } items; + int fmt; } _queue; static int -_queue_init(_queue *queue, Py_ssize_t maxsize) +_queue_init(_queue *queue, Py_ssize_t maxsize, int fmt) { PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { @@ -408,6 +416,7 @@ _queue_init(_queue *queue, Py_ssize_t maxsize) .items = { .maxsize = maxsize, }, + .fmt = fmt, }; return 0; } @@ -486,7 +495,7 @@ _queue_unlock(_queue *queue) } static int -_queue_add(_queue *queue, _PyCrossInterpreterData *data) +_queue_add(_queue *queue, _PyCrossInterpreterData *data, int fmt) { int err = _queue_lock(queue); if (err < 0) { @@ -502,7 +511,7 @@ _queue_add(_queue *queue, _PyCrossInterpreterData *data) return ERR_QUEUE_FULL; } - _queueitem *item = _queueitem_new(data); + _queueitem *item = _queueitem_new(data, fmt); if (item == NULL) { _queue_unlock(queue); return -1; @@ -522,7 +531,8 @@ _queue_add(_queue *queue, _PyCrossInterpreterData *data) } static int -_queue_next(_queue *queue, _PyCrossInterpreterData **p_data) +_queue_next(_queue *queue, + _PyCrossInterpreterData **p_data, int *p_fmt) { int err = _queue_lock(queue); if (err < 0) { @@ -541,7 +551,7 @@ _queue_next(_queue *queue, _PyCrossInterpreterData **p_data) } queue->items.count -= 1; - _queueitem_popped(item, p_data); + _queueitem_popped(item, p_data, p_fmt); _queue_unlock(queue); return 0; @@ -843,18 +853,26 @@ _queues_decref(_queues *queues, int64_t qid) PyThread_release_lock(queues->mutex); } -static int64_t * +struct queue_id_and_fmt { + int64_t id; + int fmt; +}; + +static struct queue_id_and_fmt * _queues_list_all(_queues *queues, int64_t *count) { - int64_t *qids = NULL; + struct queue_id_and_fmt *qids = NULL; PyThread_acquire_lock(queues->mutex, WAIT_LOCK); - int64_t *ids = PyMem_NEW(int64_t, (Py_ssize_t)(queues->count)); + struct queue_id_and_fmt *ids = PyMem_NEW(struct queue_id_and_fmt, + (Py_ssize_t)(queues->count)); if (ids == NULL) { goto done; } _queueref *ref = queues->head; for (int64_t i=0; ref != NULL; ref = ref->next, i++) { - ids[i] = ref->qid; + ids[i].id = ref->qid; + assert(ref->queue != NULL); + ids[i].fmt = ref->queue->fmt; } *count = queues->count; @@ -890,13 +908,13 @@ _queue_free(_queue *queue) // Create a new queue. static int64_t -queue_create(_queues *queues, Py_ssize_t maxsize) +queue_create(_queues *queues, Py_ssize_t maxsize, int fmt) { _queue *queue = GLOBAL_MALLOC(_queue); if (queue == NULL) { return ERR_QUEUE_ALLOC; } - int err = _queue_init(queue, maxsize); + int err = _queue_init(queue, maxsize, fmt); if (err < 0) { GLOBAL_FREE(queue); return (int64_t)err; @@ -925,7 +943,7 @@ queue_destroy(_queues *queues, int64_t qid) // Push an object onto the queue. static int -queue_put(_queues *queues, int64_t qid, PyObject *obj) +queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt) { // Look up the queue. _queue *queue = NULL; @@ -948,7 +966,7 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj) } // Add the data to the queue. - int res = _queue_add(queue, data); + int res = _queue_add(queue, data, fmt); _queue_unmark_waiter(queue, queues->mutex); if (res != 0) { // We may chain an exception here: @@ -963,7 +981,7 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj) // Pop the next object off the queue. Fail if empty. // XXX Support a "wait" mutex? static int -queue_get(_queues *queues, int64_t qid, PyObject **res) +queue_get(_queues *queues, int64_t qid, PyObject **res, int *p_fmt) { int err; *res = NULL; @@ -979,7 +997,7 @@ queue_get(_queues *queues, int64_t qid, PyObject **res) // Pop off the next item from the queue. _PyCrossInterpreterData *data = NULL; - err = _queue_next(queue, &data); + err = _queue_next(queue, &data, p_fmt); _queue_unmark_waiter(queue, queues->mutex); if (err != 0) { return err; @@ -1171,7 +1189,7 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj, .label = "queue ID", }; int res = idarg_int64_converter(qidobj, &converted); - Py_DECREF(qidobj); + Py_CLEAR(qidobj); if (!res) { assert(PyErr_Occurred()); return -1; @@ -1179,12 +1197,10 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj, void *raw = _queueid_xid_new(converted.id); if (raw == NULL) { - Py_DECREF(qidobj); return -1; } _PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL, _queueobj_from_xid); - Py_DECREF(qidobj); _PyCrossInterpreterData_SET_FREE(data, _queueid_xid_free); return 0; } @@ -1267,14 +1283,15 @@ qidarg_converter(PyObject *arg, void *ptr) static PyObject * queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"maxsize", NULL}; - Py_ssize_t maxsize = -1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|n:create", kwlist, - &maxsize)) { + static char *kwlist[] = {"maxsize", "fmt", NULL}; + Py_ssize_t maxsize; + int fmt; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "ni:create", kwlist, + &maxsize, &fmt)) { return NULL; } - int64_t qid = queue_create(&_globals.queues, maxsize); + int64_t qid = queue_create(&_globals.queues, maxsize, fmt); if (qid < 0) { (void)handle_queue_error((int)qid, self, qid); return NULL; @@ -1329,7 +1346,7 @@ static PyObject * queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) { int64_t count = 0; - int64_t *qids = _queues_list_all(&_globals.queues, &count); + struct queue_id_and_fmt *qids = _queues_list_all(&_globals.queues, &count); if (qids == NULL) { if (count == 0) { return PyList_New(0); @@ -1340,14 +1357,14 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) if (ids == NULL) { goto finally; } - int64_t *cur = qids; + struct queue_id_and_fmt *cur = qids; for (int64_t i=0; i < count; cur++, i++) { - PyObject *qidobj = PyLong_FromLongLong(*cur); - if (qidobj == NULL) { + PyObject *item = Py_BuildValue("Li", cur->id, cur->fmt); + if (item == NULL) { Py_SETREF(ids, NULL); break; } - PyList_SET_ITEM(ids, (Py_ssize_t)i, qidobj); + PyList_SET_ITEM(ids, (Py_ssize_t)i, item); } finally: @@ -1363,17 +1380,18 @@ Return the list of IDs for all queues."); static PyObject * queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"qid", "obj", NULL}; + static char *kwlist[] = {"qid", "obj", "fmt", NULL}; qidarg_converter_data qidarg; PyObject *obj; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O:put", kwlist, - qidarg_converter, &qidarg, &obj)) { + int fmt; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&Oi:put", kwlist, + qidarg_converter, &qidarg, &obj, &fmt)) { return NULL; } int64_t qid = qidarg.id; /* Queue up the object. */ - int err = queue_put(&_globals.queues, qid, obj); + int err = queue_put(&_globals.queues, qid, obj, fmt); if (handle_queue_error(err, self, qid)) { return NULL; } @@ -1382,7 +1400,7 @@ queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_put_doc, -"put(qid, obj)\n\ +"put(qid, obj, sharedonly=False)\n\ \n\ Add the object's data to the queue."); @@ -1399,7 +1417,8 @@ queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) int64_t qid = qidarg.id; PyObject *obj = NULL; - int err = queue_get(&_globals.queues, qid, &obj); + int fmt = 0; + int err = queue_get(&_globals.queues, qid, &obj, &fmt); if (err == ERR_QUEUE_EMPTY && dflt != NULL) { assert(obj == NULL); obj = Py_NewRef(dflt); @@ -1407,7 +1426,10 @@ queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) else if (handle_queue_error(err, self, qid)) { return NULL; } - return obj; + + PyObject *res = Py_BuildValue("Oi", obj, fmt); + Py_DECREF(obj); + return res; } PyDoc_STRVAR(queuesmod_get_doc, @@ -1499,6 +1521,33 @@ PyDoc_STRVAR(queuesmod_get_maxsize_doc, \n\ Return the maximum number of items in the queue."); +static PyObject * +queuesmod_get_default_fmt(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"qid", NULL}; + qidarg_converter_data qidarg; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_default_fmt", kwlist, + qidarg_converter, &qidarg)) { + return NULL; + } + int64_t qid = qidarg.id; + + _queue *queue = NULL; + int err = _queues_lookup(&_globals.queues, qid, &queue); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + int fmt = queue->fmt; + _queue_unmark_waiter(queue, _globals.queues.mutex); + return PyLong_FromLong(fmt); +} + +PyDoc_STRVAR(queuesmod_get_default_fmt_doc, +"get_default_fmt(qid)\n\ +\n\ +Return the default format to use for the queue."); + static PyObject * queuesmod_is_full(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1593,6 +1642,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, queuesmod_release_doc}, {"get_maxsize", _PyCFunction_CAST(queuesmod_get_maxsize), METH_VARARGS | METH_KEYWORDS, queuesmod_get_maxsize_doc}, + {"get_default_fmt", _PyCFunction_CAST(queuesmod_get_default_fmt), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_default_fmt_doc}, {"is_full", _PyCFunction_CAST(queuesmod_is_full), METH_VARARGS | METH_KEYWORDS, queuesmod_is_full_doc}, {"get_count", _PyCFunction_CAST(queuesmod_get_count), diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index b4004d165078f7..28c2f9c08bc0da 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -902,6 +902,56 @@ The code/function must not take any arguments or be a closure\n\ If a function is provided, its code object is used and all its state\n\ is ignored, including its __globals__ dict."); +static PyObject * +interp_call(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "callable", "args", "kwargs", NULL}; + PyObject *id, *callable; + PyObject *args_obj = NULL; + PyObject *kwargs_obj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OO|OO:" MODULE_NAME_STR ".call", kwlist, + &id, &callable, &args_obj, &kwargs_obj)) { + return NULL; + } + + if (args_obj != NULL) { + PyErr_SetString(PyExc_ValueError, "got unexpected args"); + return NULL; + } + if (kwargs_obj != NULL) { + PyErr_SetString(PyExc_ValueError, "got unexpected kwargs"); + return NULL; + } + + PyObject *code = (PyObject *)convert_code_arg(callable, MODULE_NAME_STR ".call", + "argument 2", "a function"); + if (code == NULL) { + return NULL; + } + + PyObject *excinfo = NULL; + int res = _interp_exec(self, id, code, NULL, &excinfo); + Py_DECREF(code); + if (res < 0) { + assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); + return excinfo; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(call_doc, +"call(id, callable, args=None, kwargs=None)\n\ +\n\ +Call the provided object in the identified interpreter.\n\ +Pass the given args and kwargs, if possible.\n\ +\n\ +\"callable\" may be a plain function with no free vars that takes\n\ +no arguments.\n\ +\n\ +The function's code object is used and all its state\n\ +is ignored, including its __globals__ dict."); + static PyObject * interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1085,6 +1135,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, is_running_doc}, {"exec", _PyCFunction_CAST(interp_exec), METH_VARARGS | METH_KEYWORDS, exec_doc}, + {"call", _PyCFunction_CAST(interp_call), + METH_VARARGS | METH_KEYWORDS, call_doc}, {"run_string", _PyCFunction_CAST(interp_run_string), METH_VARARGS | METH_KEYWORDS, run_string_doc}, {"run_func", _PyCFunction_CAST(interp_run_func), @@ -1113,6 +1165,7 @@ The 'interpreters' module provides a more convenient interface."); static int module_exec(PyObject *mod) { + PyInterpreterState *interp = PyInterpreterState_Get(); module_state *state = get_module_state(mod); // exceptions @@ -1122,6 +1175,11 @@ module_exec(PyObject *mod) if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterNotFoundError) < 0) { goto error; } + PyObject *PyExc_NotShareableError = \ + _PyInterpreterState_GetXIState(interp)->PyExc_NotShareableError; + if (PyModule_AddType(mod, (PyTypeObject *)PyExc_NotShareableError) < 0) { + goto error; + } if (register_memoryview_xid(mod, &state->XIBufferViewType) < 0) { goto error; diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 9622591a1aa855..10b2dd1c15f7fd 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -1169,6 +1169,23 @@ _elementtree_XMLParser_close(XMLParserObject *self, PyObject *Py_UNUSED(ignored) return _elementtree_XMLParser_close_impl(self); } +PyDoc_STRVAR(_elementtree_XMLParser_flush__doc__, +"flush($self, /)\n" +"--\n" +"\n"); + +#define _ELEMENTTREE_XMLPARSER_FLUSH_METHODDEF \ + {"flush", (PyCFunction)_elementtree_XMLParser_flush, METH_NOARGS, _elementtree_XMLParser_flush__doc__}, + +static PyObject * +_elementtree_XMLParser_flush_impl(XMLParserObject *self); + +static PyObject * +_elementtree_XMLParser_flush(XMLParserObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _elementtree_XMLParser_flush_impl(self); +} + PyDoc_STRVAR(_elementtree_XMLParser_feed__doc__, "feed($self, data, /)\n" "--\n" @@ -1219,4 +1236,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=218ec9e6a889f796 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=aed9f53eeb0404e0 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index d50d170589a2cd..9fff4da616ba00 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -469,6 +469,25 @@ PyDoc_STRVAR(gc_is_tracked__doc__, #define GC_IS_TRACKED_METHODDEF \ {"is_tracked", (PyCFunction)gc_is_tracked, METH_O, gc_is_tracked__doc__}, +static int +gc_is_tracked_impl(PyObject *module, PyObject *obj); + +static PyObject * +gc_is_tracked(PyObject *module, PyObject *obj) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = gc_is_tracked_impl(module, obj); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(gc_is_finalized__doc__, "is_finalized($module, obj, /)\n" "--\n" @@ -478,6 +497,25 @@ PyDoc_STRVAR(gc_is_finalized__doc__, #define GC_IS_FINALIZED_METHODDEF \ {"is_finalized", (PyCFunction)gc_is_finalized, METH_O, gc_is_finalized__doc__}, +static int +gc_is_finalized_impl(PyObject *module, PyObject *obj); + +static PyObject * +gc_is_finalized(PyObject *module, PyObject *obj) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = gc_is_finalized_impl(module, obj); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(gc_freeze__doc__, "freeze($module, /)\n" "--\n" @@ -547,4 +585,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=258f92524c1141fc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0a7e91917adcb937 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index a5b93e68598204..343cb91b975038 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -8,6 +8,53 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(pyexpat_xmlparser_SetReparseDeferralEnabled__doc__, +"SetReparseDeferralEnabled($self, enabled, /)\n" +"--\n" +"\n" +"Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0."); + +#define PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF \ + {"SetReparseDeferralEnabled", (PyCFunction)pyexpat_xmlparser_SetReparseDeferralEnabled, METH_O, pyexpat_xmlparser_SetReparseDeferralEnabled__doc__}, + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self, + int enabled); + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled(xmlparseobject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int enabled; + + enabled = PyObject_IsTrue(arg); + if (enabled < 0) { + goto exit; + } + return_value = pyexpat_xmlparser_SetReparseDeferralEnabled_impl(self, enabled); + +exit: + return return_value; +} + +PyDoc_STRVAR(pyexpat_xmlparser_GetReparseDeferralEnabled__doc__, +"GetReparseDeferralEnabled($self, /)\n" +"--\n" +"\n" +"Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0."); + +#define PYEXPAT_XMLPARSER_GETREPARSEDEFERRALENABLED_METHODDEF \ + {"GetReparseDeferralEnabled", (PyCFunction)pyexpat_xmlparser_GetReparseDeferralEnabled, METH_NOARGS, pyexpat_xmlparser_GetReparseDeferralEnabled__doc__}, + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled_impl(xmlparseobject *self); + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled(xmlparseobject *self, PyObject *Py_UNUSED(ignored)) +{ + return pyexpat_xmlparser_GetReparseDeferralEnabled_impl(self); +} + PyDoc_STRVAR(pyexpat_xmlparser_Parse__doc__, "Parse($self, data, isfinal=False, /)\n" "--\n" @@ -498,4 +545,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=48c4296e43777df4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=892e48e41f9b6e4b input=a9049054013a1b77]*/ diff --git a/Modules/expat/pyexpatns.h b/Modules/expat/pyexpatns.h index d45d9b6c457159..8ee03ef0792815 100644 --- a/Modules/expat/pyexpatns.h +++ b/Modules/expat/pyexpatns.h @@ -108,6 +108,7 @@ #define XML_SetNotStandaloneHandler PyExpat_XML_SetNotStandaloneHandler #define XML_SetParamEntityParsing PyExpat_XML_SetParamEntityParsing #define XML_SetProcessingInstructionHandler PyExpat_XML_SetProcessingInstructionHandler +#define XML_SetReparseDeferralEnabled PyExpat_XML_SetReparseDeferralEnabled #define XML_SetReturnNSTriplet PyExpat_XML_SetReturnNSTriplet #define XML_SetSkippedEntityHandler PyExpat_XML_SetSkippedEntityHandler #define XML_SetStartCdataSectionHandler PyExpat_XML_SetStartCdataSectionHandler diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 95d646c9c65b3c..02e94a21191483 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -4,6 +4,7 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_signal.h" // Py_NSIG #include "pycore_sysmodule.h" // _PySys_GetAttr() +#include "pycore_time.h" // _PyTime_FromSecondsObject() #include "pycore_traceback.h" // _Py_DumpTracebackThreads #ifdef HAVE_UNISTD_H @@ -623,7 +624,7 @@ cancel_dump_traceback_later(void) #define SEC_TO_US (1000 * 1000) static char* -format_timeout(_PyTime_t us) +format_timeout(PyTime_t us) { unsigned long sec, min, hour; char buffer[100]; @@ -656,7 +657,7 @@ faulthandler_dump_traceback_later(PyObject *self, { static char *kwlist[] = {"timeout", "repeat", "file", "exit", NULL}; PyObject *timeout_obj; - _PyTime_t timeout, timeout_us; + PyTime_t timeout, timeout_us; int repeat = 0; PyObject *file = NULL; int fd; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index a2b66b9b78c169..9807d2e7d48a36 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -201,6 +201,16 @@ gc_get_count_impl(PyObject *module) /*[clinic end generated code: output=354012e67b16398f input=a392794a08251751]*/ { GCState *gcstate = get_gc_state(); + +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct _gc_thread_state *gc = &tstate->gc; + + // Flush the local allocation count to the global count + _Py_atomic_add_int(&gcstate->generations[0].count, (int)gc->alloc_count); + gc->alloc_count = 0; +#endif + return Py_BuildValue("(iii)", gcstate->generations[0].count, gcstate->generations[1].count, @@ -373,7 +383,7 @@ gc_get_stats_impl(PyObject *module) /*[clinic input] -gc.is_tracked +gc.is_tracked -> bool obj: object / @@ -383,21 +393,15 @@ Returns true if the object is tracked by the garbage collector. Simple atomic objects will return false. [clinic start generated code]*/ -static PyObject * -gc_is_tracked(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=14f0103423b28e31 input=d83057f170ea2723]*/ +static int +gc_is_tracked_impl(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=91c8d086b7f47a33 input=423b98ec680c3126]*/ { - PyObject *result; - - if (_PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) - result = Py_True; - else - result = Py_False; - return Py_NewRef(result); + return PyObject_GC_IsTracked(obj); } /*[clinic input] -gc.is_finalized +gc.is_finalized -> bool obj: object / @@ -405,14 +409,11 @@ gc.is_finalized Returns true if the object has been already finalized by the GC. [clinic start generated code]*/ -static PyObject * -gc_is_finalized(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=e1516ac119a918ed input=201d0c58f69ae390]*/ +static int +gc_is_finalized_impl(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=401ff5d6fc660429 input=ca4d111c8f8c4e3a]*/ { - if (_PyObject_IS_GC(obj) && _PyGC_FINALIZED(obj)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + return PyObject_GC_IsFinalized(obj); } /*[clinic input] diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 164741495c7baf..d377c2c57006aa 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4623,15 +4623,15 @@ batched(p, n) --> [p0, p1, ..., p_n-1], [p_n, p_n+1, ..., p_2n-1], ...\n\ chain(p, q, ...) --> p0, p1, ... plast, q0, q1, ...\n\ chain.from_iterable([p, q, ...]) --> p0, p1, ... plast, q0, q1, ...\n\ compress(data, selectors) --> (d[0] if s[0]), (d[1] if s[1]), ...\n\ -dropwhile(pred, seq) --> seq[n], seq[n+1], starting when pred fails\n\ +dropwhile(predicate, seq) --> seq[n], seq[n+1], starting when predicate fails\n\ groupby(iterable[, keyfunc]) --> sub-iterators grouped by value of keyfunc(v)\n\ -filterfalse(pred, seq) --> elements of seq where pred(elem) is False\n\ +filterfalse(predicate, seq) --> elements of seq where predicate(elem) is False\n\ islice(seq, [start,] stop [, step]) --> elements from\n\ seq[start:stop:step]\n\ pairwise(s) --> (s[0],s[1]), (s[1],s[2]), (s[2], s[3]), ...\n\ starmap(fun, seq) --> fun(*seq[0]), fun(*seq[1]), ...\n\ tee(it, n=2) --> (it1, it2 , ... itn) splits one iterator into n\n\ -takewhile(pred, seq) --> seq[0], seq[1], until pred fails\n\ +takewhile(predicate, seq) --> seq[0], seq[1], until predicate fails\n\ zip_longest(p, q, ...) --> (p[0], q[0]), (p[1], q[1]), ...\n\ \n\ Combinatoric generators:\n\ diff --git a/Modules/makesetup b/Modules/makesetup index f000c9cd67310e..d41b6640bb5186 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -87,18 +87,6 @@ esac NL='\ ' -# Setup to link with extra libraries when making shared extensions. -# Currently, only Cygwin needs this baggage. -case `uname -s` in -CYGWIN*) if test $libdir = . - then - ExtraLibDir=. - else - ExtraLibDir='$(LIBPL)' - fi - ExtraLibs="-L$ExtraLibDir -lpython\$(LDVERSION)";; -esac - # Main loop for i in ${*-Setup} do @@ -286,7 +274,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | ;; esac rule="$file: $objs" - rule="$rule; \$(BLDSHARED) $objs $libs $ExtraLibs -o $file" + rule="$rule; \$(BLDSHARED) $objs $libs \$(MODULE_LDFLAGS) -o $file" echo "$rule" >>$rulesf done done diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 958b5a5e6e2406..fd70b38bddec79 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -23,6 +23,7 @@ #include "pycore_pylifecycle.h" // _PyOS_URandom() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_signal.h" // Py_NSIG +#include "pycore_time.h" // _PyLong_FromTime_t() #ifdef HAVE_UNISTD_H # include // symlink() @@ -645,6 +646,7 @@ PyOS_AfterFork_Child(void) #ifdef Py_GIL_DISABLED _Py_brc_after_fork(tstate->interp); + _Py_qsbr_after_fork((_PyThreadStateImpl *)tstate); #endif status = _PyEval_ReInitThreads(tstate); @@ -10409,7 +10411,7 @@ build_itimerspec(const struct itimerspec* curr_value) static PyObject * build_itimerspec_ns(const struct itimerspec* curr_value) { - _PyTime_t value, interval; + PyTime_t value, interval; if (_PyTime_FromTimespec(&value, &curr_value->it_value) < 0) { return NULL; } diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 62cd262a7885e9..f04f96bc2f7601 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -7,6 +7,7 @@ #include "pycore_pyhash.h" // _Py_HashSecret #include "pycore_traceback.h" // _PyTraceback_Add() +#include #include // offsetof() #include "expat.h" #include "pyexpat.h" @@ -81,6 +82,12 @@ typedef struct { /* NULL if not enabled */ int buffer_size; /* Size of buffer, in XML_Char units */ int buffer_used; /* Buffer units in use */ + bool reparse_deferral_enabled; /* Whether to defer reparsing of + unfinished XML tokens; a de-facto cache of + what Expat has the authority on, for lack + of a getter API function + "XML_GetReparseDeferralEnabled" in Expat + 2.6.0 */ PyObject *intern; /* Dictionary to intern strings */ PyObject **handlers; } xmlparseobject; @@ -703,6 +710,40 @@ get_parse_result(pyexpat_state *state, xmlparseobject *self, int rv) #define MAX_CHUNK_SIZE (1 << 20) +/*[clinic input] +pyexpat.xmlparser.SetReparseDeferralEnabled + + enabled: bool + / + +Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self, + int enabled) +/*[clinic end generated code: output=5ec539e3b63c8c49 input=021eb9e0bafc32c5]*/ +{ +#if XML_COMBINED_VERSION >= 20600 + XML_SetReparseDeferralEnabled(self->itself, enabled ? XML_TRUE : XML_FALSE); + self->reparse_deferral_enabled = (bool)enabled; +#endif + Py_RETURN_NONE; +} + +/*[clinic input] +pyexpat.xmlparser.GetReparseDeferralEnabled + +Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_GetReparseDeferralEnabled_impl(xmlparseobject *self) +/*[clinic end generated code: output=4e91312e88a595a8 input=54b5f11d32b20f3e]*/ +{ + return PyBool_FromLong(self->reparse_deferral_enabled); +} + /*[clinic input] pyexpat.xmlparser.Parse @@ -1063,6 +1104,8 @@ static struct PyMethodDef xmlparse_methods[] = { #if XML_COMBINED_VERSION >= 19505 PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif + PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF + PYEXPAT_XMLPARSER_GETREPARSEDEFERRALENABLED_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -1158,6 +1201,11 @@ newxmlparseobject(pyexpat_state *state, const char *encoding, self->ns_prefixes = 0; self->handlers = NULL; self->intern = Py_XNewRef(intern); +#if XML_COMBINED_VERSION >= 20600 + self->reparse_deferral_enabled = true; +#else + self->reparse_deferral_enabled = false; +#endif /* namespace_separator is either NULL or contains one char + \0 */ self->itself = XML_ParserCreate_MM(encoding, &ExpatMemoryHandler, @@ -2019,6 +2067,11 @@ pyexpat_exec(PyObject *mod) #else capi->SetHashSalt = NULL; #endif +#if XML_COMBINED_VERSION >= 20600 + capi->SetReparseDeferralEnabled = XML_SetReparseDeferralEnabled; +#else + capi->SetReparseDeferralEnabled = NULL; +#endif /* export using capsule */ PyObject *capi_object = PyCapsule_New(capi, PyExpat_CAPSULE_NAME, diff --git a/Modules/rotatingtree.c b/Modules/rotatingtree.c index 07e08bc3167c0a..217e495b3d2a9d 100644 --- a/Modules/rotatingtree.c +++ b/Modules/rotatingtree.c @@ -1,3 +1,9 @@ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "Python.h" +#include "pycore_lock.h" #include "rotatingtree.h" #define KEY_LOWER_THAN(key1, key2) ((char*)(key1) < (char*)(key2)) @@ -10,17 +16,20 @@ static unsigned int random_value = 1; static unsigned int random_stream = 0; +static PyMutex random_mutex = {0}; static int randombits(int bits) { int result; + PyMutex_Lock(&random_mutex); if (random_stream < (1U << bits)) { random_value *= 1082527; random_stream = random_value; } result = random_stream & ((1<>= bits; + PyMutex_Unlock(&random_mutex); return result; } diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 1dbde3e9e6ca5d..f16173aafa7d3c 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -15,7 +15,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_import.h" // _PyImport_GetModuleAttrString() -#include "pycore_time.h" // _PyTime_t +#include "pycore_time.h" // _PyTime_FromSecondsObject() #include #include // offsetof() @@ -297,7 +297,7 @@ select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, struct timeval tv, *tvp; int imax, omax, emax, max; int n; - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; if (timeout_obj == Py_None) tvp = (struct timeval *)NULL; @@ -619,7 +619,7 @@ select_poll_poll_impl(pollObject *self, PyObject *timeout_obj) PyObject *result_list = NULL; int poll_result, i, j; PyObject *value = NULL, *num = NULL; - _PyTime_t timeout = -1, ms = -1, deadline = 0; + PyTime_t timeout = -1, ms = -1, deadline = 0; int async_err = 0; if (timeout_obj != Py_None) { @@ -946,7 +946,7 @@ select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) PyObject *result_list = NULL; int poll_result, i; PyObject *value, *num1, *num2; - _PyTime_t timeout, ms, deadline = 0; + PyTime_t timeout, ms, deadline = 0; if (self->fd_devpoll < 0) return devpoll_err_closed(); @@ -1559,7 +1559,7 @@ select_epoll_poll_impl(pyEpoll_Object *self, PyObject *timeout_obj, int nfds, i; PyObject *elist = NULL, *etuple = NULL; struct epoll_event *evs = NULL; - _PyTime_t timeout = -1, ms = -1, deadline = 0; + PyTime_t timeout = -1, ms = -1, deadline = 0; if (self->epfd < 0) return pyepoll_err_closed(); @@ -2242,7 +2242,7 @@ select_kqueue_control_impl(kqueue_queue_Object *self, PyObject *changelist, struct kevent *chl = NULL; struct timespec timeoutspec; struct timespec *ptimeoutspec; - _PyTime_t timeout, deadline = 0; + PyTime_t timeout, deadline = 0; _selectstate *state = _selectstate_by_type(Py_TYPE(self)); if (self->kqfd < 0) diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 394a997b20c06d..5804e30af1b426 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -13,6 +13,7 @@ #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_signal.h" // _Py_RestoreSignals() +#include "pycore_time.h" // _PyTime_FromSecondsObject() #ifndef MS_WINDOWS # include "posixmodule.h" // _PyLong_FromUid() @@ -173,7 +174,7 @@ timeval_from_double(PyObject *obj, struct timeval *tv) return 0; } - _PyTime_t t; + PyTime_t t; if (_PyTime_FromSecondsObject(&t, obj, _PyTime_ROUND_CEILING) < 0) { return -1; } @@ -276,11 +277,7 @@ trip_signal(int sig_num) cleared in PyErr_CheckSignals() before .tripped. */ _Py_atomic_store_int(&is_tripped, 1); - /* Signals are always handled by the main interpreter */ - PyInterpreterState *interp = _PyInterpreterState_Main(); - - /* Notify ceval.c */ - _PyEval_SignalReceived(interp); + _PyEval_SignalReceived(); /* And then write to the wakeup fd *after* setting all the globals and doing the _PyEval_SignalReceived. We used to write to the wakeup fd @@ -303,6 +300,7 @@ trip_signal(int sig_num) int fd = wakeup.fd; if (fd != INVALID_FD) { + PyInterpreterState *interp = _PyInterpreterState_Main(); unsigned char byte = (unsigned char)sig_num; #ifdef MS_WINDOWS if (wakeup.use_send) { @@ -1210,7 +1208,7 @@ signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, PyObject *timeout_obj) /*[clinic end generated code: output=59c8971e8ae18a64 input=87fd39237cf0b7ba]*/ { - _PyTime_t timeout; + PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_CEILING) < 0) return NULL; @@ -1220,7 +1218,7 @@ signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, return NULL; } - _PyTime_t deadline = _PyDeadline_Init(timeout); + PyTime_t deadline = _PyDeadline_Init(timeout); siginfo_t si; do { @@ -1770,8 +1768,8 @@ PyErr_CheckSignals(void) Python code to ensure signals are handled. Checking for the GC here allows long running native code to clean cycles created using the C-API even if it doesn't run the evaluation loop */ - if (_Py_eval_breaker_bit_is_set(tstate->interp, _PY_GC_SCHEDULED_BIT)) { - _Py_set_eval_breaker_bit(tstate->interp, _PY_GC_SCHEDULED_BIT, 0); + if (_Py_eval_breaker_bit_is_set(tstate, _PY_GC_SCHEDULED_BIT)) { + _Py_unset_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT); _Py_RunGC(tstate); } diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 0a0e0e78656f76..cd9a803648be71 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -109,6 +109,7 @@ Local naming conventions: #include "pycore_capsule.h" // _PyCapsule_SetTraverse() #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState +#include "pycore_time.h" // _PyTime_AsMilliseconds() #ifdef _Py_MEMORY_SANITIZER # include @@ -547,7 +548,7 @@ typedef struct _socket_state { PyObject *socket_gaierror; /* Default timeout for new sockets */ - _PyTime_t defaulttimeout; + PyTime_t defaulttimeout; #if defined(HAVE_ACCEPT) || defined(HAVE_ACCEPT4) #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) @@ -772,13 +773,13 @@ internal_setblocking(PySocketSockObject *s, int block) } static int -internal_select(PySocketSockObject *s, int writing, _PyTime_t interval, +internal_select(PySocketSockObject *s, int writing, PyTime_t interval, int connect) { int n; #ifdef HAVE_POLL struct pollfd pollfd; - _PyTime_t ms; + PyTime_t ms; #else fd_set fds, efds; struct timeval tv, *tvp; @@ -888,10 +889,10 @@ sock_call_ex(PySocketSockObject *s, void *data, int connect, int *err, - _PyTime_t timeout) + PyTime_t timeout) { int has_timeout = (timeout > 0); - _PyTime_t deadline = 0; + PyTime_t deadline = 0; int deadline_initialized = 0; int res; @@ -905,7 +906,7 @@ sock_call_ex(PySocketSockObject *s, runs asynchronously. */ if (has_timeout || connect) { if (has_timeout) { - _PyTime_t interval; + PyTime_t interval; if (deadline_initialized) { /* recompute the timeout */ @@ -3011,13 +3012,13 @@ Returns True if socket is in blocking mode, or False if it\n\ is in non-blocking mode."); static int -socket_parse_timeout(_PyTime_t *timeout, PyObject *timeout_obj) +socket_parse_timeout(PyTime_t *timeout, PyObject *timeout_obj) { #ifdef MS_WINDOWS struct timeval tv; #endif #ifndef HAVE_POLL - _PyTime_t ms; + PyTime_t ms; #endif int overflow = 0; @@ -3060,7 +3061,7 @@ socket_parse_timeout(_PyTime_t *timeout, PyObject *timeout_obj) static PyObject * sock_settimeout(PySocketSockObject *s, PyObject *arg) { - _PyTime_t timeout; + PyTime_t timeout; if (socket_parse_timeout(&timeout, arg) < 0) return NULL; @@ -3112,7 +3113,7 @@ sock_gettimeout(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } else { - double seconds = _PyTime_AsSecondsDouble(s->sock_timeout); + double seconds = PyTime_AsSecondsDouble(s->sock_timeout); return PyFloat_FromDouble(seconds); } } @@ -4382,8 +4383,8 @@ sock_sendall(PySocketSockObject *s, PyObject *args) Py_buffer pbuf; struct sock_send ctx; int has_timeout = (s->sock_timeout > 0); - _PyTime_t timeout = s->sock_timeout; - _PyTime_t deadline = 0; + PyTime_t timeout = s->sock_timeout; + PyTime_t deadline = 0; int deadline_initialized = 0; PyObject *res = NULL; @@ -6916,7 +6917,7 @@ socket_getdefaulttimeout(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } else { - double seconds = _PyTime_AsSecondsDouble(state->defaulttimeout); + double seconds = PyTime_AsSecondsDouble(state->defaulttimeout); return PyFloat_FromDouble(seconds); } } @@ -6931,7 +6932,7 @@ When the socket module is first imported, the default is None."); static PyObject * socket_setdefaulttimeout(PyObject *self, PyObject *arg) { - _PyTime_t timeout; + PyTime_t timeout; if (socket_parse_timeout(&timeout, arg) < 0) return NULL; @@ -7926,6 +7927,9 @@ socket_exec(PyObject *m) #ifdef SO_BINDTODEVICE ADD_INT_MACRO(m, SO_BINDTODEVICE); #endif +#ifdef SO_BINDTOIFINDEX + ADD_INT_MACRO(m, SO_BINDTOIFINDEX); +#endif #ifdef SO_PRIORITY ADD_INT_MACRO(m, SO_PRIORITY); #endif diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 47146a28e02c8f..09fd70f351f1d8 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -1,7 +1,5 @@ /* Socket module header file */ -#include "pycore_time.h" // _PyTime_t - /* Includes needed for the sockaddr_* symbols below */ #ifndef MS_WINDOWS #ifdef __VMS @@ -324,7 +322,7 @@ typedef struct { PyObject *(*errorhandler)(void); /* Error handler; checks errno, returns NULL and sets a Python exception */ - _PyTime_t sock_timeout; /* Operation timeout in seconds; + PyTime_t sock_timeout; /* Operation timeout in seconds; 0.0 means non-blocking */ struct _socket_state *state; } PySocketSockObject; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 2b0d3900dbddd6..ed41ffd3662aa8 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -5,6 +5,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_time.h" // _PyTimeFraction #include // clock() #ifdef HAVE_SYS_TIMES_H @@ -70,12 +71,13 @@ module time /* Forward declarations */ -static int pysleep(_PyTime_t timeout); +static int pysleep(PyTime_t timeout); typedef struct { PyTypeObject *struct_time_type; -#ifdef HAVE_TIMES +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) // times() clock frequency in hertz _PyTimeFraction times_base; #endif @@ -95,26 +97,18 @@ get_time_state(PyObject *module) static PyObject* -_PyFloat_FromPyTime(_PyTime_t t) +_PyFloat_FromPyTime(PyTime_t t) { - double d = _PyTime_AsSecondsDouble(t); + double d = PyTime_AsSecondsDouble(t); return PyFloat_FromDouble(d); } -static int -get_system_time(_PyTime_t *t) -{ - // Avoid _PyTime_GetSystemClock() which silently ignores errors. - return _PyTime_GetSystemClockWithInfo(t, NULL); -} - - static PyObject * time_time(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_system_time(&t) < 0) { + PyTime_t t; + if (PyTime_Time(&t) < 0) { return NULL; } return _PyFloat_FromPyTime(t); @@ -130,11 +124,11 @@ Fractions of a second may be present if the system clock provides them."); static PyObject * time_time_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_system_time(&t) < 0) { + PyTime_t t; + if (PyTime_Time(&t) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(time_ns_doc, @@ -153,7 +147,7 @@ Return the current time in nanoseconds since the Epoch."); #endif static int -py_clock(time_module_state *state, _PyTime_t *tp, _Py_clock_info_t *info) +py_clock(time_module_state *state, PyTime_t *tp, _Py_clock_info_t *info) { _PyTimeFraction *base = &state->clock_base; @@ -171,8 +165,7 @@ py_clock(time_module_state *state, _PyTime_t *tp, _Py_clock_info_t *info) "or its value cannot be represented"); return -1; } - _PyTime_t ns = _PyTimeFraction_Mul(ticks, base); - *tp = _PyTime_FromNanoseconds(ns); + *tp = _PyTimeFraction_Mul(ticks, base); return 0; } #endif /* HAVE_CLOCK */ @@ -262,11 +255,11 @@ time_clock_gettime_ns_impl(PyObject *module, clockid_t clk_id) return NULL; } - _PyTime_t t; + PyTime_t t; if (_PyTime_FromTimespec(&t, &ts) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } #endif /* HAVE_CLOCK_GETTIME */ @@ -276,7 +269,7 @@ time_clock_settime(PyObject *self, PyObject *args) { int clk_id; PyObject *obj; - _PyTime_t t; + PyTime_t t; struct timespec tp; int ret; @@ -307,7 +300,7 @@ time_clock_settime_ns(PyObject *self, PyObject *args) { int clk_id; PyObject *obj; - _PyTime_t t; + PyTime_t t; struct timespec ts; int ret; @@ -315,7 +308,7 @@ time_clock_settime_ns(PyObject *self, PyObject *args) return NULL; } - if (_PyTime_FromNanosecondsObject(&t, obj) < 0) { + if (_PyTime_FromLong(&t, obj) < 0) { return NULL; } if (_PyTime_AsTimespec(t, &ts) == -1) { @@ -402,7 +395,7 @@ time_sleep(PyObject *self, PyObject *timeout_obj) return NULL; } - _PyTime_t timeout; + PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_TIMEOUT)) return NULL; if (timeout < 0) { @@ -1155,19 +1148,11 @@ should not be relied on."); #endif /* HAVE_WORKING_TZSET */ -static int -get_monotonic(_PyTime_t *t) -{ - // Avoid _PyTime_GetMonotonicClock() which silently ignores errors. - return _PyTime_GetMonotonicClockWithInfo(t, NULL); -} - - static PyObject * time_monotonic(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_monotonic(&t) < 0) { + PyTime_t t; + if (PyTime_Monotonic(&t) < 0) { return NULL; } return _PyFloat_FromPyTime(t); @@ -1181,11 +1166,11 @@ Monotonic clock, cannot go backward."); static PyObject * time_monotonic_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_monotonic(&t) < 0) { + PyTime_t t; + if (PyTime_Monotonic(&t) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(monotonic_ns_doc, @@ -1194,19 +1179,11 @@ PyDoc_STRVAR(monotonic_ns_doc, Monotonic clock, cannot go backward, as nanoseconds."); -static int -get_perf_counter(_PyTime_t *t) -{ - // Avoid _PyTime_GetPerfCounter() which silently ignores errors. - return _PyTime_GetPerfCounterWithInfo(t, NULL); -} - - static PyObject * time_perf_counter(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_perf_counter(&t) < 0) { + PyTime_t t; + if (PyTime_PerfCounter(&t) < 0) { return NULL; } return _PyFloat_FromPyTime(t); @@ -1221,11 +1198,11 @@ Performance counter for benchmarking."); static PyObject * time_perf_counter_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; - if (get_perf_counter(&t) < 0) { + PyTime_t t; + if (PyTime_PerfCounter(&t) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(perf_counter_ns_doc, @@ -1234,9 +1211,10 @@ PyDoc_STRVAR(perf_counter_ns_doc, Performance counter for benchmarking as nanoseconds."); -#ifdef HAVE_TIMES +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) static int -process_time_times(time_module_state *state, _PyTime_t *tp, +process_time_times(time_module_state *state, PyTime_t *tp, _Py_clock_info_t *info) { _PyTimeFraction *base = &state->times_base; @@ -1253,24 +1231,24 @@ process_time_times(time_module_state *state, _PyTime_t *tp, info->adjustable = 0; } - _PyTime_t ns; + PyTime_t ns; ns = _PyTimeFraction_Mul(process.tms_utime, base); ns += _PyTimeFraction_Mul(process.tms_stime, base); - *tp = _PyTime_FromNanoseconds(ns); + *tp = ns; return 1; } #endif static int -py_process_time(time_module_state *state, _PyTime_t *tp, +py_process_time(time_module_state *state, PyTime_t *tp, _Py_clock_info_t *info) { #if defined(MS_WINDOWS) HANDLE process; FILETIME creation_time, exit_time, kernel_time, user_time; ULARGE_INTEGER large; - _PyTime_t ktime, utime, t; + PyTime_t ktime, utime; BOOL ok; process = GetCurrentProcess(); @@ -1297,14 +1275,15 @@ py_process_time(time_module_state *state, _PyTime_t *tp, utime = large.QuadPart; /* ktime and utime have a resolution of 100 nanoseconds */ - t = _PyTime_FromNanoseconds((ktime + utime) * 100); - *tp = t; + *tp = (ktime + utime) * 100; return 0; #else /* clock_gettime */ +// gh-115714: Don't use CLOCK_PROCESS_CPUTIME_ID on WASI. #if defined(HAVE_CLOCK_GETTIME) \ - && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) + && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) \ + && !defined(__wasi__) struct timespec ts; if (HAVE_CLOCK_GETTIME_RUNTIME) { @@ -1343,7 +1322,7 @@ py_process_time(time_module_state *state, _PyTime_t *tp, struct rusage ru; if (getrusage(RUSAGE_SELF, &ru) == 0) { - _PyTime_t utime, stime; + PyTime_t utime, stime; if (info) { info->implementation = "getrusage(RUSAGE_SELF)"; @@ -1359,14 +1338,15 @@ py_process_time(time_module_state *state, _PyTime_t *tp, return -1; } - _PyTime_t total = utime + stime; + PyTime_t total = utime + stime; *tp = total; return 0; } #endif /* times() */ -#ifdef HAVE_TIMES +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) int res = process_time_times(state, tp, info); if (res < 0) { return -1; @@ -1386,7 +1366,7 @@ static PyObject * time_process_time(PyObject *module, PyObject *unused) { time_module_state *state = get_time_state(module); - _PyTime_t t; + PyTime_t t; if (py_process_time(state, &t, NULL) < 0) { return NULL; } @@ -1402,11 +1382,11 @@ static PyObject * time_process_time_ns(PyObject *module, PyObject *unused) { time_module_state *state = get_time_state(module); - _PyTime_t t; + PyTime_t t; if (py_process_time(state, &t, NULL) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(process_time_ns_doc, @@ -1419,12 +1399,12 @@ sum of the kernel and user-space CPU time."); #if defined(MS_WINDOWS) #define HAVE_THREAD_TIME static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { HANDLE thread; FILETIME creation_time, exit_time, kernel_time, user_time; ULARGE_INTEGER large; - _PyTime_t ktime, utime, t; + PyTime_t ktime, utime; BOOL ok; thread = GetCurrentThread(); @@ -1451,15 +1431,14 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) utime = large.QuadPart; /* ktime and utime have a resolution of 100 nanoseconds */ - t = _PyTime_FromNanoseconds((ktime + utime) * 100); - *tp = t; + *tp = (ktime + utime) * 100; return 0; } #elif defined(_AIX) #define HAVE_THREAD_TIME static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { /* bpo-40192: On AIX, thread_cputime() is preferred: it has nanosecond resolution, whereas clock_gettime(CLOCK_THREAD_CPUTIME_ID) @@ -1476,14 +1455,14 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) info->adjustable = 0; info->resolution = 1e-9; } - *tp = _PyTime_FromNanoseconds(tc.stime + tc.utime); + *tp = (tc.stime + tc.utime); return 0; } #elif defined(__sun) && defined(__SVR4) #define HAVE_THREAD_TIME static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { /* bpo-35455: On Solaris, CLOCK_THREAD_CPUTIME_ID clock is not always available; use gethrvtime() to substitute this functionality. */ @@ -1493,7 +1472,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) info->monotonic = 1; info->adjustable = 0; } - *tp = _PyTime_FromNanoseconds(gethrvtime()); + *tp = gethrvtime(); return 0; } @@ -1504,7 +1483,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability) static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) __attribute__((availability(macos, introduced=10.12))) __attribute__((availability(ios, introduced=10.0))) __attribute__((availability(tvos, introduced=10.0))) @@ -1512,7 +1491,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #endif static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { struct timespec ts; const clockid_t clk_id = CLOCK_THREAD_CPUTIME_ID; @@ -1554,7 +1533,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) static PyObject * time_thread_time(PyObject *self, PyObject *unused) { - _PyTime_t t; + PyTime_t t; if (_PyTime_GetThreadTimeWithInfo(&t, NULL) < 0) { return NULL; } @@ -1569,11 +1548,11 @@ Thread time for profiling: sum of the kernel and user-space CPU time."); static PyObject * time_thread_time_ns(PyObject *self, PyObject *unused) { - _PyTime_t t; + PyTime_t t; if (_PyTime_GetThreadTimeWithInfo(&t, NULL) < 0) { return NULL; } - return _PyTime_AsNanosecondsObject(t); + return _PyTime_AsLong(t); } PyDoc_STRVAR(thread_time_ns_doc, @@ -1595,7 +1574,7 @@ time_get_clock_info(PyObject *module, PyObject *args) char *name; _Py_clock_info_t info; PyObject *obj = NULL, *dict, *ns; - _PyTime_t t; + PyTime_t t; if (!PyArg_ParseTuple(args, "s:get_clock_info", &name)) { return NULL; @@ -1614,17 +1593,17 @@ time_get_clock_info(PyObject *module, PyObject *args) #endif if (strcmp(name, "time") == 0) { - if (_PyTime_GetSystemClockWithInfo(&t, &info) < 0) { + if (_PyTime_TimeWithInfo(&t, &info) < 0) { return NULL; } } else if (strcmp(name, "monotonic") == 0) { - if (_PyTime_GetMonotonicClockWithInfo(&t, &info) < 0) { + if (_PyTime_MonotonicWithInfo(&t, &info) < 0) { return NULL; } } else if (strcmp(name, "perf_counter") == 0) { - if (_PyTime_GetPerfCounterWithInfo(&t, &info) < 0) { + if (_PyTime_PerfCounterWithInfo(&t, &info) < 0) { return NULL; } } @@ -2094,7 +2073,8 @@ time_exec(PyObject *module) } #endif -#ifdef HAVE_TIMES +// gh-115714: Don't use times() on WASI. +#if defined(HAVE_TIMES) && !defined(__wasi__) long ticks_per_second; if (_Py_GetTicksPerSecond(&ticks_per_second) < 0) { PyErr_SetString(PyExc_RuntimeError, @@ -2174,7 +2154,7 @@ PyInit_time(void) // On error, raise an exception and return -1. // On success, return 0. static int -pysleep(_PyTime_t timeout) +pysleep(PyTime_t timeout) { assert(timeout >= 0); @@ -2186,10 +2166,10 @@ pysleep(_PyTime_t timeout) #else struct timeval timeout_tv; #endif - _PyTime_t deadline, monotonic; + PyTime_t deadline, monotonic; int err = 0; - if (get_monotonic(&monotonic) < 0) { + if (PyTime_Monotonic(&monotonic) < 0) { return -1; } deadline = monotonic + timeout; @@ -2242,7 +2222,7 @@ pysleep(_PyTime_t timeout) } #ifndef HAVE_CLOCK_NANOSLEEP - if (get_monotonic(&monotonic) < 0) { + if (PyTime_Monotonic(&monotonic) < 0) { return -1; } timeout = deadline - monotonic; @@ -2255,7 +2235,7 @@ pysleep(_PyTime_t timeout) return 0; #else // MS_WINDOWS - _PyTime_t timeout_100ns = _PyTime_As100Nanoseconds(timeout, + PyTime_t timeout_100ns = _PyTime_As100Nanoseconds(timeout, _PyTime_ROUND_CEILING); // Maintain Windows Sleep() semantics for time.sleep(0) diff --git a/Objects/abstract.c b/Objects/abstract.c index 07d4b89fe188c8..8357175aa5591e 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1390,7 +1390,7 @@ _PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs) } UNARY_FUNC(PyNumber_Negative, nb_negative, __neg__, "unary -") -UNARY_FUNC(PyNumber_Positive, nb_positive, __pow__, "unary +") +UNARY_FUNC(PyNumber_Positive, nb_positive, __pos__, "unary +") UNARY_FUNC(PyNumber_Invert, nb_invert, __invert__, "unary ~") UNARY_FUNC(PyNumber_Absolute, nb_absolute, __abs__, "abs()") diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index acc59b926448ca..5e3b3affbc76c5 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1729,6 +1729,10 @@ bytearray_extend(PyByteArrayObject *self, PyObject *iterable_of_ints) while ((item = PyIter_Next(it)) != NULL) { if (! _getbytevalue(item, &value)) { + if (PyErr_ExceptionMatches(PyExc_TypeError) && PyUnicode_Check(iterable_of_ints)) { + PyErr_Format(PyExc_TypeError, + "expected iterable of integers; got: 'str'"); + } Py_DECREF(item); Py_DECREF(it); Py_DECREF(bytearray_obj); diff --git a/Objects/clinic/dictobject.c.h b/Objects/clinic/dictobject.c.h index daaef211b1db49..fb46c4c64334f9 100644 --- a/Objects/clinic/dictobject.c.h +++ b/Objects/clinic/dictobject.c.h @@ -66,21 +66,6 @@ PyDoc_STRVAR(dict___contains____doc__, #define DICT___CONTAINS___METHODDEF \ {"__contains__", (PyCFunction)dict___contains__, METH_O|METH_COEXIST, dict___contains____doc__}, -static PyObject * -dict___contains___impl(PyDictObject *self, PyObject *key); - -static PyObject * -dict___contains__(PyDictObject *self, PyObject *key) -{ - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = dict___contains___impl(self, key); - Py_END_CRITICAL_SECTION(); - - return return_value; -} - PyDoc_STRVAR(dict_get__doc__, "get($self, key, default=None, /)\n" "--\n" @@ -327,4 +312,4 @@ dict_values(PyDictObject *self, PyObject *Py_UNUSED(ignored)) { return dict_values_impl(self); } -/*[clinic end generated code: output=c8fda06bac5b05f3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f3dd5f3fb8122aef input=a9049054013a1b77]*/ diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 805de2971ba475..df546a090c28e4 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1519,22 +1519,34 @@ class property(object): self.__doc__ = doc except AttributeError: # read-only or dict-less class pass + self.__name = None + + def __set_name__(self, owner, name): + self.__name = name + + @property + def __name__(self): + return self.__name if self.__name is not None else self.fget.__name__ + + @__name__.setter + def __name__(self, value): + self.__name = value def __get__(self, inst, type=None): if inst is None: return self if self.__get is None: - raise AttributeError, "property has no getter" + raise AttributeError("property has no getter") return self.__get(inst) def __set__(self, inst, value): if self.__set is None: - raise AttributeError, "property has no setter" + raise AttributeError("property has no setter") return self.__set(inst, value) def __delete__(self, inst): if self.__del is None: - raise AttributeError, "property has no deleter" + raise AttributeError("property has no deleter") return self.__del(inst) */ @@ -1628,6 +1640,20 @@ property_dealloc(PyObject *self) Py_TYPE(self)->tp_free(self); } +static int +property_name(propertyobject *prop, PyObject **name) +{ + if (prop->prop_name != NULL) { + *name = Py_NewRef(prop->prop_name); + return 1; + } + if (prop->prop_get == NULL) { + *name = NULL; + return 0; + } + return PyObject_GetOptionalAttr(prop->prop_get, &_Py_ID(__name__), name); +} + static PyObject * property_descr_get(PyObject *self, PyObject *obj, PyObject *type) { @@ -1637,11 +1663,15 @@ property_descr_get(PyObject *self, PyObject *obj, PyObject *type) propertyobject *gs = (propertyobject *)self; if (gs->prop_get == NULL) { + PyObject *propname; + if (property_name(gs, &propname) < 0) { + return NULL; + } PyObject *qualname = PyType_GetQualName(Py_TYPE(obj)); - if (gs->prop_name != NULL && qualname != NULL) { + if (propname != NULL && qualname != NULL) { PyErr_Format(PyExc_AttributeError, "property %R of %R object has no getter", - gs->prop_name, + propname, qualname); } else if (qualname != NULL) { @@ -1652,6 +1682,7 @@ property_descr_get(PyObject *self, PyObject *obj, PyObject *type) PyErr_SetString(PyExc_AttributeError, "property has no getter"); } + Py_XDECREF(propname); Py_XDECREF(qualname); return NULL; } @@ -1673,16 +1704,20 @@ property_descr_set(PyObject *self, PyObject *obj, PyObject *value) } if (func == NULL) { + PyObject *propname; + if (property_name(gs, &propname) < 0) { + return -1; + } PyObject *qualname = NULL; if (obj != NULL) { qualname = PyType_GetQualName(Py_TYPE(obj)); } - if (gs->prop_name != NULL && qualname != NULL) { + if (propname != NULL && qualname != NULL) { PyErr_Format(PyExc_AttributeError, value == NULL ? "property %R of %R object has no deleter" : "property %R of %R object has no setter", - gs->prop_name, + propname, qualname); } else if (qualname != NULL) { @@ -1698,6 +1733,7 @@ property_descr_set(PyObject *self, PyObject *obj, PyObject *value) "property has no deleter" : "property has no setter"); } + Py_XDECREF(propname); Py_XDECREF(qualname); return -1; } @@ -1730,15 +1766,12 @@ property_copy(PyObject *old, PyObject *get, PyObject *set, PyObject *del) return NULL; if (get == NULL || get == Py_None) { - Py_XDECREF(get); get = pold->prop_get ? pold->prop_get : Py_None; } if (set == NULL || set == Py_None) { - Py_XDECREF(set); set = pold->prop_set ? pold->prop_set : Py_None; } if (del == NULL || del == Py_None) { - Py_XDECREF(del); del = pold->prop_del ? pold->prop_del : Py_None; } if (pold->getter_doc && get != Py_None) { @@ -1886,6 +1919,28 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, return 0; } +static PyObject * +property_get__name__(propertyobject *prop, void *Py_UNUSED(ignored)) +{ + PyObject *name; + if (property_name(prop, &name) < 0) { + return NULL; + } + if (name == NULL) { + PyErr_SetString(PyExc_AttributeError, + "'property' object has no attribute '__name__'"); + } + return name; +} + +static int +property_set__name__(propertyobject *prop, PyObject *value, + void *Py_UNUSED(ignored)) +{ + Py_XSETREF(prop->prop_name, Py_XNewRef(value)); + return 0; +} + static PyObject * property_get___isabstractmethod__(propertyobject *prop, void *closure) { @@ -1916,6 +1971,7 @@ property_get___isabstractmethod__(propertyobject *prop, void *closure) } static PyGetSetDef property_getsetlist[] = { + {"__name__", (getter)property_get__name__, (setter)property_set__name__}, {"__isabstractmethod__", (getter)property_get___isabstractmethod__, NULL, NULL, diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 25ab21881f8f74..39778df603f43f 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -21,6 +21,7 @@ As of Python 3.6, this is compact and ordered. Basic idea is described here: | dk_log2_size | | dk_log2_index_bytes | | dk_kind | +| dk_version | | dk_usable | | dk_nentries | +---------------------+ @@ -55,7 +56,7 @@ The DictObject can be in one of two forms. Either: A combined table: ma_values == NULL, dk_refcnt == 1. - Values are stored in the me_value field of the PyDictKeysObject. + Values are stored in the me_value field of the PyDictKeyEntry. Or: A split table: ma_values != NULL, dk_refcnt >= 1 @@ -80,6 +81,8 @@ DK_ENTRIES(keys)[index] if index >= 0): Active upon key insertion. Dummy slots cannot be made Unused again else the probe sequence in case of collision would have no way to know they were once active. + In free-threaded builds dummy slots are not re-used to allow lock-free + lookups to proceed safely. 4. Pending. index >= 0, key != NULL, and value == NULL (split only) Not yet inserted in split-table. @@ -113,19 +116,20 @@ As a consequence of this, split keys have a maximum size of 16. #define PyDict_MINSIZE 8 #include "Python.h" -#include "pycore_bitutils.h" // _Py_bit_length -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_GetBuiltin() -#include "pycore_code.h" // stats -#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION -#include "pycore_dict.h" // export _PyDict_SizeOf() -#include "pycore_freelist.h" // _PyFreeListState_GET() -#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() -#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() -#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_setobject.h" // _PySet_NextEntry() -#include "stringlib/eq.h" // unicode_eq() +#include "pycore_bitutils.h" // _Py_bit_length +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_code.h" // stats +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION +#include "pycore_dict.h" // export _PyDict_SizeOf() +#include "pycore_freelist.h" // _PyFreeListState_GET() +#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_RELAXED +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_setobject.h" // _PySet_NextEntry() +#include "stringlib/eq.h" // unicode_eq() #include @@ -150,13 +154,111 @@ ASSERT_DICT_LOCKED(PyObject *op) _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); } #define ASSERT_DICT_LOCKED(op) ASSERT_DICT_LOCKED(_Py_CAST(PyObject*, op)) +#define IS_DICT_SHARED(mp) _PyObject_GC_IS_SHARED(mp) +#define SET_DICT_SHARED(mp) _PyObject_GC_SET_SHARED(mp) +#define LOAD_INDEX(keys, size, idx) _Py_atomic_load_int##size##_relaxed(&((const int##size##_t*)keys->dk_indices)[idx]); +#define STORE_INDEX(keys, size, idx, value) _Py_atomic_store_int##size##_relaxed(&((int##size##_t*)keys->dk_indices)[idx], (int##size##_t)value); +#define ASSERT_OWNED_OR_SHARED(mp) \ + assert(_Py_IsOwnedByCurrentThread((PyObject *)mp) || IS_DICT_SHARED(mp)); +#define LOAD_KEYS_NENTRIES(d) -#else +static inline Py_ssize_t +load_keys_nentries(PyDictObject *mp) +{ + PyDictKeysObject *keys = _Py_atomic_load_ptr(&mp->ma_keys); + return _Py_atomic_load_ssize(&keys->dk_nentries); +} + +static inline void +set_keys(PyDictObject *mp, PyDictKeysObject *keys) +{ + ASSERT_OWNED_OR_SHARED(mp); + _Py_atomic_store_ptr_release(&mp->ma_keys, keys); +} + +static inline void +set_values(PyDictObject *mp, PyDictValues *values) +{ + ASSERT_OWNED_OR_SHARED(mp); + _Py_atomic_store_ptr_release(&mp->ma_values, values); +} + +// Defined until we get QSBR +#define _PyMem_FreeQsbr PyMem_Free + +#define LOCK_KEYS(keys) PyMutex_LockFlags(&keys->dk_mutex, _Py_LOCK_DONT_DETACH) +#define UNLOCK_KEYS(keys) PyMutex_Unlock(&keys->dk_mutex) + +#define ASSERT_KEYS_LOCKED(keys) assert(PyMutex_IsLocked(&keys->dk_mutex)) +#define LOAD_SHARED_KEY(key) _Py_atomic_load_ptr_acquire(&key) +#define STORE_SHARED_KEY(key, value) _Py_atomic_store_ptr_release(&key, value) +// Inc refs the keys object, giving the previous value +#define INCREF_KEYS(dk) _Py_atomic_add_ssize(&dk->dk_refcnt, 1) +// Dec refs the keys object, giving the previous value +#define DECREF_KEYS(dk) _Py_atomic_add_ssize(&dk->dk_refcnt, -1) +#define LOAD_KEYS_NENTIRES(keys) _Py_atomic_load_ssize_relaxed(&keys->dk_nentries) + +static inline void split_keys_entry_added(PyDictKeysObject *keys) +{ + ASSERT_KEYS_LOCKED(keys); + + // We increase before we decrease so we never get too small of a value + // when we're racing with reads + _Py_atomic_store_ssize_relaxed(&keys->dk_nentries, keys->dk_nentries + 1); + _Py_atomic_store_ssize_release(&keys->dk_usable, keys->dk_usable - 1); +} + +#else /* Py_GIL_DISABLED */ #define ASSERT_DICT_LOCKED(op) +#define LOCK_KEYS(keys) +#define UNLOCK_KEYS(keys) +#define ASSERT_KEYS_LOCKED(keys) +#define LOAD_SHARED_KEY(key) key +#define STORE_SHARED_KEY(key, value) key = value +#define INCREF_KEYS(dk) dk->dk_refcnt++ +#define DECREF_KEYS(dk) dk->dk_refcnt-- +#define LOAD_KEYS_NENTIRES(keys) keys->dk_nentries +#define IS_DICT_SHARED(mp) (false) +#define SET_DICT_SHARED(mp) +#define LOAD_INDEX(keys, size, idx) ((const int##size##_t*)(keys->dk_indices))[idx] +#define STORE_INDEX(keys, size, idx, value) ((int##size##_t*)(keys->dk_indices))[idx] = (int##size##_t)value + +static inline void split_keys_entry_added(PyDictKeysObject *keys) +{ + keys->dk_usable--; + keys->dk_nentries++; +} + +static inline void +set_keys(PyDictObject *mp, PyDictKeysObject *keys) +{ + mp->ma_keys = keys; +} + +static inline void +set_values(PyDictObject *mp, PyDictValues *values) +{ + mp->ma_values = values; +} + +static inline Py_ssize_t +load_keys_nentries(PyDictObject *mp) +{ + return mp->ma_keys->dk_nentries; +} + #endif +#define STORE_KEY(ep, key) FT_ATOMIC_STORE_PTR_RELEASE(ep->me_key, key) +#define STORE_VALUE(ep, value) FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, value) +#define STORE_SPLIT_VALUE(mp, idx, value) FT_ATOMIC_STORE_PTR_RELEASE(mp->ma_values->values[idx], value) +#define STORE_HASH(ep, hash) FT_ATOMIC_STORE_SSIZE_RELAXED(ep->me_hash, hash) +#define STORE_KEYS_USABLE(keys, usable) FT_ATOMIC_STORE_SSIZE_RELAXED(keys->dk_usable, usable) +#define STORE_KEYS_NENTRIES(keys, nentries) FT_ATOMIC_STORE_SSIZE_RELAXED(keys->dk_nentries, nentries) +#define STORE_USED(mp, used) FT_ATOMIC_STORE_SSIZE_RELAXED(mp->ma_used, used) + #define PERTURB_SHIFT 5 /* @@ -256,10 +358,6 @@ static int dictresize(PyInterpreterState *interp, PyDictObject *mp, static PyObject* dict_iter(PyObject *dict); -static int -contains_lock_held(PyDictObject *mp, PyObject *key); -static int -contains_known_hash_lock_held(PyDictObject *mp, PyObject *key, Py_ssize_t hash); static int setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value); static int @@ -330,7 +428,7 @@ _PyDict_DebugMallocStats(FILE *out) #define DK_MASK(dk) (DK_SIZE(dk)-1) -static void free_keys_object(PyDictKeysObject *keys); +static void free_keys_object(PyDictKeysObject *keys, bool use_qsbr); /* PyDictKeysObject has refcounts like PyObject does, so we have the following two functions to mirror what Py_INCREF() and Py_DECREF() do. @@ -348,11 +446,11 @@ dictkeys_incref(PyDictKeysObject *dk) #ifdef Py_REF_DEBUG _Py_IncRefTotal(_PyInterpreterState_GET()); #endif - dk->dk_refcnt++; + INCREF_KEYS(dk); } static inline void -dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) +dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) { if (dk->dk_refcnt == _Py_IMMORTAL_REFCNT) { return; @@ -361,7 +459,7 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) #ifdef Py_REF_DEBUG _Py_DecRefTotal(_PyInterpreterState_GET()); #endif - if (--dk->dk_refcnt == 0) { + if (DECREF_KEYS(dk) == 1) { if (DK_IS_UNICODE(dk)) { PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dk); Py_ssize_t i, n; @@ -378,7 +476,7 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) Py_XDECREF(entries[i].me_value); } } - free_keys_object(dk); + free_keys_object(dk, use_qsbr); } } @@ -390,22 +488,18 @@ dictkeys_get_index(const PyDictKeysObject *keys, Py_ssize_t i) Py_ssize_t ix; if (log2size < 8) { - const int8_t *indices = (const int8_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 8, i); } else if (log2size < 16) { - const int16_t *indices = (const int16_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 16, i); } #if SIZEOF_VOID_P > 4 else if (log2size >= 32) { - const int64_t *indices = (const int64_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 64, i); } #endif else { - const int32_t *indices = (const int32_t*)(keys->dk_indices); - ix = indices[i]; + ix = LOAD_INDEX(keys, 32, i); } assert(ix >= DKIX_DUMMY); return ix; @@ -421,25 +515,21 @@ dictkeys_set_index(PyDictKeysObject *keys, Py_ssize_t i, Py_ssize_t ix) assert(keys->dk_version == 0); if (log2size < 8) { - int8_t *indices = (int8_t*)(keys->dk_indices); assert(ix <= 0x7f); - indices[i] = (char)ix; + STORE_INDEX(keys, 8, i, ix); } else if (log2size < 16) { - int16_t *indices = (int16_t*)(keys->dk_indices); assert(ix <= 0x7fff); - indices[i] = (int16_t)ix; + STORE_INDEX(keys, 16, i, ix); } #if SIZEOF_VOID_P > 4 else if (log2size >= 32) { - int64_t *indices = (int64_t*)(keys->dk_indices); - indices[i] = ix; + STORE_INDEX(keys, 64, i, ix); } #endif else { - int32_t *indices = (int32_t*)(keys->dk_indices); assert(ix <= 0x7fffffff); - indices[i] = (int32_t)ix; + STORE_INDEX(keys, 32, i, ix); } } @@ -512,6 +602,9 @@ static PyDictKeysObject empty_keys_struct = { 0, /* dk_log2_size */ 0, /* dk_log2_index_bytes */ DICT_KEYS_UNICODE, /* dk_kind */ +#ifdef Py_GIL_DISABLED + {0}, /* dk_mutex */ +#endif 1, /* dk_version */ 0, /* dk_usable (immutable) */ 0, /* dk_nentries */ @@ -697,6 +790,9 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) dk->dk_log2_size = log2_size; dk->dk_log2_index_bytes = log2_bytes; dk->dk_kind = unicode ? DICT_KEYS_UNICODE : DICT_KEYS_GENERAL; +#ifdef Py_GIL_DISABLED + dk->dk_mutex = (PyMutex){0}; +#endif dk->dk_nentries = 0; dk->dk_usable = usable; dk->dk_version = 0; @@ -706,8 +802,14 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) } static void -free_keys_object(PyDictKeysObject *keys) +free_keys_object(PyDictKeysObject *keys, bool use_qsbr) { +#ifdef Py_GIL_DISABLED + if (use_qsbr) { + _PyMem_FreeQsbr(keys); + return; + } +#endif #ifdef WITH_FREELISTS struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE @@ -739,9 +841,15 @@ new_values(size_t size) } static inline void -free_values(PyDictValues *values) +free_values(PyDictValues *values, bool use_qsbr) { - int prefix_size = ((uint8_t *)values)[-1]; + int prefix_size = DICT_VALUES_SIZE(values); +#ifdef Py_GIL_DISABLED + if (use_qsbr) { + _PyMem_FreeQsbr(((char *)values)-prefix_size); + return; + } +#endif PyMem_Free(((char *)values)-prefix_size); } @@ -767,9 +875,9 @@ new_dict(PyInterpreterState *interp, { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); if (free_values_on_failure) { - free_values(values); + free_values(values, false); } return NULL; } @@ -785,7 +893,19 @@ new_dict(PyInterpreterState *interp, static inline size_t shared_keys_usable_size(PyDictKeysObject *keys) { +#ifdef Py_GIL_DISABLED + // dk_usable will decrease for each instance that is created and each + // value that is added. dk_nentries will increase for each value that + // is added. We want to always return the right value or larger. + // We therefore increase dk_nentries first and we decrease dk_usable + // second, and conversely here we read dk_usable first and dk_entries + // second (to avoid the case where we read entries before the increment + // and read usable after the decrement) + return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) + + _Py_atomic_load_ssize_acquire(&keys->dk_nentries)); +#else return (size_t)keys->dk_nentries + (size_t)keys->dk_usable; +#endif } /* Consumes a reference to the keys object */ @@ -795,7 +915,7 @@ new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); return PyErr_NoMemory(); } ((char *)values)[-2] = 0; @@ -1074,7 +1194,7 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu PyDictKeysObject *dk; DictKeysKind kind; Py_ssize_t ix; - + // TODO: Thread safety start: dk = mp->ma_keys; kind = dk->dk_kind; @@ -1118,6 +1238,260 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu return ix; } +static inline void +ensure_shared_on_read(PyDictObject *mp) +{ +#ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread((PyObject *)mp) && !IS_DICT_SHARED(mp)) { + // The first time we access a dict from a non-owning thread we mark it + // as shared. This ensures that a concurrent resize operation will + // delay freeing the old keys or values using QSBR, which is necessary + // to safely allow concurrent reads without locking... + Py_BEGIN_CRITICAL_SECTION(mp); + if (!IS_DICT_SHARED(mp)) { + SET_DICT_SHARED(mp); + } + Py_END_CRITICAL_SECTION(); + } +#endif +} + +static inline void +ensure_shared_on_resize(PyDictObject *mp) +{ +#ifdef Py_GIL_DISABLED + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); + + if (!_Py_IsOwnedByCurrentThread((PyObject *)mp) && !IS_DICT_SHARED(mp)) { + // We are writing to the dict from another thread that owns + // it and we haven't marked it as shared which will ensure + // that when we re-size ma_keys or ma_values that we will + // free using QSBR. We need to lock the dictionary to + // contend with writes from the owning thread, mark it as + // shared, and then we can continue with lock-free reads. + // Technically this is a little heavy handed, we could just + // free the individual old keys / old-values using qsbr + SET_DICT_SHARED(mp); + } +#endif +} + +#ifdef Py_GIL_DISABLED + +static inline Py_ALWAYS_INLINE +Py_ssize_t compare_unicode_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + assert(startkey == NULL || PyUnicode_CheckExact(ep->me_key)); + assert(!PyUnicode_CheckExact(key)); + + if (startkey != NULL) { + if (!_Py_TryIncref(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; + } + + if (unicode_get_hash(startkey) == hash) { + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; + } + if (dk == _Py_atomic_load_ptr_relaxed(&mp->ma_keys) && + startkey == _Py_atomic_load_ptr_relaxed(&ep->me_key)) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; + } + } + else { + Py_DECREF(startkey); + } + } + return 0; +} + +// Search non-Unicode key from Unicode table +static Py_ssize_t +unicodekeys_lookup_generic_threadsafe(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(mp, dk, key, hash, compare_unicode_generic_threadsafe); +} + +static inline Py_ALWAYS_INLINE Py_ssize_t +compare_unicode_unicode_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + assert(startkey == NULL || PyUnicode_CheckExact(startkey)); + if (startkey == key) { + return 1; + } + if (startkey != NULL) { + if (_Py_IsImmortal(startkey)) { + return unicode_get_hash(startkey) == hash && unicode_eq(startkey, key); + } + else { + if (!_Py_TryIncref(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; + } + if (unicode_get_hash(startkey) == hash && unicode_eq(startkey, key)) { + Py_DECREF(startkey); + return 1; + } + Py_DECREF(startkey); + } + } + return 0; +} + +static Py_ssize_t _Py_HOT_FUNCTION +unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(NULL, dk, key, hash, compare_unicode_unicode_threadsafe); +} + +static inline Py_ALWAYS_INLINE +Py_ssize_t compare_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, + void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) +{ + PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix]; + PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + if (startkey == key) { + return 1; + } + Py_ssize_t ep_hash = _Py_atomic_load_ssize_relaxed(&ep->me_hash); + if (ep_hash == hash) { + if (startkey == NULL || !_Py_TryIncref(&ep->me_key, startkey)) { + return DKIX_KEY_CHANGED; + } + int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) { + return DKIX_ERROR; + } + if (dk == _Py_atomic_load_ptr_relaxed(&mp->ma_keys) && + startkey == _Py_atomic_load_ptr_relaxed(&ep->me_key)) { + return cmp; + } + else { + /* The dict was mutated, restart */ + return DKIX_KEY_CHANGED; + } + } + return 0; +} + +static Py_ssize_t +dictkeys_generic_lookup_threadsafe(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) +{ + return do_lookup(mp, dk, key, hash, compare_generic_threadsafe); +} + +static Py_ssize_t +_Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr) +{ + PyDictKeysObject *dk; + DictKeysKind kind; + Py_ssize_t ix; + PyObject *value; + + ensure_shared_on_read(mp); + + dk = _Py_atomic_load_ptr(&mp->ma_keys); + kind = dk->dk_kind; + + if (kind != DICT_KEYS_GENERAL) { + if (PyUnicode_CheckExact(key)) { + ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); + } + else { + ix = unicodekeys_lookup_generic_threadsafe(mp, dk, key, hash); + } + if (ix == DKIX_KEY_CHANGED) { + goto read_failed; + } + + if (ix >= 0) { + if (kind == DICT_KEYS_SPLIT) { + PyDictValues *values = _Py_atomic_load_ptr(&mp->ma_values); + if (values == NULL) + goto read_failed; + + uint8_t capacity = _Py_atomic_load_uint8_relaxed(&DICT_VALUES_SIZE(values)); + if (ix >= (Py_ssize_t)capacity) + goto read_failed; + + value = _Py_TryXGetRef(&values->values[ix]); + if (value == NULL) + goto read_failed; + + if (values != _Py_atomic_load_ptr(&mp->ma_values)) { + Py_DECREF(value); + goto read_failed; + } + } + else { + value = _Py_TryXGetRef(&DK_UNICODE_ENTRIES(dk)[ix].me_value); + if (value == NULL) { + goto read_failed; + } + + if (dk != _Py_atomic_load_ptr(&mp->ma_keys)) { + Py_DECREF(value); + goto read_failed; + } + } + } + else { + value = NULL; + } + } + else { + ix = dictkeys_generic_lookup_threadsafe(mp, dk, key, hash); + if (ix == DKIX_KEY_CHANGED) { + goto read_failed; + } + if (ix >= 0) { + value = _Py_TryXGetRef(&DK_ENTRIES(dk)[ix].me_value); + if (value == NULL) + goto read_failed; + + if (dk != _Py_atomic_load_ptr(&mp->ma_keys)) { + Py_DECREF(value); + goto read_failed; + } + } + else { + value = NULL; + } + } + + *value_addr = value; + return ix; + +read_failed: + // In addition to the normal races of the dict being modified the _Py_TryXGetRef + // can all fail if they don't yet have a shared ref count. That can happen here + // or in the *_lookup_* helper. In that case we need to take the lock to avoid + // mutation and do a normal incref which will make them shared. + Py_BEGIN_CRITICAL_SECTION(mp); + ix = _Py_dict_lookup(mp, key, hash, &value); + *value_addr = value; + if (value != NULL) { + assert(ix >= 0); + Py_INCREF(value); + } + Py_END_CRITICAL_SECTION(); + return ix; +} + +#endif + int _PyDict_HasOnlyStringKeys(PyObject *dict) { @@ -1188,10 +1562,19 @@ _PyDict_MaybeUntrack(PyObject *op) _PyObject_GC_UNTRACK(op); } +static inline int +is_unusable_slot(Py_ssize_t ix) +{ +#ifdef Py_GIL_DISABLED + return ix >= 0 || ix == DKIX_DUMMY; +#else + return ix >= 0; +#endif +} + /* Internal function to find slot for an item from its hash when it is known that the key is not present in the dict. - - The dict must be combined. */ + */ static Py_ssize_t find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) { @@ -1200,7 +1583,7 @@ find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) const size_t mask = DK_MASK(keys); size_t i = hash & mask; Py_ssize_t ix = dictkeys_get_index(keys, i); - for (size_t perturb = hash; ix >= 0;) { + for (size_t perturb = hash; is_unusable_slot(ix);) { perturb >>= PERTURB_SHIFT; i = (i*5 + perturb + 1) & mask; ix = dictkeys_get_index(keys, i); @@ -1215,17 +1598,11 @@ insertion_resize(PyInterpreterState *interp, PyDictObject *mp, int unicode) } static Py_ssize_t -insert_into_dictkeys(PyDictKeysObject *keys, PyObject *name) +insert_into_splitdictkeys(PyDictKeysObject *keys, PyObject *name, Py_hash_t hash) { assert(PyUnicode_CheckExact(name)); - Py_hash_t hash = unicode_get_hash(name); - if (hash == -1) { - hash = PyUnicode_Type.tp_hash(name); - if (hash == -1) { - PyErr_Clear(); - return DKIX_EMPTY; - } - } + ASSERT_KEYS_LOCKED(keys); + Py_ssize_t ix = unicodekeys_lookup_unicode(keys, name, hash); if (ix == DKIX_EMPTY) { if (keys->dk_usable <= 0) { @@ -1239,13 +1616,80 @@ insert_into_dictkeys(PyDictKeysObject *keys, PyObject *name) dictkeys_set_index(keys, hashpos, ix); assert(ep->me_key == NULL); ep->me_key = Py_NewRef(name); - keys->dk_usable--; - keys->dk_nentries++; + split_keys_entry_added(keys); } assert (ix < SHARED_KEYS_MAX_SIZE); return ix; } +static inline int +insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, + Py_hash_t hash, PyObject *key, PyObject *value) +{ + if (mp->ma_keys->dk_usable <= 0) { + /* Need to resize. */ + if (insertion_resize(interp, mp, 1) < 0) { + return -1; + } + } + + Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); + dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); + + if (DK_IS_UNICODE(mp->ma_keys)) { + PyDictUnicodeEntry *ep; + ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; + STORE_KEY(ep, key); + STORE_VALUE(ep, value); + } + else { + PyDictKeyEntry *ep; + ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; + STORE_KEY(ep, key); + STORE_VALUE(ep, value); + STORE_HASH(ep, hash); + } + STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - 1); + STORE_KEYS_NENTRIES(mp->ma_keys, mp->ma_keys->dk_nentries + 1); + assert(mp->ma_keys->dk_usable >= 0); + return 0; +} + +static int +insert_split_dict(PyInterpreterState *interp, PyDictObject *mp, + Py_hash_t hash, PyObject *key, PyObject *value) +{ + PyDictKeysObject *keys = mp->ma_keys; + LOCK_KEYS(keys); + if (keys->dk_usable <= 0) { + /* Need to resize. */ + UNLOCK_KEYS(keys); + int ins = insertion_resize(interp, mp, 1); + if (ins < 0) { + return -1; + } + assert(!_PyDict_HasSplitTable(mp)); + return insert_combined_dict(interp, mp, hash, key, value); + } + + Py_ssize_t hashpos = find_empty_slot(keys, hash); + dictkeys_set_index(keys, hashpos, keys->dk_nentries); + + PyDictUnicodeEntry *ep; + ep = &DK_UNICODE_ENTRIES(keys)[keys->dk_nentries]; + STORE_SHARED_KEY(ep->me_key, key); + + Py_ssize_t index = keys->dk_nentries; + _PyDictValues_AddToInsertionOrder(mp->ma_values, index); + assert (mp->ma_values->values[index] == NULL); + STORE_SPLIT_VALUE(mp, index, value); + + split_keys_entry_added(keys); + assert(keys->dk_usable >= 0); + UNLOCK_KEYS(keys); + return 0; +} + /* Internal routine to insert a new item into the table. Used both by the internal resize routine and by the public insert routine. @@ -1278,41 +1722,19 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, /* Insert into new slot. */ mp->ma_keys->dk_version = 0; assert(old_value == NULL); - if (mp->ma_keys->dk_usable <= 0) { - /* Need to resize. */ - if (insertion_resize(interp, mp, 1) < 0) - goto Fail; - } - - Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); - dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); - if (DK_IS_UNICODE(mp->ma_keys)) { - PyDictUnicodeEntry *ep; - ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = key; - if (mp->ma_values) { - Py_ssize_t index = mp->ma_keys->dk_nentries; - _PyDictValues_AddToInsertionOrder(mp->ma_values, index); - assert (mp->ma_values->values[index] == NULL); - mp->ma_values->values[index] = value; - } - else { - ep->me_value = value; + if (!_PyDict_HasSplitTable(mp)) { + if (insert_combined_dict(interp, mp, hash, key, value) < 0) { + goto Fail; } } else { - PyDictKeyEntry *ep; - ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = key; - ep->me_hash = hash; - ep->me_value = value; + if (insert_split_dict(interp, mp, hash, key, value) < 0) + goto Fail; } + mp->ma_used++; mp->ma_version_tag = new_version; - mp->ma_keys->dk_usable--; - mp->ma_keys->dk_nentries++; - assert(mp->ma_keys->dk_usable >= 0); ASSERT_CONSISTENT(mp); return 0; } @@ -1349,7 +1771,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, return -1; } -// Same to insertdict but specialized for ma_keys = Py_EMPTY_KEYS. +// Same as insertdict but specialized for ma_keys == Py_EMPTY_KEYS. // Consumes key and value references. static int insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, @@ -1370,28 +1792,37 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, return -1; } /* We don't decref Py_EMPTY_KEYS here because it is immortal. */ - mp->ma_keys = newkeys; - mp->ma_values = NULL; + assert(mp->ma_values == NULL); MAINTAIN_TRACKING(mp, key, value); size_t hashpos = (size_t)hash & (PyDict_MINSIZE-1); - dictkeys_set_index(mp->ma_keys, hashpos, 0); + dictkeys_set_index(newkeys, hashpos, 0); if (unicode) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mp->ma_keys); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(newkeys); ep->me_key = key; ep->me_value = value; } else { - PyDictKeyEntry *ep = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *ep = DK_ENTRIES(newkeys); ep->me_key = key; ep->me_hash = hash; ep->me_value = value; } mp->ma_used++; mp->ma_version_tag = new_version; - mp->ma_keys->dk_usable--; - mp->ma_keys->dk_nentries++; + newkeys->dk_usable--; + newkeys->dk_nentries++; + // We store the keys last so no one can see them in a partially inconsistent + // state so that we don't need to switch the keys to being shared yet for + // the case where we're inserting from the non-owner thread. We don't use + // set_keys here because the transition from empty to non-empty is safe + // as the empty keys will never be freed. +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ptr_release(&mp->ma_keys, newkeys); +#else + mp->ma_keys = newkeys; +#endif return 0; } @@ -1447,7 +1878,7 @@ static int dictresize(PyInterpreterState *interp, PyDictObject *mp, uint8_t log2_newsize, int unicode) { - PyDictKeysObject *oldkeys; + PyDictKeysObject *oldkeys, *newkeys; PyDictValues *oldvalues; ASSERT_DICT_LOCKED(mp); @@ -1465,30 +1896,31 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, unicode = 0; } + ensure_shared_on_resize(mp); /* NOTE: Current odict checks mp->ma_keys to detect resize happen. * So we can't reuse oldkeys even if oldkeys->dk_size == newsize. * TODO: Try reusing oldkeys when reimplement odict. */ /* Allocate a new table. */ - mp->ma_keys = new_keys_object(interp, log2_newsize, unicode); - if (mp->ma_keys == NULL) { - mp->ma_keys = oldkeys; + newkeys = new_keys_object(interp, log2_newsize, unicode); + if (newkeys == NULL) { return -1; } // New table must be large enough. - assert(mp->ma_keys->dk_usable >= mp->ma_used); + assert(newkeys->dk_usable >= mp->ma_used); Py_ssize_t numentries = mp->ma_used; if (oldvalues != NULL) { - PyDictUnicodeEntry *oldentries = DK_UNICODE_ENTRIES(oldkeys); + LOCK_KEYS(oldkeys); + PyDictUnicodeEntry *oldentries = DK_UNICODE_ENTRIES(oldkeys); /* Convert split table into new combined table. * We must incref keys; we can transfer values. */ - if (mp->ma_keys->dk_kind == DICT_KEYS_GENERAL) { + if (newkeys->dk_kind == DICT_KEYS_GENERAL) { // split -> generic - PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *newentries = DK_ENTRIES(newkeys); for (Py_ssize_t i = 0; i < numentries; i++) { int index = get_index_from_order(mp, i); @@ -1498,10 +1930,10 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i].me_hash = unicode_get_hash(ep->me_key); newentries[i].me_value = oldvalues->values[index]; } - build_indices_generic(mp->ma_keys, newentries, numentries); + build_indices_generic(newkeys, newentries, numentries); } else { // split -> combined unicode - PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(mp->ma_keys); + PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(newkeys); for (Py_ssize_t i = 0; i < numentries; i++) { int index = get_index_from_order(mp, i); @@ -1510,18 +1942,20 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i].me_key = Py_NewRef(ep->me_key); newentries[i].me_value = oldvalues->values[index]; } - build_indices_unicode(mp->ma_keys, newentries, numentries); + build_indices_unicode(newkeys, newentries, numentries); } - dictkeys_decref(interp, oldkeys); - mp->ma_values = NULL; - free_values(oldvalues); + UNLOCK_KEYS(oldkeys); + set_keys(mp, newkeys); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + set_values(mp, NULL); + free_values(oldvalues, IS_DICT_SHARED(mp)); } else { // oldkeys is combined. if (oldkeys->dk_kind == DICT_KEYS_GENERAL) { // generic -> generic - assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); + assert(newkeys->dk_kind == DICT_KEYS_GENERAL); PyDictKeyEntry *oldentries = DK_ENTRIES(oldkeys); - PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *newentries = DK_ENTRIES(newkeys); if (oldkeys->dk_nentries == numentries) { memcpy(newentries, oldentries, numentries * sizeof(PyDictKeyEntry)); } @@ -1533,12 +1967,12 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i] = *ep++; } } - build_indices_generic(mp->ma_keys, newentries, numentries); + build_indices_generic(newkeys, newentries, numentries); } else { // oldkeys is combined unicode PyDictUnicodeEntry *oldentries = DK_UNICODE_ENTRIES(oldkeys); if (unicode) { // combined unicode -> combined unicode - PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(mp->ma_keys); + PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(newkeys); if (oldkeys->dk_nentries == numentries && mp->ma_keys->dk_kind == DICT_KEYS_UNICODE) { memcpy(newentries, oldentries, numentries * sizeof(PyDictUnicodeEntry)); } @@ -1550,10 +1984,10 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i] = *ep++; } } - build_indices_unicode(mp->ma_keys, newentries, numentries); + build_indices_unicode(newkeys, newentries, numentries); } else { // combined unicode -> generic - PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *newentries = DK_ENTRIES(newkeys); PyDictUnicodeEntry *ep = oldentries; for (Py_ssize_t i = 0; i < numentries; i++) { while (ep->me_value == NULL) @@ -1563,22 +1997,24 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, newentries[i].me_value = ep->me_value; ep++; } - build_indices_generic(mp->ma_keys, newentries, numentries); + build_indices_generic(newkeys, newentries, numentries); } } + set_keys(mp, newkeys); + if (oldkeys != Py_EMPTY_KEYS) { #ifdef Py_REF_DEBUG _Py_DecRefTotal(_PyInterpreterState_GET()); #endif assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); - free_keys_object(oldkeys); + free_keys_object(oldkeys, IS_DICT_SHARED(mp)); } } - mp->ma_keys->dk_usable -= numentries; - mp->ma_keys->dk_nentries = numentries; + STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - numentries); + STORE_KEYS_NENTRIES(mp->ma_keys, numentries); ASSERT_CONSISTENT(mp); return 0; } @@ -1695,9 +2131,13 @@ dict_getitem(PyObject *op, PyObject *key, const char *warnmsg) PyObject *value; Py_ssize_t ix; (void)ix; - PyObject *exc = _PyErr_GetRaisedException(tstate); +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif /* Ignore any exception raised by the lookup */ PyObject *exc2 = _PyErr_Occurred(tstate); @@ -1753,11 +2193,47 @@ _PyDict_GetItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return NULL; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif assert(ix >= 0 || value == NULL); return value; // borrowed reference } +/* Gets an item and provides a new reference if the value is present. + * Returns 1 if the key is present, 0 if the key is missing, and -1 if an + * exception occurred. +*/ +int +_PyDict_GetItemRef_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash, PyObject **result) +{ + PyDictObject*mp = (PyDictObject *)op; + + PyObject *value; +#ifdef Py_GIL_DISABLED + Py_ssize_t ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); +#else + Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); +#endif + assert(ix >= 0 || value == NULL); + if (ix == DKIX_ERROR) { + *result = NULL; + return -1; + } + if (value == NULL) { + *result = NULL; + return 0; // missing key + } +#ifdef Py_GIL_DISABLED + *result = value; +#else + *result = Py_NewRef(value); +#endif + return 1; // key is present +} int PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) @@ -1767,7 +2243,6 @@ PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) *result = NULL; return -1; } - PyDictObject*mp = (PyDictObject *)op; Py_hash_t hash; if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) @@ -1779,19 +2254,7 @@ PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) } } - PyObject *value; - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); - assert(ix >= 0 || value == NULL); - if (ix == DKIX_ERROR) { - *result = NULL; - return -1; - } - if (value == NULL) { - *result = NULL; - return 0; // missing key - } - *result = Py_NewRef(value); - return 1; // key is present + return _PyDict_GetItemRef_KnownHash(op, key, hash, result); } @@ -1819,7 +2282,12 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key) } } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_XDECREF(value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif assert(ix >= 0 || value == NULL); return value; // borrowed reference } @@ -1860,7 +2328,6 @@ _PyDict_GetItemStringWithError(PyObject *v, const char *key) return rv; } - /* Fast version of global value lookup (LOAD_GLOBAL). * Lookup in globals, then builtins. * @@ -1874,6 +2341,7 @@ _PyDict_GetItemStringWithError(PyObject *v, const char *key) PyObject * _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) { + // TODO: Thread safety Py_ssize_t ix; Py_hash_t hash; PyObject *value; @@ -2025,7 +2493,7 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, mp->ma_used--; mp->ma_version_tag = new_version; - if (mp->ma_values) { + if (_PyDict_HasSplitTable(mp)) { assert(old_value == mp->ma_values->values[ix]); mp->ma_values->values[ix] = NULL; assert(ix < SHARED_KEYS_MAX_SIZE); @@ -2039,15 +2507,15 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, if (DK_IS_UNICODE(mp->ma_keys)) { PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; old_key = ep->me_key; - ep->me_key = NULL; - ep->me_value = NULL; + STORE_KEY(ep, NULL); + STORE_VALUE(ep, NULL); } else { PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; old_key = ep->me_key; - ep->me_key = NULL; - ep->me_value = NULL; - ep->me_hash = 0; + STORE_KEY(ep, NULL); + STORE_VALUE(ep, NULL); + STORE_HASH(ep, 0); } Py_DECREF(old_key); } @@ -2195,9 +2663,11 @@ clear_lock_held(PyObject *op) PyInterpreterState *interp = _PyInterpreterState_GET(); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); - dictkeys_incref(Py_EMPTY_KEYS); - mp->ma_keys = Py_EMPTY_KEYS; - mp->ma_values = NULL; + // We don't inc ref empty keys because they're immortal + ensure_shared_on_resize(mp); + + set_keys(mp, Py_EMPTY_KEYS); + set_values(mp, NULL); mp->ma_used = 0; mp->ma_version_tag = new_version; /* ...then clear the keys and values */ @@ -2205,12 +2675,12 @@ clear_lock_held(PyObject *op) n = oldkeys->dk_nentries; for (i = 0; i < n; i++) Py_CLEAR(oldvalues->values[i]); - free_values(oldvalues); - dictkeys_decref(interp, oldkeys); + free_values(oldvalues, IS_DICT_SHARED(mp)); + dictkeys_decref(interp, oldkeys, false); } else { assert(oldkeys->dk_refcnt == 1); - dictkeys_decref(interp, oldkeys); + dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); } ASSERT_CONSISTENT(mp); } @@ -2244,14 +2714,13 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, mp = (PyDictObject *)op; i = *ppos; - if (mp->ma_values) { + if (_PyDict_HasSplitTable(mp)) { assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); if (i < 0 || i >= mp->ma_used) return 0; int index = get_index_from_order(mp, i); value = mp->ma_values->values[index]; - - key = DK_UNICODE_ENTRIES(mp->ma_keys)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(mp->ma_keys)[index].me_key); hash = unicode_get_hash(key); assert(value != NULL); } @@ -2596,12 +3065,12 @@ dict_dealloc(PyObject *self) for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { Py_XDECREF(values->values[i]); } - free_values(values); - dictkeys_decref(interp, keys); + free_values(values, false); + dictkeys_decref(interp, keys, false); } else if (keys != NULL) { assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); } #ifdef WITH_FREELISTS struct _Py_dict_freelist *freelist = get_dict_freelist(); @@ -2721,7 +3190,7 @@ dict_length(PyObject *self) } static PyObject * -dict_subscript_lock_held(PyObject *self, PyObject *key) +dict_subscript(PyObject *self, PyObject *key) { PyDictObject *mp = (PyDictObject *)self; Py_ssize_t ix; @@ -2733,7 +3202,11 @@ dict_subscript_lock_held(PyObject *self, PyObject *key) if (hash == -1) return NULL; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif if (ix == DKIX_ERROR) return NULL; if (ix == DKIX_EMPTY || value == NULL) { @@ -2753,17 +3226,11 @@ dict_subscript_lock_held(PyObject *self, PyObject *key) _PyErr_SetKeyError(key); return NULL; } +#ifdef Py_GIL_DISABLED + return value; +#else return Py_NewRef(value); -} - -static PyObject * -dict_subscript(PyObject *self, PyObject *key) -{ - PyObject *res; - Py_BEGIN_CRITICAL_SECTION(self); - res = dict_subscript_lock_held(self, key); - Py_END_CRITICAL_SECTION(); - return res; +#endif } static int @@ -3141,10 +3608,11 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe if (keys == NULL) return -1; - dictkeys_decref(interp, mp->ma_keys); + ensure_shared_on_resize(mp); + dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); mp->ma_keys = keys; - if (mp->ma_values != NULL) { - free_values(mp->ma_values); + if (_PyDict_HasSplitTable(mp)) { + free_values(mp->ma_values, IS_DICT_SHARED(mp)); mp->ma_values = NULL; } @@ -3187,7 +3655,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe Py_NewRef(key), hash, Py_NewRef(value)); } else { - err = contains_known_hash_lock_held(mp, key, hash); + err = _PyDict_Contains_KnownHash((PyObject *)mp, key, hash); if (err == 0) { err = insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); @@ -3270,7 +3738,7 @@ dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) for (key = PyIter_Next(iter); key; key = PyIter_Next(iter)) { if (override != 1) { - status = contains_lock_held(mp, key); + status = PyDict_Contains(a, key); if (status != 0) { if (status > 0) { if (override == 0) { @@ -3374,7 +3842,7 @@ copy_lock_held(PyObject *o) return PyErr_NoMemory(); split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type); if (split_copy == NULL) { - free_values(newvalues); + free_values(newvalues, false); return NULL; } size_t prefix_size = ((uint8_t *)newvalues)[-1]; @@ -3484,7 +3952,7 @@ dict_equal_lock_held(PyDictObject *a, PyDictObject *b) /* can't be equal if # of entries differ */ return 0; /* Same # of entries -- check all of 'em. Exit early on any diff. */ - for (i = 0; i < a->ma_keys->dk_nentries; i++) { + for (i = 0; i < LOAD_KEYS_NENTIRES(a->ma_keys); i++) { PyObject *key, *aval; Py_hash_t hash; if (DK_IS_UNICODE(a->ma_keys)) { @@ -3494,7 +3962,7 @@ dict_equal_lock_held(PyDictObject *a, PyDictObject *b) continue; } hash = unicode_get_hash(key); - if (a->ma_values) + if (_PyDict_HasSplitTable(a)) aval = a->ma_values->values[i]; else aval = ep->me_value; @@ -3568,7 +4036,6 @@ dict_richcompare(PyObject *v, PyObject *w, int op) /*[clinic input] @coexist -@critical_section dict.__contains__ key: object @@ -3578,25 +4045,17 @@ True if the dictionary has the specified key, else False. [clinic start generated code]*/ static PyObject * -dict___contains___impl(PyDictObject *self, PyObject *key) -/*[clinic end generated code: output=1b314e6da7687dae input=bc76ec9c157cb81b]*/ +dict___contains__(PyDictObject *self, PyObject *key) +/*[clinic end generated code: output=a3d03db709ed6e6b input=fe1cb42ad831e820]*/ { - register PyDictObject *mp = self; - Py_hash_t hash; - Py_ssize_t ix; - PyObject *value; - - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; - } - ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) + int contains = PyDict_Contains((PyObject *)self, key); + if (contains < 0) { return NULL; - if (ix == DKIX_EMPTY || value == NULL) - Py_RETURN_FALSE; - Py_RETURN_TRUE; + } + if (contains) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; } /*[clinic input] @@ -3623,13 +4082,24 @@ dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value) if (hash == -1) return NULL; } +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(self, key, hash, &val); +#else ix = _Py_dict_lookup(self, key, hash, &val); +#endif if (ix == DKIX_ERROR) return NULL; +#ifdef Py_GIL_DISABLED + if (ix == DKIX_EMPTY || val == NULL) { + val = Py_NewRef(default_value); + } + return val; +#else if (ix == DKIX_EMPTY || val == NULL) { val = default_value; } return Py_NewRef(val); +#endif } static int @@ -3697,42 +4167,31 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu interp, PyDict_EVENT_ADDED, mp, key, default_value); mp->ma_keys->dk_version = 0; value = default_value; - if (mp->ma_keys->dk_usable <= 0) { - if (insertion_resize(interp, mp, 1) < 0) { + + if (!_PyDict_HasSplitTable(mp)) { + if (insert_combined_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { + Py_DECREF(key); + Py_DECREF(value); if (result) { *result = NULL; } return -1; } } - Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); - dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); - if (DK_IS_UNICODE(mp->ma_keys)) { - assert(PyUnicode_CheckExact(key)); - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = Py_NewRef(key); - if (_PyDict_HasSplitTable(mp)) { - Py_ssize_t index = (int)mp->ma_keys->dk_nentries; - assert(index < SHARED_KEYS_MAX_SIZE); - assert(mp->ma_values->values[index] == NULL); - mp->ma_values->values[index] = Py_NewRef(value); - _PyDictValues_AddToInsertionOrder(mp->ma_values, index); - } - else { - ep->me_value = Py_NewRef(value); - } - } else { - PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; - ep->me_key = Py_NewRef(key); - ep->me_hash = hash; - ep->me_value = Py_NewRef(value); + if (insert_split_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { + Py_DECREF(key); + Py_DECREF(value); + if (result) { + *result = NULL; + } + return -1; + } } + MAINTAIN_TRACKING(mp, key, value); mp->ma_used++; mp->ma_version_tag = new_version; - mp->ma_keys->dk_usable--; - mp->ma_keys->dk_nentries++; assert(mp->ma_keys->dk_usable >= 0); ASSERT_CONSISTENT(mp); if (result) { @@ -3881,8 +4340,8 @@ dict_popitem_impl(PyDictObject *self) return NULL; } /* Convert split table to combined table */ - if (self->ma_keys->dk_kind == DICT_KEYS_SPLIT) { - if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1)) { + if (_PyDict_HasSplitTable(self)) { + if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1) < 0) { Py_DECREF(res); return NULL; } @@ -3934,8 +4393,8 @@ dict_popitem_impl(PyDictObject *self) PyTuple_SET_ITEM(res, 0, key); PyTuple_SET_ITEM(res, 1, value); /* We can't dk_usable++ since there is DKIX_DUMMY in indices */ - self->ma_keys->dk_nentries = i; - self->ma_used--; + STORE_KEYS_NENTRIES(self->ma_keys, i); + STORE_USED(self, self->ma_used - 1); self->ma_version_tag = new_version; ASSERT_CONSISTENT(self); return res; @@ -3949,7 +4408,7 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) Py_ssize_t i, n = keys->dk_nentries; if (DK_IS_UNICODE(keys)) { - if (mp->ma_values != NULL) { + if (_PyDict_HasSplitTable(mp)) { for (i = 0; i < n; i++) { Py_VISIT(mp->ma_values->values[i]); } @@ -3986,7 +4445,7 @@ static Py_ssize_t sizeof_lock_held(PyDictObject *mp) { size_t res = _PyObject_SIZE(Py_TYPE(mp)); - if (mp->ma_values) { + if (_PyDict_HasSplitTable(mp)) { res += shared_keys_usable_size(mp->ma_keys) * sizeof(PyObject*); } /* If the dictionary is split, the keys portion is accounted-for @@ -4092,49 +4551,19 @@ static PyMethodDef mapp_methods[] = { {NULL, NULL} /* sentinel */ }; -static int -contains_known_hash_lock_held(PyDictObject *mp, PyObject *key, Py_ssize_t hash) -{ - Py_ssize_t ix; - PyObject *value; - - ASSERT_DICT_LOCKED(mp); - - ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) - return -1; - return (ix != DKIX_EMPTY && value != NULL); -} - -static int -contains_lock_held(PyDictObject *mp, PyObject *key) +/* Return 1 if `key` is in dict `op`, 0 if not, and -1 on error. */ +int +PyDict_Contains(PyObject *op, PyObject *key) { Py_hash_t hash; - Py_ssize_t ix; - PyObject *value; - - ASSERT_DICT_LOCKED(mp); if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { hash = PyObject_Hash(key); if (hash == -1) return -1; } - ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) - return -1; - return (ix != DKIX_EMPTY && value != NULL); -} -/* Return 1 if `key` is in dict `op`, 0 if not, and -1 on error. */ -int -PyDict_Contains(PyObject *op, PyObject *key) -{ - int res; - Py_BEGIN_CRITICAL_SECTION(op); - res = contains_lock_held((PyDictObject *)op, key); - Py_END_CRITICAL_SECTION(); - return res; + return _PyDict_Contains_KnownHash(op, key, hash); } int @@ -4157,10 +4586,20 @@ _PyDict_Contains_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) PyObject *value; Py_ssize_t ix; +#ifdef Py_GIL_DISABLED + ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); +#else ix = _Py_dict_lookup(mp, key, hash, &value); +#endif if (ix == DKIX_ERROR) return -1; - return (ix != DKIX_EMPTY && value != NULL); + if (ix != DKIX_EMPTY && value != NULL) { +#ifdef Py_GIL_DISABLED + Py_DECREF(value); +#endif + return 1; + } + return 0; } int @@ -4431,11 +4870,11 @@ dictiter_new(PyDictObject *dict, PyTypeObject *itertype) if (itertype == &PyDictRevIterKey_Type || itertype == &PyDictRevIterItem_Type || itertype == &PyDictRevIterValue_Type) { - if (dict->ma_values) { + if (_PyDict_HasSplitTable(dict)) { di->di_pos = dict->ma_used - 1; } else { - di->di_pos = dict->ma_keys->dk_nentries - 1; + di->di_pos = load_keys_nentries(dict) - 1; } } else { @@ -4502,6 +4941,14 @@ static PyMethodDef dictiter_methods[] = { {NULL, NULL} /* sentinel */ }; +#ifdef Py_GIL_DISABLED + +static int +dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, + PyObject **out_key, PyObject **out_value); + +#else /* Py_GIL_DISABLED */ + static PyObject* dictiter_iternextkey_lock_held(PyDictObject *d, PyObject *self) { @@ -4523,11 +4970,11 @@ dictiter_iternextkey_lock_held(PyDictObject *d, PyObject *self) i = di->di_pos; k = d->ma_keys; assert(i >= 0); - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { if (i >= d->ma_used) goto fail; int index = get_index_from_order(d, i); - key = DK_UNICODE_ENTRIES(k)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(k)[index].me_key); assert(d->ma_values->values[index] != NULL); } else { @@ -4569,6 +5016,8 @@ dictiter_iternextkey_lock_held(PyDictObject *d, PyObject *self) return NULL; } +#endif /* Py_GIL_DISABLED */ + static PyObject* dictiter_iternextkey(PyObject *self) { @@ -4579,9 +5028,13 @@ dictiter_iternextkey(PyObject *self) return NULL; PyObject *value; - Py_BEGIN_CRITICAL_SECTION(d); +#ifdef Py_GIL_DISABLED + if (dictiter_iternext_threadsafe(d, self, &value, NULL) < 0) { + value = NULL; + } +#else value = dictiter_iternextkey_lock_held(d, self); - Py_END_CRITICAL_SECTION(); +#endif return value; } @@ -4619,6 +5072,8 @@ PyTypeObject PyDictIterKey_Type = { 0, }; +#ifndef Py_GIL_DISABLED + static PyObject * dictiter_iternextvalue_lock_held(PyDictObject *d, PyObject *self) { @@ -4638,7 +5093,7 @@ dictiter_iternextvalue_lock_held(PyDictObject *d, PyObject *self) i = di->di_pos; assert(i >= 0); - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { if (i >= d->ma_used) goto fail; int index = get_index_from_order(d, i); @@ -4684,6 +5139,8 @@ dictiter_iternextvalue_lock_held(PyDictObject *d, PyObject *self) return NULL; } +#endif /* Py_GIL_DISABLED */ + static PyObject * dictiter_iternextvalue(PyObject *self) { @@ -4694,9 +5151,13 @@ dictiter_iternextvalue(PyObject *self) return NULL; PyObject *value; - Py_BEGIN_CRITICAL_SECTION(d); +#ifdef Py_GIL_DISABLED + if (dictiter_iternext_threadsafe(d, self, NULL, &value) < 0) { + value = NULL; + } +#else value = dictiter_iternextvalue_lock_held(d, self); - Py_END_CRITICAL_SECTION(); +#endif return value; } @@ -4734,11 +5195,12 @@ PyTypeObject PyDictIterValue_Type = { 0, }; -static PyObject * -dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self) +static int +dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self, + PyObject **out_key, PyObject **out_value) { dictiterobject *di = (dictiterobject *)self; - PyObject *key, *value, *result; + PyObject *key, *value; Py_ssize_t i; assert (PyDict_Check(d)); @@ -4747,16 +5209,17 @@ dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self) PyErr_SetString(PyExc_RuntimeError, "dictionary changed size during iteration"); di->di_used = -1; /* Make this state sticky */ - return NULL; + return -1; } - i = di->di_pos; + i = FT_ATOMIC_LOAD_SSIZE_RELAXED(di->di_pos); + assert(i >= 0); - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { if (i >= d->ma_used) goto fail; int index = get_index_from_order(d, i); - key = DK_UNICODE_ENTRIES(d->ma_keys)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(d->ma_keys)[index].me_key); value = d->ma_values->values[index]; assert(value != NULL); } @@ -4793,34 +5256,184 @@ dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self) } di->di_pos = i+1; di->len--; - result = di->di_result; - if (Py_REFCNT(result) == 1) { - PyObject *oldkey = PyTuple_GET_ITEM(result, 0); - PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); - PyTuple_SET_ITEM(result, 0, Py_NewRef(key)); - PyTuple_SET_ITEM(result, 1, Py_NewRef(value)); - Py_INCREF(result); - Py_DECREF(oldkey); - Py_DECREF(oldvalue); - // bpo-42536: The GC may have untracked this result tuple. Since we're - // recycling it, make sure it's tracked again: - if (!_PyObject_GC_IS_TRACKED(result)) { - _PyObject_GC_TRACK(result); + if (out_key != NULL) { + *out_key = Py_NewRef(key); + } + if (out_value != NULL) { + *out_value = Py_NewRef(value); + } + return 0; + +fail: + di->di_dict = NULL; + Py_DECREF(d); + return -1; +} + +#ifdef Py_GIL_DISABLED + +// Grabs the key and/or value from the provided locations and if successful +// returns them with an increased reference count. If either one is unsucessful +// nothing is incref'd and returns -1. +static int +acquire_key_value(PyObject **key_loc, PyObject *value, PyObject **value_loc, + PyObject **out_key, PyObject **out_value) +{ + if (out_key) { + *out_key = _Py_TryXGetRef(key_loc); + if (*out_key == NULL) { + return -1; + } + } + + if (out_value) { + if (!_Py_TryIncref(value_loc, value)) { + if (out_key) { + Py_DECREF(*out_key); + } + return -1; + } + *out_value = value; + } + + return 0; +} + +static Py_ssize_t +load_values_used_size(PyDictValues *values) +{ + return (Py_ssize_t)_Py_atomic_load_uint8(&DICT_VALUES_USED_SIZE(values)); +} + +static int +dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, + PyObject **out_key, PyObject **out_value) +{ + dictiterobject *di = (dictiterobject *)self; + Py_ssize_t i; + PyDictKeysObject *k; + + assert (PyDict_Check(d)); + + if (di->di_used != _Py_atomic_load_ssize_relaxed(&d->ma_used)) { + PyErr_SetString(PyExc_RuntimeError, + "dictionary changed size during iteration"); + di->di_used = -1; /* Make this state sticky */ + return -1; + } + + ensure_shared_on_read(d); + + i = _Py_atomic_load_ssize_relaxed(&di->di_pos); + k = _Py_atomic_load_ptr_relaxed(&d->ma_keys); + assert(i >= 0); + if (_PyDict_HasSplitTable(d)) { + PyDictValues *values = _Py_atomic_load_ptr_relaxed(&d->ma_values); + if (values == NULL) { + goto concurrent_modification; + } + + Py_ssize_t used = load_values_used_size(values); + if (i >= used) { + goto fail; + } + + // We're racing against writes to the order from delete_index_from_values, but + // single threaded can suffer from concurrent modification to those as well and + // can have either duplicated or skipped attributes, so we strive to do no better + // here. + int index = get_index_from_order(d, i); + PyObject *value = _Py_atomic_load_ptr(&values->values[index]); + if (acquire_key_value(&DK_UNICODE_ENTRIES(k)[index].me_key, value, + &values->values[index], out_key, out_value) < 0) { + goto try_locked; } } else { - result = PyTuple_New(2); - if (result == NULL) - return NULL; - PyTuple_SET_ITEM(result, 0, Py_NewRef(key)); - PyTuple_SET_ITEM(result, 1, Py_NewRef(value)); + Py_ssize_t n = _Py_atomic_load_ssize_relaxed(&k->dk_nentries); + if (DK_IS_UNICODE(k)) { + PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(k)[i]; + PyObject *value; + while (i < n && + (value = _Py_atomic_load_ptr(&entry_ptr->me_value)) == NULL) { + entry_ptr++; + i++; + } + if (i >= n) + goto fail; + + if (acquire_key_value(&entry_ptr->me_key, value, + &entry_ptr->me_value, out_key, out_value) < 0) { + goto try_locked; + } + } + else { + PyDictKeyEntry *entry_ptr = &DK_ENTRIES(k)[i]; + PyObject *value; + while (i < n && + (value = _Py_atomic_load_ptr(&entry_ptr->me_value)) == NULL) { + entry_ptr++; + i++; + } + + if (i >= n) + goto fail; + + if (acquire_key_value(&entry_ptr->me_key, value, + &entry_ptr->me_value, out_key, out_value) < 0) { + goto try_locked; + } + } } - return result; + // We found an element (key), but did not expect it + Py_ssize_t len; + if ((len = _Py_atomic_load_ssize_relaxed(&di->len)) == 0) { + goto concurrent_modification; + } + + _Py_atomic_store_ssize_relaxed(&di->di_pos, i + 1); + _Py_atomic_store_ssize_relaxed(&di->len, len - 1); + return 0; + +concurrent_modification: + PyErr_SetString(PyExc_RuntimeError, + "dictionary keys changed during iteration"); fail: di->di_dict = NULL; Py_DECREF(d); - return NULL; + return -1; + + int res; +try_locked: + Py_BEGIN_CRITICAL_SECTION(d); + res = dictiter_iternextitem_lock_held(d, self, out_key, out_value); + Py_END_CRITICAL_SECTION(); + return res; +} + +#endif + +static bool +has_unique_reference(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + return (_Py_IsOwnedByCurrentThread(op) && + op->ob_ref_local == 1 && + _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared) == 0); +#else + return Py_REFCNT(op) == 1; +#endif +} + +static bool +acquire_iter_result(PyObject *result) +{ + if (has_unique_reference(result)) { + Py_INCREF(result); + return true; + } + return false; } static PyObject * @@ -4832,11 +5445,37 @@ dictiter_iternextitem(PyObject *self) if (d == NULL) return NULL; - PyObject *item; - Py_BEGIN_CRITICAL_SECTION(d); - item = dictiter_iternextitem_lock_held(d, self); - Py_END_CRITICAL_SECTION(); - return item; + PyObject *key, *value; +#ifdef Py_GIL_DISABLED + if (dictiter_iternext_threadsafe(d, self, &key, &value) == 0) { +#else + if (dictiter_iternextitem_lock_held(d, self, &key, &value) == 0) { + +#endif + PyObject *result = di->di_result; + if (acquire_iter_result(result)) { + PyObject *oldkey = PyTuple_GET_ITEM(result, 0); + PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, key); + PyTuple_SET_ITEM(result, 1, value); + Py_DECREF(oldkey); + Py_DECREF(oldvalue); + // bpo-42536: The GC may have untracked this result tuple. Since we're + // recycling it, make sure it's tracked again: + if (!_PyObject_GC_IS_TRACKED(result)) { + _PyObject_GC_TRACK(result); + } + } + else { + result = PyTuple_New(2); + if (result == NULL) + return NULL; + PyTuple_SET_ITEM(result, 0, key); + PyTuple_SET_ITEM(result, 1, value); + } + return result; + } + return NULL; } PyTypeObject PyDictIterItem_Type = { @@ -4876,7 +5515,7 @@ PyTypeObject PyDictIterItem_Type = { /* dictreviter */ static PyObject * -dictreviter_iter_PyDict_Next(PyDictObject *d, PyObject *self) +dictreviter_iter_lock_held(PyDictObject *d, PyObject *self) { dictiterobject *di = (dictiterobject *)self; @@ -4897,9 +5536,9 @@ dictreviter_iter_PyDict_Next(PyDictObject *d, PyObject *self) if (i < 0) { goto fail; } - if (d->ma_values) { + if (_PyDict_HasSplitTable(d)) { int index = get_index_from_order(d, i); - key = DK_UNICODE_ENTRIES(k)[index].me_key; + key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(k)[index].me_key); value = d->ma_values->values[index]; assert (value != NULL); } @@ -4983,7 +5622,7 @@ dictreviter_iternext(PyObject *self) PyObject *value; Py_BEGIN_CRITICAL_SECTION(d); - value = dictreviter_iter_PyDict_Next(d, self); + value = dictreviter_iter_lock_held(d, self); Py_END_CRITICAL_SECTION(); return value; @@ -5912,9 +6551,20 @@ _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(tp); assert(keys != NULL); +#ifdef Py_GIL_DISABLED + Py_ssize_t usable = _Py_atomic_load_ssize_relaxed(&keys->dk_usable); + if (usable > 1) { + LOCK_KEYS(keys); + if (keys->dk_usable > 1) { + _Py_atomic_store_ssize(&keys->dk_usable, keys->dk_usable - 1); + } + UNLOCK_KEYS(keys); + } +#else if (keys->dk_usable > 1) { keys->dk_usable--; } +#endif size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { @@ -5989,18 +6639,6 @@ _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) return make_dict_from_instance_attributes(interp, keys, values); } -static bool -has_unique_reference(PyObject *op) -{ -#ifdef Py_GIL_DISABLED - return (_Py_IsOwnedByCurrentThread(op) && - op->ob_ref_local == 1 && - _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared) == 0); -#else - return Py_REFCNT(op) == 1; -#endif -} - // Return true if the dict was dematerialized, false otherwise. bool _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) @@ -6022,6 +6660,8 @@ _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) { return false; } + ensure_shared_on_resize(dict); + assert(dict->ma_values); // We have an opportunity to do something *really* cool: dematerialize it! _PyDictKeys_DecRef(dict->ma_keys); @@ -6045,22 +6685,42 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); Py_ssize_t ix = DKIX_EMPTY; if (PyUnicode_CheckExact(name)) { - ix = insert_into_dictkeys(keys, name); - } - if (ix == DKIX_EMPTY) { + Py_hash_t hash = unicode_get_hash(name); + if (hash == -1) { + hash = PyUnicode_Type.tp_hash(name); + assert(hash != -1); + } + +#ifdef Py_GIL_DISABLED + // Try a thread-safe lookup to see if the index is already allocated + ix = unicodekeys_lookup_unicode_threadsafe(keys, name, hash); + if (ix == DKIX_EMPTY) { + // Lock keys and do insert + LOCK_KEYS(keys); + ix = insert_into_splitdictkeys(keys, name, hash); + UNLOCK_KEYS(keys); + } +#else + ix = insert_into_splitdictkeys(keys, name, hash); +#endif + #ifdef Py_STATS - if (PyUnicode_CheckExact(name)) { - if (shared_keys_usable_size(keys) == SHARED_KEYS_MAX_SIZE) { - OBJECT_STAT_INC(dict_materialized_too_big); + if (ix == DKIX_EMPTY) { + if (PyUnicode_CheckExact(name)) { + if (shared_keys_usable_size(keys) == SHARED_KEYS_MAX_SIZE) { + OBJECT_STAT_INC(dict_materialized_too_big); + } + else { + OBJECT_STAT_INC(dict_materialized_new_key); + } } else { - OBJECT_STAT_INC(dict_materialized_new_key); + OBJECT_STAT_INC(dict_materialized_str_subclass); } } - else { - OBJECT_STAT_INC(dict_materialized_str_subclass); - } #endif + } + if (ix == DKIX_EMPTY) { PyObject *dict = make_dict_from_instance_attributes( interp, keys, values); if (dict == NULL) { @@ -6185,7 +6845,7 @@ _PyObject_FreeInstanceAttributes(PyObject *self) for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { Py_XDECREF(values->values[i]); } - free_values(values); + free_values(values, IS_DICT_SHARED((PyDictObject*)self)); } int @@ -6226,7 +6886,7 @@ PyObject_ClearManagedDict(PyObject *obj) Py_CLEAR(values->values[i]); } dorv_ptr->dict = NULL; - free_values(values); + free_values(values, IS_DICT_SHARED((PyDictObject*)obj)); } else { PyObject *dict = dorv_ptr->dict; @@ -6336,7 +6996,7 @@ void _PyDictKeys_DecRef(PyDictKeysObject *keys) { PyInterpreterState *interp = _PyInterpreterState_GET(); - dictkeys_decref(interp, keys); + dictkeys_decref(interp, keys, false); } uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp, diff --git a/Objects/genobject.c b/Objects/genobject.c index a1b6db1b5889d3..8d1dbb72ba9ec2 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1635,6 +1635,13 @@ get_async_gen_freelist(void) struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); return &freelists->async_gens; } + +static struct _Py_async_gen_asend_freelist * +get_async_gen_asend_freelist(void) +{ + struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + return &freelists->async_gen_asends; +} #endif @@ -1659,25 +1666,27 @@ void _PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelist_state, int is_finalization) { #ifdef WITH_FREELISTS - struct _Py_async_gen_freelist *state = &freelist_state->async_gens; + struct _Py_async_gen_freelist *freelist = &freelist_state->async_gens; - while (state->value_numfree > 0) { + while (freelist->numfree > 0) { _PyAsyncGenWrappedValue *o; - o = state->value_freelist[--state->value_numfree]; + o = freelist->items[--freelist->numfree]; assert(_PyAsyncGenWrappedValue_CheckExact(o)); PyObject_GC_Del(o); } - while (state->asend_numfree > 0) { + struct _Py_async_gen_asend_freelist *asend_freelist = &freelist_state->async_gen_asends; + + while (asend_freelist->numfree > 0) { PyAsyncGenASend *o; - o = state->asend_freelist[--state->asend_numfree]; + o = asend_freelist->items[--asend_freelist->numfree]; assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type)); PyObject_GC_Del(o); } if (is_finalization) { - state->value_numfree = -1; - state->asend_numfree = -1; + freelist->numfree = -1; + asend_freelist->numfree = -1; } #endif } @@ -1726,11 +1735,11 @@ async_gen_asend_dealloc(PyAsyncGenASend *o) Py_CLEAR(o->ags_gen); Py_CLEAR(o->ags_sendval); #ifdef WITH_FREELISTS - struct _Py_async_gen_freelist *async_gen_freelist = get_async_gen_freelist(); - if (async_gen_freelist->asend_numfree >= 0 && async_gen_freelist->asend_numfree < _PyAsyncGen_MAXFREELIST) { + struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist(); + if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) { assert(PyAsyncGenASend_CheckExact(o)); _PyGC_CLEAR_FINALIZED((PyObject *)o); - async_gen_freelist->asend_freelist[async_gen_freelist->asend_numfree++] = o; + freelist->items[freelist->numfree++] = o; } else #endif @@ -1896,10 +1905,10 @@ async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) { PyAsyncGenASend *o; #ifdef WITH_FREELISTS - struct _Py_async_gen_freelist *async_gen_freelist = get_async_gen_freelist(); - if (async_gen_freelist->asend_numfree > 0) { - async_gen_freelist->asend_numfree--; - o = async_gen_freelist->asend_freelist[async_gen_freelist->asend_numfree]; + struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist(); + if (freelist->numfree > 0) { + freelist->numfree--; + o = freelist->items[freelist->numfree]; _Py_NewReference((PyObject *)o); } else @@ -1931,10 +1940,10 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o) _PyObject_GC_UNTRACK((PyObject *)o); Py_CLEAR(o->agw_val); #ifdef WITH_FREELISTS - struct _Py_async_gen_freelist *async_gen_freelist = get_async_gen_freelist(); - if (async_gen_freelist->value_numfree >= 0 && async_gen_freelist->value_numfree < _PyAsyncGen_MAXFREELIST) { + struct _Py_async_gen_freelist *freelist = get_async_gen_freelist(); + if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) { assert(_PyAsyncGenWrappedValue_CheckExact(o)); - async_gen_freelist->value_freelist[async_gen_freelist->value_numfree++] = o; + freelist->items[freelist->numfree++] = o; OBJECT_STAT_INC(to_freelist); } else @@ -2004,10 +2013,10 @@ _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val) assert(val); #ifdef WITH_FREELISTS - struct _Py_async_gen_freelist *async_gen_freelist = get_async_gen_freelist(); - if (async_gen_freelist->value_numfree > 0) { - async_gen_freelist->value_numfree--; - o = async_gen_freelist->value_freelist[async_gen_freelist->value_numfree]; + struct _Py_async_gen_freelist *freelist = get_async_gen_freelist(); + if (freelist->numfree > 0) { + freelist->numfree--; + o = freelist->items[freelist->numfree]; OBJECT_STAT_INC(from_freelist); assert(_PyAsyncGenWrappedValue_CheckExact(o)); _Py_NewReference((PyObject*)o); diff --git a/Objects/listobject.c b/Objects/listobject.c index eb466260318ec1..87effb1b3a65fa 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_interp.h" // PyInterpreterState.list #include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject #include "pycore_long.h" // _PyLong_DigitCount @@ -20,14 +21,6 @@ class list "PyListObject *" "&PyList_Type" _Py_DECLARE_STR(list_err, "list index out of range"); -#ifdef Py_GIL_DISABLED -# define LOAD_SSIZE(value) _Py_atomic_load_ssize_relaxed(&value) -# define STORE_SSIZE(value, new_value) _Py_atomic_store_ssize_relaxed(&value, new_value) -#else -# define LOAD_SSIZE(value) value -# define STORE_SSIZE(value, new_value) value = new_value -#endif - #ifdef WITH_FREELISTS static struct _Py_list_freelist * get_list_freelist(void) @@ -509,7 +502,16 @@ list_item(PyObject *aa, Py_ssize_t i) PyErr_SetObject(PyExc_IndexError, &_Py_STR(list_err)); return NULL; } - return Py_NewRef(a->ob_item[i]); + PyObject *item; + Py_BEGIN_CRITICAL_SECTION(a); +#ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread((PyObject *)a) && !_PyObject_GC_IS_SHARED(a)) { + _PyObject_GC_SET_SHARED(a); + } +#endif + item = Py_NewRef(a->ob_item[i]); + Py_END_CRITICAL_SECTION(); + return item; } static PyObject * @@ -563,20 +565,12 @@ PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) } static PyObject * -list_concat(PyObject *aa, PyObject *bb) +list_concat_lock_held(PyListObject *a, PyListObject *b) { - PyListObject *a = (PyListObject *)aa; Py_ssize_t size; Py_ssize_t i; PyObject **src, **dest; PyListObject *np; - if (!PyList_Check(bb)) { - PyErr_Format(PyExc_TypeError, - "can only concatenate list (not \"%.200s\") to list", - Py_TYPE(bb)->tp_name); - return NULL; - } -#define b ((PyListObject *)bb) assert((size_t)Py_SIZE(a) + (size_t)Py_SIZE(b) < PY_SSIZE_T_MAX); size = Py_SIZE(a) + Py_SIZE(b); if (size == 0) { @@ -590,23 +584,39 @@ list_concat(PyObject *aa, PyObject *bb) dest = np->ob_item; for (i = 0; i < Py_SIZE(a); i++) { PyObject *v = src[i]; - dest[i] = Py_NewRef(v); + FT_ATOMIC_STORE_PTR_RELAXED(dest[i], Py_NewRef(v)); } src = b->ob_item; dest = np->ob_item + Py_SIZE(a); for (i = 0; i < Py_SIZE(b); i++) { PyObject *v = src[i]; - dest[i] = Py_NewRef(v); + FT_ATOMIC_STORE_PTR_RELAXED(dest[i], Py_NewRef(v)); } Py_SET_SIZE(np, size); return (PyObject *)np; -#undef b } static PyObject * -list_repeat(PyObject *aa, Py_ssize_t n) +list_concat(PyObject *aa, PyObject *bb) { + if (!PyList_Check(bb)) { + PyErr_Format(PyExc_TypeError, + "can only concatenate list (not \"%.200s\") to list", + Py_TYPE(bb)->tp_name); + return NULL; + } PyListObject *a = (PyListObject *)aa; + PyListObject *b = (PyListObject *)bb; + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION2(a, b); + ret = list_concat_lock_held(a, b); + Py_END_CRITICAL_SECTION2(); + return ret; +} + +static PyObject * +list_repeat_lock_held(PyListObject *a, Py_ssize_t n) +{ const Py_ssize_t input_size = Py_SIZE(a); if (input_size == 0 || n <= 0) return PyList_New(0); @@ -626,7 +636,7 @@ list_repeat(PyObject *aa, Py_ssize_t n) _Py_RefcntAdd(elem, n); PyObject **dest_end = dest + output_size; while (dest < dest_end) { - *dest++ = elem; + FT_ATOMIC_STORE_PTR_RELAXED(*dest++, elem); } } else { @@ -634,7 +644,7 @@ list_repeat(PyObject *aa, Py_ssize_t n) PyObject **src_end = src + input_size; while (src < src_end) { _Py_RefcntAdd(*src, n); - *dest++ = *src++; + FT_ATOMIC_STORE_PTR_RELAXED(*dest++, *src++); } _Py_memory_repeat((char *)np->ob_item, sizeof(PyObject *)*output_size, @@ -645,8 +655,19 @@ list_repeat(PyObject *aa, Py_ssize_t n) return (PyObject *) np; } +static PyObject * +list_repeat(PyObject *aa, Py_ssize_t n) +{ + PyObject *ret; + PyListObject *a = (PyListObject *)aa; + Py_BEGIN_CRITICAL_SECTION(a); + ret = list_repeat_lock_held(a, n); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void -list_clear(PyListObject *a) +list_clear_impl(PyListObject *a, bool is_resize) { PyObject **items = a->ob_item; if (items == NULL) { @@ -657,21 +678,36 @@ list_clear(PyListObject *a) this list, we make it empty first. */ Py_ssize_t i = Py_SIZE(a); Py_SET_SIZE(a, 0); - a->ob_item = NULL; + FT_ATOMIC_STORE_PTR_RELEASE(a->ob_item, NULL); a->allocated = 0; while (--i >= 0) { Py_XDECREF(items[i]); } - PyMem_Free(items); - +#ifdef Py_GIL_DISABLED + bool use_qsbr = is_resize && _PyObject_GC_IS_SHARED(a); +#else + bool use_qsbr = false; +#endif + if (use_qsbr) { + _PyMem_FreeDelayed(items); + } + else { + PyMem_Free(items); + } // Note that there is no guarantee that the list is actually empty // at this point, because XDECREF may have populated it indirectly again! } +static void +list_clear(PyListObject *a) +{ + list_clear_impl(a, true); +} + static int list_clear_slot(PyObject *self) { - list_clear((PyListObject *)self); + list_clear_impl((PyListObject *)self, false); return 0; } @@ -798,9 +834,8 @@ PyList_SetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) } static PyObject * -list_inplace_repeat(PyObject *_self, Py_ssize_t n) +list_inplace_repeat_lock_held(PyListObject *self, Py_ssize_t n) { - PyListObject *self = (PyListObject *)_self; Py_ssize_t input_size = PyList_GET_SIZE(self); if (input_size == 0 || n == 1) { return Py_NewRef(self); @@ -829,21 +864,51 @@ list_inplace_repeat(PyObject *_self, Py_ssize_t n) return Py_NewRef(self); } +static PyObject * +list_inplace_repeat(PyObject *_self, Py_ssize_t n) +{ + PyObject *ret; + PyListObject *self = (PyListObject *) _self; + Py_BEGIN_CRITICAL_SECTION(self); + ret = list_inplace_repeat_lock_held(self, n); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int -list_ass_item(PyObject *aa, Py_ssize_t i, PyObject *v) +list_ass_item_lock_held(PyListObject *a, Py_ssize_t i, PyObject *v) { - PyListObject *a = (PyListObject *)aa; if (!valid_index(i, Py_SIZE(a))) { PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); return -1; } - if (v == NULL) - return list_ass_slice(a, i, i+1, v); - Py_SETREF(a->ob_item[i], Py_NewRef(v)); + PyObject *tmp = a->ob_item[i]; + if (v == NULL) { + Py_ssize_t size = Py_SIZE(a); + for (Py_ssize_t idx = i; idx < size - 1; idx++) { + FT_ATOMIC_STORE_PTR_RELAXED(a->ob_item[idx], a->ob_item[idx + 1]); + } + Py_SET_SIZE(a, size - 1); + } + else { + FT_ATOMIC_STORE_PTR_RELEASE(a->ob_item[i], Py_NewRef(v)); + } + Py_DECREF(tmp); return 0; } +static int +list_ass_item(PyObject *aa, Py_ssize_t i, PyObject *v) +{ + int ret; + PyListObject *a = (PyListObject *)aa; + Py_BEGIN_CRITICAL_SECTION(a); + ret = list_ass_item_lock_held(a, i, v); + Py_END_CRITICAL_SECTION(); + return ret; +} + /*[clinic input] @critical_section list.insert @@ -2979,7 +3044,7 @@ list___sizeof___impl(PyListObject *self) /*[clinic end generated code: output=3417541f95f9a53e input=b8030a5d5ce8a187]*/ { size_t res = _PyObject_SIZE(Py_TYPE(self)); - Py_ssize_t allocated = LOAD_SSIZE(self->allocated); + Py_ssize_t allocated = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->allocated); res += (size_t)allocated * sizeof(void*); return PyLong_FromSize_t(res); } @@ -3382,7 +3447,7 @@ static PyObject * listiter_next(PyObject *self) { _PyListIterObject *it = (_PyListIterObject *)self; - Py_ssize_t index = LOAD_SSIZE(it->it_index); + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); if (index < 0) { return NULL; } @@ -3390,7 +3455,7 @@ listiter_next(PyObject *self) PyObject *item = list_get_item_ref(it->it_seq, index); if (item == NULL) { // out-of-bounds - STORE_SSIZE(it->it_index, -1); + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, -1); #ifndef Py_GIL_DISABLED PyListObject *seq = it->it_seq; it->it_seq = NULL; @@ -3398,7 +3463,7 @@ listiter_next(PyObject *self) #endif return NULL; } - STORE_SSIZE(it->it_index, index + 1); + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, index + 1); return item; } @@ -3407,7 +3472,7 @@ listiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { assert(self != NULL); _PyListIterObject *it = (_PyListIterObject *)self; - Py_ssize_t index = LOAD_SSIZE(it->it_index); + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); if (index >= 0) { Py_ssize_t len = PyList_GET_SIZE(it->it_seq) - index; if (len >= 0) @@ -3537,19 +3602,19 @@ listreviter_next(PyObject *self) { listreviterobject *it = (listreviterobject *)self; assert(it != NULL); - PyListObject *seq = it->it_seq; - assert(PyList_Check(seq)); - - Py_ssize_t index = LOAD_SSIZE(it->it_index); + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); if (index < 0) { return NULL; } + + PyListObject *seq = it->it_seq; + assert(PyList_Check(seq)); PyObject *item = list_get_item_ref(seq, index); if (item != NULL) { - STORE_SSIZE(it->it_index, index - 1); + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, index - 1); return item; } - STORE_SSIZE(it->it_index, -1); + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, -1); #ifndef Py_GIL_DISABLED it->it_seq = NULL; Py_DECREF(seq); @@ -3561,7 +3626,7 @@ static PyObject * listreviter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { listreviterobject *it = (listreviterobject *)self; - Py_ssize_t index = LOAD_SSIZE(it->it_index); + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); Py_ssize_t len = index + 1; if (it->it_seq == NULL || PyList_GET_SIZE(it->it_seq) < len) len = 0; diff --git a/Objects/longobject.c b/Objects/longobject.c index fe782983334323..2d1c6ad788e281 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1183,7 +1183,7 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness) for (Py_ssize_t i = sizeof(cv.b); i > 0; --i) { *b++ = cv.b[i - 1]; } - for (Py_ssize_t i = 0; i < n - sizeof(cv.b); ++i) { + for (Py_ssize_t i = 0; i < n - (int)sizeof(cv.b); ++i) { *b++ = fill; } } diff --git a/Objects/mimalloc/alloc.c b/Objects/mimalloc/alloc.c index f96c6f0b37f873..e6286b54bedc14 100644 --- a/Objects/mimalloc/alloc.c +++ b/Objects/mimalloc/alloc.c @@ -26,6 +26,15 @@ terms of the MIT license. A copy of the license can be found in the file // Allocation // ------------------------------------------------------ +#if (MI_DEBUG>0) +static inline void mi_debug_fill(mi_page_t* page, mi_block_t* block, int c, size_t size) { + size_t offset = (size_t)page->debug_offset; + if (offset < size) { + memset((char*)block + offset, c, size - offset); + } +} +#endif + // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { @@ -65,7 +74,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN if (!zero && !mi_page_is_huge(page)) { - memset(block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page)); + mi_debug_fill(page, block, MI_DEBUG_UNINIT, mi_page_usable_block_size(page)); } #elif (MI_SECURE!=0) if (!zero) { block->next = 0; } // don't leak internal data @@ -426,7 +435,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading if (segment->kind != MI_SEGMENT_HUGE) { // not for huge segments as we just reset the content - memset(block, MI_DEBUG_FREED, mi_usable_size(block)); + mi_debug_fill(page, block, MI_DEBUG_FREED, mi_usable_size(block)); } #endif @@ -480,7 +489,7 @@ static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block mi_check_padding(page, block); #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN if (!mi_page_is_huge(page)) { // huge page content may be already decommitted - memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); + mi_debug_fill(page, block, MI_DEBUG_FREED, mi_page_block_size(page)); } #endif mi_block_set_next(page, block, page->local_free); @@ -575,7 +584,7 @@ void mi_free(void* p) mi_attr_noexcept mi_check_padding(page, block); mi_stat_free(page, block); #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN - memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); + mi_debug_fill(page, block, MI_DEBUG_FREED, mi_page_block_size(page)); #endif mi_track_free_size(p, mi_page_usable_size_of(page,block)); // faster then mi_usable_size as we already know the page and that p is unaligned mi_block_set_next(page, block, page->local_free); diff --git a/Objects/mimalloc/init.c b/Objects/mimalloc/init.c index 5897f0512f8ef9..cb0ef6642803cc 100644 --- a/Objects/mimalloc/init.c +++ b/Objects/mimalloc/init.c @@ -13,27 +13,7 @@ terms of the MIT license. A copy of the license can be found in the file // Empty page used to initialize the small free pages array -const mi_page_t _mi_page_empty = { - 0, false, false, false, 0, - 0, // capacity - 0, // reserved capacity - { 0 }, // flags - false, // is_zero - 0, // retire_expire - NULL, // free - 0, // used - 0, // xblock_size - NULL, // local_free - #if (MI_PADDING || MI_ENCODE_FREELIST) - { 0, 0 }, - #endif - MI_ATOMIC_VAR_INIT(0), // xthread_free - MI_ATOMIC_VAR_INIT(0), // xheap - NULL, NULL - #if MI_INTPTR_SIZE==8 - , { 0 } // padding - #endif -}; +const mi_page_t _mi_page_empty; #define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty) @@ -122,6 +102,7 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { MI_BIN_FULL, 0, // page retired min/max NULL, // next false, + 0, 0 }; diff --git a/Objects/mimalloc/page.c b/Objects/mimalloc/page.c index 8f0ce920156e04..63db893e49405c 100644 --- a/Objects/mimalloc/page.c +++ b/Objects/mimalloc/page.c @@ -661,6 +661,7 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi // set fields mi_page_set_heap(page, heap); page->tag = heap->tag; + page->debug_offset = heap->debug_offset; page->xblock_size = (block_size < MI_HUGE_BLOCK_SIZE ? (uint32_t)block_size : MI_HUGE_BLOCK_SIZE); // initialize before _mi_segment_page_start size_t page_size; const void* page_start = _mi_segment_page_start(segment, page, &page_size); diff --git a/Objects/object.c b/Objects/object.c index 23eab8288a41e8..df14fe0c6fbfec 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -299,13 +299,13 @@ _Py_DecRef(PyObject *o) #ifdef Py_GIL_DISABLED # ifdef Py_REF_DEBUG -static inline int -is_shared_refcnt_dead(Py_ssize_t shared) +static int +is_dead(PyObject *o) { # if SIZEOF_SIZE_T == 8 - return shared == (Py_ssize_t)0xDDDDDDDDDDDDDDDD; + return (uintptr_t)o->ob_type == 0xDDDDDDDDDDDDDDDD; # else - return shared == (Py_ssize_t)0xDDDDDDDD; + return (uintptr_t)o->ob_type == 0xDDDDDDDD; # endif } # endif @@ -335,8 +335,8 @@ _Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) } #ifdef Py_REF_DEBUG - if ((_Py_REF_IS_MERGED(new_shared) && new_shared < 0) || - is_shared_refcnt_dead(shared)) + if ((new_shared < 0 && _Py_REF_IS_MERGED(new_shared)) || + (should_queue && is_dead(o))) { _Py_NegativeRefcount(filename, lineno, o); } diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 6a12c3dca38b36..b2a2286ef22b66 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -948,6 +948,196 @@ _PyMem_Strdup(const char *str) return copy; } +/***********************************************/ +/* Delayed freeing support for Py_GIL_DISABLED */ +/***********************************************/ + +// So that sizeof(struct _mem_work_chunk) is 4096 bytes on 64-bit platforms. +#define WORK_ITEMS_PER_CHUNK 254 + +// A pointer to be freed once the QSBR read sequence reaches qsbr_goal. +struct _mem_work_item { + void *ptr; + uint64_t qsbr_goal; +}; + +// A fixed-size buffer of pointers to be freed +struct _mem_work_chunk { + // Linked list node of chunks in queue + struct llist_node node; + + Py_ssize_t rd_idx; // index of next item to read + Py_ssize_t wr_idx; // index of next item to write + struct _mem_work_item array[WORK_ITEMS_PER_CHUNK]; +}; + +void +_PyMem_FreeDelayed(void *ptr) +{ +#ifndef Py_GIL_DISABLED + PyMem_Free(ptr); +#else + if (_PyRuntime.stoptheworld.world_stopped) { + // Free immediately if the world is stopped, including during + // interpreter shutdown. + PyMem_Free(ptr); + return; + } + + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + struct llist_node *head = &tstate->mem_free_queue; + + struct _mem_work_chunk *buf = NULL; + if (!llist_empty(head)) { + // Try to re-use the last buffer + buf = llist_data(head->prev, struct _mem_work_chunk, node); + if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) { + // already full + buf = NULL; + } + } + + if (buf == NULL) { + buf = PyMem_Calloc(1, sizeof(*buf)); + if (buf != NULL) { + llist_insert_tail(head, &buf->node); + } + } + + if (buf == NULL) { + // failed to allocate a buffer, free immediately + _PyEval_StopTheWorld(tstate->base.interp); + PyMem_Free(ptr); + _PyEval_StartTheWorld(tstate->base.interp); + return; + } + + assert(buf != NULL && buf->wr_idx < WORK_ITEMS_PER_CHUNK); + uint64_t seq = _Py_qsbr_deferred_advance(tstate->qsbr); + buf->array[buf->wr_idx].ptr = ptr; + buf->array[buf->wr_idx].qsbr_goal = seq; + buf->wr_idx++; + + if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) { + _PyMem_ProcessDelayed((PyThreadState *)tstate); + } +#endif +} + +static struct _mem_work_chunk * +work_queue_first(struct llist_node *head) +{ + return llist_data(head->next, struct _mem_work_chunk, node); +} + +static void +process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, + bool keep_empty) +{ + while (!llist_empty(head)) { + struct _mem_work_chunk *buf = work_queue_first(head); + + while (buf->rd_idx < buf->wr_idx) { + struct _mem_work_item *item = &buf->array[buf->rd_idx]; + if (!_Py_qsbr_poll(qsbr, item->qsbr_goal)) { + return; + } + + PyMem_Free(item->ptr); + buf->rd_idx++; + } + + assert(buf->rd_idx == buf->wr_idx); + if (keep_empty && buf->node.next == head) { + // Keep the last buffer in the queue to reduce re-allocations + buf->rd_idx = buf->wr_idx = 0; + return; + } + + llist_remove(&buf->node); + PyMem_Free(buf); + } +} + +static void +process_interp_queue(struct _Py_mem_interp_free_queue *queue, + struct _qsbr_thread_state *qsbr) +{ + if (!_Py_atomic_load_int_relaxed(&queue->has_work)) { + return; + } + + // Try to acquire the lock, but don't block if it's already held. + if (_PyMutex_LockTimed(&queue->mutex, 0, 0) == PY_LOCK_ACQUIRED) { + process_queue(&queue->head, qsbr, false); + + int more_work = !llist_empty(&queue->head); + _Py_atomic_store_int_relaxed(&queue->has_work, more_work); + + PyMutex_Unlock(&queue->mutex); + } +} + +void +_PyMem_ProcessDelayed(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + + // Process thread-local work + process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true); + + // Process shared interpreter work + process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr); +} + +void +_PyMem_AbandonDelayed(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + struct llist_node *queue = &((_PyThreadStateImpl *)tstate)->mem_free_queue; + + if (llist_empty(queue)) { + return; + } + + // Check if the queue contains one empty buffer + struct _mem_work_chunk *buf = work_queue_first(queue); + if (buf->rd_idx == buf->wr_idx) { + llist_remove(&buf->node); + PyMem_Free(buf); + assert(llist_empty(queue)); + return; + } + + // Merge the thread's work queue into the interpreter's work queue. + PyMutex_Lock(&interp->mem_free_queue.mutex); + llist_concat(&interp->mem_free_queue.head, queue); + _Py_atomic_store_int_relaxed(&interp->mem_free_queue.has_work, 1); + PyMutex_Unlock(&interp->mem_free_queue.mutex); + + assert(llist_empty(queue)); // the thread's queue is now empty +} + +void +_PyMem_FiniDelayed(PyInterpreterState *interp) +{ + struct llist_node *head = &interp->mem_free_queue.head; + while (!llist_empty(head)) { + struct _mem_work_chunk *buf = work_queue_first(head); + + while (buf->rd_idx < buf->wr_idx) { + // Free the remaining items immediately. There should be no other + // threads accessing the memory at this point during shutdown. + struct _mem_work_item *item = &buf->array[buf->rd_idx]; + PyMem_Free(item->ptr); + buf->rd_idx++; + } + + llist_remove(&buf->node); + PyMem_Free(buf); + } +} /**************************/ /* the "object" allocator */ @@ -2269,6 +2459,33 @@ write_size_t(void *p, size_t n) } } +static void +fill_mem_debug(debug_alloc_api_t *api, void *data, int c, size_t nbytes, + bool is_alloc) +{ +#ifdef Py_GIL_DISABLED + if (api->api_id == 'o') { + // Don't overwrite the first few bytes of a PyObject allocation in the + // free-threaded build + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + size_t debug_offset; + if (is_alloc) { + debug_offset = tstate->mimalloc.current_object_heap->debug_offset; + } + else { + char *alloc = (char *)data - 2*SST; // start of the allocation + debug_offset = _mi_ptr_page(alloc)->debug_offset; + } + debug_offset -= 2*SST; // account for pymalloc extra bytes + if (debug_offset < nbytes) { + memset((char *)data + debug_offset, c, nbytes - debug_offset); + } + return; + } +#endif + memset(data, c, nbytes); +} + /* Let S = sizeof(size_t). The debug malloc asks for 4 * S extra bytes and fills them with useful stuff, here calling the underlying malloc's result p: @@ -2345,7 +2562,7 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) memset(p + SST + 1, PYMEM_FORBIDDENBYTE, SST-1); if (nbytes > 0 && !use_calloc) { - memset(data, PYMEM_CLEANBYTE, nbytes); + fill_mem_debug(api, data, PYMEM_CLEANBYTE, nbytes, true); } /* at tail, write pad (SST bytes) and serialno (SST bytes) */ @@ -2393,8 +2610,9 @@ _PyMem_DebugRawFree(void *ctx, void *p) _PyMem_DebugCheckAddress(__func__, api->api_id, p); nbytes = read_size_t(q); - nbytes += PYMEM_DEBUG_EXTRA_BYTES; - memset(q, PYMEM_DEADBYTE, nbytes); + nbytes += PYMEM_DEBUG_EXTRA_BYTES - 2*SST; + memset(q, PYMEM_DEADBYTE, 2*SST); + fill_mem_debug(api, p, PYMEM_DEADBYTE, nbytes, false); api->alloc.free(api->alloc.ctx, q); } @@ -2414,7 +2632,6 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) size_t total; /* 2 * SST + nbytes + 2 * SST */ size_t original_nbytes; #define ERASED_SIZE 64 - uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */ _PyMem_DebugCheckAddress(__func__, api->api_id, p); @@ -2431,9 +2648,11 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) #ifdef PYMEM_DEBUG_SERIALNO size_t block_serialno = read_size_t(tail + SST); #endif +#ifndef Py_GIL_DISABLED /* Mark the header, the trailer, ERASED_SIZE bytes at the begin and ERASED_SIZE bytes at the end as dead and save the copy of erased bytes. */ + uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */ if (original_nbytes <= sizeof(save)) { memcpy(save, data, original_nbytes); memset(data - 2 * SST, PYMEM_DEADBYTE, @@ -2446,6 +2665,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) memset(tail - ERASED_SIZE, PYMEM_DEADBYTE, ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST); } +#endif /* Resize and add decorations. */ r = (uint8_t *)api->alloc.realloc(api->alloc.ctx, head, total); @@ -2473,6 +2693,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) write_size_t(tail + SST, block_serialno); #endif +#ifndef Py_GIL_DISABLED /* Restore saved bytes. */ if (original_nbytes <= sizeof(save)) { memcpy(data, save, Py_MIN(nbytes, original_nbytes)); @@ -2485,6 +2706,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) Py_MIN(nbytes - i, ERASED_SIZE)); } } +#endif if (r == NULL) { return NULL; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 0118ee255ef017..181d0323284ebc 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1835,6 +1835,8 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) if (presize) { ((PyObject **)alloc)[0] = NULL; ((PyObject **)alloc)[1] = NULL; + } + if (PyType_IS_GC(type)) { _PyObject_GC_Link(obj); } memset(obj, '\0', size); @@ -4943,7 +4945,7 @@ update_cache_gil_disabled(struct type_cache_entry *entry, PyObject *name, #endif void -_PyTypes_AfterFork() +_PyTypes_AfterFork(void) { #ifdef Py_GIL_DISABLED struct type_cache *cache = get_type_cache(); @@ -6547,6 +6549,7 @@ reduce_newobj(PyObject *obj) } else { /* args == NULL */ + Py_DECREF(copyreg); Py_DECREF(kwargs); PyErr_BadInternalCall(); return NULL; diff --git a/PC/launcher2.c b/PC/launcher2.c index 90b0fdebd3bdfb..139aa61bbe5cc2 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -1962,6 +1962,7 @@ struct AppxSearchInfo { struct AppxSearchInfo APPX_SEARCH[] = { // Releases made through the Store + { L"PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0", L"3.13", 10 }, { L"PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0", L"3.12", 10 }, { L"PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0", L"3.11", 10 }, { L"PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0", L"3.10", 10 }, @@ -1971,6 +1972,7 @@ struct AppxSearchInfo APPX_SEARCH[] = { // Side-loadable releases. Note that the publisher ID changes whenever we // renew our code-signing certificate, so the newer ID has a higher // priority (lower sortKey) + { L"PythonSoftwareFoundation.Python.3.13_3847v3x7pw1km", L"3.13", 11 }, { L"PythonSoftwareFoundation.Python.3.12_3847v3x7pw1km", L"3.12", 11 }, { L"PythonSoftwareFoundation.Python.3.11_3847v3x7pw1km", L"3.11", 11 }, { L"PythonSoftwareFoundation.Python.3.11_hd69rhyc2wevp", L"3.11", 12 }, @@ -2052,7 +2054,8 @@ struct StoreSearchInfo { struct StoreSearchInfo STORE_SEARCH[] = { - { L"3", /* 3.11 */ L"9NRWMJP3717K" }, + { L"3", /* 3.12 */ L"9NCVDN91XZQP" }, + { L"3.13", L"9PNRBTZXMB4Z" }, { L"3.12", L"9NCVDN91XZQP" }, { L"3.11", L"9NRWMJP3717K" }, { L"3.10", L"9PJPW5LDXLZ5" }, diff --git a/PC/pyconfig.h.in b/PC/pyconfig.h.in index 8bbf877a5bb5ed..d72d6282c2806f 100644 --- a/PC/pyconfig.h.in +++ b/PC/pyconfig.h.in @@ -95,7 +95,12 @@ WIN32 is still required for the locale module. #endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ /* Define to 1 if you want to disable the GIL */ -#undef Py_GIL_DISABLED +/* Uncomment the definition for free-threaded builds, or define it manually + * when compiling extension modules. Note that we test with #ifdef, so + * defining as 0 will still disable the GIL. */ +#ifndef Py_GIL_DISABLED +/* #define Py_GIL_DISABLED 1 */ +#endif /* Compiler specific defines */ diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 49f529ebbc2f9b..3a8a417a6bf47a 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -235,6 +235,7 @@ + @@ -253,6 +254,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 5b1bd7552b4cd9..5b34440af9322b 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -310,6 +310,9 @@ Source Files + + Python + Source Files @@ -376,6 +379,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index d3f62c93869003..af85f6d362466e 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -29,6 +29,9 @@ :begin_search @set PYTHON= +@rem If PYTHON_FOR_BUILD is set, use that +@if NOT "%PYTHON_FOR_BUILD%"=="" @(set PYTHON="%PYTHON_FOR_BUILD%") && (set _Py_Python_Source=found as PYTHON_FOR_BUILD) && goto :found + @rem If there is an active virtual env, use that one @if NOT "%VIRTUAL_ENV%"=="" (set PYTHON="%VIRTUAL_ENV%\Scripts\python.exe") & (set _Py_Python_Source=found in virtual env) & goto :found @@ -42,7 +45,9 @@ @if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 9)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found @rem If py.exe finds a recent enough version, use that one -@for %%p in (3.11 3.10 3.9) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@rem It is fine to add new versions to this list when they have released, +@rem but we do not use prerelease builds here. +@for %%p in (3.12 3.11 3.10 3.9) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index abfafbb2a32f45..88a4a7c9564309 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -276,6 +276,7 @@ + @@ -600,6 +601,7 @@ + @@ -614,6 +616,7 @@ + @@ -675,7 +678,7 @@ $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) - $(PyConfigHText.Replace('#undef Py_GIL_DISABLED', '#define Py_GIL_DISABLED 1')) + $(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1')) Include\internal + + Include\internal + Include\internal @@ -1379,6 +1382,9 @@ Python + + Python + Python @@ -1421,6 +1427,9 @@ Python + + Python + Python diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 387565515fa0b0..a7c2a68928ca84 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -226,12 +226,18 @@ directory. This script extracts all the external sub-projects from and https://github.com/python/cpython-bin-deps via a Python script called "get_external.py", located in this directory. -If Python 3.6 or later is not available via the "py.exe" launcher, the -path or command to use for Python can be provided in the PYTHON_FOR_BUILD -environment variable, or get_externals.bat will download the latest -version of NuGet and use it to download the latest "pythonx86" package -for use with get_external.py. Everything downloaded by these scripts is -stored in ..\externals (relative to this directory). +Everything downloaded by these scripts is stored in ..\externals +(relative to this directory), or the path specified by the EXTERNALS_DIR +environment variable. + +The path or command to use for Python can be provided with the +PYTHON_FOR_BUILD environment variable. If this is not set, an active +virtual environment will be used. If none is active, and HOST_PYTHON is +set to a recent enough version or "py.exe" is able to find a recent +enough version, those will be used. If all else fails, a copy of Python +will be downloaded from NuGet and extracted to the externals directory. +This will then be used for later builds (see PCbuild/find_python.bat +for the full logic). It is also possible to download sources from each project's homepage, though you may have to change folder names or pass the names to MSBuild diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index a90620d6ca8b7d..f36387231a0d3a 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -31,11 +31,13 @@ <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/> <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> + <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> + <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> - @@ -79,7 +81,31 @@ - + + + + + + + + + + + + + + - + diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 293f99ae135faa..ac530a5206271f 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -38,18 +38,18 @@ set regrtestargs=--fast-ci set exe= :CheckOpts -if "%1"=="-O" (set dashO=-O) & shift & goto CheckOpts -if "%1"=="-q" (set qmode=yes) & shift & goto CheckOpts -if "%1"=="-d" (set suffix=_d) & shift & goto CheckOpts +if "%~1"=="-O" (set dashO=-O) & shift & goto CheckOpts +if "%~1"=="-q" (set qmode=yes) & shift & goto CheckOpts +if "%~1"=="-d" (set suffix=_d) & shift & goto CheckOpts rem HACK: Need some way to infer the version number in this script -if "%1"=="--disable-gil" (set pyname=python3.13t) & shift & goto CheckOpts -if "%1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts -if "%1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts -if "%1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts -if "%1"=="-arm64" (set prefix=%pcbuild%arm64) & shift & goto CheckOpts -if "%1"=="-arm32" (set prefix=%pcbuild%arm32) & shift & goto CheckOpts -if "%1"=="-p" (call :SetPlatform %~2) & shift & shift & goto CheckOpts -if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts +if "%~1"=="--disable-gil" (set pyname=python3.13t) & shift & goto CheckOpts +if "%~1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts +if "%~1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts +if "%~1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts +if "%~1"=="-arm64" (set prefix=%pcbuild%arm64) & shift & goto CheckOpts +if "%~1"=="-arm32" (set prefix=%pcbuild%arm32) & shift & goto CheckOpts +if "%~1"=="-p" (call :SetPlatform %~2) & shift & shift & goto CheckOpts +if NOT "%~1"=="" (set regrtestargs=%regrtestargs% %~1) & shift & goto CheckOpts if not defined prefix set prefix=%pcbuild%amd64 set exe=%prefix%\%pyname%%suffix%.exe diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index ce92672bf00776..865fd76acf697d 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -15,6 +15,13 @@ MAX_COL = 80 AUTOGEN_MESSAGE = "// File automatically generated by {}.\n\n" +builtin_type_to_c_type = { + "identifier": "PyUnicode_Type", + "string": "PyUnicode_Type", + "int": "PyLong_Type", + "constant": "PyBaseObject_Type", +} + def get_c_type(name): """Return a string for the C name of the type. @@ -764,6 +771,67 @@ def visitConstructor(self, cons, name): self.emit("};",0) +class AnnotationsVisitor(PickleVisitor): + def visitModule(self, mod): + self.file.write(textwrap.dedent(''' + static int + add_ast_annotations(struct ast_state *state) + { + bool cond; + ''')) + for dfn in mod.dfns: + self.visit(dfn) + self.file.write(textwrap.dedent(''' + return 1; + } + ''')) + + def visitProduct(self, prod, name): + self.emit_annotations(name, prod.fields) + + def visitSum(self, sum, name): + for t in sum.types: + self.visitConstructor(t, name) + + def visitConstructor(self, cons, name): + self.emit_annotations(cons.name, cons.fields) + + def emit_annotations(self, name, fields): + self.emit(f"PyObject *{name}_annotations = PyDict_New();", 1) + self.emit(f"if (!{name}_annotations) return 0;", 1) + for field in fields: + self.emit("{", 1) + if field.type in builtin_type_to_c_type: + self.emit(f"PyObject *type = (PyObject *)&{builtin_type_to_c_type[field.type]};", 2) + else: + self.emit(f"PyObject *type = state->{field.type}_type;", 2) + if field.opt: + self.emit("type = _Py_union_type_or(type, Py_None);", 2) + self.emit("cond = type != NULL;", 2) + self.emit_annotations_error(name, 2) + elif field.seq: + self.emit("type = Py_GenericAlias((PyObject *)&PyList_Type, type);", 2) + self.emit("cond = type != NULL;", 2) + self.emit_annotations_error(name, 2) + else: + self.emit("Py_INCREF(type);", 2) + self.emit(f"cond = PyDict_SetItemString({name}_annotations, \"{field.name}\", type) == 0;", 2) + self.emit("Py_DECREF(type);", 2) + self.emit_annotations_error(name, 2) + self.emit("}", 1) + self.emit(f'cond = PyObject_SetAttrString(state->{name}_type, "_field_types", {name}_annotations) == 0;', 1) + self.emit_annotations_error(name, 1) + self.emit(f'cond = PyObject_SetAttrString(state->{name}_type, "__annotations__", {name}_annotations) == 0;', 1) + self.emit_annotations_error(name, 1) + self.emit(f"Py_DECREF({name}_annotations);", 1) + + def emit_annotations_error(self, name, depth): + self.emit("if (!cond) {", depth) + self.emit(f"Py_DECREF({name}_annotations);", depth + 1) + self.emit("return 0;", depth + 1) + self.emit("}", depth) + + class PyTypesVisitor(PickleVisitor): def visitModule(self, mod): @@ -812,7 +880,7 @@ def visitModule(self, mod): Py_ssize_t i, numfields = 0; int res = -1; - PyObject *key, *value, *fields; + PyObject *key, *value, *fields, *remaining_fields = NULL; if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { goto cleanup; } @@ -821,6 +889,13 @@ def visitModule(self, mod): if (numfields == -1) { goto cleanup; } + remaining_fields = PySet_New(fields); + } + else { + remaining_fields = PySet_New(NULL); + } + if (remaining_fields == NULL) { + goto cleanup; } res = 0; /* if no error occurs, this stays 0 to the end */ @@ -840,6 +915,11 @@ def visitModule(self, mod): goto cleanup; } res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); + if (PySet_Discard(remaining_fields, name) < 0) { + res = -1; + Py_DECREF(name); + goto cleanup; + } Py_DECREF(name); if (res < 0) { goto cleanup; @@ -852,13 +932,14 @@ def visitModule(self, mod): if (contains == -1) { res = -1; goto cleanup; - } else if (contains == 1) { - Py_ssize_t p = PySequence_Index(fields, key); + } + else if (contains == 1) { + int p = PySet_Discard(remaining_fields, key); if (p == -1) { res = -1; goto cleanup; } - if (p < PyTuple_GET_SIZE(args)) { + if (p == 0) { PyErr_Format(PyExc_TypeError, "%.400s got multiple values for argument '%U'", Py_TYPE(self)->tp_name, key); @@ -866,15 +947,91 @@ def visitModule(self, mod): goto cleanup; } } + else if ( + PyUnicode_CompareWithASCIIString(key, "lineno") != 0 && + PyUnicode_CompareWithASCIIString(key, "col_offset") != 0 && + PyUnicode_CompareWithASCIIString(key, "end_lineno") != 0 && + PyUnicode_CompareWithASCIIString(key, "end_col_offset") != 0 + ) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ got an unexpected keyword argument '%U'. " + "Support for arbitrary keyword arguments is deprecated " + "and will be removed in Python 3.15.", + Py_TYPE(self)->tp_name, key + ) < 0) { + res = -1; + goto cleanup; + } + } res = PyObject_SetAttr(self, key, value); if (res < 0) { goto cleanup; } } } + Py_ssize_t size = PySet_Size(remaining_fields); + PyObject *field_types = NULL, *remaining_list = NULL; + if (size > 0) { + if (!PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), &_Py_ID(_field_types), + &field_types)) { + res = -1; + goto cleanup; + } + remaining_list = PySequence_List(remaining_fields); + if (!remaining_list) { + goto set_remaining_cleanup; + } + for (Py_ssize_t i = 0; i < size; i++) { + PyObject *name = PyList_GET_ITEM(remaining_list, i); + PyObject *type = PyDict_GetItemWithError(field_types, name); + if (!type) { + if (!PyErr_Occurred()) { + PyErr_SetObject(PyExc_KeyError, name); + } + goto set_remaining_cleanup; + } + if (_PyUnion_Check(type)) { + // optional field + // do nothing, we'll have set a None default on the class + } + else if (Py_IS_TYPE(type, &Py_GenericAliasType)) { + // list field + PyObject *empty = PyList_New(0); + if (!empty) { + goto set_remaining_cleanup; + } + res = PyObject_SetAttr(self, name, empty); + Py_DECREF(empty); + if (res < 0) { + goto set_remaining_cleanup; + } + } + else { + // simple field (e.g., identifier) + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ missing 1 required positional argument: '%U'. " + "This will become an error in Python 3.15.", + Py_TYPE(self)->tp_name, name + ) < 0) { + res = -1; + goto cleanup; + } + } + } + Py_DECREF(remaining_list); + Py_DECREF(field_types); + } cleanup: Py_XDECREF(fields); + Py_XDECREF(remaining_fields); return res; + set_remaining_cleanup: + Py_XDECREF(remaining_list); + Py_XDECREF(field_types); + res = -1; + goto cleanup; } /* Pickling support */ @@ -886,14 +1043,75 @@ def visitModule(self, mod): return NULL; } - PyObject *dict; + PyObject *dict = NULL, *fields = NULL, *remaining_fields = NULL, + *remaining_dict = NULL, *positional_args = NULL; if (PyObject_GetOptionalAttr(self, state->__dict__, &dict) < 0) { return NULL; } + PyObject *result = NULL; if (dict) { - return Py_BuildValue("O()N", Py_TYPE(self), dict); + // Serialize the fields as positional args if possible, because if we + // serialize them as a dict, during unpickling they are set only *after* + // the object is constructed, which will now trigger a DeprecationWarning + // if the AST type has required fields. + if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { + goto cleanup; + } + if (fields) { + Py_ssize_t numfields = PySequence_Size(fields); + if (numfields == -1) { + Py_DECREF(dict); + goto cleanup; + } + remaining_dict = PyDict_Copy(dict); + Py_DECREF(dict); + if (!remaining_dict) { + goto cleanup; + } + positional_args = PyList_New(0); + if (!positional_args) { + goto cleanup; + } + for (Py_ssize_t i = 0; i < numfields; i++) { + PyObject *name = PySequence_GetItem(fields, i); + if (!name) { + goto cleanup; + } + PyObject *value = PyDict_GetItemWithError(remaining_dict, name); + if (!value) { + if (PyErr_Occurred()) { + goto cleanup; + } + break; + } + if (PyList_Append(positional_args, value) < 0) { + goto cleanup; + } + if (PyDict_DelItem(remaining_dict, name) < 0) { + goto cleanup; + } + Py_DECREF(name); + } + PyObject *args_tuple = PyList_AsTuple(positional_args); + if (!args_tuple) { + goto cleanup; + } + result = Py_BuildValue("ONO", Py_TYPE(self), args_tuple, + remaining_dict); + } + else { + result = Py_BuildValue("O()N", Py_TYPE(self), dict); + } + } + else { + result = Py_BuildValue("O()", Py_TYPE(self)); } - return Py_BuildValue("O()", Py_TYPE(self)); +cleanup: + Py_XDECREF(fields); + Py_XDECREF(remaining_fields); + Py_XDECREF(remaining_dict); + Py_XDECREF(positional_args); + return result; } static PyMemberDef ast_type_members[] = { @@ -1117,6 +1335,9 @@ def visitModule(self, mod): for dfn in mod.dfns: self.visit(dfn) self.file.write(textwrap.dedent(''' + if (!add_ast_annotations(state)) { + return -1; + } return 0; } ''')) @@ -1534,6 +1755,8 @@ def generate_module_def(mod, metadata, f, internal_h): #include "pycore_lock.h" // _PyOnceFlag #include "pycore_interp.h" // _PyInterpreterState.ast #include "pycore_pystate.h" // _PyInterpreterState_GET() + #include "pycore_unionobject.h" // _Py_union_type_or + #include "structmember.h" #include struct validator { @@ -1651,6 +1874,7 @@ def write_source(mod, metadata, f, internal_h_file): v = ChainOfVisitors( SequenceConstructorVisitor(f), PyTypesDeclareVisitor(f), + AnnotationsVisitor(f), PyTypesVisitor(f), Obj2ModPrototypeVisitor(f), FunctionVisitor(f), diff --git a/Parser/parser.c b/Parser/parser.c index 779b18e9650e9f..f1170c26197452 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -6603,7 +6603,7 @@ with_stmt_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = CHECK_VERSION ( stmt_ty , 9 , "Parenthesized context managers are" , _PyAST_With ( a , b , NEW_TYPE_COMMENT ( p , tc ) , EXTRA ) ); + _res = _PyAST_With ( a , b , NEW_TYPE_COMMENT ( p , tc ) , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index e15673d02dd3b0..e8f11a67e50fa0 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -369,20 +369,18 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, Py_ssize_t col_number = col_offset; Py_ssize_t end_col_number = end_col_offset; - if (p->tok->encoding != NULL) { - col_number = _PyPegen_byte_offset_to_character_offset(error_line, col_offset); - if (col_number < 0) { + col_number = _PyPegen_byte_offset_to_character_offset(error_line, col_offset); + if (col_number < 0) { + goto error; + } + + if (end_col_offset > 0) { + end_col_number = _PyPegen_byte_offset_to_character_offset(error_line, end_col_offset); + if (end_col_number < 0) { goto error; } - if (end_col_number > 0) { - Py_ssize_t end_col_offset = _PyPegen_byte_offset_to_character_offset(error_line, end_col_number); - if (end_col_offset < 0) { - goto error; - } else { - end_col_number = end_col_offset; - } - } } + tmp = Py_BuildValue("(OnnNnn)", p->tok->filename, lineno, col_number, error_line, end_lineno, end_col_number); if (!tmp) { goto error; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index d77e986ba067a3..46387493214829 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -7,6 +7,8 @@ #include "pycore_lock.h" // _PyOnceFlag #include "pycore_interp.h" // _PyInterpreterState.ast #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_unionobject.h" // _Py_union_type_or +#include "structmember.h" #include struct validator { @@ -816,6 +818,4170 @@ static const char * const TypeVarTuple_fields[]={ }; +static int +add_ast_annotations(struct ast_state *state) +{ + bool cond; + PyObject *Module_annotations = PyDict_New(); + if (!Module_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + cond = PyDict_SetItemString(Module_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + } + { + PyObject *type = state->type_ignore_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + cond = PyDict_SetItemString(Module_annotations, "type_ignores", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Module_type, "_field_types", + Module_annotations) == 0; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Module_type, "__annotations__", + Module_annotations) == 0; + if (!cond) { + Py_DECREF(Module_annotations); + return 0; + } + Py_DECREF(Module_annotations); + PyObject *Interactive_annotations = PyDict_New(); + if (!Interactive_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + cond = PyDict_SetItemString(Interactive_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Interactive_type, "_field_types", + Interactive_annotations) == 0; + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Interactive_type, "__annotations__", + Interactive_annotations) == 0; + if (!cond) { + Py_DECREF(Interactive_annotations); + return 0; + } + Py_DECREF(Interactive_annotations); + PyObject *Expression_annotations = PyDict_New(); + if (!Expression_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Expression_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Expression_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Expression_type, "_field_types", + Expression_annotations) == 0; + if (!cond) { + Py_DECREF(Expression_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Expression_type, "__annotations__", + Expression_annotations) == 0; + if (!cond) { + Py_DECREF(Expression_annotations); + return 0; + } + Py_DECREF(Expression_annotations); + PyObject *FunctionType_annotations = PyDict_New(); + if (!FunctionType_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionType_annotations, "argtypes", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(FunctionType_annotations, "returns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->FunctionType_type, "_field_types", + FunctionType_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FunctionType_type, "__annotations__", + FunctionType_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionType_annotations); + return 0; + } + Py_DECREF(FunctionType_annotations); + PyObject *FunctionDef_annotations = PyDict_New(); + if (!FunctionDef_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(FunctionDef_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->arguments_type; + Py_INCREF(type); + cond = PyDict_SetItemString(FunctionDef_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "decorator_list", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "returns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "type_comment", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(FunctionDef_annotations, "type_params", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->FunctionDef_type, "_field_types", + FunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FunctionDef_type, "__annotations__", + FunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(FunctionDef_annotations); + return 0; + } + Py_DECREF(FunctionDef_annotations); + PyObject *AsyncFunctionDef_annotations = PyDict_New(); + if (!AsyncFunctionDef_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "name", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->arguments_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "args", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "body", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, + "decorator_list", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, "returns", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, + "type_comment", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFunctionDef_annotations, + "type_params", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AsyncFunctionDef_type, "_field_types", + AsyncFunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AsyncFunctionDef_type, + "__annotations__", + AsyncFunctionDef_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFunctionDef_annotations); + return 0; + } + Py_DECREF(AsyncFunctionDef_annotations); + PyObject *ClassDef_annotations = PyDict_New(); + if (!ClassDef_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(ClassDef_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "bases", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->keyword_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "keywords", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "decorator_list", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyDict_SetItemString(ClassDef_annotations, "type_params", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ClassDef_type, "_field_types", + ClassDef_annotations) == 0; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ClassDef_type, "__annotations__", + ClassDef_annotations) == 0; + if (!cond) { + Py_DECREF(ClassDef_annotations); + return 0; + } + Py_DECREF(ClassDef_annotations); + PyObject *Return_annotations = PyDict_New(); + if (!Return_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + cond = PyDict_SetItemString(Return_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Return_type, "_field_types", + Return_annotations) == 0; + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Return_type, "__annotations__", + Return_annotations) == 0; + if (!cond) { + Py_DECREF(Return_annotations); + return 0; + } + Py_DECREF(Return_annotations); + PyObject *Delete_annotations = PyDict_New(); + if (!Delete_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + cond = PyDict_SetItemString(Delete_annotations, "targets", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Delete_type, "_field_types", + Delete_annotations) == 0; + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Delete_type, "__annotations__", + Delete_annotations) == 0; + if (!cond) { + Py_DECREF(Delete_annotations); + return 0; + } + Py_DECREF(Delete_annotations); + PyObject *Assign_annotations = PyDict_New(); + if (!Assign_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + cond = PyDict_SetItemString(Assign_annotations, "targets", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Assign_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + cond = PyDict_SetItemString(Assign_annotations, "type_comment", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Assign_type, "_field_types", + Assign_annotations) == 0; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Assign_type, "__annotations__", + Assign_annotations) == 0; + if (!cond) { + Py_DECREF(Assign_annotations); + return 0; + } + Py_DECREF(Assign_annotations); + PyObject *TypeAlias_annotations = PyDict_New(); + if (!TypeAlias_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeAlias_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + } + { + PyObject *type = state->type_param_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + cond = PyDict_SetItemString(TypeAlias_annotations, "type_params", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeAlias_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeAlias_type, "_field_types", + TypeAlias_annotations) == 0; + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeAlias_type, "__annotations__", + TypeAlias_annotations) == 0; + if (!cond) { + Py_DECREF(TypeAlias_annotations); + return 0; + } + Py_DECREF(TypeAlias_annotations); + PyObject *AugAssign_annotations = PyDict_New(); + if (!AugAssign_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AugAssign_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + } + { + PyObject *type = state->operator_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AugAssign_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AugAssign_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AugAssign_type, "_field_types", + AugAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AugAssign_type, "__annotations__", + AugAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AugAssign_annotations); + return 0; + } + Py_DECREF(AugAssign_annotations); + PyObject *AnnAssign_annotations = PyDict_New(); + if (!AnnAssign_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AnnAssign_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AnnAssign_annotations, "annotation", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + cond = PyDict_SetItemString(AnnAssign_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(AnnAssign_annotations, "simple", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AnnAssign_type, "_field_types", + AnnAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AnnAssign_type, "__annotations__", + AnnAssign_annotations) == 0; + if (!cond) { + Py_DECREF(AnnAssign_annotations); + return 0; + } + Py_DECREF(AnnAssign_annotations); + PyObject *For_annotations = PyDict_New(); + if (!For_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(For_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(For_annotations, "iter", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyDict_SetItemString(For_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyDict_SetItemString(For_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyDict_SetItemString(For_annotations, "type_comment", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->For_type, "_field_types", + For_annotations) == 0; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->For_type, "__annotations__", + For_annotations) == 0; + if (!cond) { + Py_DECREF(For_annotations); + return 0; + } + Py_DECREF(For_annotations); + PyObject *AsyncFor_annotations = PyDict_New(); + if (!AsyncFor_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFor_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(AsyncFor_annotations, "iter", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFor_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFor_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncFor_annotations, "type_comment", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AsyncFor_type, "_field_types", + AsyncFor_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AsyncFor_type, "__annotations__", + AsyncFor_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncFor_annotations); + return 0; + } + Py_DECREF(AsyncFor_annotations); + PyObject *While_annotations = PyDict_New(); + if (!While_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(While_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + cond = PyDict_SetItemString(While_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + cond = PyDict_SetItemString(While_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->While_type, "_field_types", + While_annotations) == 0; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->While_type, "__annotations__", + While_annotations) == 0; + if (!cond) { + Py_DECREF(While_annotations); + return 0; + } + Py_DECREF(While_annotations); + PyObject *If_annotations = PyDict_New(); + if (!If_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(If_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + cond = PyDict_SetItemString(If_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + cond = PyDict_SetItemString(If_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->If_type, "_field_types", + If_annotations) == 0; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->If_type, "__annotations__", + If_annotations) == 0; + if (!cond) { + Py_DECREF(If_annotations); + return 0; + } + Py_DECREF(If_annotations); + PyObject *With_annotations = PyDict_New(); + if (!With_annotations) return 0; + { + PyObject *type = state->withitem_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyDict_SetItemString(With_annotations, "items", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyDict_SetItemString(With_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyDict_SetItemString(With_annotations, "type_comment", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->With_type, "_field_types", + With_annotations) == 0; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->With_type, "__annotations__", + With_annotations) == 0; + if (!cond) { + Py_DECREF(With_annotations); + return 0; + } + Py_DECREF(With_annotations); + PyObject *AsyncWith_annotations = PyDict_New(); + if (!AsyncWith_annotations) return 0; + { + PyObject *type = state->withitem_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncWith_annotations, "items", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncWith_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyDict_SetItemString(AsyncWith_annotations, "type_comment", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->AsyncWith_type, "_field_types", + AsyncWith_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->AsyncWith_type, "__annotations__", + AsyncWith_annotations) == 0; + if (!cond) { + Py_DECREF(AsyncWith_annotations); + return 0; + } + Py_DECREF(AsyncWith_annotations); + PyObject *Match_annotations = PyDict_New(); + if (!Match_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Match_annotations, "subject", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + } + { + PyObject *type = state->match_case_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + cond = PyDict_SetItemString(Match_annotations, "cases", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Match_type, "_field_types", + Match_annotations) == 0; + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Match_type, "__annotations__", + Match_annotations) == 0; + if (!cond) { + Py_DECREF(Match_annotations); + return 0; + } + Py_DECREF(Match_annotations); + PyObject *Raise_annotations = PyDict_New(); + if (!Raise_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + cond = PyDict_SetItemString(Raise_annotations, "exc", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + cond = PyDict_SetItemString(Raise_annotations, "cause", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Raise_type, "_field_types", + Raise_annotations) == 0; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Raise_type, "__annotations__", + Raise_annotations) == 0; + if (!cond) { + Py_DECREF(Raise_annotations); + return 0; + } + Py_DECREF(Raise_annotations); + PyObject *Try_annotations = PyDict_New(); + if (!Try_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + { + PyObject *type = state->excepthandler_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "handlers", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyDict_SetItemString(Try_annotations, "finalbody", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Try_type, "_field_types", + Try_annotations) == 0; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Try_type, "__annotations__", + Try_annotations) == 0; + if (!cond) { + Py_DECREF(Try_annotations); + return 0; + } + Py_DECREF(Try_annotations); + PyObject *TryStar_annotations = PyDict_New(); + if (!TryStar_annotations) return 0; + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + { + PyObject *type = state->excepthandler_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "handlers", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyDict_SetItemString(TryStar_annotations, "finalbody", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TryStar_type, "_field_types", + TryStar_annotations) == 0; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TryStar_type, "__annotations__", + TryStar_annotations) == 0; + if (!cond) { + Py_DECREF(TryStar_annotations); + return 0; + } + Py_DECREF(TryStar_annotations); + PyObject *Assert_annotations = PyDict_New(); + if (!Assert_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Assert_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + cond = PyDict_SetItemString(Assert_annotations, "msg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Assert_type, "_field_types", + Assert_annotations) == 0; + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Assert_type, "__annotations__", + Assert_annotations) == 0; + if (!cond) { + Py_DECREF(Assert_annotations); + return 0; + } + Py_DECREF(Assert_annotations); + PyObject *Import_annotations = PyDict_New(); + if (!Import_annotations) return 0; + { + PyObject *type = state->alias_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + cond = PyDict_SetItemString(Import_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Import_type, "_field_types", + Import_annotations) == 0; + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Import_type, "__annotations__", + Import_annotations) == 0; + if (!cond) { + Py_DECREF(Import_annotations); + return 0; + } + Py_DECREF(Import_annotations); + PyObject *ImportFrom_annotations = PyDict_New(); + if (!ImportFrom_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyDict_SetItemString(ImportFrom_annotations, "module", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + } + { + PyObject *type = state->alias_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyDict_SetItemString(ImportFrom_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyDict_SetItemString(ImportFrom_annotations, "level", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ImportFrom_type, "_field_types", + ImportFrom_annotations) == 0; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ImportFrom_type, "__annotations__", + ImportFrom_annotations) == 0; + if (!cond) { + Py_DECREF(ImportFrom_annotations); + return 0; + } + Py_DECREF(ImportFrom_annotations); + PyObject *Global_annotations = PyDict_New(); + if (!Global_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + cond = PyDict_SetItemString(Global_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Global_type, "_field_types", + Global_annotations) == 0; + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Global_type, "__annotations__", + Global_annotations) == 0; + if (!cond) { + Py_DECREF(Global_annotations); + return 0; + } + Py_DECREF(Global_annotations); + PyObject *Nonlocal_annotations = PyDict_New(); + if (!Nonlocal_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + cond = PyDict_SetItemString(Nonlocal_annotations, "names", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Nonlocal_type, "_field_types", + Nonlocal_annotations) == 0; + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Nonlocal_type, "__annotations__", + Nonlocal_annotations) == 0; + if (!cond) { + Py_DECREF(Nonlocal_annotations); + return 0; + } + Py_DECREF(Nonlocal_annotations); + PyObject *Expr_annotations = PyDict_New(); + if (!Expr_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Expr_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Expr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Expr_type, "_field_types", + Expr_annotations) == 0; + if (!cond) { + Py_DECREF(Expr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Expr_type, "__annotations__", + Expr_annotations) == 0; + if (!cond) { + Py_DECREF(Expr_annotations); + return 0; + } + Py_DECREF(Expr_annotations); + PyObject *Pass_annotations = PyDict_New(); + if (!Pass_annotations) return 0; + cond = PyObject_SetAttrString(state->Pass_type, "_field_types", + Pass_annotations) == 0; + if (!cond) { + Py_DECREF(Pass_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Pass_type, "__annotations__", + Pass_annotations) == 0; + if (!cond) { + Py_DECREF(Pass_annotations); + return 0; + } + Py_DECREF(Pass_annotations); + PyObject *Break_annotations = PyDict_New(); + if (!Break_annotations) return 0; + cond = PyObject_SetAttrString(state->Break_type, "_field_types", + Break_annotations) == 0; + if (!cond) { + Py_DECREF(Break_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Break_type, "__annotations__", + Break_annotations) == 0; + if (!cond) { + Py_DECREF(Break_annotations); + return 0; + } + Py_DECREF(Break_annotations); + PyObject *Continue_annotations = PyDict_New(); + if (!Continue_annotations) return 0; + cond = PyObject_SetAttrString(state->Continue_type, "_field_types", + Continue_annotations) == 0; + if (!cond) { + Py_DECREF(Continue_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Continue_type, "__annotations__", + Continue_annotations) == 0; + if (!cond) { + Py_DECREF(Continue_annotations); + return 0; + } + Py_DECREF(Continue_annotations); + PyObject *BoolOp_annotations = PyDict_New(); + if (!BoolOp_annotations) return 0; + { + PyObject *type = state->boolop_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BoolOp_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + cond = PyDict_SetItemString(BoolOp_annotations, "values", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->BoolOp_type, "_field_types", + BoolOp_annotations) == 0; + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BoolOp_type, "__annotations__", + BoolOp_annotations) == 0; + if (!cond) { + Py_DECREF(BoolOp_annotations); + return 0; + } + Py_DECREF(BoolOp_annotations); + PyObject *NamedExpr_annotations = PyDict_New(); + if (!NamedExpr_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(NamedExpr_annotations, "target", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(NamedExpr_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->NamedExpr_type, "_field_types", + NamedExpr_annotations) == 0; + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->NamedExpr_type, "__annotations__", + NamedExpr_annotations) == 0; + if (!cond) { + Py_DECREF(NamedExpr_annotations); + return 0; + } + Py_DECREF(NamedExpr_annotations); + PyObject *BinOp_annotations = PyDict_New(); + if (!BinOp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BinOp_annotations, "left", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + } + { + PyObject *type = state->operator_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BinOp_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(BinOp_annotations, "right", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->BinOp_type, "_field_types", + BinOp_annotations) == 0; + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BinOp_type, "__annotations__", + BinOp_annotations) == 0; + if (!cond) { + Py_DECREF(BinOp_annotations); + return 0; + } + Py_DECREF(BinOp_annotations); + PyObject *UnaryOp_annotations = PyDict_New(); + if (!UnaryOp_annotations) return 0; + { + PyObject *type = state->unaryop_type; + Py_INCREF(type); + cond = PyDict_SetItemString(UnaryOp_annotations, "op", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(UnaryOp_annotations, "operand", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->UnaryOp_type, "_field_types", + UnaryOp_annotations) == 0; + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->UnaryOp_type, "__annotations__", + UnaryOp_annotations) == 0; + if (!cond) { + Py_DECREF(UnaryOp_annotations); + return 0; + } + Py_DECREF(UnaryOp_annotations); + PyObject *Lambda_annotations = PyDict_New(); + if (!Lambda_annotations) return 0; + { + PyObject *type = state->arguments_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Lambda_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Lambda_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Lambda_type, "_field_types", + Lambda_annotations) == 0; + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Lambda_type, "__annotations__", + Lambda_annotations) == 0; + if (!cond) { + Py_DECREF(Lambda_annotations); + return 0; + } + Py_DECREF(Lambda_annotations); + PyObject *IfExp_annotations = PyDict_New(); + if (!IfExp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(IfExp_annotations, "test", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(IfExp_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(IfExp_annotations, "orelse", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->IfExp_type, "_field_types", + IfExp_annotations) == 0; + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->IfExp_type, "__annotations__", + IfExp_annotations) == 0; + if (!cond) { + Py_DECREF(IfExp_annotations); + return 0; + } + Py_DECREF(IfExp_annotations); + PyObject *Dict_annotations = PyDict_New(); + if (!Dict_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + cond = PyDict_SetItemString(Dict_annotations, "keys", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + cond = PyDict_SetItemString(Dict_annotations, "values", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Dict_type, "_field_types", + Dict_annotations) == 0; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Dict_type, "__annotations__", + Dict_annotations) == 0; + if (!cond) { + Py_DECREF(Dict_annotations); + return 0; + } + Py_DECREF(Dict_annotations); + PyObject *Set_annotations = PyDict_New(); + if (!Set_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + cond = PyDict_SetItemString(Set_annotations, "elts", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Set_type, "_field_types", + Set_annotations) == 0; + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Set_type, "__annotations__", + Set_annotations) == 0; + if (!cond) { + Py_DECREF(Set_annotations); + return 0; + } + Py_DECREF(Set_annotations); + PyObject *ListComp_annotations = PyDict_New(); + if (!ListComp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(ListComp_annotations, "elt", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + cond = PyDict_SetItemString(ListComp_annotations, "generators", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ListComp_type, "_field_types", + ListComp_annotations) == 0; + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ListComp_type, "__annotations__", + ListComp_annotations) == 0; + if (!cond) { + Py_DECREF(ListComp_annotations); + return 0; + } + Py_DECREF(ListComp_annotations); + PyObject *SetComp_annotations = PyDict_New(); + if (!SetComp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(SetComp_annotations, "elt", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + cond = PyDict_SetItemString(SetComp_annotations, "generators", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->SetComp_type, "_field_types", + SetComp_annotations) == 0; + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->SetComp_type, "__annotations__", + SetComp_annotations) == 0; + if (!cond) { + Py_DECREF(SetComp_annotations); + return 0; + } + Py_DECREF(SetComp_annotations); + PyObject *DictComp_annotations = PyDict_New(); + if (!DictComp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(DictComp_annotations, "key", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(DictComp_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + cond = PyDict_SetItemString(DictComp_annotations, "generators", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->DictComp_type, "_field_types", + DictComp_annotations) == 0; + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->DictComp_type, "__annotations__", + DictComp_annotations) == 0; + if (!cond) { + Py_DECREF(DictComp_annotations); + return 0; + } + Py_DECREF(DictComp_annotations); + PyObject *GeneratorExp_annotations = PyDict_New(); + if (!GeneratorExp_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(GeneratorExp_annotations, "elt", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + } + { + PyObject *type = state->comprehension_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + cond = PyDict_SetItemString(GeneratorExp_annotations, "generators", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->GeneratorExp_type, "_field_types", + GeneratorExp_annotations) == 0; + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->GeneratorExp_type, "__annotations__", + GeneratorExp_annotations) == 0; + if (!cond) { + Py_DECREF(GeneratorExp_annotations); + return 0; + } + Py_DECREF(GeneratorExp_annotations); + PyObject *Await_annotations = PyDict_New(); + if (!Await_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Await_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Await_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Await_type, "_field_types", + Await_annotations) == 0; + if (!cond) { + Py_DECREF(Await_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Await_type, "__annotations__", + Await_annotations) == 0; + if (!cond) { + Py_DECREF(Await_annotations); + return 0; + } + Py_DECREF(Await_annotations); + PyObject *Yield_annotations = PyDict_New(); + if (!Yield_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + cond = PyDict_SetItemString(Yield_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Yield_type, "_field_types", + Yield_annotations) == 0; + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Yield_type, "__annotations__", + Yield_annotations) == 0; + if (!cond) { + Py_DECREF(Yield_annotations); + return 0; + } + Py_DECREF(Yield_annotations); + PyObject *YieldFrom_annotations = PyDict_New(); + if (!YieldFrom_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(YieldFrom_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(YieldFrom_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->YieldFrom_type, "_field_types", + YieldFrom_annotations) == 0; + if (!cond) { + Py_DECREF(YieldFrom_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->YieldFrom_type, "__annotations__", + YieldFrom_annotations) == 0; + if (!cond) { + Py_DECREF(YieldFrom_annotations); + return 0; + } + Py_DECREF(YieldFrom_annotations); + PyObject *Compare_annotations = PyDict_New(); + if (!Compare_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Compare_annotations, "left", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + } + { + PyObject *type = state->cmpop_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + cond = PyDict_SetItemString(Compare_annotations, "ops", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + cond = PyDict_SetItemString(Compare_annotations, "comparators", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Compare_type, "_field_types", + Compare_annotations) == 0; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Compare_type, "__annotations__", + Compare_annotations) == 0; + if (!cond) { + Py_DECREF(Compare_annotations); + return 0; + } + Py_DECREF(Compare_annotations); + PyObject *Call_annotations = PyDict_New(); + if (!Call_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Call_annotations, "func", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + cond = PyDict_SetItemString(Call_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + } + { + PyObject *type = state->keyword_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + cond = PyDict_SetItemString(Call_annotations, "keywords", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Call_type, "_field_types", + Call_annotations) == 0; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Call_type, "__annotations__", + Call_annotations) == 0; + if (!cond) { + Py_DECREF(Call_annotations); + return 0; + } + Py_DECREF(Call_annotations); + PyObject *FormattedValue_annotations = PyDict_New(); + if (!FormattedValue_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(FormattedValue_annotations, "value", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(FormattedValue_annotations, "conversion", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + cond = PyDict_SetItemString(FormattedValue_annotations, "format_spec", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->FormattedValue_type, "_field_types", + FormattedValue_annotations) == 0; + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FormattedValue_type, + "__annotations__", + FormattedValue_annotations) == 0; + if (!cond) { + Py_DECREF(FormattedValue_annotations); + return 0; + } + Py_DECREF(FormattedValue_annotations); + PyObject *JoinedStr_annotations = PyDict_New(); + if (!JoinedStr_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + cond = PyDict_SetItemString(JoinedStr_annotations, "values", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->JoinedStr_type, "_field_types", + JoinedStr_annotations) == 0; + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->JoinedStr_type, "__annotations__", + JoinedStr_annotations) == 0; + if (!cond) { + Py_DECREF(JoinedStr_annotations); + return 0; + } + Py_DECREF(JoinedStr_annotations); + PyObject *Constant_annotations = PyDict_New(); + if (!Constant_annotations) return 0; + { + PyObject *type = (PyObject *)&PyBaseObject_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(Constant_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + cond = PyDict_SetItemString(Constant_annotations, "kind", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Constant_type, "_field_types", + Constant_annotations) == 0; + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Constant_type, "__annotations__", + Constant_annotations) == 0; + if (!cond) { + Py_DECREF(Constant_annotations); + return 0; + } + Py_DECREF(Constant_annotations); + PyObject *Attribute_annotations = PyDict_New(); + if (!Attribute_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Attribute_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(Attribute_annotations, "attr", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Attribute_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Attribute_type, "_field_types", + Attribute_annotations) == 0; + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Attribute_type, "__annotations__", + Attribute_annotations) == 0; + if (!cond) { + Py_DECREF(Attribute_annotations); + return 0; + } + Py_DECREF(Attribute_annotations); + PyObject *Subscript_annotations = PyDict_New(); + if (!Subscript_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Subscript_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Subscript_annotations, "slice", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Subscript_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Subscript_type, "_field_types", + Subscript_annotations) == 0; + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Subscript_type, "__annotations__", + Subscript_annotations) == 0; + if (!cond) { + Py_DECREF(Subscript_annotations); + return 0; + } + Py_DECREF(Subscript_annotations); + PyObject *Starred_annotations = PyDict_New(); + if (!Starred_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Starred_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Starred_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Starred_type, "_field_types", + Starred_annotations) == 0; + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Starred_type, "__annotations__", + Starred_annotations) == 0; + if (!cond) { + Py_DECREF(Starred_annotations); + return 0; + } + Py_DECREF(Starred_annotations); + PyObject *Name_annotations = PyDict_New(); + if (!Name_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(Name_annotations, "id", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Name_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Name_type, "_field_types", + Name_annotations) == 0; + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Name_type, "__annotations__", + Name_annotations) == 0; + if (!cond) { + Py_DECREF(Name_annotations); + return 0; + } + Py_DECREF(Name_annotations); + PyObject *List_annotations = PyDict_New(); + if (!List_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + cond = PyDict_SetItemString(List_annotations, "elts", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(List_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->List_type, "_field_types", + List_annotations) == 0; + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->List_type, "__annotations__", + List_annotations) == 0; + if (!cond) { + Py_DECREF(List_annotations); + return 0; + } + Py_DECREF(List_annotations); + PyObject *Tuple_annotations = PyDict_New(); + if (!Tuple_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + cond = PyDict_SetItemString(Tuple_annotations, "elts", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + } + { + PyObject *type = state->expr_context_type; + Py_INCREF(type); + cond = PyDict_SetItemString(Tuple_annotations, "ctx", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Tuple_type, "_field_types", + Tuple_annotations) == 0; + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Tuple_type, "__annotations__", + Tuple_annotations) == 0; + if (!cond) { + Py_DECREF(Tuple_annotations); + return 0; + } + Py_DECREF(Tuple_annotations); + PyObject *Slice_annotations = PyDict_New(); + if (!Slice_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyDict_SetItemString(Slice_annotations, "lower", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyDict_SetItemString(Slice_annotations, "upper", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyDict_SetItemString(Slice_annotations, "step", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->Slice_type, "_field_types", + Slice_annotations) == 0; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Slice_type, "__annotations__", + Slice_annotations) == 0; + if (!cond) { + Py_DECREF(Slice_annotations); + return 0; + } + Py_DECREF(Slice_annotations); + PyObject *Load_annotations = PyDict_New(); + if (!Load_annotations) return 0; + cond = PyObject_SetAttrString(state->Load_type, "_field_types", + Load_annotations) == 0; + if (!cond) { + Py_DECREF(Load_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Load_type, "__annotations__", + Load_annotations) == 0; + if (!cond) { + Py_DECREF(Load_annotations); + return 0; + } + Py_DECREF(Load_annotations); + PyObject *Store_annotations = PyDict_New(); + if (!Store_annotations) return 0; + cond = PyObject_SetAttrString(state->Store_type, "_field_types", + Store_annotations) == 0; + if (!cond) { + Py_DECREF(Store_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Store_type, "__annotations__", + Store_annotations) == 0; + if (!cond) { + Py_DECREF(Store_annotations); + return 0; + } + Py_DECREF(Store_annotations); + PyObject *Del_annotations = PyDict_New(); + if (!Del_annotations) return 0; + cond = PyObject_SetAttrString(state->Del_type, "_field_types", + Del_annotations) == 0; + if (!cond) { + Py_DECREF(Del_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Del_type, "__annotations__", + Del_annotations) == 0; + if (!cond) { + Py_DECREF(Del_annotations); + return 0; + } + Py_DECREF(Del_annotations); + PyObject *And_annotations = PyDict_New(); + if (!And_annotations) return 0; + cond = PyObject_SetAttrString(state->And_type, "_field_types", + And_annotations) == 0; + if (!cond) { + Py_DECREF(And_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->And_type, "__annotations__", + And_annotations) == 0; + if (!cond) { + Py_DECREF(And_annotations); + return 0; + } + Py_DECREF(And_annotations); + PyObject *Or_annotations = PyDict_New(); + if (!Or_annotations) return 0; + cond = PyObject_SetAttrString(state->Or_type, "_field_types", + Or_annotations) == 0; + if (!cond) { + Py_DECREF(Or_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Or_type, "__annotations__", + Or_annotations) == 0; + if (!cond) { + Py_DECREF(Or_annotations); + return 0; + } + Py_DECREF(Or_annotations); + PyObject *Add_annotations = PyDict_New(); + if (!Add_annotations) return 0; + cond = PyObject_SetAttrString(state->Add_type, "_field_types", + Add_annotations) == 0; + if (!cond) { + Py_DECREF(Add_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Add_type, "__annotations__", + Add_annotations) == 0; + if (!cond) { + Py_DECREF(Add_annotations); + return 0; + } + Py_DECREF(Add_annotations); + PyObject *Sub_annotations = PyDict_New(); + if (!Sub_annotations) return 0; + cond = PyObject_SetAttrString(state->Sub_type, "_field_types", + Sub_annotations) == 0; + if (!cond) { + Py_DECREF(Sub_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Sub_type, "__annotations__", + Sub_annotations) == 0; + if (!cond) { + Py_DECREF(Sub_annotations); + return 0; + } + Py_DECREF(Sub_annotations); + PyObject *Mult_annotations = PyDict_New(); + if (!Mult_annotations) return 0; + cond = PyObject_SetAttrString(state->Mult_type, "_field_types", + Mult_annotations) == 0; + if (!cond) { + Py_DECREF(Mult_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Mult_type, "__annotations__", + Mult_annotations) == 0; + if (!cond) { + Py_DECREF(Mult_annotations); + return 0; + } + Py_DECREF(Mult_annotations); + PyObject *MatMult_annotations = PyDict_New(); + if (!MatMult_annotations) return 0; + cond = PyObject_SetAttrString(state->MatMult_type, "_field_types", + MatMult_annotations) == 0; + if (!cond) { + Py_DECREF(MatMult_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatMult_type, "__annotations__", + MatMult_annotations) == 0; + if (!cond) { + Py_DECREF(MatMult_annotations); + return 0; + } + Py_DECREF(MatMult_annotations); + PyObject *Div_annotations = PyDict_New(); + if (!Div_annotations) return 0; + cond = PyObject_SetAttrString(state->Div_type, "_field_types", + Div_annotations) == 0; + if (!cond) { + Py_DECREF(Div_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Div_type, "__annotations__", + Div_annotations) == 0; + if (!cond) { + Py_DECREF(Div_annotations); + return 0; + } + Py_DECREF(Div_annotations); + PyObject *Mod_annotations = PyDict_New(); + if (!Mod_annotations) return 0; + cond = PyObject_SetAttrString(state->Mod_type, "_field_types", + Mod_annotations) == 0; + if (!cond) { + Py_DECREF(Mod_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Mod_type, "__annotations__", + Mod_annotations) == 0; + if (!cond) { + Py_DECREF(Mod_annotations); + return 0; + } + Py_DECREF(Mod_annotations); + PyObject *Pow_annotations = PyDict_New(); + if (!Pow_annotations) return 0; + cond = PyObject_SetAttrString(state->Pow_type, "_field_types", + Pow_annotations) == 0; + if (!cond) { + Py_DECREF(Pow_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Pow_type, "__annotations__", + Pow_annotations) == 0; + if (!cond) { + Py_DECREF(Pow_annotations); + return 0; + } + Py_DECREF(Pow_annotations); + PyObject *LShift_annotations = PyDict_New(); + if (!LShift_annotations) return 0; + cond = PyObject_SetAttrString(state->LShift_type, "_field_types", + LShift_annotations) == 0; + if (!cond) { + Py_DECREF(LShift_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->LShift_type, "__annotations__", + LShift_annotations) == 0; + if (!cond) { + Py_DECREF(LShift_annotations); + return 0; + } + Py_DECREF(LShift_annotations); + PyObject *RShift_annotations = PyDict_New(); + if (!RShift_annotations) return 0; + cond = PyObject_SetAttrString(state->RShift_type, "_field_types", + RShift_annotations) == 0; + if (!cond) { + Py_DECREF(RShift_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->RShift_type, "__annotations__", + RShift_annotations) == 0; + if (!cond) { + Py_DECREF(RShift_annotations); + return 0; + } + Py_DECREF(RShift_annotations); + PyObject *BitOr_annotations = PyDict_New(); + if (!BitOr_annotations) return 0; + cond = PyObject_SetAttrString(state->BitOr_type, "_field_types", + BitOr_annotations) == 0; + if (!cond) { + Py_DECREF(BitOr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BitOr_type, "__annotations__", + BitOr_annotations) == 0; + if (!cond) { + Py_DECREF(BitOr_annotations); + return 0; + } + Py_DECREF(BitOr_annotations); + PyObject *BitXor_annotations = PyDict_New(); + if (!BitXor_annotations) return 0; + cond = PyObject_SetAttrString(state->BitXor_type, "_field_types", + BitXor_annotations) == 0; + if (!cond) { + Py_DECREF(BitXor_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BitXor_type, "__annotations__", + BitXor_annotations) == 0; + if (!cond) { + Py_DECREF(BitXor_annotations); + return 0; + } + Py_DECREF(BitXor_annotations); + PyObject *BitAnd_annotations = PyDict_New(); + if (!BitAnd_annotations) return 0; + cond = PyObject_SetAttrString(state->BitAnd_type, "_field_types", + BitAnd_annotations) == 0; + if (!cond) { + Py_DECREF(BitAnd_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->BitAnd_type, "__annotations__", + BitAnd_annotations) == 0; + if (!cond) { + Py_DECREF(BitAnd_annotations); + return 0; + } + Py_DECREF(BitAnd_annotations); + PyObject *FloorDiv_annotations = PyDict_New(); + if (!FloorDiv_annotations) return 0; + cond = PyObject_SetAttrString(state->FloorDiv_type, "_field_types", + FloorDiv_annotations) == 0; + if (!cond) { + Py_DECREF(FloorDiv_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->FloorDiv_type, "__annotations__", + FloorDiv_annotations) == 0; + if (!cond) { + Py_DECREF(FloorDiv_annotations); + return 0; + } + Py_DECREF(FloorDiv_annotations); + PyObject *Invert_annotations = PyDict_New(); + if (!Invert_annotations) return 0; + cond = PyObject_SetAttrString(state->Invert_type, "_field_types", + Invert_annotations) == 0; + if (!cond) { + Py_DECREF(Invert_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Invert_type, "__annotations__", + Invert_annotations) == 0; + if (!cond) { + Py_DECREF(Invert_annotations); + return 0; + } + Py_DECREF(Invert_annotations); + PyObject *Not_annotations = PyDict_New(); + if (!Not_annotations) return 0; + cond = PyObject_SetAttrString(state->Not_type, "_field_types", + Not_annotations) == 0; + if (!cond) { + Py_DECREF(Not_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Not_type, "__annotations__", + Not_annotations) == 0; + if (!cond) { + Py_DECREF(Not_annotations); + return 0; + } + Py_DECREF(Not_annotations); + PyObject *UAdd_annotations = PyDict_New(); + if (!UAdd_annotations) return 0; + cond = PyObject_SetAttrString(state->UAdd_type, "_field_types", + UAdd_annotations) == 0; + if (!cond) { + Py_DECREF(UAdd_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->UAdd_type, "__annotations__", + UAdd_annotations) == 0; + if (!cond) { + Py_DECREF(UAdd_annotations); + return 0; + } + Py_DECREF(UAdd_annotations); + PyObject *USub_annotations = PyDict_New(); + if (!USub_annotations) return 0; + cond = PyObject_SetAttrString(state->USub_type, "_field_types", + USub_annotations) == 0; + if (!cond) { + Py_DECREF(USub_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->USub_type, "__annotations__", + USub_annotations) == 0; + if (!cond) { + Py_DECREF(USub_annotations); + return 0; + } + Py_DECREF(USub_annotations); + PyObject *Eq_annotations = PyDict_New(); + if (!Eq_annotations) return 0; + cond = PyObject_SetAttrString(state->Eq_type, "_field_types", + Eq_annotations) == 0; + if (!cond) { + Py_DECREF(Eq_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Eq_type, "__annotations__", + Eq_annotations) == 0; + if (!cond) { + Py_DECREF(Eq_annotations); + return 0; + } + Py_DECREF(Eq_annotations); + PyObject *NotEq_annotations = PyDict_New(); + if (!NotEq_annotations) return 0; + cond = PyObject_SetAttrString(state->NotEq_type, "_field_types", + NotEq_annotations) == 0; + if (!cond) { + Py_DECREF(NotEq_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->NotEq_type, "__annotations__", + NotEq_annotations) == 0; + if (!cond) { + Py_DECREF(NotEq_annotations); + return 0; + } + Py_DECREF(NotEq_annotations); + PyObject *Lt_annotations = PyDict_New(); + if (!Lt_annotations) return 0; + cond = PyObject_SetAttrString(state->Lt_type, "_field_types", + Lt_annotations) == 0; + if (!cond) { + Py_DECREF(Lt_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Lt_type, "__annotations__", + Lt_annotations) == 0; + if (!cond) { + Py_DECREF(Lt_annotations); + return 0; + } + Py_DECREF(Lt_annotations); + PyObject *LtE_annotations = PyDict_New(); + if (!LtE_annotations) return 0; + cond = PyObject_SetAttrString(state->LtE_type, "_field_types", + LtE_annotations) == 0; + if (!cond) { + Py_DECREF(LtE_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->LtE_type, "__annotations__", + LtE_annotations) == 0; + if (!cond) { + Py_DECREF(LtE_annotations); + return 0; + } + Py_DECREF(LtE_annotations); + PyObject *Gt_annotations = PyDict_New(); + if (!Gt_annotations) return 0; + cond = PyObject_SetAttrString(state->Gt_type, "_field_types", + Gt_annotations) == 0; + if (!cond) { + Py_DECREF(Gt_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Gt_type, "__annotations__", + Gt_annotations) == 0; + if (!cond) { + Py_DECREF(Gt_annotations); + return 0; + } + Py_DECREF(Gt_annotations); + PyObject *GtE_annotations = PyDict_New(); + if (!GtE_annotations) return 0; + cond = PyObject_SetAttrString(state->GtE_type, "_field_types", + GtE_annotations) == 0; + if (!cond) { + Py_DECREF(GtE_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->GtE_type, "__annotations__", + GtE_annotations) == 0; + if (!cond) { + Py_DECREF(GtE_annotations); + return 0; + } + Py_DECREF(GtE_annotations); + PyObject *Is_annotations = PyDict_New(); + if (!Is_annotations) return 0; + cond = PyObject_SetAttrString(state->Is_type, "_field_types", + Is_annotations) == 0; + if (!cond) { + Py_DECREF(Is_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->Is_type, "__annotations__", + Is_annotations) == 0; + if (!cond) { + Py_DECREF(Is_annotations); + return 0; + } + Py_DECREF(Is_annotations); + PyObject *IsNot_annotations = PyDict_New(); + if (!IsNot_annotations) return 0; + cond = PyObject_SetAttrString(state->IsNot_type, "_field_types", + IsNot_annotations) == 0; + if (!cond) { + Py_DECREF(IsNot_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->IsNot_type, "__annotations__", + IsNot_annotations) == 0; + if (!cond) { + Py_DECREF(IsNot_annotations); + return 0; + } + Py_DECREF(IsNot_annotations); + PyObject *In_annotations = PyDict_New(); + if (!In_annotations) return 0; + cond = PyObject_SetAttrString(state->In_type, "_field_types", + In_annotations) == 0; + if (!cond) { + Py_DECREF(In_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->In_type, "__annotations__", + In_annotations) == 0; + if (!cond) { + Py_DECREF(In_annotations); + return 0; + } + Py_DECREF(In_annotations); + PyObject *NotIn_annotations = PyDict_New(); + if (!NotIn_annotations) return 0; + cond = PyObject_SetAttrString(state->NotIn_type, "_field_types", + NotIn_annotations) == 0; + if (!cond) { + Py_DECREF(NotIn_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->NotIn_type, "__annotations__", + NotIn_annotations) == 0; + if (!cond) { + Py_DECREF(NotIn_annotations); + return 0; + } + Py_DECREF(NotIn_annotations); + PyObject *comprehension_annotations = PyDict_New(); + if (!comprehension_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(comprehension_annotations, "target", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(comprehension_annotations, "iter", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + cond = PyDict_SetItemString(comprehension_annotations, "ifs", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(comprehension_annotations, "is_async", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->comprehension_type, "_field_types", + comprehension_annotations) == 0; + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->comprehension_type, "__annotations__", + comprehension_annotations) == 0; + if (!cond) { + Py_DECREF(comprehension_annotations); + return 0; + } + Py_DECREF(comprehension_annotations); + PyObject *ExceptHandler_annotations = PyDict_New(); + if (!ExceptHandler_annotations) return 0; + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyDict_SetItemString(ExceptHandler_annotations, "type", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyDict_SetItemString(ExceptHandler_annotations, "name", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyDict_SetItemString(ExceptHandler_annotations, "body", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ExceptHandler_type, "_field_types", + ExceptHandler_annotations) == 0; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ExceptHandler_type, "__annotations__", + ExceptHandler_annotations) == 0; + if (!cond) { + Py_DECREF(ExceptHandler_annotations); + return 0; + } + Py_DECREF(ExceptHandler_annotations); + PyObject *arguments_annotations = PyDict_New(); + if (!arguments_annotations) return 0; + { + PyObject *type = state->arg_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "posonlyargs", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "args", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "vararg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "kwonlyargs", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "kw_defaults", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->arg_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "kwarg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyDict_SetItemString(arguments_annotations, "defaults", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->arguments_type, "_field_types", + arguments_annotations) == 0; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->arguments_type, "__annotations__", + arguments_annotations) == 0; + if (!cond) { + Py_DECREF(arguments_annotations); + return 0; + } + Py_DECREF(arguments_annotations); + PyObject *arg_annotations = PyDict_New(); + if (!arg_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(arg_annotations, "arg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + cond = PyDict_SetItemString(arg_annotations, "annotation", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + cond = PyDict_SetItemString(arg_annotations, "type_comment", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->arg_type, "_field_types", + arg_annotations) == 0; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->arg_type, "__annotations__", + arg_annotations) == 0; + if (!cond) { + Py_DECREF(arg_annotations); + return 0; + } + Py_DECREF(arg_annotations); + PyObject *keyword_annotations = PyDict_New(); + if (!keyword_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + cond = PyDict_SetItemString(keyword_annotations, "arg", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(keyword_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->keyword_type, "_field_types", + keyword_annotations) == 0; + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->keyword_type, "__annotations__", + keyword_annotations) == 0; + if (!cond) { + Py_DECREF(keyword_annotations); + return 0; + } + Py_DECREF(keyword_annotations); + PyObject *alias_annotations = PyDict_New(); + if (!alias_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(alias_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + cond = PyDict_SetItemString(alias_annotations, "asname", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->alias_type, "_field_types", + alias_annotations) == 0; + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->alias_type, "__annotations__", + alias_annotations) == 0; + if (!cond) { + Py_DECREF(alias_annotations); + return 0; + } + Py_DECREF(alias_annotations); + PyObject *withitem_annotations = PyDict_New(); + if (!withitem_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(withitem_annotations, "context_expr", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + cond = PyDict_SetItemString(withitem_annotations, "optional_vars", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->withitem_type, "_field_types", + withitem_annotations) == 0; + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->withitem_type, "__annotations__", + withitem_annotations) == 0; + if (!cond) { + Py_DECREF(withitem_annotations); + return 0; + } + Py_DECREF(withitem_annotations); + PyObject *match_case_annotations = PyDict_New(); + if (!match_case_annotations) return 0; + { + PyObject *type = state->pattern_type; + Py_INCREF(type); + cond = PyDict_SetItemString(match_case_annotations, "pattern", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + cond = PyDict_SetItemString(match_case_annotations, "guard", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + } + { + PyObject *type = state->stmt_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + cond = PyDict_SetItemString(match_case_annotations, "body", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->match_case_type, "_field_types", + match_case_annotations) == 0; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->match_case_type, "__annotations__", + match_case_annotations) == 0; + if (!cond) { + Py_DECREF(match_case_annotations); + return 0; + } + Py_DECREF(match_case_annotations); + PyObject *MatchValue_annotations = PyDict_New(); + if (!MatchValue_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(MatchValue_annotations, "value", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchValue_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchValue_type, "_field_types", + MatchValue_annotations) == 0; + if (!cond) { + Py_DECREF(MatchValue_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchValue_type, "__annotations__", + MatchValue_annotations) == 0; + if (!cond) { + Py_DECREF(MatchValue_annotations); + return 0; + } + Py_DECREF(MatchValue_annotations); + PyObject *MatchSingleton_annotations = PyDict_New(); + if (!MatchSingleton_annotations) return 0; + { + PyObject *type = (PyObject *)&PyBaseObject_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(MatchSingleton_annotations, "value", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchSingleton_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchSingleton_type, "_field_types", + MatchSingleton_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSingleton_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchSingleton_type, + "__annotations__", + MatchSingleton_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSingleton_annotations); + return 0; + } + Py_DECREF(MatchSingleton_annotations); + PyObject *MatchSequence_annotations = PyDict_New(); + if (!MatchSequence_annotations) return 0; + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchSequence_annotations, "patterns", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchSequence_type, "_field_types", + MatchSequence_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchSequence_type, "__annotations__", + MatchSequence_annotations) == 0; + if (!cond) { + Py_DECREF(MatchSequence_annotations); + return 0; + } + Py_DECREF(MatchSequence_annotations); + PyObject *MatchMapping_annotations = PyDict_New(); + if (!MatchMapping_annotations) return 0; + { + PyObject *type = state->expr_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchMapping_annotations, "keys", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + } + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchMapping_annotations, "patterns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchMapping_annotations, "rest", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchMapping_type, "_field_types", + MatchMapping_annotations) == 0; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchMapping_type, "__annotations__", + MatchMapping_annotations) == 0; + if (!cond) { + Py_DECREF(MatchMapping_annotations); + return 0; + } + Py_DECREF(MatchMapping_annotations); + PyObject *MatchClass_annotations = PyDict_New(); + if (!MatchClass_annotations) return 0; + { + PyObject *type = state->expr_type; + Py_INCREF(type); + cond = PyDict_SetItemString(MatchClass_annotations, "cls", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchClass_annotations, "patterns", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchClass_annotations, "kwd_attrs", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchClass_annotations, "kwd_patterns", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchClass_type, "_field_types", + MatchClass_annotations) == 0; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchClass_type, "__annotations__", + MatchClass_annotations) == 0; + if (!cond) { + Py_DECREF(MatchClass_annotations); + return 0; + } + Py_DECREF(MatchClass_annotations); + PyObject *MatchStar_annotations = PyDict_New(); + if (!MatchStar_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchStar_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchStar_type, "_field_types", + MatchStar_annotations) == 0; + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchStar_type, "__annotations__", + MatchStar_annotations) == 0; + if (!cond) { + Py_DECREF(MatchStar_annotations); + return 0; + } + Py_DECREF(MatchStar_annotations); + PyObject *MatchAs_annotations = PyDict_New(); + if (!MatchAs_annotations) return 0; + { + PyObject *type = state->pattern_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchAs_annotations, "pattern", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchAs_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchAs_type, "_field_types", + MatchAs_annotations) == 0; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchAs_type, "__annotations__", + MatchAs_annotations) == 0; + if (!cond) { + Py_DECREF(MatchAs_annotations); + return 0; + } + Py_DECREF(MatchAs_annotations); + PyObject *MatchOr_annotations = PyDict_New(); + if (!MatchOr_annotations) return 0; + { + PyObject *type = state->pattern_type; + type = Py_GenericAlias((PyObject *)&PyList_Type, type); + cond = type != NULL; + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + cond = PyDict_SetItemString(MatchOr_annotations, "patterns", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->MatchOr_type, "_field_types", + MatchOr_annotations) == 0; + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->MatchOr_type, "__annotations__", + MatchOr_annotations) == 0; + if (!cond) { + Py_DECREF(MatchOr_annotations); + return 0; + } + Py_DECREF(MatchOr_annotations); + PyObject *TypeIgnore_annotations = PyDict_New(); + if (!TypeIgnore_annotations) return 0; + { + PyObject *type = (PyObject *)&PyLong_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeIgnore_annotations, "lineno", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + } + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeIgnore_annotations, "tag", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeIgnore_type, "_field_types", + TypeIgnore_annotations) == 0; + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeIgnore_type, "__annotations__", + TypeIgnore_annotations) == 0; + if (!cond) { + Py_DECREF(TypeIgnore_annotations); + return 0; + } + Py_DECREF(TypeIgnore_annotations); + PyObject *TypeVar_annotations = PyDict_New(); + if (!TypeVar_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeVar_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + cond = PyDict_SetItemString(TypeVar_annotations, "bound", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeVar_type, "_field_types", + TypeVar_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeVar_type, "__annotations__", + TypeVar_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + Py_DECREF(TypeVar_annotations); + PyObject *ParamSpec_annotations = PyDict_New(); + if (!ParamSpec_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(ParamSpec_annotations, "name", type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->ParamSpec_type, "_field_types", + ParamSpec_annotations) == 0; + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->ParamSpec_type, "__annotations__", + ParamSpec_annotations) == 0; + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + Py_DECREF(ParamSpec_annotations); + PyObject *TypeVarTuple_annotations = PyDict_New(); + if (!TypeVarTuple_annotations) return 0; + { + PyObject *type = (PyObject *)&PyUnicode_Type; + Py_INCREF(type); + cond = PyDict_SetItemString(TypeVarTuple_annotations, "name", type) == + 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + } + cond = PyObject_SetAttrString(state->TypeVarTuple_type, "_field_types", + TypeVarTuple_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + cond = PyObject_SetAttrString(state->TypeVarTuple_type, "__annotations__", + TypeVarTuple_annotations) == 0; + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + Py_DECREF(TypeVarTuple_annotations); + + return 1; +} + + typedef struct { PyObject_HEAD @@ -860,7 +5026,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) Py_ssize_t i, numfields = 0; int res = -1; - PyObject *key, *value, *fields; + PyObject *key, *value, *fields, *remaining_fields = NULL; if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { goto cleanup; } @@ -869,6 +5035,13 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) if (numfields == -1) { goto cleanup; } + remaining_fields = PySet_New(fields); + } + else { + remaining_fields = PySet_New(NULL); + } + if (remaining_fields == NULL) { + goto cleanup; } res = 0; /* if no error occurs, this stays 0 to the end */ @@ -888,6 +5061,11 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto cleanup; } res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); + if (PySet_Discard(remaining_fields, name) < 0) { + res = -1; + Py_DECREF(name); + goto cleanup; + } Py_DECREF(name); if (res < 0) { goto cleanup; @@ -900,13 +5078,14 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) if (contains == -1) { res = -1; goto cleanup; - } else if (contains == 1) { - Py_ssize_t p = PySequence_Index(fields, key); + } + else if (contains == 1) { + int p = PySet_Discard(remaining_fields, key); if (p == -1) { res = -1; goto cleanup; } - if (p < PyTuple_GET_SIZE(args)) { + if (p == 0) { PyErr_Format(PyExc_TypeError, "%.400s got multiple values for argument '%U'", Py_TYPE(self)->tp_name, key); @@ -914,15 +5093,91 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto cleanup; } } + else if ( + PyUnicode_CompareWithASCIIString(key, "lineno") != 0 && + PyUnicode_CompareWithASCIIString(key, "col_offset") != 0 && + PyUnicode_CompareWithASCIIString(key, "end_lineno") != 0 && + PyUnicode_CompareWithASCIIString(key, "end_col_offset") != 0 + ) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ got an unexpected keyword argument '%U'. " + "Support for arbitrary keyword arguments is deprecated " + "and will be removed in Python 3.15.", + Py_TYPE(self)->tp_name, key + ) < 0) { + res = -1; + goto cleanup; + } + } res = PyObject_SetAttr(self, key, value); if (res < 0) { goto cleanup; } } } + Py_ssize_t size = PySet_Size(remaining_fields); + PyObject *field_types = NULL, *remaining_list = NULL; + if (size > 0) { + if (!PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), &_Py_ID(_field_types), + &field_types)) { + res = -1; + goto cleanup; + } + remaining_list = PySequence_List(remaining_fields); + if (!remaining_list) { + goto set_remaining_cleanup; + } + for (Py_ssize_t i = 0; i < size; i++) { + PyObject *name = PyList_GET_ITEM(remaining_list, i); + PyObject *type = PyDict_GetItemWithError(field_types, name); + if (!type) { + if (!PyErr_Occurred()) { + PyErr_SetObject(PyExc_KeyError, name); + } + goto set_remaining_cleanup; + } + if (_PyUnion_Check(type)) { + // optional field + // do nothing, we'll have set a None default on the class + } + else if (Py_IS_TYPE(type, &Py_GenericAliasType)) { + // list field + PyObject *empty = PyList_New(0); + if (!empty) { + goto set_remaining_cleanup; + } + res = PyObject_SetAttr(self, name, empty); + Py_DECREF(empty); + if (res < 0) { + goto set_remaining_cleanup; + } + } + else { + // simple field (e.g., identifier) + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ missing 1 required positional argument: '%U'. " + "This will become an error in Python 3.15.", + Py_TYPE(self)->tp_name, name + ) < 0) { + res = -1; + goto cleanup; + } + } + } + Py_DECREF(remaining_list); + Py_DECREF(field_types); + } cleanup: Py_XDECREF(fields); + Py_XDECREF(remaining_fields); return res; + set_remaining_cleanup: + Py_XDECREF(remaining_list); + Py_XDECREF(field_types); + res = -1; + goto cleanup; } /* Pickling support */ @@ -934,14 +5189,75 @@ ast_type_reduce(PyObject *self, PyObject *unused) return NULL; } - PyObject *dict; + PyObject *dict = NULL, *fields = NULL, *remaining_fields = NULL, + *remaining_dict = NULL, *positional_args = NULL; if (PyObject_GetOptionalAttr(self, state->__dict__, &dict) < 0) { return NULL; } + PyObject *result = NULL; if (dict) { - return Py_BuildValue("O()N", Py_TYPE(self), dict); + // Serialize the fields as positional args if possible, because if we + // serialize them as a dict, during unpickling they are set only *after* + // the object is constructed, which will now trigger a DeprecationWarning + // if the AST type has required fields. + if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { + goto cleanup; + } + if (fields) { + Py_ssize_t numfields = PySequence_Size(fields); + if (numfields == -1) { + Py_DECREF(dict); + goto cleanup; + } + remaining_dict = PyDict_Copy(dict); + Py_DECREF(dict); + if (!remaining_dict) { + goto cleanup; + } + positional_args = PyList_New(0); + if (!positional_args) { + goto cleanup; + } + for (Py_ssize_t i = 0; i < numfields; i++) { + PyObject *name = PySequence_GetItem(fields, i); + if (!name) { + goto cleanup; + } + PyObject *value = PyDict_GetItemWithError(remaining_dict, name); + if (!value) { + if (PyErr_Occurred()) { + goto cleanup; + } + break; + } + if (PyList_Append(positional_args, value) < 0) { + goto cleanup; + } + if (PyDict_DelItem(remaining_dict, name) < 0) { + goto cleanup; + } + Py_DECREF(name); + } + PyObject *args_tuple = PyList_AsTuple(positional_args); + if (!args_tuple) { + goto cleanup; + } + result = Py_BuildValue("ONO", Py_TYPE(self), args_tuple, + remaining_dict); + } + else { + result = Py_BuildValue("O()N", Py_TYPE(self), dict); + } + } + else { + result = Py_BuildValue("O()", Py_TYPE(self)); } - return Py_BuildValue("O()", Py_TYPE(self)); +cleanup: + Py_XDECREF(fields); + Py_XDECREF(remaining_fields); + Py_XDECREF(remaining_dict); + Py_XDECREF(positional_args); + return result; } static PyMemberDef ast_type_members[] = { @@ -1939,6 +6255,9 @@ init_types(struct ast_state *state) "TypeVarTuple(identifier name)"); if (!state->TypeVarTuple_type) return -1; + if (!add_ast_annotations(state)) { + return -1; + } return 0; } diff --git a/Python/brc.c b/Python/brc.c index f1fd57a2964cf5..b73c721e71aef6 100644 --- a/Python/brc.c +++ b/Python/brc.c @@ -94,7 +94,7 @@ _Py_brc_queue_object(PyObject *ob) } // Notify owning thread - _Py_set_eval_breaker_bit(interp, _PY_EVAL_EXPLICIT_MERGE_BIT, 1); + _Py_set_eval_breaker_bit(&tstate->base, _PY_EVAL_EXPLICIT_MERGE_BIT); PyMutex_Unlock(&bucket->mutex); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6822e772e913e8..396a8f09f3feca 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -8,7 +8,6 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() -#include "pycore_ceval.h" // _PyEval_SignalAsyncExc() #include "pycore_code.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_function.h" @@ -54,6 +53,8 @@ #define guard #define override #define specializing +#define split +#define replicate(TIMES) // Dummy variables for stack effects. static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub; @@ -140,11 +141,10 @@ dummy_func( RESUME_CHECK, }; - inst(RESUME, (--)) { - TIER_ONE_ONLY + tier1 inst(RESUME, (--)) { assert(frame == tstate->current_frame); uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((code_version & 255) == 0); @@ -166,14 +166,14 @@ dummy_func( DEOPT_IF(_Py_emscripten_signal_clock == 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); DEOPT_IF(eval_breaker != version); } inst(INSTRUMENTED_RESUME, (--)) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { @@ -208,7 +208,7 @@ dummy_func( Py_INCREF(value); } - pure inst(LOAD_FAST, (-- value)) { + replicate(8) pure inst(LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); @@ -234,7 +234,7 @@ dummy_func( Py_INCREF(value); } - inst(STORE_FAST, (value --)) { + replicate(8) inst(STORE_FAST, (value --)) { SETLOCAL(oparg, value); } @@ -267,8 +267,7 @@ dummy_func( macro(END_FOR) = POP_TOP; - inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { - TIER_ONE_ONLY + tier1 inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -285,8 +284,7 @@ dummy_func( Py_DECREF(receiver); } - inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { - TIER_ONE_ONLY + tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { @@ -318,7 +316,6 @@ dummy_func( }; specializing op(_SPECIALIZE_TO_BOOL, (counter/1, value -- value)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -340,12 +337,12 @@ dummy_func( macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + unused/2 + _TO_BOOL; inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) { - DEOPT_IF(!PyBool_Check(value)); + EXIT_IF(!PyBool_Check(value)); STAT_INC(TO_BOOL, hit); } inst(TO_BOOL_INT, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyLong_CheckExact(value)); + EXIT_IF(!PyLong_CheckExact(value)); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -358,7 +355,7 @@ dummy_func( } inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyList_CheckExact(value)); + EXIT_IF(!PyList_CheckExact(value)); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; DECREF_INPUTS(); @@ -366,13 +363,13 @@ dummy_func( inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value)); + EXIT_IF(!Py_IsNone(value)); STAT_INC(TO_BOOL, hit); res = Py_False; } inst(TO_BOOL_STR, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyUnicode_CheckExact(value)); + EXIT_IF(!PyUnicode_CheckExact(value)); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -388,7 +385,7 @@ dummy_func( inst(TO_BOOL_ALWAYS_TRUE, (unused/1, version/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version); + EXIT_IF(Py_TYPE(value)->tp_version_tag != version); STAT_INC(TO_BOOL, hit); DECREF_INPUTS(); res = Py_True; @@ -412,8 +409,8 @@ dummy_func( }; op(_GUARD_BOTH_INT, (left, right -- left, right)) { - DEOPT_IF(!PyLong_CheckExact(left)); - DEOPT_IF(!PyLong_CheckExact(right)); + EXIT_IF(!PyLong_CheckExact(left)); + EXIT_IF(!PyLong_CheckExact(right)); } pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { @@ -448,8 +445,8 @@ dummy_func( _GUARD_BOTH_INT + unused/1 + _BINARY_OP_SUBTRACT_INT; op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { - DEOPT_IF(!PyFloat_CheckExact(left)); - DEOPT_IF(!PyFloat_CheckExact(right)); + EXIT_IF(!PyFloat_CheckExact(left)); + EXIT_IF(!PyFloat_CheckExact(right)); } pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { @@ -484,8 +481,8 @@ dummy_func( _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_SUBTRACT_FLOAT; op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { - DEOPT_IF(!PyUnicode_CheckExact(left)); - DEOPT_IF(!PyUnicode_CheckExact(right)); + EXIT_IF(!PyUnicode_CheckExact(left)); + EXIT_IF(!PyUnicode_CheckExact(right)); } pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { @@ -505,8 +502,7 @@ dummy_func( // So the inputs are the same as for all BINARY_OP // specializations, but there is no output. // At the end we just skip over the STORE_FAST. - op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { - TIER_ONE_ONLY + tier1 op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { assert(next_instr->op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(next_instr->op.arg); DEOPT_IF(*target_local != left); @@ -544,7 +540,6 @@ dummy_func( }; specializing op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -692,7 +687,6 @@ dummy_func( }; specializing op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -761,8 +755,7 @@ dummy_func( ERROR_IF(res == NULL, error); } - inst(RAISE_VARARGS, (args[oparg] -- )) { - TIER_ONE_ONLY + tier1 inst(RAISE_VARARGS, (args[oparg] -- )) { PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -786,8 +779,7 @@ dummy_func( ERROR_IF(true, error); } - inst(INTERPRETER_EXIT, (retval --)) { - TIER_ONE_ONLY + tier1 inst(INTERPRETER_EXIT, (retval --)) { assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ @@ -979,7 +971,6 @@ dummy_func( }; specializing op(_SPECIALIZE_SEND, (counter/1, receiver, unused -- receiver, unused)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -1074,8 +1065,7 @@ dummy_func( goto resume_frame; } - inst(YIELD_VALUE, (retval -- unused)) { - TIER_ONE_ONLY + tier1 inst(YIELD_VALUE, (retval -- unused)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1104,8 +1094,7 @@ dummy_func( Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); } - inst(RERAISE, (values[oparg], exc -- values[oparg])) { - TIER_ONE_ONLY + tier1 inst(RERAISE, (values[oparg], exc -- values[oparg])) { assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1126,8 +1115,7 @@ dummy_func( goto exception_unwind; } - inst(END_ASYNC_FOR, (awaitable, exc -- )) { - TIER_ONE_ONLY + tier1 inst(END_ASYNC_FOR, (awaitable, exc -- )) { assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { DECREF_INPUTS(); @@ -1140,8 +1128,7 @@ dummy_func( } } - inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value -- none, value)) { - TIER_ONE_ONLY + tier1 inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value -- none, value)) { assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { @@ -1213,7 +1200,6 @@ dummy_func( }; specializing op(_SPECIALIZE_UNPACK_SEQUENCE, (counter/1, seq -- seq)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -1283,7 +1269,6 @@ dummy_func( }; specializing op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -1402,7 +1387,6 @@ dummy_func( }; specializing op(_SPECIALIZE_LOAD_GLOBAL, (counter/1 -- )) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); @@ -1730,7 +1714,6 @@ dummy_func( }; specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super, class, unused -- global_super, class, unused)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION int load_method = oparg & 1; if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -1743,8 +1726,7 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - op(_LOAD_SUPER_ATTR, (global_super, class, self -- attr, null if (oparg & 1))) { - TIER_ONE_ONLY + tier1 op(_LOAD_SUPER_ATTR, (global_super, class, self -- attr, null if (oparg & 1))) { if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( @@ -1846,7 +1828,6 @@ dummy_func( }; specializing op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); @@ -1904,7 +1885,7 @@ dummy_func( op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version); + EXIT_IF(tp->tp_version_tag != type_version); } op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { @@ -1914,7 +1895,7 @@ dummy_func( DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); } - op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { + split op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; DEOPT_IF(attr == NULL); @@ -1931,11 +1912,11 @@ dummy_func( _LOAD_ATTR_INSTANCE_VALUE + unused/5; // Skip over rest of cache - op(_CHECK_ATTR_MODULE, (type_version/2, owner -- owner)) { + op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) { DEOPT_IF(!PyModule_CheckExact(owner)); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version); + DEOPT_IF(dict->ma_keys->dk_version != dict_version); } op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { @@ -1995,7 +1976,7 @@ dummy_func( _LOAD_ATTR_WITH_HINT + unused/5; - op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { + split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { char *addr = (char *)owner + index; attr = *(PyObject **)addr; DEOPT_IF(attr == NULL); @@ -2018,7 +1999,7 @@ dummy_func( } - op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { + split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = Py_NewRef(descr); @@ -2171,7 +2152,6 @@ dummy_func( }; specializing op(_SPECIALIZE_COMPARE_OP, (counter/1, left, right -- left, right)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -2198,9 +2178,16 @@ dummy_func( macro(COMPARE_OP) = _SPECIALIZE_COMPARE_OP + _COMPARE_OP; - inst(COMPARE_OP_FLOAT, (unused/1, left, right -- res)) { - DEOPT_IF(!PyFloat_CheckExact(left)); - DEOPT_IF(!PyFloat_CheckExact(right)); + macro(COMPARE_OP_FLOAT) = + _GUARD_BOTH_FLOAT + unused/1 + _COMPARE_OP_FLOAT; + + macro(COMPARE_OP_INT) = + _GUARD_BOTH_INT + unused/1 + _COMPARE_OP_INT; + + macro(COMPARE_OP_STR) = + _GUARD_BOTH_UNICODE + unused/1 + _COMPARE_OP_STR; + + op(_COMPARE_OP_FLOAT, (left, right -- res)) { STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); @@ -2213,9 +2200,7 @@ dummy_func( } // Similar to COMPARE_OP_FLOAT - inst(COMPARE_OP_INT, (unused/1, left, right -- res)) { - DEOPT_IF(!PyLong_CheckExact(left)); - DEOPT_IF(!PyLong_CheckExact(right)); + op(_COMPARE_OP_INT, (left, right -- res)) { DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left)); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right)); STAT_INC(COMPARE_OP, hit); @@ -2232,9 +2217,7 @@ dummy_func( } // Similar to COMPARE_OP_FLOAT, but for ==, != only - inst(COMPARE_OP_STR, (unused/1, left, right -- res)) { - DEOPT_IF(!PyUnicode_CheckExact(left)); - DEOPT_IF(!PyUnicode_CheckExact(right)); + op(_COMPARE_OP_STR, (left, right -- res)) { STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left, right); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); @@ -2293,27 +2276,24 @@ dummy_func( b = res ? Py_True : Py_False; } - inst(IMPORT_NAME, (level, fromlist -- res)) { - TIER_ONE_ONLY + tier1 inst(IMPORT_NAME, (level, fromlist -- res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_name(tstate, frame, name, fromlist, level); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } - inst(IMPORT_FROM, (from -- from, res)) { - TIER_ONE_ONLY + tier1 inst(IMPORT_FROM, (from -- from, res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_from(tstate, from, name); ERROR_IF(res == NULL, error); } - inst(JUMP_FORWARD, (--)) { - TIER_ONE_ONLY + tier1 inst(JUMP_FORWARD, (--)) { JUMPBY(oparg); } - inst(JUMP_BACKWARD, (unused/1 --)) { + tier1 inst(JUMP_BACKWARD, (unused/1 --)) { CHECK_EVAL_BREAKER(); assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); @@ -2335,13 +2315,13 @@ dummy_func( oparg >>= 8; start--; } - int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer); + _PyExecutorObject *executor; + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); ERROR_IF(optimized < 0, error); if (optimized) { - // Rewind and enter the executor: - assert(start->op.code == ENTER_EXECUTOR); - next_instr = start; - this_instr[1].cache &= OPTIMIZER_BITS_MASK; + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); } else { int backoff = this_instr[1].cache & OPTIMIZER_BITS_MASK; @@ -2368,17 +2348,17 @@ dummy_func( JUMP_BACKWARD_NO_INTERRUPT, }; - inst(ENTER_EXECUTOR, (--)) { - TIER_ONE_ONLY + tier1 inst(ENTER_EXECUTOR, (--)) { CHECK_EVAL_BREAKER(); - PyCodeObject *code = _PyFrame_GetCode(frame); - current_executor = code->co_executors->executors[oparg & 255]; - assert(current_executor->vm_data.index == INSTR_OFFSET() - 1); - assert(current_executor->vm_data.code == code); - assert(current_executor->vm_data.valid); - Py_INCREF(current_executor); - GOTO_TIER_TWO(); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + Py_INCREF(executor); + GOTO_TIER_TWO(executor); } replaced op(_POP_JUMP_IF_FALSE, (cond -- )) { @@ -2417,8 +2397,7 @@ dummy_func( macro(POP_JUMP_IF_NOT_NONE) = unused/1 + _IS_NONE + _POP_JUMP_IF_FALSE; - inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { - TIER_ONE_ONLY + tier1 inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -2514,7 +2493,6 @@ dummy_func( }; specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -2606,7 +2584,7 @@ dummy_func( assert(Py_TYPE(iter) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); PyListObject *seq = it->it_seq; - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { it->it_index = -1; #ifndef Py_GIL_DISABLED if (seq != NULL) { @@ -2627,6 +2605,7 @@ dummy_func( _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; + DEOPT_IF(seq == NULL); DEOPT_IF((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)); } @@ -2775,7 +2754,7 @@ dummy_func( GOTO_ERROR(error); } DECREF_INPUTS(); - res = _PyObject_CallNoArgsTstate(tstate, enter); + res = PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -2810,7 +2789,7 @@ dummy_func( GOTO_ERROR(error); } DECREF_INPUTS(); - res = _PyObject_CallNoArgsTstate(tstate, enter); + res = PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -2886,7 +2865,7 @@ dummy_func( DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); } - op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { + split op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); @@ -3011,7 +2990,6 @@ dummy_func( }; specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -3128,7 +3106,7 @@ dummy_func( DEOPT_IF(tstate->py_recursion_remaining <= 1); } - pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { int argcount = oparg; if (self_or_null != NULL) { args--; @@ -3473,8 +3451,7 @@ dummy_func( } // This is secretly a super-instruction - inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { - TIER_ONE_ONLY + tier1 inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append); @@ -3812,8 +3789,7 @@ dummy_func( } } - inst(RETURN_GENERATOR, (--)) { - TIER_ONE_ONLY + tier1 inst(RETURN_GENERATOR, (--)) { assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -3845,9 +3821,9 @@ dummy_func( } inst(CONVERT_VALUE, (value -- result)) { - convertion_func_ptr conv_fn; + conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; + conv_fn = _PyEval_ConversionFuncs[oparg]; result = conv_fn(value); Py_DECREF(value); ERROR_IF(result == NULL, error); @@ -3879,7 +3855,6 @@ dummy_func( } specializing op(_SPECIALIZE_BINARY_OP, (counter/1, lhs, rhs -- lhs, rhs)) { - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -3985,8 +3960,7 @@ dummy_func( INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } - inst(EXTENDED_ARG, ( -- )) { - TIER_ONE_ONLY + tier1 inst(EXTENDED_ARG, ( -- )) { assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -3994,29 +3968,27 @@ dummy_func( DISPATCH_GOTO(); } - inst(CACHE, (--)) { - TIER_ONE_ONLY + tier1 inst(CACHE, (--)) { assert(0 && "Executing a cache."); - Py_UNREACHABLE(); + Py_FatalError("Executing a cache."); } - inst(RESERVED, (--)) { - TIER_ONE_ONLY + tier1 inst(RESERVED, (--)) { assert(0 && "Executing RESERVED instruction."); - Py_UNREACHABLE(); + Py_FatalError("Executing RESERVED instruction."); } ///////// Tier-2 only opcodes ///////// op (_GUARD_IS_TRUE_POP, (flag -- )) { SYNC_SP(); - DEOPT_IF(!Py_IsTrue(flag)); + EXIT_IF(!Py_IsTrue(flag)); assert(Py_IsTrue(flag)); } op (_GUARD_IS_FALSE_POP, (flag -- )) { SYNC_SP(); - DEOPT_IF(!Py_IsFalse(flag)); + EXIT_IF(!Py_IsFalse(flag)); assert(Py_IsFalse(flag)); } @@ -4024,23 +3996,24 @@ dummy_func( SYNC_SP(); if (!Py_IsNone(val)) { Py_DECREF(val); - DEOPT_IF(1); + EXIT_IF(1); } } op (_GUARD_IS_NOT_NONE_POP, (val -- )) { SYNC_SP(); - DEOPT_IF(Py_IsNone(val)); + EXIT_IF(Py_IsNone(val)); Py_DECREF(val); } op(_JUMP_TO_TOP, (--)) { - next_uop = current_executor->trace; +#ifndef _Py_JIT + next_uop = ¤t_executor->trace[1]; +#endif CHECK_EVAL_BREAKER(); } - op(_SET_IP, (instr_ptr/4 --)) { - TIER_TWO_ONLY + tier2 op(_SET_IP, (instr_ptr/4 --)) { frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; } @@ -4053,45 +4026,42 @@ dummy_func( #endif } - op(_EXIT_TRACE, (--)) { - TIER_TWO_ONLY - DEOPT_IF(1); + tier2 op(_EXIT_TRACE, (--)) { + EXIT_IF(1); } - op(_CHECK_VALIDITY, (--)) { - TIER_TWO_ONLY + tier2 op(_CHECK_VALIDITY, (--)) { DEOPT_IF(!current_executor->vm_data.valid); } - pure op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - TIER_TWO_ONLY + tier2 pure op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { value = Py_NewRef(ptr); } - pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - TIER_TWO_ONLY + tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { value = ptr; } - pure op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { - TIER_TWO_ONLY + tier2 pure op (_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { + Py_DECREF(pop); + value = ptr; + } + + tier2 pure op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { value = Py_NewRef(ptr); null = NULL; } - pure op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { - TIER_TWO_ONLY + tier2 pure op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { value = ptr; null = NULL; } - op(_CHECK_GLOBALS, (dict/4 -- )) { - TIER_TWO_ONLY + tier2 op(_CHECK_GLOBALS, (dict/4 -- )) { DEOPT_IF(GLOBALS() != dict); } - op(_CHECK_BUILTINS, (dict/4 -- )) { - TIER_TWO_ONLY + tier2 op(_CHECK_BUILTINS, (dict/4 -- )) { DEOPT_IF(BUILTINS() != dict); } @@ -4101,8 +4071,56 @@ dummy_func( exe->count++; } - op(_CHECK_VALIDITY_AND_SET_IP, (instr_ptr/4 --)) { - TIER_TWO_ONLY + /* Only used for handling cold side exits, should never appear in + * a normal trace or as part of an instruction. + */ + tier2 op(_COLD_EXIT, (--)) { + _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor; + _PyExitData *exit = &previous->exits[oparg]; + exit->temperature++; + PyCodeObject *code = _PyFrame_GetCode(frame); + _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; + if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { + GOTO_TIER_ONE(target); + } + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } else { + int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); + if (optimized <= 0) { + int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold; + exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp; + if (optimized < 0) { + Py_DECREF(previous); + tstate->previous_executor = Py_None; + ERROR_IF(1, error); + } + GOTO_TIER_ONE(target); + } + } + /* We need two references. One to store in exit->executor and + * one to keep the executor alive when executing. */ + Py_INCREF(executor); + exit->executor = executor; + GOTO_TIER_TWO(executor); + } + + tier2 op(_START_EXECUTOR, (executor/4 --)) { + Py_DECREF(tstate->previous_executor); + tstate->previous_executor = NULL; +#ifndef _Py_JIT + current_executor = (_PyExecutorObject*)executor; +#endif + } + + tier2 op(_FATAL_ERROR, (--)) { + assert(0); + Py_FatalError("Fatal error uop executed."); + } + + tier2 op(_CHECK_VALIDITY_AND_SET_IP, (instr_ptr/4 --)) { DEOPT_IF(!current_executor->vm_data.valid); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; } diff --git a/Python/ceval.c b/Python/ceval.c index 4f208009086191..f817f288903694 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5,7 +5,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_SignalAsyncExc() +#include "pycore_ceval.h" #include "pycore_code.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_function.h" @@ -16,6 +16,7 @@ #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_opcode_metadata.h" // EXTRA_CASES +#include "pycore_optimizer.h" // _PyUOpExecutor_Type #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -336,6 +337,12 @@ const binaryfunc _PyEval_BinaryOps[] = { [NB_INPLACE_XOR] = PyNumber_InPlaceXor, }; +const conversion_func _PyEval_ConversionFuncs[4] = { + [FVC_STR] = PyObject_Str, + [FVC_REPR] = PyObject_Repr, + [FVC_ASCII] = PyObject_ASCII +}; + // PEP 634: Structural Pattern Matching @@ -648,7 +655,10 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { extern const struct _PyCode_DEF(8) _Py_InitCleanup; -extern const char *_PyUOpName(int index); +#ifdef Py_DEBUG +extern void _PyUOpPrint(const _PyUOpInstruction *uop); +#endif + /* Disable unused label warnings. They are handy for debugging, even if computed gotos aren't used. */ @@ -738,15 +748,16 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto resume_with_error; } - /* State shared between Tier 1 and Tier 2 interpreter */ - _PyExecutorObject *current_executor = NULL; - /* Local "register" variables. * These are cached values from the frame and code object. */ - _Py_CODEUNIT *next_instr; PyObject **stack_pointer; +#ifndef _Py_JIT + /* Tier 2 interpreter state */ + _PyExecutorObject *current_executor = NULL; + const _PyUOpInstruction *next_uop = NULL; +#endif start_frame: if (_Py_EnterRecursivePy(tstate)) { @@ -960,18 +971,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int enter_tier_two: #ifdef _Py_JIT - - ; // ;) - jit_func jitted = current_executor->jit_code; - next_instr = jitted(frame, stack_pointer, tstate); - frame = tstate->current_frame; - Py_DECREF(current_executor); - if (next_instr == NULL) { - goto resume_with_error; - } - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - + assert(0); #else #undef LOAD_IP @@ -1007,22 +1007,22 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif OPT_STAT_INC(traces_executed); - _PyUOpInstruction *next_uop = current_executor->trace; uint16_t uopcode; #ifdef Py_STATS uint64_t trace_uop_execution_counter = 0; #endif + assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); for (;;) { uopcode = next_uop->opcode; - DPRINTF(3, - "%4d: uop %s, oparg %d, operand %" PRIu64 ", target %d, stack_level %d\n", - (int)(next_uop - current_executor->trace), - _PyUOpName(uopcode), - next_uop->oparg, - next_uop->operand, - next_uop->target, +#ifdef Py_DEBUG + if (lltrace >= 3) { + printf("%4d uop: ", (int)(next_uop - (current_executor == NULL ? next_uop : current_executor->trace))); + _PyUOpPrint(next_uop); + printf(" stack_level=%d\n", (int)(stack_pointer - _PyFrame_Stackbase(frame))); + } +#endif next_uop++; OPT_STAT_INC(uops_executed); UOP_STAT_INC(uopcode, execution_count); @@ -1037,9 +1037,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int default: #ifdef Py_DEBUG { - fprintf(stderr, "Unknown uop %d, oparg %d, operand %" PRIu64 " @ %d\n", - opcode, next_uop[-1].oparg, next_uop[-1].operand, - (int)(next_uop - current_executor->trace - 1)); + printf("Unknown uop: "); + _PyUOpPrint(&next_uop[-1]); + printf(" @ %d\n", (int)(next_uop - current_executor->trace - 1)); Py_FatalError("Unknown uop"); } #else @@ -1067,31 +1067,63 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int pop_1_error_tier_two: STACK_SHRINK(1); error_tier_two: - DPRINTF(2, "Error: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d -> %s]\n", - uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, - (int)(next_uop - current_executor->trace - 1), - _PyOpcode_OpName[frame->instr_ptr->op.code]); +#ifdef Py_DEBUG + if (lltrace >= 2) { + printf("Error: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(" @ %d -> %s]\n", + (int)(next_uop - current_executor->trace - 1), + _PyOpcode_OpName[frame->instr_ptr->op.code]); + } +#endif OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); frame->return_offset = 0; // Don't leave this random _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(current_executor); + tstate->previous_executor = NULL; goto resume_with_error; // Jump here from DEOPT_IF() deoptimize: next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); - DPRINTF(2, "DEOPT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d -> %s]\n", - uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, - (int)(next_uop - current_executor->trace - 1), - _PyOpcode_OpName[frame->instr_ptr->op.code]); +#ifdef Py_DEBUG + if (lltrace >= 2) { + printf("DEOPT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(" -> %s]\n", + _PyOpcode_OpName[frame->instr_ptr->op.code]); + } +#endif OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); Py_DECREF(current_executor); + tstate->previous_executor = NULL; DISPATCH(); +// Jump here from EXIT_IF() +side_exit: + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + UOP_STAT_INC(uopcode, miss); + uint32_t exit_index = next_uop[-1].exit_index; + assert(exit_index < current_executor->exit_count); + _PyExitData *exit = ¤t_executor->exits[exit_index]; +#ifdef Py_DEBUG + if (lltrace >= 2) { + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %u, temp %d, target %d -> %s]\n", + exit_index, exit->temperature, exit->target, + _PyOpcode_OpName[frame->instr_ptr->op.code]); + } +#endif + Py_INCREF(exit->executor); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_TWO(exit->executor); + #endif // _Py_JIT } + #if defined(__GNUC__) # pragma GCC diagnostic pop #elif defined(_MSC_VER) /* MS_WINDOWS */ diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index deb9741291fca7..f5c44307a513f8 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -56,60 +56,52 @@ #define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL) #endif -/* bpo-40010: eval_breaker should be recomputed if there - is a pending signal: signal received by another thread which cannot - handle signals. - Similarly, we set CALLS_TO_DO and ASYNC_EXCEPTION to match the thread. -*/ +// Atomically copy the bits indicated by mask between two values. static inline void -update_eval_breaker_from_thread(PyInterpreterState *interp, PyThreadState *tstate) +copy_eval_breaker_bits(uintptr_t *from, uintptr_t *to, uintptr_t mask) { - if (tstate == NULL) { + uintptr_t from_bits = _Py_atomic_load_uintptr_relaxed(from) & mask; + uintptr_t old_value = _Py_atomic_load_uintptr_relaxed(to); + uintptr_t to_bits = old_value & mask; + if (from_bits == to_bits) { return; } - if (_Py_IsMainThread()) { - int32_t calls_to_do = _Py_atomic_load_int32_relaxed( - &_PyRuntime.ceval.pending_mainthread.calls_to_do); - if (calls_to_do) { - _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 1); - } - if (_Py_ThreadCanHandleSignals(interp)) { - if (_Py_atomic_load_int(&_PyRuntime.signals.is_tripped)) { - _Py_set_eval_breaker_bit(interp, _PY_SIGNALS_PENDING_BIT, 1); - } - } - } - if (tstate->async_exc != NULL) { - _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 1); - } + uintptr_t new_value; + do { + new_value = (old_value & ~mask) | from_bits; + } while (!_Py_atomic_compare_exchange_uintptr(to, &old_value, new_value)); } +// When attaching a thread, set the global instrumentation version and +// _PY_CALLS_TO_DO_BIT from the current state of the interpreter. static inline void -SET_GIL_DROP_REQUEST(PyInterpreterState *interp) +update_eval_breaker_for_thread(PyInterpreterState *interp, PyThreadState *tstate) { - _Py_set_eval_breaker_bit(interp, _PY_GIL_DROP_REQUEST_BIT, 1); -} - - -static inline void -RESET_GIL_DROP_REQUEST(PyInterpreterState *interp) -{ - _Py_set_eval_breaker_bit(interp, _PY_GIL_DROP_REQUEST_BIT, 0); -} - - -static inline void -SIGNAL_PENDING_CALLS(PyInterpreterState *interp) -{ - _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 1); -} +#ifdef Py_GIL_DISABLED + // Free-threaded builds eagerly update the eval_breaker on *all* threads as + // needed, so this function doesn't apply. + return; +#endif + int32_t calls_to_do = _Py_atomic_load_int32_relaxed( + &interp->ceval.pending.calls_to_do); + if (calls_to_do) { + _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); + } + else if (_Py_IsMainThread()) { + calls_to_do = _Py_atomic_load_int32_relaxed( + &_PyRuntime.ceval.pending_mainthread.calls_to_do); + if (calls_to_do) { + _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); + } + } -static inline void -UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp) -{ - _Py_set_eval_breaker_bit(interp, _PY_CALLS_TO_DO_BIT, 0); + // _PY_CALLS_TO_DO_BIT was derived from other state above, so the only bits + // we copy from our interpreter's state are the instrumentation version. + copy_eval_breaker_bits(&interp->ceval.instrumentation_version, + &tstate->eval_breaker, + ~_PY_EVAL_EVENTS_MASK); } /* @@ -254,13 +246,14 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) the GIL, and that's the only time we might delete the interpreter, so checking tstate first prevents the crash. See https://github.com/python/cpython/issues/104341. */ - if (tstate != NULL && _Py_eval_breaker_bit_is_set(interp, _PY_GIL_DROP_REQUEST_BIT)) { + if (tstate != NULL && + _Py_eval_breaker_bit_is_set(tstate, _PY_GIL_DROP_REQUEST_BIT)) { MUTEX_LOCK(gil->switch_mutex); /* Not switched yet => wait */ if (((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) == tstate) { assert(_PyThreadState_CheckConsistency(tstate)); - RESET_GIL_DROP_REQUEST(tstate->interp); + _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); /* NOTE: if COND_WAIT does not atomically start waiting when releasing the mutex, another thread can run through, take the GIL and drop it again, and reset the condition @@ -321,6 +314,8 @@ take_gil(PyThreadState *tstate) _Py_atomic_load_int_relaxed(&gil->locked) && gil->switch_number == saved_switchnum) { + PyThreadState *holder_tstate = + (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder); if (_PyThreadState_MustExit(tstate)) { MUTEX_UNLOCK(gil->mutex); // gh-96387: If the loop requested a drop request in a previous @@ -330,13 +325,13 @@ take_gil(PyThreadState *tstate) // may have to request again a drop request (iterate one more // time). if (drop_requested) { - RESET_GIL_DROP_REQUEST(interp); + _Py_unset_eval_breaker_bit(holder_tstate, _PY_GIL_DROP_REQUEST_BIT); } PyThread_exit_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); - SET_GIL_DROP_REQUEST(interp); + _Py_set_eval_breaker_bit(holder_tstate, _PY_GIL_DROP_REQUEST_BIT); drop_requested = 1; } } @@ -369,13 +364,15 @@ take_gil(PyThreadState *tstate) in take_gil() while the main thread called wait_for_thread_shutdown() from Py_Finalize(). */ MUTEX_UNLOCK(gil->mutex); - drop_gil(interp, tstate); + /* Passing NULL to drop_gil() indicates that this thread is about to + terminate and will never hold the GIL again. */ + drop_gil(interp, NULL); PyThread_exit_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); - RESET_GIL_DROP_REQUEST(interp); - update_eval_breaker_from_thread(interp, tstate); + _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); + update_eval_breaker_for_thread(interp, tstate); MUTEX_UNLOCK(gil->mutex); @@ -590,15 +587,6 @@ _PyEval_ReInitThreads(PyThreadState *tstate) } #endif -/* This function is used to signal that async exceptions are waiting to be - raised. */ - -void -_PyEval_SignalAsyncExc(PyInterpreterState *interp) -{ - _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 1); -} - PyThreadState * PyEval_SaveThread(void) { @@ -646,11 +634,9 @@ PyEval_RestoreThread(PyThreadState *tstate) */ void -_PyEval_SignalReceived(PyInterpreterState *interp) +_PyEval_SignalReceived(void) { - if (_Py_ThreadCanHandleSignals(interp)) { - _Py_set_eval_breaker_bit(interp, _PY_SIGNALS_PENDING_BIT, 1); - } + _Py_set_eval_breaker_bit(_PyRuntime.main_tstate, _PY_SIGNALS_PENDING_BIT); } /* Push one item onto the queue while holding the lock. */ @@ -702,6 +688,26 @@ _pop_pending_call(struct _pending_calls *pending, } } +#ifndef Py_GIL_DISABLED +static void +signal_active_thread(PyInterpreterState *interp, uintptr_t bit) +{ + struct _gil_runtime_state *gil = interp->ceval.gil; + + // If a thread from the targeted interpreter is holding the GIL, signal + // that thread. Otherwise, the next thread to run from the targeted + // interpreter will have its bit set as part of taking the GIL. + MUTEX_LOCK(gil->mutex); + if (_Py_atomic_load_int_relaxed(&gil->locked)) { + PyThreadState *holder = (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder); + if (holder->interp == interp) { + _Py_set_eval_breaker_bit(holder, bit); + } + } + MUTEX_UNLOCK(gil->mutex); +} +#endif + /* This implementation is thread-safe. It allows scheduling to be made from any thread, and even from an executing callback. @@ -711,10 +717,9 @@ int _PyEval_AddPendingCall(PyInterpreterState *interp, _Py_pending_call_func func, void *arg, int flags) { - assert(!(flags & _Py_PENDING_MAINTHREADONLY) - || _Py_IsMainInterpreter(interp)); struct _pending_calls *pending = &interp->ceval.pending; - if (flags & _Py_PENDING_MAINTHREADONLY) { + int main_only = (flags & _Py_PENDING_MAINTHREADONLY) != 0; + if (main_only) { /* The main thread only exists in the main interpreter. */ assert(_Py_IsMainInterpreter(interp)); pending = &_PyRuntime.ceval.pending_mainthread; @@ -724,8 +729,17 @@ _PyEval_AddPendingCall(PyInterpreterState *interp, int result = _push_pending_call(pending, func, arg, flags); PyMutex_Unlock(&pending->mutex); - /* signal main loop */ - SIGNAL_PENDING_CALLS(interp); + if (main_only) { + _Py_set_eval_breaker_bit(_PyRuntime.main_tstate, _PY_CALLS_TO_DO_BIT); + } + else { +#ifdef Py_GIL_DISABLED + _Py_set_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT); +#else + signal_active_thread(interp, _PY_CALLS_TO_DO_BIT); +#endif + } + return result; } @@ -742,13 +756,13 @@ static int handle_signals(PyThreadState *tstate) { assert(_PyThreadState_CheckConsistency(tstate)); - _Py_set_eval_breaker_bit(tstate->interp, _PY_SIGNALS_PENDING_BIT, 0); + _Py_unset_eval_breaker_bit(tstate, _PY_SIGNALS_PENDING_BIT); if (!_Py_ThreadCanHandleSignals(tstate->interp)) { return 0; } if (_PyErr_CheckSignalsTstate(tstate) < 0) { /* On failure, re-schedule a call to handle_signals(). */ - _Py_set_eval_breaker_bit(tstate->interp, _PY_SIGNALS_PENDING_BIT, 1); + _Py_set_eval_breaker_bit(tstate, _PY_SIGNALS_PENDING_BIT); return -1; } return 0; @@ -783,9 +797,30 @@ _make_pending_calls(struct _pending_calls *pending) return 0; } +static void +signal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + _Py_set_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT); +#else + _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); +#endif +} + +static void +unsignal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + _Py_unset_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT); +#else + _Py_unset_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); +#endif +} + static int -make_pending_calls(PyInterpreterState *interp) +make_pending_calls(PyThreadState *tstate) { + PyInterpreterState *interp = tstate->interp; struct _pending_calls *pending = &interp->ceval.pending; struct _pending_calls *pending_main = &_PyRuntime.ceval.pending_mainthread; @@ -811,12 +846,12 @@ make_pending_calls(PyInterpreterState *interp) /* unsignal before starting to call callbacks, so that any callback added in-between re-signals */ - UNSIGNAL_PENDING_CALLS(interp); + unsignal_pending_calls(tstate, interp); if (_make_pending_calls(pending) != 0) { pending->busy = 0; /* There might not be more calls to make, but we play it safe. */ - SIGNAL_PENDING_CALLS(interp); + signal_pending_calls(tstate, interp); return -1; } @@ -824,7 +859,7 @@ make_pending_calls(PyInterpreterState *interp) if (_make_pending_calls(pending_main) != 0) { pending->busy = 0; /* There might not be more calls to make, but we play it safe. */ - SIGNAL_PENDING_CALLS(interp); + signal_pending_calls(tstate, interp); return -1; } } @@ -833,13 +868,37 @@ make_pending_calls(PyInterpreterState *interp) return 0; } +void +_Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit) +{ + _PyRuntimeState *runtime = &_PyRuntime; + + HEAD_LOCK(runtime); + for (PyThreadState *tstate = interp->threads.head; tstate != NULL; tstate = tstate->next) { + _Py_set_eval_breaker_bit(tstate, bit); + } + HEAD_UNLOCK(runtime); +} + +void +_Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit) +{ + _PyRuntimeState *runtime = &_PyRuntime; + + HEAD_LOCK(runtime); + for (PyThreadState *tstate = interp->threads.head; tstate != NULL; tstate = tstate->next) { + _Py_unset_eval_breaker_bit(tstate, bit); + } + HEAD_UNLOCK(runtime); +} + void _Py_FinishPendingCalls(PyThreadState *tstate) { assert(PyGILState_Check()); assert(_PyThreadState_CheckConsistency(tstate)); - if (make_pending_calls(tstate->interp) < 0) { + if (make_pending_calls(tstate) < 0) { PyObject *exc = _PyErr_GetRaisedException(tstate); PyErr_BadInternalCall(); _PyErr_ChainExceptions1(exc); @@ -862,7 +921,7 @@ _PyEval_MakePendingCalls(PyThreadState *tstate) } } - res = make_pending_calls(tstate->interp); + res = make_pending_calls(tstate); if (res != 0) { return res; } @@ -955,11 +1014,11 @@ _PyEval_InitState(PyInterpreterState *interp) int _Py_HandlePending(PyThreadState *tstate) { - PyInterpreterState *interp = tstate->interp; + uintptr_t breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); /* Stop-the-world */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_EVAL_PLEASE_STOP_BIT)) { - _Py_set_eval_breaker_bit(interp, _PY_EVAL_PLEASE_STOP_BIT, 0); + if ((breaker & _PY_EVAL_PLEASE_STOP_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_PLEASE_STOP_BIT); _PyThreadState_Suspend(tstate); /* The attach blocks until the stop-the-world event is complete. */ @@ -967,35 +1026,35 @@ _Py_HandlePending(PyThreadState *tstate) } /* Pending signals */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_SIGNALS_PENDING_BIT)) { + if ((breaker & _PY_SIGNALS_PENDING_BIT) != 0) { if (handle_signals(tstate) != 0) { return -1; } } /* Pending calls */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_CALLS_TO_DO_BIT)) { - if (make_pending_calls(interp) != 0) { + if ((breaker & _PY_CALLS_TO_DO_BIT) != 0) { + if (make_pending_calls(tstate) != 0) { return -1; } } #ifdef Py_GIL_DISABLED /* Objects with refcounts to merge */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_EVAL_EXPLICIT_MERGE_BIT)) { - _Py_set_eval_breaker_bit(interp, _PY_EVAL_EXPLICIT_MERGE_BIT, 0); + if ((breaker & _PY_EVAL_EXPLICIT_MERGE_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_EXPLICIT_MERGE_BIT); _Py_brc_merge_refcounts(tstate); } #endif /* GC scheduled to run */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_GC_SCHEDULED_BIT)) { - _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 0); + if ((breaker & _PY_GC_SCHEDULED_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT); _Py_RunGC(tstate); } /* GIL drop request */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_GIL_DROP_REQUEST_BIT)) { + if ((breaker & _PY_GIL_DROP_REQUEST_BIT) != 0) { /* Give another thread a chance */ _PyThreadState_Detach(tstate); @@ -1005,11 +1064,10 @@ _Py_HandlePending(PyThreadState *tstate) } /* Check for asynchronous exception. */ - if (_Py_eval_breaker_bit_is_set(interp, _PY_ASYNC_EXCEPTION_BIT)) { - _Py_set_eval_breaker_bit(interp, _PY_ASYNC_EXCEPTION_BIT, 0); - if (tstate->async_exc != NULL) { - PyObject *exc = tstate->async_exc; - tstate->async_exc = NULL; + if ((breaker & _PY_ASYNC_EXCEPTION_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_ASYNC_EXCEPTION_BIT); + PyObject *exc = _Py_atomic_exchange_ptr(&tstate->async_exc, NULL); + if (exc != NULL) { _PyErr_SetNone(tstate, exc); Py_DECREF(exc); return -1; @@ -1017,4 +1075,3 @@ _Py_HandlePending(PyThreadState *tstate) } return 0; } - diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c2550f53ad6eaa..22992aa09e1f38 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -86,6 +86,12 @@ #define PRE_DISPATCH_GOTO() ((void)0) #endif +#ifdef Py_GIL_DISABLED +#define QSBR_QUIESCENT_STATE(tstate) _Py_qsbr_quiescent_state(((_PyThreadStateImpl *)tstate)->qsbr) +#else +#define QSBR_QUIESCENT_STATE(tstate) +#endif + /* Do interpreter dispatch accounting for tracing and instrumentation */ #define DISPATCH() \ @@ -117,7 +123,8 @@ #define CHECK_EVAL_BREAKER() \ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & _PY_EVAL_EVENTS_MASK) { \ + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { \ if (_Py_HandlePending(tstate) != 0) { \ GOTO_ERROR(error); \ } \ @@ -289,11 +296,19 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define ADAPTIVE_COUNTER_IS_MAX(COUNTER) \ (((COUNTER) >> ADAPTIVE_BACKOFF_BITS) == ((1 << MAX_BACKOFF_VALUE) - 1)) +#ifdef Py_GIL_DISABLED +#define DECREMENT_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + /* gh-115999 tracks progress on addressing this. */ \ + static_assert(0, "The specializing interpreter is not yet thread-safe"); \ + } while (0); +#else #define DECREMENT_ADAPTIVE_COUNTER(COUNTER) \ do { \ assert(!ADAPTIVE_COUNTER_IS_ZERO((COUNTER))); \ (COUNTER) -= (1 << ADAPTIVE_BACKOFF_BITS); \ } while (0); +#endif #define INCREMENT_ADAPTIVE_COUNTER(COUNTER) \ do { \ @@ -341,13 +356,6 @@ do { \ } \ } while (0); -typedef PyObject *(*convertion_func_ptr)(PyObject *); - -static const convertion_func_ptr CONVERSION_FUNCTIONS[4] = { - [FVC_STR] = PyObject_Str, - [FVC_REPR] = PyObject_Repr, - [FVC_ASCII] = PyObject_ASCII -}; // GH-89279: Force inlining by using a macro. #if defined(_MSC_VER) && SIZEOF_INT == 4 @@ -365,12 +373,6 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { tstate->py_recursion_remaining++; } -/* Marker to specify tier 1 only instructions */ -#define TIER_ONE_ONLY - -/* Marker to specify tier 2 only instructions */ -#define TIER_TWO_ONLY - /* Implementation of "macros" that modify the instruction pointer, * stack pointer, or frame pointer. * These need to treated differently by tier 1 and 2. @@ -387,7 +389,36 @@ stack_pointer = _PyFrame_GetStackPointer(frame); /* Tier-switching macros. */ -#define GOTO_TIER_TWO() goto enter_tier_two; +#ifdef _Py_JIT +#define GOTO_TIER_TWO(EXECUTOR) \ +do { \ + jit_func jitted = (EXECUTOR)->jit_code; \ + next_instr = jitted(frame, stack_pointer, tstate); \ + Py_DECREF(tstate->previous_executor); \ + tstate->previous_executor = NULL; \ + frame = tstate->current_frame; \ + if (next_instr == NULL) { \ + goto resume_with_error; \ + } \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + DISPATCH(); \ +} while (0) +#else +#define GOTO_TIER_TWO(EXECUTOR) \ +do { \ + next_uop = (EXECUTOR)->trace; \ + assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); \ + goto enter_tier_two; \ +} while (0) +#endif + +#define GOTO_TIER_ONE(TARGET) \ +do { \ + Py_DECREF(tstate->previous_executor); \ + tstate->previous_executor = NULL; \ + next_instr = target; \ + DISPATCH(); \ +} while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) diff --git a/Python/compile.c b/Python/compile.c index d857239690e7b5..6b17f3bcaf2264 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -921,11 +921,10 @@ dict_add_o(PyObject *dict, PyObject *o) PyObject *v; Py_ssize_t arg; - v = PyDict_GetItemWithError(dict, o); + if (PyDict_GetItemRef(dict, o, &v) < 0) { + return ERROR; + } if (!v) { - if (PyErr_Occurred()) { - return ERROR; - } arg = PyDict_GET_SIZE(dict); v = PyLong_FromSsize_t(arg); if (!v) { @@ -935,10 +934,10 @@ dict_add_o(PyObject *dict, PyObject *o) Py_DECREF(v); return ERROR; } - Py_DECREF(v); } else arg = PyLong_AsLong(v); + Py_DECREF(v); return arg; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 11e2a1fe85d51d..56ee93862743d5 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -17,7 +17,7 @@ if (_Py_emscripten_signal_clock == 0) goto deoptimize; _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) goto deoptimize; @@ -37,6 +37,102 @@ break; } + case _LOAD_FAST_0: { + PyObject *value; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_1: { + PyObject *value; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_2: { + PyObject *value; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_3: { + PyObject *value; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_4: { + PyObject *value; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_5: { + PyObject *value; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_6: { + PyObject *value; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_7: { + PyObject *value; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + case _LOAD_FAST: { PyObject *value; oparg = CURRENT_OPARG(); @@ -69,6 +165,86 @@ break; } + case _STORE_FAST_0: { + PyObject *value; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_1: { + PyObject *value; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_2: { + PyObject *value; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_3: { + PyObject *value; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_4: { + PyObject *value; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_5: { + PyObject *value; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_6: { + PyObject *value; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + + case _STORE_FAST_7: { + PyObject *value; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + break; + } + case _STORE_FAST: { PyObject *value; oparg = CURRENT_OPARG(); @@ -141,7 +317,7 @@ case _TO_BOOL_BOOL: { PyObject *value; value = stack_pointer[-1]; - if (!PyBool_Check(value)) goto deoptimize; + if (!PyBool_Check(value)) goto side_exit; STAT_INC(TO_BOOL, hit); break; } @@ -150,7 +326,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyLong_CheckExact(value)) goto deoptimize; + if (!PyLong_CheckExact(value)) goto side_exit; STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -168,7 +344,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyList_CheckExact(value)) goto deoptimize; + if (!PyList_CheckExact(value)) goto side_exit; STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); @@ -181,7 +357,7 @@ PyObject *res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - if (!Py_IsNone(value)) goto deoptimize; + if (!Py_IsNone(value)) goto side_exit; STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; @@ -192,7 +368,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyUnicode_CheckExact(value)) goto deoptimize; + if (!PyUnicode_CheckExact(value)) goto side_exit; STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -214,7 +390,7 @@ uint32_t version = (uint32_t)CURRENT_OPERAND(); // This one is a bit weird, because we expect *some* failures: assert(version); - if (Py_TYPE(value)->tp_version_tag != version) goto deoptimize; + if (Py_TYPE(value)->tp_version_tag != version) goto side_exit; STAT_INC(TO_BOOL, hit); Py_DECREF(value); res = Py_True; @@ -238,8 +414,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyLong_CheckExact(left)) goto deoptimize; - if (!PyLong_CheckExact(right)) goto deoptimize; + if (!PyLong_CheckExact(left)) goto side_exit; + if (!PyLong_CheckExact(right)) goto side_exit; break; } @@ -296,8 +472,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyFloat_CheckExact(left)) goto deoptimize; - if (!PyFloat_CheckExact(right)) goto deoptimize; + if (!PyFloat_CheckExact(left)) goto side_exit; + if (!PyFloat_CheckExact(right)) goto side_exit; break; } @@ -354,8 +530,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyUnicode_CheckExact(left)) goto deoptimize; - if (!PyUnicode_CheckExact(right)) goto deoptimize; + if (!PyUnicode_CheckExact(left)) goto side_exit; + if (!PyUnicode_CheckExact(right)) goto side_exit; break; } @@ -1534,7 +1710,7 @@ Py_DECREF(self); if (attr == NULL) goto pop_3_error_tier_two; stack_pointer[-3] = attr; - stack_pointer += -2 + ((0) ? 1 : 0); + stack_pointer += -2; break; } @@ -1623,7 +1799,7 @@ uint32_t type_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - if (tp->tp_version_tag != type_version) goto deoptimize; + if (tp->tp_version_tag != type_version) goto side_exit; break; } @@ -1637,11 +1813,11 @@ break; } - case _LOAD_ATTR_INSTANCE_VALUE: { + case _LOAD_ATTR_INSTANCE_VALUE_0: { PyObject *owner; PyObject *attr; PyObject *null = NULL; - oparg = CURRENT_OPARG(); + (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); @@ -1652,19 +1828,39 @@ null = NULL; Py_DECREF(owner); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); break; } + case _LOAD_ATTR_INSTANCE_VALUE_1: { + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + (void)null; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)CURRENT_OPERAND(); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + attr = _PyDictOrValues_GetValues(dorv)->values[index]; + if (attr == NULL) goto deoptimize; + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + break; + } + + /* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */ + case _CHECK_ATTR_MODULE: { PyObject *owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND(); + uint32_t dict_version = (uint32_t)CURRENT_OPERAND(); if (!PyModule_CheckExact(owner)) goto deoptimize; PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - if (dict->ma_keys->dk_version != type_version) goto deoptimize; + if (dict->ma_keys->dk_version != dict_version) goto deoptimize; break; } @@ -1735,11 +1931,11 @@ break; } - case _LOAD_ATTR_SLOT: { + case _LOAD_ATTR_SLOT_0: { PyObject *owner; PyObject *attr; PyObject *null = NULL; - oparg = CURRENT_OPARG(); + (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; @@ -1750,11 +1946,31 @@ null = NULL; Py_DECREF(owner); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); break; } + case _LOAD_ATTR_SLOT_1: { + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + (void)null; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)CURRENT_OPERAND(); + char *addr = (char *)owner + index; + attr = *(PyObject **)addr; + if (attr == NULL) goto deoptimize; + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + break; + } + + /* _LOAD_ATTR_SLOT is split on (oparg & 1) */ + case _CHECK_ATTR_CLASS: { PyObject *owner; owner = stack_pointer[-1]; @@ -1765,11 +1981,11 @@ break; } - case _LOAD_ATTR_CLASS: { + case _LOAD_ATTR_CLASS_0: { PyObject *owner; PyObject *attr; PyObject *null = NULL; - oparg = CURRENT_OPARG(); + (void)null; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); STAT_INC(LOAD_ATTR, hit); @@ -1778,11 +1994,29 @@ null = NULL; Py_DECREF(owner); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); break; } + case _LOAD_ATTR_CLASS_1: { + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + (void)null; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)CURRENT_OPERAND(); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + null = NULL; + Py_DECREF(owner); + stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + break; + } + + /* _LOAD_ATTR_CLASS is split on (oparg & 1) */ + /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */ /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ @@ -1866,8 +2100,6 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyFloat_CheckExact(left)) goto deoptimize; - if (!PyFloat_CheckExact(right)) goto deoptimize; STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); @@ -1889,8 +2121,6 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyLong_CheckExact(left)) goto deoptimize; - if (!PyLong_CheckExact(right)) goto deoptimize; if (!_PyLong_IsCompact((PyLongObject *)left)) goto deoptimize; if (!_PyLong_IsCompact((PyLongObject *)right)) goto deoptimize; STAT_INC(COMPARE_OP, hit); @@ -1916,8 +2146,6 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyUnicode_CheckExact(left)) goto deoptimize; - if (!PyUnicode_CheckExact(right)) goto deoptimize; STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left, right); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); @@ -2013,8 +2241,6 @@ break; } - /* _JUMP_BACKWARD is not a viable micro-op for tier 2 */ - /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ @@ -2201,6 +2427,7 @@ _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; + if (seq == NULL) goto deoptimize; if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) goto deoptimize; break; } @@ -2321,7 +2548,7 @@ GOTO_ERROR(error); } Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); + res = PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -2364,7 +2591,7 @@ GOTO_ERROR(error); } Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); + res = PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -2466,8 +2693,8 @@ assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); self = owner; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } @@ -2486,8 +2713,8 @@ attr = Py_NewRef(descr); self = owner; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } @@ -2503,7 +2730,6 @@ Py_DECREF(owner); attr = Py_NewRef(descr); stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); break; } @@ -2520,7 +2746,6 @@ Py_DECREF(owner); attr = Py_NewRef(descr); stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); break; } @@ -2549,8 +2774,8 @@ attr = Py_NewRef(descr); self = owner; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } @@ -2617,6 +2842,136 @@ break; } + case _INIT_CALL_PY_EXACT_ARGS_0: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_1: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_2: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_3: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_4: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer += -1 - oparg; + break; + } + case _INIT_CALL_PY_EXACT_ARGS: { PyObject **args; PyObject *self_or_null; @@ -2662,7 +3017,6 @@ goto exit_unwind; } #endif - stack_pointer += ((0) ? 1 : 0); break; } @@ -3216,9 +3570,9 @@ PyObject *result; oparg = CURRENT_OPARG(); value = stack_pointer[-1]; - convertion_func_ptr conv_fn; + conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; + conv_fn = _PyEval_ConversionFuncs[oparg]; result = conv_fn(value); Py_DECREF(value); if (result == NULL) goto pop_1_error_tier_two; @@ -3318,7 +3672,7 @@ PyObject *flag; flag = stack_pointer[-1]; stack_pointer += -1; - if (!Py_IsTrue(flag)) goto deoptimize; + if (!Py_IsTrue(flag)) goto side_exit; assert(Py_IsTrue(flag)); break; } @@ -3327,7 +3681,7 @@ PyObject *flag; flag = stack_pointer[-1]; stack_pointer += -1; - if (!Py_IsFalse(flag)) goto deoptimize; + if (!Py_IsFalse(flag)) goto side_exit; assert(Py_IsFalse(flag)); break; } @@ -3338,7 +3692,7 @@ stack_pointer += -1; if (!Py_IsNone(val)) { Py_DECREF(val); - if (1) goto deoptimize; + if (1) goto side_exit; } break; } @@ -3347,20 +3701,21 @@ PyObject *val; val = stack_pointer[-1]; stack_pointer += -1; - if (Py_IsNone(val)) goto deoptimize; + if (Py_IsNone(val)) goto side_exit; Py_DECREF(val); break; } case _JUMP_TO_TOP: { - next_uop = current_executor->trace; + #ifndef _Py_JIT + next_uop = ¤t_executor->trace[1]; + #endif CHECK_EVAL_BREAKER(); break; } case _SET_IP: { PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); - TIER_TWO_ONLY frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; break; } @@ -3377,13 +3732,11 @@ } case _EXIT_TRACE: { - TIER_TWO_ONLY - if (1) goto deoptimize; + if (1) goto side_exit; break; } case _CHECK_VALIDITY: { - TIER_TWO_ONLY if (!current_executor->vm_data.valid) goto deoptimize; break; } @@ -3391,7 +3744,6 @@ case _LOAD_CONST_INLINE: { PyObject *value; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - TIER_TWO_ONLY value = Py_NewRef(ptr); stack_pointer[0] = value; stack_pointer += 1; @@ -3401,18 +3753,27 @@ case _LOAD_CONST_INLINE_BORROW: { PyObject *value; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - TIER_TWO_ONLY value = ptr; stack_pointer[0] = value; stack_pointer += 1; break; } + case _POP_TOP_LOAD_CONST_INLINE_BORROW: { + PyObject *pop; + PyObject *value; + pop = stack_pointer[-1]; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + Py_DECREF(pop); + value = ptr; + stack_pointer[-1] = value; + break; + } + case _LOAD_CONST_INLINE_WITH_NULL: { PyObject *value; PyObject *null; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - TIER_TWO_ONLY value = Py_NewRef(ptr); null = NULL; stack_pointer[0] = value; @@ -3425,7 +3786,6 @@ PyObject *value; PyObject *null; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - TIER_TWO_ONLY value = ptr; null = NULL; stack_pointer[0] = value; @@ -3436,14 +3796,12 @@ case _CHECK_GLOBALS: { PyObject *dict = (PyObject *)CURRENT_OPERAND(); - TIER_TWO_ONLY if (GLOBALS() != dict) goto deoptimize; break; } case _CHECK_BUILTINS: { PyObject *dict = (PyObject *)CURRENT_OPERAND(); - TIER_TWO_ONLY if (BUILTINS() != dict) goto deoptimize; break; } @@ -3457,9 +3815,59 @@ break; } + case _COLD_EXIT: { + oparg = CURRENT_OPARG(); + _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor; + _PyExitData *exit = &previous->exits[oparg]; + exit->temperature++; + PyCodeObject *code = _PyFrame_GetCode(frame); + _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; + if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { + GOTO_TIER_ONE(target); + } + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } else { + int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); + if (optimized <= 0) { + int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold; + exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp; + if (optimized < 0) { + Py_DECREF(previous); + tstate->previous_executor = Py_None; + if (1) goto error_tier_two; + } + GOTO_TIER_ONE(target); + } + } + /* We need two references. One to store in exit->executor and + * one to keep the executor alive when executing. */ + Py_INCREF(executor); + exit->executor = executor; + GOTO_TIER_TWO(executor); + break; + } + + case _START_EXECUTOR: { + PyObject *executor = (PyObject *)CURRENT_OPERAND(); + Py_DECREF(tstate->previous_executor); + tstate->previous_executor = NULL; + #ifndef _Py_JIT + current_executor = (_PyExecutorObject*)executor; + #endif + break; + } + + case _FATAL_ERROR: { + assert(0); + Py_FatalError("Fatal error uop executed."); + break; + } + case _CHECK_VALIDITY_AND_SET_IP: { PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); - TIER_TWO_ONLY if (!current_executor->vm_data.valid) goto deoptimize; frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; break; diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 4d9ba9eceb8637..2f47e47bf9d29d 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -665,12 +665,6 @@ translate_jump_labels_to_targets(basicblock *entryblock) return SUCCESS; } -int -_PyCfg_JumpLabelsToTargets(cfg_builder *g) -{ - return translate_jump_labels_to_targets(g->g_entryblock); -} - static int mark_except_handlers(basicblock *entryblock) { #ifndef NDEBUG @@ -2790,3 +2784,14 @@ _PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g, return SUCCESS; } + +/* This is used by _PyCompile_Assemble to fill in the jump and exception + * targets in a synthetic CFG (which is not the ouptut of the builtin compiler). + */ +int +_PyCfg_JumpLabelsToTargets(cfg_builder *g) +{ + RETURN_IF_ERROR(translate_jump_labels_to_targets(g->g_entryblock)); + RETURN_IF_ERROR(label_exception_targets(g->g_entryblock)); + return SUCCESS; +} diff --git a/Python/gc.c b/Python/gc.c index c6831f4c74bcac..ea3b596d1713df 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -12,6 +12,7 @@ #include "pycore_object_alloc.h" // _PyObject_MallocWithType() #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_time.h" // _PyTime_PerfCounterUnchecked() #include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" @@ -1285,7 +1286,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) PyGC_Head unreachable; /* non-problematic unreachable trash */ PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ PyGC_Head *gc; - _PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ + PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ GCState *gcstate = &tstate->interp->gc; // gc_collect_main() must not be called before _PyGC_Init @@ -1326,7 +1327,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) if (gcstate->debug & _PyGC_DEBUG_STATS) { PySys_WriteStderr("gc: collecting generation %d...\n", generation); show_stats_each_generations(gcstate); - t1 = _PyTime_GetPerfCounter(); + t1 = _PyTime_PerfCounterUnchecked(); } if (PyDTrace_GC_START_ENABLED()) { @@ -1427,7 +1428,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) debug_cycle("uncollectable", FROM_GC(gc)); } if (gcstate->debug & _PyGC_DEBUG_STATS) { - double d = _PyTime_AsSecondsDouble(_PyTime_GetPerfCounter() - t1); + double d = PyTime_AsSecondsDouble(_PyTime_PerfCounterUnchecked() - t1); PySys_WriteStderr( "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", n+m, n, d); @@ -1771,9 +1772,12 @@ PyObject_IS_GC(PyObject *obj) } void -_Py_ScheduleGC(PyInterpreterState *interp) +_Py_ScheduleGC(PyThreadState *tstate) { - _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 1); + if (!_Py_eval_breaker_bit_is_set(tstate, _PY_GC_SCHEDULED_BIT)) + { + _Py_set_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT); + } } void @@ -1794,7 +1798,7 @@ _PyObject_GC_Link(PyObject *op) !_Py_atomic_load_int_relaxed(&gcstate->collecting) && !_PyErr_Occurred(tstate)) { - _Py_ScheduleGC(tstate->interp); + _Py_ScheduleGC(tstate); } } diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 3dc1dc19182eb4..18893c6c391fff 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -11,6 +11,7 @@ #include "pycore_object_stack.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_time.h" // _PyTime_GetPerfCounter() #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" @@ -23,6 +24,11 @@ typedef struct _gc_runtime_state GCState; # define GC_DEBUG #endif +// Each thread buffers the count of allocated objects in a thread-local +// variable up to +/- this amount to reduce the overhead of updating +// the global count. +#define LOCAL_ALLOC_COUNT_THRESHOLD 512 + // Automatically choose the generation that needs collecting. #define GENERATION_AUTO (-1) @@ -318,6 +324,23 @@ merge_all_queued_objects(PyInterpreterState *interp, struct collection_state *st HEAD_UNLOCK(&_PyRuntime); } +static void +process_delayed_frees(PyInterpreterState *interp) +{ + // In STW status, we can observe the latest write sequence by + // advancing the write sequence immediately. + _Py_qsbr_advance(&interp->qsbr); + _PyThreadStateImpl *current_tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + _Py_qsbr_quiescent_state(current_tstate->qsbr); + HEAD_LOCK(&_PyRuntime); + PyThreadState *tstate = interp->threads.head; + while (tstate != NULL) { + _PyMem_ProcessDelayed(tstate); + tstate = (PyThreadState *)tstate->next; + } + HEAD_UNLOCK(&_PyRuntime); +} + // Subtract an incoming reference from the computed "gc_refs" refcount. static int visit_decref(PyObject *op, void *arg) @@ -959,12 +982,48 @@ gc_should_collect(GCState *gcstate) gcstate->generations[1].threshold == 0); } +static void +record_allocation(PyThreadState *tstate) +{ + struct _gc_thread_state *gc = &((_PyThreadStateImpl *)tstate)->gc; + + // We buffer the allocation count to avoid the overhead of atomic + // operations for every allocation. + gc->alloc_count++; + if (gc->alloc_count >= LOCAL_ALLOC_COUNT_THRESHOLD) { + // TODO: Use Py_ssize_t for the generation count. + GCState *gcstate = &tstate->interp->gc; + _Py_atomic_add_int(&gcstate->generations[0].count, (int)gc->alloc_count); + gc->alloc_count = 0; + + if (gc_should_collect(gcstate) && + !_Py_atomic_load_int_relaxed(&gcstate->collecting)) + { + _Py_ScheduleGC(tstate); + } + } +} + +static void +record_deallocation(PyThreadState *tstate) +{ + struct _gc_thread_state *gc = &((_PyThreadStateImpl *)tstate)->gc; + + gc->alloc_count--; + if (gc->alloc_count <= -LOCAL_ALLOC_COUNT_THRESHOLD) { + GCState *gcstate = &tstate->interp->gc; + _Py_atomic_add_int(&gcstate->generations[0].count, (int)gc->alloc_count); + gc->alloc_count = 0; + } +} + static void gc_collect_internal(PyInterpreterState *interp, struct collection_state *state) { _PyEval_StopTheWorld(interp); // merge refcounts for all queued objects merge_all_queued_objects(interp, state); + process_delayed_frees(interp); // Find unreachable objects int err = deduce_unreachable_heap(interp, state); @@ -981,6 +1040,9 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state) } } + // Record the number of live GC objects + interp->gc.long_lived_total = state->long_lived_total; + // Clear weakrefs and enqueue callbacks (but do not call them). clear_weakrefs(state); _PyEval_StartTheWorld(interp); @@ -1028,7 +1090,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) int i; Py_ssize_t m = 0; /* # objects collected */ Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ - _PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ + PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ GCState *gcstate = &tstate->interp->gc; // gc_collect_main() must not be called before _PyGC_Init @@ -1064,7 +1126,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) if (gcstate->debug & _PyGC_DEBUG_STATS) { PySys_WriteStderr("gc: collecting generation %d...\n", generation); show_stats_each_generations(gcstate); - t1 = _PyTime_GetPerfCounter(); + t1 = _PyTime_PerfCounterUnchecked(); } if (PyDTrace_GC_START_ENABLED()) { @@ -1090,10 +1152,9 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) m = state.collected; n = state.uncollectable; - gcstate->long_lived_total = state.long_lived_total; if (gcstate->debug & _PyGC_DEBUG_STATS) { - double d = _PyTime_AsSecondsDouble(_PyTime_GetPerfCounter() - t1); + double d = PyTime_AsSecondsDouble(_PyTime_PerfCounterUnchecked() - t1); PySys_WriteStderr( "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", n+m, n, d); @@ -1522,23 +1583,18 @@ PyObject_IS_GC(PyObject *obj) } void -_Py_ScheduleGC(PyInterpreterState *interp) +_Py_ScheduleGC(PyThreadState *tstate) { - _Py_set_eval_breaker_bit(interp, _PY_GC_SCHEDULED_BIT, 1); + if (!_Py_eval_breaker_bit_is_set(tstate, _PY_GC_SCHEDULED_BIT)) + { + _Py_set_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT); + } } void _PyObject_GC_Link(PyObject *op) { - PyThreadState *tstate = _PyThreadState_GET(); - GCState *gcstate = &tstate->interp->gc; - gcstate->generations[0].count++; - - if (gc_should_collect(gcstate) && - !_Py_atomic_load_int_relaxed(&gcstate->collecting)) - { - _Py_ScheduleGC(tstate->interp); - } + record_allocation(_PyThreadState_GET()); } void @@ -1564,7 +1620,7 @@ gc_alloc(PyTypeObject *tp, size_t basicsize, size_t presize) ((PyObject **)mem)[1] = NULL; } PyObject *op = (PyObject *)(mem + presize); - _PyObject_GC_Link(op); + record_allocation(tstate); return op; } @@ -1646,29 +1702,22 @@ PyObject_GC_Del(void *op) PyErr_SetRaisedException(exc); #endif } - GCState *gcstate = get_gc_state(); - if (gcstate->generations[0].count > 0) { - gcstate->generations[0].count--; - } + + record_deallocation(_PyThreadState_GET()); + PyObject_Free(((char *)op)-presize); } int PyObject_GC_IsTracked(PyObject* obj) { - if (_PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) { - return 1; - } - return 0; + return _PyObject_GC_IS_TRACKED(obj); } int PyObject_GC_IsFinalized(PyObject *obj) { - if (_PyObject_IS_GC(obj) && _PyGC_FINALIZED(obj)) { - return 1; - } - return 0; + return _PyGC_FINALIZED(obj); } struct custom_visitor_args { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6c19adc60c690f..e612c9e4c37632 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -40,7 +40,7 @@ GOTO_ERROR(error); } Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); + res = PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -86,7 +86,7 @@ GOTO_ERROR(error); } Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); + res = PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -104,6 +104,7 @@ INSTRUCTION_STATS(BINARY_OP); PREDICTED(BINARY_OP); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *rhs; PyObject *lhs; PyObject *res; @@ -112,7 +113,7 @@ lhs = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -242,7 +243,6 @@ /* Skip 1 cache entry */ // _BINARY_OP_INPLACE_ADD_UNICODE { - TIER_ONE_ONLY assert(next_instr->op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(next_instr->op.arg); DEOPT_IF(*target_local != left, BINARY_OP); @@ -421,6 +421,7 @@ INSTRUCTION_STATS(BINARY_SUBSCR); PREDICTED(BINARY_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *sub; PyObject *container; PyObject *res; @@ -429,7 +430,7 @@ container = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -739,9 +740,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); - TIER_ONE_ONLY assert(0 && "Executing a cache."); - Py_UNREACHABLE(); + Py_FatalError("Executing a cache."); + DISPATCH(); } TARGET(CALL) { @@ -750,6 +751,7 @@ INSTRUCTION_STATS(CALL); PREDICTED(CALL); _Py_CODEUNIT *this_instr = next_instr - 4; + (void)this_instr; PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -760,7 +762,7 @@ callable = stack_pointer[-2 - oparg]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -1004,7 +1006,6 @@ } #endif } - stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -1181,6 +1182,7 @@ INSTRUCTION_STATS(CALL_FUNCTION_EX); PREDICTED(CALL_FUNCTION_EX); _Py_CODEUNIT *this_instr = next_instr - 1; + (void)this_instr; PyObject *kwargs = NULL; PyObject *callargs; PyObject *func; @@ -1341,6 +1343,7 @@ INSTRUCTION_STATS(CALL_KW); PREDICTED(CALL_KW); _Py_CODEUNIT *this_instr = next_instr - 1; + (void)this_instr; PyObject *kwnames; PyObject **args; PyObject *self_or_null; @@ -1477,7 +1480,6 @@ args = &stack_pointer[-oparg]; self = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - TIER_ONE_ONLY assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append, CALL); @@ -1754,7 +1756,6 @@ } #endif } - stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -1944,6 +1945,7 @@ TARGET(CLEANUP_THROW) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); PyObject *exc_value; @@ -1954,7 +1956,6 @@ exc_value = stack_pointer[-1]; last_sent_val = stack_pointer[-2]; sub_iter = stack_pointer[-3]; - TIER_ONE_ONLY assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { @@ -1981,6 +1982,7 @@ INSTRUCTION_STATS(COMPARE_OP); PREDICTED(COMPARE_OP); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *right; PyObject *left; PyObject *res; @@ -1989,7 +1991,7 @@ left = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -2027,20 +2029,26 @@ PyObject *right; PyObject *left; PyObject *res; - /* Skip 1 cache entry */ + // _GUARD_BOTH_FLOAT right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. + { + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT + { + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + } stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -2054,24 +2062,30 @@ PyObject *right; PyObject *left; PyObject *res; - /* Skip 1 cache entry */ + // _GUARD_BOTH_INT right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && + { + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + } stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -2085,21 +2099,27 @@ PyObject *right; PyObject *left; PyObject *res; - /* Skip 1 cache entry */ + // _GUARD_BOTH_UNICODE right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. + { + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left, right); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + } stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -2131,9 +2151,9 @@ PyObject *value; PyObject *result; value = stack_pointer[-1]; - convertion_func_ptr conv_fn; + conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; + conv_fn = _PyEval_ConversionFuncs[oparg]; result = conv_fn(value); Py_DECREF(value); if (result == NULL) goto pop_1_error; @@ -2315,13 +2335,13 @@ TARGET(END_ASYNC_FOR) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(END_ASYNC_FOR); PyObject *exc; PyObject *awaitable; exc = stack_pointer[-1]; awaitable = stack_pointer[-2]; - TIER_ONE_ONLY assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { Py_DECREF(awaitable); @@ -2366,15 +2386,16 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); - TIER_ONE_ONLY CHECK_EVAL_BREAKER(); PyCodeObject *code = _PyFrame_GetCode(frame); - current_executor = code->co_executors->executors[oparg & 255]; - assert(current_executor->vm_data.index == INSTR_OFFSET() - 1); - assert(current_executor->vm_data.code == code); - assert(current_executor->vm_data.valid); - Py_INCREF(current_executor); - GOTO_TIER_TWO(); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + Py_INCREF(executor); + GOTO_TIER_TWO(executor); DISPATCH(); } @@ -2399,7 +2420,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); - TIER_ONE_ONLY assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -2452,13 +2472,14 @@ INSTRUCTION_STATS(FOR_ITER); PREDICTED(FOR_ITER); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *iter; PyObject *next; // _SPECIALIZE_FOR_ITER iter = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -2541,7 +2562,7 @@ assert(Py_TYPE(iter) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); PyListObject *seq = it->it_seq; - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { it->it_index = -1; #ifndef Py_GIL_DISABLED if (seq != NULL) { @@ -2848,7 +2869,6 @@ PyObject *from; PyObject *res; from = stack_pointer[-1]; - TIER_ONE_ONLY PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; @@ -2866,7 +2886,6 @@ PyObject *res; fromlist = stack_pointer[-1]; level = stack_pointer[-2]; - TIER_ONE_ONLY PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_name(tstate, frame, name, fromlist, level); Py_DECREF(level); @@ -2879,6 +2898,7 @@ TARGET(INSTRUMENTED_CALL) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); /* Skip 3 cache entries */ @@ -2904,6 +2924,7 @@ TARGET(INSTRUMENTED_CALL_KW) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); int is_meth = PEEK(oparg + 2) != NULL; @@ -2920,13 +2941,13 @@ TARGET(INSTRUMENTED_END_FOR) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_FOR); PyObject *value; PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - TIER_ONE_ONLY /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -2943,13 +2964,13 @@ TARGET(INSTRUMENTED_END_SEND) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); PyObject *value; PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - TIER_ONE_ONLY if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { @@ -2965,6 +2986,7 @@ TARGET(INSTRUMENTED_FOR_ITER) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); /* Skip 1 cache entry */ @@ -2997,6 +3019,7 @@ TARGET(INSTRUMENTED_INSTRUCTION) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); int next_opcode = _Py_call_instrumentation_instruction( @@ -3013,6 +3036,7 @@ TARGET(INSTRUMENTED_JUMP_BACKWARD) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); /* Skip 1 cache entry */ @@ -3023,6 +3047,7 @@ TARGET(INSTRUMENTED_JUMP_FORWARD) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); @@ -3031,6 +3056,7 @@ TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); /* Skip 1 cache entry */ @@ -3042,6 +3068,7 @@ TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); /* Skip 1 cache entry */ @@ -3058,6 +3085,7 @@ TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); /* Skip 1 cache entry */ @@ -3080,6 +3108,7 @@ TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); /* Skip 1 cache entry */ @@ -3102,6 +3131,7 @@ TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); /* Skip 1 cache entry */ @@ -3118,9 +3148,10 @@ TARGET(INSTRUMENTED_RESUME) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { @@ -3148,6 +3179,7 @@ TARGET(INSTRUMENTED_RETURN_CONST) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); @@ -3171,6 +3203,7 @@ TARGET(INSTRUMENTED_RETURN_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); PyObject *retval; @@ -3195,6 +3228,7 @@ TARGET(INSTRUMENTED_YIELD_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); PyObject *retval; @@ -3229,7 +3263,6 @@ INSTRUCTION_STATS(INTERPRETER_EXIT); PyObject *retval; retval = stack_pointer[-1]; - TIER_ONE_ONLY assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ @@ -3259,6 +3292,7 @@ TARGET(JUMP_BACKWARD) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); /* Skip 1 cache entry */ @@ -3283,13 +3317,13 @@ oparg >>= 8; start--; } - int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer); + _PyExecutorObject *executor; + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); if (optimized < 0) goto error; if (optimized) { - // Rewind and enter the executor: - assert(start->op.code == ENTER_EXECUTOR); - next_instr = start; - this_instr[1].cache &= OPTIMIZER_BITS_MASK; + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); } else { int backoff = this_instr[1].cache & OPTIMIZER_BITS_MASK; @@ -3311,7 +3345,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - TIER_ONE_ONLY /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -3325,7 +3358,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); - TIER_ONE_ONLY JUMPBY(oparg); DISPATCH(); } @@ -3387,6 +3419,7 @@ INSTRUCTION_STATS(LOAD_ATTR); PREDICTED(LOAD_ATTR); _Py_CODEUNIT *this_instr = next_instr - 10; + (void)this_instr; PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; @@ -3394,7 +3427,7 @@ owner = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); @@ -3593,8 +3626,8 @@ self = owner; } stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; DISPATCH(); } @@ -3628,8 +3661,8 @@ self = owner; } stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; DISPATCH(); } @@ -3675,8 +3708,8 @@ self = owner; } stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; DISPATCH(); } @@ -3692,11 +3725,11 @@ // _CHECK_ATTR_MODULE owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t dict_version = read_u32(&this_instr[2].cache); DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); + DEOPT_IF(dict->ma_keys->dk_version != dict_version, LOAD_ATTR); } // _LOAD_ATTR_MODULE { @@ -3747,7 +3780,6 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -3790,7 +3822,6 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); DISPATCH(); } @@ -4086,12 +4117,13 @@ INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED(LOAD_GLOBAL); _Py_CODEUNIT *this_instr = next_instr - 5; + (void)this_instr; PyObject *res; PyObject *null = NULL; // _SPECIALIZE_LOAD_GLOBAL { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); @@ -4283,6 +4315,7 @@ INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED(LOAD_SUPER_ATTR); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *class; PyObject *global_super; PyObject *self; @@ -4293,7 +4326,7 @@ global_super = stack_pointer[-3]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY + (void)counter; #if ENABLE_SPECIALIZATION int load_method = oparg & 1; if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -4308,7 +4341,6 @@ // _LOAD_SUPER_ATTR self = stack_pointer[-1]; { - TIER_ONE_ONLY if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( @@ -4376,7 +4408,7 @@ Py_DECREF(self); if (attr == NULL) goto pop_3_error; stack_pointer[-3] = attr; - stack_pointer += -2 + ((0) ? 1 : 0); + stack_pointer += -2; DISPATCH(); } @@ -4571,6 +4603,7 @@ TARGET(POP_JUMP_IF_FALSE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); PyObject *cond; @@ -4588,6 +4621,7 @@ TARGET(POP_JUMP_IF_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); PyObject *value; @@ -4621,6 +4655,7 @@ TARGET(POP_JUMP_IF_NOT_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); PyObject *value; @@ -4654,6 +4689,7 @@ TARGET(POP_JUMP_IF_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); PyObject *cond; @@ -4715,11 +4751,11 @@ TARGET(RAISE_VARARGS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); PyObject **args; args = &stack_pointer[-oparg]; - TIER_ONE_ONLY PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -4745,13 +4781,13 @@ TARGET(RERAISE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; next_instr += 1; INSTRUCTION_STATS(RERAISE); PyObject *exc; PyObject **values; exc = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; - TIER_ONE_ONLY assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -4776,9 +4812,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); - TIER_ONE_ONLY assert(0 && "Executing RESERVED instruction."); - Py_UNREACHABLE(); + Py_FatalError("Executing RESERVED instruction."); + DISPATCH(); } TARGET(RESUME) { @@ -4787,10 +4823,10 @@ INSTRUCTION_STATS(RESUME); PREDICTED(RESUME); _Py_CODEUNIT *this_instr = next_instr - 1; - TIER_ONE_ONLY + (void)this_instr; assert(frame == tstate->current_frame); uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((code_version & 255) == 0); @@ -4817,7 +4853,7 @@ DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); DEOPT_IF(eval_breaker != version, RESUME); @@ -4865,7 +4901,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); - TIER_ONE_ONLY assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4925,6 +4960,7 @@ INSTRUCTION_STATS(SEND); PREDICTED(SEND); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *receiver; PyObject *v; PyObject *retval; @@ -4932,7 +4968,7 @@ receiver = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -5113,13 +5149,14 @@ INSTRUCTION_STATS(STORE_ATTR); PREDICTED(STORE_ATTR); _Py_CODEUNIT *this_instr = next_instr - 5; + (void)this_instr; PyObject *owner; PyObject *v; // _SPECIALIZE_STORE_ATTR owner = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -5403,6 +5440,7 @@ INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED(STORE_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *sub; PyObject *container; PyObject *v; @@ -5411,7 +5449,7 @@ container = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -5507,13 +5545,14 @@ INSTRUCTION_STATS(TO_BOOL); PREDICTED(TO_BOOL); _Py_CODEUNIT *this_instr = next_instr - 4; + (void)this_instr; PyObject *value; PyObject *res; // _SPECIALIZE_TO_BOOL value = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -5717,12 +5756,13 @@ INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED(UNPACK_SEQUENCE); _Py_CODEUNIT *this_instr = next_instr - 2; + (void)this_instr; PyObject *seq; // _SPECIALIZE_UNPACK_SEQUENCE seq = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY + (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -5857,7 +5897,6 @@ INSTRUCTION_STATS(YIELD_VALUE); PyObject *retval; retval = stack_pointer[-1]; - TIER_ONE_ONLY // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. diff --git a/Python/import.c b/Python/import.c index 2fd0c08a6bb5ae..dc92708c8b6ea0 100644 --- a/Python/import.c +++ b/Python/import.c @@ -13,6 +13,7 @@ #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_time.h" // _PyTime_PerfCounterUnchecked() #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "marshal.h" // PyMarshal_ReadObjectFromString() @@ -2719,7 +2720,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #define import_level FIND_AND_LOAD(interp).import_level #define accumulated FIND_AND_LOAD(interp).accumulated - _PyTime_t t1 = 0, accumulated_copy = accumulated; + PyTime_t t1 = 0, accumulated_copy = accumulated; PyObject *sys_path = PySys_GetObject("path"); PyObject *sys_meta_path = PySys_GetObject("meta_path"); @@ -2747,7 +2748,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #undef header import_level++; - t1 = _PyTime_GetPerfCounter(); + t1 = _PyTime_PerfCounterUnchecked(); accumulated = 0; } @@ -2762,7 +2763,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) mod != NULL); if (import_time) { - _PyTime_t cum = _PyTime_GetPerfCounter() - t1; + PyTime_t cum = _PyTime_PerfCounterUnchecked() - t1; import_level--; fprintf(stderr, "import time: %9ld | %10ld | %*s%s\n", diff --git a/Python/initconfig.c b/Python/initconfig.c index a6d8c176156617..74f28f3b39175b 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -153,7 +153,8 @@ Options (and corresponding environment variables):\n\ .pyc extension; also PYTHONOPTIMIZE=x\n\ -OO : do -O changes and also discard docstrings; add .opt-2 before\n\ .pyc extension\n\ --P : don't prepend a potentially unsafe path to sys.path; also PYTHONSAFEPATH\n\ +-P : don't prepend a potentially unsafe path to sys.path; also\n\ + PYTHONSAFEPATH\n\ -q : don't print version and copyright messages on interactive startup\n\ -s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ -S : don't imply 'import site' on initialization\n\ @@ -169,9 +170,10 @@ Options (and corresponding environment variables):\n\ -X opt : set implementation-specific option\n\ --check-hash-based-pycs always|default|never:\n\ control how Python invalidates hash-based .pyc files\n\ ---help-env : print help about Python environment variables and exit\n\ ---help-xoptions : print help about implementation-specific -X options and exit\n\ ---help-all : print complete help information and exit\n\ +--help-env: print help about Python environment variables and exit\n\ +--help-xoptions: print help about implementation-specific -X options and exit\n\ +--help-all: print complete help information and exit\n\ +\n\ Arguments:\n\ file : program read from script file\n\ - : program read from stdin (default; interactive mode if a tty)\n\ @@ -180,73 +182,61 @@ arg ...: arguments passed to program in sys.argv[1:]\n\ static const char usage_xoptions[] = "\ The following implementation-specific options are available:\n\ -\n\ -X faulthandler: enable faulthandler\n\ -\n\ -X showrefcount: output the total reference count and number of used\n\ - memory blocks when the program finishes or after each statement in the\n\ - interactive interpreter. This only works on debug builds\n\ -\n\ + memory blocks when the program finishes or after each statement in\n\ + the interactive interpreter. This only works on debug builds\n\ -X tracemalloc: start tracing Python memory allocations using the\n\ - tracemalloc module. By default, only the most recent frame is stored in a\n\ - traceback of a trace. Use -X tracemalloc=NFRAME to start tracing with a\n\ - traceback limit of NFRAME frames\n\ -\n\ --X importtime: show how long each import takes. It shows module name,\n\ - cumulative time (including nested imports) and self time (excluding\n\ - nested imports). Note that its output may be broken in multi-threaded\n\ - application. Typical usage is python3 -X importtime -c 'import asyncio'\n\ -\n\ --X dev: enable CPython's \"development mode\", introducing additional runtime\n\ - checks which are too expensive to be enabled by default. Effect of the\n\ - developer mode:\n\ - * Add default warning filter, as -W default\n\ - * Install debug hooks on memory allocators: see the PyMem_SetupDebugHooks()\n\ - C function\n\ - * Enable the faulthandler module to dump the Python traceback on a crash\n\ - * Enable asyncio debug mode\n\ - * Set the dev_mode attribute of sys.flags to True\n\ - * io.IOBase destructor logs close() exceptions\n\ -\n\ --X utf8: enable UTF-8 mode for operating system interfaces, overriding the default\n\ - locale-aware mode. -X utf8=0 explicitly disables UTF-8 mode (even when it would\n\ - otherwise activate automatically)\n\ -\n\ --X pycache_prefix=PATH: enable writing .pyc files to a parallel tree rooted at the\n\ - given directory instead of to the code tree\n\ -\n\ + tracemalloc module. By default, only the most recent frame is stored\n\ + in a traceback of a trace. Use -X tracemalloc=NFRAME to start\n\ + tracing with a traceback limit of NFRAME frames\n\ +-X importtime: show how long each import takes. It shows module name,\n\ + cumulative time (including nested imports) and self time (excluding\n\ + nested imports). Note that its output may be broken in\n\ + multi-threaded application.\n\ + Typical usage is python3 -X importtime -c 'import asyncio'\n\ +-X dev : enable CPython's \"development mode\", introducing additional runtime\n\ + checks which are too expensive to be enabled by default. Effect of\n\ + the developer mode:\n\ + * Add default warning filter, as -W default\n\ + * Install debug hooks on memory allocators: see the\n\ + PyMem_SetupDebugHooks() C function\n\ + * Enable the faulthandler module to dump the Python traceback on\n\ + a crash\n\ + * Enable asyncio debug mode\n\ + * Set the dev_mode attribute of sys.flags to True\n\ + * io.IOBase destructor logs close() exceptions\n\ +-X utf8: enable UTF-8 mode for operating system interfaces, overriding the\n\ + default locale-aware mode. -X utf8=0 explicitly disables UTF-8 mode\n\ + (even when it would otherwise activate automatically)\n\ +-X pycache_prefix=PATH: enable writing .pyc files to a parallel tree rooted\n\ + at the given directory instead of to the code tree\n\ -X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None'\n\ -\n\ --X no_debug_ranges: disable the inclusion of the tables mapping extra location \n\ - information (end line, start column offset and end column offset) to every \n\ - instruction in code objects. This is useful when smaller code objects and pyc \n\ - files are desired as well as suppressing the extra visual location indicators \n\ - when the interpreter displays tracebacks.\n\ -\n\ --X perf: activate support for the Linux \"perf\" profiler by activating the \"perf\"\n\ - trampoline. When this option is activated, the Linux \"perf\" profiler will be \n\ - able to report Python calls. This option is only available on some platforms and will \n\ - do nothing if is not supported on the current system. The default value is \"off\".\n\ -\n\ +-X no_debug_ranges: disable the inclusion of the tables mapping extra location\n\ + information (end line, start column offset and end column offset) to\n\ + every instruction in code objects. This is useful when smaller code\n\ + objects and pyc files are desired as well as suppressing the extra\n\ + visual location indicators when the interpreter displays tracebacks.\n\ +-X perf: activate support for the Linux \"perf\" profiler by activating the\n\ + \"perf\" trampoline. When this option is activated, the Linux \"perf\"\n\ + profiler will be able to report Python calls. This option is only\n\ + available on some platforms and will do nothing if is not supported\n\ + on the current system. The default value is \"off\".\n\ -X frozen_modules=[on|off]: whether or not frozen modules should be used.\n\ - The default is \"on\" (or \"off\" if you are running a local build).\n\ -\n\ + The default is \"on\" (or \"off\" if you are running a local build).\n\ -X int_max_str_digits=number: limit the size of int<->str conversions.\n\ - This helps avoid denial of service attacks when parsing untrusted data.\n\ - The default is sys.int_info.default_max_str_digits. 0 disables.\n\ -\n\ + This helps avoid denial of service attacks when parsing untrusted\n\ + data. The default is sys.int_info.default_max_str_digits.\n\ + 0 disables.\n\ -X cpu_count=[n|default]: Override the return value of os.cpu_count(),\n\ - os.process_cpu_count(), and multiprocessing.cpu_count(). This can help users who need\n\ - to limit resources in a container." - + os.process_cpu_count(), and multiprocessing.cpu_count(). This can\n\ + help users who need to limit resources in a container." #ifdef Py_STATS "\n\ -\n\ -X pystats: Enable pystats collection at startup." #endif #ifdef Py_DEBUG "\n\ -\n\ -X presite=package.module: import this module before site.py is run." #endif ; @@ -254,65 +244,73 @@ The following implementation-specific options are available:\n\ /* Envvars that don't have equivalent command-line options are listed first */ static const char usage_envvars[] = "Environment variables that change behavior:\n" -"PYTHONSTARTUP: file executed on interactive startup (no default)\n" -"PYTHONPATH : '%lc'-separated list of directories prefixed to the\n" -" default module search path. The result is sys.path.\n" -"PYTHONHOME : alternate directory (or %lc).\n" -" The default module search path uses %s.\n" -"PYTHONPLATLIBDIR : override sys.platlibdir.\n" -"PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" -"PYTHONUTF8: if set to 1, enable the UTF-8 mode.\n" +"PYTHONSTARTUP : file executed on interactive startup (no default)\n" +"PYTHONPATH : '%lc'-separated list of directories prefixed to the\n" +" default module search path. The result is sys.path.\n" +"PYTHONHOME : alternate directory (or %lc).\n" +" The default module search path uses %s.\n" +"PYTHONPLATLIBDIR: override sys.platlibdir.\n" +"PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" +"PYTHONUTF8 : if set to 1, enable the UTF-8 mode.\n" "PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" "PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n" -"PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n" -" to seed the hashes of str and bytes objects. It can also be set to an\n" -" integer in the range [0,4294967295] to get hash values with a\n" -" predictable seed.\n" +"PYTHONHASHSEED : if this variable is set to 'random', a random value is used\n" +" to seed the hashes of str and bytes objects. It can also be\n" +" set to an integer in the range [0,4294967295] to get hash\n" +" values with a predictable seed.\n" "PYTHONINTMAXSTRDIGITS: limits the maximum digit characters in an int value\n" -" when converting from a string and when converting an int back to a str.\n" -" A value of 0 disables the limit. Conversions to or from bases 2, 4, 8,\n" -" 16, and 32 are never limited.\n" -"PYTHONMALLOC: set the Python memory allocators and/or install debug hooks\n" -" on Python memory allocators. Use PYTHONMALLOC=debug to install debug\n" -" hooks.\n" +" when converting from a string and when converting an int\n" +" back to a str. A value of 0 disables the limit.\n" +" Conversions to or from bases 2, 4, 8, 16, and 32 are never\n" +" limited.\n" +"PYTHONMALLOC : set the Python memory allocators and/or install debug hooks\n" +" on Python memory allocators. Use PYTHONMALLOC=debug to\n" +" install debug hooks.\n" "PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" -" coercion behavior. Use PYTHONCOERCECLOCALE=warn to request display of\n" -" locale coercion and locale compatibility warnings on stderr.\n" +" coercion behavior. Use PYTHONCOERCECLOCALE=warn to request\n" +" display of locale coercion and locale compatibility warnings\n" +" on stderr.\n" "PYTHONBREAKPOINT: if this variable is set to 0, it disables the default\n" -" debugger. It can be set to the callable of your debugger of choice.\n" +" debugger. It can be set to the callable of your debugger of\n" +" choice.\n" "PYTHON_CPU_COUNT: Overrides the return value of os.process_cpu_count(),\n" -" os.cpu_count(), and multiprocessing.cpu_count() if set to a positive integer.\n" -"PYTHONDEVMODE: enable the development mode.\n" +" os.cpu_count(), and multiprocessing.cpu_count() if set to\n" +" a positive integer.\n" +"PYTHONDEVMODE : enable the development mode.\n" "PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n" "PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'.\n" -"PYTHONNODEBUGRANGES: if this variable is set, it disables the inclusion of the \n" -" tables mapping extra location information (end line, start column offset \n" -" and end column offset) to every instruction in code objects. This is useful \n" -" when smaller code objects and pyc files are desired as well as suppressing the \n" -" extra visual location indicators when the interpreter displays tracebacks.\n" -"PYTHON_FROZEN_MODULES : if this variable is set, it determines whether or not \n" -" frozen modules should be used. The default is \"on\" (or \"off\" if you are \n" -" running a local build).\n" -"PYTHON_COLORS : If this variable is set to 1, the interpreter will" -" colorize various kinds of output. Setting it to 0 deactivates this behavior.\n" -"PYTHON_HISTORY : the location of a .python_history file.\n" -"These variables have equivalent command-line parameters (see --help for details):\n" -"PYTHONDEBUG : enable parser debug mode (-d)\n" -"PYTHONDONTWRITEBYTECODE : don't write .pyc files (-B)\n" -"PYTHONINSPECT : inspect interactively after running script (-i)\n" -"PYTHONINTMAXSTRDIGITS : limit max digit characters in an int value\n" -" (-X int_max_str_digits=number)\n" -"PYTHONNOUSERSITE : disable user site directory (-s)\n" -"PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" -"PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path (-P)\n" -"PYTHONUNBUFFERED : disable stdout/stderr buffering (-u)\n" -"PYTHONVERBOSE : trace import statements (-v)\n" -"PYTHONWARNINGS=arg : warning control (-W arg)\n" +"PYTHONNODEBUGRANGES: if this variable is set, it disables the inclusion of\n" +" the tables mapping extra location information (end line,\n" +" start column offset and end column offset) to every\n" +" instruction in code objects. This is useful when smaller\n" +" code objects and pyc files are desired as well as\n" +" suppressing the extra visual location indicators when the\n" +" interpreter displays tracebacks.\n" +"PYTHON_FROZEN_MODULES: if this variable is set, it determines whether or not\n" +" frozen modules should be used. The default is \"on\" (or\n" +" \"off\" if you are running a local build).\n" +"PYTHON_COLORS : if this variable is set to 1, the interpreter will colorize\n" +" various kinds of output. Setting it to 0 deactivates\n" +" this behavior.\n" +"PYTHON_HISTORY : the location of a .python_history file.\n" +"\n" +"These variables have equivalent command-line options (see --help for details):\n" +"PYTHONDEBUG : enable parser debug mode (-d)\n" +"PYTHONDONTWRITEBYTECODE: don't write .pyc files (-B)\n" +"PYTHONINSPECT : inspect interactively after running script (-i)\n" +"PYTHONINTMAXSTRDIGITS: limit max digit characters in an int value\n" +" (-X int_max_str_digits=number)\n" +"PYTHONNOUSERSITE: disable user site directory (-s)\n" +"PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" +"PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path.\n" +"PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n" +"PYTHONVERBOSE : trace import statements (-v)\n" +"PYTHONWARNINGS=arg: warning control (-W arg)\n" #ifdef Py_STATS -"PYTHONSTATS : turns on statistics gathering\n" +"PYTHONSTATS : turns on statistics gathering\n" #endif #ifdef Py_DEBUG -"PYTHON_PRESITE=pkg.mod : import this module before site.py is run\n" +"PYTHON_PRESITE=pkg.mod: import this module before site.py is run\n" #endif ; @@ -2387,9 +2385,9 @@ static void config_complete_usage(const wchar_t* program) { config_usage(0, program); - puts("\n"); + putchar('\n'); config_envvars_usage(); - puts("\n"); + putchar('\n'); config_xoptions_usage(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 533aece210202b..018cd662b1561a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -891,18 +891,51 @@ static inline int most_significant_bit(uint8_t bits) { static uint32_t global_version(PyInterpreterState *interp) { - return interp->ceval.eval_breaker & ~_PY_EVAL_EVENTS_MASK; + uint32_t version = (uint32_t)_Py_atomic_load_uintptr_relaxed( + &interp->ceval.instrumentation_version); +#ifdef Py_DEBUG + PyThreadState *tstate = _PyThreadState_GET(); + uint32_t thread_version = + (uint32_t)(_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK); + assert(thread_version == version); +#endif + return version; } +/* Atomically set the given version in the given location, without touching + anything in _PY_EVAL_EVENTS_MASK. */ static void -set_global_version(PyInterpreterState *interp, uint32_t version) +set_version_raw(uintptr_t *ptr, uint32_t version) { - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - uintptr_t old = _Py_atomic_load_uintptr(&interp->ceval.eval_breaker); - intptr_t new; + uintptr_t old = _Py_atomic_load_uintptr_relaxed(ptr); + uintptr_t new; do { new = (old & _PY_EVAL_EVENTS_MASK) | version; - } while (!_Py_atomic_compare_exchange_uintptr(&interp->ceval.eval_breaker, &old, new)); + } while (!_Py_atomic_compare_exchange_uintptr(ptr, &old, new)); +} + +static void +set_global_version(PyThreadState *tstate, uint32_t version) +{ + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + PyInterpreterState *interp = tstate->interp; + set_version_raw(&interp->ceval.instrumentation_version, version); + +#ifdef Py_GIL_DISABLED + // Set the version on all threads in free-threaded builds. + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + for (tstate = interp->threads.head; tstate; + tstate = PyThreadState_Next(tstate)) { + set_version_raw(&tstate->eval_breaker, version); + }; + HEAD_UNLOCK(runtime); +#else + // Normal builds take the current version from instrumentation_version when + // attaching a thread, so we only have to set the current thread's version. + set_version_raw(&tstate->eval_breaker, version); +#endif } static bool @@ -1566,7 +1599,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { if (is_version_up_to_date(code, interp)) { assert( - (interp->ceval.eval_breaker & ~_PY_EVAL_EVENTS_MASK) == 0 || + interp->ceval.instrumentation_version == 0 || instrumentation_cross_checks(interp, code) ); return 0; @@ -1574,7 +1607,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (code->co_executors != NULL) { _PyCode_Clear_Executors(code); } - _Py_Executors_InvalidateDependency(interp, code); + _Py_Executors_InvalidateDependency(interp, code, 1); int code_len = (int)Py_SIZE(code); /* Exit early to avoid creating instrumentation * data for potential statically allocated code @@ -1778,7 +1811,8 @@ int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = tstate->interp; assert(events < (1 << _PY_MONITORING_UNGROUPED_EVENTS)); if (check_tool(interp, tool_id)) { return -1; @@ -1793,8 +1827,8 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) PyErr_Format(PyExc_OverflowError, "events set too many times"); return -1; } - set_global_version(interp, new_version); - _Py_Executors_InvalidateAll(interp); + set_global_version(tstate, new_version); + _Py_Executors_InvalidateAll(interp, 1); return instrument_all_executing_code_objects(interp); } @@ -1824,7 +1858,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent /* Force instrumentation update */ code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT; } - _Py_Executors_InvalidateDependency(interp, code); + _Py_Executors_InvalidateDependency(interp, code, 1); if (_Py_Instrument(code, interp)) { return -1; } @@ -2122,7 +2156,8 @@ monitoring_restart_events_impl(PyObject *module) * last restart version > instrumented version for all code objects * last restart version < current version */ - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = tstate->interp; uint32_t restart_version = global_version(interp) + MONITORING_VERSION_INCREMENT; uint32_t new_version = restart_version + MONITORING_VERSION_INCREMENT; if (new_version <= MONITORING_VERSION_INCREMENT) { @@ -2130,7 +2165,7 @@ monitoring_restart_events_impl(PyObject *module) return NULL; } interp->last_restart_version = restart_version; - set_global_version(interp, new_version); + set_global_version(tstate, new_version); if (instrument_all_executing_code_objects(interp)) { return NULL; } diff --git a/Python/jit.c b/Python/jit.c index 22949c082da05a..dae25166b1f106 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -47,18 +47,18 @@ jit_error(const char *message) PyErr_Format(PyExc_RuntimeWarning, "JIT %s (%d)", message, hint); } -static char * +static unsigned char * jit_alloc(size_t size) { assert(size); assert(size % get_page_size() == 0); #ifdef MS_WINDOWS int flags = MEM_COMMIT | MEM_RESERVE; - char *memory = VirtualAlloc(NULL, size, flags, PAGE_READWRITE); + unsigned char *memory = VirtualAlloc(NULL, size, flags, PAGE_READWRITE); int failed = memory == NULL; #else int flags = MAP_ANONYMOUS | MAP_PRIVATE; - char *memory = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); + unsigned char *memory = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0); int failed = memory == MAP_FAILED; #endif if (failed) { @@ -69,7 +69,7 @@ jit_alloc(size_t size) } static int -jit_free(char *memory, size_t size) +jit_free(unsigned char *memory, size_t size) { assert(size); assert(size % get_page_size() == 0); @@ -86,7 +86,7 @@ jit_free(char *memory, size_t size) } static int -mark_executable(char *memory, size_t size) +mark_executable(unsigned char *memory, size_t size) { if (size == 0) { return 0; @@ -113,7 +113,7 @@ mark_executable(char *memory, size_t size) } static int -mark_readable(char *memory, size_t size) +mark_readable(unsigned char *memory, size_t size) { if (size == 0) { return 0; @@ -169,20 +169,24 @@ set_bits(uint32_t *loc, uint8_t loc_start, uint64_t value, uint8_t value_start, // Fill all of stencil's holes in the memory pointed to by base, using the // values in patches. static void -patch(char *base, const Stencil *stencil, uint64_t *patches) +patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) { for (uint64_t i = 0; i < stencil->holes_size; i++) { const Hole *hole = &stencil->holes[i]; - void *location = base + hole->offset; + unsigned char *location = base + hole->offset; uint64_t value = patches[hole->value] + (uint64_t)hole->symbol + hole->addend; + uint8_t *loc8 = (uint8_t *)location; uint32_t *loc32 = (uint32_t *)location; uint64_t *loc64 = (uint64_t *)location; // LLD is a great reference for performing relocations... just keep in // mind that Tools/jit/build.py does filtering and preprocessing for us! // Here's a good place to start for each platform: // - aarch64-apple-darwin: + // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64.cpp // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.cpp // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.h + // - aarch64-pc-windows-msvc: + // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp // - aarch64-unknown-linux-gnu: // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/AArch64.cpp // - i686-pc-windows-msvc: @@ -201,13 +205,56 @@ patch(char *base, const Stencil *stencil, uint64_t *patches) *loc32 = (uint32_t)value; continue; case HoleKind_ARM64_RELOC_UNSIGNED: - case HoleKind_IMAGE_REL_AMD64_ADDR64: case HoleKind_R_AARCH64_ABS64: case HoleKind_X86_64_RELOC_UNSIGNED: case HoleKind_R_X86_64_64: // 64-bit absolute address. *loc64 = value; continue; + case HoleKind_IMAGE_REL_AMD64_REL32: + case HoleKind_IMAGE_REL_I386_REL32: + case HoleKind_R_X86_64_GOTPCRELX: + case HoleKind_R_X86_64_REX_GOTPCRELX: + case HoleKind_X86_64_RELOC_GOT: + case HoleKind_X86_64_RELOC_GOT_LOAD: { + // 32-bit relative address. + // Try to relax the GOT load into an immediate value: + uint64_t relaxed = *(uint64_t *)(value + 4) - 4; + if ((int64_t)relaxed - (int64_t)location >= -(1LL << 31) && + (int64_t)relaxed - (int64_t)location + 1 < (1LL << 31)) + { + if (loc8[-2] == 0x8B) { + // mov reg, dword ptr [rip + AAA] -> lea reg, [rip + XXX] + loc8[-2] = 0x8D; + value = relaxed; + } + else if (loc8[-2] == 0xFF && loc8[-1] == 0x15) { + // call qword ptr [rip + AAA] -> nop; call XXX + loc8[-2] = 0x90; + loc8[-1] = 0xE8; + value = relaxed; + } + else if (loc8[-2] == 0xFF && loc8[-1] == 0x25) { + // jmp qword ptr [rip + AAA] -> nop; jmp XXX + loc8[-2] = 0x90; + loc8[-1] = 0xE9; + value = relaxed; + } + } + } + // Fall through... + case HoleKind_R_X86_64_GOTPCREL: + case HoleKind_R_X86_64_PC32: + case HoleKind_X86_64_RELOC_SIGNED: + case HoleKind_X86_64_RELOC_BRANCH: + // 32-bit relative address. + value -= (uint64_t)location; + // Check that we're not out of range of 32 signed bits: + assert((int64_t)value >= -(1LL << 31)); + assert((int64_t)value < (1LL << 31)); + *loc32 = (uint32_t)value; + continue; + case HoleKind_IMAGE_REL_ARM64_BRANCH26: case HoleKind_R_AARCH64_CALL26: case HoleKind_R_AARCH64_JUMP26: // 28-bit relative branch. @@ -249,10 +296,57 @@ patch(char *base, const Stencil *stencil, uint64_t *patches) set_bits(loc32, 5, value, 48, 16); continue; case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21: + case HoleKind_IMAGE_REL_ARM64_PAGEBASE_REL21: + case HoleKind_R_AARCH64_ADR_GOT_PAGE: // 21-bit count of pages between this page and an absolute address's // page... I know, I know, it's weird. Pairs nicely with // ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below). assert(IS_AARCH64_ADRP(*loc32)); + // Try to relax the pair of GOT loads into an immediate value: + const Hole *next_hole = &stencil->holes[i + 1]; + if (i + 1 < stencil->holes_size && + (next_hole->kind == HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12 || + next_hole->kind == HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L || + next_hole->kind == HoleKind_R_AARCH64_LD64_GOT_LO12_NC) && + next_hole->offset == hole->offset + 4 && + next_hole->symbol == hole->symbol && + next_hole->addend == hole->addend && + next_hole->value == hole->value) + { + unsigned char reg = get_bits(loc32[0], 0, 5); + assert(IS_AARCH64_LDR_OR_STR(loc32[1])); + // There should be only one register involved: + assert(reg == get_bits(loc32[1], 0, 5)); // ldr's output register. + assert(reg == get_bits(loc32[1], 5, 5)); // ldr's input register. + uint64_t relaxed = *(uint64_t *)value; + if (relaxed < (1UL << 16)) { + // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; nop + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; + loc32[1] = 0xD503201F; + i++; + continue; + } + if (relaxed < (1ULL << 32)) { + // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; movk reg, YYY + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; + loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | reg; + i++; + continue; + } + relaxed = (uint64_t)value - (uint64_t)location; + if ((relaxed & 0x3) == 0 && + (int64_t)relaxed >= -(1L << 19) && + (int64_t)relaxed < (1L << 19)) + { + // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr reg, XXX; nop + loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | reg; + loc32[1] = 0xD503201F; + i++; + continue; + } + } + // Fall through... + case HoleKind_ARM64_RELOC_PAGE21: // Number of pages between this page and the value's page: value = (value >> 12) - ((uint64_t)location >> 12); // Check that we're not out of range of 21 signed bits: @@ -264,6 +358,10 @@ patch(char *base, const Stencil *stencil, uint64_t *patches) set_bits(loc32, 5, value, 2, 19); continue; case HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12: + case HoleKind_ARM64_RELOC_PAGEOFF12: + case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12A: + case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L: + case HoleKind_R_AARCH64_LD64_GOT_LO12_NC: // 12-bit low part of an absolute address. Pairs nicely with // ARM64_RELOC_GOT_LOAD_PAGE21 (above). assert(IS_AARCH64_LDR_OR_STR(*loc32) || IS_AARCH64_ADD_OR_SUB(*loc32)); @@ -285,7 +383,7 @@ patch(char *base, const Stencil *stencil, uint64_t *patches) } static void -copy_and_patch(char *base, const Stencil *stencil, uint64_t *patches) +copy_and_patch(unsigned char *base, const Stencil *stencil, uint64_t *patches) { memcpy(base, stencil->body, stencil->body_size); patch(base, stencil, patches); @@ -294,19 +392,19 @@ copy_and_patch(char *base, const Stencil *stencil, uint64_t *patches) static void emit(const StencilGroup *group, uint64_t patches[]) { - copy_and_patch((char *)patches[HoleValue_CODE], &group->code, patches); - copy_and_patch((char *)patches[HoleValue_DATA], &group->data, patches); + copy_and_patch((unsigned char *)patches[HoleValue_DATA], &group->data, patches); + copy_and_patch((unsigned char *)patches[HoleValue_CODE], &group->code, patches); } // Compiles executor in-place. Don't forget to call _PyJIT_Free later! int -_PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t length) +_PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length) { // Loop once to find the total compiled size: size_t code_size = 0; size_t data_size = 0; for (size_t i = 0; i < length; i++) { - _PyUOpInstruction *instruction = &trace[i]; + _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; const StencilGroup *group = &stencil_groups[instruction->opcode]; code_size += group->code.body_size; data_size += group->data.body_size; @@ -316,15 +414,20 @@ _PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t len assert((page_size & (page_size - 1)) == 0); code_size += page_size - (code_size & (page_size - 1)); data_size += page_size - (data_size & (page_size - 1)); - char *memory = jit_alloc(code_size + data_size); + unsigned char *memory = jit_alloc(code_size + data_size); if (memory == NULL) { return -1; } // Loop again to emit the code: - char *code = memory; - char *data = memory + code_size; + unsigned char *code = memory; + unsigned char *data = memory + code_size; + unsigned char *top = code; + if (trace[0].opcode == _START_EXECUTOR) { + // Don't want to execute this more than once: + top += stencil_groups[_START_EXECUTOR].code.body_size; + } for (size_t i = 0; i < length; i++) { - _PyUOpInstruction *instruction = &trace[i]; + _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; const StencilGroup *group = &stencil_groups[instruction->opcode]; // Think of patches as a dictionary mapping HoleValue to uint64_t: uint64_t patches[] = GET_PATCHES(); @@ -335,7 +438,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t len patches[HoleValue_OPARG] = instruction->oparg; patches[HoleValue_OPERAND] = instruction->operand; patches[HoleValue_TARGET] = instruction->target; - patches[HoleValue_TOP] = (uint64_t)memory; + patches[HoleValue_TOP] = (uint64_t)top; patches[HoleValue_ZERO] = 0; emit(group, patches); code += group->code.body_size; @@ -355,7 +458,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t len void _PyJIT_Free(_PyExecutorObject *executor) { - char *memory = (char *)executor->jit_code; + unsigned char *memory = (unsigned char *)executor->jit_code; size_t size = executor->jit_size; if (memory) { executor->jit_code = NULL; diff --git a/Python/lock.c b/Python/lock.c index bf0143654bd692..de25adce385105 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -5,18 +5,19 @@ #include "pycore_lock.h" #include "pycore_parking_lot.h" #include "pycore_semaphore.h" +#include "pycore_time.h" // _PyTime_MonotonicUnchecked() #ifdef MS_WINDOWS -#define WIN32_LEAN_AND_MEAN -#include // SwitchToThread() +# define WIN32_LEAN_AND_MEAN +# include // SwitchToThread() #elif defined(HAVE_SCHED_H) -#include // sched_yield() +# include // sched_yield() #endif // If a thread waits on a lock for longer than TIME_TO_BE_FAIR_NS (1 ms), then // the unlocking thread directly hands off ownership of the lock. This avoids // starvation. -static const _PyTime_t TIME_TO_BE_FAIR_NS = 1000*1000; +static const PyTime_t TIME_TO_BE_FAIR_NS = 1000*1000; // Spin for a bit before parking the thread. This is only enabled for // `--disable-gil` builds because it is unlikely to be helpful if the GIL is @@ -30,7 +31,7 @@ static const int MAX_SPIN_COUNT = 0; struct mutex_entry { // The time after which the unlocking thread should hand off lock ownership // directly to the waiting thread. Written by the waiting thread. - _PyTime_t time_to_be_fair; + PyTime_t time_to_be_fair; // Set to 1 if the lock was handed off. Written by the unlocking thread. int handed_off; @@ -53,7 +54,7 @@ _PyMutex_LockSlow(PyMutex *m) } PyLockStatus -_PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout, _PyLockFlags flags) +_PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) { uint8_t v = _Py_atomic_load_uint8_relaxed(&m->v); if ((v & _Py_LOCKED) == 0) { @@ -65,8 +66,8 @@ _PyMutex_LockTimed(PyMutex *m, _PyTime_t timeout, _PyLockFlags flags) return PY_LOCK_FAILURE; } - _PyTime_t now = _PyTime_GetMonotonicClock(); - _PyTime_t endtime = 0; + PyTime_t now = _PyTime_MonotonicUnchecked(); + PyTime_t endtime = 0; if (timeout > 0) { endtime = _PyTime_Add(now, timeout); } @@ -142,7 +143,7 @@ mutex_unpark(PyMutex *m, struct mutex_entry *entry, int has_more_waiters) { uint8_t v = 0; if (entry) { - _PyTime_t now = _PyTime_GetMonotonicClock(); + PyTime_t now = _PyTime_MonotonicUnchecked(); int should_be_fair = now > entry->time_to_be_fair; entry->handed_off = should_be_fair; @@ -248,6 +249,13 @@ _PyRawMutex_UnlockSlow(_PyRawMutex *m) } } +int +_PyEvent_IsSet(PyEvent *evt) +{ + uint8_t v = _Py_atomic_load_uint8(&evt->v); + return v == _Py_LOCKED; +} + void _PyEvent_Notify(PyEvent *evt) { @@ -274,7 +282,7 @@ PyEvent_Wait(PyEvent *evt) } int -PyEvent_WaitTimed(PyEvent *evt, _PyTime_t timeout_ns) +PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns) { for (;;) { uint8_t v = _Py_atomic_load_uint8(&evt->v); @@ -296,6 +304,30 @@ PyEvent_WaitTimed(PyEvent *evt, _PyTime_t timeout_ns) } } +_PyEventRc * +_PyEventRc_New(void) +{ + _PyEventRc *erc = (_PyEventRc *)PyMem_RawCalloc(1, sizeof(_PyEventRc)); + if (erc != NULL) { + erc->refcount = 1; + } + return erc; +} + +void +_PyEventRc_Incref(_PyEventRc *erc) +{ + _Py_atomic_add_ssize(&erc->refcount, 1); +} + +void +_PyEventRc_Decref(_PyEventRc *erc) +{ + if (_Py_atomic_add_ssize(&erc->refcount, -1) == 1) { + PyMem_RawFree(erc); + } +} + static int unlock_once(_PyOnceFlag *o, int res) { diff --git a/Python/optimizer.c b/Python/optimizer.c index efa19680c9b1f3..acd6d52c4a885f 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -2,6 +2,7 @@ #include "opcode.h" #include "pycore_interp.h" #include "pycore_bitutils.h" // _Py_popcount32() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode_metadata.h" // _PyOpcode_OpName[] #include "pycore_opcode_utils.h" // MAX_REAL_OPCODE #include "pycore_optimizer.h" // _Py_uop_analyze_and_optimize() @@ -128,10 +129,11 @@ static _PyOptimizerObject _PyOptimizer_Default = { .optimize = never_optimize, .resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, .backedge_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, + .side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, }; static uint32_t -shift_and_offset_threshold(uint16_t threshold) +shift_and_offset_threshold(uint32_t threshold) { return (threshold << OPTIMIZER_BITS_IN_COUNTER) + (1 << 15); } @@ -140,41 +142,74 @@ _PyOptimizerObject * PyUnstable_GetOptimizer(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->optimizer == &_PyOptimizer_Default) { - return NULL; - } assert(interp->optimizer_backedge_threshold == shift_and_offset_threshold(interp->optimizer->backedge_threshold)); assert(interp->optimizer_resume_threshold == shift_and_offset_threshold(interp->optimizer->resume_threshold)); + if (interp->optimizer == &_PyOptimizer_Default) { + return NULL; + } Py_INCREF(interp->optimizer); return interp->optimizer; } +static _PyExecutorObject * +make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies); + +static int +init_cold_exit_executor(_PyExecutorObject *executor, int oparg); + +static int cold_exits_initialized = 0; +static _PyExecutorObject COLD_EXITS[UOP_MAX_TRACE_LENGTH] = { 0 }; + +static const _PyBloomFilter EMPTY_FILTER = { 0 }; + _PyOptimizerObject * _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) { if (optimizer == NULL) { optimizer = &_PyOptimizer_Default; } + else if (cold_exits_initialized == 0) { + cold_exits_initialized = 1; + for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { + if (init_cold_exit_executor(&COLD_EXITS[i], i)) { + return NULL; + } + } + } _PyOptimizerObject *old = interp->optimizer; + if (old == NULL) { + old = &_PyOptimizer_Default; + } Py_INCREF(optimizer); interp->optimizer = optimizer; interp->optimizer_backedge_threshold = shift_and_offset_threshold(optimizer->backedge_threshold); interp->optimizer_resume_threshold = shift_and_offset_threshold(optimizer->resume_threshold); + interp->optimizer_side_threshold = optimizer->side_threshold; + if (optimizer == &_PyOptimizer_Default) { + assert(interp->optimizer_backedge_threshold > (1 << 16)); + assert(interp->optimizer_resume_threshold > (1 << 16)); + } return old; } -void +int PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer) { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyOptimizerObject *old = _Py_SetOptimizer(interp, optimizer); - Py_DECREF(old); + Py_XDECREF(old); + return old == NULL ? -1 : 0; } +/* Returns 1 if optimized, 0 if not optimized, and -1 for an error. + * If optimized, *executor_ptr contains a new reference to the executor + */ int -_PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer) +_PyOptimizer_Optimize( + _PyInterpreterFrame *frame, _Py_CODEUNIT *start, + PyObject **stack_pointer, _PyExecutorObject **executor_ptr) { PyCodeObject *code = (PyCodeObject *)frame->f_executable; assert(PyCode_Check(code)); @@ -183,12 +218,11 @@ _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject return 0; } _PyOptimizerObject *opt = interp->optimizer; - _PyExecutorObject *executor = NULL; - int err = opt->optimize(opt, frame, start, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); + int err = opt->optimize(opt, frame, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame))); if (err <= 0) { - assert(executor == NULL); return err; } + assert(*executor_ptr != NULL); int index = get_index_for_executor(code, start); if (index < 0) { /* Out of memory. Don't raise and assume that the @@ -197,11 +231,11 @@ _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject * If an optimizer has already produced an executor, * it might get confused by the executor disappearing, * but there is not much we can do about that here. */ - Py_DECREF(executor); + Py_DECREF(*executor_ptr); return 0; } - insert_executor(code, start, index, executor); - Py_DECREF(executor); + insert_executor(code, start, index, *executor_ptr); + assert((*executor_ptr)->vm_data.valid); return 1; } @@ -228,8 +262,22 @@ is_valid(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyBool_FromLong(((_PyExecutorObject *)self)->vm_data.valid); } +static PyObject * +get_opcode(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromUnsignedLong(((_PyExecutorObject *)self)->vm_data.opcode); +} + +static PyObject * +get_oparg(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromUnsignedLong(((_PyExecutorObject *)self)->vm_data.oparg); +} + static PyMethodDef executor_methods[] = { { "is_valid", is_valid, METH_NOARGS, NULL }, + { "get_opcode", get_opcode, METH_NOARGS, NULL }, + { "get_oparg", get_oparg, METH_NOARGS, NULL }, { NULL, NULL }, }; @@ -237,23 +285,45 @@ static PyMethodDef executor_methods[] = { static void uop_dealloc(_PyExecutorObject *self) { + _PyObject_GC_UNTRACK(self); _Py_ExecutorClear(self); #ifdef _Py_JIT _PyJIT_Free(self); #endif - PyObject_Free(self); + PyObject_GC_Del(self); } const char * _PyUOpName(int index) { + if (index < 0 || index > MAX_UOP_ID) { + return NULL; + } return _PyOpcode_uop_name[index]; } +#ifdef Py_DEBUG +void +_PyUOpPrint(const _PyUOpInstruction *uop) +{ + const char *name = _PyUOpName(uop->opcode); + if (name == NULL) { + printf("", uop->opcode); + } + else { + printf("%s", name); + } + printf(" (%d, target=%d, operand=%" PRIx64 ")", + uop->oparg, + uop->target, + (uint64_t)uop->operand); +} +#endif + static Py_ssize_t uop_len(_PyExecutorObject *self) { - return Py_SIZE(self); + return self->code_size; } static PyObject * @@ -277,14 +347,21 @@ uop_item(_PyExecutorObject *self, Py_ssize_t index) Py_DECREF(oname); return NULL; } + PyObject *target = PyLong_FromUnsignedLong(self->trace[index].target); + if (oparg == NULL) { + Py_DECREF(oparg); + Py_DECREF(oname); + return NULL; + } PyObject *operand = PyLong_FromUnsignedLongLong(self->trace[index].operand); if (operand == NULL) { + Py_DECREF(target); Py_DECREF(oparg); Py_DECREF(oname); return NULL; } - PyObject *args[3] = { oname, oparg, operand }; - return _PyTuple_FromArraySteal(args, 3); + PyObject *args[4] = { oname, oparg, target, operand }; + return _PyTuple_FromArraySteal(args, 4); } PySequenceMethods uop_as_sequence = { @@ -292,15 +369,34 @@ PySequenceMethods uop_as_sequence = { .sq_item = (ssizeargfunc)uop_item, }; +static int +executor_clear(PyObject *o) +{ + _Py_ExecutorClear((_PyExecutorObject *)o); + return 0; +} + +static int +executor_traverse(PyObject *o, visitproc visit, void *arg) +{ + _PyExecutorObject *executor = (_PyExecutorObject *)o; + for (uint32_t i = 0; i < executor->exit_count; i++) { + Py_VISIT(executor->exits[i].executor); + } + return 0; +} + PyTypeObject _PyUOpExecutor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "uop_executor", - .tp_basicsize = offsetof(_PyExecutorObject, trace), - .tp_itemsize = sizeof(_PyUOpInstruction), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_basicsize = offsetof(_PyExecutorObject, exits), + .tp_itemsize = 1, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC, .tp_dealloc = (destructor)uop_dealloc, .tp_as_sequence = &uop_as_sequence, .tp_methods = executor_methods, + .tp_traverse = executor_traverse, + .tp_clear = executor_clear, }; /* TO DO -- Generate these tables */ @@ -324,6 +420,7 @@ BRANCH_TO_GUARD[4][2] = { [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NOT_NONE_POP, }; + #define CONFIDENCE_RANGE 1000 #define CONFIDENCE_CUTOFF 333 @@ -335,19 +432,29 @@ BRANCH_TO_GUARD[4][2] = { #endif +// Beware: Macro arg order differs from struct member order +#ifdef Py_DEBUG #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ - DPRINTF(2, \ - " ADD_TO_TRACE(%s, %d, %" PRIu64 ", %d)\n", \ - _PyUOpName(OPCODE), \ - (OPARG), \ - (uint64_t)(OPERAND), \ - TARGET); \ assert(trace_length < max_length); \ trace[trace_length].opcode = (OPCODE); \ trace[trace_length].oparg = (OPARG); \ + trace[trace_length].target = (TARGET); \ trace[trace_length].operand = (OPERAND); \ + if (lltrace >= 2) { \ + printf("%4d ADD_TO_TRACE: ", trace_length); \ + _PyUOpPrint(&trace[trace_length]); \ + printf("\n"); \ + } \ + trace_length++; +#else +#define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ + assert(trace_length < max_length); \ + trace[trace_length].opcode = (OPCODE); \ + trace[trace_length].oparg = (OPARG); \ trace[trace_length].target = (TARGET); \ + trace[trace_length].operand = (OPERAND); \ trace_length++; +#endif #define INSTR_IP(INSTR, CODE) \ ((uint32_t)((INSTR) - ((_Py_CODEUNIT *)(CODE)->co_code_adaptive))) @@ -439,6 +546,8 @@ translate_bytecode_to_trace( uint32_t oparg = instr->op.arg; uint32_t extended = 0; + DPRINTF(3, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); + if (opcode == ENTER_EXECUTOR) { assert(oparg < 256); _PyExecutorObject *executor = code->co_executors->executors[oparg]; @@ -486,21 +595,25 @@ translate_bytecode_to_trace( int counter = instr[1].cache; int bitcount = _Py_popcount32(counter); int jump_likely = bitcount > 8; + /* If bitcount is 8 (half the jumps were taken), adjust confidence by 50%. + If it's 16 or 0 (all or none were taken), adjust by 10% + (since the future is still somewhat uncertain). + For values in between, adjust proportionally. */ if (jump_likely) { - confidence = confidence * bitcount / 16; + confidence = confidence * (bitcount + 2) / 20; } else { - confidence = confidence * (16 - bitcount) / 16; + confidence = confidence * (18 - bitcount) / 20; } + uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; + DPRINTF(2, "%d: %s(%d): counter=%x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", + target, _PyOpcode_OpName[opcode], oparg, + counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); if (confidence < CONFIDENCE_CUTOFF) { - DPRINTF(2, "Confidence too low (%d)\n", confidence); + DPRINTF(2, "Confidence too low (%d < %d)\n", confidence, CONFIDENCE_CUTOFF); OPT_STAT_INC(low_confidence); goto done; } - uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - DPRINTF(2, "%s(%d): counter=%x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", - _PyOpcode_OpName[opcode], oparg, - counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; _Py_CODEUNIT *target_instr = next_instr + oparg; if (jump_likely) { @@ -726,9 +839,10 @@ translate_bytecode_to_trace( * NOPs are excluded from the count. */ static int -compute_used(_PyUOpInstruction *buffer, uint32_t *used) +compute_used(_PyUOpInstruction *buffer, uint32_t *used, int *exit_count_ptr) { int count = 0; + int exit_count = 0; SET_BIT(used, 0); for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { if (!BIT_IS_SET(used, i)) { @@ -736,6 +850,9 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used) } count++; int opcode = buffer[i].opcode; + if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { + exit_count++; + } if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { continue; } @@ -751,44 +868,76 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used) UNSET_BIT(used, i); } } + *exit_count_ptr = exit_count; return count; } +/* Executor side exits */ + +static _PyExecutorObject * +allocate_executor(int exit_count, int length) +{ + int size = exit_count*sizeof(_PyExitData) + length*sizeof(_PyUOpInstruction); + _PyExecutorObject *res = PyObject_GC_NewVar(_PyExecutorObject, &_PyUOpExecutor_Type, size); + if (res == NULL) { + return NULL; + } + res->trace = (_PyUOpInstruction *)(res->exits + exit_count); + res->code_size = length; + res->exit_count = exit_count; + return res; +} + /* Makes an executor from a buffer of uops. * Account for the buffer having gaps and NOPs by computing a "used" * bit vector and only copying the used uops. Here "used" means reachable * and not a NOP. */ static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) +make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies) { uint32_t used[(UOP_MAX_TRACE_LENGTH + 31)/32] = { 0 }; - int length = compute_used(buffer, used); - _PyExecutorObject *executor = PyObject_NewVar(_PyExecutorObject, &_PyUOpExecutor_Type, length); + int exit_count; + int length = compute_used(buffer, used, &exit_count); + _PyExecutorObject *executor = allocate_executor(exit_count, length+1); if (executor == NULL) { return NULL; } - int dest = length - 1; + /* Initialize exits */ + for (int i = 0; i < exit_count; i++) { + executor->exits[i].executor = &COLD_EXITS[i]; + executor->exits[i].temperature = 0; + } + int next_exit = exit_count-1; + _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; /* Scan backwards, so that we see the destinations of jumps before the jumps themselves. */ for (int i = UOP_MAX_TRACE_LENGTH-1; i >= 0; i--) { if (!BIT_IS_SET(used, i)) { continue; } - executor->trace[dest] = buffer[i]; + *dest = buffer[i]; int opcode = buffer[i].opcode; if (opcode == _POP_JUMP_IF_FALSE || opcode == _POP_JUMP_IF_TRUE) { /* The oparg of the target will already have been set to its new offset */ - int oparg = executor->trace[dest].oparg; - executor->trace[dest].oparg = buffer[oparg].oparg; + int oparg = dest->oparg; + dest->oparg = buffer[oparg].oparg; + } + if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { + executor->exits[next_exit].target = buffer[i].target; + dest->exit_index = next_exit; + next_exit--; } /* Set the oparg to be the destination offset, * so that we can set the oparg of earlier jumps correctly. */ - buffer[i].oparg = dest; + buffer[i].oparg = (uint16_t)(dest - executor->trace); dest--; } - assert(dest == -1); + assert(next_exit == -1); + assert(dest == executor->trace); + dest->opcode = _START_EXECUTOR; + dest->operand = (uintptr_t)executor; _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -799,26 +948,49 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) if (lltrace >= 2) { printf("Optimized executor (length %d):\n", length); for (int i = 0; i < length; i++) { - printf("%4d %s(%d, %d, %" PRIu64 ")\n", - i, - _PyUOpName(executor->trace[i].opcode), - executor->trace[i].oparg, - executor->trace[i].target, - executor->trace[i].operand); + printf("%4d OPTIMIZED: ", i); + _PyUOpPrint(&executor->trace[i]); + printf("\n"); } } #endif #ifdef _Py_JIT executor->jit_code = NULL; executor->jit_size = 0; - if (_PyJIT_Compile(executor, executor->trace, Py_SIZE(executor))) { + if (_PyJIT_Compile(executor, executor->trace, length+1)) { Py_DECREF(executor); return NULL; } #endif + _PyObject_GC_TRACK(executor); return executor; } +static int +init_cold_exit_executor(_PyExecutorObject *executor, int oparg) +{ + _Py_SetImmortal(executor); + Py_SET_TYPE(executor, &_PyUOpExecutor_Type); + executor->trace = (_PyUOpInstruction *)executor->exits; + executor->code_size = 1; + executor->exit_count = 0; + _PyUOpInstruction *inst = (_PyUOpInstruction *)&executor->trace[0]; + inst->opcode = _COLD_EXIT; + inst->oparg = oparg; + executor->vm_data.valid = true; + for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { + assert(executor->vm_data.bloom.bits[i] == 0); + } +#ifdef _Py_JIT + executor->jit_code = NULL; + executor->jit_size = 0; + if (_PyJIT_Compile(executor, executor->trace, 1)) { + return -1; + } +#endif + return 0; +} + static int uop_optimize( _PyOptimizerObject *self, @@ -836,8 +1008,8 @@ uop_optimize( return err; } OPT_STAT_INC(traces_created); - char *uop_optimize = Py_GETENV("PYTHONUOPSOPTIMIZE"); - if (uop_optimize == NULL || *uop_optimize > '0') { + char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); + if (env_var == NULL || *env_var == '\0' || *env_var > '0') { err = _Py_uop_analyze_and_optimize(frame, buffer, UOP_MAX_TRACE_LENGTH, curr_stackentries, &dependencies); @@ -846,6 +1018,21 @@ uop_optimize( } } assert(err == 1); + /* Fix up */ + for (int pc = 0; pc < UOP_MAX_TRACE_LENGTH; pc++) { + int opcode = buffer[pc].opcode; + int oparg = buffer[pc].oparg; + if (_PyUop_Flags[opcode] & HAS_OPARG_AND_1_FLAG) { + buffer[pc].opcode = opcode + 1 + (oparg & 1); + } + else if (oparg < _PyUop_Replication[opcode]) { + buffer[pc].opcode = opcode + oparg + 1; + } + else if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { + break; + } + assert(_PyOpcode_uop_name[buffer[pc].opcode]); + } _PyExecutorObject *executor = make_executor_from_uops(buffer, &dependencies); if (executor == NULL) { return -1; @@ -880,13 +1067,15 @@ PyUnstable_Optimizer_NewUOpOptimizer(void) opt->resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD; // Need a few iterations to settle specializations, // and to ammortize the cost of optimization. + opt->side_threshold = 16; opt->backedge_threshold = 16; return (PyObject *)opt; } static void counter_dealloc(_PyExecutorObject *self) { - PyObject *opt = (PyObject *)self->trace[0].operand; + /* The optimizer is the operand of the second uop. */ + PyObject *opt = (PyObject *)self->trace[1].operand; Py_DECREF(opt); uop_dealloc(self); } @@ -894,11 +1083,13 @@ counter_dealloc(_PyExecutorObject *self) { PyTypeObject _PyCounterExecutor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "counting_executor", - .tp_basicsize = offsetof(_PyExecutorObject, trace), - .tp_itemsize = sizeof(_PyUOpInstruction), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_basicsize = offsetof(_PyExecutorObject, exits), + .tp_itemsize = 1, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC, .tp_dealloc = (destructor)counter_dealloc, .tp_methods = executor_methods, + .tp_traverse = executor_traverse, + .tp_clear = executor_clear, }; static int @@ -926,9 +1117,7 @@ counter_optimize( { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)) } }; - _PyBloomFilter empty; - _Py_BloomFilter_Init(&empty); - _PyExecutorObject *executor = make_executor_from_uops(buffer, &empty); + _PyExecutorObject *executor = make_executor_from_uops(buffer, &EMPTY_FILTER); if (executor == NULL) { return -1; } @@ -968,6 +1157,7 @@ PyUnstable_Optimizer_NewCounter(void) } opt->base.optimize = counter_optimize; opt->base.resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD; + opt->base.side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD; opt->base.backedge_threshold = 0; opt->count = 0; return (PyObject *)opt; @@ -1091,9 +1281,6 @@ link_executor(_PyExecutorObject *executor) static void unlink_executor(_PyExecutorObject *executor) { - if (!executor->vm_data.valid) { - return; - } _PyExecutorLinkListNode *links = &executor->vm_data.links; _PyExecutorObject *next = links->next; _PyExecutorObject *prev = links->previous; @@ -1114,7 +1301,7 @@ unlink_executor(_PyExecutorObject *executor) /* This must be called by optimizers before using the executor */ void -_Py_ExecutorInit(_PyExecutorObject *executor, _PyBloomFilter *dependency_set) +_Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_set) { executor->vm_data.valid = true; for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { @@ -1127,11 +1314,19 @@ _Py_ExecutorInit(_PyExecutorObject *executor, _PyBloomFilter *dependency_set) void _Py_ExecutorClear(_PyExecutorObject *executor) { + if (!executor->vm_data.valid) { + return; + } unlink_executor(executor); PyCodeObject *code = executor->vm_data.code; if (code == NULL) { return; } + for (uint32_t i = 0; i < executor->exit_count; i++) { + Py_DECREF(executor->exits[i].executor); + executor->exits[i].executor = &COLD_EXITS[i]; + executor->exits[i].temperature = INT16_MIN; + } _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; assert(instruction->op.code == ENTER_EXECUTOR); int index = instruction->op.arg; @@ -1153,7 +1348,7 @@ _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj) * May cause other executors to be invalidated as well */ void -_Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) +_Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation) { _PyBloomFilter obj_filter; _Py_BloomFilter_Init(&obj_filter); @@ -1165,6 +1360,9 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) _PyExecutorObject *next = exec->vm_data.links.next; if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) { _Py_ExecutorClear(exec); + if (is_invalidation) { + OPT_STAT_INC(executors_invalidated); + } } exec = next; } @@ -1172,7 +1370,7 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj) /* Invalidate all executors */ void -_Py_Executors_InvalidateAll(PyInterpreterState *interp) +_Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) { while (interp->executor_list_head) { _PyExecutorObject *executor = interp->executor_list_head; @@ -1183,5 +1381,8 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp) else { _Py_ExecutorClear(executor); } + if (is_invalidation) { + OPT_STAT_INC(executors_invalidated); + } } } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index d73bc310345f41..a326e2249bb4de 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -1,5 +1,5 @@ /* - * This file contains the support code for CPython's uops redundancy eliminator. + * This file contains the support code for CPython's uops optimizer. * It also performs some simple optimizations. * It performs a traditional data-flow analysis[1] over the trace of uops. * Using the information gained, it chooses to emit, or skip certain instructions @@ -33,17 +33,8 @@ #include #include -// Holds locals, stack, locals, stack ... co_consts (in that order) -#define MAX_ABSTRACT_INTERP_SIZE 4096 - -#define OVERALLOCATE_FACTOR 5 - -#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * OVERALLOCATE_FACTOR) - -// Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) -#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) - #ifdef Py_DEBUG + extern const char *_PyUOpName(int index); static const char *const DEBUG_ENV = "PYTHON_OPT_DEBUG"; static inline int get_lltrace(void) { char *uop_debug = Py_GETENV(DEBUG_ENV); @@ -60,321 +51,6 @@ #endif -// Flags for below. -#define KNOWN 1 << 0 -#define TRUE_CONST 1 << 1 -#define IS_NULL 1 << 2 -#define NOT_NULL 1 << 3 - -typedef struct { - int flags; - PyTypeObject *typ; - // constant propagated value (might be NULL) - PyObject *const_val; -} _Py_UOpsSymType; - - -typedef struct _Py_UOpsAbstractFrame { - // Max stacklen - int stack_len; - int locals_len; - - _Py_UOpsSymType **stack_pointer; - _Py_UOpsSymType **stack; - _Py_UOpsSymType **locals; -} _Py_UOpsAbstractFrame; - - -typedef struct ty_arena { - int ty_curr_number; - int ty_max_number; - _Py_UOpsSymType arena[TY_ARENA_SIZE]; -} ty_arena; - -// Tier 2 types meta interpreter -typedef struct _Py_UOpsAbstractInterpContext { - PyObject_HEAD - // The current "executing" frame. - _Py_UOpsAbstractFrame *frame; - _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; - int curr_frame_depth; - - // Arena for the symbolic types. - ty_arena t_arena; - - _Py_UOpsSymType **n_consumed; - _Py_UOpsSymType **limit; - _Py_UOpsSymType *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; -} _Py_UOpsAbstractInterpContext; - -static inline _Py_UOpsSymType* sym_new_unknown(_Py_UOpsAbstractInterpContext *ctx); - -// 0 on success, -1 on error. -static _Py_UOpsAbstractFrame * -ctx_frame_new( - _Py_UOpsAbstractInterpContext *ctx, - PyCodeObject *co, - _Py_UOpsSymType **localsplus_start, - int n_locals_already_filled, - int curr_stackentries -) -{ - assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); - _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; - - frame->stack_len = co->co_stacksize; - frame->locals_len = co->co_nlocalsplus; - - frame->locals = localsplus_start; - frame->stack = frame->locals + co->co_nlocalsplus; - frame->stack_pointer = frame->stack + curr_stackentries; - ctx->n_consumed = localsplus_start + (co->co_nlocalsplus + co->co_stacksize); - if (ctx->n_consumed >= ctx->limit) { - return NULL; - } - - - // Initialize with the initial state of all local variables - for (int i = n_locals_already_filled; i < co->co_nlocalsplus; i++) { - _Py_UOpsSymType *local = sym_new_unknown(ctx); - if (local == NULL) { - return NULL; - } - frame->locals[i] = local; - } - - - // Initialize the stack as well - for (int i = 0; i < curr_stackentries; i++) { - _Py_UOpsSymType *stackvar = sym_new_unknown(ctx); - if (stackvar == NULL) { - return NULL; - } - frame->stack[i] = stackvar; - } - - return frame; -} - -static void -abstractcontext_fini(_Py_UOpsAbstractInterpContext *ctx) -{ - if (ctx == NULL) { - return; - } - ctx->curr_frame_depth = 0; - int tys = ctx->t_arena.ty_curr_number; - for (int i = 0; i < tys; i++) { - Py_CLEAR(ctx->t_arena.arena[i].const_val); - } -} - -static int -abstractcontext_init( - _Py_UOpsAbstractInterpContext *ctx, - PyCodeObject *co, - int curr_stacklen, - int ir_entries -) -{ - ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; - ctx->n_consumed = ctx->locals_and_stack; -#ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. - for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { - ctx->locals_and_stack[i] = NULL; - } -#endif - - // Setup the arena for sym expressions. - ctx->t_arena.ty_curr_number = 0; - ctx->t_arena.ty_max_number = TY_ARENA_SIZE; - - // Frame setup - ctx->curr_frame_depth = 0; - _Py_UOpsAbstractFrame *frame = ctx_frame_new(ctx, co, ctx->n_consumed, 0, curr_stacklen); - if (frame == NULL) { - return -1; - } - ctx->curr_frame_depth++; - ctx->frame = frame; - return 0; -} - - -static int -ctx_frame_pop( - _Py_UOpsAbstractInterpContext *ctx -) -{ - _Py_UOpsAbstractFrame *frame = ctx->frame; - - ctx->n_consumed = frame->locals; - ctx->curr_frame_depth--; - assert(ctx->curr_frame_depth >= 1); - ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; - - return 0; -} - - -// Takes a borrowed reference to const_val, turns that into a strong reference. -static _Py_UOpsSymType* -sym_new(_Py_UOpsAbstractInterpContext *ctx, - PyObject *const_val) -{ - _Py_UOpsSymType *self = &ctx->t_arena.arena[ctx->t_arena.ty_curr_number]; - if (ctx->t_arena.ty_curr_number >= ctx->t_arena.ty_max_number) { - OPT_STAT_INC(optimizer_failure_reason_no_memory); - DPRINTF(1, "out of space for symbolic expression type\n"); - return NULL; - } - ctx->t_arena.ty_curr_number++; - self->const_val = NULL; - self->typ = NULL; - self->flags = 0; - - if (const_val != NULL) { - self->const_val = Py_NewRef(const_val); - } - - return self; -} - -static inline void -sym_set_flag(_Py_UOpsSymType *sym, int flag) -{ - sym->flags |= flag; -} - -static inline void -sym_clear_flag(_Py_UOpsSymType *sym, int flag) -{ - sym->flags &= (~flag); -} - -static inline bool -sym_has_flag(_Py_UOpsSymType *sym, int flag) -{ - return (sym->flags & flag) != 0; -} - -static inline bool -sym_is_known(_Py_UOpsSymType *sym) -{ - return sym_has_flag(sym, KNOWN); -} - -static inline bool -sym_is_not_null(_Py_UOpsSymType *sym) -{ - return (sym->flags & (IS_NULL | NOT_NULL)) == NOT_NULL; -} - -static inline bool -sym_is_null(_Py_UOpsSymType *sym) -{ - return (sym->flags & (IS_NULL | NOT_NULL)) == IS_NULL; -} - -static inline void -sym_set_type(_Py_UOpsSymType *sym, PyTypeObject *tp) -{ - assert(PyType_Check(tp)); - sym->typ = tp; - sym_set_flag(sym, KNOWN); - sym_set_flag(sym, NOT_NULL); -} - -static inline void -sym_set_null(_Py_UOpsSymType *sym) -{ - sym_set_flag(sym, IS_NULL); - sym_set_flag(sym, KNOWN); -} - - -static inline _Py_UOpsSymType* -sym_new_unknown(_Py_UOpsAbstractInterpContext *ctx) -{ - return sym_new(ctx,NULL); -} - -static inline _Py_UOpsSymType* -sym_new_known_notnull(_Py_UOpsAbstractInterpContext *ctx) -{ - _Py_UOpsSymType *res = sym_new_unknown(ctx); - if (res == NULL) { - return NULL; - } - sym_set_flag(res, NOT_NULL); - return res; -} - -static inline _Py_UOpsSymType* -sym_new_known_type(_Py_UOpsAbstractInterpContext *ctx, - PyTypeObject *typ) -{ - _Py_UOpsSymType *res = sym_new(ctx,NULL); - if (res == NULL) { - return NULL; - } - sym_set_type(res, typ); - return res; -} - -// Takes a borrowed reference to const_val. -static inline _Py_UOpsSymType* -sym_new_const(_Py_UOpsAbstractInterpContext *ctx, PyObject *const_val) -{ - assert(const_val != NULL); - _Py_UOpsSymType *temp = sym_new( - ctx, - const_val - ); - if (temp == NULL) { - return NULL; - } - sym_set_type(temp, Py_TYPE(const_val)); - sym_set_flag(temp, TRUE_CONST); - sym_set_flag(temp, KNOWN); - sym_set_flag(temp, NOT_NULL); - return temp; -} - -static inline bool -is_const(_Py_UOpsSymType *sym) -{ - return sym->const_val != NULL; -} - -static inline PyObject * -get_const(_Py_UOpsSymType *sym) -{ - return sym->const_val; -} - -static _Py_UOpsSymType* -sym_new_null(_Py_UOpsAbstractInterpContext *ctx) -{ - _Py_UOpsSymType *null_sym = sym_new_unknown(ctx); - if (null_sym == NULL) { - return NULL; - } - sym_set_null(null_sym); - return null_sym; -} - - -static inline bool -sym_matches_type(_Py_UOpsSymType *sym, PyTypeObject *typ) -{ - assert(typ == NULL || PyType_Check(typ)); - if (!sym_has_flag(sym, KNOWN)) { - return false; - } - return sym->typ == typ; -} - static inline bool op_is_end(uint32_t opcode) @@ -407,24 +83,27 @@ globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, { RARE_EVENT_STAT_INC(watched_globals_modification); assert(get_mutations(dict) < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS); - _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict); + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict, 1); increment_mutations(dict); PyDict_Unwatch(GLOBALS_WATCHER_ID, dict); return 0; } -static void -global_to_const(_PyUOpInstruction *inst, PyObject *obj) +static PyObject * +convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) { - assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS); + assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS || inst->opcode == _LOAD_ATTR_MODULE); assert(PyDict_CheckExact(obj)); PyDictObject *dict = (PyDictObject *)obj; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); assert(inst->operand <= UINT16_MAX); + if ((int)inst->operand >= dict->ma_keys->dk_nentries) { + return NULL; + } PyObject *res = entries[inst->operand].me_value; if (res == NULL) { - return; + return NULL; } if (_Py_IsImmortal(res)) { inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_BORROW_WITH_NULL : _LOAD_CONST_INLINE_BORROW; @@ -433,6 +112,7 @@ global_to_const(_PyUOpInstruction *inst, PyObject *obj) inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_WITH_NULL : _LOAD_CONST_INLINE; } inst->operand = (uint64_t)res; + return res; } static int @@ -529,12 +209,12 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, break; case _LOAD_GLOBAL_BUILTINS: if (globals_checked & builtins_checked & globals_watched & builtins_watched & 1) { - global_to_const(inst, builtins); + convert_global_to_const(inst, builtins); } break; case _LOAD_GLOBAL_MODULE: if (globals_checked & globals_watched & 1) { - global_to_const(inst, globals); + convert_global_to_const(inst, globals); } break; case _PUSH_FRAME: @@ -588,38 +268,63 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, INST->oparg = ARG; \ INST->operand = OPERAND; +#define OUT_OF_SPACE_IF_NULL(EXPR) \ + do { \ + if ((EXPR) == NULL) { \ + goto out_of_space; \ + } \ + } while (0); + #define _LOAD_ATTR_NOT_NULL \ do { \ - attr = sym_new_known_notnull(ctx); \ - if (attr == NULL) { \ - goto error; \ - } \ - null = sym_new_null(ctx); \ - if (null == NULL) { \ - goto error; \ - } \ + OUT_OF_SPACE_IF_NULL(attr = _Py_uop_sym_new_not_null(ctx)); \ + OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); \ } while (0); +/* Shortened forms for convenience, used in optimizer_bytecodes.c */ +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_set_null _Py_uop_sym_set_null +#define sym_set_non_null _Py_uop_sym_set_non_null +#define sym_set_type _Py_uop_sym_set_type +#define sym_set_const _Py_uop_sym_set_const +#define sym_is_bottom _Py_uop_sym_is_bottom +#define frame_new _Py_uop_frame_new +#define frame_pop _Py_uop_frame_pop + + /* 1 for success, 0 for not ready, cannot error at the moment. */ static int -uop_redundancy_eliminator( +optimize_uops( PyCodeObject *co, _PyUOpInstruction *trace, int trace_len, - int curr_stacklen + int curr_stacklen, + _PyBloomFilter *dependencies ) { - _Py_UOpsAbstractInterpContext context; - _Py_UOpsAbstractInterpContext *ctx = &context; + _Py_UOpsContext context; + _Py_UOpsContext *ctx = &context; - if (abstractcontext_init( - ctx, - co, curr_stacklen, - trace_len) < 0) { + if (_Py_uop_abstractcontext_init(ctx) < 0) { goto out_of_space; } + _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, ctx->n_consumed, 0, curr_stacklen); + if (frame == NULL) { + return -1; + } + ctx->curr_frame_depth++; + ctx->frame = frame; for (_PyUOpInstruction *this_instr = trace; this_instr < trace + trace_len && !op_is_end(this_instr->opcode); @@ -628,13 +333,13 @@ uop_redundancy_eliminator( int oparg = this_instr->oparg; uint32_t opcode = this_instr->opcode; - _Py_UOpsSymType **stack_pointer = ctx->frame->stack_pointer; + _Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer; DPRINTF(3, "Abstract interpreting %s:%d ", - _PyOpcode_uop_name[opcode], + _PyUOpName(opcode), oparg); switch (opcode) { -#include "tier2_redundancy_eliminator_cases.c.h" +#include "optimizer_cases.c.h" default: DPRINTF(1, "Unknown opcode in abstract interpreter\n"); @@ -646,17 +351,26 @@ uop_redundancy_eliminator( assert(STACK_LEVEL() >= 0); } - abstractcontext_fini(ctx); + _Py_uop_abstractcontext_fini(ctx); return 1; out_of_space: DPRINTF(1, "Out of space in abstract interpreter\n"); - abstractcontext_fini(ctx); + _Py_uop_abstractcontext_fini(ctx); return 0; error: DPRINTF(1, "Encountered error in abstract interpreter\n"); - abstractcontext_fini(ctx); + _Py_uop_abstractcontext_fini(ctx); + return 0; + +hit_bottom: + // Attempted to push a "bottom" (contradition) symbol onto the stack. + // This means that the abstract interpreter has hit unreachable code. + // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but it's + // simpler to just admit failure and not create the executor. + DPRINTF(1, "Hit bottom in abstract interpreter\n"); + _Py_uop_abstractcontext_fini(ctx); return 0; } @@ -669,7 +383,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) * could error. _CHECK_VALIDITY is needed if the previous * instruction could have escaped. */ int last_set_ip = -1; - bool may_have_escaped = false; + bool may_have_escaped = true; for (int pc = 0; pc < buffer_size; pc++) { int opcode = buffer[pc].opcode; switch (opcode) { @@ -695,6 +409,22 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } last_set_ip = pc; break; + case _POP_TOP: + { + _PyUOpInstruction *last = &buffer[pc-1]; + while (last->opcode == _NOP) { + last--; + } + if (last->opcode == _LOAD_CONST_INLINE || + last->opcode == _LOAD_CONST_INLINE_BORROW || + last->opcode == _LOAD_FAST || + last->opcode == _COPY + ) { + last->opcode = _NOP; + buffer[pc].opcode = NOP; + } + break; + } case _JUMP_TO_TOP: case _EXIT_TRACE: return; @@ -708,9 +438,6 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { needs_ip = true; } - if (opcode == _PUSH_FRAME) { - needs_ip = true; - } if (needs_ip && last_set_ip >= 0) { if (buffer[last_set_ip].opcode == _CHECK_VALIDITY) { buffer[last_set_ip].opcode = _CHECK_VALIDITY_AND_SET_IP; @@ -793,9 +520,9 @@ _Py_uop_analyze_and_optimize( peephole_opt(frame, buffer, buffer_size); - err = uop_redundancy_eliminator( + err = optimize_uops( (PyCodeObject *)frame->f_executable, buffer, - buffer_size, curr_stacklen); + buffer_size, curr_stacklen, dependencies); if (err == 0) { goto not_ready; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c new file mode 100644 index 00000000000000..786d884fc5a1a8 --- /dev/null +++ b/Python/optimizer_bytecodes.c @@ -0,0 +1,548 @@ +#include "Python.h" +#include "pycore_optimizer.h" +#include "pycore_uops.h" +#include "pycore_uop_ids.h" +#include "internal/pycore_moduleobject.h" + +#define op(name, ...) /* NAME is ignored */ + +typedef struct _Py_UopsSymbol _Py_UopsSymbol; +typedef struct _Py_UOpsContext _Py_UOpsContext; +typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; + +/* Shortened forms for convenience */ +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_set_null _Py_uop_sym_set_null +#define sym_set_non_null _Py_uop_sym_set_non_null +#define sym_set_type _Py_uop_sym_set_type +#define sym_set_const _Py_uop_sym_set_const +#define sym_is_bottom _Py_uop_sym_is_bottom +#define frame_new _Py_uop_frame_new +#define frame_pop _Py_uop_frame_pop + +static int +dummy_func(void) { + + PyCodeObject *code; + int oparg; + _Py_UopsSymbol *flag; + _Py_UopsSymbol *left; + _Py_UopsSymbol *right; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + _Py_UopsSymbol *iter; + _Py_UopsSymbol *top; + _Py_UopsSymbol *bottom; + _Py_UOpsAbstractFrame *frame; + _Py_UOpsContext *ctx; + _PyUOpInstruction *this_instr; + _PyBloomFilter *dependencies; + int modified; + +// BEGIN BYTECODES // + + op(_LOAD_FAST_CHECK, (-- value)) { + value = GETLOCAL(oparg); + // We guarantee this will error - just bail and don't optimize it. + if (sym_is_null(value)) { + goto out_of_space; + } + } + + op(_LOAD_FAST, (-- value)) { + value = GETLOCAL(oparg); + } + + op(_LOAD_FAST_AND_CLEAR, (-- value)) { + value = GETLOCAL(oparg); + _Py_UopsSymbol *temp; + OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx)); + GETLOCAL(oparg) = temp; + } + + op(_STORE_FAST, (value --)) { + GETLOCAL(oparg) = value; + } + + op(_PUSH_NULL, (-- res)) { + res = sym_new_null(ctx); + if (res == NULL) { + goto out_of_space; + }; + } + + op(_GUARD_BOTH_INT, (left, right -- left, right)) { + if (sym_matches_type(left, &PyLong_Type) && + sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + if (!sym_set_type(left, &PyLong_Type)) { + goto hit_bottom; + } + if (!sym_set_type(right, &PyLong_Type)) { + goto hit_bottom; + } + } + + op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { + if (sym_matches_type(left, &PyFloat_Type) && + sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + if (!sym_set_type(left, &PyFloat_Type)) { + goto hit_bottom; + } + if (!sym_set_type(right, &PyFloat_Type)) { + goto hit_bottom; + } + } + + op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { + if (sym_matches_type(left, &PyUnicode_Type) && + sym_matches_type(right, &PyUnicode_Type)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + if (!sym_set_type(left, &PyUnicode_Type)) { + goto hit_bottom; + } + if (!sym_set_type(right, &PyUnicode_Type)) { + goto hit_bottom; + } + } + + op(_BINARY_OP_ADD_INT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + } + } + + op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + } + } + + op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); + } + } + + op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) + + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + } + } + + op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) - + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + } + } + + op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) * + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + } + } + + op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { + PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyUnicode_Type)); + } + } + + op(_TO_BOOL, (value -- res)) { + (void)value; + res = sym_new_type(ctx, &PyBool_Type); + OUT_OF_SPACE_IF_NULL(res); + } + + op(_TO_BOOL_BOOL, (value -- value)) { + if (sym_matches_type(value, &PyBool_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + if(!sym_set_type(value, &PyBool_Type)) { + goto hit_bottom; + } + } + } + + op(_TO_BOOL_INT, (value -- res)) { + if (sym_is_const(value) && sym_matches_type(value, &PyLong_Type)) { + PyObject *load = _PyLong_IsZero((PyLongObject *)sym_get_const(value)) + ? Py_False : Py_True; + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + } + if(!sym_set_type(value, &PyLong_Type)) { + goto hit_bottom; + } + } + + op(_TO_BOOL_LIST, (value -- res)) { + if(!sym_set_type(value, &PyList_Type)) { + goto hit_bottom; + } + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + } + + op(_TO_BOOL_NONE, (value -- res)) { + if (sym_get_const(value) == Py_None) { + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_False); + } + sym_set_const(value, Py_None); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, Py_False)); + } + + op(_TO_BOOL_STR, (value -- res)) { + if (sym_is_const(value) && sym_matches_type(value, &PyUnicode_Type)) { + PyObject *load = sym_get_const(value) == &_Py_STR(empty) ? Py_False : Py_True; + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + } + if(!sym_set_type(value, &PyUnicode_Type)) { + goto hit_bottom; + } + } + + op(_LOAD_CONST, (-- value)) { + // There should be no LOAD_CONST. It should be all + // replaced by peephole_opt. + Py_UNREACHABLE(); + } + + op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + } + + op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + } + + op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + } + + op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + } + + + op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { + assert(oparg > 0); + top = bottom; + } + + op(_SWAP, (bottom, unused[oparg-2], top -- + top, unused[oparg-2], bottom)) { + } + + op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { + _LOAD_ATTR_NOT_NULL + (void)index; + (void)owner; + } + + op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) { + (void)dict_version; + if (sym_is_const(owner)) { + PyObject *cnst = sym_get_const(owner); + if (PyModule_CheckExact(cnst)) { + PyModuleObject *mod = (PyModuleObject *)cnst; + PyObject *dict = mod->md_dict; + uint64_t watched_mutations = get_mutations(dict); + if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + PyDict_Watch(GLOBALS_WATCHER_ID, dict); + _Py_BloomFilter_Add(dependencies, dict); + this_instr->opcode = _NOP; + } + } + } + } + + op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { + (void)index; + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + attr = NULL; + if (this_instr[-1].opcode == _NOP) { + // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. + assert(sym_is_const(owner)); + PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); + assert(PyModule_CheckExact(mod)); + PyObject *dict = mod->md_dict; + PyObject *res = convert_global_to_const(this_instr, dict); + if (res != NULL) { + this_instr[-1].opcode = _POP_TOP; + OUT_OF_SPACE_IF_NULL(attr = sym_new_const(ctx, res)); + } + } + if (attr == NULL) { + /* No conversion made. We don't know what `attr` is. */ + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + } + } + + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { + _LOAD_ATTR_NOT_NULL + (void)hint; + (void)owner; + } + + op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { + _LOAD_ATTR_NOT_NULL + (void)index; + (void)owner; + } + + op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { + _LOAD_ATTR_NOT_NULL + (void)descr; + (void)owner; + } + + op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { + (void)descr; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + self = owner; + } + + op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { + (void)descr; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + self = owner; + } + + op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self if (1))) { + (void)descr; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + self = owner; + } + + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { + (void)callable; + OUT_OF_SPACE_IF_NULL(func = sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(self = sym_new_not_null(ctx)); + } + + + op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + if (!sym_set_type(callable, &PyFunction_Type)) { + goto hit_bottom; + } + (void)self_or_null; + (void)func_version; + } + + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { + if (!sym_set_null(null)) { + goto hit_bottom; + } + if (!sym_set_type(callable, &PyMethod_Type)) { + goto hit_bottom; + } + } + + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + int argcount = oparg; + + (void)callable; + + PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; + if (func == NULL) { + goto error; + } + PyCodeObject *co = (PyCodeObject *)func->func_code; + + assert(self_or_null != NULL); + assert(args != NULL); + if (sym_is_not_null(self_or_null)) { + // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM + args--; + argcount++; + } + + _Py_UopsSymbol **localsplus_start = ctx->n_consumed; + int n_locals_already_filled = 0; + // Can determine statically, so we interleave the new locals + // and make the current stack the new locals. + // This also sets up for true call inlining. + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + localsplus_start = args; + n_locals_already_filled = argcount; + } + OUT_OF_SPACE_IF_NULL(new_frame = + frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); + } + + op(_POP_FRAME, (retval -- res)) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = retval; + } + + op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) { + SYNC_SP(); + ctx->frame->stack_pointer = stack_pointer; + ctx->frame = new_frame; + ctx->curr_frame_depth++; + stack_pointer = new_frame->stack_pointer; + } + + op(_UNPACK_SEQUENCE, (seq -- values[oparg])) { + /* This has to be done manually */ + (void)seq; + for (int i = 0; i < oparg; i++) { + OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + } + } + + op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8])) { + /* This has to be done manually */ + (void)seq; + int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; + for (int i = 0; i < totalargs; i++) { + OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); + } + } + + op(_ITER_NEXT_RANGE, (iter -- iter, next)) { + OUT_OF_SPACE_IF_NULL(next = sym_new_type(ctx, &PyLong_Type)); + (void)iter; + } + + + + +// END BYTECODES // + +} diff --git a/Python/tier2_redundancy_eliminator_cases.c.h b/Python/optimizer_cases.c.h similarity index 66% rename from Python/tier2_redundancy_eliminator_cases.c.h rename to Python/optimizer_cases.c.h index a9617f51ef4615..6d3488f2118589 100644 --- a/Python/tier2_redundancy_eliminator_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1,6 +1,6 @@ -// This file is generated by Tools/cases_generator/tier2_abstract_generator.py +// This file is generated by Tools/cases_generator/optimizer_generator.py // from: -// Python/tier2_redundancy_eliminator_bytecodes.c +// Python/optimizer_bytecodes.c // Do not edit! case _NOP: { @@ -14,7 +14,7 @@ /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. if (sym_is_null(value)) { @@ -26,7 +26,7 @@ } case _LOAD_FAST: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = GETLOCAL(oparg); stack_pointer[0] = value; stack_pointer += 1; @@ -34,12 +34,10 @@ } case _LOAD_FAST_AND_CLEAR: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = GETLOCAL(oparg); - _Py_UOpsSymType *temp = sym_new_null(ctx); - if (temp == NULL) { - goto out_of_space; - } + _Py_UopsSymbol *temp; + OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx)); GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; @@ -47,7 +45,7 @@ } case _LOAD_CONST: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; // There should be no LOAD_CONST. It should be all // replaced by peephole_opt. Py_UNREACHABLE(); @@ -57,7 +55,7 @@ } case _STORE_FAST: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; stack_pointer += -1; @@ -70,7 +68,7 @@ } case _PUSH_NULL: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_null(ctx); if (res == NULL) { goto out_of_space; @@ -81,7 +79,7 @@ } case _END_SEND: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[-2] = value; @@ -90,7 +88,7 @@ } case _UNARY_NEGATIVE: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -98,7 +96,7 @@ } case _UNARY_NOT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -106,51 +104,96 @@ } case _TO_BOOL: { - _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + value = stack_pointer[-1]; + (void)value; + res = sym_new_type(ctx, &PyBool_Type); + OUT_OF_SPACE_IF_NULL(res); stack_pointer[-1] = res; break; } case _TO_BOOL_BOOL: { + _Py_UopsSymbol *value; + value = stack_pointer[-1]; + if (sym_matches_type(value, &PyBool_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + if(!sym_set_type(value, &PyBool_Type)) { + goto hit_bottom; + } + } break; } case _TO_BOOL_INT: { - _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + value = stack_pointer[-1]; + if (sym_is_const(value) && sym_matches_type(value, &PyLong_Type)) { + PyObject *load = _PyLong_IsZero((PyLongObject *)sym_get_const(value)) + ? Py_False : Py_True; + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + } + if(!sym_set_type(value, &PyLong_Type)) { + goto hit_bottom; + } stack_pointer[-1] = res; break; } case _TO_BOOL_LIST: { - _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + value = stack_pointer[-1]; + if(!sym_set_type(value, &PyList_Type)) { + goto hit_bottom; + } + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); stack_pointer[-1] = res; break; } case _TO_BOOL_NONE: { - _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + value = stack_pointer[-1]; + if (sym_get_const(value) == Py_None) { + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_False); + } + sym_set_const(value, Py_None); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, Py_False)); stack_pointer[-1] = res; break; } case _TO_BOOL_STR: { - _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + _Py_UopsSymbol *value; + _Py_UopsSymbol *res; + value = stack_pointer[-1]; + if (sym_is_const(value) && sym_matches_type(value, &PyUnicode_Type)) { + PyObject *load = sym_get_const(value) == &_Py_STR(empty) ? Py_False : Py_True; + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); + } + if(!sym_set_type(value, &PyUnicode_Type)) { + goto hit_bottom; + } stack_pointer[-1] = res; break; } case _TO_BOOL_ALWAYS_TRUE: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -158,7 +201,7 @@ } case _UNARY_INVERT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -166,41 +209,47 @@ } case _GUARD_BOTH_INT: { - _Py_UOpsSymType *right; - _Py_UOpsSymType *left; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(left, &PyLong_Type); - sym_set_type(right, &PyLong_Type); + if (!sym_set_type(left, &PyLong_Type)) { + goto hit_bottom; + } + if (!sym_set_type(right, &PyLong_Type)) { + goto hit_bottom; + } break; } case _BINARY_OP_MULTIPLY_INT: { - _Py_UOpsSymType *right; - _Py_UOpsSymType *left; - _Py_UOpsSymType *res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (is_const(left) && is_const(right)) { - assert(PyLong_CheckExact(get_const(left))); - assert(PyLong_CheckExact(get_const(right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)get_const(left), - (PyLongObject *)get_const(right)); + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } res = sym_new_const(ctx, temp); - // TODO replace opcode with constant propagated one and add tests! + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! } else { - res = sym_new_known_type(ctx, &PyLong_Type); - if (res == NULL) { - goto out_of_space; - } + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -208,27 +257,29 @@ } case _BINARY_OP_ADD_INT: { - _Py_UOpsSymType *right; - _Py_UOpsSymType *left; - _Py_UOpsSymType *res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (is_const(left) && is_const(right)) { - assert(PyLong_CheckExact(get_const(left))); - assert(PyLong_CheckExact(get_const(right))); - PyObject *temp = _PyLong_Add((PyLongObject *)get_const(left), - (PyLongObject *)get_const(right)); + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } res = sym_new_const(ctx, temp); - // TODO replace opcode with constant propagated one and add tests! + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! } else { - res = sym_new_known_type(ctx, &PyLong_Type); - if (res == NULL) { - goto out_of_space; - } + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -236,27 +287,29 @@ } case _BINARY_OP_SUBTRACT_INT: { - _Py_UOpsSymType *right; - _Py_UOpsSymType *left; - _Py_UOpsSymType *res; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (is_const(left) && is_const(right)) { - assert(PyLong_CheckExact(get_const(left))); - assert(PyLong_CheckExact(get_const(right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)get_const(left), - (PyLongObject *)get_const(right)); + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } res = sym_new_const(ctx, temp); - // TODO replace opcode with constant propagated one and add tests! + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! } else { - res = sym_new_known_type(ctx, &PyLong_Type); - if (res == NULL) { - goto out_of_space; - } + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyLong_Type)); } stack_pointer[-2] = res; stack_pointer += -1; @@ -264,61 +317,160 @@ } case _GUARD_BOTH_FLOAT: { - _Py_UOpsSymType *right; - _Py_UOpsSymType *left; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0 ,0); } - sym_set_type(left, &PyFloat_Type); - sym_set_type(right, &PyFloat_Type); + if (!sym_set_type(left, &PyFloat_Type)) { + goto hit_bottom; + } + if (!sym_set_type(right, &PyFloat_Type)) { + goto hit_bottom; + } break; } case _BINARY_OP_MULTIPLY_FLOAT: { - _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) * + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + } stack_pointer[-2] = res; stack_pointer += -1; break; } case _BINARY_OP_ADD_FLOAT: { - _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) + + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + } stack_pointer[-2] = res; stack_pointer += -1; break; } case _BINARY_OP_SUBTRACT_FLOAT: { - _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) - + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyFloat_Type)); + } stack_pointer[-2] = res; stack_pointer += -1; break; } case _GUARD_BOTH_UNICODE: { + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyUnicode_Type) && + sym_matches_type(right, &PyUnicode_Type)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + if (!sym_set_type(left, &PyUnicode_Type)) { + goto hit_bottom; + } + if (!sym_set_type(right, &PyUnicode_Type)) { + goto hit_bottom; + } break; } case _BINARY_OP_ADD_UNICODE: { - _Py_UOpsSymType *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; + _Py_UopsSymbol *right; + _Py_UopsSymbol *left; + _Py_UopsSymbol *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { + PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + OUT_OF_SPACE_IF_NULL(res); + } + else { + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyUnicode_Type)); + } stack_pointer[-2] = res; stack_pointer += -1; break; } case _BINARY_SUBSCR: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -327,7 +479,7 @@ } case _BINARY_SLICE: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-3] = res; @@ -341,7 +493,7 @@ } case _BINARY_SUBSCR_LIST_INT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -350,7 +502,7 @@ } case _BINARY_SUBSCR_STR_INT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -359,7 +511,7 @@ } case _BINARY_SUBSCR_TUPLE_INT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -368,7 +520,7 @@ } case _BINARY_SUBSCR_DICT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -409,7 +561,7 @@ } case _CALL_INTRINSIC_1: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -417,7 +569,7 @@ } case _CALL_INTRINSIC_2: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -426,12 +578,12 @@ } case _POP_FRAME: { - _Py_UOpsSymType *retval; - _Py_UOpsSymType *res; + _Py_UopsSymbol *retval; + _Py_UopsSymbol *res; retval = stack_pointer[-1]; stack_pointer += -1; ctx->frame->stack_pointer = stack_pointer; - ctx_frame_pop(ctx); + frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; res = retval; stack_pointer[0] = res; @@ -444,7 +596,7 @@ /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 */ case _GET_AITER: { - _Py_UOpsSymType *iter; + _Py_UopsSymbol *iter; iter = sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; @@ -452,7 +604,7 @@ } case _GET_ANEXT: { - _Py_UOpsSymType *awaitable; + _Py_UopsSymbol *awaitable; awaitable = sym_new_unknown(ctx); if (awaitable == NULL) goto out_of_space; stack_pointer[0] = awaitable; @@ -461,7 +613,7 @@ } case _GET_AWAITABLE: { - _Py_UOpsSymType *iter; + _Py_UopsSymbol *iter; iter = sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; @@ -480,7 +632,7 @@ } case _LOAD_ASSERTION_ERROR: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[0] = value; @@ -489,7 +641,7 @@ } case _LOAD_BUILD_CLASS: { - _Py_UOpsSymType *bc; + _Py_UopsSymbol *bc; bc = sym_new_unknown(ctx); if (bc == NULL) goto out_of_space; stack_pointer[0] = bc; @@ -507,24 +659,21 @@ } case _UNPACK_SEQUENCE: { - _Py_UOpsSymType *seq; - _Py_UOpsSymType **values; + _Py_UopsSymbol *seq; + _Py_UopsSymbol **values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ (void)seq; for (int i = 0; i < oparg; i++) { - values[i] = sym_new_unknown(ctx); - if (values[i] == NULL) { - goto out_of_space; - } + OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); } stack_pointer += -1 + oparg; break; } case _UNPACK_SEQUENCE_TWO_TUPLE: { - _Py_UOpsSymType **values; + _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_unknown(ctx); @@ -535,7 +684,7 @@ } case _UNPACK_SEQUENCE_TUPLE: { - _Py_UOpsSymType **values; + _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_unknown(ctx); @@ -546,7 +695,7 @@ } case _UNPACK_SEQUENCE_LIST: { - _Py_UOpsSymType **values; + _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_unknown(ctx); @@ -557,18 +706,15 @@ } case _UNPACK_EX: { - _Py_UOpsSymType *seq; - _Py_UOpsSymType **values; + _Py_UopsSymbol *seq; + _Py_UopsSymbol **values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ (void)seq; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { - values[i] = sym_new_unknown(ctx); - if (values[i] == NULL) { - goto out_of_space; - } + OUT_OF_SPACE_IF_NULL(values[i] = sym_new_unknown(ctx)); } stack_pointer += (oparg >> 8) + (oparg & 0xFF); break; @@ -594,7 +740,7 @@ } case _LOAD_LOCALS: { - _Py_UOpsSymType *locals; + _Py_UopsSymbol *locals; locals = sym_new_unknown(ctx); if (locals == NULL) goto out_of_space; stack_pointer[0] = locals; @@ -603,7 +749,7 @@ } case _LOAD_FROM_DICT_OR_GLOBALS: { - _Py_UOpsSymType *v; + _Py_UopsSymbol *v; v = sym_new_unknown(ctx); if (v == NULL) goto out_of_space; stack_pointer[-1] = v; @@ -611,7 +757,7 @@ } case _LOAD_NAME: { - _Py_UOpsSymType *v; + _Py_UopsSymbol *v; v = sym_new_unknown(ctx); if (v == NULL) goto out_of_space; stack_pointer[0] = v; @@ -620,8 +766,8 @@ } case _LOAD_GLOBAL: { - _Py_UOpsSymType *res; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; null = sym_new_null(ctx); @@ -641,8 +787,8 @@ } case _LOAD_GLOBAL_MODULE: { - _Py_UOpsSymType *res; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; null = sym_new_null(ctx); @@ -654,8 +800,8 @@ } case _LOAD_GLOBAL_BUILTINS: { - _Py_UOpsSymType *res; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *res; + _Py_UopsSymbol *null = NULL; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; null = sym_new_null(ctx); @@ -679,7 +825,7 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[-1] = value; @@ -687,7 +833,7 @@ } case _LOAD_DEREF: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; value = sym_new_unknown(ctx); if (value == NULL) goto out_of_space; stack_pointer[0] = value; @@ -705,7 +851,7 @@ } case _BUILD_STRING: { - _Py_UOpsSymType *str; + _Py_UopsSymbol *str; str = sym_new_unknown(ctx); if (str == NULL) goto out_of_space; stack_pointer[-oparg] = str; @@ -714,7 +860,7 @@ } case _BUILD_TUPLE: { - _Py_UOpsSymType *tup; + _Py_UopsSymbol *tup; tup = sym_new_unknown(ctx); if (tup == NULL) goto out_of_space; stack_pointer[-oparg] = tup; @@ -723,7 +869,7 @@ } case _BUILD_LIST: { - _Py_UOpsSymType *list; + _Py_UopsSymbol *list; list = sym_new_unknown(ctx); if (list == NULL) goto out_of_space; stack_pointer[-oparg] = list; @@ -742,7 +888,7 @@ } case _BUILD_SET: { - _Py_UOpsSymType *set; + _Py_UopsSymbol *set; set = sym_new_unknown(ctx); if (set == NULL) goto out_of_space; stack_pointer[-oparg] = set; @@ -751,7 +897,7 @@ } case _BUILD_MAP: { - _Py_UOpsSymType *map; + _Py_UopsSymbol *map; map = sym_new_unknown(ctx); if (map == NULL) goto out_of_space; stack_pointer[-oparg*2] = map; @@ -764,7 +910,7 @@ } case _BUILD_CONST_KEY_MAP: { - _Py_UOpsSymType *map; + _Py_UopsSymbol *map; map = sym_new_unknown(ctx); if (map == NULL) goto out_of_space; stack_pointer[-1 - oparg] = map; @@ -790,17 +936,17 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ case _LOAD_SUPER_ATTR_ATTR: { - _Py_UOpsSymType *attr; + _Py_UopsSymbol *attr; attr = sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-3] = attr; - stack_pointer += -2 + ((0) ? 1 : 0); + stack_pointer += -2; break; } case _LOAD_SUPER_ATTR_METHOD: { - _Py_UOpsSymType *attr; - _Py_UOpsSymType *self_or_null; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null; attr = sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; self_or_null = sym_new_unknown(ctx); @@ -812,8 +958,8 @@ } case _LOAD_ATTR: { - _Py_UOpsSymType *attr; - _Py_UOpsSymType *self_or_null = NULL; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self_or_null = NULL; attr = sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; self_or_null = sym_new_unknown(ctx); @@ -833,9 +979,9 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - _Py_UOpsSymType *owner; - _Py_UOpsSymType *attr; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; _LOAD_ATTR_NOT_NULL @@ -848,18 +994,51 @@ } case _CHECK_ATTR_MODULE: { + _Py_UopsSymbol *owner; + owner = stack_pointer[-1]; + uint32_t dict_version = (uint32_t)this_instr->operand; + (void)dict_version; + if (sym_is_const(owner)) { + PyObject *cnst = sym_get_const(owner); + if (PyModule_CheckExact(cnst)) { + PyModuleObject *mod = (PyModuleObject *)cnst; + PyObject *dict = mod->md_dict; + uint64_t watched_mutations = get_mutations(dict); + if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + PyDict_Watch(GLOBALS_WATCHER_ID, dict); + _Py_BloomFilter_Add(dependencies, dict); + this_instr->opcode = _NOP; + } + } + } break; } case _LOAD_ATTR_MODULE: { - _Py_UOpsSymType *owner; - _Py_UOpsSymType *attr; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; - _LOAD_ATTR_NOT_NULL (void)index; - (void)owner; + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); + attr = NULL; + if (this_instr[-1].opcode == _NOP) { + // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. + assert(sym_is_const(owner)); + PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); + assert(PyModule_CheckExact(mod)); + PyObject *dict = mod->md_dict; + PyObject *res = convert_global_to_const(this_instr, dict); + if (res != NULL) { + this_instr[-1].opcode = _POP_TOP; + OUT_OF_SPACE_IF_NULL(attr = sym_new_const(ctx, res)); + } + } + if (attr == NULL) { + /* No conversion made. We don't know what `attr` is. */ + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -871,9 +1050,9 @@ } case _LOAD_ATTR_WITH_HINT: { - _Py_UOpsSymType *owner; - _Py_UOpsSymType *attr; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t hint = (uint16_t)this_instr->operand; _LOAD_ATTR_NOT_NULL @@ -886,9 +1065,9 @@ } case _LOAD_ATTR_SLOT: { - _Py_UOpsSymType *owner; - _Py_UOpsSymType *attr; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; _LOAD_ATTR_NOT_NULL @@ -905,9 +1084,9 @@ } case _LOAD_ATTR_CLASS: { - _Py_UOpsSymType *owner; - _Py_UOpsSymType *attr; - _Py_UOpsSymType *null = NULL; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *null = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; _LOAD_ATTR_NOT_NULL @@ -940,7 +1119,7 @@ } case _COMPARE_OP: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -949,7 +1128,7 @@ } case _COMPARE_OP_FLOAT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -958,7 +1137,7 @@ } case _COMPARE_OP_INT: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -967,7 +1146,7 @@ } case _COMPARE_OP_STR: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -976,7 +1155,7 @@ } case _IS_OP: { - _Py_UOpsSymType *b; + _Py_UopsSymbol *b; b = sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-2] = b; @@ -985,7 +1164,7 @@ } case _CONTAINS_OP: { - _Py_UOpsSymType *b; + _Py_UopsSymbol *b; b = sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-2] = b; @@ -994,8 +1173,8 @@ } case _CHECK_EG_MATCH: { - _Py_UOpsSymType *rest; - _Py_UOpsSymType *match; + _Py_UopsSymbol *rest; + _Py_UopsSymbol *match; rest = sym_new_unknown(ctx); if (rest == NULL) goto out_of_space; match = sym_new_unknown(ctx); @@ -1006,21 +1185,19 @@ } case _CHECK_EXC_MATCH: { - _Py_UOpsSymType *b; + _Py_UopsSymbol *b; b = sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-1] = b; break; } - /* _JUMP_BACKWARD is not a viable micro-op for tier 2 */ - /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { - _Py_UOpsSymType *b; + _Py_UopsSymbol *b; b = sym_new_unknown(ctx); if (b == NULL) goto out_of_space; stack_pointer[-1] = b; @@ -1028,7 +1205,7 @@ } case _GET_LEN: { - _Py_UOpsSymType *len_o; + _Py_UopsSymbol *len_o; len_o = sym_new_unknown(ctx); if (len_o == NULL) goto out_of_space; stack_pointer[0] = len_o; @@ -1037,7 +1214,7 @@ } case _MATCH_CLASS: { - _Py_UOpsSymType *attrs; + _Py_UopsSymbol *attrs; attrs = sym_new_unknown(ctx); if (attrs == NULL) goto out_of_space; stack_pointer[-3] = attrs; @@ -1046,7 +1223,7 @@ } case _MATCH_MAPPING: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; @@ -1055,7 +1232,7 @@ } case _MATCH_SEQUENCE: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; @@ -1064,7 +1241,7 @@ } case _MATCH_KEYS: { - _Py_UOpsSymType *values_or_none; + _Py_UopsSymbol *values_or_none; values_or_none = sym_new_unknown(ctx); if (values_or_none == NULL) goto out_of_space; stack_pointer[0] = values_or_none; @@ -1073,7 +1250,7 @@ } case _GET_ITER: { - _Py_UOpsSymType *iter; + _Py_UopsSymbol *iter; iter = sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; @@ -1081,7 +1258,7 @@ } case _GET_YIELD_FROM_ITER: { - _Py_UOpsSymType *iter; + _Py_UopsSymbol *iter; iter = sym_new_unknown(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; @@ -1091,7 +1268,7 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { - _Py_UOpsSymType *next; + _Py_UopsSymbol *next; next = sym_new_unknown(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; @@ -1112,7 +1289,7 @@ } case _ITER_NEXT_LIST: { - _Py_UOpsSymType *next; + _Py_UopsSymbol *next; next = sym_new_unknown(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; @@ -1131,7 +1308,7 @@ } case _ITER_NEXT_TUPLE: { - _Py_UOpsSymType *next; + _Py_UopsSymbol *next; next = sym_new_unknown(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; @@ -1150,13 +1327,10 @@ } case _ITER_NEXT_RANGE: { - _Py_UOpsSymType *iter; - _Py_UOpsSymType *next; + _Py_UopsSymbol *iter; + _Py_UopsSymbol *next; iter = stack_pointer[-1]; - next = sym_new_known_type(ctx, &PyLong_Type); - if (next == NULL) { - goto out_of_space; - } + OUT_OF_SPACE_IF_NULL(next = sym_new_type(ctx, &PyLong_Type)); (void)iter; stack_pointer[0] = next; stack_pointer += 1; @@ -1166,8 +1340,8 @@ /* _FOR_ITER_GEN is not a viable micro-op for tier 2 */ case _BEFORE_ASYNC_WITH: { - _Py_UOpsSymType *exit; - _Py_UOpsSymType *res; + _Py_UopsSymbol *exit; + _Py_UopsSymbol *res; exit = sym_new_unknown(ctx); if (exit == NULL) goto out_of_space; res = sym_new_unknown(ctx); @@ -1179,8 +1353,8 @@ } case _BEFORE_WITH: { - _Py_UOpsSymType *exit; - _Py_UOpsSymType *res; + _Py_UopsSymbol *exit; + _Py_UopsSymbol *res; exit = sym_new_unknown(ctx); if (exit == NULL) goto out_of_space; res = sym_new_unknown(ctx); @@ -1192,7 +1366,7 @@ } case _WITH_EXCEPT_START: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; @@ -1201,8 +1375,8 @@ } case _PUSH_EXC_INFO: { - _Py_UOpsSymType *prev_exc; - _Py_UOpsSymType *new_exc; + _Py_UopsSymbol *prev_exc; + _Py_UopsSymbol *new_exc; prev_exc = sym_new_unknown(ctx); if (prev_exc == NULL) goto out_of_space; new_exc = sym_new_unknown(ctx); @@ -1222,46 +1396,48 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - _Py_UOpsSymType *attr; - _Py_UOpsSymType *self = NULL; - attr = sym_new_unknown(ctx); - if (attr == NULL) goto out_of_space; - self = sym_new_unknown(ctx); - if (self == NULL) goto out_of_space; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + self = owner; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } case _LOAD_ATTR_METHOD_NO_DICT: { - _Py_UOpsSymType *attr; - _Py_UOpsSymType *self = NULL; - attr = sym_new_unknown(ctx); - if (attr == NULL) goto out_of_space; - self = sym_new_unknown(ctx); - if (self == NULL) goto out_of_space; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + self = owner; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - _Py_UOpsSymType *attr; + _Py_UopsSymbol *attr; attr = sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - _Py_UOpsSymType *attr; + _Py_UopsSymbol *attr; attr = sym_new_unknown(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; - stack_pointer += ((0) ? 1 : 0); break; } @@ -1270,15 +1446,17 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - _Py_UOpsSymType *attr; - _Py_UOpsSymType *self = NULL; - attr = sym_new_unknown(ctx); - if (attr == NULL) goto out_of_space; - self = sym_new_unknown(ctx); - if (self == NULL) goto out_of_space; + _Py_UopsSymbol *owner; + _Py_UopsSymbol *attr; + _Py_UopsSymbol *self = NULL; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + self = owner; stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += ((1) ? 1 : 0); + stack_pointer[0] = self; + stack_pointer += 1; break; } @@ -1287,22 +1465,27 @@ /* _CALL is not a viable micro-op for tier 2 */ case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UOpsSymType *null; - _Py_UOpsSymType *callable; + _Py_UopsSymbol *null; + _Py_UopsSymbol *callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - sym_set_null(null); - sym_set_type(callable, &PyMethod_Type); + if (!sym_set_null(null)) { + goto hit_bottom; + } + if (!sym_set_type(callable, &PyMethod_Type)) { + goto hit_bottom; + } break; } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UOpsSymType *func; - _Py_UOpsSymType *self; - func = sym_new_unknown(ctx); - if (func == NULL) goto out_of_space; - self = sym_new_unknown(ctx); - if (self == NULL) goto out_of_space; + _Py_UopsSymbol *callable; + _Py_UopsSymbol *func; + _Py_UopsSymbol *self; + callable = stack_pointer[-2 - oparg]; + (void)callable; + OUT_OF_SPACE_IF_NULL(func = sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(self = sym_new_not_null(ctx)); stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = self; break; @@ -1313,12 +1496,14 @@ } case _CHECK_FUNCTION_EXACT_ARGS: { - _Py_UOpsSymType *self_or_null; - _Py_UOpsSymType *callable; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand; - sym_set_type(callable, &PyFunction_Type); + if (!sym_set_type(callable, &PyFunction_Type)) { + goto hit_bottom; + } (void)self_or_null; (void)func_version; break; @@ -1329,9 +1514,9 @@ } case _INIT_CALL_PY_EXACT_ARGS: { - _Py_UOpsSymType **args; - _Py_UOpsSymType *self_or_null; - _Py_UOpsSymType *callable; + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; _Py_UOpsAbstractFrame *new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; @@ -1350,20 +1535,18 @@ args--; argcount++; } - _Py_UOpsSymType **localsplus_start = ctx->n_consumed; + _Py_UopsSymbol **localsplus_start = ctx->n_consumed; int n_locals_already_filled = 0; // Can determine statically, so we interleave the new locals // and make the current stack the new locals. // This also sets up for true call inlining. - if (sym_is_known(self_or_null)) { + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { localsplus_start = args; n_locals_already_filled = argcount; } - new_frame = ctx_frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0); - if (new_frame == NULL){ - goto out_of_space; - } - stack_pointer[-2 - oparg] = (_Py_UOpsSymType *)new_frame; + OUT_OF_SPACE_IF_NULL(new_frame = + frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; stack_pointer += -1 - oparg; break; } @@ -1376,14 +1559,13 @@ ctx->frame = new_frame; ctx->curr_frame_depth++; stack_pointer = new_frame->stack_pointer; - stack_pointer += ((0) ? 1 : 0); break; } /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ case _CALL_TYPE_1: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1392,7 +1574,7 @@ } case _CALL_STR_1: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1401,7 +1583,7 @@ } case _CALL_TUPLE_1: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1417,7 +1599,7 @@ } case _CALL_BUILTIN_CLASS: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1426,7 +1608,7 @@ } case _CALL_BUILTIN_O: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1435,7 +1617,7 @@ } case _CALL_BUILTIN_FAST: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1444,7 +1626,7 @@ } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1453,7 +1635,7 @@ } case _CALL_LEN: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1462,7 +1644,7 @@ } case _CALL_ISINSTANCE: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1471,7 +1653,7 @@ } case _CALL_METHOD_DESCRIPTOR_O: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1480,7 +1662,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1489,7 +1671,7 @@ } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1498,7 +1680,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; @@ -1515,7 +1697,7 @@ /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { - _Py_UOpsSymType *func; + _Py_UopsSymbol *func; func = sym_new_unknown(ctx); if (func == NULL) goto out_of_space; stack_pointer[-1] = func; @@ -1523,7 +1705,7 @@ } case _SET_FUNCTION_ATTRIBUTE: { - _Py_UOpsSymType *func; + _Py_UopsSymbol *func; func = sym_new_unknown(ctx); if (func == NULL) goto out_of_space; stack_pointer[-2] = func; @@ -1532,7 +1714,7 @@ } case _BUILD_SLICE: { - _Py_UOpsSymType *slice; + _Py_UopsSymbol *slice; slice = sym_new_unknown(ctx); if (slice == NULL) goto out_of_space; stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; @@ -1541,7 +1723,7 @@ } case _CONVERT_VALUE: { - _Py_UOpsSymType *result; + _Py_UopsSymbol *result; result = sym_new_unknown(ctx); if (result == NULL) goto out_of_space; stack_pointer[-1] = result; @@ -1549,7 +1731,7 @@ } case _FORMAT_SIMPLE: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; @@ -1557,7 +1739,7 @@ } case _FORMAT_WITH_SPEC: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -1566,8 +1748,8 @@ } case _COPY: { - _Py_UOpsSymType *bottom; - _Py_UOpsSymType *top; + _Py_UopsSymbol *bottom; + _Py_UopsSymbol *top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; @@ -1577,7 +1759,7 @@ } case _BINARY_OP: { - _Py_UOpsSymType *res; + _Py_UopsSymbol *res; res = sym_new_unknown(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; @@ -1586,8 +1768,8 @@ } case _SWAP: { - _Py_UOpsSymType *top; - _Py_UOpsSymType *bottom; + _Py_UopsSymbol *top; + _Py_UopsSymbol *bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; stack_pointer[-2 - (oparg-2)] = top; @@ -1650,41 +1832,37 @@ } case _LOAD_CONST_INLINE: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; PyObject *ptr = (PyObject *)this_instr->operand; - value = sym_new_const(ctx, ptr); - if (value == NULL) { - goto out_of_space; - } + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; break; } case _LOAD_CONST_INLINE_BORROW: { - _Py_UOpsSymType *value; + _Py_UopsSymbol *value; PyObject *ptr = (PyObject *)this_instr->operand; - value = sym_new_const(ctx, ptr); - if (value == NULL) { - goto out_of_space; - } + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; break; } + case _POP_TOP_LOAD_CONST_INLINE_BORROW: { + _Py_UopsSymbol *value; + value = sym_new_unknown(ctx); + if (value == NULL) goto out_of_space; + stack_pointer[-1] = value; + break; + } + case _LOAD_CONST_INLINE_WITH_NULL: { - _Py_UOpsSymType *value; - _Py_UOpsSymType *null; + _Py_UopsSymbol *value; + _Py_UopsSymbol *null; PyObject *ptr = (PyObject *)this_instr->operand; - value = sym_new_const(ctx, ptr); - if (value == NULL) { - goto out_of_space; - } - null = sym_new_null(ctx); - if (null == NULL) { - goto out_of_space; - } + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); stack_pointer[0] = value; stack_pointer[1] = null; stack_pointer += 2; @@ -1692,17 +1870,11 @@ } case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { - _Py_UOpsSymType *value; - _Py_UOpsSymType *null; + _Py_UopsSymbol *value; + _Py_UopsSymbol *null; PyObject *ptr = (PyObject *)this_instr->operand; - value = sym_new_const(ctx, ptr); - if (value == NULL) { - goto out_of_space; - } - null = sym_new_null(ctx); - if (null == NULL) { - goto out_of_space; - } + OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr)); + OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); stack_pointer[0] = value; stack_pointer[1] = null; stack_pointer += 2; @@ -1722,6 +1894,18 @@ break; } + case _COLD_EXIT: { + break; + } + + case _START_EXECUTOR: { + break; + } + + case _FATAL_ERROR: { + break; + } + case _CHECK_VALIDITY_AND_SET_IP: { break; } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c new file mode 100644 index 00000000000000..5c3ec2b5ed1a4c --- /dev/null +++ b/Python/optimizer_symbols.c @@ -0,0 +1,449 @@ + +#include "Python.h" + +#include "cpython/optimizer.h" +#include "pycore_code.h" +#include "pycore_frame.h" +#include "pycore_optimizer.h" + +#include +#include +#include + +/* Symbols + ======= + + See the diagram at + https://github.com/faster-cpython/ideas/blob/main/3.13/redundancy_eliminator.md + + We represent the nodes in the diagram as follows + (the flag bits are only defined in optimizer_symbols.c): + - Top: no flag bits, typ and const_val are NULL. + - NULL: IS_NULL flag set, type and const_val NULL. + - Not NULL: NOT_NULL flag set, type and const_val NULL. + - None/not None: not used. (None could be represented as any other constant.) + - Known type: NOT_NULL flag set and typ set; const_val is NULL. + - Known constant: NOT_NULL flag set, type set, const_val set. + - Bottom: IS_NULL and NOT_NULL flags set, type and const_val NULL. + */ + +// Flags for below. +#define IS_NULL 1 << 0 +#define NOT_NULL 1 << 1 + +#ifdef Py_DEBUG +static inline int get_lltrace(void) { + char *uop_debug = Py_GETENV("PYTHON_OPT_DEBUG"); + int lltrace = 0; + if (uop_debug != NULL && *uop_debug >= '0') { + lltrace = *uop_debug - '0'; // TODO: Parse an int and all that + } + return lltrace; +} +#define DPRINTF(level, ...) \ + if (get_lltrace() >= (level)) { printf(__VA_ARGS__); } +#else +#define DPRINTF(level, ...) +#endif + +static _Py_UopsSymbol * +sym_new(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *self = &ctx->t_arena.arena[ctx->t_arena.ty_curr_number]; + if (ctx->t_arena.ty_curr_number >= ctx->t_arena.ty_max_number) { + OPT_STAT_INC(optimizer_failure_reason_no_memory); + DPRINTF(1, "out of space for symbolic expression type\n"); + return NULL; + } + ctx->t_arena.ty_curr_number++; + self->flags = 0; + self->typ = NULL; + self->const_val = NULL; + + return self; +} + +static inline void +sym_set_flag(_Py_UopsSymbol *sym, int flag) +{ + sym->flags |= flag; +} + +static inline void +sym_set_bottom(_Py_UopsSymbol *sym) +{ + sym_set_flag(sym, IS_NULL | NOT_NULL); + sym->typ = NULL; + Py_CLEAR(sym->const_val); +} + +bool +_Py_uop_sym_is_bottom(_Py_UopsSymbol *sym) +{ + if ((sym->flags & IS_NULL) && (sym->flags & NOT_NULL)) { + assert(sym->flags == (IS_NULL | NOT_NULL)); + assert(sym->typ == NULL); + assert(sym->const_val == NULL); + return true; + } + return false; +} + +bool +_Py_uop_sym_is_not_null(_Py_UopsSymbol *sym) +{ + return sym->flags == NOT_NULL; +} + +bool +_Py_uop_sym_is_null(_Py_UopsSymbol *sym) +{ + return sym->flags == IS_NULL; +} + +bool +_Py_uop_sym_is_const(_Py_UopsSymbol *sym) +{ + return sym->const_val != NULL; +} + +PyObject * +_Py_uop_sym_get_const(_Py_UopsSymbol *sym) +{ + return sym->const_val; +} + +bool +_Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ) +{ + assert(typ != NULL && PyType_Check(typ)); + if (sym->flags & IS_NULL) { + sym_set_bottom(sym); + return false; + } + if (sym->typ != NULL) { + if (sym->typ != typ) { + sym_set_bottom(sym); + return false; + } + } + else { + sym_set_flag(sym, NOT_NULL); + sym->typ = typ; + } + return true; +} + +bool +_Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val) +{ + assert(const_val != NULL); + if (sym->flags & IS_NULL) { + sym_set_bottom(sym); + return false; + } + PyTypeObject *typ = Py_TYPE(const_val); + if (sym->typ != NULL && sym->typ != typ) { + sym_set_bottom(sym); + return false; + } + if (sym->const_val != NULL) { + if (sym->const_val != const_val) { + // TODO: What if they're equal? + sym_set_bottom(sym); + return false; + } + } + else { + sym_set_flag(sym, NOT_NULL); + sym->typ = typ; + sym->const_val = Py_NewRef(const_val); + } + return true; +} + + +bool +_Py_uop_sym_set_null(_Py_UopsSymbol *sym) +{ + sym_set_flag(sym, IS_NULL); + return !_Py_uop_sym_is_bottom(sym); +} + +bool +_Py_uop_sym_set_non_null(_Py_UopsSymbol *sym) +{ + sym_set_flag(sym, NOT_NULL); + return !_Py_uop_sym_is_bottom(sym); +} + + +_Py_UopsSymbol * +_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx) +{ + return sym_new(ctx); +} + +_Py_UopsSymbol * +_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *res = _Py_uop_sym_new_unknown(ctx); + if (res == NULL) { + return NULL; + } + sym_set_flag(res, NOT_NULL); + return res; +} + +_Py_UopsSymbol * +_Py_uop_sym_new_type(_Py_UOpsContext *ctx, PyTypeObject *typ) +{ + _Py_UopsSymbol *res = sym_new(ctx); + if (res == NULL) { + return NULL; + } + _Py_uop_sym_set_type(res, typ); + return res; +} + +// Adds a new reference to const_val, owned by the symbol. +_Py_UopsSymbol * +_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) +{ + assert(const_val != NULL); + _Py_UopsSymbol *res = sym_new(ctx); + if (res == NULL) { + return NULL; + } + _Py_uop_sym_set_const(res, const_val); + return res; +} + +_Py_UopsSymbol * +_Py_uop_sym_new_null(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *null_sym = _Py_uop_sym_new_unknown(ctx); + if (null_sym == NULL) { + return NULL; + } + _Py_uop_sym_set_null(null_sym); + return null_sym; +} + +bool +_Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) +{ + assert(typ != NULL && PyType_Check(typ)); + if (_Py_uop_sym_is_bottom(sym)) { + return false; + } + return sym->typ == typ; +} + +// 0 on success, -1 on error. +_Py_UOpsAbstractFrame * +_Py_uop_frame_new( + _Py_UOpsContext *ctx, + PyCodeObject *co, + _Py_UopsSymbol **localsplus_start, + int n_locals_already_filled, + int curr_stackentries) +{ + assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); + _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; + + frame->stack_len = co->co_stacksize; + frame->locals_len = co->co_nlocalsplus; + + frame->locals = localsplus_start; + frame->stack = frame->locals + co->co_nlocalsplus; + frame->stack_pointer = frame->stack + curr_stackentries; + ctx->n_consumed = localsplus_start + (co->co_nlocalsplus + co->co_stacksize); + if (ctx->n_consumed >= ctx->limit) { + return NULL; + } + + + // Initialize with the initial state of all local variables + for (int i = n_locals_already_filled; i < co->co_nlocalsplus; i++) { + _Py_UopsSymbol *local = _Py_uop_sym_new_unknown(ctx); + if (local == NULL) { + return NULL; + } + frame->locals[i] = local; + } + + + // Initialize the stack as well + for (int i = 0; i < curr_stackentries; i++) { + _Py_UopsSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); + if (stackvar == NULL) { + return NULL; + } + frame->stack[i] = stackvar; + } + + return frame; +} + +void +_Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx) +{ + if (ctx == NULL) { + return; + } + ctx->curr_frame_depth = 0; + int tys = ctx->t_arena.ty_curr_number; + for (int i = 0; i < tys; i++) { + Py_CLEAR(ctx->t_arena.arena[i].const_val); + } +} + +int +_Py_uop_abstractcontext_init(_Py_UOpsContext *ctx) +{ + ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; + ctx->n_consumed = ctx->locals_and_stack; +#ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. + for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { + ctx->locals_and_stack[i] = NULL; + } +#endif + + // Setup the arena for sym expressions. + ctx->t_arena.ty_curr_number = 0; + ctx->t_arena.ty_max_number = TY_ARENA_SIZE; + + // Frame setup + ctx->curr_frame_depth = 0; + + return 0; +} + +int +_Py_uop_frame_pop(_Py_UOpsContext *ctx) +{ + _Py_UOpsAbstractFrame *frame = ctx->frame; + ctx->n_consumed = frame->locals; + ctx->curr_frame_depth--; + assert(ctx->curr_frame_depth >= 1); + ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + return 0; +} + +#define TEST_PREDICATE(PRED, MSG) \ +do { \ + if (!(PRED)) { \ + PyErr_SetString( \ + PyExc_AssertionError, \ + (MSG)); \ + goto fail; \ + } \ +} while (0) + +static _Py_UopsSymbol * +make_bottom(_Py_UOpsContext *ctx) +{ + _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); + _Py_uop_sym_set_null(sym); + _Py_uop_sym_set_non_null(sym); + return sym; +} + +PyObject * +_Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) +{ + _Py_UOpsContext context; + _Py_UOpsContext *ctx = &context; + _Py_uop_abstractcontext_init(ctx); + PyObject *val_42 = NULL; + PyObject *val_43 = NULL; + + // Use a single 'sym' variable so copy-pasting tests is easier. + _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "top is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "top as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); + + sym = make_bottom(ctx); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "bottom is not NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "bottom matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "bottom is a constant is not false"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "bottom as constant is not NULL"); + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "int is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "int isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "int isn't int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "int matches float"); + TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "int is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "int as constant is not NULL"); + + _Py_uop_sym_set_type(sym, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(int and int) isn't int"); + + _Py_uop_sym_set_type(sym, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(int and float) isn't bottom"); + + val_42 = PyLong_FromLong(42); + assert(val_42 != NULL); + assert(_Py_IsImmortal(val_42)); + + val_43 = PyLong_FromLong(43); + assert(val_43 != NULL); + assert(_Py_IsImmortal(val_43)); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + _Py_uop_sym_set_const(sym, val_42); + TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "42 isn't an int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "42 matches float"); + TEST_PREDICATE(_Py_uop_sym_is_const(sym), "42 is not a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) != NULL, "42 as constant is NULL"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "42 as constant isn't 42"); + + _Py_uop_sym_set_type(sym, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(42 and 42) isn't an int"); + TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "(42 and 42) as constant isn't 42"); + + _Py_uop_sym_set_type(sym, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); + + sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (sym == NULL) { + goto fail; + } + _Py_uop_sym_set_const(sym, val_42); + _Py_uop_sym_set_const(sym, val_43); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); + + _Py_uop_abstractcontext_fini(ctx); + Py_DECREF(val_42); + Py_DECREF(val_43); + Py_RETURN_NONE; + +fail: + _Py_uop_abstractcontext_fini(ctx); + Py_XDECREF(val_42); + Py_XDECREF(val_43); + return NULL; +} diff --git a/Python/parking_lot.c b/Python/parking_lot.c index 8ba50fc1353ebd..0a897f9952f648 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -1,11 +1,12 @@ #include "Python.h" #include "pycore_llist.h" -#include "pycore_lock.h" // _PyRawMutex +#include "pycore_lock.h" // _PyRawMutex #include "pycore_parking_lot.h" -#include "pycore_pyerrors.h" // _Py_FatalErrorFormat -#include "pycore_pystate.h" // _PyThreadState_GET -#include "pycore_semaphore.h" // _PySemaphore +#include "pycore_pyerrors.h" // _Py_FatalErrorFormat +#include "pycore_pystate.h" // _PyThreadState_GET +#include "pycore_semaphore.h" // _PySemaphore +#include "pycore_time.h" //_PyTime_MonotonicUnchecked() #include @@ -91,7 +92,7 @@ _PySemaphore_Destroy(_PySemaphore *sema) } static int -_PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) +_PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) { int res; #if defined(MS_WINDOWS) @@ -119,13 +120,13 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) struct timespec ts; #if defined(CLOCK_MONOTONIC) && defined(HAVE_SEM_CLOCKWAIT) - _PyTime_t deadline = _PyTime_Add(_PyTime_GetMonotonicClock(), timeout); + PyTime_t deadline = _PyTime_Add(_PyTime_MonotonicUnchecked(), timeout); _PyTime_AsTimespec_clamp(deadline, &ts); err = sem_clockwait(&sema->platform_sem, CLOCK_MONOTONIC, &ts); #else - _PyTime_t deadline = _PyTime_Add(_PyTime_GetSystemClock(), timeout); + PyTime_t deadline = _PyTime_Add(_PyTime_TimeUnchecked(), timeout); _PyTime_AsTimespec_clamp(deadline, &ts); @@ -162,7 +163,7 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) _PyTime_AsTimespec_clamp(timeout, &ts); err = pthread_cond_timedwait_relative_np(&sema->cond, &sema->mutex, &ts); #else - _PyTime_t deadline = _PyTime_Add(_PyTime_GetSystemClock(), timeout); + PyTime_t deadline = _PyTime_Add(_PyTime_TimeUnchecked(), timeout); _PyTime_AsTimespec_clamp(deadline, &ts); err = pthread_cond_timedwait(&sema->cond, &sema->mutex, &ts); @@ -188,7 +189,7 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) } int -_PySemaphore_Wait(_PySemaphore *sema, _PyTime_t timeout, int detach) +_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout, int detach) { PyThreadState *tstate = NULL; if (detach) { @@ -283,7 +284,7 @@ atomic_memcmp(const void *addr, const void *expected, size_t addr_size) int _PyParkingLot_Park(const void *addr, const void *expected, size_t size, - _PyTime_t timeout_ns, void *park_arg, int detach) + PyTime_t timeout_ns, void *park_arg, int detach) { struct wait_entry wait = { .park_arg = park_arg, diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 5e5db98481150e..3a2c0a450ac9d9 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -612,7 +612,7 @@ builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, Py { PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { - _Py_Executors_InvalidateAll(interp); + _Py_Executors_InvalidateAll(interp, 1); } RARE_EVENT_INTERP_INC(interp, builtin_dict); return 0; @@ -663,6 +663,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, if (tstate == NULL) { return _PyStatus_ERR("can't make first thread"); } + runtime->main_tstate = tstate; _PyThreadState_Bind(tstate); init_interp_create_gil(tstate, config.gil); @@ -1109,7 +1110,6 @@ run_presite(PyThreadState *tstate) ); if (presite_modname == NULL) { fprintf(stderr, "Could not convert pre-site module name to unicode\n"); - Py_DECREF(presite_modname); } else { PyObject *presite = PyImport_Import(presite_modname); @@ -1262,7 +1262,9 @@ init_interp_main(PyThreadState *tstate) if (opt == NULL) { return _PyStatus_ERR("can't initialize optimizer"); } - PyUnstable_SetOptimizer((_PyOptimizerObject *)opt); + if (PyUnstable_SetOptimizer((_PyOptimizerObject *)opt)) { + return _PyStatus_ERR("can't initialize optimizer"); + } Py_DECREF(opt); } } @@ -1626,7 +1628,7 @@ finalize_modules(PyThreadState *tstate) PyInterpreterState *interp = tstate->interp; // Invalidate all executors and turn off tier 2 optimizer - _Py_Executors_InvalidateAll(interp); + _Py_Executors_InvalidateAll(interp, 0); _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL); Py_XDECREF(old); @@ -1835,6 +1837,9 @@ finalize_interp_clear(PyThreadState *tstate) finalize_interp_types(tstate->interp); + /* Free any delayed free requests immediately */ + _PyMem_FiniDelayed(tstate->interp); + /* finalize_interp_types may allocate Python objects so we may need to abandon mimalloc segments again */ _PyThreadState_ClearMimallocHeaps(tstate); diff --git a/Python/pystate.c b/Python/pystate.c index 24f9b7790915ab..3d6394f81da816 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -617,6 +617,7 @@ init_interpreter(PyInterpreterState *interp, #ifdef Py_GIL_DISABLED _Py_brc_init_state(interp); #endif + llist_init(&interp->mem_free_queue.head); for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->monitors.tools[i] = 0; } @@ -782,6 +783,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) } _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL); + assert(old != NULL); Py_DECREF(old); /* It is possible that any of the objects below have a finalizer @@ -793,9 +795,10 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); - // At this time, all the threads should be cleared so we don't need - // atomic operations for eval_breaker - interp->ceval.eval_breaker = 0; + // At this time, all the threads should be cleared so we don't need atomic + // operations for instrumentation_version or eval_breaker. + interp->ceval.instrumentation_version = 0; + tstate->eval_breaker = 0; for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->monitors.tools[i] = 0; @@ -953,6 +956,8 @@ PyInterpreterState_Delete(PyInterpreterState *interp) PyThread_free_lock(interp->id_mutex); } + _Py_qsbr_fini(interp); + _PyObject_FiniState(interp); free_interpreter(interp); @@ -1315,6 +1320,8 @@ init_threadstate(_PyThreadStateImpl *_tstate, assert(interp != NULL); tstate->interp = interp; + tstate->eval_breaker = + _Py_atomic_load_uintptr_relaxed(&interp->ceval.instrumentation_version); // next/prev are set in add_threadstate(). assert(tstate->next == NULL); @@ -1344,11 +1351,13 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->datastack_top = NULL; tstate->datastack_limit = NULL; tstate->what_event = -1; + tstate->previous_executor = NULL; #ifdef Py_GIL_DISABLED // Initialize biased reference counting inter-thread queue _Py_brc_init_thread(tstate); #endif + llist_init(&_tstate->mem_free_queue); if (interp->stoptheworld.requested || _PyRuntime.stoptheworld.requested) { // Start in the suspended state if there is an ongoing stop-the-world. @@ -1386,6 +1395,14 @@ new_threadstate(PyInterpreterState *interp, int whence) if (new_tstate == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + Py_ssize_t qsbr_idx = _Py_qsbr_reserve(interp); + if (qsbr_idx < 0) { + PyMem_RawFree(new_tstate); + return NULL; + } +#endif + /* We serialize concurrent creation to protect global state. */ HEAD_LOCK(runtime); @@ -1420,6 +1437,12 @@ new_threadstate(PyInterpreterState *interp, int whence) // Must be called with lock unlocked to avoid re-entrancy deadlock. PyMem_RawFree(new_tstate); } + +#ifdef Py_GIL_DISABLED + // Must be called with lock unlocked to avoid lock ordering deadlocks. + _Py_qsbr_register(tstate, interp, qsbr_idx); +#endif + return (PyThreadState *)tstate; } @@ -1556,6 +1579,7 @@ PyThreadState_Clear(PyThreadState *tstate) // don't call _PyInterpreterState_SetNotRunningMain() yet. tstate->on_delete(tstate->on_delete_data); } + #ifdef Py_GIL_DISABLED // Each thread should clear own freelists in free-threading builds. struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); @@ -1565,6 +1589,9 @@ PyThreadState_Clear(PyThreadState *tstate) _Py_brc_remove_thread(tstate); #endif + // Merge our queue of pointers to be freed into the interpreter queue. + _PyMem_AbandonDelayed(tstate); + _PyThreadState_ClearMimallocHeaps(tstate); tstate->_status.cleared = 1; @@ -1611,6 +1638,10 @@ tstate_delete_common(PyThreadState *tstate) } HEAD_UNLOCK(runtime); +#ifdef Py_GIL_DISABLED + _Py_qsbr_unregister((_PyThreadStateImpl *)tstate); +#endif + // XXX Unbind in PyThreadState_Clear(), or earlier // (and assert not-equal here)? if (tstate->_status.bound_gilstate) { @@ -1652,6 +1683,9 @@ void _PyThreadState_DeleteCurrent(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); +#ifdef Py_GIL_DISABLED + _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif tstate_set_detached(tstate); tstate_delete_common(tstate); current_fast_clear(tstate->interp->runtime); @@ -1873,6 +1907,10 @@ _PyThreadState_Attach(PyThreadState *tstate) tstate_wait_attach(tstate); } +#ifdef Py_GIL_DISABLED + _Py_qsbr_attach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif + // Resume previous critical section. This acquires the lock(s) from the // top-most critical section. if (tstate->critical_section != 0) { @@ -1893,6 +1931,9 @@ detach_thread(PyThreadState *tstate, int detached_state) if (tstate->critical_section != 0) { _PyCriticalSection_SuspendAll(tstate); } +#ifdef Py_GIL_DISABLED + _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif tstate_deactivate(tstate); tstate_set_detached(tstate); current_fast_clear(&_PyRuntime); @@ -1989,8 +2030,7 @@ park_detached_threads(struct _stoptheworld_state *stw) } } else if (state == _Py_THREAD_ATTACHED && t != stw->requester) { - // TODO: set this per-thread, rather than per-interpreter. - _Py_set_eval_breaker_bit(t->interp, _PY_EVAL_PLEASE_STOP_BIT, 1); + _Py_set_eval_breaker_bit(t, _PY_EVAL_PLEASE_STOP_BIT); } } stw->thread_countdown -= num_parked; @@ -2041,7 +2081,7 @@ stop_the_world(struct _stoptheworld_state *stw) break; } - _PyTime_t wait_ns = 1000*1000; // 1ms (arbitrary, may need tuning) + PyTime_t wait_ns = 1000*1000; // 1ms (arbitrary, may need tuning) if (PyEvent_WaitTimed(&stw->stop_event, wait_ns)) { assert(stw->thread_countdown == 0); break; @@ -2154,19 +2194,18 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) * deadlock, we need to release head_mutex before * the decref. */ - PyObject *old_exc = tstate->async_exc; - tstate->async_exc = Py_XNewRef(exc); + Py_XINCREF(exc); + PyObject *old_exc = _Py_atomic_exchange_ptr(&tstate->async_exc, exc); HEAD_UNLOCK(runtime); Py_XDECREF(old_exc); - _PyEval_SignalAsyncExc(tstate->interp); + _Py_set_eval_breaker_bit(tstate, _PY_ASYNC_EXCEPTION_BIT); return 1; } HEAD_UNLOCK(runtime); return 0; } - //--------------------------------- // API for the current thread state //--------------------------------- @@ -2492,16 +2531,7 @@ PyGILState_Check(void) return 0; } -#ifdef MS_WINDOWS - int err = GetLastError(); -#endif - PyThreadState *tcur = gilstate_tss_get(runtime); - -#ifdef MS_WINDOWS - SetLastError(err); -#endif - return (tstate == tcur); } @@ -2630,7 +2660,7 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, return; } if (eval_frame != NULL) { - _Py_Executors_InvalidateAll(interp); + _Py_Executors_InvalidateAll(interp, 1); } RARE_EVENT_INC(set_eval_frame_func); interp->eval_frame = eval_frame; @@ -2815,9 +2845,24 @@ tstate_mimalloc_bind(PyThreadState *tstate) // pools to keep Python objects from different interpreters separate. tld->segments.abandoned = &tstate->interp->mimalloc.abandoned_pool; + // Don't fill in the first N bytes up to ob_type in debug builds. We may + // access ob_tid and the refcount fields in the dict and list lock-less + // accesses, so they must remain valid for a while after deallocation. + size_t base_offset = offsetof(PyObject, ob_type); + if (_PyMem_DebugEnabled()) { + // The debug allocator adds two words at the beginning of each block. + base_offset += 2 * sizeof(size_t); + } + size_t debug_offsets[_Py_MIMALLOC_HEAP_COUNT] = { + [_Py_MIMALLOC_HEAP_OBJECT] = base_offset, + [_Py_MIMALLOC_HEAP_GC] = base_offset, + [_Py_MIMALLOC_HEAP_GC_PRE] = base_offset + 2 * sizeof(PyObject *), + }; + // Initialize each heap for (uint8_t i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) { _mi_heap_init_ex(&mts->heaps[i], tld, _mi_arena_id_none(), false, i); + mts->heaps[i].debug_offset = (uint8_t)debug_offsets[i]; } // By default, object allocations use _Py_MIMALLOC_HEAP_OBJECT. diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 5f305aa00e08b9..f87c53fb28fbea 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -89,7 +89,7 @@ int PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) { - PyObject *filename_obj; + PyObject *filename_obj = NULL; if (filename != NULL) { filename_obj = PyUnicode_DecodeFSDefault(filename); if (filename_obj == NULL) { @@ -97,9 +97,6 @@ PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, return -1; } } - else { - filename_obj = NULL; - } int res = _PyRun_AnyFileObject(fp, filename_obj, closeit, flags); Py_XDECREF(filename_obj); return res; diff --git a/Python/pytime.c b/Python/pytime.c index fb0ed85c541e68..90ef2eeb546f7f 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -1,5 +1,5 @@ #include "Python.h" -#include "pycore_time.h" // _PyTime_t +#include "pycore_time.h" // PyTime_t #include // gmtime_r() #ifdef HAVE_SYS_TIME_H @@ -51,18 +51,18 @@ #endif #if PyTime_MIN + PyTime_MAX != -1 -# error "_PyTime_t is not a two's complement integer type" +# error "PyTime_t is not a two's complement integer type" #endif -static _PyTime_t -_PyTime_GCD(_PyTime_t x, _PyTime_t y) +static PyTime_t +_PyTime_GCD(PyTime_t x, PyTime_t y) { // Euclidean algorithm assert(x >= 1); assert(y >= 1); while (y != 0) { - _PyTime_t tmp = y; + PyTime_t tmp = y; y = x % y; x = tmp; } @@ -72,13 +72,13 @@ _PyTime_GCD(_PyTime_t x, _PyTime_t y) int -_PyTimeFraction_Set(_PyTimeFraction *frac, _PyTime_t numer, _PyTime_t denom) +_PyTimeFraction_Set(_PyTimeFraction *frac, PyTime_t numer, PyTime_t denom) { if (numer < 1 || denom < 1) { return -1; } - _PyTime_t gcd = _PyTime_GCD(numer, denom); + PyTime_t gcd = _PyTime_GCD(numer, denom); frac->numer = numer / gcd; frac->denom = denom / gcd; return 0; @@ -104,29 +104,13 @@ static void pytime_overflow(void) { PyErr_SetString(PyExc_OverflowError, - "timestamp too large to convert to C _PyTime_t"); -} - - -static inline _PyTime_t -pytime_from_nanoseconds(_PyTime_t t) -{ - // _PyTime_t is a number of nanoseconds - return t; -} - - -static inline _PyTime_t -pytime_as_nanoseconds(_PyTime_t t) -{ - // _PyTime_t is a number of nanoseconds: see pytime_from_nanoseconds() - return t; + "timestamp too large to convert to C PyTime_t"); } // Compute t1 + t2. Clamp to [PyTime_MIN; PyTime_MAX] on overflow. static inline int -pytime_add(_PyTime_t *t1, _PyTime_t t2) +pytime_add(PyTime_t *t1, PyTime_t t2) { if (t2 > 0 && *t1 > PyTime_MAX - t2) { *t1 = PyTime_MAX; @@ -143,8 +127,8 @@ pytime_add(_PyTime_t *t1, _PyTime_t t2) } -_PyTime_t -_PyTime_Add(_PyTime_t t1, _PyTime_t t2) +PyTime_t +_PyTime_Add(PyTime_t t1, PyTime_t t2) { (void)pytime_add(&t1, t2); return t1; @@ -152,7 +136,7 @@ _PyTime_Add(_PyTime_t t1, _PyTime_t t2) static inline int -pytime_mul_check_overflow(_PyTime_t a, _PyTime_t b) +pytime_mul_check_overflow(PyTime_t a, PyTime_t b) { if (b != 0) { assert(b > 0); @@ -166,7 +150,7 @@ pytime_mul_check_overflow(_PyTime_t a, _PyTime_t b) // Compute t * k. Clamp to [PyTime_MIN; PyTime_MAX] on overflow. static inline int -pytime_mul(_PyTime_t *t, _PyTime_t k) +pytime_mul(PyTime_t *t, PyTime_t k) { assert(k >= 0); if (pytime_mul_check_overflow(*t, k)) { @@ -181,19 +165,19 @@ pytime_mul(_PyTime_t *t, _PyTime_t k) // Compute t * k. Clamp to [PyTime_MIN; PyTime_MAX] on overflow. -static inline _PyTime_t -_PyTime_Mul(_PyTime_t t, _PyTime_t k) +static inline PyTime_t +_PyTime_Mul(PyTime_t t, PyTime_t k) { (void)pytime_mul(&t, k); return t; } -_PyTime_t -_PyTimeFraction_Mul(_PyTime_t ticks, const _PyTimeFraction *frac) +PyTime_t +_PyTimeFraction_Mul(PyTime_t ticks, const _PyTimeFraction *frac) { - const _PyTime_t mul = frac->numer; - const _PyTime_t div = frac->denom; + const PyTime_t mul = frac->numer; + const PyTime_t div = frac->denom; if (div == 1) { // Fast-path taken by mach_absolute_time() with 1/1 time base. @@ -205,7 +189,7 @@ _PyTimeFraction_Mul(_PyTime_t ticks, const _PyTimeFraction *frac) (ticks * mul) / div == (ticks / div) * mul + (ticks % div) * mul / div */ - _PyTime_t intpart, remaining; + PyTime_t intpart, remaining; intpart = ticks / div; ticks %= div; remaining = _PyTime_Mul(ticks, mul) / div; @@ -247,17 +231,17 @@ _PyLong_FromTime_t(time_t t) } -// Convert _PyTime_t to time_t. +// Convert PyTime_t to time_t. // Return 0 on success. Return -1 and clamp the value on overflow. static int -_PyTime_AsTime_t(_PyTime_t t, time_t *t2) +_PyTime_AsTime_t(PyTime_t t, time_t *t2) { #if SIZEOF_TIME_T < _SIZEOF_PYTIME_T - if ((_PyTime_t)PY_TIME_T_MAX < t) { + if ((PyTime_t)PY_TIME_T_MAX < t) { *t2 = PY_TIME_T_MAX; return -1; } - if (t < (_PyTime_t)PY_TIME_T_MIN) { + if (t < (PyTime_t)PY_TIME_T_MIN) { *t2 = PY_TIME_T_MIN; return -1; } @@ -268,17 +252,17 @@ _PyTime_AsTime_t(_PyTime_t t, time_t *t2) #ifdef MS_WINDOWS -// Convert _PyTime_t to long. +// Convert PyTime_t to long. // Return 0 on success. Return -1 and clamp the value on overflow. static int -_PyTime_AsLong(_PyTime_t t, long *t2) +_PyTime_AsCLong(PyTime_t t, long *t2) { #if SIZEOF_LONG < _SIZEOF_PYTIME_T - if ((_PyTime_t)LONG_MAX < t) { + if ((PyTime_t)LONG_MAX < t) { *t2 = LONG_MAX; return -1; } - if (t < (_PyTime_t)LONG_MIN) { + if (t < (PyTime_t)LONG_MIN) { *t2 = LONG_MIN; return -1; } @@ -453,50 +437,42 @@ _PyTime_ObjectToTimeval(PyObject *obj, time_t *sec, long *usec, } -_PyTime_t +PyTime_t _PyTime_FromSeconds(int seconds) { /* ensure that integer overflow cannot happen, int type should have 32 - bits, whereas _PyTime_t type has at least 64 bits (SEC_TO_NS takes 30 + bits, whereas PyTime_t type has at least 64 bits (SEC_TO_NS takes 30 bits). */ - static_assert(INT_MAX <= PyTime_MAX / SEC_TO_NS, "_PyTime_t overflow"); - static_assert(INT_MIN >= PyTime_MIN / SEC_TO_NS, "_PyTime_t underflow"); + static_assert(INT_MAX <= PyTime_MAX / SEC_TO_NS, "PyTime_t overflow"); + static_assert(INT_MIN >= PyTime_MIN / SEC_TO_NS, "PyTime_t underflow"); - _PyTime_t t = (_PyTime_t)seconds; + PyTime_t t = (PyTime_t)seconds; assert((t >= 0 && t <= PyTime_MAX / SEC_TO_NS) || (t < 0 && t >= PyTime_MIN / SEC_TO_NS)); t *= SEC_TO_NS; - return pytime_from_nanoseconds(t); -} - - -_PyTime_t -_PyTime_FromNanoseconds(_PyTime_t ns) -{ - return pytime_from_nanoseconds(ns); + return t; } -_PyTime_t -_PyTime_FromMicrosecondsClamp(_PyTime_t us) +PyTime_t +_PyTime_FromMicrosecondsClamp(PyTime_t us) { - _PyTime_t ns = _PyTime_Mul(us, US_TO_NS); - return pytime_from_nanoseconds(ns); + PyTime_t ns = _PyTime_Mul(us, US_TO_NS); + return ns; } int -_PyTime_FromNanosecondsObject(_PyTime_t *tp, PyObject *obj) +_PyTime_FromLong(PyTime_t *tp, PyObject *obj) { - if (!PyLong_Check(obj)) { PyErr_Format(PyExc_TypeError, "expect int, got %s", Py_TYPE(obj)->tp_name); return -1; } - static_assert(sizeof(long long) == sizeof(_PyTime_t), - "_PyTime_t is not long long"); + static_assert(sizeof(long long) == sizeof(PyTime_t), + "PyTime_t is not long long"); long long nsec = PyLong_AsLongLong(obj); if (nsec == -1 && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_OverflowError)) { @@ -505,28 +481,28 @@ _PyTime_FromNanosecondsObject(_PyTime_t *tp, PyObject *obj) return -1; } - _PyTime_t t = (_PyTime_t)nsec; - *tp = pytime_from_nanoseconds(t); + PyTime_t t = (PyTime_t)nsec; + *tp = t; return 0; } #ifdef HAVE_CLOCK_GETTIME static int -pytime_fromtimespec(_PyTime_t *tp, const struct timespec *ts, int raise_exc) +pytime_fromtimespec(PyTime_t *tp, const struct timespec *ts, int raise_exc) { - _PyTime_t t, tv_nsec; + PyTime_t t, tv_nsec; - static_assert(sizeof(ts->tv_sec) <= sizeof(_PyTime_t), - "timespec.tv_sec is larger than _PyTime_t"); - t = (_PyTime_t)ts->tv_sec; + static_assert(sizeof(ts->tv_sec) <= sizeof(PyTime_t), + "timespec.tv_sec is larger than PyTime_t"); + t = (PyTime_t)ts->tv_sec; int res1 = pytime_mul(&t, SEC_TO_NS); tv_nsec = ts->tv_nsec; int res2 = pytime_add(&t, tv_nsec); - *tp = pytime_from_nanoseconds(t); + *tp = t; if (raise_exc && (res1 < 0 || res2 < 0)) { pytime_overflow(); @@ -536,7 +512,7 @@ pytime_fromtimespec(_PyTime_t *tp, const struct timespec *ts, int raise_exc) } int -_PyTime_FromTimespec(_PyTime_t *tp, const struct timespec *ts) +_PyTime_FromTimespec(PyTime_t *tp, const struct timespec *ts) { return pytime_fromtimespec(tp, ts, 1); } @@ -545,18 +521,18 @@ _PyTime_FromTimespec(_PyTime_t *tp, const struct timespec *ts) #ifndef MS_WINDOWS static int -pytime_fromtimeval(_PyTime_t *tp, struct timeval *tv, int raise_exc) +pytime_fromtimeval(PyTime_t *tp, struct timeval *tv, int raise_exc) { - static_assert(sizeof(tv->tv_sec) <= sizeof(_PyTime_t), - "timeval.tv_sec is larger than _PyTime_t"); - _PyTime_t t = (_PyTime_t)tv->tv_sec; + static_assert(sizeof(tv->tv_sec) <= sizeof(PyTime_t), + "timeval.tv_sec is larger than PyTime_t"); + PyTime_t t = (PyTime_t)tv->tv_sec; int res1 = pytime_mul(&t, SEC_TO_NS); - _PyTime_t usec = (_PyTime_t)tv->tv_usec * US_TO_NS; + PyTime_t usec = (PyTime_t)tv->tv_usec * US_TO_NS; int res2 = pytime_add(&t, usec); - *tp = pytime_from_nanoseconds(t); + *tp = t; if (raise_exc && (res1 < 0 || res2 < 0)) { pytime_overflow(); @@ -567,7 +543,7 @@ pytime_fromtimeval(_PyTime_t *tp, struct timeval *tv, int raise_exc) int -_PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv) +_PyTime_FromTimeval(PyTime_t *tp, struct timeval *tv) { return pytime_fromtimeval(tp, tv, 1); } @@ -575,7 +551,7 @@ _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv) static int -pytime_from_double(_PyTime_t *tp, double value, _PyTime_round_t round, +pytime_from_double(PyTime_t *tp, double value, _PyTime_round_t round, long unit_to_ns) { /* volatile avoids optimization changing how numbers are rounded */ @@ -591,15 +567,15 @@ pytime_from_double(_PyTime_t *tp, double value, _PyTime_round_t round, pytime_time_t_overflow(); return -1; } - _PyTime_t ns = (_PyTime_t)d; + PyTime_t ns = (PyTime_t)d; - *tp = pytime_from_nanoseconds(ns); + *tp = ns; return 0; } static int -pytime_from_object(_PyTime_t *tp, PyObject *obj, _PyTime_round_t round, +pytime_from_object(PyTime_t *tp, PyObject *obj, _PyTime_round_t round, long unit_to_ns) { if (PyFloat_Check(obj)) { @@ -620,45 +596,44 @@ pytime_from_object(_PyTime_t *tp, PyObject *obj, _PyTime_round_t round, return -1; } - static_assert(sizeof(long long) <= sizeof(_PyTime_t), - "_PyTime_t is smaller than long long"); - _PyTime_t ns = (_PyTime_t)sec; + static_assert(sizeof(long long) <= sizeof(PyTime_t), + "PyTime_t is smaller than long long"); + PyTime_t ns = (PyTime_t)sec; if (pytime_mul(&ns, unit_to_ns) < 0) { pytime_overflow(); return -1; } - *tp = pytime_from_nanoseconds(ns); + *tp = ns; return 0; } } int -_PyTime_FromSecondsObject(_PyTime_t *tp, PyObject *obj, _PyTime_round_t round) +_PyTime_FromSecondsObject(PyTime_t *tp, PyObject *obj, _PyTime_round_t round) { return pytime_from_object(tp, obj, round, SEC_TO_NS); } int -_PyTime_FromMillisecondsObject(_PyTime_t *tp, PyObject *obj, _PyTime_round_t round) +_PyTime_FromMillisecondsObject(PyTime_t *tp, PyObject *obj, _PyTime_round_t round) { return pytime_from_object(tp, obj, round, MS_TO_NS); } double -PyTime_AsSecondsDouble(PyTime_t t) +PyTime_AsSecondsDouble(PyTime_t ns) { /* volatile avoids optimization changing how numbers are rounded */ volatile double d; - PyTime_t ns = pytime_as_nanoseconds(t); if (ns % SEC_TO_NS == 0) { /* Divide using integers to avoid rounding issues on the integer part. 1e-9 cannot be stored exactly in IEEE 64-bit. */ - _PyTime_t secs = ns / SEC_TO_NS; + PyTime_t secs = ns / SEC_TO_NS; d = (double)secs; } else { @@ -670,18 +645,17 @@ PyTime_AsSecondsDouble(PyTime_t t) PyObject * -_PyTime_AsNanosecondsObject(_PyTime_t t) +_PyTime_AsLong(PyTime_t ns) { - _PyTime_t ns = pytime_as_nanoseconds(t); - static_assert(sizeof(long long) >= sizeof(_PyTime_t), - "_PyTime_t is larger than long long"); + static_assert(sizeof(long long) >= sizeof(PyTime_t), + "PyTime_t is larger than long long"); return PyLong_FromLongLong((long long)ns); } -_PyTime_t +PyTime_t _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round) { - _PyTime_t tp; + PyTime_t tp; if(pytime_from_double(&tp, seconds, round, SEC_TO_NS) < 0) { return -1; } @@ -689,14 +663,14 @@ _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round) } -static _PyTime_t -pytime_divide_round_up(const _PyTime_t t, const _PyTime_t k) +static PyTime_t +pytime_divide_round_up(const PyTime_t t, const PyTime_t k) { assert(k > 1); if (t >= 0) { // Don't use (t + k - 1) / k to avoid integer overflow // if t is equal to PyTime_MAX - _PyTime_t q = t / k; + PyTime_t q = t / k; if (t % k) { q += 1; } @@ -705,7 +679,7 @@ pytime_divide_round_up(const _PyTime_t t, const _PyTime_t k) else { // Don't use (t - (k - 1)) / k to avoid integer overflow // if t is equals to PyTime_MIN. - _PyTime_t q = t / k; + PyTime_t q = t / k; if (t % k) { q -= 1; } @@ -714,15 +688,15 @@ pytime_divide_round_up(const _PyTime_t t, const _PyTime_t k) } -static _PyTime_t -pytime_divide(const _PyTime_t t, const _PyTime_t k, +static PyTime_t +pytime_divide(const PyTime_t t, const PyTime_t k, const _PyTime_round_t round) { assert(k > 1); if (round == _PyTime_ROUND_HALF_EVEN) { - _PyTime_t x = t / k; - _PyTime_t r = t % k; - _PyTime_t abs_r = Py_ABS(r); + PyTime_t x = t / k; + PyTime_t r = t % k; + PyTime_t abs_r = Py_ABS(r); if (abs_r > k / 2 || (abs_r == k / 2 && (Py_ABS(x) & 1))) { if (t >= 0) { x++; @@ -761,12 +735,12 @@ pytime_divide(const _PyTime_t t, const _PyTime_t k, // Return 0 on success. // Return -1 on underflow and store (PyTime_MIN, 0) in (pq, pr). static int -pytime_divmod(const _PyTime_t t, const _PyTime_t k, - _PyTime_t *pq, _PyTime_t *pr) +pytime_divmod(const PyTime_t t, const PyTime_t k, + PyTime_t *pq, PyTime_t *pr) { assert(k > 1); - _PyTime_t q = t / k; - _PyTime_t r = t % k; + PyTime_t q = t / k; + PyTime_t r = t % k; if (r < 0) { if (q == PyTime_MIN) { *pq = PyTime_MIN; @@ -785,39 +759,35 @@ pytime_divmod(const _PyTime_t t, const _PyTime_t k, #ifdef MS_WINDOWS -_PyTime_t -_PyTime_As100Nanoseconds(_PyTime_t t, _PyTime_round_t round) +PyTime_t +_PyTime_As100Nanoseconds(PyTime_t ns, _PyTime_round_t round) { - _PyTime_t ns = pytime_as_nanoseconds(t); return pytime_divide(ns, NS_TO_100NS, round); } #endif -_PyTime_t -_PyTime_AsMicroseconds(_PyTime_t t, _PyTime_round_t round) +PyTime_t +_PyTime_AsMicroseconds(PyTime_t ns, _PyTime_round_t round) { - _PyTime_t ns = pytime_as_nanoseconds(t); return pytime_divide(ns, NS_TO_US, round); } -_PyTime_t -_PyTime_AsMilliseconds(_PyTime_t t, _PyTime_round_t round) +PyTime_t +_PyTime_AsMilliseconds(PyTime_t ns, _PyTime_round_t round) { - _PyTime_t ns = pytime_as_nanoseconds(t); return pytime_divide(ns, NS_TO_MS, round); } static int -pytime_as_timeval(_PyTime_t t, _PyTime_t *ptv_sec, int *ptv_usec, +pytime_as_timeval(PyTime_t ns, PyTime_t *ptv_sec, int *ptv_usec, _PyTime_round_t round) { - _PyTime_t ns = pytime_as_nanoseconds(t); - _PyTime_t us = pytime_divide(ns, US_TO_NS, round); + PyTime_t us = pytime_divide(ns, US_TO_NS, round); - _PyTime_t tv_sec, tv_usec; + PyTime_t tv_sec, tv_usec; int res = pytime_divmod(us, SEC_TO_US, &tv_sec, &tv_usec); *ptv_sec = tv_sec; *ptv_usec = (int)tv_usec; @@ -826,16 +796,16 @@ pytime_as_timeval(_PyTime_t t, _PyTime_t *ptv_sec, int *ptv_usec, static int -pytime_as_timeval_struct(_PyTime_t t, struct timeval *tv, +pytime_as_timeval_struct(PyTime_t t, struct timeval *tv, _PyTime_round_t round, int raise_exc) { - _PyTime_t tv_sec; + PyTime_t tv_sec; int tv_usec; int res = pytime_as_timeval(t, &tv_sec, &tv_usec, round); int res2; #ifdef MS_WINDOWS // On Windows, timeval.tv_sec type is long - res2 = _PyTime_AsLong(tv_sec, &tv->tv_sec); + res2 = _PyTime_AsCLong(tv_sec, &tv->tv_sec); #else res2 = _PyTime_AsTime_t(tv_sec, &tv->tv_sec); #endif @@ -853,24 +823,24 @@ pytime_as_timeval_struct(_PyTime_t t, struct timeval *tv, int -_PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +_PyTime_AsTimeval(PyTime_t t, struct timeval *tv, _PyTime_round_t round) { return pytime_as_timeval_struct(t, tv, round, 1); } void -_PyTime_AsTimeval_clamp(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +_PyTime_AsTimeval_clamp(PyTime_t t, struct timeval *tv, _PyTime_round_t round) { (void)pytime_as_timeval_struct(t, tv, round, 0); } int -_PyTime_AsTimevalTime_t(_PyTime_t t, time_t *p_secs, int *us, +_PyTime_AsTimevalTime_t(PyTime_t t, time_t *p_secs, int *us, _PyTime_round_t round) { - _PyTime_t secs; + PyTime_t secs; if (pytime_as_timeval(t, &secs, us, round) < 0) { pytime_time_t_overflow(); return -1; @@ -886,10 +856,9 @@ _PyTime_AsTimevalTime_t(_PyTime_t t, time_t *p_secs, int *us, #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) static int -pytime_as_timespec(_PyTime_t t, struct timespec *ts, int raise_exc) +pytime_as_timespec(PyTime_t ns, struct timespec *ts, int raise_exc) { - _PyTime_t ns = pytime_as_nanoseconds(t); - _PyTime_t tv_sec, tv_nsec; + PyTime_t tv_sec, tv_nsec; int res = pytime_divmod(ns, SEC_TO_NS, &tv_sec, &tv_nsec); int res2 = _PyTime_AsTime_t(tv_sec, &ts->tv_sec); @@ -906,13 +875,13 @@ pytime_as_timespec(_PyTime_t t, struct timespec *ts, int raise_exc) } void -_PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts) +_PyTime_AsTimespec_clamp(PyTime_t t, struct timespec *ts) { (void)pytime_as_timespec(t, ts, 0); } int -_PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) +_PyTime_AsTimespec(PyTime_t t, struct timespec *ts) { return pytime_as_timespec(t, ts, 1); } @@ -921,7 +890,7 @@ _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) // N.B. If raise_exc=0, this may be called without the GIL. static int -py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) +py_get_system_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); @@ -935,8 +904,8 @@ py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) /* 11,644,473,600,000,000,000: number of nanoseconds between the 1st january 1601 and the 1st january 1970 (369 years + 89 leap days). */ - _PyTime_t ns = large.QuadPart * 100 - 11644473600000000000; - *tp = pytime_from_nanoseconds(ns); + PyTime_t ns = large.QuadPart * 100 - 11644473600000000000; + *tp = ns; if (info) { DWORD timeAdjustment, timeIncrement; BOOL isTimeAdjustmentDisabled, ok; @@ -1031,10 +1000,10 @@ py_get_system_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) } -_PyTime_t -_PyTime_GetSystemClock(void) +PyTime_t +_PyTime_TimeUnchecked(void) { - _PyTime_t t; + PyTime_t t; if (py_get_system_clock(&t, NULL, 0) < 0) { // If clock_gettime(CLOCK_REALTIME) or gettimeofday() fails: // silently ignore the failure and return 0. @@ -1048,16 +1017,14 @@ int PyTime_Time(PyTime_t *result) { if (py_get_system_clock(result, NULL, 1) < 0) { - // If clock_gettime(CLOCK_REALTIME) or gettimeofday() fails: - // silently ignore the failure and return 0. *result = 0; return -1; } - return 1; + return 0; } int -_PyTime_GetSystemClockWithInfo(_PyTime_t *t, _Py_clock_info_t *info) +_PyTime_TimeWithInfo(PyTime_t *t, _Py_clock_info_t *info) { return py_get_system_clock(t, info, 1); } @@ -1073,13 +1040,13 @@ py_mach_timebase_info(_PyTimeFraction *base, int raise) (void)mach_timebase_info(&timebase); // Check that timebase.numer and timebase.denom can be casted to - // _PyTime_t. In practice, timebase uses uint32_t, so casting cannot + // PyTime_t. In practice, timebase uses uint32_t, so casting cannot // overflow. At the end, only make sure that the type is uint32_t - // (_PyTime_t is 64-bit long). - Py_BUILD_ASSERT(sizeof(timebase.numer) <= sizeof(_PyTime_t)); - Py_BUILD_ASSERT(sizeof(timebase.denom) <= sizeof(_PyTime_t)); - _PyTime_t numer = (_PyTime_t)timebase.numer; - _PyTime_t denom = (_PyTime_t)timebase.denom; + // (PyTime_t is 64-bit long). + Py_BUILD_ASSERT(sizeof(timebase.numer) <= sizeof(PyTime_t)); + Py_BUILD_ASSERT(sizeof(timebase.denom) <= sizeof(PyTime_t)); + PyTime_t numer = (PyTime_t)timebase.numer; + PyTime_t denom = (PyTime_t)timebase.denom; // Known time bases: // @@ -1100,21 +1067,21 @@ py_mach_timebase_info(_PyTimeFraction *base, int raise) // N.B. If raise_exc=0, this may be called without the GIL. static int -py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) +py_get_monotonic_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); #if defined(MS_WINDOWS) ULONGLONG ticks = GetTickCount64(); - static_assert(sizeof(ticks) <= sizeof(_PyTime_t), - "ULONGLONG is larger than _PyTime_t"); - _PyTime_t t; + static_assert(sizeof(ticks) <= sizeof(PyTime_t), + "ULONGLONG is larger than PyTime_t"); + PyTime_t t; if (ticks <= (ULONGLONG)PyTime_MAX) { - t = (_PyTime_t)ticks; + t = (PyTime_t)ticks; } else { - // GetTickCount64() maximum is larger than _PyTime_t maximum: - // ULONGLONG is unsigned, whereas _PyTime_t is signed. + // GetTickCount64() maximum is larger than PyTime_t maximum: + // ULONGLONG is unsigned, whereas PyTime_t is signed. t = PyTime_MAX; } @@ -1159,15 +1126,13 @@ py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) uint64_t uticks = mach_absolute_time(); // unsigned => signed assert(uticks <= (uint64_t)PyTime_MAX); - _PyTime_t ticks = (_PyTime_t)uticks; + PyTime_t ticks = (PyTime_t)uticks; - _PyTime_t ns = _PyTimeFraction_Mul(ticks, &base); - *tp = pytime_from_nanoseconds(ns); + PyTime_t ns = _PyTimeFraction_Mul(ticks, &base); + *tp = ns; #elif defined(__hpux) - hrtime_t time; - - time = gethrtime(); + hrtime_t time = gethrtime(); if (time == -1) { if (raise_exc) { PyErr_SetFromErrno(PyExc_OSError); @@ -1175,7 +1140,7 @@ py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) return -1; } - *tp = pytime_from_nanoseconds(time); + *tp = time; if (info) { info->implementation = "gethrtime()"; @@ -1223,10 +1188,10 @@ py_get_monotonic_clock(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) } -_PyTime_t -_PyTime_GetMonotonicClock(void) +PyTime_t +_PyTime_MonotonicUnchecked(void) { - _PyTime_t t; + PyTime_t t; if (py_get_monotonic_clock(&t, NULL, 0) < 0) { // If mach_timebase_info(), clock_gettime() or gethrtime() fails: // silently ignore the failure and return 0. @@ -1248,7 +1213,7 @@ PyTime_Monotonic(PyTime_t *result) int -_PyTime_GetMonotonicClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +_PyTime_MonotonicWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { return py_get_monotonic_clock(tp, info, 1); } @@ -1268,8 +1233,8 @@ py_win_perf_counter_frequency(_PyTimeFraction *base, int raise) // Since Windows XP, frequency cannot be zero. assert(frequency >= 1); - Py_BUILD_ASSERT(sizeof(_PyTime_t) == sizeof(frequency)); - _PyTime_t denom = (_PyTime_t)frequency; + Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(frequency)); + PyTime_t denom = (PyTime_t)frequency; // Known QueryPerformanceFrequency() values: // @@ -1288,7 +1253,7 @@ py_win_perf_counter_frequency(_PyTimeFraction *base, int raise) // N.B. If raise_exc=0, this may be called without the GIL. static int -py_get_win_perf_counter(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) +py_get_win_perf_counter(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); @@ -1310,35 +1275,35 @@ py_get_win_perf_counter(_PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) QueryPerformanceCounter(&now); LONGLONG ticksll = now.QuadPart; - /* Make sure that casting LONGLONG to _PyTime_t cannot overflow, + /* Make sure that casting LONGLONG to PyTime_t cannot overflow, both types are signed */ - _PyTime_t ticks; + PyTime_t ticks; static_assert(sizeof(ticksll) <= sizeof(ticks), - "LONGLONG is larger than _PyTime_t"); - ticks = (_PyTime_t)ticksll; + "LONGLONG is larger than PyTime_t"); + ticks = (PyTime_t)ticksll; - _PyTime_t ns = _PyTimeFraction_Mul(ticks, &base); - *tp = pytime_from_nanoseconds(ns); + PyTime_t ns = _PyTimeFraction_Mul(ticks, &base); + *tp = ns; return 0; } #endif // MS_WINDOWS int -_PyTime_GetPerfCounterWithInfo(_PyTime_t *t, _Py_clock_info_t *info) +_PyTime_PerfCounterWithInfo(PyTime_t *t, _Py_clock_info_t *info) { #ifdef MS_WINDOWS return py_get_win_perf_counter(t, info, 1); #else - return _PyTime_GetMonotonicClockWithInfo(t, info); + return _PyTime_MonotonicWithInfo(t, info); #endif } -_PyTime_t -_PyTime_GetPerfCounter(void) +PyTime_t +_PyTime_PerfCounterUnchecked(void) { - _PyTime_t t; + PyTime_t t; int res; #ifdef MS_WINDOWS res = py_get_win_perf_counter(&t, NULL, 0); @@ -1440,17 +1405,17 @@ _PyTime_gmtime(time_t t, struct tm *tm) } -_PyTime_t -_PyDeadline_Init(_PyTime_t timeout) +PyTime_t +_PyDeadline_Init(PyTime_t timeout) { - _PyTime_t now = _PyTime_GetMonotonicClock(); + PyTime_t now = _PyTime_MonotonicUnchecked(); return _PyTime_Add(now, timeout); } -_PyTime_t -_PyDeadline_Get(_PyTime_t deadline) +PyTime_t +_PyDeadline_Get(PyTime_t deadline) { - _PyTime_t now = _PyTime_GetMonotonicClock(); + PyTime_t now = _PyTime_MonotonicUnchecked(); return deadline - now; } diff --git a/Python/qsbr.c b/Python/qsbr.c new file mode 100644 index 00000000000000..7f7ae03cf60d22 --- /dev/null +++ b/Python/qsbr.c @@ -0,0 +1,286 @@ +/* + * Implementation of safe memory reclamation scheme using + * quiescent states. + * + * This is dervied from the "GUS" safe memory reclamation technique + * in FreeBSD written by Jeffrey Roberson. It is heavily modified. Any bugs + * in this code are likely due to the modifications. + * + * The original copyright is preserved below. + * + * Copyright (c) 2019,2020 Jeffrey Roberson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "Python.h" +#include "pycore_initconfig.h" // _PyStatus_NO_MEMORY() +#include "pycore_lock.h" // PyMutex_Lock() +#include "pycore_qsbr.h" +#include "pycore_pystate.h" // _PyThreadState_GET() + + +// Wrap-around safe comparison. This is a holdover from the FreeBSD +// implementation, which uses 32-bit sequence numbers. We currently use 64-bit +// sequence numbers, so wrap-around is unlikely. +#define QSBR_LT(a, b) ((int64_t)((a)-(b)) < 0) +#define QSBR_LEQ(a, b) ((int64_t)((a)-(b)) <= 0) + +// Starting size of the array of qsbr thread states +#define MIN_ARRAY_SIZE 8 + +// For _Py_qsbr_deferred_advance(): the number of deferrals before advancing +// the write sequence. +#define QSBR_DEFERRED_LIMIT 10 + +// Allocate a QSBR thread state from the freelist +static struct _qsbr_thread_state * +qsbr_allocate(struct _qsbr_shared *shared) +{ + struct _qsbr_thread_state *qsbr = shared->freelist; + if (qsbr == NULL) { + return NULL; + } + shared->freelist = qsbr->freelist_next; + qsbr->freelist_next = NULL; + qsbr->shared = shared; + qsbr->allocated = true; + return qsbr; +} + +// Initialize (or reintialize) the freelist of QSBR thread states +static void +initialize_new_array(struct _qsbr_shared *shared) +{ + for (Py_ssize_t i = 0; i != shared->size; i++) { + struct _qsbr_thread_state *qsbr = &shared->array[i].qsbr; + if (qsbr->tstate != NULL) { + // Update the thread state pointer to its QSBR state + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)qsbr->tstate; + tstate->qsbr = qsbr; + } + if (!qsbr->allocated) { + // Push to freelist + qsbr->freelist_next = shared->freelist; + shared->freelist = qsbr; + } + } +} + +// Grow the array of QSBR thread states. Returns 0 on success, -1 on failure. +static int +grow_thread_array(struct _qsbr_shared *shared) +{ + Py_ssize_t new_size = shared->size * 2; + if (new_size < MIN_ARRAY_SIZE) { + new_size = MIN_ARRAY_SIZE; + } + + struct _qsbr_pad *array = PyMem_RawCalloc(new_size, sizeof(*array)); + if (array == NULL) { + return -1; + } + + struct _qsbr_pad *old = shared->array; + if (old != NULL) { + memcpy(array, shared->array, shared->size * sizeof(*array)); + } + + shared->array = array; + shared->size = new_size; + shared->freelist = NULL; + initialize_new_array(shared); + + PyMem_RawFree(old); + return 0; +} + +uint64_t +_Py_qsbr_advance(struct _qsbr_shared *shared) +{ + // NOTE: with 64-bit sequence numbers, we don't have to worry too much + // about the wr_seq getting too far ahead of rd_seq, but if we ever use + // 32-bit sequence numbers, we'll need to be more careful. + return _Py_atomic_add_uint64(&shared->wr_seq, QSBR_INCR) + QSBR_INCR; +} + +uint64_t +_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr) +{ + if (++qsbr->deferrals < QSBR_DEFERRED_LIMIT) { + return _Py_qsbr_shared_current(qsbr->shared) + QSBR_INCR; + } + qsbr->deferrals = 0; + return _Py_qsbr_advance(qsbr->shared); +} + +static uint64_t +qsbr_poll_scan(struct _qsbr_shared *shared) +{ + // Synchronize with store in _Py_qsbr_attach(). We need to ensure that + // the reads from each thread's sequence number are not reordered to see + // earlier "offline" states. + _Py_atomic_fence_seq_cst(); + + // Compute the minimum sequence number of all attached threads + uint64_t min_seq = _Py_atomic_load_uint64(&shared->wr_seq); + struct _qsbr_pad *array = shared->array; + for (Py_ssize_t i = 0, size = shared->size; i != size; i++) { + struct _qsbr_thread_state *qsbr = &array[i].qsbr; + + uint64_t seq = _Py_atomic_load_uint64(&qsbr->seq); + if (seq != QSBR_OFFLINE && QSBR_LT(seq, min_seq)) { + min_seq = seq; + } + } + + // Update the shared read sequence + uint64_t rd_seq = _Py_atomic_load_uint64(&shared->rd_seq); + if (QSBR_LT(rd_seq, min_seq)) { + // It's okay if the compare-exchange failed: another thread updated it + (void)_Py_atomic_compare_exchange_uint64(&shared->rd_seq, &rd_seq, min_seq); + rd_seq = min_seq; + } + + return rd_seq; +} + +bool +_Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal) +{ + assert(_PyThreadState_GET()->state == _Py_THREAD_ATTACHED); + + uint64_t rd_seq = _Py_atomic_load_uint64(&qsbr->shared->rd_seq); + if (QSBR_LEQ(goal, rd_seq)) { + return true; + } + + rd_seq = qsbr_poll_scan(qsbr->shared); + return QSBR_LEQ(goal, rd_seq); +} + +void +_Py_qsbr_attach(struct _qsbr_thread_state *qsbr) +{ + assert(qsbr->seq == 0 && "already attached"); + + uint64_t seq = _Py_qsbr_shared_current(qsbr->shared); + _Py_atomic_store_uint64(&qsbr->seq, seq); // needs seq_cst +} + +void +_Py_qsbr_detach(struct _qsbr_thread_state *qsbr) +{ + assert(qsbr->seq != 0 && "already detached"); + + _Py_atomic_store_uint64_release(&qsbr->seq, QSBR_OFFLINE); +} + +Py_ssize_t +_Py_qsbr_reserve(PyInterpreterState *interp) +{ + struct _qsbr_shared *shared = &interp->qsbr; + + PyMutex_Lock(&shared->mutex); + // Try allocating from our internal freelist + struct _qsbr_thread_state *qsbr = qsbr_allocate(shared); + + // If there are no free entries, we pause all threads, grow the array, + // and update the pointers in PyThreadState to entries in the new array. + if (qsbr == NULL) { + _PyEval_StopTheWorld(interp); + if (grow_thread_array(shared) == 0) { + qsbr = qsbr_allocate(shared); + } + _PyEval_StartTheWorld(interp); + } + PyMutex_Unlock(&shared->mutex); + + if (qsbr == NULL) { + return -1; + } + + // Return an index rather than the pointer because the array may be + // resized and the pointer invalidated. + return (struct _qsbr_pad *)qsbr - shared->array; +} + +void +_Py_qsbr_register(_PyThreadStateImpl *tstate, PyInterpreterState *interp, + Py_ssize_t index) +{ + // Associate the QSBR state with the thread state + struct _qsbr_shared *shared = &interp->qsbr; + + PyMutex_Lock(&shared->mutex); + struct _qsbr_thread_state *qsbr = &interp->qsbr.array[index].qsbr; + assert(qsbr->allocated && qsbr->tstate == NULL); + qsbr->tstate = (PyThreadState *)tstate; + tstate->qsbr = qsbr; + PyMutex_Unlock(&shared->mutex); +} + +void +_Py_qsbr_unregister(_PyThreadStateImpl *tstate) +{ + struct _qsbr_thread_state *qsbr = tstate->qsbr; + struct _qsbr_shared *shared = qsbr->shared; + + assert(qsbr->seq == 0 && "thread state must be detached"); + + PyMutex_Lock(&shared->mutex); + assert(qsbr->allocated && qsbr->tstate == (PyThreadState *)tstate); + tstate->qsbr = NULL; + qsbr->tstate = NULL; + qsbr->allocated = false; + qsbr->freelist_next = shared->freelist; + shared->freelist = qsbr; + PyMutex_Unlock(&shared->mutex); +} + +void +_Py_qsbr_fini(PyInterpreterState *interp) +{ + struct _qsbr_shared *shared = &interp->qsbr; + PyMem_RawFree(shared->array); + shared->array = NULL; + shared->size = 0; + shared->freelist = NULL; +} + +void +_Py_qsbr_after_fork(_PyThreadStateImpl *tstate) +{ + struct _qsbr_thread_state *this_qsbr = tstate->qsbr; + struct _qsbr_shared *shared = this_qsbr->shared; + + _PyMutex_at_fork_reinit(&shared->mutex); + + for (Py_ssize_t i = 0; i != shared->size; i++) { + struct _qsbr_thread_state *qsbr = &shared->array[i].qsbr; + if (qsbr != this_qsbr && qsbr->allocated) { + qsbr->tstate = NULL; + qsbr->allocated = false; + qsbr->freelist_next = shared->freelist; + shared->freelist = qsbr; + } + } +} diff --git a/Python/specialize.c b/Python/specialize.c index 2256d79b387c56..f83d8a9ceb0282 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -17,6 +17,7 @@ #include // rand() +extern const char *_PyUOpName(int index); /* For guidance on adding or extending families of instructions see * ./adaptive.md @@ -235,6 +236,7 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); + fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated); print_histogram(out, "Trace length", stats->trace_length_hist); print_histogram(out, "Trace run length", stats->trace_run_length_hist); @@ -246,17 +248,12 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) stats->optimizer_failure_reason_no_memory); const char* const* names; - for (int i = 0; i < 512; i++) { - if (i < 256) { - names = _PyOpcode_OpName; - } else { - names = _PyOpcode_uop_name; - } + for (int i = 0; i <= MAX_UOP_ID; i++) { if (stats->opcode[i].execution_count) { - fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", names[i], stats->opcode[i].execution_count); + fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); } if (stats->opcode[i].miss) { - fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", names[i], stats->opcode[i].miss); + fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss); } } diff --git a/Python/symtable.c b/Python/symtable.c index d69516351efba2..b69452bf77c517 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1359,16 +1359,22 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, } static long -symtable_lookup(struct symtable *st, PyObject *name) +symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name) { PyObject *mangled = _Py_Mangle(st->st_private, name); if (!mangled) return 0; - long ret = _PyST_GetSymbol(st->st_cur, mangled); + long ret = _PyST_GetSymbol(ste, mangled); Py_DECREF(mangled); return ret; } +static long +symtable_lookup(struct symtable *st, PyObject *name) +{ + return symtable_lookup_entry(st, st->st_cur, name); +} + static int symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _symtable_entry *ste, int lineno, int col_offset, int end_lineno, int end_col_offset) @@ -2009,7 +2015,7 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) * binding conflict with iteration variables, otherwise skip it */ if (ste->ste_comprehension) { - long target_in_scope = _PyST_GetSymbol(ste, target_name); + long target_in_scope = symtable_lookup_entry(st, ste, target_name); if ((target_in_scope & DEF_COMP_ITER) && (target_in_scope & DEF_LOCAL)) { PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_CONFLICT, target_name); @@ -2025,7 +2031,7 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) /* If we find a FunctionBlock entry, add as GLOBAL/LOCAL or NONLOCAL/LOCAL */ if (ste->ste_type == FunctionBlock) { - long target_in_scope = _PyST_GetSymbol(ste, target_name); + long target_in_scope = symtable_lookup_entry(st, ste, target_name); if (target_in_scope & DEF_GLOBAL) { if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e))) VISIT_QUIT(st, 0); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 69b6d886ccc3e9..1bfd031fdd26b2 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2138,7 +2138,7 @@ sys__clear_internal_caches_impl(PyObject *module) /*[clinic end generated code: output=0ee128670a4966d6 input=253e741ca744f6e8]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - _Py_Executors_InvalidateAll(interp); + _Py_Executors_InvalidateAll(interp, 0); PyType_ClearCache(); Py_RETURN_NONE; } diff --git a/Python/thread.c b/Python/thread.c index fefae8391617f7..b31d1dc5e770ef 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -20,8 +20,8 @@ // Define PY_TIMEOUT_MAX constant. #ifdef _POSIX_THREADS - // PyThread_acquire_lock_timed() uses _PyTime_FromNanoseconds(us * 1000), - // convert microseconds to nanoseconds. + // PyThread_acquire_lock_timed() uses (us * 1000) to convert microseconds + // to nanoseconds. # define PY_TIMEOUT_MAX_VALUE (LLONG_MAX / 1000) #elif defined (NT_THREADS) // WaitForSingleObject() accepts timeout in milliseconds in the range @@ -107,7 +107,7 @@ PyThread_ParseTimeoutArg(PyObject *arg, int blocking, PY_TIMEOUT_T *timeout_p) return -1; } - _PyTime_t timeout; + PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, arg, _PyTime_ROUND_TIMEOUT) < 0) { return -1; } @@ -132,14 +132,14 @@ PyThread_acquire_lock_timed_with_retries(PyThread_type_lock lock, PY_TIMEOUT_T timeout) { PyThreadState *tstate = _PyThreadState_GET(); - _PyTime_t endtime = 0; + PyTime_t endtime = 0; if (timeout > 0) { endtime = _PyDeadline_Init(timeout); } PyLockStatus r; do { - _PyTime_t microseconds; + PyTime_t microseconds; microseconds = _PyTime_AsMicroseconds(timeout, _PyTime_ROUND_CEILING); /* first a simple non-blocking try without releasing the GIL */ diff --git a/Python/thread_nt.h b/Python/thread_nt.h index ad467e0e7840e7..9dca833ff203ca 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -1,4 +1,5 @@ -#include "pycore_interp.h" // _PyInterpreterState.threads.stacksize +#include "pycore_interp.h" // _PyInterpreterState.threads.stacksize +#include "pycore_time.h" // _PyTime_AsMicroseconds() /* This code implemented by Dag.Gruneau@elsa.preseco.comm.se */ /* Fast NonRecursiveMutex support by Yakov Markovitch, markovitch@iso.ru */ @@ -76,16 +77,16 @@ EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds) } } else if (milliseconds != 0) { /* wait at least until the deadline */ - _PyTime_t nanoseconds = _PyTime_FromNanoseconds((_PyTime_t)milliseconds * 1000000); - _PyTime_t deadline = _PyTime_Add(_PyTime_GetPerfCounter(), nanoseconds); + PyTime_t nanoseconds = (PyTime_t)milliseconds * (1000 * 1000); + PyTime_t deadline = _PyTime_Add(_PyTime_PerfCounterUnchecked(), nanoseconds); while (mutex->locked) { - _PyTime_t microseconds = _PyTime_AsMicroseconds(nanoseconds, + PyTime_t microseconds = _PyTime_AsMicroseconds(nanoseconds, _PyTime_ROUND_TIMEOUT); if (PyCOND_TIMEDWAIT(&mutex->cv, &mutex->cs, microseconds) < 0) { result = WAIT_FAILED; break; } - nanoseconds = deadline - _PyTime_GetPerfCounter(); + nanoseconds = deadline - _PyTime_PerfCounterUnchecked(); if (nanoseconds <= 0) { break; } @@ -512,5 +513,10 @@ void * PyThread_tss_get(Py_tss_t *key) { assert(key != NULL); - return TlsGetValue(key->_key); + int err = GetLastError(); + void *r = TlsGetValue(key->_key); + if (r || !GetLastError()) { + SetLastError(err); + } + return r; } diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 556e3de0b071f8..64cc60053e6cf7 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -1,5 +1,6 @@ #include "pycore_interp.h" // _PyInterpreterState.threads.stacksize #include "pycore_pythread.h" // _POSIX_SEMAPHORES +#include "pycore_time.h" // _PyTime_FromMicrosecondsClamup() /* Posix threads interface */ @@ -149,16 +150,16 @@ _PyThread_cond_init(PyCOND_T *cond) void _PyThread_cond_after(long long us, struct timespec *abs) { - _PyTime_t timeout = _PyTime_FromMicrosecondsClamp(us); - _PyTime_t t; + PyTime_t timeout = _PyTime_FromMicrosecondsClamp(us); + PyTime_t t; #ifdef CONDATTR_MONOTONIC if (condattr_monotonic) { - t = _PyTime_GetMonotonicClock(); + t = _PyTime_MonotonicUnchecked(); } else #endif { - t = _PyTime_GetSystemClock(); + t = _PyTime_TimeUnchecked(); } t = _PyTime_Add(t, timeout); _PyTime_AsTimespec_clamp(t, abs); @@ -481,31 +482,31 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, (void) error; /* silence unused-but-set-variable warning */ - _PyTime_t timeout; // relative timeout + PyTime_t timeout; // relative timeout if (microseconds >= 0) { // bpo-41710: PyThread_acquire_lock_timed() cannot report timeout // overflow to the caller, so clamp the timeout to - // [_PyTime_MIN, _PyTime_MAX]. + // [PyTime_MIN, PyTime_MAX]. // - // _PyTime_MAX nanoseconds is around 292.3 years. + // PyTime_MAX nanoseconds is around 292.3 years. // // _thread.Lock.acquire() and _thread.RLock.acquire() raise an // OverflowError if microseconds is greater than PY_TIMEOUT_MAX. timeout = _PyTime_FromMicrosecondsClamp(microseconds); } else { - timeout = _PyTime_FromNanoseconds(-1); + timeout = -1; } #ifdef HAVE_SEM_CLOCKWAIT struct timespec abs_timeout; // Local scope for deadline { - _PyTime_t deadline = _PyTime_Add(_PyTime_GetMonotonicClock(), timeout); + PyTime_t deadline = _PyTime_Add(_PyTime_MonotonicUnchecked(), timeout); _PyTime_AsTimespec_clamp(deadline, &abs_timeout); } #else - _PyTime_t deadline = 0; + PyTime_t deadline = 0; if (timeout > 0 && !intr_flag) { deadline = _PyDeadline_Init(timeout); } @@ -517,7 +518,7 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, status = fix_status(sem_clockwait(thelock, CLOCK_MONOTONIC, &abs_timeout)); #else - _PyTime_t abs_time = _PyTime_Add(_PyTime_GetSystemClock(), + PyTime_t abs_time = _PyTime_Add(_PyTime_TimeUnchecked(), timeout); struct timespec ts; _PyTime_AsTimespec_clamp(abs_time, &ts); diff --git a/Python/tier2_engine.md b/Python/tier2_engine.md new file mode 100644 index 00000000000000..df9f6c124509bd --- /dev/null +++ b/Python/tier2_engine.md @@ -0,0 +1,150 @@ +# The tier 2 execution engine + +## General idea + +When execution in tier 1 becomes "hot", that is the counter for that point in +the code reaches some threshold, we create an executor and execute that +instead of the tier 1 bytecode. + +Since each executor must exit, we also track the "hotness" of those +exits and attach new executors to those exits. + +As the program executes, and the hot parts of the program get optimized, +a graph of executors forms. + +## Superblocks and Executors + +Once a point in the code has become hot enough, we want to optimize it. +Starting from that point we project the likely path of execution, +using information gathered by tier 1 to guide that projection to +form a "superblock", a mostly linear sequence of micro-ops. +Although mostly linear, it may include a single loop. + +We then optimize this superblock to form an optimized superblock, +which is equivalent but more efficient. + +A superblock is a representation of the code we want to execute, +but it is not in executable form. +The executable form is known as an executor. + +Executors are semantically equivalent to the superblock they are +created from, but are in a form that can be efficiently executable. + +There are two execution engines for executors, and two types of executors: +* The hardware which runs machine code executors created by the JIT compiler. +* The tier 2 interpreter runs bytecode executors. + +It would be very wasteful to support both a tier 2 interpreter and +JIT compiler in the same process. +For now, we will make the choice of engine a configuration option, +but we could make it a command line option in the future if that would prove useful. + + +### Tier 2 Interpreter + +For platforms without a JIT and for testing, we need an interpreter +for executors. It is similar in design to the tier 1 interpreter, but has a +different instruction set, and does not adapt. + +### JIT compiler + +The JIT compiler converts superblocks into machine code executors. +These have identical behavior to interpreted executors, except that +they consume more memory for the generated machine code and are a lot faster. + +## Transfering control + +There are three types of control transfer that we need to consider: +* Tier 1 to tier 2 +* Tier 2 to tier 1 +* One executor to another within tier 2 + +Since we expect the graph of executors to span most of the hot +part of the program, transfers from one executor to another should +be the most common. +Therefore, we want to make those transfers fast. + +### Tier 2 to tier 2 + +#### Cold exits + +All side exits start cold and most stay cold, but a few become +hot. We want to keep the memory consumption small for the many +cold exits, but those that become hot need to be fast. +However we cannot know in advance, which will be which. + +So that tier 2 to tier 2 transfers are fast for hot exits, +exits must be implemented as executors. In order to patch +executor exits when they get hot, a pointer to the current +executor must be passed to the exit executor. + +#### Handling reference counts + +There must be an implicit reference to the currently executing +executor, otherwise it might be freed. +Consequently, we must increment the reference count of an +executor just before executing it, and decrement it just after +executing it. + +We want to minimize the amount of data that is passed from +one executor to the next. In the JIT, this reduces the number +of arguments in the tailcall, freeing up registers for other uses. +It is less important in the interpreter, but following the same +design as the JIT simplifies debugging and is good for performance. + +Provided that we incref the new executor before executing it, we +can jump directly to the code of the executor, without needing +to pass a reference to that executor object. +However, we do need a reference to the previous executor, +so that it can be decref'd and for handling of cold exits. +To avoid messing up the JIT's register allocation, we pass a +reference to the previous executor in the thread state's +`previous_executor` field. + +#### The interpreter + +The tier 2 interpreter has a variable `current_executor` which +points to the currently live executor. When transfering from executor +`A` to executor `B` we do the following: +(Initially `current_executor` points to `A`, and the refcount of +`A` is elevated by one) + +1. Set the instruction pointer to start at the beginning of `B` +2. Increment the reference count of `B` +3. Start executing `B` + +We also make the first instruction in `B` do the following: +1. Set `current_executor` to point to `B` +2. Decrement the reference count of `A` (`A` is referenced by `tstate->previous_executor`) + +The net effect of the above is to safely decrement the refcount of `A`, +increment the refcount of `B` and set `current_executor` to point to `B`. + +#### In the JIT + +Transfering control from one executor to another is done via tailcalls. + +The compiled executor should do the same, except that there is no local +variable `current_executor`. + +### Tier 1 to tier 2 + +Since the executor doesn't know if the previous code was tier 1 or tier 2, +we need to make a transfer from tier 1 to tier 2 look like a tier 2 to tier 2 +transfer to the executor. + +We can then perform a tier 1 to tier 2 transfer by setting `current_executor` +to `None`, and then performing a tier 2 to tier 2 transfer as above. + +### Tier 2 to tier 1 + +Each micro-op that might exit to tier 1 contains a `target` value, +which is the offset of the tier 1 instruction to exit to in the +current code object. + +## Counters + +TO DO. +The implementation will change soon, so there is no point in +documenting it until then. + diff --git a/Python/tier2_redundancy_eliminator_bytecodes.c b/Python/tier2_redundancy_eliminator_bytecodes.c deleted file mode 100644 index 39ea0eef627632..00000000000000 --- a/Python/tier2_redundancy_eliminator_bytecodes.c +++ /dev/null @@ -1,322 +0,0 @@ -#include "Python.h" -#include "pycore_uops.h" -#include "pycore_uop_ids.h" - -#define op(name, ...) /* NAME is ignored */ - -typedef struct _Py_UOpsSymType _Py_UOpsSymType; -typedef struct _Py_UOpsAbstractInterpContext _Py_UOpsAbstractInterpContext; -typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; - -static int -dummy_func(void) { - - PyCodeObject *code; - int oparg; - _Py_UOpsSymType *flag; - _Py_UOpsSymType *left; - _Py_UOpsSymType *right; - _Py_UOpsSymType *value; - _Py_UOpsSymType *res; - _Py_UOpsSymType *iter; - _Py_UOpsSymType *top; - _Py_UOpsSymType *bottom; - _Py_UOpsAbstractFrame *frame; - _Py_UOpsAbstractInterpContext *ctx; - _PyUOpInstruction *this_instr; - _PyBloomFilter *dependencies; - int modified; - -// BEGIN BYTECODES // - - op(_LOAD_FAST_CHECK, (-- value)) { - value = GETLOCAL(oparg); - // We guarantee this will error - just bail and don't optimize it. - if (sym_is_null(value)) { - goto out_of_space; - } - } - - op(_LOAD_FAST, (-- value)) { - value = GETLOCAL(oparg); - } - - op(_LOAD_FAST_AND_CLEAR, (-- value)) { - value = GETLOCAL(oparg); - _Py_UOpsSymType *temp = sym_new_null(ctx); - if (temp == NULL) { - goto out_of_space; - } - GETLOCAL(oparg) = temp; - } - - op(_STORE_FAST, (value --)) { - GETLOCAL(oparg) = value; - } - - op(_PUSH_NULL, (-- res)) { - res = sym_new_null(ctx); - if (res == NULL) { - goto out_of_space; - }; - } - - op(_GUARD_BOTH_INT, (left, right -- left, right)) { - if (sym_matches_type(left, &PyLong_Type) && - sym_matches_type(right, &PyLong_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); - } - sym_set_type(left, &PyLong_Type); - sym_set_type(right, &PyLong_Type); - } - - op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { - if (sym_matches_type(left, &PyFloat_Type) && - sym_matches_type(right, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0 ,0); - } - sym_set_type(left, &PyFloat_Type); - sym_set_type(right, &PyFloat_Type); - } - - - op(_BINARY_OP_ADD_INT, (left, right -- res)) { - if (is_const(left) && is_const(right)) { - assert(PyLong_CheckExact(get_const(left))); - assert(PyLong_CheckExact(get_const(right))); - PyObject *temp = _PyLong_Add((PyLongObject *)get_const(left), - (PyLongObject *)get_const(right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - // TODO replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_known_type(ctx, &PyLong_Type); - if (res == NULL) { - goto out_of_space; - } - } - } - - op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { - if (is_const(left) && is_const(right)) { - assert(PyLong_CheckExact(get_const(left))); - assert(PyLong_CheckExact(get_const(right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)get_const(left), - (PyLongObject *)get_const(right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - // TODO replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_known_type(ctx, &PyLong_Type); - if (res == NULL) { - goto out_of_space; - } - } - } - - op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { - if (is_const(left) && is_const(right)) { - assert(PyLong_CheckExact(get_const(left))); - assert(PyLong_CheckExact(get_const(right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)get_const(left), - (PyLongObject *)get_const(right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - // TODO replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_known_type(ctx, &PyLong_Type); - if (res == NULL) { - goto out_of_space; - } - } - } - - op(_LOAD_CONST, (-- value)) { - // There should be no LOAD_CONST. It should be all - // replaced by peephole_opt. - Py_UNREACHABLE(); - } - - op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); - if (value == NULL) { - goto out_of_space; - } - } - - op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); - if (value == NULL) { - goto out_of_space; - } - } - - op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { - value = sym_new_const(ctx, ptr); - if (value == NULL) { - goto out_of_space; - } - null = sym_new_null(ctx); - if (null == NULL) { - goto out_of_space; - } - } - - op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { - value = sym_new_const(ctx, ptr); - if (value == NULL) { - goto out_of_space; - } - null = sym_new_null(ctx); - if (null == NULL) { - goto out_of_space; - } - } - - - op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { - assert(oparg > 0); - top = bottom; - } - - op(_SWAP, (bottom, unused[oparg-2], top -- - top, unused[oparg-2], bottom)) { - } - - op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { - _LOAD_ATTR_NOT_NULL - (void)index; - (void)owner; - } - - op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { - _LOAD_ATTR_NOT_NULL - (void)index; - (void)owner; - } - - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { - _LOAD_ATTR_NOT_NULL - (void)hint; - (void)owner; - } - - op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { - _LOAD_ATTR_NOT_NULL - (void)index; - (void)owner; - } - - op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { - _LOAD_ATTR_NOT_NULL - (void)descr; - (void)owner; - } - - op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - sym_set_type(callable, &PyFunction_Type); - (void)self_or_null; - (void)func_version; - } - - op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - sym_set_null(null); - sym_set_type(callable, &PyMethod_Type); - } - - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { - int argcount = oparg; - - (void)callable; - - PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; - if (func == NULL) { - goto error; - } - PyCodeObject *co = (PyCodeObject *)func->func_code; - - assert(self_or_null != NULL); - assert(args != NULL); - if (sym_is_not_null(self_or_null)) { - // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM - args--; - argcount++; - } - - _Py_UOpsSymType **localsplus_start = ctx->n_consumed; - int n_locals_already_filled = 0; - // Can determine statically, so we interleave the new locals - // and make the current stack the new locals. - // This also sets up for true call inlining. - if (sym_is_known(self_or_null)) { - localsplus_start = args; - n_locals_already_filled = argcount; - } - new_frame = ctx_frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0); - if (new_frame == NULL){ - goto out_of_space; - } - } - - op(_POP_FRAME, (retval -- res)) { - SYNC_SP(); - ctx->frame->stack_pointer = stack_pointer; - ctx_frame_pop(ctx); - stack_pointer = ctx->frame->stack_pointer; - res = retval; - } - - op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) { - SYNC_SP(); - ctx->frame->stack_pointer = stack_pointer; - ctx->frame = new_frame; - ctx->curr_frame_depth++; - stack_pointer = new_frame->stack_pointer; - } - - op(_UNPACK_SEQUENCE, (seq -- values[oparg])) { - /* This has to be done manually */ - (void)seq; - for (int i = 0; i < oparg; i++) { - values[i] = sym_new_unknown(ctx); - if (values[i] == NULL) { - goto out_of_space; - } - } - } - - op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8])) { - /* This has to be done manually */ - (void)seq; - int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; - for (int i = 0; i < totalargs; i++) { - values[i] = sym_new_unknown(ctx); - if (values[i] == NULL) { - goto out_of_space; - } - } - } - - op(_ITER_NEXT_RANGE, (iter -- iter, next)) { - next = sym_new_known_type(ctx, &PyLong_Type); - if (next == NULL) { - goto out_of_space; - } - (void)iter; - } - - - - -// END BYTECODES // - -} \ No newline at end of file diff --git a/Tools/README b/Tools/README index 9c4b6d86e990ba..09bd6fb4798950 100644 --- a/Tools/README +++ b/Tools/README @@ -10,8 +10,6 @@ c-analyzer Tools to check no new global variables have been added. cases_generator Tooling to generate interpreters. -ccbench A Python threads-based concurrency benchmark. (*) - clinic A preprocessor for CPython C files in order to automate the boilerplate involved with writing argument parsing code for "builtins". @@ -28,8 +26,6 @@ i18n Tools for internationalization. pygettext.py importbench A set of micro-benchmarks for various import scenarios. -iobench Benchmark for the new Python I/O system. (*) - msi Support for packaging Python as an MSI package on Windows. nuget Files for the NuGet package manager for .NET. @@ -45,9 +41,6 @@ scripts A number of useful single-file programs, e.g. run_tests.py ssl Scripts to generate ssl_data.h from OpenSSL sources, and run tests against multiple installations of OpenSSL and LibreSSL. -stringbench A suite of micro-benchmarks for various operations on - strings (both 8-bit and unicode). (*) - tz A script to dump timezone from /usr/share/zoneinfo. unicode Tools for generating unicodedata and codecs from unicode.org @@ -60,6 +53,4 @@ unittestgui A Tkinter based GUI test runner for unittest, with test wasm Config and helpers to facilitate cross compilation of CPython to WebAssembly (WASM). -(*) A generic benchmark suite is maintained separately at https://github.com/python/performance - Note: The pynche color editor has moved to https://gitlab.com/warsaw/pynche diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 201c81c4d14d79..6aa4946ee227e7 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -7,9 +7,8 @@ import pathlib import subprocess import sys +import urllib.request import typing -import zipfile -from urllib.request import urlopen CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent @@ -125,30 +124,41 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]: return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")]) -def main() -> None: - sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json" - sbom_data = json.loads(sbom_path.read_bytes()) +def get_externals() -> list[str]: + """ + Parses 'PCbuild/get_externals.bat' for external libraries. + Returns a list of (git tag, name, version) tuples. + """ + get_externals_bat_path = CPYTHON_ROOT_DIR / "PCbuild/get_externals.bat" + externals = re.findall( + r"set\s+libraries\s*=\s*%libraries%\s+([a-zA-Z0-9.-]+)\s", + get_externals_bat_path.read_text() + ) + return externals - # We regenerate all of this information. Package information - # should be preserved though since that is edited by humans. - sbom_data["files"] = [] - sbom_data["relationships"] = [] - # Ensure all packages in this tool are represented also in the SBOM file. - actual_names = {package["name"] for package in sbom_data["packages"]} - expected_names = set(PACKAGE_TO_FILES) - error_if( - actual_names != expected_names, - f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}", - ) +def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None: + """Make a bunch of assertions about the SBOM package data to ensure it's consistent.""" - # Make a bunch of assertions about the SBOM data to ensure it's consistent. for package in sbom_data["packages"]: # Properties and ID must be properly formed. error_if( "name" not in package, "Package is missing the 'name' field" ) + + # Verify that the checksum matches the expected value + # and that the download URL is valid. + if "checksums" not in package or "CI" in os.environ: + download_location = package["downloadLocation"] + resp = urllib.request.urlopen(download_location) + error_if(resp.status != 200, f"Couldn't access URL: {download_location}'") + + package["checksums"] = [{ + "algorithm": "SHA256", + "checksumValue": hashlib.sha256(resp.read()).hexdigest() + }] + missing_required_keys = REQUIRED_PROPERTIES_PACKAGE - set(package.keys()) error_if( bool(missing_required_keys), @@ -180,6 +190,26 @@ def main() -> None: f"License identifier must be 'NOASSERTION'" ) + +def create_source_sbom() -> None: + sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json" + sbom_data = json.loads(sbom_path.read_bytes()) + + # We regenerate all of this information. Package information + # should be preserved though since that is edited by humans. + sbom_data["files"] = [] + sbom_data["relationships"] = [] + + # Ensure all packages in this tool are represented also in the SBOM file. + actual_names = {package["name"] for package in sbom_data["packages"]} + expected_names = set(PACKAGE_TO_FILES) + error_if( + actual_names != expected_names, + f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}", + ) + + check_sbom_packages(sbom_data) + # We call 'sorted()' here a lot to avoid filesystem scan order issues. for name, files in sorted(PACKAGE_TO_FILES.items()): package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}") @@ -224,5 +254,49 @@ def main() -> None: sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True)) +def create_externals_sbom() -> None: + sbom_path = CPYTHON_ROOT_DIR / "Misc/externals.spdx.json" + sbom_data = json.loads(sbom_path.read_bytes()) + + externals = get_externals() + externals_name_to_version = {} + externals_name_to_git_tag = {} + for git_tag in externals: + name, _, version = git_tag.rpartition("-") + externals_name_to_version[name] = version + externals_name_to_git_tag[name] = git_tag + + # Ensure all packages in this tool are represented also in the SBOM file. + actual_names = {package["name"] for package in sbom_data["packages"]} + expected_names = set(externals_name_to_version) + error_if( + actual_names != expected_names, + f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}", + ) + + # Set the versionInfo and downloadLocation fields for all packages. + for package in sbom_data["packages"]: + package["versionInfo"] = externals_name_to_version[package["name"]] + download_location = ( + f"https://github.com/python/cpython-source-deps/archive/refs/tags/{externals_name_to_git_tag[package['name']]}.tar.gz" + ) + download_location_changed = download_location != package["downloadLocation"] + package["downloadLocation"] = download_location + + # If the download URL has changed we want one to get recalulated. + if download_location_changed: + package.pop("checksums", None) + + check_sbom_packages(sbom_data) + + # Update the SBOM on disk + sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True)) + + +def main() -> None: + create_source_sbom() + create_externals_sbom() + + if __name__ == "__main__": main() diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 5dce4e042d1eb4..588dfda50658be 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -34,6 +34,7 @@ '_testinternalcapi', '_testmultiphase', '_testsinglephase', + '_testexternalinspection', '_xxsubinterpreters', '_xxinterpchannels', '_xxinterpqueues', diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 781f9a4c8206c8..0c47470a0ecb7a 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -7,17 +7,10 @@ set here=%~dp0 set rt_opts=-q -d set regrtest_args= set arm32_ssh= +set cmdline_args=%* +set cmdline_args=%cmdline_args:,=#COMMA#% -:CheckOpts -if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-arm64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-arm32" (set rt_opts=%rt_opts% %1) & (set arm32_ssh=true) & shift & goto CheckOpts -if "%1"=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts -if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts -if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts -if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts +call:CheckOpts %cmdline_args% if "%PROCESSOR_ARCHITECTURE%"=="ARM" if "%arm32_ssh%"=="true" goto NativeExecution if "%arm32_ssh%"=="true" goto :Arm32Ssh @@ -49,3 +42,16 @@ echo The test worker should have the SSH agent running. echo Also a key must be created with ssh-keygen and added to both the buildbot worker machine echo and the ARM32 worker device: see https://docs.microsoft.com/en-us/windows/iot-core/connect-your-device/ssh exit /b 127 + +:CheckOpts +set arg="%~1" +if %arg%=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-arm64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-arm32" (set rt_opts=%rt_opts% %1) & (set arm32_ssh=true) & shift & goto CheckOpts +if %arg%=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts +if %arg%=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts +if %arg%=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts +if NOT %arg%=="" (set regrtest_args=%regrtest_args% %arg:#COMMA#=,%) & shift & goto CheckOpts +goto:eof diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 61cd41ea8f31c1..12010f0e9c0549 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -83,11 +83,11 @@ def clean_lines(text): Python/frozen_modules/*.h Python/generated_cases.c.h Python/executor_cases.c.h -Python/tier2_redundancy_eliminator_cases.c.h +Python/optimizer_cases.c.h # not actually source Python/bytecodes.c -Python/tier2_redundancy_eliminator_bytecodes.c +Python/optimizer_bytecodes.c # mimalloc Objects/mimalloc/*.c @@ -321,6 +321,7 @@ def clean_lines(text): _abs('Objects/stringlib/unicode_format.h'): (10_000, 400), _abs('Objects/typeobject.c'): (35_000, 200), _abs('Python/compile.c'): (20_000, 500), + _abs('Python/optimizer.c'): (100_000, 5_000), _abs('Python/parking_lot.c'): (40_000, 1000), _abs('Python/pylifecycle.c'): (500_000, 5000), _abs('Python/pystate.c'): (500_000, 5000), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 45119664af4362..686a3d3160cc90 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -482,3 +482,4 @@ Modules/readline.c - sigwinch_ohandler - Modules/readline.c - completed_input_string - Modules/rotatingtree.c - random_stream - Modules/rotatingtree.c - random_value - +Modules/rotatingtree.c - random_mutex - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 14bcd85b9eae59..0f212ecb528ac4 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -361,6 +361,7 @@ Python/import.c - _PyImport_Inittab - Python/import.c - _PySys_ImplCacheTag - Python/intrinsics.c - _PyIntrinsics_UnaryFunctions - Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - +Python/lock.c - TIME_TO_BE_FAIR_NS - Python/opcode_targets.h - opcode_targets - Python/perf_trampoline.c - _Py_perfmap_callbacks - Python/pyhash.c - PyHash_Func - @@ -382,6 +383,11 @@ Python/optimizer.c - _PyCounterOptimizer_Type - Python/optimizer.c - _PyUOpExecutor_Type - Python/optimizer.c - _PyUOpOptimizer_Type - Python/optimizer.c - _PyOptimizer_Default - +Python/optimizer.c - _ColdExit_Type - +Python/optimizer.c - COLD_EXITS - +Python/optimizer.c - Py_FatalErrorExecutor - +Python/optimizer.c - EMPTY_FILTER - +Python/optimizer.c - cold_exits_initialized - ##----------------------- ## test code @@ -734,6 +740,4 @@ Modules/expat/xmlrole.c - error - ## other Modules/_io/_iomodule.c - _PyIO_Module - Modules/_sqlite/module.c - _sqlite3module - -Python/optimizer_analysis.c - _Py_UOpsAbstractFrame_Type - -Python/optimizer_analysis.c - _Py_UOpsAbstractInterpContext_Type - Modules/clinic/md5module.c.h _md5_md5 _keywords - diff --git a/Tools/cases_generator/README.md b/Tools/cases_generator/README.md index d35a868b42ea9e..fb512c4646b851 100644 --- a/Tools/cases_generator/README.md +++ b/Tools/cases_generator/README.md @@ -13,9 +13,9 @@ What's currently here: - `parser.py` helper for interactions with `parsing.py` - `tierN_generator.py`: a couple of driver scripts to read `Python/bytecodes.c` and write `Python/generated_cases.c.h` (and several other files) -- `tier2_abstract_generator.py`: reads `Python/bytecodes.c` and - `Python/tier2_redundancy_eliminator_bytecodes.c` and writes - `Python/tier2_redundancy_eliminator_cases.c.h` +- `optimizer_generator.py`: reads `Python/bytecodes.c` and + `Python/optimizer_bytecodes.c` and writes + `Python/optimizer_cases.c.h` - `stack.py`: code to handle generalized stack effects - `cwriter.py`: code which understands tokens and how to format C code; main class: `CWriter` diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 3497b7fcdf35d3..b0a15e6d87c2c6 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field import lexer import parser +import re from typing import Optional @@ -16,14 +17,16 @@ class Properties: needs_this: bool always_exits: bool stores_sp: bool - tier_one_only: bool uses_co_consts: bool uses_co_names: bool uses_locals: bool has_free: bool - + side_exit: bool pure: bool passthrough: bool + tier: int | None = None + oparg_and_1: bool = False + const_oparg: int = -1 def dump(self, indent: str) -> None: print(indent, end="") @@ -43,11 +46,11 @@ def from_list(properties: list["Properties"]) -> "Properties": needs_this=any(p.needs_this for p in properties), always_exits=any(p.always_exits for p in properties), stores_sp=any(p.stores_sp for p in properties), - tier_one_only=any(p.tier_one_only for p in properties), uses_co_consts=any(p.uses_co_consts for p in properties), uses_co_names=any(p.uses_co_names for p in properties), uses_locals=any(p.uses_locals for p in properties), has_free=any(p.has_free for p in properties), + side_exit=any(p.side_exit for p in properties), pure=all(p.pure for p in properties), passthrough=all(p.passthrough for p in properties), ) @@ -64,11 +67,11 @@ def from_list(properties: list["Properties"]) -> "Properties": needs_this=False, always_exits=False, stores_sp=False, - tier_one_only=False, uses_co_consts=False, uses_co_names=False, uses_locals=False, has_free=False, + side_exit=False, pure=False, passthrough=False, ) @@ -138,6 +141,8 @@ class Uop: properties: Properties _size: int = -1 implicitly_created: bool = False + replicated = 0 + replicates : "Uop | None" = None def dump(self, indent: str) -> None: print( @@ -268,17 +273,19 @@ def override_error( ) -def convert_stack_item(item: parser.StackEffect) -> StackItem: +def convert_stack_item(item: parser.StackEffect, replace_op_arg_1: str | None) -> StackItem: + cond = item.cond + if replace_op_arg_1 and OPARG_AND_1.match(item.cond): + cond = replace_op_arg_1 return StackItem( - item.name, item.type, item.cond, (item.size or "1") + item.name, item.type, cond, (item.size or "1") ) - -def analyze_stack(op: parser.InstDef) -> StackEffect: +def analyze_stack(op: parser.InstDef, replace_op_arg_1: str | None = None) -> StackEffect: inputs: list[StackItem] = [ - convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect) + convert_stack_item(i, replace_op_arg_1) for i in op.inputs if isinstance(i, parser.StackEffect) ] - outputs: list[StackItem] = [convert_stack_item(i) for i in op.outputs] + outputs: list[StackItem] = [convert_stack_item(i, replace_op_arg_1) for i in op.outputs] for input, output in zip(inputs, outputs): if input.name == output.name: input.peek = output.peek = True @@ -303,6 +310,15 @@ def variable_used(node: parser.InstDef, name: str) -> bool: token.kind == "IDENTIFIER" and token.text == name for token in node.tokens ) +def tier_variable(node: parser.InstDef) -> int | None: + """Determine whether a tier variable is used in a node.""" + for token in node.tokens: + if token.kind == "ANNOTATION": + if token.text == "specializing": + return 1 + if re.fullmatch(r"tier\d", token.text): + return int(token.text[-1]) + return None def is_infallible(op: parser.InstDef) -> bool: return not ( @@ -441,6 +457,22 @@ def stack_effect_only_peeks(instr: parser.InstDef) -> bool: for s, other in zip(stack_inputs, instr.outputs) ) +OPARG_AND_1 = re.compile("\\(*oparg *& *1") + +def effect_depends_on_oparg_1(op: parser.InstDef) -> bool: + for effect in op.inputs: + if isinstance(effect, parser.CacheEffect): + continue + if not effect.cond: + continue + if OPARG_AND_1.match(effect.cond): + return True + for effect in op.outputs: + if not effect.cond: + continue + if OPARG_AND_1.match(effect.cond): + return True + return False def compute_properties(op: parser.InstDef) -> Properties: has_free = ( @@ -448,13 +480,24 @@ def compute_properties(op: parser.InstDef) -> Properties: or variable_used(op, "PyCell_GET") or variable_used(op, "PyCell_SET") ) + deopts_if = variable_used(op, "DEOPT_IF") + exits_if = variable_used(op, "EXIT_IF") + if deopts_if and exits_if: + tkn = op.tokens[0] + raise lexer.make_syntax_error( + "Op cannot contain both EXIT_IF and DEOPT_IF", + tkn.filename, + tkn.line, + tkn.column, + op.name, + ) infallible = is_infallible(op) - deopts = variable_used(op, "DEOPT_IF") passthrough = stack_effect_only_peeks(op) and infallible return Properties( escapes=makes_escaping_api_call(op), infallible=infallible, - deopts=deopts, + deopts=deopts_if or exits_if, + side_exit=exits_if, oparg=variable_used(op, "oparg"), jumps=variable_used(op, "JUMPBY"), eval_breaker=variable_used(op, "CHECK_EVAL_BREAKER"), @@ -462,7 +505,6 @@ def compute_properties(op: parser.InstDef) -> Properties: needs_this=variable_used(op, "this_instr"), always_exits=always_exits(op), stores_sp=variable_used(op, "SYNC_SP"), - tier_one_only=variable_used(op, "TIER_ONE_ONLY"), uses_co_consts=variable_used(op, "FRAME_CO_CONSTS"), uses_co_names=variable_used(op, "FRAME_CO_NAMES"), uses_locals=(variable_used(op, "GETLOCAL") or variable_used(op, "SETLOCAL")) @@ -470,11 +512,12 @@ def compute_properties(op: parser.InstDef) -> Properties: has_free=has_free, pure="pure" in op.annotations, passthrough=passthrough, + tier=tier_variable(op), ) -def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect]) -> Uop: - return Uop( +def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect], uops: dict[str, Uop]) -> Uop: + result = Uop( name=name, context=op.context, annotations=op.annotations, @@ -483,6 +526,49 @@ def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect]) -> body=op.block.tokens, properties=compute_properties(op), ) + if effect_depends_on_oparg_1(op) and "split" in op.annotations: + result.properties.oparg_and_1 = True + for bit in ("0", "1"): + name_x = name + "_" + bit + properties = compute_properties(op) + if properties.oparg: + # May not need oparg anymore + properties.oparg = any(token.text == "oparg" for token in op.block.tokens) + rep = Uop( + name=name_x, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op, bit), + caches=analyze_caches(inputs), + body=op.block.tokens, + properties=properties, + ) + rep.replicates = result + uops[name_x] = rep + for anno in op.annotations: + if anno.startswith("replicate"): + result.replicated = int(anno[10:-1]) + break + else: + return result + for oparg in range(result.replicated): + name_x = name + "_" + str(oparg) + properties = compute_properties(op) + properties.oparg = False + properties.const_oparg = oparg + rep = Uop( + name=name_x, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op), + caches=analyze_caches(inputs), + body=op.block.tokens, + properties=properties, + ) + rep.replicates = result + uops[name_x] = rep + + return result def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: @@ -492,7 +578,7 @@ def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: raise override_error( op.name, op.context, uops[op.name].context, op.tokens[0] ) - uops[op.name] = make_uop(op.name, op, op.inputs) + uops[op.name] = make_uop(op.name, op, op.inputs, uops) def add_instruction( @@ -519,7 +605,7 @@ def desugar_inst( uop_index = len(parts) # Place holder for the uop. parts.append(Skip(0)) - uop = make_uop("_" + inst.name, inst, op_inputs) + uop = make_uop("_" + inst.name, inst, op_inputs, uops) uop.implicitly_created = True uops[inst.name] = uop if uop_index < 0: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 2fc2ab115321cf..0b4b99c60768b5 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -119,7 +119,10 @@ def replace_decrefs( out.emit(f"Py_DECREF({var.name}[_i]);\n") out.emit("}\n") elif var.condition: - out.emit(f"Py_XDECREF({var.name});\n") + if var.condition == "1": + out.emit(f"Py_DECREF({var.name});\n") + elif var.condition != "0": + out.emit(f"Py_XDECREF({var.name});\n") else: out.emit(f"Py_DECREF({var.name});\n") @@ -154,6 +157,7 @@ def replace_check_eval_breaker( REPLACEMENT_FUNCTIONS = { + "EXIT_IF": replace_deopt, "DEOPT_IF": replace_deopt, "ERROR_IF": replace_error, "DECREF_INPUTS": replace_decrefs, @@ -205,6 +209,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_EVAL_BREAK_FLAG") if p.deopts: flags.append("HAS_DEOPT_FLAG") + if p.side_exit: + flags.append("HAS_EXIT_FLAG") if not p.infallible: flags.append("HAS_ERROR_FLAG") if p.escapes: @@ -213,6 +219,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.passthrough: flags.append("HAS_PASSTHROUGH_FLAG") + if p.oparg_and_1: + flags.append("HAS_OPARG_AND_1_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index 9b5733562f77b4..889f58fc3e1a75 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -168,6 +168,7 @@ list of annotations and their meanings are as follows: * `override`. For external use by other interpreter definitions to override the current instruction definition. * `pure`. This instruction has no side effects. +* 'tierN'. This instruction only used by tier N interpreter. ### Special functions/macros diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 4f8d01c5492f51..13aee94f2b957c 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -222,6 +222,10 @@ def choice(*opts: str) -> str: "register", "replaced", "pure", + "split", + "replicate", + "tier1", + "tier2", } __all__ = [] diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 3e9fa3e26daa53..ab597834a8892f 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -50,8 +50,10 @@ "DEOPT", "ERROR", "ESCAPES", + "EXIT", "PURE", "PASSTHROUGH", + "OPARG_AND_1", ] @@ -282,7 +284,7 @@ def is_viable_expansion(inst: Instruction) -> bool: continue if "replaced" in part.annotations: continue - if part.properties.tier_one_only or not part.is_viable(): + if part.properties.tier == 1 or not part.is_viable(): return False return True diff --git a/Tools/cases_generator/tier2_abstract_generator.py b/Tools/cases_generator/optimizer_generator.py similarity index 90% rename from Tools/cases_generator/tier2_abstract_generator.py rename to Tools/cases_generator/optimizer_generator.py index cc29b1660d26ed..fca42b51fbd689 100644 --- a/Tools/cases_generator/tier2_abstract_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -1,19 +1,15 @@ -"""Generate the cases for the tier 2 redundancy eliminator/abstract interpreter. -Reads the instruction definitions from bytecodes.c. and tier2_redundancy_eliminator.bytecodes.c -Writes the cases to tier2_redundancy_eliminator_cases.c.h, which is #included in Python/optimizer_analysis.c. +"""Generate the cases for the tier 2 optimizer. +Reads the instruction definitions from bytecodes.c and optimizer_bytecodes.c +Writes the cases to optimizer_cases.c.h, which is #included in Python/optimizer_analysis.c. """ import argparse -import os.path -import sys from analyzer import ( Analysis, Instruction, Uop, - Part, analyze_files, - Skip, StackItem, analysis_error, ) @@ -28,10 +24,10 @@ from cwriter import CWriter from typing import TextIO, Iterator from lexer import Token -from stack import StackOffset, Stack, SizeMismatch, UNUSED +from stack import Stack, SizeMismatch, UNUSED -DEFAULT_OUTPUT = ROOT / "Python/tier2_redundancy_eliminator_cases.c.h" -DEFAULT_ABSTRACT_INPUT = ROOT / "Python/tier2_redundancy_eliminator_bytecodes.c" +DEFAULT_OUTPUT = ROOT / "Python/optimizer_cases.c.h" +DEFAULT_ABSTRACT_INPUT = ROOT / "Python/optimizer_bytecodes.c" def validate_uop(override: Uop, uop: Uop) -> None: @@ -41,10 +37,10 @@ def validate_uop(override: Uop, uop: Uop) -> None: def type_name(var: StackItem) -> str: if var.is_array(): - return f"_Py_UOpsSymType **" + return f"_Py_UopsSymbol **" if var.type: return var.type - return f"_Py_UOpsSymType *" + return f"_Py_UopsSymbol *" def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: @@ -148,7 +144,7 @@ def write_uop( if not var.peek or is_override: out.emit(stack.push(var)) out.start_line() - stack.flush(out, cast_type="_Py_UOpsSymType *") + stack.flush(out, cast_type="_Py_UopsSymbol *") except SizeMismatch as ex: raise analysis_error(ex.args[0], uop.body[0]) @@ -176,7 +172,9 @@ def generate_abstract_interpreter( if uop.name in abstract.uops: override = abstract.uops[uop.name] validate_uop(override, uop) - if uop.properties.tier_one_only: + if uop.properties.tier == 1: + continue + if uop.replicates: continue if uop.is_super(): continue diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index a8961f28babea1..0d54820e4e71fb 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -179,7 +179,13 @@ def inst_header(self) -> InstHeader | None: # | annotation* op(NAME, (inputs -- outputs)) annotations = [] while anno := self.expect(lx.ANNOTATION): - annotations.append(anno.text) + if anno.text == "replicate": + self.require(lx.LPAREN) + times = self.require(lx.NUMBER) + self.require(lx.RPAREN) + annotations.append(f"replicate({times.text})") + else: + annotations.append(anno.text) tkn = self.expect(lx.INST) if not tkn: tkn = self.expect(lx.OP) diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 97a301142d59c7..5aecac39aef5e2 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -23,8 +23,12 @@ def maybe_parenthesize(sym: str) -> str: def var_size(var: StackItem) -> str: if var.condition: - # Special case simplification - if var.condition == "oparg & 1" and var.size == "1": + # Special case simplifications + if var.condition == "0": + return "0" + elif var.condition == "1": + return var.size + elif var.condition == "oparg & 1" and var.size == "1": return f"({var.condition})" else: return f"(({var.condition}) ? {var.size} : 0)" @@ -154,7 +158,12 @@ def pop(self, var: StackItem) -> str: f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}];" ) if var.condition: - return f"if ({var.condition}) {{ {assign} }}\n" + if var.condition == "1": + return f"{assign}\n" + elif var.condition == "0": + return "" + else: + return f"if ({var.condition}) {{ {assign} }}\n" return f"{assign}\n" def push(self, var: StackItem) -> str: @@ -175,7 +184,10 @@ def flush(self, out: CWriter, cast_type: str = "PyObject *") -> None: cast = f"({cast_type})" if var.type else "" if var.name not in UNUSED and not var.is_array(): if var.condition: - out.emit(f"if ({var.condition}) ") + if var.condition == "0": + continue + elif var.condition != "1": + out.emit(f"if ({var.condition}) ") out.emit( f"stack_pointer[{self.base_offset.to_c()}] = {cast}{var.name};\n" ) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index aba36ec74e5766..fb2ab931b1c108 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -87,6 +87,8 @@ def write_uop( out.emit( f"{type}{cache.name} = {reader}(&this_instr[{offset}].cache);\n" ) + if inst.family is None: + out.emit(f"(void){cache.name};\n") offset += cache.size emit_tokens(out, uop, stack, inst) if uop.properties.stores_sp: @@ -131,8 +133,10 @@ def generate_tier1( needs_this = uses_this(inst) out.emit("\n") out.emit(f"TARGET({name}) {{\n") + unused_guard = "(void)this_instr;\n" if inst.family is None else "" if needs_this and not inst.is_target: out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;\n") + out.emit(unused_guard) else: out.emit(f"frame->instr_ptr = next_instr;\n") out.emit(f"next_instr += {inst.size};\n") @@ -141,6 +145,7 @@ def generate_tier1( out.emit(f"PREDICTED({name});\n") if needs_this: out.emit(f"_Py_CODEUNIT *this_instr = next_instr - {inst.size};\n") + out.emit(unused_guard) if inst.family is not None: out.emit( f"static_assert({inst.family.size} == {inst.size-1}" diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 7897b89b2752a7..d8eed1078b0914 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -33,24 +33,29 @@ DEFAULT_OUTPUT = ROOT / "Python/executor_cases.c.h" +def declare_variable( + var: StackItem, uop: Uop, variables: set[str], out: CWriter +) -> None: + if var.name in variables: + return + type = var.type if var.type else "PyObject *" + variables.add(var.name) + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + if uop.replicates: + # Replicas may not use all their conditional variables + # So avoid a compiler warning with a fake use + out.emit(f"(void){var.name};\n") + else: + out.emit(f"{type}{var.name};\n") + + def declare_variables(uop: Uop, out: CWriter) -> None: variables = {"unused"} for var in reversed(uop.stack.inputs): - if var.name not in variables: - type = var.type if var.type else "PyObject *" - variables.add(var.name) - if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") - else: - out.emit(f"{type}{var.name};\n") + declare_variable(var, uop, variables, out) for var in uop.stack.outputs: - if var.name not in variables: - variables.add(var.name) - type = var.type if var.type else "PyObject *" - if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") - else: - out.emit(f"{type}{var.name};\n") + declare_variable(var, uop, variables, out) def tier2_replace_error( @@ -98,9 +103,47 @@ def tier2_replace_deopt( out.emit(") goto deoptimize;\n") +def tier2_replace_exit_if( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("if ", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(") goto side_exit;\n") + + +def tier2_replace_oparg( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + if not uop.name.endswith("_0") and not uop.name.endswith("_1"): + out.emit(tkn) + return + amp = next(tkn_iter) + if amp.text != "&": + out.emit(tkn) + out.emit(amp) + return + one = next(tkn_iter) + assert one.text == "1" + out.emit_at(uop.name[-1], tkn) + + TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy() TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt +TIER2_REPLACEMENT_FUNCTIONS["oparg"] = tier2_replace_oparg +TIER2_REPLACEMENT_FUNCTIONS["EXIT_IF"] = tier2_replace_exit_if def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: @@ -108,6 +151,10 @@ def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: out.start_line() if uop.properties.oparg: out.emit("oparg = CURRENT_OPARG();\n") + assert uop.properties.const_oparg < 0 + elif uop.properties.const_oparg >= 0: + out.emit(f"oparg = {uop.properties.const_oparg};\n") + out.emit(f"assert(oparg == CURRENT_OPARG());\n") for var in reversed(uop.stack.inputs): out.emit(stack.pop(var)) if not uop.properties.stores_sp: @@ -147,7 +194,10 @@ def generate_tier2( out = CWriter(outfile, 2, lines) out.emit("\n") for name, uop in analysis.uops.items(): - if uop.properties.tier_one_only: + if uop.properties.tier == 1: + continue + if uop.properties.oparg_and_1: + out.emit(f"/* {uop.name} is split on (oparg & 1) */\n\n") continue if uop.is_super(): continue diff --git a/Tools/cases_generator/uop_id_generator.py b/Tools/cases_generator/uop_id_generator.py index 633249f1c6b1fe..eb5e3f4a324735 100644 --- a/Tools/cases_generator/uop_id_generator.py +++ b/Tools/cases_generator/uop_id_generator.py @@ -38,15 +38,17 @@ def generate_uop_ids( next_id += 1 PRE_DEFINED = {"_EXIT_TRACE", "_SET_IP"} - for uop in analysis.uops.values(): - if uop.name in PRE_DEFINED: + uops = [(uop.name, uop) for uop in analysis.uops.values()] + # Sort so that _BASE comes immediately before _BASE_0, etc. + for name, uop in sorted(uops): + if name in PRE_DEFINED: continue - if uop.properties.tier_one_only: + if uop.properties.tier == 1: continue - if uop.implicitly_created and not distinct_namespace: - out.emit(f"#define {uop.name} {uop.name[1:]}\n") + if uop.implicitly_created and not distinct_namespace and not uop.replicated: + out.emit(f"#define {name} {name[1:]}\n") else: - out.emit(f"#define {uop.name} {next_id}\n") + out.emit(f"#define {name} {next_id}\n") next_id += 1 out.emit(f"#define MAX_UOP_ID {next_id-1}\n") diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 9083ecc48bdf5b..72eed3041c55c9 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -24,17 +24,24 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") + out.emit("extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1];\n") out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") for uop in analysis.uops.values(): - if uop.is_viable() and not uop.properties.tier_one_only: + if uop.is_viable() and uop.properties.tier != 1: out.emit(f"[{uop.name}] = {cflags(uop.properties)},\n") + out.emit("};\n\n") + out.emit("const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = {\n") + for uop in analysis.uops.values(): + if uop.replicated: + out.emit(f"[{uop.name}] = {uop.replicated},\n") + out.emit("};\n\n") out.emit("const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {\n") for uop in sorted(analysis.uops.values(), key=lambda t: t.name): - if uop.is_viable() and not uop.properties.tier_one_only: + if uop.is_viable() and uop.properties.tier != 1: out.emit(f'[{uop.name}] = "{uop.name}",\n') out.emit("};\n") out.emit("#endif // NEED_OPCODE_METADATA\n\n") diff --git a/Tools/ccbench/ccbench.py b/Tools/ccbench/ccbench.py deleted file mode 100644 index d52701a82948da..00000000000000 --- a/Tools/ccbench/ccbench.py +++ /dev/null @@ -1,606 +0,0 @@ -# This file should be kept compatible with both Python 2.6 and Python >= 3.0. - -from __future__ import division -from __future__ import print_function - -""" -ccbench, a Python concurrency benchmark. -""" - -import time -import os -import sys -import itertools -import threading -import subprocess -import socket -from optparse import OptionParser, SUPPRESS_HELP -import platform - -# Compatibility -try: - xrange -except NameError: - xrange = range - -try: - map = itertools.imap -except AttributeError: - pass - - -THROUGHPUT_DURATION = 2.0 - -LATENCY_PING_INTERVAL = 0.1 -LATENCY_DURATION = 2.0 - -BANDWIDTH_PACKET_SIZE = 1024 -BANDWIDTH_DURATION = 2.0 - - -def task_pidigits(): - """Pi calculation (Python)""" - _map = map - _count = itertools.count - _islice = itertools.islice - - def calc_ndigits(n): - # From http://shootout.alioth.debian.org/ - def gen_x(): - return _map(lambda k: (k, 4*k + 2, 0, 2*k + 1), _count(1)) - - def compose(a, b): - aq, ar, as_, at = a - bq, br, bs, bt = b - return (aq * bq, - aq * br + ar * bt, - as_ * bq + at * bs, - as_ * br + at * bt) - - def extract(z, j): - q, r, s, t = z - return (q*j + r) // (s*j + t) - - def pi_digits(): - z = (1, 0, 0, 1) - x = gen_x() - while 1: - y = extract(z, 3) - while y != extract(z, 4): - z = compose(z, next(x)) - y = extract(z, 3) - z = compose((10, -10*y, 0, 1), z) - yield y - - return list(_islice(pi_digits(), n)) - - return calc_ndigits, (50, ) - -def task_regex(): - """regular expression (C)""" - # XXX this task gives horrendous latency results. - import re - # Taken from the `inspect` module - pat = re.compile(r'^(\s*def\s)|(.*(? return the previous one. - if end_event: - return niters, duration - niters += step - duration = t2 - start_time - if duration >= min_duration: - end_event.append(None) - return niters, duration - if t2 - t1 < 0.01: - # Minimize interference of measurement on overall runtime - step = step * 3 // 2 - elif do_yield: - # OS scheduling of Python threads is sometimes so bad that we - # have to force thread switching ourselves, otherwise we get - # completely useless results. - _sleep(0.0001) - t1 = t2 - - -def run_throughput_test(func, args, nthreads): - assert nthreads >= 1 - - # Warm up - func(*args) - - results = [] - loop = TimedLoop(func, args) - end_event = [] - - if nthreads == 1: - # Pure single-threaded performance, without any switching or - # synchronization overhead. - start_time = time.time() - results.append(loop(start_time, THROUGHPUT_DURATION, - end_event, do_yield=False)) - return results - - started = False - ready_cond = threading.Condition() - start_cond = threading.Condition() - ready = [] - - def run(): - with ready_cond: - ready.append(None) - ready_cond.notify() - with start_cond: - while not started: - start_cond.wait() - results.append(loop(start_time, THROUGHPUT_DURATION, - end_event, do_yield=True)) - - threads = [] - for i in range(nthreads): - threads.append(threading.Thread(target=run)) - for t in threads: - t.daemon = True - t.start() - # We don't want measurements to include thread startup overhead, - # so we arrange for timing to start after all threads are ready. - with ready_cond: - while len(ready) < nthreads: - ready_cond.wait() - with start_cond: - start_time = time.time() - started = True - start_cond.notify(nthreads) - for t in threads: - t.join() - - return results - -def run_throughput_tests(max_threads): - for task in throughput_tasks: - print(task.__doc__) - print() - func, args = task() - nthreads = 1 - baseline_speed = None - while nthreads <= max_threads: - results = run_throughput_test(func, args, nthreads) - # Taking the max duration rather than average gives pessimistic - # results rather than optimistic. - speed = sum(r[0] for r in results) / max(r[1] for r in results) - print("threads=%d: %d" % (nthreads, speed), end="") - if baseline_speed is None: - print(" iterations/s.") - baseline_speed = speed - else: - print(" ( %d %%)" % (speed / baseline_speed * 100)) - nthreads += 1 - print() - - -LAT_END = "END" - -def _sendto(sock, s, addr): - sock.sendto(s.encode('ascii'), addr) - -def _recv(sock, n): - return sock.recv(n).decode('ascii') - -def latency_client(addr, nb_pings, interval): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - _time = time.time - _sleep = time.sleep - def _ping(): - _sendto(sock, "%r\n" % _time(), addr) - # The first ping signals the parent process that we are ready. - _ping() - # We give the parent a bit of time to notice. - _sleep(1.0) - for i in range(nb_pings): - _sleep(interval) - _ping() - _sendto(sock, LAT_END + "\n", addr) - finally: - sock.close() - -def run_latency_client(**kwargs): - cmd_line = [sys.executable, '-E', os.path.abspath(__file__)] - cmd_line.extend(['--latclient', repr(kwargs)]) - return subprocess.Popen(cmd_line) #, stdin=subprocess.PIPE, - #stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - -def run_latency_test(func, args, nthreads): - # Create a listening socket to receive the pings. We use UDP which should - # be painlessly cross-platform. - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind(("127.0.0.1", 0)) - addr = sock.getsockname() - - interval = LATENCY_PING_INTERVAL - duration = LATENCY_DURATION - nb_pings = int(duration / interval) - - results = [] - threads = [] - end_event = [] - start_cond = threading.Condition() - started = False - if nthreads > 0: - # Warm up - func(*args) - - results = [] - loop = TimedLoop(func, args) - ready = [] - ready_cond = threading.Condition() - - def run(): - with ready_cond: - ready.append(None) - ready_cond.notify() - with start_cond: - while not started: - start_cond.wait() - loop(start_time, duration * 1.5, end_event, do_yield=False) - - for i in range(nthreads): - threads.append(threading.Thread(target=run)) - for t in threads: - t.daemon = True - t.start() - # Wait for threads to be ready - with ready_cond: - while len(ready) < nthreads: - ready_cond.wait() - - # Run the client and wait for the first ping(s) to arrive before - # unblocking the background threads. - chunks = [] - process = run_latency_client(addr=sock.getsockname(), - nb_pings=nb_pings, interval=interval) - s = _recv(sock, 4096) - _time = time.time - - with start_cond: - start_time = _time() - started = True - start_cond.notify(nthreads) - - while LAT_END not in s: - s = _recv(sock, 4096) - t = _time() - chunks.append((t, s)) - - # Tell the background threads to stop. - end_event.append(None) - for t in threads: - t.join() - process.wait() - sock.close() - - for recv_time, chunk in chunks: - # NOTE: it is assumed that a line sent by a client wasn't received - # in two chunks because the lines are very small. - for line in chunk.splitlines(): - line = line.strip() - if line and line != LAT_END: - send_time = eval(line) - assert isinstance(send_time, float) - results.append((send_time, recv_time)) - - return results - -def run_latency_tests(max_threads): - for task in latency_tasks: - print("Background CPU task:", task.__doc__) - print() - func, args = task() - nthreads = 0 - while nthreads <= max_threads: - results = run_latency_test(func, args, nthreads) - n = len(results) - # We print out milliseconds - lats = [1000 * (t2 - t1) for (t1, t2) in results] - #print(list(map(int, lats))) - avg = sum(lats) / n - dev = (sum((x - avg) ** 2 for x in lats) / n) ** 0.5 - print("CPU threads=%d: %d ms. (std dev: %d ms.)" % (nthreads, avg, dev), end="") - print() - #print(" [... from %d samples]" % n) - nthreads += 1 - print() - - -BW_END = "END" - -def bandwidth_client(addr, packet_size, duration): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind(("127.0.0.1", 0)) - local_addr = sock.getsockname() - _time = time.time - _sleep = time.sleep - def _send_chunk(msg): - _sendto(sock, ("%r#%s\n" % (local_addr, msg)).rjust(packet_size), addr) - # We give the parent some time to be ready. - _sleep(1.0) - try: - start_time = _time() - end_time = start_time + duration * 2.0 - i = 0 - while _time() < end_time: - _send_chunk(str(i)) - s = _recv(sock, packet_size) - assert len(s) == packet_size - i += 1 - _send_chunk(BW_END) - finally: - sock.close() - -def run_bandwidth_client(**kwargs): - cmd_line = [sys.executable, '-E', os.path.abspath(__file__)] - cmd_line.extend(['--bwclient', repr(kwargs)]) - return subprocess.Popen(cmd_line) #, stdin=subprocess.PIPE, - #stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - -def run_bandwidth_test(func, args, nthreads): - # Create a listening socket to receive the packets. We use UDP which should - # be painlessly cross-platform. - with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: - sock.bind(("127.0.0.1", 0)) - addr = sock.getsockname() - - duration = BANDWIDTH_DURATION - packet_size = BANDWIDTH_PACKET_SIZE - - results = [] - threads = [] - end_event = [] - start_cond = threading.Condition() - started = False - if nthreads > 0: - # Warm up - func(*args) - - results = [] - loop = TimedLoop(func, args) - ready = [] - ready_cond = threading.Condition() - - def run(): - with ready_cond: - ready.append(None) - ready_cond.notify() - with start_cond: - while not started: - start_cond.wait() - loop(start_time, duration * 1.5, end_event, do_yield=False) - - for i in range(nthreads): - threads.append(threading.Thread(target=run)) - for t in threads: - t.daemon = True - t.start() - # Wait for threads to be ready - with ready_cond: - while len(ready) < nthreads: - ready_cond.wait() - - # Run the client and wait for the first packet to arrive before - # unblocking the background threads. - process = run_bandwidth_client(addr=addr, - packet_size=packet_size, - duration=duration) - _time = time.time - # This will also wait for the parent to be ready - s = _recv(sock, packet_size) - remote_addr = eval(s.partition('#')[0]) - - with start_cond: - start_time = _time() - started = True - start_cond.notify(nthreads) - - n = 0 - first_time = None - while not end_event and BW_END not in s: - _sendto(sock, s, remote_addr) - s = _recv(sock, packet_size) - if first_time is None: - first_time = _time() - n += 1 - end_time = _time() - - end_event.append(None) - for t in threads: - t.join() - process.kill() - - return (n - 1) / (end_time - first_time) - -def run_bandwidth_tests(max_threads): - for task in bandwidth_tasks: - print("Background CPU task:", task.__doc__) - print() - func, args = task() - nthreads = 0 - baseline_speed = None - while nthreads <= max_threads: - results = run_bandwidth_test(func, args, nthreads) - speed = results - #speed = len(results) * 1.0 / results[-1][0] - print("CPU threads=%d: %.1f" % (nthreads, speed), end="") - if baseline_speed is None: - print(" packets/s.") - baseline_speed = speed - else: - print(" ( %d %%)" % (speed / baseline_speed * 100)) - nthreads += 1 - print() - - -def main(): - usage = "usage: %prog [-h|--help] [options]" - parser = OptionParser(usage=usage) - parser.add_option("-t", "--throughput", - action="store_true", dest="throughput", default=False, - help="run throughput tests") - parser.add_option("-l", "--latency", - action="store_true", dest="latency", default=False, - help="run latency tests") - parser.add_option("-b", "--bandwidth", - action="store_true", dest="bandwidth", default=False, - help="run I/O bandwidth tests") - parser.add_option("-i", "--interval", - action="store", type="int", dest="check_interval", default=None, - help="sys.setcheckinterval() value " - "(Python 3.8 and older)") - parser.add_option("-I", "--switch-interval", - action="store", type="float", dest="switch_interval", default=None, - help="sys.setswitchinterval() value " - "(Python 3.2 and newer)") - parser.add_option("-n", "--num-threads", - action="store", type="int", dest="nthreads", default=4, - help="max number of threads in tests") - - # Hidden option to run the pinging and bandwidth clients - parser.add_option("", "--latclient", - action="store", dest="latclient", default=None, - help=SUPPRESS_HELP) - parser.add_option("", "--bwclient", - action="store", dest="bwclient", default=None, - help=SUPPRESS_HELP) - - options, args = parser.parse_args() - if args: - parser.error("unexpected arguments") - - if options.latclient: - kwargs = eval(options.latclient) - latency_client(**kwargs) - return - - if options.bwclient: - kwargs = eval(options.bwclient) - bandwidth_client(**kwargs) - return - - if not options.throughput and not options.latency and not options.bandwidth: - options.throughput = options.latency = options.bandwidth = True - if options.check_interval: - sys.setcheckinterval(options.check_interval) - if options.switch_interval: - sys.setswitchinterval(options.switch_interval) - - print("== %s %s (%s) ==" % ( - platform.python_implementation(), - platform.python_version(), - platform.python_build()[0], - )) - # Processor identification often has repeated spaces - cpu = ' '.join(platform.processor().split()) - print("== %s %s on '%s' ==" % ( - platform.machine(), - platform.system(), - cpu, - )) - print() - - if options.throughput: - print("--- Throughput ---") - print() - run_throughput_tests(options.nthreads) - - if options.latency: - print("--- Latency ---") - print() - run_latency_tests(options.nthreads) - - if options.bandwidth: - print("--- I/O bandwidth ---") - print() - run_bandwidth_tests(options.nthreads) - -if __name__ == "__main__": - main() diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 4925f27b2937b1..0a8546247cc326 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -138,31 +138,6 @@ def fail( warn_or_fail(*args, filename=filename, line_number=line_number, fail=True) -is_legal_c_identifier = re.compile('^[A-Za-z_][A-Za-z0-9_]*$').match - -def is_legal_py_identifier(s: str) -> bool: - return all(is_legal_c_identifier(field) for field in s.split('.')) - -# identifiers that are okay in Python but aren't a good idea in C. -# so if they're used Argument Clinic will add "_value" to the end -# of the name in C. -c_keywords = set(""" -asm auto break case char const continue default do double -else enum extern float for goto if inline int long -register return short signed sizeof static struct switch -typedef typeof union unsigned void volatile while -""".strip().split()) - -def ensure_legal_c_identifier(s: str) -> str: - # for now, just complain if what we're given isn't legal - if not is_legal_c_identifier(s): - fail("Illegal C identifier:", s) - # but if we picked a C keyword, pick something else - if s in c_keywords: - return s + "_value" - return s - - class CRenderData: def __init__(self) -> None: @@ -884,9 +859,6 @@ def parser_body( limited_capi = False parsearg: str | None - if f.kind in {GETTER, SETTER} and parameters: - fail(f"@{f.kind.name.lower()} method cannot define parameters") - if not parameters: parser_code: list[str] | None if f.kind is GETTER: @@ -1640,12 +1612,9 @@ def render_function( for converter in converters: converter.set_template_dict(template_dict) - f.return_converter.render(f, data) - if f.kind is SETTER: - # All setters return an int. - template_dict['impl_return_type'] = 'int' - else: - template_dict['impl_return_type'] = f.return_converter.type + if f.kind not in {SETTER, METHOD_INIT}: + f.return_converter.render(f, data) + template_dict['impl_return_type'] = f.return_converter.type template_dict['declarations'] = libclinic.format_escape("\n".join(data.declarations)) template_dict['initializers'] = "\n\n".join(data.initializers) @@ -2954,7 +2923,7 @@ def __init__(self, unused: bool = False, **kwargs: Any ) -> None: - self.name = ensure_legal_c_identifier(name) + self.name = libclinic.ensure_legal_c_identifier(name) self.py_name = py_name self.unused = unused self.includes: list[Include] = [] @@ -4094,8 +4063,6 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st # mapping from arguments to format unit *and* registers the # legacy C converter for that format unit. # -ConverterKeywordDict = dict[str, TypeSet | bool] - def r(format_unit: str, *, accept: TypeSet, @@ -4111,7 +4078,7 @@ def r(format_unit: str, # # also don't add the converter for 's' because # the metaclass for CConverter adds it for us. - kwargs: ConverterKeywordDict = {} + kwargs: dict[str, Any] = {} if accept != {str}: kwargs['accept'] = accept if zeroes: @@ -4592,20 +4559,6 @@ class int_return_converter(long_return_converter): cast = '(long)' -class init_return_converter(long_return_converter): - """ - Special return converter for __init__ functions. - """ - type = 'int' - cast = '(long)' - - def render( - self, - function: Function, - data: CRenderData - ) -> None: ... - - class unsigned_long_return_converter(long_return_converter): type = 'unsigned long' conversion_fn = 'PyLong_FromUnsignedLong' @@ -5083,9 +5036,9 @@ def parse_function_names(self, line: str) -> FunctionNames: if fields[-1] == '__new__': fields.pop() c_basename = "_".join(fields) - if not is_legal_py_identifier(full_name): + if not libclinic.is_legal_py_identifier(full_name): fail(f"Illegal function name: {full_name!r}") - if not is_legal_c_identifier(c_basename): + if not libclinic.is_legal_c_identifier(c_basename): fail(f"Illegal C basename: {c_basename!r}") names = FunctionNames(full_name=full_name, c_basename=c_basename) self.normalize_function_kind(names.full_name) @@ -5119,6 +5072,8 @@ def resolve_return_converter( if forced_converter: if self.kind in {GETTER, SETTER}: fail(f"@{self.kind.name.lower()} method cannot define a return type") + if self.kind is METHOD_INIT: + fail("__init__ methods cannot define a return type") ast_input = f"def x() -> {forced_converter}: pass" try: module_node = ast.parse(ast_input) @@ -5136,8 +5091,8 @@ def resolve_return_converter( except ValueError: fail(f"Badly formed annotation for {full_name!r}: {forced_converter!r}") - if self.kind is METHOD_INIT: - return init_return_converter() + if self.kind in {METHOD_INIT, SETTER}: + return int_return_converter() return CReturnConverter() def parse_cloned_function(self, names: FunctionNames, existing: str) -> None: @@ -5207,7 +5162,7 @@ def state_modulename_name(self, line: str) -> None: before, equals, existing = line.rpartition('=') if equals: existing = existing.strip() - if is_legal_py_identifier(existing): + if libclinic.is_legal_py_identifier(existing): # we're cloning! names = self.parse_function_names(before) return self.parse_cloned_function(names, existing) @@ -5319,6 +5274,11 @@ def state_parameters_start(self, line: str) -> None: if not self.indent.infer(line): return self.next(self.state_function_docstring, line) + assert self.function is not None + if self.function.kind in {GETTER, SETTER}: + getset = self.function.kind.name.lower() + fail(f"@{getset} methods cannot define parameters") + self.parameter_continuation = '' return self.next(self.state_parameter, line) diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py index 6237809764d9e1..738864a48c08d3 100644 --- a/Tools/clinic/libclinic/__init__.py +++ b/Tools/clinic/libclinic/__init__.py @@ -16,6 +16,11 @@ wrap_declarations, wrapped_c_string_literal, ) +from .identifiers import ( + ensure_legal_c_identifier, + is_legal_c_identifier, + is_legal_py_identifier, +) from .utils import ( FormatCounterFormatter, compute_checksum, @@ -41,6 +46,11 @@ "wrap_declarations", "wrapped_c_string_literal", + # Identifier helpers + "ensure_legal_c_identifier", + "is_legal_c_identifier", + "is_legal_py_identifier", + # Utility functions "FormatCounterFormatter", "compute_checksum", diff --git a/Tools/clinic/libclinic/identifiers.py b/Tools/clinic/libclinic/identifiers.py new file mode 100644 index 00000000000000..d3b80bbcef3b2b --- /dev/null +++ b/Tools/clinic/libclinic/identifiers.py @@ -0,0 +1,31 @@ +import re +from .errors import ClinicError + + +is_legal_c_identifier = re.compile("^[A-Za-z_][A-Za-z0-9_]*$").match + + +def is_legal_py_identifier(identifier: str) -> bool: + return all(is_legal_c_identifier(field) for field in identifier.split(".")) + + +# Identifiers that are okay in Python but aren't a good idea in C. +# So if they're used Argument Clinic will add "_value" to the end +# of the name in C. +_c_keywords = frozenset(""" +asm auto break case char const continue default do double +else enum extern float for goto if inline int long +register return short signed sizeof static struct switch +typedef typeof union unsigned void volatile while +""".strip().split() +) + + +def ensure_legal_c_identifier(identifier: str) -> str: + # For now, just complain if what we're given isn't legal. + if not is_legal_c_identifier(identifier): + raise ClinicError(f"Illegal C identifier: {identifier}") + # But if we picked a C keyword, pick something else. + if identifier in _c_keywords: + return identifier + "_value" + return identifier diff --git a/Tools/iobench/iobench.py b/Tools/iobench/iobench.py deleted file mode 100644 index 4017149ec91630..00000000000000 --- a/Tools/iobench/iobench.py +++ /dev/null @@ -1,568 +0,0 @@ -import itertools -import os -import platform -import re -import sys -import time -from optparse import OptionParser - -out = sys.stdout - -TEXT_ENCODING = 'utf8' -NEWLINES = 'lf' - - -def text_open(fn, mode, encoding=None): - try: - return open(fn, mode, encoding=encoding or TEXT_ENCODING) - except TypeError: - return open(fn, mode) - - -def get_file_sizes(): - for s in ['20 KiB', '400 KiB', '10 MiB']: - size, unit = s.split() - size = int(size) * {'KiB': 1024, 'MiB': 1024 ** 2}[unit] - yield s.replace(' ', ''), size - - -def get_binary_files(): - return ((name + ".bin", size) for name, size in get_file_sizes()) - - -def get_text_files(): - return ((f"{name}-{TEXT_ENCODING}-{NEWLINES}.txt", size) - for name, size in get_file_sizes()) - - -def with_open_mode(mode): - def decorate(f): - f.file_open_mode = mode - return f - return decorate - - -def with_sizes(*sizes): - def decorate(f): - f.file_sizes = sizes - return f - return decorate - - -# Here begin the tests - -@with_open_mode("r") -@with_sizes("medium") -def read_bytewise(f): - """ read one unit at a time """ - f.seek(0) - while f.read(1): - pass - - -@with_open_mode("r") -@with_sizes("medium") -def read_small_chunks(f): - """ read 20 units at a time """ - f.seek(0) - while f.read(20): - pass - - -@with_open_mode("r") -@with_sizes("medium") -def read_big_chunks(f): - """ read 4096 units at a time """ - f.seek(0) - while f.read(4096): - pass - - -@with_open_mode("r") -@with_sizes("small", "medium", "large") -def read_whole_file(f): - """ read whole contents at once """ - f.seek(0) - while f.read(): - pass - - -@with_open_mode("rt") -@with_sizes("medium") -def read_lines(f): - """ read one line at a time """ - f.seek(0) - for line in f: - pass - - -@with_open_mode("r") -@with_sizes("medium") -def seek_forward_bytewise(f): - """ seek forward one unit at a time """ - f.seek(0, 2) - size = f.tell() - f.seek(0, 0) - for i in range(0, size - 1): - f.seek(i, 0) - - -@with_open_mode("r") -@with_sizes("medium") -def seek_forward_blockwise(f): - """ seek forward 1000 units at a time """ - f.seek(0, 2) - size = f.tell() - f.seek(0, 0) - for i in range(0, size - 1, 1000): - f.seek(i, 0) - - -@with_open_mode("rb") -@with_sizes("medium") -def read_seek_bytewise(f): - """ alternate read & seek one unit """ - f.seek(0) - while f.read(1): - f.seek(1, 1) - - -@with_open_mode("rb") -@with_sizes("medium") -def read_seek_blockwise(f): - """ alternate read & seek 1000 units """ - f.seek(0) - while f.read(1000): - f.seek(1000, 1) - - -@with_open_mode("w") -@with_sizes("small") -def write_bytewise(f, source): - """ write one unit at a time """ - for i in range(0, len(source)): - f.write(source[i:i+1]) - - -@with_open_mode("w") -@with_sizes("medium") -def write_small_chunks(f, source): - """ write 20 units at a time """ - for i in range(0, len(source), 20): - f.write(source[i:i+20]) - - -@with_open_mode("w") -@with_sizes("medium") -def write_medium_chunks(f, source): - """ write 4096 units at a time """ - for i in range(0, len(source), 4096): - f.write(source[i:i+4096]) - - -@with_open_mode("w") -@with_sizes("large") -def write_large_chunks(f, source): - """ write 1e6 units at a time """ - for i in range(0, len(source), 1000000): - f.write(source[i:i+1000000]) - - -@with_open_mode("w+") -@with_sizes("small") -def modify_bytewise(f, source): - """ modify one unit at a time """ - f.seek(0) - for i in range(0, len(source)): - f.write(source[i:i+1]) - - -@with_open_mode("w+") -@with_sizes("medium") -def modify_small_chunks(f, source): - """ modify 20 units at a time """ - f.seek(0) - for i in range(0, len(source), 20): - f.write(source[i:i+20]) - - -@with_open_mode("w+") -@with_sizes("medium") -def modify_medium_chunks(f, source): - """ modify 4096 units at a time """ - f.seek(0) - for i in range(0, len(source), 4096): - f.write(source[i:i+4096]) - - -@with_open_mode("wb+") -@with_sizes("medium") -def modify_seek_forward_bytewise(f, source): - """ alternate write & seek one unit """ - f.seek(0) - for i in range(0, len(source), 2): - f.write(source[i:i+1]) - f.seek(i+2) - - -@with_open_mode("wb+") -@with_sizes("medium") -def modify_seek_forward_blockwise(f, source): - """ alternate write & seek 1000 units """ - f.seek(0) - for i in range(0, len(source), 2000): - f.write(source[i:i+1000]) - f.seek(i+2000) - - -# XXX the 2 following tests don't work with py3k's text IO -@with_open_mode("wb+") -@with_sizes("medium") -def read_modify_bytewise(f, source): - """ alternate read & write one unit """ - f.seek(0) - for i in range(0, len(source), 2): - f.read(1) - f.write(source[i+1:i+2]) - - -@with_open_mode("wb+") -@with_sizes("medium") -def read_modify_blockwise(f, source): - """ alternate read & write 1000 units """ - f.seek(0) - for i in range(0, len(source), 2000): - f.read(1000) - f.write(source[i+1000:i+2000]) - - -read_tests = [ - read_bytewise, read_small_chunks, read_lines, read_big_chunks, - None, read_whole_file, None, - seek_forward_bytewise, seek_forward_blockwise, - read_seek_bytewise, read_seek_blockwise, -] - -write_tests = [ - write_bytewise, write_small_chunks, write_medium_chunks, write_large_chunks, -] - -modify_tests = [ - modify_bytewise, modify_small_chunks, modify_medium_chunks, - None, - modify_seek_forward_bytewise, modify_seek_forward_blockwise, - read_modify_bytewise, read_modify_blockwise, -] - - -def run_during(duration, func): - _t = time.time - n = 0 - start = os.times() - start_timestamp = _t() - real_start = start[4] or start_timestamp - while True: - func() - n += 1 - if _t() - start_timestamp > duration: - break - end = os.times() - real = (end[4] if start[4] else time.time()) - real_start - return n, real, sum(end[0:2]) - sum(start[0:2]) - - -def warm_cache(filename): - with open(filename, "rb") as f: - f.read() - - -def run_all_tests(options): - def print_label(filename, func): - name = re.split(r'[-.]', filename)[0] - out.write( - f"[{name.center(7)}] {func.__doc__.strip()}... ".ljust(52)) - out.flush() - - def print_results(size, n, real, cpu): - bw = n * float(size) / 1024 ** 2 / real - bw = ("%4d MiB/s" if bw > 100 else "%.3g MiB/s") % bw - out.write(bw.rjust(12) + "\n") - if cpu < 0.90 * real: - out.write(" warning: test above used only " - f"{cpu / real:%} CPU, " - "result may be flawed!\n") - - def run_one_test(name, size, open_func, test_func, *args): - mode = test_func.file_open_mode - print_label(name, test_func) - if "w" not in mode or "+" in mode: - warm_cache(name) - with open_func(name) as f: - n, real, cpu = run_during(1.5, lambda: test_func(f, *args)) - print_results(size, n, real, cpu) - - def run_test_family(tests, mode_filter, files, open_func, *make_args): - for test_func in tests: - if test_func is None: - out.write("\n") - continue - if mode_filter in test_func.file_open_mode: - continue - for s in test_func.file_sizes: - name, size = files[size_names[s]] - #name += file_ext - args = tuple(f(name, size) for f in make_args) - run_one_test(name, size, - open_func, test_func, *args) - - size_names = { - "small": 0, - "medium": 1, - "large": 2, - } - - print(f"Python {sys.version}") - print("Unicode: PEP 393") - print(platform.platform()) - binary_files = list(get_binary_files()) - text_files = list(get_text_files()) - if "b" in options: - print("Binary unit = one byte") - if "t" in options: - print(f"Text unit = one character ({TEXT_ENCODING}-decoded)") - - # Binary reads - if "b" in options and "r" in options: - print("\n** Binary input **\n") - run_test_family(read_tests, "t", binary_files, lambda fn: open(fn, "rb")) - - # Text reads - if "t" in options and "r" in options: - print("\n** Text input **\n") - run_test_family(read_tests, "b", text_files, lambda fn: text_open(fn, "r")) - - # Binary writes - if "b" in options and "w" in options: - print("\n** Binary append **\n") - - def make_test_source(name, size): - with open(name, "rb") as f: - return f.read() - run_test_family(write_tests, "t", binary_files, - lambda fn: open(os.devnull, "wb"), make_test_source) - - # Text writes - if "t" in options and "w" in options: - print("\n** Text append **\n") - - def make_test_source(name, size): - with text_open(name, "r") as f: - return f.read() - run_test_family(write_tests, "b", text_files, - lambda fn: text_open(os.devnull, "w"), make_test_source) - - # Binary overwrites - if "b" in options and "w" in options: - print("\n** Binary overwrite **\n") - - def make_test_source(name, size): - with open(name, "rb") as f: - return f.read() - run_test_family(modify_tests, "t", binary_files, - lambda fn: open(fn, "r+b"), make_test_source) - - # Text overwrites - if "t" in options and "w" in options: - print("\n** Text overwrite **\n") - - def make_test_source(name, size): - with text_open(name, "r") as f: - return f.read() - run_test_family(modify_tests, "b", text_files, - lambda fn: text_open(fn, "r+"), make_test_source) - - -def prepare_files(): - print("Preparing files...") - # Binary files - for name, size in get_binary_files(): - if os.path.isfile(name) and os.path.getsize(name) == size: - continue - with open(name, "wb") as f: - f.write(os.urandom(size)) - # Text files - chunk = [] - with text_open(__file__, "r", encoding='utf8') as f: - for line in f: - if line.startswith("# "): - break - else: - raise RuntimeError( - f"Couldn't find chunk marker in {__file__} !") - if NEWLINES == "all": - it = itertools.cycle(["\n", "\r", "\r\n"]) - else: - it = itertools.repeat( - {"cr": "\r", "lf": "\n", "crlf": "\r\n"}[NEWLINES]) - chunk = "".join(line.replace("\n", next(it)) for line in f) - if isinstance(chunk, bytes): - chunk = chunk.decode('utf8') - chunk = chunk.encode(TEXT_ENCODING) - for name, size in get_text_files(): - if os.path.isfile(name) and os.path.getsize(name) == size: - continue - head = chunk * (size // len(chunk)) - tail = chunk[:size % len(chunk)] - # Adjust tail to end on a character boundary - while True: - try: - tail.decode(TEXT_ENCODING) - break - except UnicodeDecodeError: - tail = tail[:-1] - with open(name, "wb") as f: - f.write(head) - f.write(tail) - - -def main(): - global TEXT_ENCODING, NEWLINES - - usage = "usage: %prog [-h|--help] [options]" - parser = OptionParser(usage=usage) - parser.add_option("-b", "--binary", - action="store_true", dest="binary", default=False, - help="run binary I/O tests") - parser.add_option("-t", "--text", - action="store_true", dest="text", default=False, - help="run text I/O tests") - parser.add_option("-r", "--read", - action="store_true", dest="read", default=False, - help="run read tests") - parser.add_option("-w", "--write", - action="store_true", dest="write", default=False, - help="run write & modify tests") - parser.add_option("-E", "--encoding", - action="store", dest="encoding", default=None, - help=f"encoding for text tests (default: {TEXT_ENCODING})") - parser.add_option("-N", "--newlines", - action="store", dest="newlines", default='lf', - help="line endings for text tests " - "(one of: {lf (default), cr, crlf, all})") - parser.add_option("-m", "--io-module", - action="store", dest="io_module", default=None, - help="io module to test (default: builtin open())") - options, args = parser.parse_args() - if args: - parser.error("unexpected arguments") - NEWLINES = options.newlines.lower() - if NEWLINES not in ('lf', 'cr', 'crlf', 'all'): - parser.error(f"invalid 'newlines' option: {NEWLINES!r}") - - test_options = "" - if options.read: - test_options += "r" - if options.write: - test_options += "w" - elif not options.read: - test_options += "rw" - if options.text: - test_options += "t" - if options.binary: - test_options += "b" - elif not options.text: - test_options += "tb" - - if options.encoding: - TEXT_ENCODING = options.encoding - - if options.io_module: - globals()['open'] = __import__(options.io_module, {}, {}, ['open']).open - - prepare_files() - run_all_tests(test_options) - - -if __name__ == "__main__": - main() - - -# -- This part to exercise text reading. Don't change anything! -- -# - -""" -1. -Gáttir allar, -áðr gangi fram, -um skoðask skyli, -um skyggnast skyli, -því at óvíst er at vita, -hvar óvinir -sitja á fleti fyrir. - -2. -Gefendr heilir! -Gestr er inn kominn, -hvar skal sitja sjá? -Mjök er bráðr, -sá er á bröndum skal -síns of freista frama. - -3. -Elds er þörf, -þeims inn er kominn -ok á kné kalinn; -matar ok váða -er manni þörf, -þeim er hefr um fjall farit. - -4. -Vatns er þörf, -þeim er til verðar kemr, -þerru ok þjóðlaðar, -góðs of æðis, -ef sér geta mætti, -orðs ok endrþögu. - -5. -Vits er þörf, -þeim er víða ratar; -dælt er heima hvat; -at augabragði verðr, -sá er ekki kann -ok með snotrum sitr. - -6. -At hyggjandi sinni -skyli-t maðr hræsinn vera, -heldr gætinn at geði; -þá er horskr ok þögull -kemr heimisgarða til, -sjaldan verðr víti vörum, -því at óbrigðra vin -fær maðr aldregi -en mannvit mikit. - -7. -Inn vari gestr, -er til verðar kemr, -þunnu hljóði þegir, -eyrum hlýðir, -en augum skoðar; -svá nýsisk fróðra hverr fyrir. - -8. -Hinn er sæll, -er sér of getr -lof ok líknstafi; -ódælla er við þat, -er maðr eiga skal -annars brjóstum í. -""" - -""" -C'est revenir tard, je le sens, sur un sujet trop rebattu et déjà presque oublié. Mon état, qui ne me permet plus aucun travail suivi, mon aversion pour le genre polémique, ont causé ma lenteur à écrire et ma répugnance à publier. J'aurais même tout à fait supprimé ces Lettres, ou plutôt je lie les aurais point écrites, s'il n'eût été question que de moi : Mais ma patrie ne m'est pas tellement devenue étrangère que je puisse voir tranquillement opprimer ses citoyens, surtout lorsqu'ils n'ont compromis leurs droits qu'en défendant ma cause. Je serais le dernier des hommes si dans une telle occasion j'écoutais un sentiment qui n'est plus ni douceur ni patience, mais faiblesse et lâcheté, dans celui qu'il empêche de remplir son devoir. -Rien de moins important pour le public, j'en conviens, que la matière de ces lettres. La constitution d'une petite République, le sort d'un petit particulier, l'exposé de quelques injustices, la réfutation de quelques sophismes ; tout cela n'a rien en soi d'assez considérable pour mériter beaucoup de lecteurs : mais si mes sujets sont petits mes objets sont grands, et dignes de l'attention de tout honnête homme. Laissons Genève à sa place, et Rousseau dans sa dépression ; mais la religion, mais la liberté, la justice ! voilà, qui que vous soyez, ce qui n'est pas au-dessous de vous. -Qu'on ne cherche pas même ici dans le style le dédommagement de l'aridité de la matière. Ceux que quelques traits heureux de ma plume ont si fort irrités trouveront de quoi s'apaiser dans ces lettres, L'honneur de défendre un opprimé eût enflammé mon coeur si j'avais parlé pour un autre. Réduit au triste emploi de me défendre moi-même, j'ai dû me borner à raisonner ; m'échauffer eût été m'avilir. J'aurai donc trouvé grâce en ce point devant ceux qui s'imaginent qu'il est essentiel à la vérité d'être dite froidement ; opinion que pourtant j'ai peine à comprendre. Lorsqu'une vive persuasion nous anime, le moyen d'employer un langage glacé ? Quand Archimède tout transporté courait nu dans les rues de Syracuse, en avait-il moins trouvé la vérité parce qu'il se passionnait pour elle ? Tout au contraire, celui qui la sent ne peut s'abstenir de l'adorer ; celui qui demeure froid ne l'a pas vue. -Quoi qu'il en soit, je prie les lecteurs de vouloir bien mettre à part mon beau style, et d'examiner seulement si je raisonne bien ou mal ; car enfin, de cela seul qu'un auteur s'exprime en bons termes, je ne vois pas comment il peut s'ensuivre que cet auteur ne sait ce qu'il dit. -""" diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index 8eeb78e6cd69ee..045fd502a03c12 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -4,17 +4,34 @@ HoleKind: typing.TypeAlias = typing.Literal[ "ARM64_RELOC_GOT_LOAD_PAGE21", "ARM64_RELOC_GOT_LOAD_PAGEOFF12", + "ARM64_RELOC_PAGE21", + "ARM64_RELOC_PAGEOFF12", "ARM64_RELOC_UNSIGNED", - "IMAGE_REL_AMD64_ADDR64", + "IMAGE_REL_AMD64_REL32", + "IMAGE_REL_ARM64_BRANCH26", + "IMAGE_REL_ARM64_PAGEBASE_REL21", + "IMAGE_REL_ARM64_PAGEOFFSET_12A", + "IMAGE_REL_ARM64_PAGEOFFSET_12L", "IMAGE_REL_I386_DIR32", + "IMAGE_REL_I386_REL32", "R_AARCH64_ABS64", + "R_AARCH64_ADR_GOT_PAGE", "R_AARCH64_CALL26", "R_AARCH64_JUMP26", + "R_AARCH64_LD64_GOT_LO12_NC", "R_AARCH64_MOVW_UABS_G0_NC", "R_AARCH64_MOVW_UABS_G1_NC", "R_AARCH64_MOVW_UABS_G2_NC", "R_AARCH64_MOVW_UABS_G3", "R_X86_64_64", + "R_X86_64_GOTPCREL", + "R_X86_64_GOTPCRELX", + "R_X86_64_PC32", + "R_X86_64_REX_GOTPCRELX", + "X86_64_RELOC_BRANCH", + "X86_64_RELOC_GOT", + "X86_64_RELOC_GOT_LOAD", + "X86_64_RELOC_SIGNED", "X86_64_RELOC_UNSIGNED", ] diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 71c678e04fbfd5..78c566d9c8a7ef 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -63,7 +63,7 @@ def as_c(self) -> str: f"HoleKind_{self.kind}", f"HoleValue_{self.value.name}", f"&{self.symbol}" if self.symbol else "NULL", - _format_addend(self.addend), + f"{_signed(self.addend):#x}", ] return f"{{{', '.join(parts)}}}" @@ -96,7 +96,7 @@ def emit_aarch64_trampoline(self, hole: Hole) -> None: instruction |= ((base - hole.offset) >> 2) & 0x03FFFFFF self.body[where] = instruction.to_bytes(4, sys.byteorder) self.disassembly += [ - f"{base + 4 * 0: x}: d2800008 mov x8, #0x0", + f"{base + 4 * 0:x}: d2800008 mov x8, #0x0", f"{base + 4 * 0:016x}: R_AARCH64_MOVW_UABS_G0_NC {hole.symbol}", f"{base + 4 * 1:x}: f2a00008 movk x8, #0x0, lsl #16", f"{base + 4 * 1:016x}: R_AARCH64_MOVW_UABS_G1_NC {hole.symbol}", @@ -124,6 +124,56 @@ def emit_aarch64_trampoline(self, hole: Hole) -> None: ): self.holes.append(hole.replace(offset=base + 4 * i, kind=kind)) + def remove_jump(self) -> None: + """Remove a zero-length continuation jump, if it exists.""" + hole = max(self.holes, key=lambda hole: hole.offset) + match hole: + case Hole( + offset=offset, + kind="IMAGE_REL_AMD64_REL32", + value=HoleValue.GOT, + symbol="_JIT_CONTINUE", + addend=-4, + ) as hole: + # jmp qword ptr [rip] + jump = b"\x48\xFF\x25\x00\x00\x00\x00" + offset -= 3 + case Hole( + offset=offset, + kind="IMAGE_REL_I386_REL32" | "X86_64_RELOC_BRANCH", + value=HoleValue.CONTINUE, + symbol=None, + addend=-4, + ) as hole: + # jmp 5 + jump = b"\xE9\x00\x00\x00\x00" + offset -= 1 + case Hole( + offset=offset, + kind="R_AARCH64_JUMP26", + value=HoleValue.CONTINUE, + symbol=None, + addend=0, + ) as hole: + # b #4 + jump = b"\x00\x00\x00\x14" + case Hole( + offset=offset, + kind="R_X86_64_GOTPCRELX", + value=HoleValue.GOT, + symbol="_JIT_CONTINUE", + addend=addend, + ) as hole: + assert _signed(addend) == -4 + # jmp qword ptr [rip] + jump = b"\xFF\x25\x00\x00\x00\x00" + offset -= 2 + case _: + return + if self.body[offset:] == jump: + self.body = self.body[:offset] + self.holes.remove(hole) + @dataclasses.dataclass class StencilGroup: @@ -142,10 +192,19 @@ class StencilGroup: def process_relocations(self, *, alignment: int = 1) -> None: """Fix up all GOT and internal relocations for this stencil group.""" + for hole in self.code.holes.copy(): + if ( + hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"} + and hole.value is HoleValue.ZERO + ): + self.code.pad(alignment) + self.code.emit_aarch64_trampoline(hole) + self.code.pad(alignment) + self.code.holes.remove(hole) + self.code.remove_jump() self.code.pad(alignment) self.data.pad(8) for stencil in [self.code, self.data]: - holes = [] for hole in stencil.holes: if hole.value is HoleValue.GOT: assert hole.symbol is not None @@ -157,14 +216,12 @@ def process_relocations(self, *, alignment: int = 1) -> None: hole.addend += addend hole.symbol = None elif ( - hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"} + hole.kind in {"IMAGE_REL_AMD64_REL32"} and hole.value is HoleValue.ZERO ): - self.code.emit_aarch64_trampoline(hole) - continue - holes.append(hole) - stencil.holes[:] = holes - self.code.pad(alignment) + raise ValueError( + f"Add PyAPI_FUNC(...) or PyAPI_DATA(...) to declaration of {hole.symbol}!" + ) self._emit_global_offset_table() self.code.holes.sort(key=lambda hole: hole.offset) self.data.holes.sort(key=lambda hole: hole.offset) @@ -188,8 +245,9 @@ def _emit_global_offset_table(self) -> None: if value_part and not symbol and not addend: addend_part = "" else: + signed = "+" if symbol is not None else "" addend_part = f"&{symbol}" if symbol else "" - addend_part += _format_addend(addend, signed=symbol is not None) + addend_part += f"{_signed(addend):{signed}#x}" if value_part: value_part += "+" self.data.disassembly.append( @@ -213,8 +271,8 @@ def symbol_to_value(symbol: str) -> tuple[HoleValue, str | None]: return HoleValue.ZERO, symbol -def _format_addend(addend: int, signed: bool = False) -> str: - addend %= 1 << 64 - if addend & (1 << 63): - addend -= 1 << 64 - return f"{addend:{'+#x' if signed else '#x'}}" +def _signed(value: int) -> int: + value %= 1 << 64 + if value & (1 << 63): + value -= 1 << 64 + return value diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 51b091eb246413..417fdb56ccf7a1 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -37,6 +37,7 @@ class _Target(typing.Generic[_S, _R]): triple: str _: dataclasses.KW_ONLY alignment: int = 1 + args: typing.Sequence[str] = () prefix: str = "" debug: bool = False force: bool = False @@ -45,8 +46,7 @@ class _Target(typing.Generic[_S, _R]): def _compute_digest(self, out: pathlib.Path) -> str: hasher = hashlib.sha256() hasher.update(self.triple.encode()) - hasher.update(self.alignment.to_bytes()) - hasher.update(self.prefix.encode()) + hasher.update(self.debug.to_bytes()) # These dependencies are also reflected in _JITSources in regen.targets: hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes()) hasher.update((out / "pyconfig.h").read_bytes()) @@ -106,7 +106,7 @@ async def _compile( o = tempdir / f"{opname}.o" args = [ f"--target={self.triple}", - "-DPy_BUILD_CORE", + "-DPy_BUILD_CORE_MODULE", "-D_DEBUG" if self.debug else "-DNDEBUG", f"-D_JIT_OPCODE={opname}", "-D_PyJIT_ACTIVE", @@ -118,24 +118,23 @@ async def _compile( f"-I{CPYTHON / 'Python'}", "-O3", "-c", + # This debug info isn't necessary, and bloats out the JIT'ed code. + # We *may* be able to re-enable this, process it, and JIT it for a + # nicer debugging experience... but that needs a lot more research: "-fno-asynchronous-unwind-tables", - # SET_FUNCTION_ATTRIBUTE on 32-bit Windows debug builds: - "-fno-jump-tables", - # Position-independent code adds indirection to every load and jump: - "-fno-pic", - # Don't make calls to weird stack-smashing canaries: + # Don't call built-in functions that we can't find or patch: + "-fno-builtin", + # Emit relaxable 64-bit calls/jumps, so we don't have to worry about + # about emitting in-range trampolines for out-of-range targets. + # We can probably remove this and emit trampolines in the future: + "-fno-plt", + # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", - # We have three options for code model: - # - "small": the default, assumes that code and data reside in the - # lowest 2GB of memory (128MB on aarch64) - # - "medium": assumes that code resides in the lowest 2GB of memory, - # and makes no assumptions about data (not available on aarch64) - # - "large": makes no assumptions about either code or data - "-mcmodel=large", "-o", f"{o}", "-std=c11", f"{c}", + *self.args, ] await _llvm.run("clang", args, echo=self.verbose) return await self._parse(o) @@ -166,7 +165,7 @@ def build(self, out: pathlib.Path, *, comment: str = "") -> None: with jit_stencils.open("w") as file: file.write(digest) if comment: - file.write(f"// {comment}\n") + file.write(f"// {comment}\n\n") file.write("") for line in _writer.dump(stencil_groups): file.write(f"{line}\n") @@ -200,12 +199,21 @@ def _handle_section( offset = base + symbol["Value"] name = symbol["Name"] name = name.removeprefix(self.prefix) - group.symbols[name] = value, offset + if name not in group.symbols: + group.symbols[name] = value, offset for wrapped_relocation in section["Relocations"]: relocation = wrapped_relocation["Relocation"] hole = self._handle_relocation(base, relocation, stencil.body) stencil.holes.append(hole) + def _unwrap_dllimport(self, name: str) -> tuple[_stencils.HoleValue, str | None]: + if name.startswith("__imp_"): + name = name.removeprefix("__imp_") + name = name.removeprefix(self.prefix) + return _stencils.HoleValue.GOT, name + name = name.removeprefix(self.prefix) + return _stencils.symbol_to_value(name) + def _handle_relocation( self, base: int, relocation: _schema.COFFRelocation, raw: bytes ) -> _stencils.Hole: @@ -213,21 +221,36 @@ def _handle_relocation( case { "Offset": offset, "Symbol": s, - "Type": {"Value": "IMAGE_REL_AMD64_ADDR64" as kind}, + "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind}, }: offset += base - s = s.removeprefix(self.prefix) - value, symbol = _stencils.symbol_to_value(s) - addend = int.from_bytes(raw[offset : offset + 8], "little") + value, symbol = self._unwrap_dllimport(s) + addend = int.from_bytes(raw[offset : offset + 4], "little") case { "Offset": offset, "Symbol": s, - "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind}, + "Type": { + "Value": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind + }, }: offset += base - s = s.removeprefix(self.prefix) - value, symbol = _stencils.symbol_to_value(s) - addend = int.from_bytes(raw[offset : offset + 4], "little") + value, symbol = self._unwrap_dllimport(s) + addend = ( + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + ) + case { + "Offset": offset, + "Symbol": s, + "Type": { + "Value": "IMAGE_REL_ARM64_BRANCH26" + | "IMAGE_REL_ARM64_PAGEBASE_REL21" + | "IMAGE_REL_ARM64_PAGEOFFSET_12A" + | "IMAGE_REL_ARM64_PAGEOFFSET_12L" as kind + }, + }: + offset += base + value, symbol = self._unwrap_dllimport(s) + addend = 0 case _: raise NotImplementedError(relocation) return _stencils.Hole(offset, kind, value, symbol, addend) @@ -284,7 +307,23 @@ def _handle_section( def _handle_relocation( self, base: int, relocation: _schema.ELFRelocation, raw: bytes ) -> _stencils.Hole: + symbol: str | None match relocation: + case { + "Addend": addend, + "Offset": offset, + "Symbol": {"Value": s}, + "Type": { + "Value": "R_AARCH64_ADR_GOT_PAGE" + | "R_AARCH64_LD64_GOT_LO12_NC" + | "R_X86_64_GOTPCREL" + | "R_X86_64_GOTPCRELX" + | "R_X86_64_REX_GOTPCRELX" as kind + }, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.HoleValue.GOT, s case { "Addend": addend, "Offset": offset, @@ -310,6 +349,8 @@ def _handle_section( flags = {flag["Name"] for flag in section["Attributes"]["Flags"]} name = section["Name"]["Value"] name = name.removeprefix(self.prefix) + if "Debug" in flags: + return if "SomeInstructions" in flags: value = _stencils.HoleValue.CODE stencil = group.code @@ -356,6 +397,34 @@ def _handle_relocation( s = s.removeprefix(self.prefix) value, symbol = _stencils.HoleValue.GOT, s addend = 0 + case { + "Offset": offset, + "Symbol": {"Value": s}, + "Type": {"Value": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.HoleValue.GOT, s + addend = ( + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + ) + case { + "Offset": offset, + "Section": {"Value": s}, + "Type": {"Value": "X86_64_RELOC_SIGNED" as kind}, + } | { + "Offset": offset, + "Symbol": {"Value": s}, + "Type": { + "Value": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind + }, + }: + offset += base + s = s.removeprefix(self.prefix) + value, symbol = _stencils.symbol_to_value(s) + addend = ( + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + ) case { "Offset": offset, "Section": {"Value": s}, @@ -371,24 +440,28 @@ def _handle_relocation( addend = 0 case _: raise NotImplementedError(relocation) - # Turn Clang's weird __bzero calls into normal bzero calls: - if symbol == "__bzero": - symbol = "bzero" return _stencils.Hole(offset, kind, value, symbol, addend) def get_target(host: str) -> _COFF | _ELF | _MachO: """Build a _Target for the given host "triple" and options.""" if re.fullmatch(r"aarch64-apple-darwin.*", host): - return _MachO(host, alignment=8, prefix="_") + args = ["-mcmodel=large"] + return _MachO(host, alignment=8, args=args, prefix="_") + if re.fullmatch(r"aarch64-pc-windows-msvc", host): + args = ["-fms-runtime-lib=dll"] + return _COFF(host, alignment=8, args=args) if re.fullmatch(r"aarch64-.*-linux-gnu", host): - return _ELF(host, alignment=8) + args = ["-mcmodel=large"] + return _ELF(host, alignment=8, args=args) if re.fullmatch(r"i686-pc-windows-msvc", host): - return _COFF(host, prefix="_") + args = ["-DPy_NO_ENABLE_SHARED"] + return _COFF(host, args=args, prefix="_") if re.fullmatch(r"x86_64-apple-darwin.*", host): return _MachO(host, prefix="_") if re.fullmatch(r"x86_64-pc-windows-msvc", host): - return _COFF(host) + args = ["-fms-runtime-lib=dll"] + return _COFF(host, args=args) if re.fullmatch(r"x86_64-.*-linux-gnu", host): return _ELF(host) raise ValueError(host) diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 12303a550d8879..8aaf4581de362d 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -9,6 +9,7 @@ #include "pycore_long.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" +#include "pycore_optimizer.h" #include "pycore_range.h" #include "pycore_setobject.h" #include "pycore_sliceobject.h" @@ -38,17 +39,31 @@ goto LABEL ## _tier_two; \ } while (0) +#undef GOTO_TIER_TWO +#define GOTO_TIER_TWO(EXECUTOR) \ +do { \ + __attribute__((musttail)) \ + return ((jit_func)((EXECUTOR)->jit_code))(frame, stack_pointer, tstate); \ +} while (0) + +#undef GOTO_TIER_ONE +#define GOTO_TIER_ONE(TARGET) \ +do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + return TARGET; \ +} while (0) + #undef LOAD_IP #define LOAD_IP(UNUSED) \ do { \ } while (0) #define PATCH_VALUE(TYPE, NAME, ALIAS) \ - extern void ALIAS; \ + PyAPI_DATA(void) ALIAS; \ TYPE NAME = (TYPE)(uint64_t)&ALIAS; #define PATCH_JUMP(ALIAS) \ - extern void ALIAS; \ + PyAPI_DATA(void) ALIAS; \ __attribute__((musttail)) \ return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); @@ -59,7 +74,6 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) int oparg; int opcode = _JIT_OPCODE; - _PyUOpInstruction *next_uop; // Other stuff we need handy: PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) @@ -90,9 +104,16 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * pop_1_error_tier_two: STACK_SHRINK(1); error_tier_two: - _PyFrame_SetStackPointer(frame, stack_pointer); - return NULL; + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(NULL); deoptimize: - _PyFrame_SetStackPointer(frame, stack_pointer); - return _PyCode_CODE(_PyFrame_GetCode(frame)) + _target; + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target); +side_exit: + { + _PyExitData *exit = ¤t_executor->exits[_target]; + Py_INCREF(exit->executor); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_TWO(exit->executor); + } } diff --git a/Tools/msi/bundle/Default.wxl b/Tools/msi/bundle/Default.wxl index 1540f050159a54..0014204e89d1bb 100644 --- a/Tools/msi/bundle/Default.wxl +++ b/Tools/msi/bundle/Default.wxl @@ -88,6 +88,7 @@ Select Customize to review current options. Install Python [ShortVersion] for &all users for &all users (requires admin privileges) Use admin privi&leges when installing py.exe + Python Launcher is already installed &Precompile standard library Download debugging &symbols Download debu&g binaries (requires VS 2017 or later) diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp index 3a17ffbaa0b655..e0e179e3aede6d 100644 --- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp +++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp @@ -442,6 +442,14 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { ThemeControlElevates(_theme, ID_INSTALL_BUTTON, elevated); ThemeControlElevates(_theme, ID_INSTALL_SIMPLE_BUTTON, elevated); ThemeControlElevates(_theme, ID_INSTALL_UPGRADE_BUTTON, elevated); + + LONGLONG blockedLauncher; + if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) && blockedLauncher) { + LOC_STRING *pLocString = nullptr; + if (SUCCEEDED(LocGetString(_wixLoc, L"#(loc.ShortInstallLauncherBlockedLabel)", &pLocString)) && pLocString) { + ThemeSetTextControl(_theme, ID_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX, pLocString->wzText); + } + } } void Custom1Page_Show() { @@ -718,25 +726,67 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { __in DWORD64 /*dw64Version*/, __in BOOTSTRAPPER_RELATED_OPERATION operation ) { - if (BOOTSTRAPPER_RELATED_OPERATION_MAJOR_UPGRADE == operation && - (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_AllUsers", -1) || - CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_JustForMe", -1))) { - auto hr = LoadAssociateFilesStateFromKey(_engine, fPerMachine ? HKEY_LOCAL_MACHINE : HKEY_CURRENT_USER); - if (hr == S_OK) { - _engine->SetVariableNumeric(L"AssociateFiles", 1); - } else if (hr == S_FALSE) { - _engine->SetVariableNumeric(L"AssociateFiles", 0); - } else if (FAILED(hr)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr); + // Only check launcher_AllUsers because we'll find the same packages + // twice if we check launcher_JustForMe as well. + if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_AllUsers", -1)) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Detected existing launcher install"); + + LONGLONG blockedLauncher, detectedLauncher; + if (FAILED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher))) { + blockedLauncher = 0; + } + + // Get the prior DetectedLauncher value so we can see if we've + // detected more than one, and then update the stored variable + // (we use the original value later on via the local). + if (FAILED(BalGetNumericVariable(L"DetectedLauncher", &detectedLauncher))) { + detectedLauncher = 0; + } + if (!detectedLauncher) { + _engine->SetVariableNumeric(L"DetectedLauncher", 1); + } + + if (blockedLauncher) { + // Nothing else to do, we're already blocking + } + else if (BOOTSTRAPPER_RELATED_OPERATION_DOWNGRADE == operation) { + // Found a higher version, so we can't install ours. + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Higher version launcher has been detected."); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Launcher will not be installed"); + _engine->SetVariableNumeric(L"BlockedLauncher", 1); + } + else if (detectedLauncher) { + if (!blockedLauncher) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Multiple launcher installs have been detected."); + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "No launcher will be installed or upgraded until one has been removed."); + _engine->SetVariableNumeric(L"BlockedLauncher", 1); + } } + else if (BOOTSTRAPPER_RELATED_OPERATION_MAJOR_UPGRADE == operation) { + // Found an older version, so let's run the equivalent as an upgrade + // This overrides "unknown" all users options, but will leave alone + // any that have already been set/detected. + // User can deselect the option to include the launcher, but cannot + // change it from the current per user/machine setting. + LONGLONG includeLauncher, includeLauncherAllUsers; + if (FAILED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))) { + includeLauncher = -1; + } + if (FAILED(BalGetNumericVariable(L"InstallLauncherAllUsers", &includeLauncherAllUsers))) { + includeLauncherAllUsers = -1; + } - LONGLONG includeLauncher; - if (FAILED(BalGetNumericVariable(L"Include_launcher", &includeLauncher)) - || includeLauncher == -1) { - _engine->SetVariableNumeric(L"Include_launcher", 1); - _engine->SetVariableNumeric(L"InstallLauncherAllUsers", fPerMachine ? 1 : 0); + if (includeLauncher < 0) { + _engine->SetVariableNumeric(L"Include_launcher", 1); + } + if (includeLauncherAllUsers < 0) { + _engine->SetVariableNumeric(L"InstallLauncherAllUsers", fPerMachine ? 1 : 0); + } else if (includeLauncherAllUsers != fPerMachine ? 1 : 0) { + // Requested AllUsers option is inconsistent, so block + _engine->SetVariableNumeric(L"BlockedLauncher", 1); + } + _engine->SetVariableNumeric(L"DetectedOldLauncher", 1); } - _engine->SetVariableNumeric(L"DetectedOldLauncher", 1); } return CheckCanceled() ? IDCANCEL : IDNOACTION; } @@ -784,48 +834,7 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { __in LPCWSTR wzPackageId, __in HRESULT hrStatus, __in BOOTSTRAPPER_PACKAGE_STATE state - ) { - if (FAILED(hrStatus)) { - return; - } - - BOOL detectedLauncher = FALSE; - HKEY hkey = HKEY_LOCAL_MACHINE; - if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_AllUsers", -1)) { - if (BOOTSTRAPPER_PACKAGE_STATE_PRESENT == state || BOOTSTRAPPER_PACKAGE_STATE_OBSOLETE == state) { - detectedLauncher = TRUE; - _engine->SetVariableNumeric(L"InstallLauncherAllUsers", 1); - } - } else if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_JustForMe", -1)) { - if (BOOTSTRAPPER_PACKAGE_STATE_PRESENT == state || BOOTSTRAPPER_PACKAGE_STATE_OBSOLETE == state) { - detectedLauncher = TRUE; - _engine->SetVariableNumeric(L"InstallLauncherAllUsers", 0); - } - } - - LONGLONG includeLauncher; - if (SUCCEEDED(BalGetNumericVariable(L"Include_launcher", &includeLauncher)) - && includeLauncher != -1) { - detectedLauncher = FALSE; - } - - if (detectedLauncher) { - /* When we detect the current version of the launcher. */ - _engine->SetVariableNumeric(L"Include_launcher", 1); - _engine->SetVariableNumeric(L"DetectedLauncher", 1); - _engine->SetVariableString(L"Include_launcherState", L"disable"); - _engine->SetVariableString(L"InstallLauncherAllUsersState", L"disable"); - - auto hr = LoadAssociateFilesStateFromKey(_engine, hkey); - if (hr == S_OK) { - _engine->SetVariableNumeric(L"AssociateFiles", 1); - } else if (hr == S_FALSE) { - _engine->SetVariableNumeric(L"AssociateFiles", 0); - } else if (FAILED(hr)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr); - } - } - } + ) { } virtual STDMETHODIMP_(void) OnDetectComplete(__in HRESULT hrStatus) { @@ -835,19 +844,67 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { } if (SUCCEEDED(hrStatus)) { - LONGLONG includeLauncher; - if (SUCCEEDED(BalGetNumericVariable(L"Include_launcher", &includeLauncher)) - && includeLauncher == -1) { - if (BOOTSTRAPPER_ACTION_LAYOUT == _command.action || - (BOOTSTRAPPER_ACTION_INSTALL == _command.action && !_upgrading)) { - // When installing/downloading, we want to include the launcher - // by default. - _engine->SetVariableNumeric(L"Include_launcher", 1); - } else { - // Any other action, if we didn't detect the MSI then we want to - // keep it excluded - _engine->SetVariableNumeric(L"Include_launcher", 0); - _engine->SetVariableNumeric(L"AssociateFiles", 0); + // Update launcher install states + // If we didn't detect any existing installs, Include_launcher and + // InstallLauncherAllUsers will both be -1, so we will set to their + // defaults and leave the options enabled. + // Otherwise, if we detected an existing install, we disable the + // options so they remain fixed. + // The code in OnDetectRelatedMsiPackage is responsible for figuring + // out whether existing installs are compatible with the settings in + // place during detection. + LONGLONG blockedLauncher; + if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) + && blockedLauncher) { + _engine->SetVariableNumeric(L"Include_launcher", 0); + _engine->SetVariableNumeric(L"InstallLauncherAllUsers", 0); + _engine->SetVariableString(L"InstallLauncherAllUsersState", L"disable"); + _engine->SetVariableString(L"Include_launcherState", L"disable"); + } + else { + LONGLONG includeLauncher, includeLauncherAllUsers, associateFiles; + + if (FAILED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))) { + includeLauncher = -1; + } + if (FAILED(BalGetNumericVariable(L"InstallLauncherAllUsers", &includeLauncherAllUsers))) { + includeLauncherAllUsers = -1; + } + if (FAILED(BalGetNumericVariable(L"AssociateFiles", &associateFiles))) { + associateFiles = -1; + } + + if (includeLauncherAllUsers < 0) { + includeLauncherAllUsers = 0; + _engine->SetVariableNumeric(L"InstallLauncherAllUsers", includeLauncherAllUsers); + } + + if (includeLauncher < 0) { + if (BOOTSTRAPPER_ACTION_LAYOUT == _command.action || + (BOOTSTRAPPER_ACTION_INSTALL == _command.action && !_upgrading)) { + // When installing/downloading, we include the launcher + // (though downloads should ignore this setting anyway) + _engine->SetVariableNumeric(L"Include_launcher", 1); + } else { + // Any other action, we should have detected an existing + // install (e.g. on remove/modify), so if we didn't, we + // assume it's not selected. + _engine->SetVariableNumeric(L"Include_launcher", 0); + _engine->SetVariableNumeric(L"AssociateFiles", 0); + } + } + + if (associateFiles < 0) { + auto hr = LoadAssociateFilesStateFromKey( + _engine, + includeLauncherAllUsers ? HKEY_LOCAL_MACHINE : HKEY_CURRENT_USER + ); + if (FAILED(hr)) { + BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr); + } else if (hr == S_OK) { + associateFiles = 1; + } + _engine->SetVariableNumeric(L"AssociateFiles", associateFiles); } } } diff --git a/Tools/msi/bundle/bundle.wxs b/Tools/msi/bundle/bundle.wxs index 9b4f072152d5c0..abfeb88784890c 100644 --- a/Tools/msi/bundle/bundle.wxs +++ b/Tools/msi/bundle/bundle.wxs @@ -28,10 +28,11 @@ - + - + + @@ -91,10 +92,11 @@ + - + diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index c0a63b40ff4155..2fb616e894a734 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -3,5 +3,5 @@ mypy==1.8.0 # needed for peg_generator: -types-psutil==5.9.5.20240106 -types-setuptools==69.0.0.20240125 +types-psutil==5.9.5.20240205 +types-setuptools==69.1.0.20240301 diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 064731a236ee86..1a45d1c431dd11 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.97.4 +hypothesis==6.98.15 diff --git a/Tools/scripts/sortperf.py b/Tools/scripts/sortperf.py new file mode 100644 index 00000000000000..b54681524ac173 --- /dev/null +++ b/Tools/scripts/sortperf.py @@ -0,0 +1,196 @@ +""" +List sort performance test. + +To install `pyperf` you would need to: + + python3 -m pip install pyperf + +To run: + + python3 Tools/scripts/sortperf + +Options: + + * `benchmark` name to run + * `--rnd-seed` to set random seed + * `--size` to set the sorted list size + +Based on https://github.com/python/cpython/blob/963904335e579bfe39101adf3fd6a0cf705975ff/Lib/test/sortperf.py +""" + +from __future__ import annotations + +import argparse +import time +import random + + +# =============== +# Data generation +# =============== + +def _random_data(size: int, rand: random.Random) -> list[float]: + result = [rand.random() for _ in range(size)] + # Shuffle it a bit... + for i in range(10): + i = rand.randrange(size) + temp = result[:i] + del result[:i] + temp.reverse() + result.extend(temp) + del temp + assert len(result) == size + return result + + +def list_sort(size: int, rand: random.Random) -> list[float]: + return _random_data(size, rand) + + +def list_sort_descending(size: int, rand: random.Random) -> list[float]: + return list(reversed(list_sort_ascending(size, rand))) + + +def list_sort_ascending(size: int, rand: random.Random) -> list[float]: + return sorted(_random_data(size, rand)) + + +def list_sort_ascending_exchanged(size: int, rand: random.Random) -> list[float]: + result = list_sort_ascending(size, rand) + # Do 3 random exchanges. + for _ in range(3): + i1 = rand.randrange(size) + i2 = rand.randrange(size) + result[i1], result[i2] = result[i2], result[i1] + return result + + +def list_sort_ascending_random(size: int, rand: random.Random) -> list[float]: + assert size >= 10, "This benchmark requires size to be >= 10" + result = list_sort_ascending(size, rand) + # Replace the last 10 with random floats. + result[-10:] = [rand.random() for _ in range(10)] + return result + + +def list_sort_ascending_one_percent(size: int, rand: random.Random) -> list[float]: + result = list_sort_ascending(size, rand) + # Replace 1% of the elements at random. + for _ in range(size // 100): + result[rand.randrange(size)] = rand.random() + return result + + +def list_sort_duplicates(size: int, rand: random.Random) -> list[float]: + assert size >= 4 + result = list_sort_ascending(4, rand) + # Arrange for lots of duplicates. + result = result * (size // 4) + # Force the elements to be distinct objects, else timings can be + # artificially low. + return list(map(abs, result)) + + +def list_sort_equal(size: int, rand: random.Random) -> list[float]: + # All equal. Again, force the elements to be distinct objects. + return list(map(abs, [-0.519012] * size)) + + +def list_sort_worst_case(size: int, rand: random.Random) -> list[float]: + # This one looks like [3, 2, 1, 0, 0, 1, 2, 3]. It was a bad case + # for an older implementation of quicksort, which used the median + # of the first, last and middle elements as the pivot. + half = size // 2 + result = list(range(half - 1, -1, -1)) + result.extend(range(half)) + # Force to float, so that the timings are comparable. This is + # significantly faster if we leave them as ints. + return list(map(float, result)) + + +# ========= +# Benchmark +# ========= + +class Benchmark: + def __init__(self, name: str, size: int, seed: int) -> None: + self._name = name + self._size = size + self._seed = seed + self._random = random.Random(self._seed) + + def run(self, loops: int) -> float: + all_data = self._prepare_data(loops) + start = time.perf_counter() + + for data in all_data: + data.sort() # Benching this method! + + return time.perf_counter() - start + + def _prepare_data(self, loops: int) -> list[float]: + bench = BENCHMARKS[self._name] + return [bench(self._size, self._random)] * loops + + +def add_cmdline_args(cmd: list[str], args) -> None: + if args.benchmark: + cmd.append(args.benchmark) + cmd.append(f"--size={args.size}") + cmd.append(f"--rng-seed={args.rng_seed}") + + +def add_parser_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "benchmark", + choices=BENCHMARKS, + nargs="?", + help="Can be any of: {0}".format(", ".join(BENCHMARKS)), + ) + parser.add_argument( + "--size", + type=int, + default=DEFAULT_SIZE, + help=f"Size of the lists to sort (default: {DEFAULT_SIZE})", + ) + parser.add_argument( + "--rng-seed", + type=int, + default=DEFAULT_RANDOM_SEED, + help=f"Random number generator seed (default: {DEFAULT_RANDOM_SEED})", + ) + + +DEFAULT_SIZE = 1 << 14 +DEFAULT_RANDOM_SEED = 0 +BENCHMARKS = { + "list_sort": list_sort, + "list_sort_descending": list_sort_descending, + "list_sort_ascending": list_sort_ascending, + "list_sort_ascending_exchanged": list_sort_ascending_exchanged, + "list_sort_ascending_random": list_sort_ascending_random, + "list_sort_ascending_one_percent": list_sort_ascending_one_percent, + "list_sort_duplicates": list_sort_duplicates, + "list_sort_equal": list_sort_equal, + "list_sort_worst_case": list_sort_worst_case, +} + +if __name__ == "__main__": + # This needs `pyperf` 3rd party library: + import pyperf + + runner = pyperf.Runner(add_cmdline_args=add_cmdline_args) + add_parser_args(runner.argparser) + args = runner.parse_args() + + runner.metadata["description"] = "Test `list.sort()` with different data" + runner.metadata["list_sort_size"] = args.size + runner.metadata["list_sort_random_seed"] = args.rng_seed + + if args.benchmark: + benchmarks = (args.benchmark,) + else: + benchmarks = sorted(BENCHMARKS) + for bench in benchmarks: + benchmark = Benchmark(bench, args.size, args.rng_seed) + runner.bench_time_func(bench, benchmark.run) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 7891b9cf923d33..2925e096f4d95e 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -11,6 +11,7 @@ import argparse import collections from collections.abc import KeysView +from dataclasses import dataclass from datetime import date import enum import functools @@ -21,6 +22,7 @@ from pathlib import Path import re import sys +import textwrap from typing import Any, Callable, TextIO, TypeAlias @@ -100,6 +102,10 @@ def load_raw_data(input: Path) -> RawData: file=sys.stderr, ) continue + # Hack to handle older data files where some uops + # are missing an underscore prefix in their name + if key.startswith("uops[") and key[5:6] != "_": + key = "uops[_" + key[5:] stats[key.strip()] += int(value) stats["__nfiles__"] += 1 @@ -115,6 +121,64 @@ def save_raw_data(data: RawData, json_output: TextIO): json.dump(data, json_output) +@dataclass(frozen=True) +class Doc: + text: str + doc: str + + def markdown(self) -> str: + return textwrap.dedent( + f""" + {self.text} +
+ ⓘ + + {self.doc} +
+ """ + ) + + +class Count(int): + def markdown(self) -> str: + return format(self, ",d") + + +@dataclass(frozen=True) +class Ratio: + num: int + den: int | None = None + percentage: bool = True + + def __float__(self): + if self.den == 0: + return 0.0 + elif self.den is None: + return self.num + else: + return self.num / self.den + + def markdown(self) -> str: + if self.den is None: + return "" + elif self.den == 0: + if self.num != 0: + return f"{self.num:,} / 0 !!" + return "" + elif self.percentage: + return f"{self.num / self.den:,.01%}" + else: + return f"{self.num / self.den:,.02f}" + + +class DiffRatio(Ratio): + def __init__(self, base: int | str, head: int | str): + if isinstance(base, str) or isinstance(head, str): + super().__init__(0, 0) + else: + super().__init__(head - base, base) + + class OpcodeStats: """ Manages the data related to specific set of opcodes, e.g. tier1 (with prefix @@ -387,19 +451,65 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: inner_loop = self._data["Optimization inner loop"] recursive_call = self._data["Optimization recursive call"] low_confidence = self._data["Optimization low confidence"] + executors_invalidated = self._data["Executors invalidated"] return { - "Optimization attempts": (attempts, None), - "Traces created": (created, attempts), - "Trace stack overflow": (trace_stack_overflow, attempts), - "Trace stack underflow": (trace_stack_underflow, attempts), - "Trace too long": (trace_too_long, attempts), - "Trace too short": (trace_too_short, attempts), - "Inner loop found": (inner_loop, attempts), - "Recursive call": (recursive_call, attempts), - "Low confidence": (low_confidence, attempts), - "Traces executed": (executed, None), - "Uops executed": (uops, executed), + Doc( + "Optimization attempts", + "The number of times a potential trace is identified. Specifically, this " + "occurs in the JUMP BACKWARD instruction when the counter reaches a " + "threshold.", + ): ( + attempts, + None, + ), + Doc( + "Traces created", "The number of traces that were successfully created." + ): (created, attempts), + Doc( + "Trace stack overflow", + "A trace is truncated because it would require more than 5 stack frames.", + ): (trace_stack_overflow, attempts), + Doc( + "Trace stack underflow", + "A potential trace is abandoned because it pops more frames than it pushes.", + ): (trace_stack_underflow, attempts), + Doc( + "Trace too long", + "A trace is truncated because it is longer than the instruction buffer.", + ): (trace_too_long, attempts), + Doc( + "Trace too short", + "A potential trace is abandoced because it it too short.", + ): (trace_too_short, attempts), + Doc( + "Inner loop found", "A trace is truncated because it has an inner loop" + ): (inner_loop, attempts), + Doc( + "Recursive call", + "A trace is truncated because it has a recursive call.", + ): (recursive_call, attempts), + Doc( + "Low confidence", + "A trace is abandoned because the likelihood of the jump to top being taken " + "is too low.", + ): (low_confidence, attempts), + Doc( + "Executors invalidated", + "The number of executors that were invalidated due to watched " + "dictionary changes.", + ): (executors_invalidated, created), + Doc("Traces executed", "The number of traces that were executed"): ( + executed, + None, + ), + Doc( + "Uops executed", + "The total number of uops (micro-operations) that were executed", + ): ( + uops, + executed, + ), } def get_histogram(self, prefix: str) -> list[tuple[int, int]]: @@ -415,52 +525,12 @@ def get_histogram(self, prefix: str) -> list[tuple[int, int]]: def get_rare_events(self) -> list[tuple[str, int]]: prefix = "Rare event " return [ - (key[len(prefix) + 1:-1].replace("_", " "), val) + (key[len(prefix) + 1 : -1].replace("_", " "), val) for key, val in self._data.items() if key.startswith(prefix) ] -class Count(int): - def markdown(self) -> str: - return format(self, ",d") - - -class Ratio: - def __init__(self, num: int, den: int | None, percentage: bool = True): - self.num = num - self.den = den - self.percentage = percentage - - def __float__(self): - if self.den == 0: - return 0.0 - elif self.den is None: - return self.num - else: - return self.num / self.den - - def markdown(self) -> str: - if self.den is None: - return "" - elif self.den == 0: - if self.num != 0: - return f"{self.num:,} / 0 !!" - return "" - elif self.percentage: - return f"{self.num / self.den:,.01%}" - else: - return f"{self.num / self.den:,.02f}" - - -class DiffRatio(Ratio): - def __init__(self, base: int | str, head: int | str): - if isinstance(base, str) or isinstance(head, str): - super().__init__(0, 0) - else: - super().__init__(head - base, base) - - class JoinMode(enum.Enum): # Join using the first column as a key SIMPLE = 0 @@ -568,13 +638,16 @@ def __init__( title: str = "", summary: str = "", part_iter=None, + *, comparative: bool = True, + doc: str = "", ): self.title = title if not summary: self.summary = title.lower() else: self.summary = summary + self.doc = textwrap.dedent(doc) if part_iter is None: part_iter = [] if isinstance(part_iter, list): @@ -620,7 +693,7 @@ def calc(stats: Stats) -> Rows: def execution_count_section() -> Section: return Section( "Execution counts", - "execution counts for all instructions", + "Execution counts for Tier 1 instructions.", [ Table( ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), @@ -628,6 +701,11 @@ def execution_count_section() -> Section: join_mode=JoinMode.CHANGE_ONE_COLUMN, ) ], + doc=""" + The "miss ratio" column shows the percentage of times the instruction + executed that it deoptimized. When this happens, the base unspecialized + instruction is not counted. + """, ) @@ -655,7 +733,7 @@ def calc_pair_count_table(stats: Stats) -> Rows: return Section( "Pair counts", - "Pair counts for top 100 pairs", + "Pair counts for top 100 Tier 1 instructions", [ Table( ("Pair", "Count:", "Self:", "Cumulative:"), @@ -663,6 +741,10 @@ def calc_pair_count_table(stats: Stats) -> Rows: ) ], comparative=False, + doc=""" + Pairs of specialized operations that deoptimize and are then followed by + the corresponding unspecialized instruction are not counted as pairs. + """, ) @@ -705,22 +787,33 @@ def iter_pre_succ_pairs_tables(base_stats: Stats, head_stats: Stats | None = Non return Section( "Predecessor/Successor Pairs", - "Top 5 predecessors and successors of each opcode", + "Top 5 predecessors and successors of each Tier 1 opcode.", iter_pre_succ_pairs_tables, comparative=False, + doc=""" + This does not include the unspecialized instructions that occur after a + specialized instruction deoptimizes. + """, ) def specialization_section() -> Section: def calc_specialization_table(opcode: str) -> RowCalculator: def calc(stats: Stats) -> Rows: + DOCS = { + "deferred": 'Lists the number of "deferred" (i.e. not specialized) instructions executed.', + "hit": "Specialized instructions that complete.", + "miss": "Specialized instructions that deopt.", + "deopt": "Specialized instructions that deopt.", + } + opcode_stats = stats.get_opcode_stats("opcode") total = opcode_stats.get_specialization_total(opcode) specialization_counts = opcode_stats.get_specialization_counts(opcode) return [ ( - f"{label:>12}", + Doc(label, DOCS[label]), Count(count), Ratio(count, total), ) @@ -790,7 +883,7 @@ def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = Non JoinMode.CHANGE, ), Table( - ("", "Count:", "Ratio:"), + ("Success", "Count:", "Ratio:"), calc_specialization_success_failure_table(opcode), JoinMode.CHANGE, ), @@ -804,7 +897,7 @@ def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = Non return Section( "Specialization stats", - "specialization stats by family", + "Specialization stats by family", iter_specialization_tables, ) @@ -822,19 +915,35 @@ def calc_specialization_effectiveness_table(stats: Stats) -> Rows: ) = opcode_stats.get_specialized_total_counts() return [ - ("Basic", Count(basic), Ratio(basic, total)), ( - "Not specialized", + Doc( + "Basic", + "Instructions that are not and cannot be specialized, e.g. `LOAD_FAST`.", + ), + Count(basic), + Ratio(basic, total), + ), + ( + Doc( + "Not specialized", + "Instructions that could be specialized but aren't, e.g. `LOAD_ATTR`, `BINARY_SLICE`.", + ), Count(not_specialized), Ratio(not_specialized, total), ), ( - "Specialized hits", + Doc( + "Specialized hits", + "Specialized instructions, e.g. `LOAD_ATTR_MODULE` that complete.", + ), Count(specialized_hits), Ratio(specialized_hits, total), ), ( - "Specialized misses", + Doc( + "Specialized misses", + "Specialized instructions, e.g. `LOAD_ATTR_MODULE` that deopt.", + ), Count(specialized_misses), Ratio(specialized_misses, total), ), @@ -879,7 +988,7 @@ def calc_misses_by_table(stats: Stats) -> Rows: ), Section( "Deferred by instruction", - "", + "Breakdown of deferred (not specialized) instruction counts by family", [ Table( ("Name", "Count:", "Ratio:"), @@ -890,7 +999,7 @@ def calc_misses_by_table(stats: Stats) -> Rows: ), Section( "Misses by instruction", - "", + "Breakdown of misses (specialized deopts) instruction counts by family", [ Table( ("Name", "Count:", "Ratio:"), @@ -900,6 +1009,10 @@ def calc_misses_by_table(stats: Stats) -> Rows: ], ), ], + doc=""" + All entries are execution counts. Should add up to the total number of + Tier 1 instructions executed. + """, ) @@ -922,6 +1035,13 @@ def calc_call_stats_table(stats: Stats) -> Rows: JoinMode.CHANGE, ) ], + doc=""" + This shows what fraction of calls to Python functions are inlined (i.e. + not having a call at the C level) and for those that are not, where the + call comes from. The various categories overlap. + + Also includes the count of frame objects created. + """, ) @@ -935,7 +1055,7 @@ def calc_object_stats_table(stats: Stats) -> Rows: return Section( "Object stats", - "allocations, frees and dict materializatons", + "Allocations, frees and dict materializatons", [ Table( ("", "Count:", "Ratio:"), @@ -943,6 +1063,16 @@ def calc_object_stats_table(stats: Stats) -> Rows: JoinMode.CHANGE, ) ], + doc=""" + Below, "allocations" means "allocations that are not from a freelist". + Total allocations = "Allocations from freelist" + "Allocations". + + "New values" is the number of values arrays created for objects with + managed dicts. + + The cache hit/miss numbers are for the MRO cache, split into dunder and + other names. + """, ) @@ -969,6 +1099,9 @@ def calc_gc_stats(stats: Stats) -> Rows: calc_gc_stats, ) ], + doc=""" + Collected/visits gives some measure of efficiency. + """, ) @@ -1074,7 +1207,19 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) def rare_event_section() -> Section: def calc_rare_event_table(stats: Stats) -> Table: - return [(x, Count(y)) for x, y in stats.get_rare_events()] + DOCS = { + "set class": "Setting an object's class, `obj.__class__ = ...`", + "set bases": "Setting the bases of a class, `cls.__bases__ = ...`", + "set eval frame func": ( + "Setting the PEP 523 frame eval function " + "`_PyInterpreterState_SetFrameEvalFunc()`" + ), + "builtin dict": "Modifying the builtins, `__builtins__.__dict__[var] = ...`", + "func modification": "Modifying a function, e.g. `func.__defaults__ = ...`, etc.", + "watched dict modification": "A watched dict has been modified", + "watched globals modification": "A watched `globals()` dict has been modified", + } + return [(Doc(x, DOCS[x]), Count(y)) for x, y in stats.get_rare_events()] return Section( "Rare events", @@ -1134,6 +1279,9 @@ def to_markdown(x): print("
", file=out) print("", obj.summary, "", file=out) print(file=out) + if obj.doc: + print(obj.doc, file=out) + if head_stats is not None and obj.comparative is False: print("Not included in comparative output.\n") else: @@ -1149,24 +1297,36 @@ def to_markdown(x): if len(rows) == 0: return - width = len(header) - header_line = "|" - under_line = "|" + alignments = [] for item in header: - under = "---" + if item.endswith(":"): + alignments.append("right") + else: + alignments.append("left") + + print("", file=out) + print("", file=out) + print("", file=out) + for item, align in zip(header, alignments): if item.endswith(":"): item = item[:-1] - under += ":" - header_line += item + " | " - under_line += under + "|" - print(header_line, file=out) - print(under_line, file=out) + print(f'', file=out) + print("", file=out) + print("", file=out) + + print("", file=out) for row in rows: - if len(row) != width: + if len(row) != len(header): raise ValueError( "Wrong number of elements in row '" + str(row) + "'" ) - print("|", " | ".join(to_markdown(i) for i in row), "|", file=out) + print("", file=out) + for col, align in zip(row, alignments): + print(f'', file=out) + print("", file=out) + print("", file=out) + + print("
{item}
{to_markdown(col)}
", file=out) print(file=out) case list(): diff --git a/Tools/stringbench/README b/Tools/stringbench/README deleted file mode 100644 index a271f12632afff..00000000000000 --- a/Tools/stringbench/README +++ /dev/null @@ -1,68 +0,0 @@ -stringbench is a set of performance tests comparing byte string -operations with unicode operations. The two string implementations -are loosely based on each other and sometimes the algorithm for one is -faster than the other. - -These test set was started at the Need For Speed sprint in Reykjavik -to identify which string methods could be sped up quickly and to -identify obvious places for improvement. - -Here is an example of a benchmark - - -@bench('"Andrew".startswith("A")', 'startswith single character', 1000) -def startswith_single(STR): - s1 = STR("Andrew") - s2 = STR("A") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - -The bench decorator takes three parameters. The first is a short -description of how the code works. In most cases this is Python code -snippet. It is not the code which is actually run because the real -code is hand-optimized to focus on the method being tested. - -The second parameter is a group title. All benchmarks with the same -group title are listed together. This lets you compare different -implementations of the same algorithm, such as "t in s" -vs. "s.find(t)". - -The last is a count. Each benchmark loops over the algorithm either -100 or 1000 times, depending on the algorithm performance. The output -time is the time per benchmark call so the reader needs a way to know -how to scale the performance. - -These parameters become function attributes. - - -Here is an example of the output - - -========== count newlines -38.54 41.60 92.7 ...text.with.2000.newlines.count("\n") (*100) -========== early match, single character -1.14 1.18 96.8 ("A"*1000).find("A") (*1000) -0.44 0.41 105.6 "A" in "A"*1000 (*1000) -1.15 1.17 98.1 ("A"*1000).index("A") (*1000) - -The first column is the run time in milliseconds for byte strings. -The second is the run time for unicode strings. The third is a -percentage; byte time / unicode time. It's the percentage by which -unicode is faster than byte strings. - -The last column contains the code snippet and the repeat count for the -internal benchmark loop. - -The times are computed with 'timeit.py' which repeats the test more -and more times until the total time takes over 0.2 seconds, returning -the best time for a single iteration. - -The final line of the output is the cumulative time for byte and -unicode strings, and the overall performance of unicode relative to -bytes. For example - -4079.83 5432.25 75.1 TOTAL - -However, this has no meaning as it evenly weights every test. - diff --git a/Tools/stringbench/stringbench.py b/Tools/stringbench/stringbench.py deleted file mode 100644 index 5d2b4146378626..00000000000000 --- a/Tools/stringbench/stringbench.py +++ /dev/null @@ -1,1482 +0,0 @@ - -# Various microbenchmarks comparing unicode and byte string performance -# Please keep this file both 2.x and 3.x compatible! - -import timeit -import itertools -import operator -import re -import sys -import datetime -import optparse - -VERSION = '2.0' - -def p(*args): - sys.stdout.write(' '.join(str(s) for s in args) + '\n') - -if sys.version_info >= (3,): - BYTES = bytes_from_str = lambda x: x.encode('ascii') - UNICODE = unicode_from_str = lambda x: x -else: - BYTES = bytes_from_str = lambda x: x - UNICODE = unicode_from_str = lambda x: x.decode('ascii') - -class UnsupportedType(TypeError): - pass - - -p('stringbench v%s' % VERSION) -p(sys.version) -p(datetime.datetime.now()) - -REPEAT = 1 -REPEAT = 3 -#REPEAT = 7 - -if __name__ != "__main__": - raise SystemExit("Must run as main program") - -parser = optparse.OptionParser() -parser.add_option("-R", "--skip-re", dest="skip_re", - action="store_true", - help="skip regular expression tests") -parser.add_option("-8", "--8-bit", dest="bytes_only", - action="store_true", - help="only do 8-bit string benchmarks") -parser.add_option("-u", "--unicode", dest="unicode_only", - action="store_true", - help="only do Unicode string benchmarks") - - -_RANGE_1000 = list(range(1000)) -_RANGE_100 = list(range(100)) -_RANGE_10 = list(range(10)) - -dups = {} -def bench(s, group, repeat_count): - def blah(f): - if f.__name__ in dups: - raise AssertionError("Multiple functions with same name: %r" % - (f.__name__,)) - dups[f.__name__] = 1 - f.comment = s - f.is_bench = True - f.group = group - f.repeat_count = repeat_count - return f - return blah - -def uses_re(f): - f.uses_re = True - -####### 'in' comparisons - -@bench('"A" in "A"*1000', "early match, single character", 1000) -def in_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - for x in _RANGE_1000: - s2 in s1 - -@bench('"B" in "A"*1000', "no match, single character", 1000) -def in_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - for x in _RANGE_1000: - s2 in s1 - - -@bench('"AB" in "AB"*1000', "early match, two characters", 1000) -def in_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - for x in _RANGE_1000: - s2 in s1 - -@bench('"BC" in "AB"*1000', "no match, two characters", 1000) -def in_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - for x in _RANGE_1000: - s2 in s1 - -@bench('"BC" in ("AB"*300+"C")', "late match, two characters", 1000) -def in_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - for x in _RANGE_1000: - s2 in s1 - -@bench('s="ABC"*33; (s+"E") in ((s+"D")*300+s+"E")', - "late match, 100 characters", 100) -def in_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*300 + m+e - s2 = m+e - for x in _RANGE_100: - s2 in s1 - -# Try with regex -@uses_re -@bench('s="ABC"*33; re.compile(s+"D").search((s+"D")*300+s+"E")', - "late match, 100 characters", 100) -def re_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*300 + m+e - s2 = m+e - pat = re.compile(s2) - search = pat.search - for x in _RANGE_100: - search(s1) - - -#### same tests as 'in' but use 'find' - -@bench('("A"*1000).find("A")', "early match, single character", 1000) -def find_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("A"*1000).find("B")', "no match, single character", 1000) -def find_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - - -@bench('("AB"*1000).find("AB")', "early match, two characters", 1000) -def find_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*1000).find("BC")', "no match, two characters", 1000) -def find_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*1000).find("CA")', "no match, two characters", 1000) -def find_test_no_match_two_character_bis(STR): - s1 = STR("AB" * 1000) - s2 = STR("CA") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*300+"C").find("BC")', "late match, two characters", 1000) -def find_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('("AB"*300+"CA").find("CA")', "late match, two characters", 1000) -def find_test_slow_match_two_characters_bis(STR): - s1 = STR("AB" * 300+"CA") - s2 = STR("CA") - s1_find = s1.find - for x in _RANGE_1000: - s1_find(s2) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").find(s+"E")', - "late match, 100 characters", 100) -def find_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_find = s1.find - for x in _RANGE_100: - s1_find(s2) - -@bench('s="ABC"*33; ((s+"D")*500+"E"+s).find("E"+s)', - "late match, 100 characters", 100) -def find_test_slow_match_100_characters_bis(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + e+m - s2 = e+m - s1_find = s1.find - for x in _RANGE_100: - s1_find(s2) - - -#### Same tests for 'rfind' - -@bench('("A"*1000).rfind("A")', "early match, single character", 1000) -def rfind_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("A"*1000).rfind("B")', "no match, single character", 1000) -def rfind_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - - -@bench('("AB"*1000).rfind("AB")', "early match, two characters", 1000) -def rfind_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("AB"*1000).rfind("BC")', "no match, two characters", 1000) -def rfind_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("AB"*1000).rfind("CA")', "no match, two characters", 1000) -def rfind_test_no_match_two_character_bis(STR): - s1 = STR("AB" * 1000) - s2 = STR("CA") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("C"+"AB"*300).rfind("CA")', "late match, two characters", 1000) -def rfind_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('("BC"+"AB"*300).rfind("BC")', "late match, two characters", 1000) -def rfind_test_slow_match_two_characters_bis(STR): - s1 = STR("BC" + "AB" * 300) - s2 = STR("BC") - s1_rfind = s1.rfind - for x in _RANGE_1000: - s1_rfind(s2) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rfind("E"+s)', - "late match, 100 characters", 100) -def rfind_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e+m + (d+m)*500 - s2 = e+m - s1_rfind = s1.rfind - for x in _RANGE_100: - s1_rfind(s2) - -@bench('s="ABC"*33; (s+"E"+("D"+s)*500).rfind(s+"E")', - "late match, 100 characters", 100) -def rfind_test_slow_match_100_characters_bis(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = m+e + (d+m)*500 - s2 = m+e - s1_rfind = s1.rfind - for x in _RANGE_100: - s1_rfind(s2) - - -#### Now with index. -# Skip the ones which fail because that would include exception overhead. - -@bench('("A"*1000).index("A")', "early match, single character", 1000) -def index_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_index = s1.index - for x in _RANGE_1000: - s1_index(s2) - -@bench('("AB"*1000).index("AB")', "early match, two characters", 1000) -def index_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_index = s1.index - for x in _RANGE_1000: - s1_index(s2) - -@bench('("AB"*300+"C").index("BC")', "late match, two characters", 1000) -def index_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_index = s1.index - for x in _RANGE_1000: - s1_index(s2) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").index(s+"E")', - "late match, 100 characters", 100) -def index_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_index = s1.index - for x in _RANGE_100: - s1_index(s2) - - -#### Same for rindex - -@bench('("A"*1000).rindex("A")', "early match, single character", 1000) -def rindex_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rindex = s1.rindex - for x in _RANGE_1000: - s1_rindex(s2) - -@bench('("AB"*1000).rindex("AB")', "early match, two characters", 1000) -def rindex_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rindex = s1.rindex - for x in _RANGE_1000: - s1_rindex(s2) - -@bench('("C"+"AB"*300).rindex("CA")', "late match, two characters", 1000) -def rindex_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rindex = s1.rindex - for x in _RANGE_1000: - s1_rindex(s2) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rindex("E"+s)', - "late match, 100 characters", 100) -def rindex_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e + m + (d+m)*500 - s2 = e + m - s1_rindex = s1.rindex - for x in _RANGE_100: - s1_rindex(s2) - - -#### Same for partition - -@bench('("A"*1000).partition("A")', "early match, single character", 1000) -def partition_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('("A"*1000).partition("B")', "no match, single character", 1000) -def partition_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - - -@bench('("AB"*1000).partition("AB")', "early match, two characters", 1000) -def partition_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('("AB"*1000).partition("BC")', "no match, two characters", 1000) -def partition_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('("AB"*300+"C").partition("BC")', "late match, two characters", 1000) -def partition_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_partition = s1.partition - for x in _RANGE_1000: - s1_partition(s2) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").partition(s+"E")', - "late match, 100 characters", 100) -def partition_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_partition = s1.partition - for x in _RANGE_100: - s1_partition(s2) - - -#### Same for rpartition - -@bench('("A"*1000).rpartition("A")', "early match, single character", 1000) -def rpartition_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('("A"*1000).rpartition("B")', "no match, single character", 1000) -def rpartition_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - - -@bench('("AB"*1000).rpartition("AB")', "early match, two characters", 1000) -def rpartition_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('("AB"*1000).rpartition("BC")', "no match, two characters", 1000) -def rpartition_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('("C"+"AB"*300).rpartition("CA")', "late match, two characters", 1000) -def rpartition_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rpartition = s1.rpartition - for x in _RANGE_1000: - s1_rpartition(s2) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rpartition("E"+s)', - "late match, 100 characters", 100) -def rpartition_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e + m + (d+m)*500 - s2 = e + m - s1_rpartition = s1.rpartition - for x in _RANGE_100: - s1_rpartition(s2) - - -#### Same for split(s, 1) - -@bench('("A"*1000).split("A", 1)', "early match, single character", 1000) -def split_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('("A"*1000).split("B", 1)', "no match, single character", 1000) -def split_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - - -@bench('("AB"*1000).split("AB", 1)', "early match, two characters", 1000) -def split_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('("AB"*1000).split("BC", 1)', "no match, two characters", 1000) -def split_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('("AB"*300+"C").split("BC", 1)', "late match, two characters", 1000) -def split_test_slow_match_two_characters(STR): - s1 = STR("AB" * 300+"C") - s2 = STR("BC") - s1_split = s1.split - for x in _RANGE_1000: - s1_split(s2, 1) - -@bench('s="ABC"*33; ((s+"D")*500+s+"E").split(s+"E", 1)', - "late match, 100 characters", 100) -def split_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = (m+d)*500 + m+e - s2 = m+e - s1_split = s1.split - for x in _RANGE_100: - s1_split(s2, 1) - - -#### Same for rsplit(s, 1) - -@bench('("A"*1000).rsplit("A", 1)', "early match, single character", 1000) -def rsplit_test_quick_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("A") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('("A"*1000).rsplit("B", 1)', "no match, single character", 1000) -def rsplit_test_no_match_single_character(STR): - s1 = STR("A" * 1000) - s2 = STR("B") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - - -@bench('("AB"*1000).rsplit("AB", 1)', "early match, two characters", 1000) -def rsplit_test_quick_match_two_characters(STR): - s1 = STR("AB" * 1000) - s2 = STR("AB") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('("AB"*1000).rsplit("BC", 1)', "no match, two characters", 1000) -def rsplit_test_no_match_two_character(STR): - s1 = STR("AB" * 1000) - s2 = STR("BC") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('("C"+"AB"*300).rsplit("CA", 1)', "late match, two characters", 1000) -def rsplit_test_slow_match_two_characters(STR): - s1 = STR("C" + "AB" * 300) - s2 = STR("CA") - s1_rsplit = s1.rsplit - for x in _RANGE_1000: - s1_rsplit(s2, 1) - -@bench('s="ABC"*33; ("E"+s+("D"+s)*500).rsplit("E"+s, 1)', - "late match, 100 characters", 100) -def rsplit_test_slow_match_100_characters(STR): - m = STR("ABC"*33) - d = STR("D") - e = STR("E") - s1 = e + m + (d+m)*500 - s2 = e + m - s1_rsplit = s1.rsplit - for x in _RANGE_100: - s1_rsplit(s2, 1) - - -#### Benchmark the operator-based methods - -@bench('"A"*10', "repeat 1 character 10 times", 1000) -def repeat_single_10_times(STR): - s = STR("A") - for x in _RANGE_1000: - s * 10 - -@bench('"A"*1000', "repeat 1 character 1000 times", 1000) -def repeat_single_1000_times(STR): - s = STR("A") - for x in _RANGE_1000: - s * 1000 - -@bench('"ABCDE"*10', "repeat 5 characters 10 times", 1000) -def repeat_5_10_times(STR): - s = STR("ABCDE") - for x in _RANGE_1000: - s * 10 - -@bench('"ABCDE"*1000', "repeat 5 characters 1000 times", 1000) -def repeat_5_1000_times(STR): - s = STR("ABCDE") - for x in _RANGE_1000: - s * 1000 - -# + for concat - -@bench('"Andrew"+"Dalke"', "concat two strings", 1000) -def concat_two_strings(STR): - s1 = STR("Andrew") - s2 = STR("Dalke") - for x in _RANGE_1000: - s1+s2 - -@bench('s1+s2+s3+s4+...+s20', "concat 20 strings of words length 4 to 15", - 1000) -def concat_many_strings(STR): - s1=STR('TIXSGYNREDCVBHJ') - s2=STR('PUMTLXBZVDO') - s3=STR('FVZNJ') - s4=STR('OGDXUW') - s5=STR('WEIMRNCOYVGHKB') - s6=STR('FCQTNMXPUZH') - s7=STR('TICZJYRLBNVUEAK') - s8=STR('REYB') - s9=STR('PWUOQ') - s10=STR('EQHCMKBS') - s11=STR('AEVDFOH') - s12=STR('IFHVD') - s13=STR('JGTCNLXWOHQ') - s14=STR('ITSKEPYLROZAWXF') - s15=STR('THEK') - s16=STR('GHPZFBUYCKMNJIT') - s17=STR('JMUZ') - s18=STR('WLZQMTB') - s19=STR('KPADCBW') - s20=STR('TNJHZQAGBU') - for x in _RANGE_1000: - (s1 + s2+ s3+ s4+ s5+ s6+ s7+ s8+ s9+s10+ - s11+s12+s13+s14+s15+s16+s17+s18+s19+s20) - - -#### Benchmark join - -def get_bytes_yielding_seq(STR, arg): - if STR is BYTES and sys.version_info >= (3,): - raise UnsupportedType - return STR(arg) - -@bench('"A".join("")', - "join empty string, with 1 character sep", 100) -def join_empty_single(STR): - sep = STR("A") - s2 = get_bytes_yielding_seq(STR, "") - sep_join = sep.join - for x in _RANGE_100: - sep_join(s2) - -@bench('"ABCDE".join("")', - "join empty string, with 5 character sep", 100) -def join_empty_5(STR): - sep = STR("ABCDE") - s2 = get_bytes_yielding_seq(STR, "") - sep_join = sep.join - for x in _RANGE_100: - sep_join(s2) - -@bench('"A".join("ABC..Z")', - "join string with 26 characters, with 1 character sep", 1000) -def join_alphabet_single(STR): - sep = STR("A") - s2 = get_bytes_yielding_seq(STR, "ABCDEFGHIJKLMnOPQRSTUVWXYZ") - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"ABCDE".join("ABC..Z")', - "join string with 26 characters, with 5 character sep", 1000) -def join_alphabet_5(STR): - sep = STR("ABCDE") - s2 = get_bytes_yielding_seq(STR, "ABCDEFGHIJKLMnOPQRSTUVWXYZ") - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"A".join(list("ABC..Z"))', - "join list of 26 characters, with 1 character sep", 1000) -def join_alphabet_list_single(STR): - sep = STR("A") - s2 = [STR(x) for x in "ABCDEFGHIJKLMnOPQRSTUVWXYZ"] - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"ABCDE".join(list("ABC..Z"))', - "join list of 26 characters, with 5 character sep", 1000) -def join_alphabet_list_five(STR): - sep = STR("ABCDE") - s2 = [STR(x) for x in "ABCDEFGHIJKLMnOPQRSTUVWXYZ"] - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"A".join(["Bob"]*100)', - "join list of 100 words, with 1 character sep", 1000) -def join_100_words_single(STR): - sep = STR("A") - s2 = [STR("Bob")]*100 - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -@bench('"ABCDE".join(["Bob"]*100))', - "join list of 100 words, with 5 character sep", 1000) -def join_100_words_5(STR): - sep = STR("ABCDE") - s2 = [STR("Bob")]*100 - sep_join = sep.join - for x in _RANGE_1000: - sep_join(s2) - -#### split tests - -@bench('("Here are some words. "*2).split()', "split whitespace (small)", 1000) -def whitespace_split(STR): - s = STR("Here are some words. "*2) - s_split = s.split - for x in _RANGE_1000: - s_split() - -@bench('("Here are some words. "*2).rsplit()', "split whitespace (small)", 1000) -def whitespace_rsplit(STR): - s = STR("Here are some words. "*2) - s_rsplit = s.rsplit - for x in _RANGE_1000: - s_rsplit() - -@bench('("Here are some words. "*2).split(None, 1)', - "split 1 whitespace", 1000) -def whitespace_split_1(STR): - s = STR("Here are some words. "*2) - s_split = s.split - N = None - for x in _RANGE_1000: - s_split(N, 1) - -@bench('("Here are some words. "*2).rsplit(None, 1)', - "split 1 whitespace", 1000) -def whitespace_rsplit_1(STR): - s = STR("Here are some words. "*2) - s_rsplit = s.rsplit - N = None - for x in _RANGE_1000: - s_rsplit(N, 1) - -@bench('("Here are some words. "*2).partition(" ")', - "split 1 whitespace", 1000) -def whitespace_partition(STR): - sep = STR(" ") - s = STR("Here are some words. "*2) - s_partition = s.partition - for x in _RANGE_1000: - s_partition(sep) - -@bench('("Here are some words. "*2).rpartition(" ")', - "split 1 whitespace", 1000) -def whitespace_rpartition(STR): - sep = STR(" ") - s = STR("Here are some words. "*2) - s_rpartition = s.rpartition - for x in _RANGE_1000: - s_rpartition(sep) - -human_text = """\ -Python is a dynamic object-oriented programming language that can be -used for many kinds of software development. It offers strong support -for integration with other languages and tools, comes with extensive -standard libraries, and can be learned in a few days. Many Python -programmers report substantial productivity gains and feel the language -encourages the development of higher quality, more maintainable code. - -Python runs on Windows, Linux/Unix, Mac OS X, Amiga, Palm -Handhelds, and Nokia mobile phones. Python has also been ported to the -Java and .NET virtual machines. - -Python is distributed under an OSI-approved open source license that -makes it free to use, even for commercial products. -"""*25 -human_text_bytes = bytes_from_str(human_text) -human_text_unicode = unicode_from_str(human_text) -def _get_human_text(STR): - if STR is UNICODE: - return human_text_unicode - if STR is BYTES: - return human_text_bytes - raise AssertionError - -@bench('human_text.split()', "split whitespace (huge)", 10) -def whitespace_split_huge(STR): - s = _get_human_text(STR) - s_split = s.split - for x in _RANGE_10: - s_split() - -@bench('human_text.rsplit()', "split whitespace (huge)", 10) -def whitespace_rsplit_huge(STR): - s = _get_human_text(STR) - s_rsplit = s.rsplit - for x in _RANGE_10: - s_rsplit() - - - -@bench('"this\\nis\\na\\ntest\\n".split("\\n")', "split newlines", 1000) -def newlines_split(STR): - s = STR("this\nis\na\ntest\n") - s_split = s.split - nl = STR("\n") - for x in _RANGE_1000: - s_split(nl) - - -@bench('"this\\nis\\na\\ntest\\n".rsplit("\\n")', "split newlines", 1000) -def newlines_rsplit(STR): - s = STR("this\nis\na\ntest\n") - s_rsplit = s.rsplit - nl = STR("\n") - for x in _RANGE_1000: - s_rsplit(nl) - -@bench('"this\\nis\\na\\ntest\\n".splitlines()', "split newlines", 1000) -def newlines_splitlines(STR): - s = STR("this\nis\na\ntest\n") - s_splitlines = s.splitlines - for x in _RANGE_1000: - s_splitlines() - -## split text with 2000 newlines - -def _make_2000_lines(): - import random - r = random.Random(100) - chars = list(map(chr, range(32, 128))) - i = 0 - while i < len(chars): - chars[i] = " " - i += r.randrange(9) - s = "".join(chars) - s = s*4 - words = [] - for i in range(2000): - start = r.randrange(96) - n = r.randint(5, 65) - words.append(s[start:start+n]) - return "\n".join(words)+"\n" - -_text_with_2000_lines = _make_2000_lines() -_text_with_2000_lines_bytes = bytes_from_str(_text_with_2000_lines) -_text_with_2000_lines_unicode = unicode_from_str(_text_with_2000_lines) -def _get_2000_lines(STR): - if STR is UNICODE: - return _text_with_2000_lines_unicode - if STR is BYTES: - return _text_with_2000_lines_bytes - raise AssertionError - - -@bench('"...text...".split("\\n")', "split 2000 newlines", 10) -def newlines_split_2000(STR): - s = _get_2000_lines(STR) - s_split = s.split - nl = STR("\n") - for x in _RANGE_10: - s_split(nl) - -@bench('"...text...".rsplit("\\n")', "split 2000 newlines", 10) -def newlines_rsplit_2000(STR): - s = _get_2000_lines(STR) - s_rsplit = s.rsplit - nl = STR("\n") - for x in _RANGE_10: - s_rsplit(nl) - -@bench('"...text...".splitlines()', "split 2000 newlines", 10) -def newlines_splitlines_2000(STR): - s = _get_2000_lines(STR) - s_splitlines = s.splitlines - for x in _RANGE_10: - s_splitlines() - - -## split text on "--" characters -@bench( - '"this--is--a--test--of--the--emergency--broadcast--system".split("--")', - "split on multicharacter separator (small)", 1000) -def split_multichar_sep_small(STR): - s = STR("this--is--a--test--of--the--emergency--broadcast--system") - s_split = s.split - pat = STR("--") - for x in _RANGE_1000: - s_split(pat) -@bench( - '"this--is--a--test--of--the--emergency--broadcast--system".rsplit("--")', - "split on multicharacter separator (small)", 1000) -def rsplit_multichar_sep_small(STR): - s = STR("this--is--a--test--of--the--emergency--broadcast--system") - s_rsplit = s.rsplit - pat = STR("--") - for x in _RANGE_1000: - s_rsplit(pat) - -## split dna text on "ACTAT" characters -@bench('dna.split("ACTAT")', - "split on multicharacter separator (dna)", 10) -def split_multichar_sep_dna(STR): - s = _get_dna(STR) - s_split = s.split - pat = STR("ACTAT") - for x in _RANGE_10: - s_split(pat) - -@bench('dna.rsplit("ACTAT")', - "split on multicharacter separator (dna)", 10) -def rsplit_multichar_sep_dna(STR): - s = _get_dna(STR) - s_rsplit = s.rsplit - pat = STR("ACTAT") - for x in _RANGE_10: - s_rsplit(pat) - - - -## split with limits - -GFF3_example = "\t".join([ - "I", "Genomic_canonical", "region", "357208", "396183", ".", "+", ".", - "ID=Sequence:R119;note=Clone R119%3B Genbank AF063007;Name=R119"]) - -@bench('GFF3_example.split("\\t")', "tab split", 1000) -def tab_split_no_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_split = s.split - for x in _RANGE_1000: - s_split(sep) - -@bench('GFF3_example.split("\\t", 8)', "tab split", 1000) -def tab_split_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_split = s.split - for x in _RANGE_1000: - s_split(sep, 8) - -@bench('GFF3_example.rsplit("\\t")', "tab split", 1000) -def tab_rsplit_no_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_rsplit = s.rsplit - for x in _RANGE_1000: - s_rsplit(sep) - -@bench('GFF3_example.rsplit("\\t", 8)', "tab split", 1000) -def tab_rsplit_limit(STR): - sep = STR("\t") - s = STR(GFF3_example) - s_rsplit = s.rsplit - for x in _RANGE_1000: - s_rsplit(sep, 8) - -#### Count characters - -@bench('...text.with.2000.newlines.count("\\n")', - "count newlines", 10) -def count_newlines(STR): - s = _get_2000_lines(STR) - s_count = s.count - nl = STR("\n") - for x in _RANGE_10: - s_count(nl) - -# Orchid sequences concatenated, from Biopython -_dna = """ -CGTAACAAGGTTTCCGTAGGTGAACCTGCGGAAGGATCATTGTTGAGATCACATAATAATTGATCGGGTT -AATCTGGAGGATCTGTTTACTTTGGTCACCCATGAGCATTTGCTGTTGAAGTGACCTAGAATTGCCATCG -AGCCTCCTTGGGAGCTTTCTTGTTGGCGAGATCTAAACCCTTGCCCGGCGCAGTTTTGCTCCAAGTCGTT -TGACACATAATTGGTGAAGGGGGTGGCATCCTTCCCTGACCCTCCCCCAACTATTTTTTTAACAACTCTC -AGCAACGGAGACTCAGTCTTCGGCAAATGCGATAAATGGTGTGAATTGCAGAATCCCGTGCACCATCGAG -TCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACGCCTGCCTGGGCATTGCGAGTCATAT -CTCTCCCTTAACGAGGCTGTCCATACATACTGTTCAGCCGGTGCGGATGTGAGTTTGGCCCCTTGTTCTT -TGGTACGGGGGGTCTAAGAGCTGCATGGGCTTTTGATGGTCCTAAATACGGCAAGAGGTGGACGAACTAT -GCTACAACAAAATTGTTGTGCAGAGGCCCCGGGTTGTCGTATTAGATGGGCCACCGTAATCTGAAGACCC -TTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATGGCCATTTGGTTGCGACCCCAGGTCAG -GTGAGCAACAGCTGTCGTAACAAGGTTTCCGTAGGGTGAACTGCGGAAGGATCATTGTTGAGATCACATA -ATAATTGATCGAGTTAATCTGGAGGATCTGTTTACTTGGGTCACCCATGGGCATTTGCTGTTGAAGTGAC -CTAGATTTGCCATCGAGCCTCCTTGGGAGCATCCTTGTTGGCGATATCTAAACCCTCAATTTTTCCCCCA -ATCAAATTACACAAAATTGGTGGAGGGGGTGGCATTCTTCCCTTACCCTCCCCCAAATATTTTTTTAACA -ACTCTCAGCAACGGATATCTCAGCTCTTGCATCGATGAAGAACCCACCGAAATGCGATAAATGGTGTGAA -TTGCAGAATCCCGTGAACCATCGAGTCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACG -CCTGCCTGGGCATTGCGAGTCATATCTCTCCCTTAACGAGGCTGTCCATACATACTGTTCAGCCGGTGCG -GATGTGAGTTTGGCCCCTTGTTCTTTGGTACGGGGGGTCTAAGAGATGCATGGGCTTTTGATGGTCCTAA -ATACGGCAAGAGGTGGACGAACTATGCTACAACAAAATTGTTGTGCAAAGGCCCCGGGTTGTCGTATAAG -ATGGGCCACCGATATCTGAAGACCCTTTTGGACCCCATTGGAGCCCATCAACCCATGTCAGTTGATGGCC -ATTCGTAACAAGGTTTCCGTAGGTGAACCTGCGGAAGGATCATTGTTGAGATCACATAATAATTGATCGA -GTTAATCTGGAGGATCTGTTTACTTGGGTCACCCATGGGCATTTGCTGTTGAAGTGACCTAGATTTGCCA -TCGAGCCTCCTTGGGAGCTTTCTTGTTGGCGATATCTAAACCCTTGCCCGGCAGAGTTTTGGGAATCCCG -TGAACCATCGAGTCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACGCCTGCCTGGGCAT -TGCGAGTCATATCTCTCCCTTAACGAGGCTGTCCATACACACCTGTTCAGCCGGTGCGGATGTGAGTTTG -GCCCCTTGTTCTTTGGTACGGGGGGTCTAAGAGCTGCATGGGCTTTTGATGGTCCTAAATACGGCAAGAG -GTGGACGAACTATGCTACAACAAAATTGTTGTGCAAAGGCCCCGGGTTGTCGTATTAGATGGGCCACCAT -AATCTGAAGACCCTTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATGGCCATTTGGTTGC -GACCCAGTCAGGTGAGGGTAGGTGAACCTGCGGAAGGATCATTGTTGAGATCACATAATAATTGATCGAG -TTAATCTGGAGGATCTGTTTACTTTGGTCACCCATGGGCATTTGCTGTTGAAGTGACCTAGATTTGCCAT -CGAGCCTCCTTGGGAGCTTTCTTGTTGGCGAGATCTAAACCCTTGCCCGGCGGAGTTTGGCGCCAAGTCA -TATGACACATAATTGGTGAAGGGGGTGGCATCCTGCCCTGACCCTCCCCAAATTATTTTTTTAACAACTC -TCAGCAACGGATATCTCGGCTCTTGCATCGATGAAGAACGCAGCGAAATGCGATAAATGGTGTGAATTGC -AGAATCCCGTGAACCATCGAGTCTTTGGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCACGCCT -GCCTGGGCATTGGGAATCATATCTCTCCCCTAACGAGGCTATCCAAACATACTGTTCATCCGGTGCGGAT -GTGAGTTTGGCCCCTTGTTCTTTGGTACCGGGGGTCTAAGAGCTGCATGGGCATTTGATGGTCCTCAAAA -CGGCAAGAGGTGGACGAACTATGCCACAACAAAATTGTTGTCCCAAGGCCCCGGGTTGTCGTATTAGATG -GGCCACCGTAACCTGAAGACCCTTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATGACCA -TTTGTTGCGACCCCAGTCAGCTGAGCAACCCGCTGAGTGGAAGGTCATTGCCGATATCACATAATAATTG -ATCGAGTTAATCTGGAGGATCTGTTTACTTGGTCACCCATGAGCATTTGCTGTTGAAGTGACCTAGATTT -GCCATCGAGCCTCCTTGGGAGTTTTCTTGTTGGCGAGATCTAAACCCTTGCCCGGCGGAGTTGTGCGCCA -AGTCATATGACACATAATTGGTGAAGGGGGTGGCATCCTGCCCTGACCCTCCCCAAATTATTTTTTTAAC -AACTCTCAGCAACGGATATCTCGGCTCTTGCATCGATGAAGAACGCAGCGAAATGCGATAAATGGTGTGA -ATTGCAGAATCCCGTGAACCATCGAGTCTTTGAACGCAAGTTGCGCCCGAGGCCATCAGGCCAAGGGCAC -GCCTGCCTGGGCATTGCGAGTCATATCTCTCCCTTAACGAGGCTGTCCATACATACTGTTCATCCGGTGC -GGATGTGAGTTTGGCCCCTTGTTCTTTGGTACGGGGGGTCTAAGAGCTGCATGGGCATTTGATGGTCCTC -AAAACGGCAAGAGGTGGACGAACTATGCTACAACCAAATTGTTGTCCCAAGGCCCCGGGTTGTCGTATTA -GATGGGCCACCGTAACCTGAAGACCCTTTTGAACCCCATTGGAGGCCCATCAACCCATGATCAGTTGATG -ACCATGTGTTGCGACCCCAGTCAGCTGAGCAACGCGCTGAGCGTAACAAGGTTTCCGTAGGTGGACCTCC -GGGAGGATCATTGTTGAGATCACATAATAATTGATCGAGGTAATCTGGAGGATCTGCATATTTTGGTCAC -""" -_dna = "".join(_dna.splitlines()) -_dna = _dna * 25 -_dna_bytes = bytes_from_str(_dna) -_dna_unicode = unicode_from_str(_dna) - -def _get_dna(STR): - if STR is UNICODE: - return _dna_unicode - if STR is BYTES: - return _dna_bytes - raise AssertionError - -@bench('dna.count("AACT")', "count AACT substrings in DNA example", 10) -def count_aact(STR): - seq = _get_dna(STR) - seq_count = seq.count - needle = STR("AACT") - for x in _RANGE_10: - seq_count(needle) - -##### startswith and endswith - -@bench('"Andrew".startswith("A")', 'startswith single character', 1000) -def startswith_single(STR): - s1 = STR("Andrew") - s2 = STR("A") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - -@bench('"Andrew".startswith("Andrew")', 'startswith multiple characters', - 1000) -def startswith_multiple(STR): - s1 = STR("Andrew") - s2 = STR("Andrew") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - -@bench('"Andrew".startswith("Anders")', - 'startswith multiple characters - not!', 1000) -def startswith_multiple_not(STR): - s1 = STR("Andrew") - s2 = STR("Anders") - s1_startswith = s1.startswith - for x in _RANGE_1000: - s1_startswith(s2) - - -# endswith - -@bench('"Andrew".endswith("w")', 'endswith single character', 1000) -def endswith_single(STR): - s1 = STR("Andrew") - s2 = STR("w") - s1_endswith = s1.endswith - for x in _RANGE_1000: - s1_endswith(s2) - -@bench('"Andrew".endswith("Andrew")', 'endswith multiple characters', 1000) -def endswith_multiple(STR): - s1 = STR("Andrew") - s2 = STR("Andrew") - s1_endswith = s1.endswith - for x in _RANGE_1000: - s1_endswith(s2) - -@bench('"Andrew".endswith("Anders")', - 'endswith multiple characters - not!', 1000) -def endswith_multiple_not(STR): - s1 = STR("Andrew") - s2 = STR("Anders") - s1_endswith = s1.endswith - for x in _RANGE_1000: - s1_endswith(s2) - -#### Strip - -@bench('"Hello!\\n".strip()', 'strip terminal newline', 1000) -def terminal_newline_strip_right(STR): - s = STR("Hello!\n") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"Hello!\\n".rstrip()', 'strip terminal newline', 1000) -def terminal_newline_rstrip(STR): - s = STR("Hello!\n") - s_rstrip = s.rstrip - for x in _RANGE_1000: - s_rstrip() - -@bench('"\\nHello!".strip()', 'strip terminal newline', 1000) -def terminal_newline_strip_left(STR): - s = STR("\nHello!") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"\\nHello!\\n".strip()', 'strip terminal newline', 1000) -def terminal_newline_strip_both(STR): - s = STR("\nHello!\n") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"\\nHello!".rstrip()', 'strip terminal newline', 1000) -def terminal_newline_lstrip(STR): - s = STR("\nHello!") - s_lstrip = s.lstrip - for x in _RANGE_1000: - s_lstrip() - -@bench('s="Hello!\\n"; s[:-1] if s[-1]=="\\n" else s', - 'strip terminal newline', 1000) -def terminal_newline_if_else(STR): - s = STR("Hello!\n") - NL = STR("\n") - for x in _RANGE_1000: - s[:-1] if (s[-1] == NL) else s - - -# Strip multiple spaces or tabs - -@bench('"Hello\\t \\t".strip()', 'strip terminal spaces and tabs', 1000) -def terminal_space_strip(STR): - s = STR("Hello\t \t!") - s_strip = s.strip - for x in _RANGE_1000: - s_strip() - -@bench('"Hello\\t \\t".rstrip()', 'strip terminal spaces and tabs', 1000) -def terminal_space_rstrip(STR): - s = STR("Hello!\t \t") - s_rstrip = s.rstrip - for x in _RANGE_1000: - s_rstrip() - -@bench('"\\t \\tHello".rstrip()', 'strip terminal spaces and tabs', 1000) -def terminal_space_lstrip(STR): - s = STR("\t \tHello!") - s_lstrip = s.lstrip - for x in _RANGE_1000: - s_lstrip() - - -#### replace -@bench('"This is a test".replace(" ", "\\t")', 'replace single character', - 1000) -def replace_single_character(STR): - s = STR("This is a test!") - from_str = STR(" ") - to_str = STR("\t") - s_replace = s.replace - for x in _RANGE_1000: - s_replace(from_str, to_str) - -@uses_re -@bench('re.sub(" ", "\\t", "This is a test"', 'replace single character', - 1000) -def replace_single_character_re(STR): - s = STR("This is a test!") - pat = re.compile(STR(" ")) - to_str = STR("\t") - pat_sub = pat.sub - for x in _RANGE_1000: - pat_sub(to_str, s) - -@bench('"...text.with.2000.lines...replace("\\n", " ")', - 'replace single character, big string', 10) -def replace_single_character_big(STR): - s = _get_2000_lines(STR) - from_str = STR("\n") - to_str = STR(" ") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str) - -@uses_re -@bench('re.sub("\\n", " ", "...text.with.2000.lines...")', - 'replace single character, big string', 10) -def replace_single_character_big_re(STR): - s = _get_2000_lines(STR) - pat = re.compile(STR("\n")) - to_str = STR(" ") - pat_sub = pat.sub - for x in _RANGE_10: - pat_sub(to_str, s) - - -@bench('dna.replace("ATC", "ATT")', - 'replace multiple characters, dna', 10) -def replace_multiple_characters_dna(STR): - seq = _get_dna(STR) - from_str = STR("ATC") - to_str = STR("ATT") - seq_replace = seq.replace - for x in _RANGE_10: - seq_replace(from_str, to_str) - -# This increases the character count -@bench('"...text.with.2000.newlines...replace("\\n", "\\r\\n")', - 'replace and expand multiple characters, big string', 10) -def replace_multiple_character_big(STR): - s = _get_2000_lines(STR) - from_str = STR("\n") - to_str = STR("\r\n") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str) - - -# This decreases the character count -@bench('"When shall we three meet again?".replace("ee", "")', - 'replace/remove multiple characters', 1000) -def replace_multiple_character_remove(STR): - s = STR("When shall we three meet again?") - from_str = STR("ee") - to_str = STR("") - s_replace = s.replace - for x in _RANGE_1000: - s_replace(from_str, to_str) - - -big_s = "A" + ("Z"*128*1024) -big_s_bytes = bytes_from_str(big_s) -big_s_unicode = unicode_from_str(big_s) -def _get_big_s(STR): - if STR is UNICODE: return big_s_unicode - if STR is BYTES: return big_s_bytes - raise AssertionError - -# The older replace implementation counted all matches in -# the string even when it only needed to make one replacement. -@bench('("A" + ("Z"*128*1024)).replace("A", "BB", 1)', - 'quick replace single character match', 10) -def quick_replace_single_match(STR): - s = _get_big_s(STR) - from_str = STR("A") - to_str = STR("BB") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str, 1) - -@bench('("A" + ("Z"*128*1024)).replace("AZZ", "BBZZ", 1)', - 'quick replace multiple character match', 10) -def quick_replace_multiple_match(STR): - s = _get_big_s(STR) - from_str = STR("AZZ") - to_str = STR("BBZZ") - s_replace = s.replace - for x in _RANGE_10: - s_replace(from_str, to_str, 1) - - -#### - -# CCP does a lot of this, for internationalisation of ingame messages. -_format = "The %(thing)s is %(place)s the %(location)s." -_format_dict = { "thing":"THING", "place":"PLACE", "location":"LOCATION", } -_format_bytes = bytes_from_str(_format) -_format_unicode = unicode_from_str(_format) -_format_dict_bytes = dict((bytes_from_str(k), bytes_from_str(v)) for (k,v) in _format_dict.items()) -_format_dict_unicode = dict((unicode_from_str(k), unicode_from_str(v)) for (k,v) in _format_dict.items()) - -def _get_format(STR): - if STR is UNICODE: - return _format_unicode - if STR is BYTES: - if sys.version_info >= (3,): - raise UnsupportedType - return _format_bytes - raise AssertionError - -def _get_format_dict(STR): - if STR is UNICODE: - return _format_dict_unicode - if STR is BYTES: - if sys.version_info >= (3,): - raise UnsupportedType - return _format_dict_bytes - raise AssertionError - -# Formatting. -@bench('"The %(k1)s is %(k2)s the %(k3)s."%{"k1":"x","k2":"y","k3":"z",}', - 'formatting a string type with a dict', 1000) -def format_with_dict(STR): - s = _get_format(STR) - d = _get_format_dict(STR) - for x in _RANGE_1000: - s % d - - -#### Upper- and lower- case conversion - -@bench('("Where in the world is Carmen San Deigo?"*10).lower()', - "case conversion -- rare", 1000) -def lower_conversion_rare(STR): - s = STR("Where in the world is Carmen San Deigo?"*10) - s_lower = s.lower - for x in _RANGE_1000: - s_lower() - -@bench('("WHERE IN THE WORLD IS CARMEN SAN DEIGO?"*10).lower()', - "case conversion -- dense", 1000) -def lower_conversion_dense(STR): - s = STR("WHERE IN THE WORLD IS CARMEN SAN DEIGO?"*10) - s_lower = s.lower - for x in _RANGE_1000: - s_lower() - - -@bench('("wHERE IN THE WORLD IS cARMEN sAN dEIGO?"*10).upper()', - "case conversion -- rare", 1000) -def upper_conversion_rare(STR): - s = STR("Where in the world is Carmen San Deigo?"*10) - s_upper = s.upper - for x in _RANGE_1000: - s_upper() - -@bench('("where in the world is carmen san deigo?"*10).upper()', - "case conversion -- dense", 1000) -def upper_conversion_dense(STR): - s = STR("where in the world is carmen san deigo?"*10) - s_upper = s.upper - for x in _RANGE_1000: - s_upper() - - -# end of benchmarks - -################# - -class BenchTimer(timeit.Timer): - def best(self, repeat=1): - for i in range(1, 10): - number = 10**i - x = self.timeit(number) - if x > 0.02: - break - times = [x] - for i in range(1, repeat): - times.append(self.timeit(number)) - return min(times) / number - -def main(): - (options, test_names) = parser.parse_args() - if options.bytes_only and options.unicode_only: - raise SystemExit("Only one of --8-bit and --unicode are allowed") - - bench_functions = [] - for (k,v) in globals().items(): - if hasattr(v, "is_bench"): - if test_names: - for name in test_names: - if name in v.group: - break - else: - # Not selected, ignore - continue - if options.skip_re and hasattr(v, "uses_re"): - continue - - bench_functions.append( (v.group, k, v) ) - bench_functions.sort() - - p("bytes\tunicode") - p("(in ms)\t(in ms)\t%\tcomment") - - bytes_total = uni_total = 0.0 - - for title, group in itertools.groupby(bench_functions, - operator.itemgetter(0)): - # Flush buffer before each group - sys.stdout.flush() - p("="*10, title) - for (_, k, v) in group: - if hasattr(v, "is_bench"): - bytes_time = 0.0 - bytes_time_s = " - " - if not options.unicode_only: - try: - bytes_time = BenchTimer("__main__.%s(__main__.BYTES)" % (k,), - "import __main__").best(REPEAT) - bytes_time_s = "%.2f" % (1000 * bytes_time) - bytes_total += bytes_time - except UnsupportedType: - bytes_time_s = "N/A" - uni_time = 0.0 - uni_time_s = " - " - if not options.bytes_only: - try: - uni_time = BenchTimer("__main__.%s(__main__.UNICODE)" % (k,), - "import __main__").best(REPEAT) - uni_time_s = "%.2f" % (1000 * uni_time) - uni_total += uni_time - except UnsupportedType: - uni_time_s = "N/A" - try: - average = bytes_time/uni_time - except (TypeError, ZeroDivisionError): - average = 0.0 - p("%s\t%s\t%.1f\t%s (*%d)" % ( - bytes_time_s, uni_time_s, 100.*average, - v.comment, v.repeat_count)) - - if bytes_total == uni_total == 0.0: - p("That was zippy!") - else: - try: - ratio = bytes_total/uni_total - except ZeroDivisionError: - ratio = 0.0 - p("%.2f\t%.2f\t%.1f\t%s" % ( - 1000*bytes_total, 1000*uni_total, 100.*ratio, - "TOTAL")) - -if __name__ == "__main__": - main() diff --git a/Tools/wasm/config.site-wasm32-wasi b/Tools/wasm/config.site-wasm32-wasi index 5e98775400f6ea..4a1a466a4ab3f1 100644 --- a/Tools/wasm/config.site-wasm32-wasi +++ b/Tools/wasm/config.site-wasm32-wasi @@ -40,3 +40,12 @@ ac_cv_header_netpacket_packet_h=no # Disable int-conversion for wask-sdk as it triggers an error from version 17. ac_cv_disable_int_conversion=yes + +# preadv(), readv(), pwritev(), and writev() under wasmtime's WASI 0.2 support +# do not use more than the first buffer provided, failing under test_posix. +# Since wasmtime will not be changing this behaviour, disable the functions. +# https://github.com/bytecodealliance/wasmtime/issues/7830 +ac_cv_func_preadv=no +ac_cv_func_readv=no +ac_cv_func_pwritev=no +ac_cv_func_writev=no diff --git a/aclocal.m4 b/aclocal.m4 index 09ae5d1aa8a608..832aec19f48f17 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -150,6 +150,80 @@ AS_VAR_IF(CACHEVAR,yes, AS_VAR_POPDEF([CACHEVAR])dnl ])dnl AX_CHECK_COMPILE_FLAGS +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_check_define.html +# =========================================================================== +# +# SYNOPSIS +# +# AC_CHECK_DEFINE([symbol], [ACTION-IF-FOUND], [ACTION-IF-NOT]) +# AX_CHECK_DEFINE([includes],[symbol], [ACTION-IF-FOUND], [ACTION-IF-NOT]) +# +# DESCRIPTION +# +# Complements AC_CHECK_FUNC but it does not check for a function but for a +# define to exist. Consider a usage like: +# +# AC_CHECK_DEFINE(__STRICT_ANSI__, CFLAGS="$CFLAGS -D_XOPEN_SOURCE=500") +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 11 + +AU_ALIAS([AC_CHECK_DEFINED], [AC_CHECK_DEFINE]) +AC_DEFUN([AC_CHECK_DEFINE],[ +AS_VAR_PUSHDEF([ac_var],[ac_cv_defined_$1])dnl +AC_CACHE_CHECK([for $1 defined], ac_var, +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[ + #ifdef $1 + int ok; + (void)ok; + #else + choke me + #endif +]])],[AS_VAR_SET(ac_var, yes)],[AS_VAR_SET(ac_var, no)])) +AS_IF([test AS_VAR_GET(ac_var) != "no"], [$2], [$3])dnl +AS_VAR_POPDEF([ac_var])dnl +]) + +AU_ALIAS([AX_CHECK_DEFINED], [AX_CHECK_DEFINE]) +AC_DEFUN([AX_CHECK_DEFINE],[ +AS_VAR_PUSHDEF([ac_var],[ac_cv_defined_$2_$1])dnl +AC_CACHE_CHECK([for $2 defined in $1], ac_var, +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <$1>]], [[ + #ifdef $2 + int ok; + (void)ok; + #else + choke me + #endif +]])],[AS_VAR_SET(ac_var, yes)],[AS_VAR_SET(ac_var, no)])) +AS_IF([test AS_VAR_GET(ac_var) != "no"], [$3], [$4])dnl +AS_VAR_POPDEF([ac_var])dnl +]) + +AC_DEFUN([AX_CHECK_FUNC], +[AS_VAR_PUSHDEF([ac_var], [ac_cv_func_$2])dnl +AC_CACHE_CHECK([for $2], ac_var, +dnl AC_LANG_FUNC_LINK_TRY +[AC_LINK_IFELSE([AC_LANG_PROGRAM([$1 + #undef $2 + char $2 ();],[ + char (*f) () = $2; + return f != $2; ])], + [AS_VAR_SET(ac_var, yes)], + [AS_VAR_SET(ac_var, no)])]) +AS_IF([test AS_VAR_GET(ac_var) = yes], [$3], [$4])dnl +AS_VAR_POPDEF([ac_var])dnl +])# AC_CHECK_FUNC + # =========================================================================== # https://www.gnu.org/software/autoconf-archive/ax_check_openssl.html # =========================================================================== diff --git a/config.sub b/config.sub index 2c6a07ab3c34ea..1bb6a05dc11026 100755 --- a/config.sub +++ b/config.sub @@ -4,6 +4,7 @@ # shellcheck disable=SC2006,SC2268 # see below for rationale +# Patched 2024-02-03 to include support for arm64_32 and iOS/tvOS/watchOS simulators timestamp='2024-01-01' # This file is free software; you can redistribute it and/or modify it @@ -1127,7 +1128,7 @@ case $cpu-$vendor in xscale-* | xscalee[bl]-*) cpu=`echo "$cpu" | sed 's/^xscale/arm/'` ;; - arm64-* | aarch64le-*) + arm64-* | aarch64le-* | arm64_32-*) cpu=aarch64 ;; @@ -1866,6 +1867,8 @@ case $kernel-$os-$obj in ;; *-eabi*- | *-gnueabi*-) ;; + ios*-simulator- | tvos*-simulator- | watchos*-simulator- ) + ;; none--*) # None (no kernel, i.e. freestanding / bare metal), # can be paired with an machine code file format diff --git a/configure b/configure index ba2d49df7c65fe..4a980fea453697 100755 --- a/configure +++ b/configure @@ -661,6 +661,8 @@ MODULE__XXTESTFUZZ_FALSE MODULE__XXTESTFUZZ_TRUE MODULE_XXSUBTYPE_FALSE MODULE_XXSUBTYPE_TRUE +MODULE__TESTEXTERNALINSPECTION_FALSE +MODULE__TESTEXTERNALINSPECTION_TRUE MODULE__TESTMULTIPHASE_FALSE MODULE__TESTMULTIPHASE_TRUE MODULE__TESTIMPORTMULTIPLE_FALSE @@ -834,7 +836,8 @@ LIBPL PY_ENABLE_SHARED PLATLIBDIR BINLIBDEST -LIBPYTHON +MODULE_LDFLAGS +MODULE_DEPS_SHARED EXT_SUFFIX ALT_SOABI SOABI @@ -969,6 +972,7 @@ LDFLAGS CFLAGS CC HAS_XCRUN +IOS_DEPLOYMENT_TARGET EXPORT_MACOSX_DEPLOYMENT_TARGET CONFIGURE_MACOSX_DEPLOYMENT_TARGET _PYTHON_HOST_PLATFORM @@ -4028,6 +4032,9 @@ then *-*-cygwin*) ac_sys_system=Cygwin ;; + *-apple-ios*) + ac_sys_system=iOS + ;; *-*-vxworks*) ac_sys_system=VxWorks ;; @@ -4193,10 +4200,18 @@ then : enableval=$enable_framework; case $enableval in yes) + if test "$ac_sys_system" = "iOS"; then + as_fn_error $? "iOS builds must provide an explicit path for --enable-framework" "$LINENO" 5 + fi + enableval=/Library/Frameworks esac case $enableval in no) + if test "$ac_sys_system" = "iOS"; then + as_fn_error $? "iOS builds must use --enable-framework=" "$LINENO" 5 + fi + PYTHONFRAMEWORK= PYTHONFRAMEWORKDIR=no-framework PYTHONFRAMEWORKPREFIX= @@ -4220,11 +4235,11 @@ then : *) PYTHONFRAMEWORKPREFIX="${enableval}" PYTHONFRAMEWORKINSTALLDIR=$PYTHONFRAMEWORKPREFIX/$PYTHONFRAMEWORKDIR - FRAMEWORKINSTALLFIRST="frameworkinstallstructure" - FRAMEWORKALTINSTALLFIRST="frameworkinstallstructure " case $ac_sys_system in #( Darwin) : + FRAMEWORKINSTALLFIRST="frameworkinstallversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallversionedstructure " FRAMEWORKINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools" FRAMEWORKALTINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkaltinstallunixtools" FRAMEWORKPYTHONW="frameworkpythonw" @@ -4286,6 +4301,21 @@ then : ac_config_files="$ac_config_files Mac/Resources/app/Info.plist" + ;; + iOS) : + FRAMEWORKINSTALLFIRST="frameworkinstallunversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallunversionedstructure " + FRAMEWORKINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKALTINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKPYTHONW= + INSTALLTARGETS="libinstall inclinstall sharedinstall" + + prefix=$PYTHONFRAMEWORKPREFIX + PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" + RESSRCDIR=iOS/Resources + + ac_config_files="$ac_config_files iOS/Resources/Info.plist" + ;; *) as_fn_error $? "Unknown platform for framework build" "$LINENO" 5 @@ -4295,6 +4325,10 @@ then : else $as_nop + if test "$ac_sys_system" = "iOS"; then + as_fn_error $? "iOS builds must use --enable-framework=" "$LINENO" 5 + fi + PYTHONFRAMEWORK= PYTHONFRAMEWORKDIR=no-framework PYTHONFRAMEWORKPREFIX= @@ -4352,6 +4386,23 @@ if test "$cross_compiling" = yes; then *-*-cygwin*) _host_ident= ;; + *-apple-ios*) + _host_os=`echo $host | cut -d '-' -f3` + _host_device=`echo $host | cut -d '-' -f4` + _host_device=${_host_device:=os} + + IOS_DEPLOYMENT_TARGET=${_host_os:3} + IOS_DEPLOYMENT_TARGET=${IOS_DEPLOYMENT_TARGET:=12.0} + + case "$host_cpu" in + aarch64) + _host_ident=${IOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} + ;; + *) + _host_ident=${IOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} + ;; + esac + ;; *-*-vxworks*) _host_ident=$host_cpu ;; @@ -4430,6 +4481,9 @@ printf "%s\n" "#define _BSD_SOURCE 1" >>confdefs.h define_xopen_source=no;; Darwin/[12][0-9].*) define_xopen_source=no;; + # On iOS, defining _POSIX_C_SOURCE also disables platform specific features. + iOS/*) + define_xopen_source=no;; # On QNX 6.3.2, defining _XOPEN_SOURCE prevents netdb.h from # defining NI_NUMERICHOST. QNX/6.3.2) @@ -4524,6 +4578,17 @@ case $host in #( ;; esac +case $ac_sys_system in #( + iOS) : + + as_fn_append CFLAGS " -mios-version-min=${IOS_DEPLOYMENT_TARGET}" + as_fn_append LDFLAGS " -mios-version-min=${IOS_DEPLOYMENT_TARGET}" + + ;; #( + *) : + ;; +esac + if test "$ac_sys_system" = "Darwin" then # Extract the first word of "xcrun", so it can be a program name with args. @@ -6786,6 +6851,8 @@ printf %s "checking for multiarch... " >&6; } case $ac_sys_system in #( Darwin*) : MULTIARCH="" ;; #( + iOS) : + MULTIARCH="" ;; #( FreeBSD*) : MULTIARCH="" ;; #( *) : @@ -6806,6 +6873,8 @@ fi printf "%s\n" "$MULTIARCH" >&6; } case $ac_sys_system in #( + iOS) : + SOABI_PLATFORM=`echo "$PLATFORM_TRIPLET" | cut -d '-' -f2` ;; #( *) : SOABI_PLATFORM=$PLATFORM_TRIPLET ;; @@ -6851,6 +6920,10 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=3 ;; #( x86_64-*-freebsd*/clang) : PY_SUPPORT_TIER=3 ;; #( + aarch64-apple-ios*-simulator/clang) : + PY_SUPPORT_TIER=3 ;; #( + aarch64-apple-ios*/clang) : + PY_SUPPORT_TIER=3 ;; #( *) : PY_SUPPORT_TIER=0 ;; @@ -7306,12 +7379,15 @@ printf %s "checking LDLIBRARY... " >&6; } # will find it with a -framework option). For this reason there is an # extra variable BLDLIBRARY against which Python and the extension # modules are linked, BLDLIBRARY. This is normally the same as -# LDLIBRARY, but empty for MacOSX framework builds. +# LDLIBRARY, but empty for MacOSX framework builds. iOS does the same, +# but uses a non-versioned framework layout. if test "$enable_framework" then case $ac_sys_system in Darwin) LDLIBRARY='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)';; + iOS) + LDLIBRARY='$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)';; *) as_fn_error $? "Unknown platform for framework build" "$LINENO" 5;; esac @@ -7330,6 +7406,7 @@ printf "%s\n" "#define Py_ENABLE_SHARED 1" >>confdefs.h case $ac_sys_system in CYGWIN*) LDLIBRARY='libpython$(LDVERSION).dll.a' + BLDLIBRARY='-L. -lpython$(LDVERSION)' DLLLIBRARY='libpython$(LDVERSION).dll' ;; SunOS*) @@ -7346,7 +7423,13 @@ printf "%s\n" "#define Py_ENABLE_SHARED 1" >>confdefs.h LDLIBRARY='libpython$(LDVERSION).so' BLDLIBRARY='-L. -lpython$(LDVERSION)' RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION + + # The Android Gradle plugin will only package libraries whose names end + # with ".so". + if test "$ac_sys_system" != "Linux-android"; then + INSTSONAME="$LDLIBRARY".$SOVERSION + fi + if test "$with_pydebug" != yes then PY3LIBRARY=libpython3.so @@ -7369,6 +7452,9 @@ printf "%s\n" "#define Py_ENABLE_SHARED 1" >>confdefs.h BLDLIBRARY='-L. -lpython$(LDVERSION)' RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} ;; + iOS) + LDLIBRARY='libpython$(LDVERSION).dylib' + ;; AIX*) LDLIBRARY='libpython$(LDVERSION).so' RUNSHARED=LIBPATH=`pwd`${LIBPATH:+:${LIBPATH}} @@ -7384,11 +7470,15 @@ else # shared is disabled ;; esac fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDLIBRARY" >&5 +printf "%s\n" "$LDLIBRARY" >&6; } if test "$cross_compiling" = yes; then RUNSHARED= fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 +printf %s "checking HOSTRUNNER... " >&6; } if test -z "$HOSTRUNNER" then @@ -7572,8 +7662,6 @@ fi esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 -printf %s "checking HOSTRUNNER... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $HOSTRUNNER" >&5 printf "%s\n" "$HOSTRUNNER" >&6; } @@ -7581,9 +7669,6 @@ if test -n "$HOSTRUNNER"; then PYTHON_FOR_BUILD="_PYTHON_HOSTRUNNER='$HOSTRUNNER' $PYTHON_FOR_BUILD" fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDLIBRARY" >&5 -printf "%s\n" "$LDLIBRARY" >&6; } - # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable case $ac_sys_system/$ac_sys_emscripten_target in #( Emscripten/browser*) : @@ -11218,42 +11303,22 @@ then : fi -# checks for typedefs - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_t in time.h" >&5 -printf %s "checking for clock_t in time.h... " >&6; } -if test ${ac_cv_clock_t_time_h+y} -then : - printf %s "(cached) " >&6 -else $as_nop - - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include - -_ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "clock_t" >/dev/null 2>&1 +# Check for clock_t in time.h. +ac_fn_c_check_type "$LINENO" "clock_t" "ac_cv_type_clock_t" "#include +" +if test "x$ac_cv_type_clock_t" = xyes then : - ac_cv_clock_t_time_h=yes -else $as_nop - ac_cv_clock_t_time_h=no -fi -rm -rf conftest* +printf "%s\n" "#define HAVE_CLOCK_T 1" >>confdefs.h -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_clock_t_time_h" >&5 -printf "%s\n" "$ac_cv_clock_t_time_h" >&6; } -if test "x$ac_cv_clock_t_time_h" = xno -then : +else $as_nop printf "%s\n" "#define clock_t long" >>confdefs.h - fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for makedev" >&5 printf %s "checking for makedev... " >&6; } if test ${ac_cv_func_makedev+y} @@ -11455,6 +11520,7 @@ printf "%s\n" "#define size_t unsigned int" >>confdefs.h fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for uid_t in sys/types.h" >&5 printf %s "checking for uid_t in sys/types.h... " >&6; } if test ${ac_cv_type_uid_t+y} @@ -11494,12 +11560,16 @@ then : printf "%s\n" "#define HAVE_SSIZE_T 1" >>confdefs.h + fi ac_fn_c_check_type "$LINENO" "__uint128_t" "ac_cv_type___uint128_t" "$ac_includes_default" if test "x$ac_cv_type___uint128_t" = xyes then : +printf "%s\n" "#define HAVE___UINT128_T 1" >>confdefs.h + + printf "%s\n" "#define HAVE_GCC_UINT128_T 1" >>confdefs.h fi @@ -12619,6 +12689,7 @@ if test -z "$SHLIB_SUFFIX"; then esac ;; CYGWIN*) SHLIB_SUFFIX=.dll;; + iOS) SHLIB_SUFFIX=.dylib;; *) SHLIB_SUFFIX=.so;; esac fi @@ -12701,6 +12772,11 @@ then BLDSHARED="$LDSHARED" fi ;; + iOS/*) + LDSHARED='$(CC) -dynamiclib -F . -framework Python' + LDCXXSHARED='$(CXX) -dynamiclib -F . -framework Python' + BLDSHARED="$LDSHARED" + ;; Emscripten|WASI) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; @@ -12789,7 +12865,6 @@ then then CCSHARED="-fPIC"; else CCSHARED="+z"; fi;; - Linux-android*) ;; Linux*|GNU*) CCSHARED="-fPIC";; Emscripten*|WASI*) if test "x$enable_wasm_dynamic_linking" = xyes @@ -12830,30 +12905,34 @@ then Linux-android*) LINKFORSHARED="-pie -Xlinker -export-dynamic";; Linux*|GNU*) LINKFORSHARED="-Xlinker -export-dynamic";; # -u libsys_s pulls in all symbols in libsys - Darwin/*) + Darwin/*|iOS/*) LINKFORSHARED="$extra_undefs -framework CoreFoundation" # Issue #18075: the default maximum stack size (8MBytes) is too # small for the default recursion limit. Increase the stack size # to ensure that tests don't crash - stack_size="1000000" # 16 MB - if test "$with_ubsan" = "yes" - then - # Undefined behavior sanitizer requires an even deeper stack - stack_size="4000000" # 64 MB - fi - - LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + stack_size="1000000" # 16 MB + if test "$with_ubsan" = "yes" + then + # Undefined behavior sanitizer requires an even deeper stack + stack_size="4000000" # 64 MB + fi printf "%s\n" "#define THREAD_STACK_SIZE 0x$stack_size" >>confdefs.h - if test "$enable_framework" - then - LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + if test $ac_sys_system = "Darwin"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + + if test "$enable_framework"; then + LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + fi + LINKFORSHARED="$LINKFORSHARED" + elif test $ac_sys_system = "iOS"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)' fi - LINKFORSHARED="$LINKFORSHARED";; + ;; OpenUNIX*|UnixWare*) LINKFORSHARED="-Wl,-Bexport";; SCO_SV*) LINKFORSHARED="-Wl,-Bexport";; ReliantUNIX*) LINKFORSHARED="-W1 -Blargedynsym";; @@ -13653,7 +13732,14 @@ then : else $as_nop if test "$cross_compiling" = yes then : + +# "yes" changes the hash function to FNV, which causes problems with Numba +# (https://github.com/numba/numba/blob/0.59.0/numba/cpython/hashing.py#L470). +if test "$ac_sys_system" = "Linux-android"; then + ac_cv_aligned_required=no +else ac_cv_aligned_required=yes +fi else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -14242,6 +14328,10 @@ then : ctypes_malloc_closure=yes ;; #( + iOS) : + + ctypes_malloc_closure=yes + ;; #( sunos5) : as_fn_append LIBFFI_LIBS " -mimpure-text" ;; #( @@ -16088,24 +16178,47 @@ else # (e.g. gnu pth with pthread emulation) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _POSIX_THREADS in unistd.h" >&5 printf %s "checking for _POSIX_THREADS in unistd.h... " >&6; } - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _POSIX_THREADS defined in unistd.h" >&5 +printf %s "checking for _POSIX_THREADS defined in unistd.h... " >&6; } +if test ${ac_cv_defined__POSIX_THREADS_unistd_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef _POSIX_THREADS -yes -#endif +int +main (void) +{ + + #ifdef _POSIX_THREADS + int ok; + (void)ok; + #else + choke me + #endif + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_defined__POSIX_THREADS_unistd_h=yes +else $as_nop + ac_cv_defined__POSIX_THREADS_unistd_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__POSIX_THREADS_unistd_h" >&5 +printf "%s\n" "$ac_cv_defined__POSIX_THREADS_unistd_h" >&6; } +if test $ac_cv_defined__POSIX_THREADS_unistd_h != "no" then : unistd_defines_pthreads=yes else $as_nop unistd_defines_pthreads=no fi -rm -rf conftest* - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $unistd_defines_pthreads" >&5 printf "%s\n" "$unistd_defines_pthreads" >&6; } @@ -16606,65 +16719,135 @@ ipv6lib=none ipv6trylibc=no if test "$ipv6" = yes -a "$cross_compiling" = no; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ipv6 stack type" >&5 -printf %s "checking ipv6 stack type... " >&6; } for i in inria kame linux-glibc linux-inet6 solaris toshiba v6d zeta; do case $i in inria) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for IPV6_INRIA_VERSION defined in netinet/in.h" >&5 +printf %s "checking for IPV6_INRIA_VERSION defined in netinet/in.h... " >&6; } +if test ${ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef IPV6_INRIA_VERSION -yes -#endif +int +main (void) +{ + + #ifdef IPV6_INRIA_VERSION + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=yes +else $as_nop + ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h" >&5 +printf "%s\n" "$ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h" >&6; } +if test $ac_cv_defined_IPV6_INRIA_VERSION_netinet_in_h != "no" then : ipv6type=$i fi -rm -rf conftest* - ;; kame) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __KAME__ defined in netinet/in.h" >&5 +printf %s "checking for __KAME__ defined in netinet/in.h... " >&6; } +if test ${ac_cv_defined___KAME___netinet_in_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef __KAME__ -yes -#endif +int +main (void) +{ + + #ifdef __KAME__ + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=inet6 - ipv6libdir=/usr/local/v6/lib - ipv6trylibc=yes + ac_cv_defined___KAME___netinet_in_h=yes +else $as_nop + ac_cv_defined___KAME___netinet_in_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___KAME___netinet_in_h" >&5 +printf "%s\n" "$ac_cv_defined___KAME___netinet_in_h" >&6; } +if test $ac_cv_defined___KAME___netinet_in_h != "no" +then : + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib + ipv6trylibc=yes fi -rm -rf conftest* - ;; linux-glibc) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __GLIBC__ defined in features.h" >&5 +printf %s "checking for __GLIBC__ defined in features.h... " >&6; } +if test ${ac_cv_defined___GLIBC___features_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#if defined(__GLIBC__) && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 1) || (__GLIBC__ > 2)) -yes -#endif +int +main (void) +{ + + #ifdef __GLIBC__ + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6trylibc=yes + ac_cv_defined___GLIBC___features_h=yes +else $as_nop + ac_cv_defined___GLIBC___features_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___GLIBC___features_h" >&5 +printf "%s\n" "$ac_cv_defined___GLIBC___features_h" >&6; } +if test $ac_cv_defined___GLIBC___features_h != "no" +then : + ipv6type=$i + ipv6trylibc=yes fi -rm -rf conftest* - ;; linux-inet6) if test -d /usr/inet6; then @@ -16683,83 +16866,159 @@ rm -rf conftest* fi ;; toshiba) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _TOSHIBA_INET6 defined in sys/param.h" >&5 +printf %s "checking for _TOSHIBA_INET6 defined in sys/param.h... " >&6; } +if test ${ac_cv_defined__TOSHIBA_INET6_sys_param_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef _TOSHIBA_INET6 -yes -#endif +int +main (void) +{ + + #ifdef _TOSHIBA_INET6 + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib + ac_cv_defined__TOSHIBA_INET6_sys_param_h=yes +else $as_nop + ac_cv_defined__TOSHIBA_INET6_sys_param_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__TOSHIBA_INET6_sys_param_h" >&5 +printf "%s\n" "$ac_cv_defined__TOSHIBA_INET6_sys_param_h" >&6; } +if test $ac_cv_defined__TOSHIBA_INET6_sys_param_h != "no" +then : + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib fi -rm -rf conftest* - ;; v6d) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __V6D__ defined in /usr/local/v6/include/sys/v6config.h" >&5 +printf %s "checking for __V6D__ defined in /usr/local/v6/include/sys/v6config.h... " >&6; } +if test ${ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef __V6D__ -yes -#endif +int +main (void) +{ + + #ifdef __V6D__ + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=v6; - ipv6libdir=/usr/local/v6/lib; - BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS" + ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=yes +else $as_nop + ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h" >&5 +printf "%s\n" "$ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h" >&6; } +if test $ac_cv_defined___V6D____usr_local_v6_include_sys_v6config_h != "no" +then : + ipv6type=$i + ipv6lib=v6 + ipv6libdir=/usr/local/v6/lib + BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS" fi -rm -rf conftest* - ;; zeta) - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _ZETA_MINAMI_INET6 defined in sys/param.h" >&5 +printf %s "checking for _ZETA_MINAMI_INET6 defined in sys/param.h... " >&6; } +if test ${ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ #include -#ifdef _ZETA_MINAMI_INET6 -yes -#endif +int +main (void) +{ + + #ifdef _ZETA_MINAMI_INET6 + int ok; + (void)ok; + #else + choke me + #endif + + ; + return 0; +} _ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "yes" >/dev/null 2>&1 +if ac_fn_c_try_compile "$LINENO" then : - ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib + ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=yes +else $as_nop + ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h" >&5 +printf "%s\n" "$ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h" >&6; } +if test $ac_cv_defined__ZETA_MINAMI_INET6_sys_param_h != "no" +then : + ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib fi -rm -rf conftest* - ;; esac if test "$ipv6type" != "unknown"; then break fi done + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ipv6 stack type" >&5 +printf %s "checking ipv6 stack type... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ipv6type" >&5 printf "%s\n" "$ipv6type" >&6; } fi if test "$ipv6" = "yes" -a "$ipv6lib" != "none"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ipv6 library" >&5 +printf %s "checking ipv6 library... " >&6; } if test -d $ipv6libdir -a -f $ipv6libdir/lib$ipv6lib.a; then LIBS="-L$ipv6libdir -l$ipv6lib $LIBS" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: using lib$ipv6lib" >&5 -printf "%s\n" "$as_me: using lib$ipv6lib" >&6;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: lib$ipv6lib" >&5 +printf "%s\n" "lib$ipv6lib" >&6; } else if test "x$ipv6trylibc" = xyes then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: using libc" >&5 -printf "%s\n" "$as_me: using libc" >&6;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: libc" >&5 +printf "%s\n" "libc" >&6; } else $as_nop @@ -17283,6 +17542,23 @@ else printf "%s\n" "$MACHDEP_OBJS" >&6; } fi +if test "$ac_sys_system" = "Linux-android"; then + # When these functions are used in an unprivileged process, they crash rather + # than returning an error. + privileged_funcs="chroot initgroups setegid seteuid setgid setregid setresgid + setresuid setreuid setuid" + + # These functions are unimplemented and always return an error. + unimplemented_funcs="sem_open sem_unlink" + + for name in $privileged_funcs $unimplemented_funcs; do + as_func_var=`printf "%s\n" "ac_cv_func_$name" | $as_tr_sh` + + eval "$as_func_var=no" + + done +fi + # checks for library functions ac_fn_c_check_func "$LINENO" "accept4" "ac_cv_func_accept4" if test "x$ac_cv_func_accept4" = xyes @@ -17493,12 +17769,6 @@ if test "x$ac_cv_func_getegid" = xyes then : printf "%s\n" "#define HAVE_GETEGID 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" -if test "x$ac_cv_func_getentropy" = xyes -then : - printf "%s\n" "#define HAVE_GETENTROPY 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "geteuid" "ac_cv_func_geteuid" if test "x$ac_cv_func_geteuid" = xyes @@ -17541,12 +17811,6 @@ if test "x$ac_cv_func_getgrouplist" = xyes then : printf "%s\n" "#define HAVE_GETGROUPLIST 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "getgroups" "ac_cv_func_getgroups" -if test "x$ac_cv_func_getgroups" = xyes -then : - printf "%s\n" "#define HAVE_GETGROUPS 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "gethostname" "ac_cv_func_gethostname" if test "x$ac_cv_func_gethostname" = xyes @@ -17913,6 +18177,12 @@ if test "x$ac_cv_func_preadv2" = xyes then : printf "%s\n" "#define HAVE_PREADV2 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "process_vm_readv" "ac_cv_func_process_vm_readv" +if test "x$ac_cv_func_process_vm_readv" = xyes +then : + printf "%s\n" "#define HAVE_PROCESS_VM_READV 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "pthread_cond_timedwait_relative_np" "ac_cv_func_pthread_cond_timedwait_relative_np" if test "x$ac_cv_func_pthread_cond_timedwait_relative_np" = xyes @@ -18273,12 +18543,6 @@ if test "x$ac_cv_func_sysconf" = xyes then : printf "%s\n" "#define HAVE_SYSCONF 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "system" "ac_cv_func_system" -if test "x$ac_cv_func_system" = xyes -then : - printf "%s\n" "#define HAVE_SYSTEM 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "tcgetpgrp" "ac_cv_func_tcgetpgrp" if test "x$ac_cv_func_tcgetpgrp" = xyes @@ -18457,6 +18721,32 @@ fi fi +# iOS defines some system methods that can be linked (so they are +# found by configure), but either raise a compilation error (because the +# header definition prevents usage - autoconf doesn't use the headers), or +# raise an error if used at runtime. Force these symbols off. +if test "$ac_sys_system" != "iOS" ; then + ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" +if test "x$ac_cv_func_getentropy" = xyes +then : + printf "%s\n" "#define HAVE_GETENTROPY 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "getgroups" "ac_cv_func_getgroups" +if test "x$ac_cv_func_getgroups" = xyes +then : + printf "%s\n" "#define HAVE_GETGROUPS 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "system" "ac_cv_func_system" +if test "x$ac_cv_func_system" = xyes +then : + printf "%s\n" "#define HAVE_SYSTEM 1" >>confdefs.h + +fi + +fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC options needed to detect all undeclared functions" >&5 printf %s "checking for $CC options needed to detect all undeclared functions... " >&6; } if test ${ac_cv_c_undeclared_builtin_options+y} @@ -21752,6 +22042,11 @@ fi done +# On Android and iOS, clock_settime can be linked (so it is found by +# configure), but when used in an unprivileged process, it crashes rather than +# returning an error. Force the symbol off. +if test "$ac_sys_system" != "Linux-android" && test "$ac_sys_system" != "iOS" +then for ac_func in clock_settime do : @@ -21762,7 +22057,7 @@ then : else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_settime in -lrt" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_settime in -lrt" >&5 printf %s "checking for clock_settime in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_settime+y} then : @@ -21800,7 +22095,7 @@ printf "%s\n" "$ac_cv_lib_rt_clock_settime" >&6; } if test "x$ac_cv_lib_rt_clock_settime" = xyes then : - printf "%s\n" "#define HAVE_CLOCK_SETTIME 1" >>confdefs.h + printf "%s\n" "#define HAVE_CLOCK_SETTIME 1" >>confdefs.h fi @@ -21809,6 +22104,7 @@ fi fi done +fi for ac_func in clock_nanosleep @@ -22030,7 +22326,9 @@ else $as_nop if test "$cross_compiling" = yes then : -if test "${enable_ipv6+set}" = set; then +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then + ac_cv_buggy_getaddrinfo="no" +elif test "${enable_ipv6+set}" = set; then ac_cv_buggy_getaddrinfo="no -- configured with --(en|dis)able-ipv6" else ac_cv_buggy_getaddrinfo=yes @@ -23957,12 +24255,21 @@ LDVERSION='$(VERSION)$(ABIFLAGS)' { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDVERSION" >&5 printf "%s\n" "$LDVERSION" >&6; } -# On Android and Cygwin the shared libraries must be linked with libpython. +# Configure the flags and dependencies used when compiling shared modules + + +MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)' +MODULE_LDFLAGS='' +# On Android and Cygwin the shared libraries must be linked with libpython. if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then - LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" -else - LIBPYTHON='' + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)" + MODULE_LDFLAGS="\$(BLDLIBRARY)" +fi + +# On iOS the shared libraries must be linked with the Python framework +if test "$ac_sys_system" == "iOS"; then + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(PYTHONFRAMEWORKDIR)/\$(PYTHONFRAMEWORK)" fi @@ -24958,6 +25265,7 @@ then : printf "%s\n" "#define HAVE_RL_COMPDISP_FUNC_T 1" >>confdefs.h + fi @@ -26693,24 +27001,28 @@ CPPFLAGS=$ac_save_cppflags { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for device files" >&5 printf "%s\n" "$as_me: checking for device files" >&6;} -if test "x$cross_compiling" = xyes; then - if test "${ac_cv_file__dev_ptmx+set}" != set; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then + ac_cv_file__dev_ptmx=no + ac_cv_file__dev_ptc=no +else + if test "x$cross_compiling" = xyes; then + if test "${ac_cv_file__dev_ptmx+set}" != set; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 printf %s "checking for /dev/ptmx... " >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 printf "%s\n" "not set" >&6; } - as_fn_error $? "set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 - fi - if test "${ac_cv_file__dev_ptc+set}" != set; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 + as_fn_error $? "set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 + fi + if test "${ac_cv_file__dev_ptc+set}" != set; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 printf %s "checking for /dev/ptc... " >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not set" >&5 printf "%s\n" "not set" >&6; } - as_fn_error $? "set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 + as_fn_error $? "set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling" "$LINENO" 5 + fi fi -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptmx" >&5 printf %s "checking for /dev/ptmx... " >&6; } if test ${ac_cv_file__dev_ptmx+y} then : @@ -26731,12 +27043,12 @@ then : fi -if test "x$ac_cv_file__dev_ptmx" = xyes; then + if test "x$ac_cv_file__dev_ptmx" = xyes; then printf "%s\n" "#define HAVE_DEV_PTMX 1" >>confdefs.h -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 + fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for /dev/ptc" >&5 printf %s "checking for /dev/ptc... " >&6; } if test ${ac_cv_file__dev_ptc+y} then : @@ -26757,10 +27069,11 @@ then : fi -if test "x$ac_cv_file__dev_ptc" = xyes; then + if test "x$ac_cv_file__dev_ptc" = xyes; then printf "%s\n" "#define HAVE_DEV_PTC 1" >>confdefs.h + fi fi if test $ac_sys_system = Darwin @@ -26780,6 +27093,9 @@ ac_fn_c_check_type "$LINENO" "socklen_t" "ac_cv_type_socklen_t" " if test "x$ac_cv_type_socklen_t" = xyes then : +printf "%s\n" "#define HAVE_SOCKLEN_T 1" >>confdefs.h + + else $as_nop printf "%s\n" "#define socklen_t int" >>confdefs.h @@ -28143,6 +28459,27 @@ case $ac_sys_system in #( ;; #( Darwin) : ;; #( + iOS) : + + + + py_cv_module__curses=n/a + py_cv_module__curses_panel=n/a + py_cv_module__gdbm=n/a + py_cv_module__multiprocessing=n/a + py_cv_module__posixshmem=n/a + py_cv_module__posixsubprocess=n/a + py_cv_module__scproxy=n/a + py_cv_module__tkinter=n/a + py_cv_module_grp=n/a + py_cv_module_nis=n/a + py_cv_module_readline=n/a + py_cv_module_pwd=n/a + py_cv_module_spwd=n/a + py_cv_module_syslog=n/a + py_cv_module_=n/a + + ;; #( CYGWIN*) : @@ -30540,6 +30877,44 @@ fi printf "%s\n" "$py_cv_module__testmultiphase" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testexternalinspection" >&5 +printf %s "checking for stdlib extension module _testexternalinspection... " >&6; } + if test "$py_cv_module__testexternalinspection" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if true +then : + py_cv_module__testexternalinspection=yes +else $as_nop + py_cv_module__testexternalinspection=missing +fi +else $as_nop + py_cv_module__testexternalinspection=disabled +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__TESTEXTERNALINSPECTION_STATE=$py_cv_module__testexternalinspection$as_nl" + if test "x$py_cv_module__testexternalinspection" = xyes +then : + + + + +fi + if test "$py_cv_module__testexternalinspection" = yes; then + MODULE__TESTEXTERNALINSPECTION_TRUE= + MODULE__TESTEXTERNALINSPECTION_FALSE='#' +else + MODULE__TESTEXTERNALINSPECTION_TRUE='#' + MODULE__TESTEXTERNALINSPECTION_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testexternalinspection" >&5 +printf "%s\n" "$py_cv_module__testexternalinspection" >&6; } + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module xxsubtype" >&5 printf %s "checking for stdlib extension module xxsubtype... " >&6; } if test "$py_cv_module_xxsubtype" != "n/a" @@ -31152,6 +31527,10 @@ if test -z "${MODULE__TESTMULTIPHASE_TRUE}" && test -z "${MODULE__TESTMULTIPHASE as_fn_error $? "conditional \"MODULE__TESTMULTIPHASE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__TESTEXTERNALINSPECTION_TRUE}" && test -z "${MODULE__TESTEXTERNALINSPECTION_FALSE}"; then + as_fn_error $? "conditional \"MODULE__TESTEXTERNALINSPECTION\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE_XXSUBTYPE_TRUE}" && test -z "${MODULE_XXSUBTYPE_FALSE}"; then as_fn_error $? "conditional \"MODULE_XXSUBTYPE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 @@ -31754,6 +32133,7 @@ do "Mac/PythonLauncher/Makefile") CONFIG_FILES="$CONFIG_FILES Mac/PythonLauncher/Makefile" ;; "Mac/Resources/framework/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/framework/Info.plist" ;; "Mac/Resources/app/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/app/Info.plist" ;; + "iOS/Resources/Info.plist") CONFIG_FILES="$CONFIG_FILES iOS/Resources/Info.plist" ;; "Makefile.pre") CONFIG_FILES="$CONFIG_FILES Makefile.pre" ;; "Misc/python.pc") CONFIG_FILES="$CONFIG_FILES Misc/python.pc" ;; "Misc/python-embed.pc") CONFIG_FILES="$CONFIG_FILES Misc/python-embed.pc" ;; diff --git a/configure.ac b/configure.ac index b39af7422c4c7c..103c24962b7b42 100644 --- a/configure.ac +++ b/configure.ac @@ -327,6 +327,9 @@ then *-*-cygwin*) ac_sys_system=Cygwin ;; + *-apple-ios*) + ac_sys_system=iOS + ;; *-*-vxworks*) ac_sys_system=VxWorks ;; @@ -484,10 +487,18 @@ AC_ARG_ENABLE([framework], [ case $enableval in yes) + if test "$ac_sys_system" = "iOS"; then + AC_MSG_ERROR([iOS builds must provide an explicit path for --enable-framework]) + fi + enableval=/Library/Frameworks esac case $enableval in no) + if test "$ac_sys_system" = "iOS"; then + AC_MSG_ERROR([iOS builds must use --enable-framework=]) + fi + PYTHONFRAMEWORK= PYTHONFRAMEWORKDIR=no-framework PYTHONFRAMEWORKPREFIX= @@ -511,11 +522,11 @@ AC_ARG_ENABLE([framework], *) PYTHONFRAMEWORKPREFIX="${enableval}" PYTHONFRAMEWORKINSTALLDIR=$PYTHONFRAMEWORKPREFIX/$PYTHONFRAMEWORKDIR - FRAMEWORKINSTALLFIRST="frameworkinstallstructure" - FRAMEWORKALTINSTALLFIRST="frameworkinstallstructure " case $ac_sys_system in #( Darwin) : + FRAMEWORKINSTALLFIRST="frameworkinstallversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallversionedstructure " FRAMEWORKINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools" FRAMEWORKALTINSTALLLAST="frameworkinstallmaclib frameworkinstallapps frameworkaltinstallunixtools" FRAMEWORKPYTHONW="frameworkpythonw" @@ -574,12 +585,30 @@ AC_ARG_ENABLE([framework], AC_CONFIG_FILES([Mac/Resources/framework/Info.plist]) AC_CONFIG_FILES([Mac/Resources/app/Info.plist]) ;; + iOS) : + FRAMEWORKINSTALLFIRST="frameworkinstallunversionedstructure" + FRAMEWORKALTINSTALLFIRST="frameworkinstallunversionedstructure " + FRAMEWORKINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKALTINSTALLLAST="frameworkinstallmobileheaders" + FRAMEWORKPYTHONW= + INSTALLTARGETS="libinstall inclinstall sharedinstall" + + prefix=$PYTHONFRAMEWORKPREFIX + PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" + RESSRCDIR=iOS/Resources + + AC_CONFIG_FILES([iOS/Resources/Info.plist]) + ;; *) AC_MSG_ERROR([Unknown platform for framework build]) ;; esac esac ],[ + if test "$ac_sys_system" = "iOS"; then + AC_MSG_ERROR([iOS builds must use --enable-framework=]) + fi + PYTHONFRAMEWORK= PYTHONFRAMEWORKDIR=no-framework PYTHONFRAMEWORKPREFIX= @@ -634,6 +663,24 @@ if test "$cross_compiling" = yes; then *-*-cygwin*) _host_ident= ;; + *-apple-ios*) + _host_os=`echo $host | cut -d '-' -f3` + _host_device=`echo $host | cut -d '-' -f4` + _host_device=${_host_device:=os} + + dnl IOS_DEPLOYMENT_TARGET is the minimum supported iOS version + IOS_DEPLOYMENT_TARGET=${_host_os:3} + IOS_DEPLOYMENT_TARGET=${IOS_DEPLOYMENT_TARGET:=12.0} + + case "$host_cpu" in + aarch64) + _host_ident=${IOS_DEPLOYMENT_TARGET}-arm64-iphone${_host_device} + ;; + *) + _host_ident=${IOS_DEPLOYMENT_TARGET}-$host_cpu-iphone${_host_device} + ;; + esac + ;; *-*-vxworks*) _host_ident=$host_cpu ;; @@ -711,6 +758,9 @@ case $ac_sys_system/$ac_sys_release in define_xopen_source=no;; Darwin/@<:@[12]@:>@@<:@0-9@:>@.*) define_xopen_source=no;; + # On iOS, defining _POSIX_C_SOURCE also disables platform specific features. + iOS/*) + define_xopen_source=no;; # On QNX 6.3.2, defining _XOPEN_SOURCE prevents netdb.h from # defining NI_NUMERICHOST. QNX/6.3.2) @@ -801,6 +851,15 @@ AS_CASE([$host], ], ) +dnl Add the compiler flag for the iOS minimum supported OS version. +AS_CASE([$ac_sys_system], + [iOS], [ + AS_VAR_APPEND([CFLAGS], [" -mios-version-min=${IOS_DEPLOYMENT_TARGET}"]) + AS_VAR_APPEND([LDFLAGS], [" -mios-version-min=${IOS_DEPLOYMENT_TARGET}"]) + AC_SUBST([IOS_DEPLOYMENT_TARGET]) + ], +) + if test "$ac_sys_system" = "Darwin" then dnl look for SDKROOT @@ -967,6 +1026,7 @@ dnl platforms. AC_MSG_CHECKING([for multiarch]) AS_CASE([$ac_sys_system], [Darwin*], [MULTIARCH=""], + [iOS], [MULTIARCH=""], [FreeBSD*], [MULTIARCH=""], [MULTIARCH=$($CC --print-multiarch 2>/dev/null)] ) @@ -988,6 +1048,7 @@ dnl will have multiple sysconfig modules (one for each CPU architecture), but dnl use a single "fat" binary at runtime. SOABI_PLATFORM is the component of dnl the PLATFORM_TRIPLET that will be used in binary module extensions. AS_CASE([$ac_sys_system], + [iOS], [SOABI_PLATFORM=`echo "$PLATFORM_TRIPLET" | cut -d '-' -f2`], [SOABI_PLATFORM=$PLATFORM_TRIPLET] ) @@ -1019,6 +1080,8 @@ AS_CASE([$host/$ac_cv_cc_name], [powerpc64le-*-linux-gnu/clang], [PY_SUPPORT_TIER=3], dnl Linux on PPC64 little endian, glibc, clang [s390x-*-linux-gnu/gcc], [PY_SUPPORT_TIER=3], dnl Linux on 64bit s390x (big endian), glibc, gcc [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 + [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 + [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 [PY_SUPPORT_TIER=0] ) @@ -1337,12 +1400,15 @@ AC_MSG_CHECKING([LDLIBRARY]) # will find it with a -framework option). For this reason there is an # extra variable BLDLIBRARY against which Python and the extension # modules are linked, BLDLIBRARY. This is normally the same as -# LDLIBRARY, but empty for MacOSX framework builds. +# LDLIBRARY, but empty for MacOSX framework builds. iOS does the same, +# but uses a non-versioned framework layout. if test "$enable_framework" then case $ac_sys_system in Darwin) LDLIBRARY='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)';; + iOS) + LDLIBRARY='$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)';; *) AC_MSG_ERROR([Unknown platform for framework build]);; esac @@ -1360,6 +1426,7 @@ if test $enable_shared = "yes"; then case $ac_sys_system in CYGWIN*) LDLIBRARY='libpython$(LDVERSION).dll.a' + BLDLIBRARY='-L. -lpython$(LDVERSION)' DLLLIBRARY='libpython$(LDVERSION).dll' ;; SunOS*) @@ -1376,7 +1443,13 @@ if test $enable_shared = "yes"; then LDLIBRARY='libpython$(LDVERSION).so' BLDLIBRARY='-L. -lpython$(LDVERSION)' RUNSHARED=LD_LIBRARY_PATH=`pwd`${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} - INSTSONAME="$LDLIBRARY".$SOVERSION + + # The Android Gradle plugin will only package libraries whose names end + # with ".so". + if test "$ac_sys_system" != "Linux-android"; then + INSTSONAME="$LDLIBRARY".$SOVERSION + fi + if test "$with_pydebug" != yes then PY3LIBRARY=libpython3.so @@ -1399,6 +1472,9 @@ if test $enable_shared = "yes"; then BLDLIBRARY='-L. -lpython$(LDVERSION)' RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} ;; + iOS) + LDLIBRARY='libpython$(LDVERSION).dylib' + ;; AIX*) LDLIBRARY='libpython$(LDVERSION).so' RUNSHARED=LIBPATH=`pwd`${LIBPATH:+:${LIBPATH}} @@ -1414,11 +1490,13 @@ else # shared is disabled ;; esac fi +AC_MSG_RESULT([$LDLIBRARY]) if test "$cross_compiling" = yes; then RUNSHARED= fi +AC_MSG_CHECKING([HOSTRUNNER]) AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform]) if test -z "$HOSTRUNNER" then @@ -1464,7 +1542,6 @@ then ) fi AC_SUBST([HOSTRUNNER]) -AC_MSG_CHECKING([HOSTRUNNER]) AC_MSG_RESULT([$HOSTRUNNER]) if test -n "$HOSTRUNNER"; then @@ -1472,8 +1549,6 @@ if test -n "$HOSTRUNNER"; then PYTHON_FOR_BUILD="_PYTHON_HOSTRUNNER='$HOSTRUNNER' $PYTHON_FOR_BUILD" fi -AC_MSG_RESULT([$LDLIBRARY]) - # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], [Emscripten/browser*], [LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB) python.html python.worker.js'], @@ -2826,15 +2901,11 @@ AC_CHECK_HEADERS( #endif ]) -# checks for typedefs -AC_CACHE_CHECK([for clock_t in time.h], [ac_cv_clock_t_time_h], [ - AC_EGREP_HEADER([clock_t], [time.h], [ac_cv_clock_t_time_h=yes], [ac_cv_clock_t_time_h=no]) -]) -dnl checks for "no" -AS_VAR_IF([ac_cv_clock_t_time_h], [no], [ - AC_DEFINE([clock_t], [long], - [Define to 'long' if doesn't define.]) -]) +# Check for clock_t in time.h. +AC_CHECK_TYPES([clock_t], [], + [AC_DEFINE([clock_t], [long], + [Define to 'long' if does not define clock_t.])], + [@%:@include ]) AC_CACHE_CHECK([for makedev], [ac_cv_func_makedev], [ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ @@ -2909,12 +2980,10 @@ AC_DEFINE_UNQUOTED([RETSIGTYPE],[void],[assume C89 semantics that RETSIGTYPE is AC_TYPE_SIZE_T AC_TYPE_UID_T -AC_CHECK_TYPE([ssize_t], - AC_DEFINE([HAVE_SSIZE_T], [1], - [Define if your compiler provides ssize_t]), [], []) -AC_CHECK_TYPE([__uint128_t], - AC_DEFINE([HAVE_GCC_UINT128_T], [1], - [Define if your compiler provides __uint128_t]), [], []) +AC_CHECK_TYPES([ssize_t]) +AC_CHECK_TYPES([__uint128_t], + [AC_DEFINE([HAVE_GCC_UINT128_T], [1], + [Define if your compiler provides __uint128_t])]) # Sizes and alignments of various common basic types # ANSI C requires sizeof(char) == 1, so no need to check it @@ -3169,6 +3238,7 @@ if test -z "$SHLIB_SUFFIX"; then esac ;; CYGWIN*) SHLIB_SUFFIX=.dll;; + iOS) SHLIB_SUFFIX=.dylib;; *) SHLIB_SUFFIX=.so;; esac fi @@ -3249,6 +3319,11 @@ then BLDSHARED="$LDSHARED" fi ;; + iOS/*) + LDSHARED='$(CC) -dynamiclib -F . -framework Python' + LDCXXSHARED='$(CXX) -dynamiclib -F . -framework Python' + BLDSHARED="$LDSHARED" + ;; Emscripten|WASI) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; @@ -3333,7 +3408,6 @@ then then CCSHARED="-fPIC"; else CCSHARED="+z"; fi;; - Linux-android*) ;; Linux*|GNU*) CCSHARED="-fPIC";; Emscripten*|WASI*) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ @@ -3369,30 +3443,34 @@ then Linux-android*) LINKFORSHARED="-pie -Xlinker -export-dynamic";; Linux*|GNU*) LINKFORSHARED="-Xlinker -export-dynamic";; # -u libsys_s pulls in all symbols in libsys - Darwin/*) + Darwin/*|iOS/*) LINKFORSHARED="$extra_undefs -framework CoreFoundation" # Issue #18075: the default maximum stack size (8MBytes) is too # small for the default recursion limit. Increase the stack size # to ensure that tests don't crash - stack_size="1000000" # 16 MB - if test "$with_ubsan" = "yes" - then - # Undefined behavior sanitizer requires an even deeper stack - stack_size="4000000" # 64 MB - fi + stack_size="1000000" # 16 MB + if test "$with_ubsan" = "yes" + then + # Undefined behavior sanitizer requires an even deeper stack + stack_size="4000000" # 64 MB + fi - LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + AC_DEFINE_UNQUOTED([THREAD_STACK_SIZE], + [0x$stack_size], + [Custom thread stack size depending on chosen sanitizer runtimes.]) - AC_DEFINE_UNQUOTED([THREAD_STACK_SIZE], - [0x$stack_size], - [Custom thread stack size depending on chosen sanitizer runtimes.]) + if test $ac_sys_system = "Darwin"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" - if test "$enable_framework" - then - LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + if test "$enable_framework"; then + LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + fi + LINKFORSHARED="$LINKFORSHARED" + elif test $ac_sys_system = "iOS"; then + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/$(PYTHONFRAMEWORK)' fi - LINKFORSHARED="$LINKFORSHARED";; + ;; OpenUNIX*|UnixWare*) LINKFORSHARED="-Wl,-Bexport";; SCO_SV*) LINKFORSHARED="-Wl,-Bexport";; ReliantUNIX*) LINKFORSHARED="-W1 -Blargedynsym";; @@ -3603,7 +3681,14 @@ int main(void) }]])], [ac_cv_aligned_required=no], [ac_cv_aligned_required=yes], -[ac_cv_aligned_required=yes]) +[ +# "yes" changes the hash function to FNV, which causes problems with Numba +# (https://github.com/numba/numba/blob/0.59.0/numba/cpython/hashing.py#L470). +if test "$ac_sys_system" = "Linux-android"; then + ac_cv_aligned_required=no +else + ac_cv_aligned_required=yes +fi]) ]) if test "$ac_cv_aligned_required" = yes ; then AC_DEFINE([HAVE_ALIGNED_REQUIRED], [1], @@ -3766,6 +3851,9 @@ AS_VAR_IF([have_libffi], [yes], [ dnl when do we need USING_APPLE_OS_LIBFFI? ctypes_malloc_closure=yes ], + [iOS], [ + ctypes_malloc_closure=yes + ], [sunos5], [AS_VAR_APPEND([LIBFFI_LIBS], [" -mimpure-text"])] ) AS_VAR_IF([ctypes_malloc_closure], [yes], [ @@ -4252,13 +4340,9 @@ else # define _POSIX_THREADS in unistd.h. Some apparently don't # (e.g. gnu pth with pthread emulation) AC_MSG_CHECKING([for _POSIX_THREADS in unistd.h]) - AC_EGREP_CPP([yes], - [ -#include -#ifdef _POSIX_THREADS -yes -#endif - ], unistd_defines_pthreads=yes, unistd_defines_pthreads=no) + AX_CHECK_DEFINE([unistd.h], [_POSIX_THREADS], + [unistd_defines_pthreads=yes], + [unistd_defines_pthreads=no]) AC_MSG_RESULT([$unistd_defines_pthreads]) AC_DEFINE([_REENTRANT]) @@ -4432,40 +4516,26 @@ ipv6lib=none ipv6trylibc=no if test "$ipv6" = yes -a "$cross_compiling" = no; then - AC_MSG_CHECKING([ipv6 stack type]) for i in inria kame linux-glibc linux-inet6 solaris toshiba v6d zeta; do case $i in inria) dnl http://www.kame.net/ - AC_EGREP_CPP([yes], [ -#include -#ifdef IPV6_INRIA_VERSION -yes -@%:@endif], - [ipv6type=$i]) + AX_CHECK_DEFINE([netinet/in.h], [IPV6_INRIA_VERSION], [ipv6type=$i]) ;; kame) dnl http://www.kame.net/ - AC_EGREP_CPP([yes], [ -#include -#ifdef __KAME__ -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6 - ipv6libdir=/usr/local/v6/lib - ipv6trylibc=yes]) + AX_CHECK_DEFINE([netinet/in.h], [__KAME__], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib + ipv6trylibc=yes]) ;; linux-glibc) - dnl http://www.v6.linux.or.jp/ - AC_EGREP_CPP([yes], [ -#include -#if defined(__GLIBC__) && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 1) || (__GLIBC__ > 2)) -yes -@%:@endif], - [ipv6type=$i; - ipv6trylibc=yes]) + dnl Advanced IPv6 support was added to glibc 2.1 in 1999. + AX_CHECK_DEFINE([features.h], [__GLIBC__], + [ipv6type=$i + ipv6trylibc=yes]) ;; linux-inet6) dnl http://www.v6.linux.or.jp/ @@ -4485,51 +4555,41 @@ yes fi ;; toshiba) - AC_EGREP_CPP([yes], [ -#include -#ifdef _TOSHIBA_INET6 -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib]) + AX_CHECK_DEFINE([sys/param.h], [_TOSHIBA_INET6], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib]) ;; v6d) - AC_EGREP_CPP([yes], [ -#include -#ifdef __V6D__ -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=v6; - ipv6libdir=/usr/local/v6/lib; - BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS"]) + AX_CHECK_DEFINE([/usr/local/v6/include/sys/v6config.h], [__V6D__], + [ipv6type=$i + ipv6lib=v6 + ipv6libdir=/usr/local/v6/lib + BASECFLAGS="-I/usr/local/v6/include $BASECFLAGS"]) ;; zeta) - AC_EGREP_CPP([yes], [ -#include -#ifdef _ZETA_MINAMI_INET6 -yes -@%:@endif], - [ipv6type=$i; - ipv6lib=inet6; - ipv6libdir=/usr/local/v6/lib]) + AX_CHECK_DEFINE([sys/param.h], [_ZETA_MINAMI_INET6], + [ipv6type=$i + ipv6lib=inet6 + ipv6libdir=/usr/local/v6/lib]) ;; esac if test "$ipv6type" != "unknown"; then break fi done + AC_MSG_CHECKING([ipv6 stack type]) AC_MSG_RESULT([$ipv6type]) fi if test "$ipv6" = "yes" -a "$ipv6lib" != "none"; then + AC_MSG_CHECKING([ipv6 library]) if test -d $ipv6libdir -a -f $ipv6libdir/lib$ipv6lib.a; then LIBS="-L$ipv6libdir -l$ipv6lib $LIBS" - AC_MSG_NOTICE([using lib$ipv6lib]) + AC_MSG_RESULT([lib$ipv6lib]) else AS_VAR_IF([ipv6trylibc], [yes], [ - AC_MSG_NOTICE([using libc]) + AC_MSG_RESULT([libc]) ], [ AC_MSG_ERROR([m4_normalize([ No $ipv6lib library found; cannot continue. @@ -4825,14 +4885,30 @@ else AC_MSG_RESULT([$MACHDEP_OBJS]) fi +if test "$ac_sys_system" = "Linux-android"; then + # When these functions are used in an unprivileged process, they crash rather + # than returning an error. + privileged_funcs="chroot initgroups setegid seteuid setgid setregid setresgid + setresuid setreuid setuid" + + # These functions are unimplemented and always return an error. + unimplemented_funcs="sem_open sem_unlink" + + for name in $privileged_funcs $unimplemented_funcs; do + AS_VAR_PUSHDEF([func_var], [ac_cv_func_$name]) + AS_VAR_SET([func_var], [no]) + AS_VAR_POPDEF([func_var]) + done +fi + # checks for library functions AC_CHECK_FUNCS([ \ accept4 alarm bind_textdomain_codeset chmod chown clock closefrom close_range confstr \ copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ - gai_strerror getegid getentropy geteuid getgid getgrent getgrgid getgrgid_r \ - getgrnam_r getgrouplist getgroups gethostname getitimer getloadavg getlogin \ + gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ + getgrnam_r getgrouplist gethostname getitimer getloadavg getlogin \ getpeername getpgid getpid getppid getpriority _getpty \ getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid getspent \ getspnam getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \ @@ -4840,7 +4916,7 @@ AC_CHECK_FUNCS([ \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ pipe2 plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ posix_spawn_file_actions_addclosefrom_np \ - pread preadv preadv2 pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ + pread preadv preadv2 process_vm_readv pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ pthread_kill ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ sched_setparam sched_setscheduler sem_clockwait sem_getvalue sem_open \ @@ -4849,7 +4925,7 @@ AC_CHECK_FUNCS([ \ setresuid setreuid setsid setuid setvbuf shutdown sigaction sigaltstack \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ - sysconf system tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ + sysconf tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ tmpnam tmpnam_r truncate ttyname umask uname unlinkat unlockpt utimensat utimes vfork \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) @@ -4861,6 +4937,14 @@ if test "$MACHDEP" != linux; then AC_CHECK_FUNCS([lchmod]) fi +# iOS defines some system methods that can be linked (so they are +# found by configure), but either raise a compilation error (because the +# header definition prevents usage - autoconf doesn't use the headers), or +# raise an error if used at runtime. Force these symbols off. +if test "$ac_sys_system" != "iOS" ; then + AC_CHECK_FUNCS([getentropy getgroups system]) +fi + AC_CHECK_DECL([dirfd], [AC_DEFINE([HAVE_DIRFD], [1], [Define if you have the 'dirfd' function or macro.])], @@ -5161,11 +5245,17 @@ AC_CHECK_FUNCS([clock_getres], [], [ ]) ]) -AC_CHECK_FUNCS([clock_settime], [], [ - AC_CHECK_LIB([rt], [clock_settime], [ - AC_DEFINE([HAVE_CLOCK_SETTIME], [1]) - ]) -]) +# On Android and iOS, clock_settime can be linked (so it is found by +# configure), but when used in an unprivileged process, it crashes rather than +# returning an error. Force the symbol off. +if test "$ac_sys_system" != "Linux-android" && test "$ac_sys_system" != "iOS" +then + AC_CHECK_FUNCS([clock_settime], [], [ + AC_CHECK_LIB([rt], [clock_settime], [ + AC_DEFINE([HAVE_CLOCK_SETTIME], [1]) + ]) + ]) +fi AC_CHECK_FUNCS([clock_nanosleep], [], [ AC_CHECK_LIB([rt], [clock_nanosleep], [ @@ -5311,7 +5401,9 @@ int main(void) [ac_cv_buggy_getaddrinfo=no], [ac_cv_buggy_getaddrinfo=yes], [ -if test "${enable_ipv6+set}" = set; then +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then + ac_cv_buggy_getaddrinfo="no" +elif test "${enable_ipv6+set}" = set; then ac_cv_buggy_getaddrinfo="no -- configured with --(en|dis)able-ipv6" else ac_cv_buggy_getaddrinfo=yes @@ -5887,12 +5979,21 @@ AC_MSG_CHECKING([LDVERSION]) LDVERSION='$(VERSION)$(ABIFLAGS)' AC_MSG_RESULT([$LDVERSION]) +# Configure the flags and dependencies used when compiling shared modules +AC_SUBST([MODULE_DEPS_SHARED]) +AC_SUBST([MODULE_LDFLAGS]) +MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)' +MODULE_LDFLAGS='' + # On Android and Cygwin the shared libraries must be linked with libpython. -AC_SUBST([LIBPYTHON]) if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then - LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" -else - LIBPYTHON='' + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)" + MODULE_LDFLAGS="\$(BLDLIBRARY)" +fi + +# On iOS the shared libraries must be linked with the Python framework +if test "$ac_sys_system" == "iOS"; then + MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(PYTHONFRAMEWORKDIR)/\$(PYTHONFRAMEWORK)" fi @@ -6151,11 +6252,7 @@ AS_VAR_IF([with_readline], [no], [ ]) # in readline as well as newer editline (April 2023) - AC_CHECK_TYPE([rl_compdisp_func_t], - [AC_DEFINE([HAVE_RL_COMPDISP_FUNC_T], [1], - [Define if readline supports rl_compdisp_func_t])], - [], - [readline_includes]) + AC_CHECK_TYPES([rl_compdisp_func_t], [], [], [readline_includes]) m4_undefine([readline_includes]) ])dnl WITH_SAVE_ENV() @@ -6524,28 +6621,35 @@ CPPFLAGS=$ac_save_cppflags AC_MSG_NOTICE([checking for device files]) dnl NOTE: Inform user how to proceed with files when cross compiling. -if test "x$cross_compiling" = xyes; then - if test "${ac_cv_file__dev_ptmx+set}" != set; then - AC_MSG_CHECKING([for /dev/ptmx]) - AC_MSG_RESULT([not set]) - AC_MSG_ERROR([set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling]) - fi - if test "${ac_cv_file__dev_ptc+set}" != set; then - AC_MSG_CHECKING([for /dev/ptc]) - AC_MSG_RESULT([not set]) - AC_MSG_ERROR([set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling]) +dnl Some cross-compile builds are predictable; they won't ever +dnl have /dev/ptmx or /dev/ptc, so we can set them explicitly. +if test "$ac_sys_system" = "Linux-android" || test "$ac_sys_system" = "iOS"; then + ac_cv_file__dev_ptmx=no + ac_cv_file__dev_ptc=no +else + if test "x$cross_compiling" = xyes; then + if test "${ac_cv_file__dev_ptmx+set}" != set; then + AC_MSG_CHECKING([for /dev/ptmx]) + AC_MSG_RESULT([not set]) + AC_MSG_ERROR([set ac_cv_file__dev_ptmx to yes/no in your CONFIG_SITE file when cross compiling]) + fi + if test "${ac_cv_file__dev_ptc+set}" != set; then + AC_MSG_CHECKING([for /dev/ptc]) + AC_MSG_RESULT([not set]) + AC_MSG_ERROR([set ac_cv_file__dev_ptc to yes/no in your CONFIG_SITE file when cross compiling]) + fi fi -fi -AC_CHECK_FILE([/dev/ptmx], [], []) -if test "x$ac_cv_file__dev_ptmx" = xyes; then - AC_DEFINE([HAVE_DEV_PTMX], [1], - [Define to 1 if you have the /dev/ptmx device file.]) -fi -AC_CHECK_FILE([/dev/ptc], [], []) -if test "x$ac_cv_file__dev_ptc" = xyes; then - AC_DEFINE([HAVE_DEV_PTC], [1], - [Define to 1 if you have the /dev/ptc device file.]) + AC_CHECK_FILE([/dev/ptmx], [], []) + if test "x$ac_cv_file__dev_ptmx" = xyes; then + AC_DEFINE([HAVE_DEV_PTMX], [1], + [Define to 1 if you have the /dev/ptmx device file.]) + fi + AC_CHECK_FILE([/dev/ptc], [], []) + if test "x$ac_cv_file__dev_ptc" = xyes; then + AC_DEFINE([HAVE_DEV_PTC], [1], + [Define to 1 if you have the /dev/ptc device file.]) + fi fi if test $ac_sys_system = Darwin @@ -6553,12 +6657,9 @@ then LIBS="$LIBS -framework CoreFoundation" fi -AC_CHECK_TYPE( - [socklen_t], [], - [AC_DEFINE( - [socklen_t], [int], - [Define to `int' if does not define.] - )], [ +AC_CHECK_TYPES([socklen_t], [], + [AC_DEFINE([socklen_t], [int], + [Define to 'int' if does not define.])], [ #ifdef HAVE_SYS_TYPES_H #include #endif @@ -7188,6 +7289,28 @@ AS_CASE([$ac_sys_system], [VxWorks*], [PY_STDLIB_MOD_SET_NA([_scproxy], [termios], [grp])], dnl The _scproxy module is available on macOS [Darwin], [], + [iOS], [ + dnl subprocess and multiprocessing are not supported (no fork syscall). + dnl curses and tkinter user interface are not available. + dnl gdbm and nis aren't available + dnl Stub implementations are provided for pwd, grp etc APIs + PY_STDLIB_MOD_SET_NA( + [_curses], + [_curses_panel], + [_gdbm], + [_multiprocessing], + [_posixshmem], + [_posixsubprocess], + [_scproxy], + [_tkinter], + [grp], + [nis], + [readline], + [pwd], + [spwd], + [syslog], + ) + ], [CYGWIN*], [PY_STDLIB_MOD_SET_NA([_scproxy])], [QNX*], [PY_STDLIB_MOD_SET_NA([_scproxy])], [FreeBSD*], [PY_STDLIB_MOD_SET_NA([_scproxy])], @@ -7457,6 +7580,7 @@ PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([_testexternalinspection], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_ctypes_test], diff --git a/iOS/README.rst b/iOS/README.rst new file mode 100644 index 00000000000000..1043b5ecebbc0c --- /dev/null +++ b/iOS/README.rst @@ -0,0 +1,321 @@ +==================== +Python on iOS README +==================== + +:Authors: + Russell Keith-Magee (2023-11) + +This document provides a quick overview of some iOS specific features in the +Python distribution. + +These instructions are only needed if you're planning to compile Python for iOS +yourself. Most users should *not* need to do this. If you're looking to +experiment with writing an iOS app in Python, tools such as `BeeWare's Briefcase +`__ and `Kivy's Buildozer +`__ will provide a much more approachable +user experience. + +Compilers for building on iOS +============================= + +Building for iOS requires the use of Apple's Xcode tooling. It is strongly +recommended that you use the most recent stable release of Xcode. This will +require the use of the most (or second-most) recently released macOS version, +as Apple does not maintain Xcode for older macOS versions. The Xcode Command +Line Tools are not sufficient for iOS development; you need a *full* Xcode +install. + +If you want to run your code on the iOS simulator, you'll also need to install +an iOS Simulator Platform. You should be prompted to select an iOS Simulator +Platform when you first run Xcode. Alternatively, you can add an iOS Simulator +Platform by selecting an open the Platforms tab of the Xcode Settings panel. + +iOS specific arguments to configure +=================================== + +* ``--enable-framework=DIR`` + + This argument specifies the location where the Python.framework will be + installed. This argument is required for all iOS builds; a directory *must* + be specified. + +* ``--with-framework-name=NAME`` + + Specify the name for the Python framework; defaults to ``Python``. + +Building Python on iOS +====================== + +ABIs and Architectures +---------------------- + +iOS apps can be deployed on physical devices, and on the iOS simulator. Although +the API used on these devices is identical, the ABI is different - you need to +link against different libraries for an iOS device build (``iphoneos``) or an +iOS simulator build (``iphonesimulator``). + +Apple uses the ``XCframework`` format to allow specifying a single dependency +that supports multiple ABIs. An ``XCframework`` is a wrapper around multiple +ABI-specific frameworks that share a common API. + +iOS can also support different CPU architectures within each ABI. At present, +there is only a single supported architecture on physical devices - ARM64. +However, the *simulator* supports 2 architectures - ARM64 (for running on Apple +Silicon machines), and x86_64 (for running on older Intel-based machines). + +To support multiple CPU architectures on a single platform, Apple uses a "fat +binary" format - a single physical file that contains support for multiple +architectures. It is possible to compile and use a "thin" single architecture +version of a binary for testing purposes; however, the "thin" binary will not be +portable to machines using other architectures. + +Building a single-architecture framework +---------------------------------------- + +The Python build system will create a ``Python.framework`` that supports a +*single* ABI with a *single* architecture. Unlike macOS, iOS does not allow a +framework to contain non-library content, so the iOS build will produce a +``bin`` and ``lib`` folder in the same output folder as ``Python.framework``. +The ``lib`` folder will be needed at runtime to support the Python library. + +If you want to use Python in a real iOS project, you need to produce multiple +``Python.framework`` builds, one for each ABI and architecture. iOS builds of +Python *must* be constructed as framework builds. To support this, you must +provide the ``--enable-framework`` flag when configuring the build. The build +also requires the use of cross-compilation. The minimal commands for building +Python for the ARM64 iOS simulator will look something like:: + + $ export PATH="`pwd`/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" + $ ./configure \ + AR=arm64-apple-ios-simulator-ar \ + CC=arm64-apple-ios-simulator-clang \ + CPP=arm64-apple-ios-simulator-cpp \ + CXX=arm64-apple-ios-simulator-clang \ + --enable-framework=/path/to/install \ + --host=arm64-apple-ios-simulator \ + --build=arm64-apple-darwin \ + --with-build-python=/path/to/python.exe + $ make + $ make install + +In this invocation: + +* ``iOS/Resources/bin`` has been added to the path, providing some shims for the + compilers and linkers needed by the build. Xcode requires the use of ``xcrun`` + to invoke compiler tooling. However, if ``xcrun`` is pre-evaluated and the + result passed to ``configure``, these results can embed user- and + version-specific paths into the sysconfig data, which limits the portability + of the compiled Python. Alternatively, if ``xcrun`` is used *as* the compiler, + it requires that compiler variables like ``CC`` include spaces, which can + cause significant problems with many C configuration systems which assume that + ``CC`` will be a single executable. + + To work around this problem, the ``iOS/Resources/bin`` folder contains some + wrapper scripts that present as simple compilers and linkers, but wrap + underlying calls to ``xcrun``. This allows configure to use a ``CC`` + definition without spaces, and without user- or version-specific paths, while + retaining the ability to adapt to the local Xcode install. These scripts are + included in the ``bin`` directory of an iOS install. + + These scripts will, by default, use the currently active Xcode installation. + If you want to use a different Xcode installation, you can use + ``xcode-select`` to set a new default Xcode globally, or you can use the + ``DEVELOPER_DIR`` environment variable to specify an Xcode install. The + scripts will use the default ``iphoneos``/``iphonesimulator`` SDK version for + the select Xcode install; if you want to use a different SDK, you can set the + ``IOS_SDK_VERSION`` environment variable. (e.g, setting + ``IOS_SDK_VERSION=17.1`` would cause the scripts to use the ``iphoneos17.1`` + and ``iphonesimulator17.1`` SDKs, regardless of the Xcode default.) + + The path has also been cleared of any user customizations. A common source of + bugs is for tools like Homebrew to accidentally leak macOS binaries into an iOS + build. Resetting the path to a known "bare bones" value is the easiest way to + avoid these problems. + +* ``/path/to/install`` is the location where the final ``Python.framework`` will + be output. + +* ``--host`` is the architecture and ABI that you want to build, in GNU compiler + triple format. This will be one of: + + - ``arm64-apple-ios`` for ARM64 iOS devices. + - ``arm64-apple-ios-simulator`` for the iOS simulator running on Apple + Silicon devices. + - ``x86_64-apple-ios-simulator`` for the iOS simulator running on Intel + devices. + +* ``--build`` is the GNU compiler triple for the machine that will be running + the compiler. This is one of: + + - ``arm64-apple-darwin`` for Apple Silicon devices. + - ``x86_64-apple-darwin`` for Intel devices. + +* ``/path/to/python.exe`` is the path to a Python binary on the machine that + will be running the compiler. This is needed because the Python compilation + process involves running some Python code. On a normal desktop build of + Python, you can compile a python interpreter and then use that interpreter to + run Python code. However, the binaries produced for iOS won't run on macOS, so + you need to provide an external Python interpreter. This interpreter must be + the same version as the Python that is being compiled. To be completely safe, + this should be the *exact* same commit hash. However, the longer a Python + release has been stable, the more likely it is that this constraint can be + relaxed - the same micro version will often be sufficient. + +For a full CPython build, you also need to specify the paths to iOS builds of +the binary libraries that CPython depends on (XZ, BZip2, LibFFI and OpenSSL). +This can be done by defining the ``LIBLZMA_CFLAGS``, ``LIBLZMA_LIBS``, +``BZIP2_CFLAGS``, ``BZIP2_LIBS``, ``LIBFFI_CFLAGS``, and ``LIBFFI_LIBS`` +environment variables, and the ``--with-openssl`` configure option. Versions of +these libraries pre-compiled for iOS can be found in `this repository +`__. + +By default, Python will be compiled with an iOS deployment target (i.e., the +minimum supported iOS version) of 12.0. To specify a different deployment +target, provide the version number as part of the ``--host`` argument - for +example, ``--host=arm64-apple-ios15.4-simulator`` would compile an ARM64 +simulator build with a deployment target of 15.4. + +Merge thin frameworks into fat frameworks +----------------------------------------- + +Once you've built a ``Python.framework`` for each ABI and and architecture, you +must produce a "fat" framework for each ABI that contains all the architectures +for that ABI. + +The ``iphoneos`` build only needs to support a single architecture, so it can be +used without modification. + +If you only want to support a single simulator architecture, (e.g., only support +ARM64 simulators), you can use a single architecture ``Python.framework`` build. +However, if you want to create ``Python.xcframework`` that supports *all* +architectures, you'll need to merge the ``iphonesimulator`` builds for ARM64 and +x86_64 into a single "fat" framework. + +The "fat" framework can be constructed by performing a directory merge of the +content of the two "thin" ``Python.framework`` directories, plus the ``bin`` and +``lib`` folders for each thin framework. When performing this merge: + +* The pure Python standard library content is identical for each architecture, + except for a handful of platform-specific files (such as the ``sysconfig`` + module). Ensure that the "fat" framework has the union of all standard library + files. + +* Any binary files in the standard library, plus the main + ``libPython3.X.dylib``, can be merged using the ``lipo`` tool, provide by + Xcode:: + + $ lipo -create -output module.dylib path/to/x86_64/module.dylib path/to/arm64/module.dylib + +* The header files will be indentical on both architectures, except for + ``pyconfig.h``. Copy all the headers from one platform (say, arm64), rename + ``pyconfig.h`` to ``pyconfig-arm64.h``, and copy the ``pyconfig.h`` for the + other architecture into the merged header folder as ``pyconfig-x86_64.h``. + Then copy the ``iOS/Resources/pyconfig.h`` file from the CPython sources into + the merged headers folder. This will allow the two Python architectures to + share a common ``pyconfig.h`` header file. + +At this point, you should have 2 Python.framework folders - one for ``iphoneos``, +and one for ``iphonesimulator`` that is a merge of x86+64 and ARM64 content. + +Merge frameworks into an XCframework +------------------------------------ + +Now that we have 2 (potentially fat) ABI-specific frameworks, we can merge those +frameworks into a single ``XCframework``. + +The initial skeleton of an ``XCframework`` is built using:: + + xcodebuild -create-xcframework -output Python.xcframework -framework path/to/iphoneos/Python.framework -framework path/to/iphonesimulator/Python.framework + +Then, copy the ``bin`` and ``lib`` folders into the architecture-specific slices of +the XCframework:: + + cp path/to/iphoneos/bin Python.xcframework/ios-arm64 + cp path/to/iphoneos/lib Python.xcframework/ios-arm64 + + cp path/to/iphonesimulator/bin Python.xcframework/ios-arm64_x86-64-simulator + cp path/to/iphonesimulator/lib Python.xcframework/ios-arm64_x86-64-simulator + +Note that the name of the architecture-specific slice for the simulator will +depend on the CPU architecture that you build. + +Then, add symbolic links to "common" platform names for each slice:: + + ln -si ios-arm64 Python.xcframework/iphoneos + ln -si ios-arm64_x86-64-simulator Python.xcframework/iphonesimulator + +You now have a Python.xcframework that can be used in a project. + +Testing Python on iOS +===================== + +The ``iOS/testbed`` folder that contains an Xcode project that is able to run +the iOS test suite. This project converts the Python test suite into a single +test case in Xcode's XCTest framework. The single XCTest passes if the test +suite passes. + +To run the test suite, configure a Python build for an iOS simulator (i.e., +``--host=arm64-apple-ios-simulator`` or ``--host=x86_64-apple-ios-simulator`` +), setting the framework location to the testbed project:: + + --enable-framework="./iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator" + +Then run ``make all install testiOS``. This will build an iOS framework for your +chosen architecture, install the Python iOS framework into the testbed project, +and run the test suite on an "iPhone SE (3rd generation)" simulator. + +While the test suite is running, Xcode does not display any console output. +After showing some Xcode build commands, the console output will print ``Testing +started``, and then appear to stop. It will remain in this state until the test +suite completes. On a 2022 M1 MacBook Pro, the test suite takes approximately 12 +minutes to run; a couple of extra minutes is required to boot and prepare the +iOS simulator. + +On success, the test suite will exit and report successful completion of the +test suite. No output of the Python test suite will be displayed. + +On failure, the output of the Python test suite *will* be displayed. This will +show the details of the tests that failed. + +Debugging test failures +----------------------- + +The easiest way to diagnose a single test failure is to open the testbed project +in Xcode and run the tests from there using the "Product > Test" menu item. + +Running specific tests +^^^^^^^^^^^^^^^^^^^^^^ + +As the test suite is being executed on an iOS simulator, it is not possible to +pass in command line arguments to configure test suite operation. To work around +this limitation, the arguments that would normally be passed as command line +arguments are configured as a static string at the start of the XCTest method +``- (void)testPython`` in ``iOSTestbedTests.m``. To pass an argument to the test +suite, add a a string to the ``argv`` defintion. These arguments will be passed +to the test suite as if they had been passed to ``python -m test`` at the +command line. + +Disabling automated breakpoints +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, Xcode will inserts an automatic breakpoint whenever a signal is +raised. The Python test suite raises many of these signals as part of normal +operation; unless you are trying to diagnose an issue with signals, the +automatic breakpoints can be inconvenient. However, they can be disabled by +creating a symbolic breakpoint that is triggered at the start of the test run. + +Select "Debug > Breakpoints > Create Symbolic Breakpoint" from the Xcode menu, and +populate the new brewpoint with the following details: + +* **Name**: IgnoreSignals +* **Symbol**: UIApplicationMain +* **Action**: Add debugger commands for: + - ``process handle SIGINT -n true -p true -s false`` + - ``process handle SIGUSR1 -n true -p true -s false`` + - ``process handle SIGUSR2 -n true -p true -s false`` + - ``process handle SIGXFSZ -n true -p true -s false`` +* Check the "Automatically continue after evaluating" box. + +All other details can be left blank. When the process executes the +``UIApplicationMain`` entry point, the breakpoint will trigger, run the debugger +commands to disable the automatic breakpoints, and automatically resume. diff --git a/iOS/Resources/Info.plist.in b/iOS/Resources/Info.plist.in new file mode 100644 index 00000000000000..3ecdc894f0a285 --- /dev/null +++ b/iOS/Resources/Info.plist.in @@ -0,0 +1,34 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + Python + CFBundleGetInfoString + Python Runtime and Library + CFBundleIdentifier + @PYTHONFRAMEWORKIDENTIFIER@ + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + Python + CFBundlePackageType + FMWK + CFBundleShortVersionString + %VERSION% + CFBundleLongVersionString + %VERSION%, (c) 2001-2024 Python Software Foundation. + CFBundleSignature + ???? + CFBundleVersion + 1 + CFBundleSupportedPlatforms + + iPhoneOS + + MinimumOSVersion + @IOS_DEPLOYMENT_TARGET@ + + diff --git a/iOS/Resources/bin/arm64-apple-ios-ar b/iOS/Resources/bin/arm64-apple-ios-ar new file mode 100755 index 00000000000000..8122332b9c1de0 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} ar $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-clang b/iOS/Resources/bin/arm64-apple-ios-clang new file mode 100755 index 00000000000000..4d525751eba798 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-cpp b/iOS/Resources/bin/arm64-apple-ios-cpp new file mode 100755 index 00000000000000..891bb25bb4318c --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios -E $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-ar b/iOS/Resources/bin/arm64-apple-ios-simulator-ar new file mode 100755 index 00000000000000..74ed3bc6df1c2b --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang b/iOS/Resources/bin/arm64-apple-ios-simulator-clang new file mode 100755 index 00000000000000..32574cad284441 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator $@ diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-cpp b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp new file mode 100755 index 00000000000000..6aaf6fbe188c32 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator -E $@ diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-ar b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar new file mode 100755 index 00000000000000..74ed3bc6df1c2b --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar $@ diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang new file mode 100755 index 00000000000000..bcbe91f6061e16 --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator $@ diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp new file mode 100755 index 00000000000000..e6a42d9b85dec7 --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator -E $@ diff --git a/iOS/Resources/dylib-Info-template.plist b/iOS/Resources/dylib-Info-template.plist new file mode 100644 index 00000000000000..f652e272f71c88 --- /dev/null +++ b/iOS/Resources/dylib-Info-template.plist @@ -0,0 +1,26 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + + CFBundleIdentifier + + CFBundleInfoDictionaryVersion + 6.0 + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleSupportedPlatforms + + iPhoneOS + + MinimumOSVersion + 12.0 + CFBundleVersion + 1 + + diff --git a/iOS/Resources/pyconfig.h b/iOS/Resources/pyconfig.h new file mode 100644 index 00000000000000..4acff2c6051637 --- /dev/null +++ b/iOS/Resources/pyconfig.h @@ -0,0 +1,7 @@ +#ifdef __arm64__ +#include "pyconfig-arm64.h" +#endif + +#ifdef __x86_64__ +#include "pyconfig-x86_64.h" +#endif diff --git a/pyconfig.h.in b/pyconfig.h.in index 2b4bb1a2b52866..e28baef51d5737 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -157,6 +157,9 @@ /* Define to 1 if you have the `clock_settime' function. */ #undef HAVE_CLOCK_SETTIME +/* Define to 1 if the system has the type `clock_t'. */ +#undef HAVE_CLOCK_T + /* Define to 1 if you have the `closefrom' function. */ #undef HAVE_CLOSEFROM @@ -933,6 +936,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_PROCESS_H +/* Define to 1 if you have the `process_vm_readv' function. */ +#undef HAVE_PROCESS_VM_READV + /* Define if your compiler supports function prototype */ #undef HAVE_PROTOTYPES @@ -1009,7 +1015,7 @@ /* Define if you can turn off readline's signal handling. */ #undef HAVE_RL_CATCH_SIGNAL -/* Define if readline supports rl_compdisp_func_t */ +/* Define to 1 if the system has the type `rl_compdisp_func_t'. */ #undef HAVE_RL_COMPDISP_FUNC_T /* Define if you have readline 2.2 */ @@ -1195,13 +1201,16 @@ /* Define if you have the 'socketpair' function. */ #undef HAVE_SOCKETPAIR +/* Define to 1 if the system has the type `socklen_t'. */ +#undef HAVE_SOCKLEN_T + /* Define to 1 if you have the header file. */ #undef HAVE_SPAWN_H /* Define to 1 if you have the `splice' function. */ #undef HAVE_SPLICE -/* Define if your compiler provides ssize_t */ +/* Define to 1 if the system has the type `ssize_t'. */ #undef HAVE_SSIZE_T /* Define to 1 if you have the `statvfs' function. */ @@ -1568,6 +1577,9 @@ /* Define to 1 if you have the `_getpty' function. */ #undef HAVE__GETPTY +/* Define to 1 if the system has the type `__uint128_t'. */ +#undef HAVE___UINT128_T + /* Define to 1 if `major', `minor', and `makedev' are declared in . */ #undef MAJOR_IN_MKDEV @@ -1932,7 +1944,7 @@ /* Define on FreeBSD to activate all library features */ #undef __BSD_VISIBLE -/* Define to 'long' if doesn't define. */ +/* Define to 'long' if does not define clock_t. */ #undef clock_t /* Define to empty if `const' does not conform to ANSI C. */ @@ -1956,7 +1968,7 @@ /* Define to `unsigned int' if does not define. */ #undef size_t -/* Define to `int' if does not define. */ +/* Define to 'int' if does not define. */ #undef socklen_t /* Define to `int' if doesn't define. */