From db64dae74517d094da5516b9c96d3e81e7108ebf Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 21 May 2024 13:35:26 +0200 Subject: [PATCH] [3.13] gh-74929: PEP 667 general docs update (gh-119291) * expand on What's New entry for PEP 667 (including porting notes) * define 'optimized scope' as a glossary term * cover comprehensions and generator expressions in locals() docs * review all mentions of "locals" in documentation (updating if needed) * review all mentions of "f_locals" in documentation (updating if needed) (cherry picked from commit e870c852c0ea96fa4e4569e9c39c7ceb80ce858d) Co-authored-by: Alyssa Coghlan --- Doc/c-api/frame.rst | 11 ++-- Doc/glossary.rst | 9 ++++ Doc/library/code.rst | 6 +-- Doc/library/functions.rst | 103 ++++++++++++++++++++++++------------ Doc/library/pdb.rst | 22 ++++---- Doc/library/profile.rst | 2 +- Doc/library/traceback.rst | 2 +- Doc/reference/datamodel.rst | 2 +- Doc/whatsnew/3.13.rst | 55 +++++++++++++++++-- Lib/code.py | 8 +-- Lib/pdb.py | 9 ++-- 11 files changed, 163 insertions(+), 66 deletions(-) diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 82e0980ad753d0..638a740e0c24da 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -121,17 +121,18 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame) Get the *frame*'s :attr:`~frame.f_locals` attribute. - If the frame refers to a function or comprehension, this returns - a write-through proxy object that allows modifying the locals. - In all other cases (classes, modules) it returns the :class:`dict` - representing the frame locals directly. + If the frame refers to an :term:`optimized scope`, this returns a + write-through proxy object that allows modifying the locals. + In all other cases (classes, modules, :func:`exec`, :func:`eval`) it returns + the mapping representing the frame locals directly (as described for + :func:`locals`). Return a :term:`strong reference`. .. versionadded:: 3.11 .. versionchanged:: 3.13 - Return a proxy object for functions and comprehensions. + As part of :pep:`667`, return a proxy object for optimized scopes. .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 2846f77feb112d..42e2a6f4e301b6 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -889,6 +889,15 @@ Glossary (methods). Also the ultimate base class of any :term:`new-style class`. + optimized scope + A scope where target local variable names are reliably known to the + compiler when the code is compiled, allowing optimization of read and + write access to these names. The local namespaces for functions, + generators, coroutines, comprehensions, and generator expressions are + optimized in this fashion. Note: most interpreter optimizations are + applied to all scopes, only those relying on a known set of local + and nonlocal variable names are restricted to optimized scopes. + package A Python :term:`module` which can contain submodules or recursively, subpackages. Technically, a package is a Python module with a diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 8c3a3e8e95a11f..8f7692df9fb22d 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -18,9 +18,9 @@ build applications which provide an interactive interpreter prompt. This class deals with parsing and interpreter state (the user's namespace); it does not deal with input buffering or prompting or input file naming (the filename is always passed in explicitly). The optional *locals* argument - specifies the dictionary in which code will be executed; it defaults to a newly - created dictionary with key ``'__name__'`` set to ``'__console__'`` and key - ``'__doc__'`` set to ``None``. + specifies a mapping to use as the namespace in which code will be executed; + it defaults to a newly created dictionary with key ``'__name__'`` set to + ``'__console__'`` and key ``'__doc__'`` set to ``None``. .. class:: InteractiveConsole(locals=None, filename="", local_exit=False) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3986ace02b561a..a879ddbca92e82 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -543,18 +543,19 @@ are always available. They are listed here in alphabetical order. The *expression* argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the *globals* and *locals* - dictionaries as global and local namespace. If the *globals* dictionary is + mappings as global and local namespace. If the *globals* dictionary is present and does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is inserted under that key before *expression* is parsed. That way you can control what builtins are available to the executed code by inserting your own ``__builtins__`` dictionary into *globals* before passing it to - :func:`eval`. If the *locals* dictionary is omitted it defaults to the - *globals* dictionary. If both dictionaries are omitted, the expression is + :func:`eval`. If the *locals* mapping is omitted it defaults to the + *globals* dictionary. If both mappings are omitted, the expression is executed with the *globals* and *locals* in the environment where - :func:`eval` is called. Note, *eval()* does not have access to the + :func:`eval` is called. Note, *eval()* will only have access to the :term:`nested scopes ` (non-locals) in the enclosing - environment. + environment if they are already referenced in the scope that is calling + :func:`eval` (e.g. via a :keyword:`nonlocal` statement). Example: @@ -587,6 +588,11 @@ are always available. They are listed here in alphabetical order. The *globals* and *locals* arguments can now be passed as keywords. + .. versionchanged:: 3.13 + + The semantics of the default *locals* namespace have been adjusted as + described for the :func:`locals` builtin. + .. index:: pair: built-in function; exec .. function:: exec(source, /, globals=None, locals=None, *, closure=None) @@ -612,9 +618,15 @@ are always available. They are listed here in alphabetical order. .. note:: - Most users should just pass a *globals* argument and never *locals*. - If exec gets two separate objects as *globals* and *locals*, the code - will be executed as if it were embedded in a class definition. + When ``exec`` gets two separate objects as *globals* and *locals*, the + code will be executed as if it were embedded in a class definition. This + means functions and classes defined in the executed code will not be able + to access variables assigned at the top level (as the "top level" + variables are treated as class variables in a class definition). + Passing a :class:`collections.ChainMap` instance as *globals* allows name + lookups to be chained across multiple mappings without triggering this + behaviour. Values assigned to top level names in the executed code can be + retrieved by passing an empty dictionary as the first entry in the chain. If the *globals* dictionary does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module @@ -635,7 +647,7 @@ are always available. They are listed here in alphabetical order. .. note:: The built-in functions :func:`globals` and :func:`locals` return the current - global and local dictionary, respectively, which may be useful to pass around + global and local namespace, respectively, which may be useful to pass around for use as the second and third argument to :func:`exec`. .. note:: @@ -651,6 +663,11 @@ are always available. They are listed here in alphabetical order. The *globals* and *locals* arguments can now be passed as keywords. + .. versionchanged:: 3.13 + + The semantics of the default *locals* namespace have been adjusted as + described for the :func:`locals` builtin. + .. function:: filter(function, iterable) @@ -1056,39 +1073,51 @@ are always available. They are listed here in alphabetical order. variable names as the keys, and their currently bound references as the values. - At module scope, as well as when using ``exec()`` or ``eval()`` with a - single namespace, this function returns the same namespace as ``globals()``. + At module scope, as well as when using :func:`exec` or :func:`eval` with + a single namespace, this function returns the same namespace as + :func:`globals`. At class scope, it returns the namespace that will be passed to the metaclass constructor. When using ``exec()`` or ``eval()`` with separate local and global - namespaces, it returns the local namespace passed in to the function call. + arguments, it returns the local namespace passed in to the function call. In all of the above cases, each call to ``locals()`` in a given frame of execution will return the *same* mapping object. Changes made through - the mapping object returned from ``locals()`` will be visible as bound, - rebound, or deleted local variables, and binding, rebinding, or deleting - local variables will immediately affect the contents of the returned mapping - object. - - At function scope (including for generators and coroutines), each call to - ``locals()`` instead returns a fresh dictionary containing the current - bindings of the function's local variables and any nonlocal cell references. - In this case, name binding changes made via the returned dict are *not* - written back to the corresponding local variables or nonlocal cell - references, and binding, rebinding, or deleting local variables and nonlocal - cell references does *not* affect the contents of previously returned - dictionaries. + the mapping object returned from ``locals()`` will be visible as assigned, + reassigned, or deleted local variables, and assigning, reassigning, or + deleting local variables will immediately affect the contents of the + returned mapping object. + + In an :term:`optimized scope` (including functions, generators, and + coroutines), each call to ``locals()`` instead returns a fresh dictionary + containing the current bindings of the function's local variables and any + nonlocal cell references. In this case, name binding changes made via the + returned dict are *not* written back to the corresponding local variables + or nonlocal cell references, and assigning, reassigning, or deleting local + variables and nonlocal cell references does *not* affect the contents + of previously returned dictionaries. + + Calling ``locals()`` as part of a comprehension in a function, generator, or + coroutine is equivalent to calling it in the containing scope, except that + the comprehension's initialised iteration variables will be included. In + other scopes, it behaves as if the comprehension were running as a nested + function. + + Calling ``locals()`` as part of a generator expression is equivalent to + calling it in a nested generator function. + + .. versionchanged:: 3.12 + The behaviour of ``locals()`` in a comprehension has been updated as + described in :pep:`709`. .. versionchanged:: 3.13 - In previous versions, the semantics of mutating the mapping object - returned from this function were formally undefined. In CPython - specifically, the mapping returned at function scope could be - implicitly refreshed by other operations, such as calling ``locals()`` - again. Obtaining the legacy CPython behaviour now requires explicit - calls to update the initially returned dictionary with the results - of subsequent calls to ``locals()``. + As part of :pep:`667`, the semantics of mutating the mapping objects + returned from this function are now defined. The behavior in + :term:`optimized scopes ` is now as described above. + Aside from being defined, the behaviour in other scopes remains + unchanged from previous versions. .. function:: map(function, iterable, *iterables) @@ -1975,14 +2004,18 @@ are always available. They are listed here in alphabetical order. :attr:`~object.__dict__` attributes (for example, classes use a :class:`types.MappingProxyType` to prevent direct dictionary updates). - Without an argument, :func:`vars` acts like :func:`locals`. Note, the - locals dictionary is only useful for reads since updates to the locals - dictionary are ignored. + Without an argument, :func:`vars` acts like :func:`locals`. A :exc:`TypeError` exception is raised if an object is specified but it doesn't have a :attr:`~object.__dict__` attribute (for example, if its class defines the :attr:`~object.__slots__` attribute). + .. versionchanged:: 3.13 + + The result of calling this function without an argument has been + updated as described for the :func:`locals` builtin. + + .. function:: zip(*iterables, strict=False) Iterate over several iterables in parallel, producing tuples with an item diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 7a47a7d5d6754f..7d67e06434b799 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -123,6 +123,11 @@ The typical usage to inspect a crashed program is:: 0 (Pdb) +.. versionchanged:: 3.13 + The implementation of :pep:`667` means that name assignments made via ``pdb`` + will immediately affect the active scope, even when running inside an + :term:`optimized scope`. + The module defines the following functions; each enters the debugger in a slightly different way: @@ -579,18 +584,17 @@ can be overridden by the local file. .. pdbcommand:: interact - Start an interactive interpreter (using the :mod:`code` module) whose global - namespace contains all the (global and local) names found in the current - scope. Use ``exit()`` or ``quit()`` to exit the interpreter and return to - the debugger. + Start an interactive interpreter (using the :mod:`code` module) in a new + global namespace initialised from the local and global namespaces for the + current scope. Use ``exit()`` or ``quit()`` to exit the interpreter and + return to the debugger. .. note:: - Because interact creates a new global namespace with the current global - and local namespace for execution, assignment to variables will not - affect the original namespaces. - However, modification to the mutable objects will be reflected in the - original namespaces. + As ``interact`` creates a new dedicated namespace for code execution, + assignments to variables will not affect the original namespaces. + However, modifications to any referenced mutable objects will be reflected + in the original namespaces as usual. .. versionadded:: 3.2 diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 3ca802e024bc27..9721da7220d54d 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -234,7 +234,7 @@ functions: .. function:: runctx(command, globals, locals, filename=None, sort=-1) This function is similar to :func:`run`, with added arguments to supply the - globals and locals dictionaries for the *command* string. This routine + globals and locals mappings for the *command* string. This routine executes:: exec(command, globals, locals) diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 9983b8da427a2a..bfd2c3efc4b1f6 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -473,7 +473,7 @@ in a :ref:`traceback `. attribute accessed (which also happens when casting it to a :class:`tuple`). :attr:`~FrameSummary.line` may be directly provided, and will prevent line lookups happening at all. *locals* is an optional local variable - dictionary, and if supplied the variable representations are stored in the + mapping, and if supplied the variable representations are stored in the summary for later display. :class:`!FrameSummary` instances have the following attributes: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index f5e87160732056..6d2da88d13c25c 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1346,7 +1346,7 @@ Special read-only attributes * - .. attribute:: frame.f_locals - The dictionary used by the frame to look up :ref:`local variables `. - If the frame refers to a function or comprehension, + If the frame refers to an :term:`optimized scope`, this may return a write-through proxy object. .. versionchanged:: 3.13 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 01a06db2856192..339391f64995e8 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -94,10 +94,11 @@ Interpreter improvements: Performance improvements are modest -- we expect to be improving this over the next few releases. -* :pep:`667`: :attr:`FrameType.f_locals ` when used in - a function now returns a write-through proxy to the frame's locals, - rather than a ``dict``. See the PEP for corresponding C API changes - and deprecations. +* :pep:`667`: The :func:`locals` builtin now has + :ref:`defined semantics ` when mutating the + returned mapping. Python debuggers and similar tools may now more reliably + update local variables in optimized frames even during concurrent code + execution. New typing features: @@ -247,6 +248,34 @@ Improved Error Messages through ``self.X`` from any function in its body. (Contributed by Irit Katriel in :gh:`115775`.) +.. _whatsnew313-locals-semantics: + +Defined mutation semantics for ``locals()`` +------------------------------------------- + +Historically, the expected result of mutating the return value of :func:`locals` +has been left to individual Python implementations to define. + +Through :pep:`667`, Python 3.13 standardises the historical behaviour of CPython +for most code execution scopes, but changes +:term:`optimized scopes ` (functions, generators, coroutines, +comprehensions, and generator expressions) to explicitly return independent +snapshots of the currently assigned local variables, including locally +referenced nonlocal variables captured in closures. + +To ensure debuggers and similar tools can reliably update local variables in +scopes affected by this change, :attr:`FrameType.f_locals ` now +returns a write-through proxy to the frame's local and locally referenced +nonlocal variables in these scopes, rather than returning an inconsistently +updated shared ``dict`` instance with undefined runtime semantics. + +See :pep:`667` for more details, including related C API changes and +deprecations. + +(PEP and implementation contributed by Mark Shannon and Tian Gao in +:gh:`74929`. Documentation updates provided by Guido van Rossum and +Alyssa Coghlan.) + Incremental Garbage Collection ------------------------------ @@ -2177,6 +2206,24 @@ Changes in the Python API returned by :meth:`zipfile.ZipFile.open` was changed from ``'r'`` to ``'rb'``. (Contributed by Serhiy Storchaka in :gh:`115961`.) +* Calling :func:`locals` in an :term:`optimized scope` now produces an + independent snapshot on each call, and hence no longer implicitly updates + previously returned references. Obtaining the legacy CPython behaviour now + requires explicit calls to update the initially returned dictionary with the + results of subsequent calls to ``locals()``. (Changed as part of :pep:`667`.) + +* Calling :func:`locals` from a comprehension at module or class scope + (including via ``exec`` or ``eval``) once more behaves as if the comprehension + were running as an independent nested function (i.e. the local variables from + the containing scope are not included). In Python 3.12, this had changed + to include the local variables from the containing scope when implementing + :pep:`709`. (Changed as part of :pep:`667`.) + +* Accessing :attr:`FrameType.f_locals ` in an + :term:`optimized scope` now returns a write-through proxy rather than a + snapshot that gets updated at ill-specified times. If a snapshot is desired, + it must be created explicitly with ``dict`` or the proxy's ``.copy()`` method. + (Changed as part of :pep:`667`.) Changes in the C API -------------------- diff --git a/Lib/code.py b/Lib/code.py index 9d124563f728c2..0c2fd2963b2118 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -25,10 +25,10 @@ class InteractiveInterpreter: def __init__(self, locals=None): """Constructor. - The optional 'locals' argument specifies the dictionary in - which code will be executed; it defaults to a newly created - dictionary with key "__name__" set to "__console__" and key - "__doc__" set to None. + The optional 'locals' argument specifies a mapping to use as the + namespace in which code will be executed; it defaults to a newly + created dictionary with key "__name__" set to "__console__" and + key "__doc__" set to None. """ if locals is None: diff --git a/Lib/pdb.py b/Lib/pdb.py index e507a9bb896611..0f791d71950099 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -392,9 +392,12 @@ def setup(self, f, tb): self.tb_lineno[tb.tb_frame] = lineno tb = tb.tb_next self.curframe = self.stack[self.curindex][0] - # The f_locals dictionary is updated from the actual frame - # locals whenever the .f_locals accessor is called, so we - # cache it here to ensure that modifications are not overwritten. + # The f_locals dictionary used to be updated from the actual frame + # locals whenever the .f_locals accessor was called, so it was + # cached here to ensure that modifications were not overwritten. While + # the caching is no longer required now that f_locals is a direct proxy + # on optimized frames, it's also harmless, so the code structure has + # been left unchanged. self.curframe_locals = self.curframe.f_locals self.set_convenience_variable(self.curframe, '_frame', self.curframe)