From 7626a0e73fff2ea97cb33f09e53809e5d1c1e7fe Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 18 Sep 2017 17:57:18 +1000 Subject: [PATCH 01/66] bpo-30744: Trace hooks no longer reset closure state Previously, trace hooks running on a nested function could incorrectly reset the state of a closure cell. This avoids that by slightly changing the semantics of the locals namespace in functions, generators, and coroutines, such that closure references are represented in the locals namespace by their actual cell objects while a trace hook implemented in Python is running. Depends on PEP 558 and bpo-17960 to cover the underlying change to frame.f_locals semantics. --- Include/descrobject.h | 4 ++ Include/frameobject.h | 3 ++ Lib/test/test_sys_settrace.py | 45 +++++++++++++++++++++- Objects/frameobject.c | 72 ++++++++++++++++++++++++++++------- Python/sysmodule.c | 7 +++- 5 files changed, 116 insertions(+), 15 deletions(-) diff --git a/Include/descrobject.h b/Include/descrobject.h index cb43174838a86b..040cfc97881347 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -99,6 +99,10 @@ PyAPI_FUNC(PyObject *) PyDescr_NewWrapper(PyTypeObject *, #endif PyAPI_FUNC(PyObject *) PyDictProxy_New(PyObject *); +#ifdef Py_BUILD_CORE +PyAPI_FUNC(PyObject *) _PyDictProxy_GetMapping(PyObject *); +#endif + PyAPI_FUNC(PyObject *) PyWrapper_New(PyObject *, PyObject *); diff --git a/Include/frameobject.h b/Include/frameobject.h index a95baf8867a360..bb3eb316858ec4 100644 --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -78,6 +78,9 @@ PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); +#ifdef Py_BUILD_CORE +int _PyFrame_FastToLocalsInternal(PyFrameObject *f, int); /* For sys.settrace */ +#endif PyAPI_FUNC(int) PyFrame_ClearFreeList(void); diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index ed9e6d4f492fec..bd081255cf5ddc 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -889,13 +889,56 @@ class fake_function: self.compare_jump_output([2, 3, 2, 3, 4], namespace["output"]) +class FrameLocalsTestCase(unittest.TestCase): + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + sys.settrace(None) + + def test_closures_are_not_implicitly_reset_to_previous_state(self): + # See https://bugs.python.org/issue30744 for details + i_from_generator = [] + x_from_generator = [] + x_from_nested_ref = [] + x_from_nested_locals = [] + def outer(): + x = 0 + + def update_nested_refs(): + x_from_nested_ref.append(x) + x_from_nested_locals.append(locals()["x"]) + + yield update_nested_refs + for i in range(1, 10): + i_from_generator.append(i) + x += 1 + yield x + + incrementing_generator = outer() + update_nested_refs = next(incrementing_generator) + + def tracefunc(frame, event, arg): + x_from_generator.append(next(incrementing_generator)) + return tracefunc + + sys.settrace(tracefunc) + try: + update_nested_refs() + update_nested_refs() + finally: + sys.settrace(None) + self.assertEqual(x_from_generator, i_from_generator) + self.assertEqual(x_from_nested_ref, [2, 6]) + self.assertEqual(x_from_nested_locals, [3, 7]) + + def test_main(): support.run_unittest( TraceTestCase, SkipLineEventsTraceTestCase, TraceOpcodesTestCase, RaisingTraceFuncTestCase, - JumpTestCase + JumpTestCase, + FrameLocalsTestCase ) if __name__ == "__main__": diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 6ab3a22950ade6..bc6f32aee868ea 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -10,6 +10,17 @@ #define OFF(x) offsetof(PyFrameObject, x) +/* PEP558: + * + * Forward declaration of fastlocalsproxy + * PyEval_GetLocals will need a new PyFrame_GetLocals() helper function + * that ensures it always gets the snapshot reference and never the proxy + * even when tracing is enabled. + * That should probably be a public API for the benefit of third party debuggers + * implemented in C. + * + */ + static PyMemberDef frame_memberlist[] = { {"f_back", T_OBJECT, OFF(f_back), READONLY}, {"f_code", T_OBJECT, OFF(f_code), READONLY}, @@ -813,18 +824,20 @@ map_to_dict(PyObject *map, Py_ssize_t nmap, PyObject *dict, PyObject **values, map and values are input arguments. map is a tuple of strings. values is an array of PyObject*. At index i, map[i] is the name of - the variable with value values[i]. The function copies the first - nmap variable from map/values into dict. If values[i] is NULL, - the variable is deleted from dict. + the variable with value values[i]. The function gets the new value + for values[i] by looking up map[i] in the dict. + + If clear is true and map[i] is missing from the dict, then values[i] is + set to NULL. If clear is false, then values[i] is left alone in that case. If deref is true, then the values being copied are cell variables - and the value is extracted from the cell variable before being put - in dict. If clear is true, then variables in map but not in dict - are set to NULL in map; if clear is false, variables missing in - dict are ignored. + and the value is inserted into the cell variable rather than overwriting + the value directly. If the value in the dict *is* the cell itself, then + the cell value is left alone, and instead that value is written back + into the dict (replacing the cell reference). - Exceptions raised while modifying the dict are silently ignored, - because there is no good way to report them. + Exceptions raised while reading or updating the dict or updating a cell + reference are silently ignored, because there is no good way to report them. */ static void @@ -847,7 +860,16 @@ dict_to_map(PyObject *map, Py_ssize_t nmap, PyObject *dict, PyObject **values, } if (deref) { assert(PyCell_Check(values[j])); - if (PyCell_GET(values[j]) != value) { + PyObject *cell_value = PyCell_GET(values[j]); + if (values[j] == value) { + /* The dict currently contains the cell itself, so write the + cell's value into the dict rather than the other way around + */ + if (PyObject_SetItem(dict, key, cell_value) != 0) { + PyErr_Clear(); + } + } else if (cell_value != value) { + /* Write the value from the dict back into the cell */ if (PyCell_Set(values[j], value) < 0) PyErr_Clear(); } @@ -860,7 +882,7 @@ dict_to_map(PyObject *map, Py_ssize_t nmap, PyObject *dict, PyObject **values, } int -PyFrame_FastToLocalsWithError(PyFrameObject *f) +_PyFrame_FastToLocalsInternal(PyFrameObject *f, int deref) { /* Merge fast locals into f->f_locals */ PyObject *locals, *map; @@ -879,6 +901,14 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f) if (locals == NULL) return -1; } + /* PEP558: + * + * If a trace function is active, ensure f_locals is a fastlocalsproxy + * instance, while locals still refers to the underlying mapping. + * If a trace function is *not* active, discard the proxy, if any (and + * invalidate its reference back to the frame) to disallow further writes. + */ + co = f->f_code; map = co->co_varnames; if (!PyTuple_Check(map)) { @@ -898,8 +928,17 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f) ncells = PyTuple_GET_SIZE(co->co_cellvars); nfreevars = PyTuple_GET_SIZE(co->co_freevars); if (ncells || nfreevars) { + /* If deref is true, we'll replace cells with their values in the + namespace. If it's false, we'll include the cells themselves, which + means PyFrame_LocalsToFast will skip writing them back (unless + they've actually been modified). + + The trace hook implementation relies on this to allow debuggers to + inject changes to local variables without inadvertently resetting + closure variables to a previous value. + */ if (map_to_dict(co->co_cellvars, ncells, - locals, fast + co->co_nlocals, 1)) + locals, fast + co->co_nlocals, deref)) return -1; /* If the namespace is unoptimized, then one of the @@ -912,13 +951,20 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f) */ if (co->co_flags & CO_OPTIMIZED) { if (map_to_dict(co->co_freevars, nfreevars, - locals, fast + co->co_nlocals + ncells, 1) < 0) + locals, fast + co->co_nlocals + ncells, deref) < 0) return -1; } } + return 0; } +int +PyFrame_FastToLocalsWithError(PyFrameObject *f) +{ + return _PyFrame_FastToLocalsInternal(f, 1); +} + void PyFrame_FastToLocals(PyFrameObject *f) { diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 6dc8e08be7d997..0d38e5c6238c02 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -458,7 +458,8 @@ call_trampoline(PyObject* callback, PyObject *result; PyObject *stack[3]; - if (PyFrame_FastToLocalsWithError(frame) < 0) { + /* Put any cell references into locals as the actual cells */ + if (_PyFrame_FastToLocalsInternal(frame, 0) < 0) { return NULL; } @@ -469,6 +470,10 @@ call_trampoline(PyObject* callback, /* call the Python-level function */ result = _PyObject_FastCall(callback, stack, 3); + /* All local references will be written back here, but cell references + will only be written back here if they were changed to refer to + something other than the cell itself. + */ PyFrame_LocalsToFast(frame, 1); if (result == NULL) { PyTraceBack_Here(frame); From 3cbb73c8d3ec30434fc82a1e43537a324ac091fb Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 5 Nov 2017 15:57:01 +1000 Subject: [PATCH 02/66] Disable the current broken writeback logic --- Objects/frameobject.c | 41 +++++------------------------------------ Python/ceval.c | 11 ++++++----- Python/sysmodule.c | 10 ---------- 3 files changed, 11 insertions(+), 51 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index bc6f32aee868ea..727fbb92a019f5 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -980,42 +980,11 @@ PyFrame_FastToLocals(PyFrameObject *f) void PyFrame_LocalsToFast(PyFrameObject *f, int clear) { - /* Merge f->f_locals into fast locals */ - PyObject *locals, *map; - PyObject **fast; - PyObject *error_type, *error_value, *error_traceback; - PyCodeObject *co; - Py_ssize_t j; - Py_ssize_t ncells, nfreevars; - if (f == NULL) - return; - locals = f->f_locals; - co = f->f_code; - map = co->co_varnames; - if (locals == NULL) - return; - if (!PyTuple_Check(map)) - return; - PyErr_Fetch(&error_type, &error_value, &error_traceback); - fast = f->f_localsplus; - j = PyTuple_GET_SIZE(map); - if (j > co->co_nlocals) - j = co->co_nlocals; - if (co->co_nlocals) - dict_to_map(co->co_varnames, j, locals, fast, 0, clear); - ncells = PyTuple_GET_SIZE(co->co_cellvars); - nfreevars = PyTuple_GET_SIZE(co->co_freevars); - if (ncells || nfreevars) { - dict_to_map(co->co_cellvars, ncells, - locals, fast + co->co_nlocals, 1, clear); - /* Same test as in PyFrame_FastToLocals() above. */ - if (co->co_flags & CO_OPTIMIZED) { - dict_to_map(co->co_freevars, nfreevars, - locals, fast + co->co_nlocals + ncells, 1, - clear); - } - } - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetString( + PyExc_RuntimeError, + "PyFrame_LocalsToFast is no longer supported. " + "Use PyFrame_GetLocals() instead." + ); } /* Clear out the free list */ diff --git a/Python/ceval.c b/Python/ceval.c index f6519cff590fe1..ccafe55521a455 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2650,10 +2650,12 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) TARGET(IMPORT_STAR) { PyObject *from = POP(), *locals; int err; - if (PyFrame_FastToLocalsWithError(f) < 0) { - Py_DECREF(from); - goto error; - } + /* TODO for PEP 558 + * Report an error here for CO_OPTIMIZED frames + * The 3.x compiler treats wildcard imports as an error inside + * functions, but they can still happen with independently + * constructed opcode sequences + */ locals = f->f_locals; if (locals == NULL) { @@ -2663,7 +2665,6 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) goto error; } err = import_all_from(locals, from); - PyFrame_LocalsToFast(f, 0); Py_DECREF(from); if (err != 0) goto error; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 0d38e5c6238c02..87785ddac6ee15 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -458,11 +458,6 @@ call_trampoline(PyObject* callback, PyObject *result; PyObject *stack[3]; - /* Put any cell references into locals as the actual cells */ - if (_PyFrame_FastToLocalsInternal(frame, 0) < 0) { - return NULL; - } - stack[0] = (PyObject *)frame; stack[1] = whatstrings[what]; stack[2] = (arg != NULL) ? arg : Py_None; @@ -470,11 +465,6 @@ call_trampoline(PyObject* callback, /* call the Python-level function */ result = _PyObject_FastCall(callback, stack, 3); - /* All local references will be written back here, but cell references - will only be written back here if they were changed to refer to - something other than the cell itself. - */ - PyFrame_LocalsToFast(frame, 1); if (result == NULL) { PyTraceBack_Here(frame); } From 01f3f34f9295fb5b8f7a52abe8e5b88497e62261 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 5 Nov 2017 16:59:12 +1000 Subject: [PATCH 03/66] Failing test case for writeback functionality --- Lib/test/test_sys_settrace.py | 100 ++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index bd081255cf5ddc..2db7ea206cf493 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -5,6 +5,7 @@ import sys import difflib import gc +import textwrap # A very basic example. If this fails, we're in deep trouble. def basic(): @@ -930,6 +931,105 @@ def tracefunc(frame, event, arg): self.assertEqual(x_from_nested_ref, [2, 6]) self.assertEqual(x_from_nested_locals, [3, 7]) + def test_locals_writeback_support(self): + # To support interactive debuggers, trace functions are expected to + # be able to reliably modify function locals. However, this should + # NOT enable writebacks via locals() at function scope. + # + # Note: the sample values have numbers in them so mixing up variable + # names in the checks can't accidentally make the test pass - + # you'd have to get both the name *and* expected number wrong + self.maxDiff = None + code = textwrap.dedent(""" + locals()['a_global'] = 'created1' # We expect this to be retained + another_global = 'original2' # Trace func will modify this + + class C: + locals()['an_attr'] = 'created3' # We expect this to be retained + another_attr = 'original4' # Trace func will modify this + a_class_attribute = C.an_attr + another_class_attribute = C.another_attr + del C + + def outer(): + a_nonlocal = 'original5' # We expect this to be retained + another_nonlocal = 'original6' # Trace func will modify this + def inner(): + nonlocal another_nonlocal + a_local = 'original7' # We expect this to be retained + another_local = 'original8' # Trace func will modify this + ns = locals() + ns['a_local'] = 'modified7' # We expect this to be reverted + ns['a_nonlocal'] = 'modified5' # We expect this to be reverted + ns['a_new_local'] = 'created9' # We expect this to be retained + return a_local, another_local, ns + outer_local = 'original10' # Trace func will modify this + # Trigger any updates from the inner function & trace function + inner_result = inner() + outer_result = a_nonlocal, another_nonlocal, outer_local, locals() + return outer_result, inner_result + outer_result, inner_result = outer() + a_nonlocal, another_nonlocal, outer_local, outer_ns = outer_result + a_nonlocal_via_ns = outer_ns['a_nonlocal'] + another_nonlocal_via_ns = outer_ns['another_nonlocal'] + outer_local_via_ns = outer_ns['outer_local'] + a_local, another_local, inner_ns = inner_result + a_local_via_ns = inner_ns['a_local'] + a_nonlocal_via_inner_ns = inner_ns['a_nonlocal'] + another_nonlocal_via_inner_ns = inner_ns['another_nonlocal'] + another_local_via_ns = inner_ns['another_local'] + a_new_local_via_ns = inner_ns['a_new_local'] + del outer, outer_result, outer_ns, inner_result, inner_ns + """ + ) + def tracefunc(frame, event, arg): + if event == "return": + # We leave any state manipulation to the very end + ns = frame.f_locals + co_name = frame.f_code.co_name + if co_name == "C": + # Modify class attributes + ns["another_attr"] = "modified4" + elif co_name == "inner": + # Modify local and nonlocal variables + ns["another_nonlocal"] = "modified6" + ns["another_local"] = "modified8" + outer_ns = frame.f_back.f_locals + outer_ns["outer_local"] = "modified10" + elif co_name == "": + # Modify globals + ns["another_global"] = "modified2" + return tracefunc + actual_ns = {} + sys.settrace(tracefunc) + try: + exec(code, actual_ns) + finally: + sys.settrace(None) + for k in list(actual_ns.keys()): + if k.startswith("_"): + del actual_ns[k] + expected_ns = { + "a_global": "created1", + "another_global": "modified2", + "a_class_attribute": "created3", + "another_class_attribute": "modified4", + "a_nonlocal": "modified5", + "a_nonlocal_via_ns": "modified5", + "a_nonlocal_via_inner_ns": "modified5", + "another_nonlocal": "modified6", + "another_nonlocal_via_ns": "modified6", + "another_nonlocal_via_inner_ns": "modified6", + "a_local": "original7", + "a_local_via_ns": "original7", + "another_local": "modified8", + "another_local_via_ns": "modified8", + "a_new_local_via_ns": "created9", + "outer_local": "modified10", + "outer_local_via_ns": "modified10", + } + self.assertEqual(actual_ns, expected_ns) + def test_main(): support.run_unittest( From 4f6dd934e61ad9deb74944b9479d320a2f28f2e5 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 5 Nov 2017 23:16:40 +1000 Subject: [PATCH 04/66] Initial skeleton for a write-through proxy --- Include/descrobject.h | 5 +- Objects/clinic/frameobject.c.h | 25 +++ Objects/descrobject.c | 5 +- Objects/frameobject.c | 390 +++++++++++++++++++++++++++------ 4 files changed, 359 insertions(+), 66 deletions(-) create mode 100644 Objects/clinic/frameobject.c.h diff --git a/Include/descrobject.h b/Include/descrobject.h index 040cfc97881347..79866675e72f99 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -98,9 +98,12 @@ PyAPI_FUNC(PyObject *) PyDescr_NewWrapper(PyTypeObject *, #define PyDescr_IsData(d) (Py_TYPE(d)->tp_descr_set != NULL) #endif +/* PyDictProxy should really have its own header/impl pair, but keeping + * it here for now... */ + PyAPI_FUNC(PyObject *) PyDictProxy_New(PyObject *); #ifdef Py_BUILD_CORE -PyAPI_FUNC(PyObject *) _PyDictProxy_GetMapping(PyObject *); +PyAPI_DATA(PyTypeObject) PyDictProxy_Type; #endif PyAPI_FUNC(PyObject *) PyWrapper_New(PyObject *, PyObject *); diff --git a/Objects/clinic/frameobject.c.h b/Objects/clinic/frameobject.c.h new file mode 100644 index 00000000000000..10325dcb41ca65 --- /dev/null +++ b/Objects/clinic/frameobject.c.h @@ -0,0 +1,25 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +static PyObject * +fastlocalsproxy_new_impl(PyTypeObject *type, PyObject *frame); + +static PyObject * +fastlocalsproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"frame", NULL}; + static _PyArg_Parser _parser = {"O:fastlocalsproxy", _keywords, 0}; + PyObject *frame; + + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, + &frame)) { + goto exit; + } + return_value = fastlocalsproxy_new_impl(type, frame); + +exit: + return return_value; +} +/*[clinic end generated code: output=5fa72522109d3584 input=a9049054013a1b77]*/ diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 71d522433a2d10..54a2a4bbbc808a 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1011,6 +1011,7 @@ PyDictProxy_New(PyObject *mapping) } + /* --- Wrapper object for "slot" methods --- */ /* This has no reason to be in this file except that adding new files is a @@ -1606,7 +1607,9 @@ PyTypeObject PyDictProxy_Type = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + Py_TPFLAGS_DEFAULT | + Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_BASETYPE, /* tp_flags */ 0, /* tp_doc */ mappingproxy_traverse, /* tp_traverse */ 0, /* tp_clear */ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 727fbb92a019f5..a33560f3779417 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -817,70 +817,6 @@ map_to_dict(PyObject *map, Py_ssize_t nmap, PyObject *dict, PyObject **values, return 0; } -/* Copy values from the "locals" dict into the fast locals. - - dict is an input argument containing string keys representing - variables names and arbitrary PyObject* as values. - - map and values are input arguments. map is a tuple of strings. - values is an array of PyObject*. At index i, map[i] is the name of - the variable with value values[i]. The function gets the new value - for values[i] by looking up map[i] in the dict. - - If clear is true and map[i] is missing from the dict, then values[i] is - set to NULL. If clear is false, then values[i] is left alone in that case. - - If deref is true, then the values being copied are cell variables - and the value is inserted into the cell variable rather than overwriting - the value directly. If the value in the dict *is* the cell itself, then - the cell value is left alone, and instead that value is written back - into the dict (replacing the cell reference). - - Exceptions raised while reading or updating the dict or updating a cell - reference are silently ignored, because there is no good way to report them. -*/ - -static void -dict_to_map(PyObject *map, Py_ssize_t nmap, PyObject *dict, PyObject **values, - int deref, int clear) -{ - Py_ssize_t j; - assert(PyTuple_Check(map)); - assert(PyDict_Check(dict)); - assert(PyTuple_Size(map) >= nmap); - for (j = nmap; --j >= 0; ) { - PyObject *key = PyTuple_GET_ITEM(map, j); - PyObject *value = PyObject_GetItem(dict, key); - assert(PyUnicode_Check(key)); - /* We only care about NULLs if clear is true. */ - if (value == NULL) { - PyErr_Clear(); - if (!clear) - continue; - } - if (deref) { - assert(PyCell_Check(values[j])); - PyObject *cell_value = PyCell_GET(values[j]); - if (values[j] == value) { - /* The dict currently contains the cell itself, so write the - cell's value into the dict rather than the other way around - */ - if (PyObject_SetItem(dict, key, cell_value) != 0) { - PyErr_Clear(); - } - } else if (cell_value != value) { - /* Write the value from the dict back into the cell */ - if (PyCell_Set(values[j], value) < 0) - PyErr_Clear(); - } - } else if (values[j] != value) { - Py_XINCREF(value); - Py_XSETREF(values[j], value); - } - Py_XDECREF(value); - } -} - int _PyFrame_FastToLocalsInternal(PyFrameObject *f, int deref) { @@ -987,6 +923,109 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear) ); } +static int +add_local_refs(PyObject *map, Py_ssize_t nmap, PyObject *fast_refs) +{ + /* Populate a lookup table from variable names to fast locals array indices */ + Py_ssize_t j; + assert(PyTuple_Check(map)); + assert(PyDict_Check(fast_refs)); + assert(PyTuple_Size(map) >= nmap); + for (j = nmap; --j >= 0; ) { + PyObject *key = PyTuple_GET_ITEM(map, j); + PyObject *value = PyLong_FromSsize_t(j); + assert(PyUnicode_Check(key)); + if (PyDict_SetItem(fast_refs, key, value) != 0) { + return -1; + } + } + return 0; +} + +static int +add_nonlocal_refs(PyObject *map, Py_ssize_t nmap, PyObject *fast_refs, PyObject **cells) +{ + /* Populate a lookup table from variable names to closure cell references */ + Py_ssize_t j; + assert(PyTuple_Check(map)); + assert(PyDict_Check(fast_refs)); + assert(PyTuple_Size(map) >= nmap); + for (j = nmap; --j >= 0; ) { + PyObject *key = PyTuple_GET_ITEM(map, j); + PyObject *value = cells[j]; + assert(PyUnicode_Check(key)); + assert(PyCell_Check(value)); + assert(PyUnicode_Check(key)); + if (PyDict_SetItem(fast_refs, key, value) != 0) { + return -1; + } + } + return 0; +} + + +static PyObject * +_PyFrame_BuildFastRefs(PyFrameObject *f) +{ + PyObject *fast_refs, *map; + PyObject **fast; + PyCodeObject *co; + Py_ssize_t j; + Py_ssize_t ncells, nfreevars; + + /* Construct a combined mapping from local variable names to indices + * in the fast locals array, and from nonlocal variable names directly + * to the corresponding cell objects + */ + co = f->f_code; + if (!(co->co_flags & CO_OPTIMIZED)) { + PyErr_SetString(PyExc_SystemError, + "attempted to build fast refs lookup table for non-optimized scope"); + return NULL; + } + map = co->co_varnames; + if (!PyTuple_Check(map)) { + PyErr_Format(PyExc_SystemError, + "co_varnames must be a tuple, not %s", + Py_TYPE(map)->tp_name); + return NULL; + } + fast_refs = PyDict_New(); + if (fast_refs == NULL) { + return NULL; + } + j = PyTuple_GET_SIZE(map); + if (j > co->co_nlocals) + j = co->co_nlocals; + if (co->co_nlocals) { + if (add_local_refs(map, j, fast_refs) < 0) { + Py_DECREF(fast_refs); + return NULL; + } + } + fast = f->f_localsplus; + ncells = PyTuple_GET_SIZE(co->co_cellvars); + if (ncells) { + if (add_nonlocal_refs(co->co_cellvars, ncells, + fast_refs, fast + co->co_nlocals)) { + Py_DECREF(fast_refs); + return NULL; + } + } + + nfreevars = PyTuple_GET_SIZE(co->co_freevars); + if (nfreevars) { + if (add_nonlocal_refs(co->co_freevars, nfreevars, + fast_refs, fast + co->co_nlocals + ncells) < 0) { + Py_DECREF(fast_refs); + return NULL; + } + } + + return fast_refs; + +} + /* Clear out the free list */ int PyFrame_ClearFreeList(void) @@ -1018,3 +1057,226 @@ _PyFrame_DebugMallocStats(FILE *out) numfree, sizeof(PyFrameObject)); } +/* PyFastLocalsProxy_Type + * + * Subclass of PyDict_Proxy (currently defined in descrobject.h/.c) + * + * Mostly works just like PyDict_Proxy (backed by the frame locals), but + * suflports setitem and delitem, with writes being delegated to both the + * referenced mapping *and* the fast locals and/or cell reference on the + * frame. + */ +/*[clinic input] +class fastlocalsproxy "fastlocalsproxyobject *" "&PyFastLocalsProxy_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b0e135835cface9f]*/ + + +typedef struct { + PyObject_HEAD /* Match mappingproxyobject in descrobject.c */ + PyObject *mapping; /* Match mappingproxyobject in descrobject.c */ + PyFrameObject *frame; + PyObject *fast_refs; /* Cell refs and local variable indices */ +} fastlocalsproxyobject; + +/* Provide __setitem__() and __delitem__() implementations that not only + * write to the namespace returned by locals(), but also write to the frame + * storage directly (either the closure cells or the fast locals array) + */ + +static int +fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObject *value) +{ + int result = 0; + PyObject *fast_ref = PyDict_GetItem(flp->fast_refs, key); + if (fast_ref != NULL) { + /* Key is also stored on the frame, so update that reference */ + if (PyCell_Check(fast_ref)) { + result = PyCell_Set(fast_ref, NULL); + } else { + /* TODO: It's Python integer mapping into the fast locals array */ + } + } + return result; +} + +static int +fastlocalsproxy_setitem(fastlocalsproxyobject *flp, PyObject *key, PyObject *value) +{ + int result; + result = PyDict_SetItem(flp->mapping, key, value); + if (result == 0) { + result = fastlocalsproxy_write_to_frame(flp, key, value); + } + return result; +} + +static int +fastlocalsproxy_delitem(fastlocalsproxyobject *flp, PyObject *key) +{ + int result; + result = PyDict_DelItem(flp->mapping, key); + if (result == 0) { + result = fastlocalsproxy_write_to_frame(flp, key, NULL); + } + return result; +} + +static int +fastlocalsproxy_mp_assign_subscript(PyObject *flp, PyObject *key, PyObject *value) +{ + if (value == NULL) + return fastlocalsproxy_delitem((fastlocalsproxyobject *)flp, key); + else + return fastlocalsproxy_setitem((fastlocalsproxyobject *)flp, key, value); +} + +static PyMappingMethods fastlocalsproxy_as_mapping = { + 0, /* mp_length */ + 0, /* mp_subscript */ + fastlocalsproxy_mp_assign_subscript, /* mp_ass_subscript */ +}; + + +/* TODO: Delegate the mutating methods not delegated by mappingproxy */ + +static PyMethodDef fastlocalsproxy_methods[] = { + {0} +}; + +static void +fastlocalsproxy_dealloc(fastlocalsproxyobject *flp) +{ + _PyObject_GC_UNTRACK(flp); + Py_CLEAR(flp->frame); + Py_CLEAR(flp->mapping); + Py_CLEAR(flp->fast_refs); + PyObject_GC_Del(flp); +} + +static PyObject * +fastlocalsproxy_repr(fastlocalsproxyobject *flp) +{ + return PyUnicode_FromFormat("fastlocalsproxy(%R)", flp->mapping); +} + +static int +fastlocalsproxy_traverse(PyObject *self, visitproc visit, void *arg) +{ + fastlocalsproxyobject *flp = (fastlocalsproxyobject *)self; + Py_VISIT(flp->frame); + Py_VISIT(flp->mapping); + Py_VISIT(flp->fast_refs); + return 0; +} + +static int +fastlocalsproxy_check_frame(PyObject *maybe_frame) +{ + /* This is an internal-only API, so getting bad arguments means something + * already went wrong elsewhere in the interpreter code. + */ + if (!PyFrame_Check(maybe_frame)) { + PyErr_Format(PyExc_SystemError, + "fastlocalsproxy() argument must be a frame, not %s", + Py_TYPE(maybe_frame)->tp_name); + return -1; + } + + PyFrameObject *frame = (PyFrameObject *) maybe_frame; + if (!(frame->f_code->co_flags & CO_OPTIMIZED)) { + PyErr_SetString(PyExc_SystemError, + "fastlocalsproxy() argument must be a frame using fast locals"); + return -1; + } else if (frame->f_locals == NULL) { + PyErr_SetString(PyExc_SystemError, + "fastlocalsproxy() argument must already have an allocated locals namespace"); + return -1; + } else if (!PyDict_CheckExact(frame->f_locals)) { + PyErr_SetString(PyExc_SystemError, + "fastlocalsproxy() argument must be a builtin dict"); + return -1; + } + return 0; +} + +static PyObject * +_PyFastLocalsProxy_New(PyObject *frame) +{ + fastlocalsproxyobject *flp; + PyObject *mapping; + + if (fastlocalsproxy_check_frame(frame) == -1) + return NULL; + + flp = PyObject_GC_New(fastlocalsproxyobject, &PyDictProxy_Type); + if (flp == NULL) + return NULL; + Py_INCREF(frame); + flp->frame = (PyFrameObject *) frame; + mapping = flp->frame->f_locals; + Py_INCREF(mapping); + flp->mapping = mapping; + flp->fast_refs = _PyFrame_BuildFastRefs(flp->frame); + _PyObject_GC_TRACK(flp); + return (PyObject *)flp; +} + +/*[clinic input] +@classmethod +fastlocalsproxy.__new__ as fastlocalsproxy_new + + frame: object + +[clinic start generated code]*/ + +static PyObject * +fastlocalsproxy_new_impl(PyTypeObject *type, PyObject *frame) +/*[clinic end generated code: output=058c588af86f1525 input=049f74502e02fc63]*/ +{ + return _PyFastLocalsProxy_New(frame); +} + +#include "clinic/frameobject.c.h" + +PyTypeObject PyFastLocalsProxy_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "fastlocalsproxy", /* tp_name */ + sizeof(fastlocalsproxyobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)fastlocalsproxy_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + (reprfunc)fastlocalsproxy_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + &fastlocalsproxy_as_mapping, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)fastlocalsproxy_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + fastlocalsproxy_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyDictProxy_Type, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + fastlocalsproxy_new, /* tp_new */ + 0, /* tp_free */ +}; From acbf5876e8a7c9b3b9b7b82f17b9e9dee726df6c Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 22 Apr 2019 01:57:07 +1000 Subject: [PATCH 05/66] Finish adding the write-through proxy --- Include/frameobject.h | 8 ++++++ Lib/test/test_sys_settrace.py | 9 ++++--- Objects/frameobject.c | 50 ++++++++++++++++++++++++++++++----- Objects/object.c | 1 + 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/Include/frameobject.h b/Include/frameobject.h index bb3eb316858ec4..46e728477b3b1a 100644 --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -89,6 +89,14 @@ PyAPI_FUNC(void) _PyFrame_DebugMallocStats(FILE *out); /* Return the line of code the frame is currently executing. */ PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *); + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03080000 +/* Fast locals proxy for reliable write-through from trace functions */ +PyTypeObject PyFastLocalsProxy_Type; +#define _PyFastLocalsProxy_CheckExact(self) \ + (Py_TYPE(self) == &PyFastLocalsProxy_Type) +#endif + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 8e9a93505899ed..0dde3c2390a081 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1525,14 +1525,17 @@ def tracefunc(frame, event, arg): for k in list(actual_ns.keys()): if k.startswith("_"): del actual_ns[k] + # CURRENT FAILURE STATUS: + # - trace functions mutating state as expected + # - proxy is incorrectly being returned by locals(), so that also mutates expected_ns = { "a_global": "created1", "another_global": "modified2", "a_class_attribute": "created3", "another_class_attribute": "modified4", - "a_nonlocal": "modified5", - "a_nonlocal_via_ns": "modified5", - "a_nonlocal_via_inner_ns": "modified5", + "a_nonlocal": "original5", + "a_nonlocal_via_ns": "original5", + "a_nonlocal_via_inner_ns": "original5", "another_nonlocal": "modified6", "another_nonlocal_via_ns": "modified6", "another_nonlocal_via_inner_ns": "modified6", diff --git a/Objects/frameobject.c b/Objects/frameobject.c index fa1d346cdc4bb2..d30a0b9aef4ac3 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -22,6 +22,10 @@ * */ +static PyObject *_PyFastLocalsProxy_New(PyObject *frame); +static PyObject *_PyFastLocalsProxy_GetLocals(PyObject *flp); + + static PyMemberDef frame_memberlist[] = { {"f_back", T_OBJECT, OFF(f_back), READONLY}, {"f_code", T_OBJECT, OFF(f_code), READONLY}, @@ -840,11 +844,21 @@ _PyFrame_FastToLocalsInternal(PyFrameObject *f, int deref) * * If a trace function is active, ensure f_locals is a fastlocalsproxy * instance, while locals still refers to the underlying mapping. - * If a trace function is *not* active, discard the proxy, if any (and - * invalidate its reference back to the frame) to disallow further writes. + * If a trace function is *not* active, leave any existing proxies alone, + * but also don't create any new ones. */ - co = f->f_code; + if (f->f_trace && (co->co_flags & CO_OPTIMIZED) && PyDict_Check(locals)) { + PyObject *flp = _PyFastLocalsProxy_New((PyObject *) f); + if (!flp) { + return -1; + } + f->f_locals = flp; + Py_DECREF(locals); // The proxy now holds the reference to the snapshot + } else if (_PyFastLocalsProxy_CheckExact(locals)) { + locals = _PyFastLocalsProxy_GetLocals(locals); + } + map = co->co_varnames; if (!PyTuple_Check(map)) { PyErr_Format(PyExc_SystemError, @@ -1061,7 +1075,7 @@ _PyFrame_DebugMallocStats(FILE *out) * Subclass of PyDict_Proxy (currently defined in descrobject.h/.c) * * Mostly works just like PyDict_Proxy (backed by the frame locals), but - * suflports setitem and delitem, with writes being delegated to both the + * supports setitem and delitem, with writes being delegated to both the * referenced mapping *and* the fast locals and/or cell reference on the * frame. */ @@ -1091,9 +1105,20 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje if (fast_ref != NULL) { /* Key is also stored on the frame, so update that reference */ if (PyCell_Check(fast_ref)) { - result = PyCell_Set(fast_ref, NULL); + result = PyCell_Set(fast_ref, value); } else { - /* TODO: It's Python integer mapping into the fast locals array */ + /* Fast ref is a Python int mapping into the fast locals array */ + Py_ssize_t offset = PyLong_AsSsize_t(fast_ref); + Py_ssize_t max_offset = flp->frame->f_code->co_nlocals; + if (offset < 0) { + result = -1; + } else if (offset > max_offset) { + PyErr_Format(PyExc_SystemError, + "Fast locals ref (%d) exceeds array bound (%d)", + offset, max_offset); + result = -1; + } + flp->frame->f_localsplus[offset] = value; } } return result; @@ -1199,6 +1224,7 @@ fastlocalsproxy_check_frame(PyObject *maybe_frame) return 0; } + static PyObject * _PyFastLocalsProxy_New(PyObject *frame) { @@ -1208,7 +1234,7 @@ _PyFastLocalsProxy_New(PyObject *frame) if (fastlocalsproxy_check_frame(frame) == -1) return NULL; - flp = PyObject_GC_New(fastlocalsproxyobject, &PyDictProxy_Type); + flp = PyObject_GC_New(fastlocalsproxyobject, &PyFastLocalsProxy_Type); if (flp == NULL) return NULL; Py_INCREF(frame); @@ -1221,6 +1247,16 @@ _PyFastLocalsProxy_New(PyObject *frame) return (PyObject *)flp; } +static PyObject * +_PyFastLocalsProxy_GetLocals(PyObject *self) +{ + if (!_PyFastLocalsProxy_CheckExact(self)) { + PyErr_BadInternalCall(); + return NULL; + } + return ((fastlocalsproxyobject *) self)->mapping; +} + /*[clinic input] @classmethod fastlocalsproxy.__new__ as fastlocalsproxy_new diff --git a/Objects/object.c b/Objects/object.c index e7ec7aec490f8b..f032316b64f215 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1813,6 +1813,7 @@ _PyTypes_Init(void) INIT_TYPE(&PyMethod_Type, "method"); INIT_TYPE(&PyFunction_Type, "function"); INIT_TYPE(&PyDictProxy_Type, "dict proxy"); + INIT_TYPE(&PyFastLocalsProxy_Type, "fast locals proxy"); INIT_TYPE(&PyGen_Type, "generator"); INIT_TYPE(&PyGetSetDescr_Type, "get-set descriptor"); INIT_TYPE(&PyWrapperDescr_Type, "wrapper"); From 5ea8bcfb9192f35e9e421efd634651b14def0b1f Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 22 May 2019 22:22:48 +1000 Subject: [PATCH 06/66] Add test case for the PEP 558 locals() behaviour --- Lib/test/test_scope.py | 54 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 4239b26408ecdf..e0e6ddec127f89 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -757,5 +757,59 @@ def dig(self): self.assertIsNone(ref()) + +LOCALS_SEMANTICS_TEST_CODE = """\ +global_ns = globals() +known_var = "original" +local_ns_1 = locals() +local_ns_1["known_var"] = "set_via_locals" +local_ns_1["unknown_var"] = "set_via_locals" +local_ns_2 = locals() +""" + + +class TestLocalsSemantics(unittest.TestCase): + # This is a new set of test cases added as part of the implementation + # of PEP 558 to cover the expected behaviour of locals() + # The expected behaviour of frame.f_locals is covered in test_sys_settrace + + def test_locals_update_semantics_at_module_scope(self): + # At module scope, globals() and locals() are the same namespace + global_ns = dict() + exec(LOCALS_SEMANTICS_TEST_CODE, global_ns) + self.assertIs(global_ns["global_ns"], global_ns) + self.assertIs(global_ns["local_ns_1"], global_ns) + self.assertIs(global_ns["local_ns_2"], global_ns) + self.assertEqual(global_ns["known_var"], "set_via_locals") + self.assertEqual(global_ns["unknown_var"], "set_via_locals") + + def test_locals_update_semantics_at_class_scope(self): + # At class scope, globals() and locals() are different namespaces + global_ns = dict() + local_ns = dict() + exec(LOCALS_SEMANTICS_TEST_CODE, global_ns, local_ns) + self.assertIs(local_ns["global_ns"], global_ns) + self.assertIs(local_ns["local_ns_1"], local_ns) + self.assertIs(local_ns["local_ns_2"], local_ns) + self.assertEqual(local_ns["known_var"], "set_via_locals") + self.assertEqual(local_ns["unknown_var"], "set_via_locals") + + def test_locals_update_semantics_at_function_scope(self): + def function_local_semantics(): + global_ns = globals() + known_var = "original" + local_ns = locals() + local_ns["known_var"] = "set_via_locals" + local_ns["unknown_var"] = "set_via_locals" + return locals() + + global_ns = globals() + local_ns = function_local_semantics() + self.assertIs(local_ns["global_ns"], global_ns) + self.assertIs(local_ns["local_ns"], local_ns) + self.assertEqual(local_ns["known_var"], "original") + self.assertEqual(local_ns["unknown_var"], "set_via_locals") + + if __name__ == '__main__': unittest.main() From fe928915d4c8fad7426ebcab33a115cf06252886 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Thu, 23 May 2019 00:34:57 +1000 Subject: [PATCH 07/66] Actually implement most of the PEP and fix the tests --- Include/frameobject.h | 26 ++++--- Lib/test/test_scope.py | 2 + Lib/test/test_sys_settrace.py | 61 ++++++++++++--- Objects/frameobject.c | 135 +++++++++++++++++----------------- Python/ceval.c | 6 +- Python/sysmodule.c | 4 + 6 files changed, 140 insertions(+), 94 deletions(-) diff --git a/Include/frameobject.h b/Include/frameobject.h index 0b8e0664fb42c3..b938ecb39c901d 100644 --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -71,16 +71,6 @@ PyAPI_FUNC(PyTryBlock *) PyFrame_BlockPop(PyFrameObject *); PyAPI_FUNC(PyObject **) PyFrame_ExtendStack(PyFrameObject *, int, int); -/* Conversions between "fast locals" and locals in dictionary */ - -PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int); - -PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); -PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); -#ifdef Py_BUILD_CORE -int _PyFrame_FastToLocalsInternal(PyFrameObject *f, int); /* For sys.settrace */ -#endif - PyAPI_FUNC(int) PyFrame_ClearFreeList(void); PyAPI_FUNC(void) _PyFrame_DebugMallocStats(FILE *out); @@ -89,13 +79,29 @@ PyAPI_FUNC(void) _PyFrame_DebugMallocStats(FILE *out); PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *); +/* Conversions between "fast locals" and locals in dictionary */ +PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); +PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03080000 /* Fast locals proxy for reliable write-through from trace functions */ PyTypeObject PyFastLocalsProxy_Type; #define _PyFastLocalsProxy_CheckExact(self) \ (Py_TYPE(self) == &PyFastLocalsProxy_Type) + +/* Access the frame locals mapping */ +PyAPI_FUNC(PyObject *) PyFrame_GetPyLocals(PyFrameObject *); // = locals() +PyAPI_FUNC(PyObject *) PyFrame_GetLocalsAttr(PyFrameObject *); // = frame.f_locals +#endif + +#ifdef Py_BUILD_CORE +PyObject *_PyFrame_BorrowPyLocals(PyFrameObject *f); /* For PyEval_GetLocals() */ #endif + +/* This always raises RuntimeError now (use PyFrame_GetLocalsAttr() instead) */ +PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index e0e6ddec127f89..88cdb0ab77bea9 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -804,7 +804,9 @@ def function_local_semantics(): return locals() global_ns = globals() + self.assertIsInstance(global_ns, dict) local_ns = function_local_semantics() + self.assertIsInstance(local_ns, dict) self.assertIs(local_ns["global_ns"], global_ns) self.assertIs(local_ns["local_ns"], local_ns) self.assertEqual(local_ns["known_var"], "original") diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 0dde3c2390a081..809b24d58517c3 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1447,11 +1447,45 @@ def tracefunc(frame, event, arg): self.assertEqual(x_from_nested_ref, [2, 6]) self.assertEqual(x_from_nested_locals, [3, 7]) - def test_locals_writeback_support(self): + def test_locals_writethrough_proxy(self): # To support interactive debuggers, trace functions are expected to # be able to reliably modify function locals. However, this should # NOT enable writebacks via locals() at function scope. # + # This is a simple scenario to test the core behaviour of the + # writethrough proxy + local_var = "original" + # Check regular locals() snapshot + locals_snapshot = locals() + self.assertEqual(locals_snapshot["local_var"], "original") + locals_snapshot["local_var"] = "modified" + self.assertEqual(local_var, "original") + # Check writethrough proxy on frame + locals_proxy = sys._getframe().f_locals + self.assertEqual(locals_proxy["local_var"], "original") + locals_proxy["local_var"] = "modified" + self.assertEqual(local_var, "modified") + # Check handling of closures + def nested_scope(): + nonlocal local_var + return local_var, locals(), sys._getframe().f_locals + closure_var, inner_snapshot, inner_proxy = nested_scope() + self.assertEqual(closure_var, "modified") + self.assertEqual(inner_snapshot["local_var"], "modified") + self.assertEqual(inner_proxy["local_var"], "modified") + inner_snapshot["local_var"] = "modified_again" + self.assertEqual(local_var, "modified") + self.assertEqual(inner_snapshot["local_var"], "modified_again") + self.assertEqual(inner_proxy["local_var"], "modified_again") # Q: Is this desirable? + inner_proxy["local_var"] = "modified_yet_again" + self.assertEqual(local_var, "modified_yet_again") + self.assertEqual(inner_snapshot["local_var"], "modified_yet_again") + self.assertEqual(inner_proxy["local_var"], "modified_yet_again") + + def test_locals_writeback_complex_scenario(self): + # Further locals writeback testing using a more complex scenario + # involving multiple scopes of different kinds + # # Note: the sample values have numbers in them so mixing up variable # names in the checks can't accidentally make the test pass - # you'd have to get both the name *and* expected number wrong @@ -1471,7 +1505,7 @@ def outer(): a_nonlocal = 'original5' # We expect this to be retained another_nonlocal = 'original6' # Trace func will modify this def inner(): - nonlocal another_nonlocal + nonlocal a_nonlocal, another_nonlocal a_local = 'original7' # We expect this to be retained another_local = 'original8' # Trace func will modify this ns = locals() @@ -1492,6 +1526,7 @@ def inner(): a_local, another_local, inner_ns = inner_result a_local_via_ns = inner_ns['a_local'] a_nonlocal_via_inner_ns = inner_ns['a_nonlocal'] + print(a_nonlocal_via_inner_ns) another_nonlocal_via_inner_ns = inner_ns['another_nonlocal'] another_local_via_ns = inner_ns['another_local'] a_new_local_via_ns = inner_ns['a_new_local'] @@ -1499,22 +1534,28 @@ def inner(): """ ) def tracefunc(frame, event, arg): + ns = frame.f_locals + co_name = frame.f_code.co_name if event == "return": - # We leave any state manipulation to the very end - ns = frame.f_locals - co_name = frame.f_code.co_name + # We leave most state manipulation to the very end if co_name == "C": # Modify class attributes ns["another_attr"] = "modified4" elif co_name == "inner": - # Modify local and nonlocal variables + # Modify variables in outer scope ns["another_nonlocal"] = "modified6" - ns["another_local"] = "modified8" outer_ns = frame.f_back.f_locals outer_ns["outer_local"] = "modified10" elif co_name == "": # Modify globals ns["another_global"] = "modified2" + elif event == "line" and co_name == "inner": + # This is the one item we can't leave to the end, as the + # return tuple is already built by the time the return event + # fires, and we want to manipulate one of the entries in that + # So instead, we just mutate it on every line trace event + ns["another_local"] = "modified8" + print(event) return tracefunc actual_ns = {} sys.settrace(tracefunc) @@ -1525,9 +1566,6 @@ def tracefunc(frame, event, arg): for k in list(actual_ns.keys()): if k.startswith("_"): del actual_ns[k] - # CURRENT FAILURE STATUS: - # - trace functions mutating state as expected - # - proxy is incorrectly being returned by locals(), so that also mutates expected_ns = { "a_global": "created1", "another_global": "modified2", @@ -1547,7 +1585,8 @@ def tracefunc(frame, event, arg): "outer_local": "modified10", "outer_local_via_ns": "modified10", } - self.assertEqual(actual_ns, expected_ns) + # Expected is first so any error diff is easier to read + self.assertEqual(expected_ns, actual_ns) if __name__ == "__main__": diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d30a0b9aef4ac3..f331faf5071cc7 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -11,20 +11,8 @@ #define OFF(x) offsetof(PyFrameObject, x) -/* PEP558: - * - * Forward declaration of fastlocalsproxy - * PyEval_GetLocals will need a new PyFrame_GetLocals() helper function - * that ensures it always gets the snapshot reference and never the proxy - * even when tracing is enabled. - * That should probably be a public API for the benefit of third party debuggers - * implemented in C. - * - */ - static PyObject *_PyFastLocalsProxy_New(PyObject *frame); -static PyObject *_PyFastLocalsProxy_GetLocals(PyObject *flp); - +static PyObject *_PyFastLocalsProxy_BorrowLocals(PyObject *flp); static PyMemberDef frame_memberlist[] = { {"f_back", T_OBJECT, OFF(f_back), READONLY}, @@ -38,14 +26,48 @@ static PyMemberDef frame_memberlist[] = { }; static PyObject * -frame_getlocals(PyFrameObject *f, void *closure) +_frame_get_updated_locals(PyFrameObject *f) { if (PyFrame_FastToLocalsWithError(f) < 0) return NULL; - Py_INCREF(f->f_locals); + assert(f->f_locals != NULL); return f->f_locals; } +PyObject * +PyFrame_GetLocalsAttr(PyFrameObject *f) +{ + PyObject *updated_locals =_frame_get_updated_locals(f); + Py_INCREF(updated_locals); + return updated_locals; +} + +PyObject * +_PyFrame_BorrowPyLocals(PyFrameObject *f) +{ + // This is called by PyEval_GetLocals(), which has historically returned + // a borrowed reference, so this does the same + PyObject *updated_locals =_frame_get_updated_locals(f); + if (_PyFastLocalsProxy_CheckExact(updated_locals)) { + updated_locals = _PyFastLocalsProxy_BorrowLocals(updated_locals); + } + return updated_locals; +} + +PyObject * +PyFrame_GetPyLocals(PyFrameObject *f) +{ + PyObject *updated_locals =_PyFrame_BorrowPyLocals(f); + Py_INCREF(updated_locals); + return updated_locals; +} + +static PyObject * +frame_getlocals(PyFrameObject *f, void *__unused) +{ + return PyFrame_GetLocalsAttr(f); +} + int PyFrame_GetLineNumber(PyFrameObject *f) { @@ -821,7 +843,7 @@ map_to_dict(PyObject *map, Py_ssize_t nmap, PyObject *dict, PyObject **values, } int -_PyFrame_FastToLocalsInternal(PyFrameObject *f, int deref) +PyFrame_FastToLocalsWithError(PyFrameObject *f) { /* Merge fast locals into f->f_locals */ PyObject *locals, *map; @@ -834,29 +856,26 @@ _PyFrame_FastToLocalsInternal(PyFrameObject *f, int deref) PyErr_BadInternalCall(); return -1; } + co = f->f_code; locals = f->f_locals; if (locals == NULL) { - locals = f->f_locals = PyDict_New(); - if (locals == NULL) - return -1; - } - /* PEP558: - * - * If a trace function is active, ensure f_locals is a fastlocalsproxy - * instance, while locals still refers to the underlying mapping. - * If a trace function is *not* active, leave any existing proxies alone, - * but also don't create any new ones. - */ - co = f->f_code; - if (f->f_trace && (co->co_flags & CO_OPTIMIZED) && PyDict_Check(locals)) { - PyObject *flp = _PyFastLocalsProxy_New((PyObject *) f); - if (!flp) { - return -1; + if (co->co_flags & CO_OPTIMIZED) { + /* PEP 558: If this is an optimized frame, ensure f_locals is a + * fastlocalsproxy instance, while locals refers to the underlying mapping. + */ + PyObject *flp = _PyFastLocalsProxy_New((PyObject *) f); + if (flp == NULL) { + return -1; + } + f->f_locals = flp; + locals = _PyFastLocalsProxy_BorrowLocals(flp); + } else { + locals = f->f_locals = PyDict_New(); + if (locals == NULL) + return -1; } - f->f_locals = flp; - Py_DECREF(locals); // The proxy now holds the reference to the snapshot } else if (_PyFastLocalsProxy_CheckExact(locals)) { - locals = _PyFastLocalsProxy_GetLocals(locals); + locals = _PyFastLocalsProxy_BorrowLocals(locals); } map = co->co_varnames; @@ -877,17 +896,11 @@ _PyFrame_FastToLocalsInternal(PyFrameObject *f, int deref) ncells = PyTuple_GET_SIZE(co->co_cellvars); nfreevars = PyTuple_GET_SIZE(co->co_freevars); if (ncells || nfreevars) { - /* If deref is true, we'll replace cells with their values in the - namespace. If it's false, we'll include the cells themselves, which - means PyFrame_LocalsToFast will skip writing them back (unless - they've actually been modified). - - The trace hook implementation relies on this to allow debuggers to - inject changes to local variables without inadvertently resetting - closure variables to a previous value. + /* Passing deref to map_to_dict means we'll replace cells with their + values in the namespace. */ if (map_to_dict(co->co_cellvars, ncells, - locals, fast + co->co_nlocals, deref)) + locals, fast + co->co_nlocals, 1)) return -1; /* If the namespace is unoptimized, then one of the @@ -900,7 +913,7 @@ _PyFrame_FastToLocalsInternal(PyFrameObject *f, int deref) */ if (co->co_flags & CO_OPTIMIZED) { if (map_to_dict(co->co_freevars, nfreevars, - locals, fast + co->co_nlocals + ncells, deref) < 0) + locals, fast + co->co_nlocals + ncells, 1) < 0) return -1; } } @@ -908,12 +921,6 @@ _PyFrame_FastToLocalsInternal(PyFrameObject *f, int deref) return 0; } -int -PyFrame_FastToLocalsWithError(PyFrameObject *f) -{ - return _PyFrame_FastToLocalsInternal(f, 1); -} - void PyFrame_FastToLocals(PyFrameObject *f) { @@ -932,7 +939,7 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear) PyErr_SetString( PyExc_RuntimeError, "PyFrame_LocalsToFast is no longer supported. " - "Use PyFrame_GetLocals() instead." + "Use PyFrame_GetPyLocals() instead." ); } @@ -1212,14 +1219,6 @@ fastlocalsproxy_check_frame(PyObject *maybe_frame) PyErr_SetString(PyExc_SystemError, "fastlocalsproxy() argument must be a frame using fast locals"); return -1; - } else if (frame->f_locals == NULL) { - PyErr_SetString(PyExc_SystemError, - "fastlocalsproxy() argument must already have an allocated locals namespace"); - return -1; - } else if (!PyDict_CheckExact(frame->f_locals)) { - PyErr_SetString(PyExc_SystemError, - "fastlocalsproxy() argument must be a builtin dict"); - return -1; } return 0; } @@ -1237,23 +1236,23 @@ _PyFastLocalsProxy_New(PyObject *frame) flp = PyObject_GC_New(fastlocalsproxyobject, &PyFastLocalsProxy_Type); if (flp == NULL) return NULL; + mapping = PyDict_New(); + if (mapping == NULL) { + Py_DECREF(flp); + return NULL; + } + flp->mapping = mapping; Py_INCREF(frame); flp->frame = (PyFrameObject *) frame; - mapping = flp->frame->f_locals; - Py_INCREF(mapping); - flp->mapping = mapping; flp->fast_refs = _PyFrame_BuildFastRefs(flp->frame); _PyObject_GC_TRACK(flp); return (PyObject *)flp; } static PyObject * -_PyFastLocalsProxy_GetLocals(PyObject *self) +_PyFastLocalsProxy_BorrowLocals(PyObject *self) { - if (!_PyFastLocalsProxy_CheckExact(self)) { - PyErr_BadInternalCall(); - return NULL; - } + assert(_PyFastLocalsProxy_CheckExact(self)); return ((fastlocalsproxyobject *) self)->mapping; } diff --git a/Python/ceval.c b/Python/ceval.c index 4da64339bec631..d658446c69985e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4683,11 +4683,7 @@ PyEval_GetLocals(void) return NULL; } - if (PyFrame_FastToLocalsWithError(current_frame) < 0) - return NULL; - - assert(current_frame->f_locals != NULL); - return current_frame->f_locals; + return _PyFrame_BorrowPyLocals(current_frame); } PyObject * diff --git a/Python/sysmodule.c b/Python/sysmodule.c index d4e069f3da5e35..e50932dcb69709 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -507,6 +507,10 @@ call_trampoline(PyObject* callback, PyObject *result; PyObject *stack[3]; + if (PyFrame_FastToLocalsWithError(frame) < 0) { + return NULL; + } + stack[0] = (PyObject *)frame; stack[1] = whatstrings[what]; stack[2] = (arg != NULL) ? arg : Py_None; From ac9e0bfc68cf24da77e35b9448f61eb5ba72f198 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 27 May 2019 23:45:49 +1000 Subject: [PATCH 08/66] Fix segfault on cleared frames --- Objects/frameobject.c | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index f331faf5071cc7..3b8a0e40ec6e2f 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -534,6 +534,8 @@ frame_tp_clear(PyFrameObject *f) for (i = slots; --i >= 0; ++fastlocals) Py_CLEAR(*fastlocals); + /* TODO: The fast locals proxy is no longer valid here... */ + /* stack */ if (oldtop != NULL) { for (p = f->f_valuestack; p < oldtop; p++) @@ -955,8 +957,11 @@ add_local_refs(PyObject *map, Py_ssize_t nmap, PyObject *fast_refs) PyObject *key = PyTuple_GET_ITEM(map, j); PyObject *value = PyLong_FromSsize_t(j); assert(PyUnicode_Check(key)); - if (PyDict_SetItem(fast_refs, key, value) != 0) { - return -1; + /* Values may be missing if the frame has been cleared */ + if (value != NULL) { + if (PyDict_SetItem(fast_refs, key, value) != 0) { + return -1; + } } } return 0; @@ -974,10 +979,13 @@ add_nonlocal_refs(PyObject *map, Py_ssize_t nmap, PyObject *fast_refs, PyObject PyObject *key = PyTuple_GET_ITEM(map, j); PyObject *value = cells[j]; assert(PyUnicode_Check(key)); - assert(PyCell_Check(value)); - assert(PyUnicode_Check(key)); - if (PyDict_SetItem(fast_refs, key, value) != 0) { - return -1; + /* Values may be missing if the frame has been cleared */ + if (value != NULL) { + assert(PyCell_Check(value)); + assert(PyUnicode_Check(key)); + if (PyDict_SetItem(fast_refs, key, value) != 0) { + return -1; + } } } return 0; @@ -1121,7 +1129,7 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje result = -1; } else if (offset > max_offset) { PyErr_Format(PyExc_SystemError, - "Fast locals ref (%d) exceeds array bound (%d)", + "Fast locals ref (%z) exceeds array bound (%z)", offset, max_offset); result = -1; } @@ -1228,7 +1236,7 @@ static PyObject * _PyFastLocalsProxy_New(PyObject *frame) { fastlocalsproxyobject *flp; - PyObject *mapping; + PyObject *mapping, *fast_refs; if (fastlocalsproxy_check_frame(frame) == -1) return NULL; @@ -1244,7 +1252,14 @@ _PyFastLocalsProxy_New(PyObject *frame) flp->mapping = mapping; Py_INCREF(frame); flp->frame = (PyFrameObject *) frame; - flp->fast_refs = _PyFrame_BuildFastRefs(flp->frame); + fast_refs = _PyFrame_BuildFastRefs(flp->frame); + if (fast_refs == NULL) { + Py_DECREF(flp); + Py_DECREF(mapping); + Py_DECREF(frame); + return NULL; + } + flp->fast_refs = fast_refs; _PyObject_GC_TRACK(flp); return (PyObject *)flp; } From 6774e71cf41de77860527d534c6c92b3b2716677 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 27 May 2019 23:46:47 +1000 Subject: [PATCH 09/66] Use correct printf formatting code --- Objects/frameobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 3b8a0e40ec6e2f..7eec952b196229 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1129,7 +1129,7 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje result = -1; } else if (offset > max_offset) { PyErr_Format(PyExc_SystemError, - "Fast locals ref (%z) exceeds array bound (%z)", + "Fast locals ref (%zd) exceeds array bound (%zd)", offset, max_offset); result = -1; } From 74c51e447e83746b11cf9996fe5eac285f019b8d Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 29 May 2019 00:51:37 +1000 Subject: [PATCH 10/66] Initial skeleton for other mutable mapping methods --- Objects/frameobject.c | 307 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 305 insertions(+), 2 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 7eec952b196229..ca3d044991df66 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1177,10 +1177,131 @@ static PyMappingMethods fastlocalsproxy_as_mapping = { }; -/* TODO: Delegate the mutating methods not delegated by mappingproxy */ +/* setdefault() */ + +PyDoc_STRVAR(fastlocalsproxy_setdefault__doc__, +"flp.setdefault(k[, d=None]) -> v, Insert key with a value of default if key\n\ + is not in the dictionary.\n\n\ + Return the value for key if key is in the dictionary, else default."); + + + +static PyObject * +fastlocalsproxy_setdefault(PyObject *flp, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"key", "default", 0}; + PyObject *key, *failobj = NULL; + + /* borrowed */ + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O:pop", kwlist, + &key, &failobj)) { + return NULL; + } + + PyObject *value = NULL; + + PyErr_Format(PyExc_NotImplementedError, + "FastLocalsProxy does not yet implement setdefault()"); + return value; +} + + +/* pop() */ + +PyDoc_STRVAR(fastlocalsproxy_pop__doc__, +"flp.pop(k[,d]) -> v, remove specified key and return the corresponding\n\ + value. If key is not found, d is returned if given, otherwise KeyError\n\ + is raised."); + +/* forward */ +static PyObject * _fastlocalsproxy_popkey(PyObject *, PyObject *, PyObject *); + +static PyObject * +fastlocalsproxy_pop(PyObject *flp, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"key", "default", 0}; + PyObject *key, *failobj = NULL; + + /* borrowed */ + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O:pop", kwlist, + &key, &failobj)) { + return NULL; + } + + return _fastlocalsproxy_popkey(flp, key, failobj); +} + +static PyObject * +_fastlocalsproxy_popkey_hash(PyObject *flp, PyObject *key, PyObject *failobj, + Py_hash_t hash) +{ + PyObject *value = NULL; + + PyErr_Format(PyExc_NotImplementedError, + "FastLocalsProxy does not yet implement pop()"); + return value; +} + +static PyObject * +_fastlocalsproxy_popkey(PyObject *flp, PyObject *key, PyObject *failobj) +{ + Py_hash_t hash = PyObject_Hash(key); + if (hash == -1) + return NULL; + + return _fastlocalsproxy_popkey_hash(flp, key, failobj, hash); +} + +/* popitem() */ + +PyDoc_STRVAR(fastlocalsproxy_popitem__doc__, +"flp.popitem() -> (k, v), remove and return some (key, value) pair as a\n\ + 2-tuple; but raise KeyError if D is empty."); + +static PyObject * +fastlocalsproxy_popitem(PyObject *flp, PyObject *Py_UNUSED(ignored)) +{ + PyErr_Format(PyExc_NotImplementedError, + "FastLocalsProxy does not yet implement popitem()"); + return NULL; +} + +/* update() */ + +/* MutableMapping.update() does not have a docstring. */ +PyDoc_STRVAR(fastlocalsproxy_update__doc__, ""); + +/* forward */ +static PyObject * mutablemapping_update(PyObject *, PyObject *, PyObject *); + +#define fastlocalsproxy_update mutablemapping_update + +/* clear() */ + +PyDoc_STRVAR(fastlocalsproxy_clear__doc__, + "flp.clear() -> None. Remove all items from snapshot and frame."); + +static PyObject * +fastlocalsproxy_clear(register PyObject *flp, PyObject *Py_UNUSED(ignored)) +{ + PyErr_Format(PyExc_NotImplementedError, + "FastLocalsProxy does not yet implement clear()"); + return NULL; +} static PyMethodDef fastlocalsproxy_methods[] = { - {0} + {"setdefault", (PyCFunction)(void(*)(void))fastlocalsproxy_setdefault, + METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_pop__doc__}, + {"pop", (PyCFunction)(void(*)(void))fastlocalsproxy_pop, + METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_pop__doc__}, + {"clear", (PyCFunction)fastlocalsproxy_popitem, + METH_NOARGS, fastlocalsproxy_popitem__doc__}, + {"update", (PyCFunction)(void(*)(void))fastlocalsproxy_update, + METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_update__doc__}, + {"clear", (PyCFunction)fastlocalsproxy_clear, + METH_NOARGS, fastlocalsproxy_clear__doc__}, + + {NULL, NULL} /* sentinel */ }; static void @@ -1329,3 +1450,185 @@ PyTypeObject PyFastLocalsProxy_Type = { fastlocalsproxy_new, /* tp_new */ 0, /* tp_free */ }; + + + +//========================================================================== +// The rest of this file is currently DUPLICATED CODE from odictobject.c +// +// TODO: move the duplicated code to abstract.c and expose it to the +// linker as a private API +// +//========================================================================== + +static int +mutablemapping_add_pairs(PyObject *self, PyObject *pairs) +{ + PyObject *pair, *iterator, *unexpected; + int res = 0; + + iterator = PyObject_GetIter(pairs); + if (iterator == NULL) + return -1; + PyErr_Clear(); + + while ((pair = PyIter_Next(iterator)) != NULL) { + /* could be more efficient (see UNPACK_SEQUENCE in ceval.c) */ + PyObject *key = NULL, *value = NULL; + PyObject *pair_iterator = PyObject_GetIter(pair); + if (pair_iterator == NULL) + goto Done; + + key = PyIter_Next(pair_iterator); + if (key == NULL) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_ValueError, + "need more than 0 values to unpack"); + goto Done; + } + + value = PyIter_Next(pair_iterator); + if (value == NULL) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_ValueError, + "need more than 1 value to unpack"); + goto Done; + } + + unexpected = PyIter_Next(pair_iterator); + if (unexpected != NULL) { + Py_DECREF(unexpected); + PyErr_SetString(PyExc_ValueError, + "too many values to unpack (expected 2)"); + goto Done; + } + else if (PyErr_Occurred()) + goto Done; + + res = PyObject_SetItem(self, key, value); + +Done: + Py_DECREF(pair); + Py_XDECREF(pair_iterator); + Py_XDECREF(key); + Py_XDECREF(value); + if (PyErr_Occurred()) + break; + } + Py_DECREF(iterator); + + if (res < 0 || PyErr_Occurred() != NULL) + return -1; + else + return 0; +} + +static PyObject * +mutablemapping_update(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int res = 0; + Py_ssize_t len; + _Py_IDENTIFIER(items); + _Py_IDENTIFIER(keys); + + /* first handle args, if any */ + assert(args == NULL || PyTuple_Check(args)); + len = (args != NULL) ? PyTuple_GET_SIZE(args) : 0; + if (len > 1) { + const char *msg = "update() takes at most 1 positional argument (%zd given)"; + PyErr_Format(PyExc_TypeError, msg, len); + return NULL; + } + + if (len) { + PyObject *func; + PyObject *other = PyTuple_GET_ITEM(args, 0); /* borrowed reference */ + assert(other != NULL); + Py_INCREF(other); + if (PyDict_CheckExact(other)) { + PyObject *items = PyDict_Items(other); + Py_DECREF(other); + if (items == NULL) + return NULL; + res = mutablemapping_add_pairs(self, items); + Py_DECREF(items); + if (res == -1) + return NULL; + goto handle_kwargs; + } + + if (_PyObject_LookupAttrId(other, &PyId_keys, &func) < 0) { + Py_DECREF(other); + return NULL; + } + if (func != NULL) { + PyObject *keys, *iterator, *key; + keys = _PyObject_CallNoArg(func); + Py_DECREF(func); + if (keys == NULL) { + Py_DECREF(other); + return NULL; + } + iterator = PyObject_GetIter(keys); + Py_DECREF(keys); + if (iterator == NULL) { + Py_DECREF(other); + return NULL; + } + while (res == 0 && (key = PyIter_Next(iterator))) { + PyObject *value = PyObject_GetItem(other, key); + if (value != NULL) { + res = PyObject_SetItem(self, key, value); + Py_DECREF(value); + } + else { + res = -1; + } + Py_DECREF(key); + } + Py_DECREF(other); + Py_DECREF(iterator); + if (res != 0 || PyErr_Occurred()) + return NULL; + goto handle_kwargs; + } + + if (_PyObject_LookupAttrId(other, &PyId_items, &func) < 0) { + Py_DECREF(other); + return NULL; + } + if (func != NULL) { + PyObject *items; + Py_DECREF(other); + items = _PyObject_CallNoArg(func); + Py_DECREF(func); + if (items == NULL) + return NULL; + res = mutablemapping_add_pairs(self, items); + Py_DECREF(items); + if (res == -1) + return NULL; + goto handle_kwargs; + } + + res = mutablemapping_add_pairs(self, other); + Py_DECREF(other); + if (res != 0) + return NULL; + } + + handle_kwargs: + /* now handle kwargs */ + assert(kwargs == NULL || PyDict_Check(kwargs)); + if (kwargs != NULL && PyDict_GET_SIZE(kwargs)) { + PyObject *items = PyDict_Items(kwargs); + if (items == NULL) + return NULL; + res = mutablemapping_add_pairs(self, items); + Py_DECREF(items); + if (res == -1) + return NULL; + } + + Py_RETURN_NONE; +} From 0e5fbf3fac82644a7ca6427007dabde27a370942 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Thu, 30 May 2019 23:19:56 +1000 Subject: [PATCH 11/66] Break ref cycle when frame finishes executing --- Include/frameobject.h | 1 + Objects/frameobject.c | 36 ++++++++++++++++++++++++++++++++---- Python/ceval.c | 7 ++++++- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/Include/frameobject.h b/Include/frameobject.h index b938ecb39c901d..d76f2852f5968e 100644 --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -96,6 +96,7 @@ PyAPI_FUNC(PyObject *) PyFrame_GetLocalsAttr(PyFrameObject *); // = frame.f_loc #ifdef Py_BUILD_CORE PyObject *_PyFrame_BorrowPyLocals(PyFrameObject *f); /* For PyEval_GetLocals() */ +void _PyFrame_PostEvalCleanup(PyFrameObject *f); /* For PyEval_EvalFrameEx() */ #endif diff --git a/Objects/frameobject.c b/Objects/frameobject.c index ca3d044991df66..2907c913a4b170 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -13,6 +13,7 @@ static PyObject *_PyFastLocalsProxy_New(PyObject *frame); static PyObject *_PyFastLocalsProxy_BorrowLocals(PyObject *flp); +static void _PyFastLocalsProxy_BreakReferenceCycle(PyObject *flp); static PyMemberDef frame_memberlist[] = { {"f_back", T_OBJECT, OFF(f_back), READONLY}, @@ -54,6 +55,18 @@ _PyFrame_BorrowPyLocals(PyFrameObject *f) return updated_locals; } +void _PyFrame_PostEvalCleanup(PyFrameObject *f) +{ + // This is called by PyEval_EvalFrameEx() to ensure that any reference + // cycle between the frame and f_locals gets broken when the frame finishes + // execution. + assert(f->f_locals); + if (_PyFastLocalsProxy_CheckExact(f->f_locals)) { + _PyFastLocalsProxy_BreakReferenceCycle(f->f_locals); + } +} + + PyObject * PyFrame_GetPyLocals(PyFrameObject *f) { @@ -534,7 +547,7 @@ frame_tp_clear(PyFrameObject *f) for (i = slots; --i >= 0; ++fastlocals) Py_CLEAR(*fastlocals); - /* TODO: The fast locals proxy is no longer valid here... */ + Py_CLEAR(f->f_locals); /* stack */ if (oldtop != NULL) { @@ -1116,6 +1129,11 @@ static int fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObject *value) { int result = 0; + if (flp->frame == NULL) { + // This indicates the frame has finished executing and the proxy's link + // back to the frame has been cleared to break the reference cycle + return 0; + } PyObject *fast_ref = PyDict_GetItem(flp->fast_refs, key); if (fast_ref != NULL) { /* Key is also stored on the frame, so update that reference */ @@ -1291,7 +1309,7 @@ fastlocalsproxy_clear(register PyObject *flp, PyObject *Py_UNUSED(ignored)) static PyMethodDef fastlocalsproxy_methods[] = { {"setdefault", (PyCFunction)(void(*)(void))fastlocalsproxy_setdefault, - METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_pop__doc__}, + METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_setdefault__doc__}, {"pop", (PyCFunction)(void(*)(void))fastlocalsproxy_pop, METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_pop__doc__}, {"clear", (PyCFunction)fastlocalsproxy_popitem, @@ -1308,8 +1326,8 @@ static void fastlocalsproxy_dealloc(fastlocalsproxyobject *flp) { _PyObject_GC_UNTRACK(flp); - Py_CLEAR(flp->frame); Py_CLEAR(flp->mapping); + Py_CLEAR(flp->frame); Py_CLEAR(flp->fast_refs); PyObject_GC_Del(flp); } @@ -1324,8 +1342,8 @@ static int fastlocalsproxy_traverse(PyObject *self, visitproc visit, void *arg) { fastlocalsproxyobject *flp = (fastlocalsproxyobject *)self; - Py_VISIT(flp->frame); Py_VISIT(flp->mapping); + Py_VISIT(flp->frame); Py_VISIT(flp->fast_refs); return 0; } @@ -1392,6 +1410,16 @@ _PyFastLocalsProxy_BorrowLocals(PyObject *self) return ((fastlocalsproxyobject *) self)->mapping; } +static void +_PyFastLocalsProxy_BreakReferenceCycle(PyObject *self) +{ + assert(_PyFastLocalsProxy_CheckExact(self)); + fastlocalsproxyobject *flp = (fastlocalsproxyobject *) self; + Py_CLEAR(flp->frame); + Py_CLEAR(flp->fast_refs); +} + + /*[clinic input] @classmethod fastlocalsproxy.__new__ as fastlocalsproxy_new diff --git a/Python/ceval.c b/Python/ceval.c index d658446c69985e..9e897200862cb6 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -682,7 +682,12 @@ PyObject * PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) { PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); - return interp->eval_frame(f, throwflag); + PyObject * result = interp->eval_frame(f, throwflag); + if (f->f_locals != NULL) { + // There may be a cyclic reference that needs to be cleaned up... + _PyFrame_PostEvalCleanup(f); + } + return result; } PyObject* _Py_HOT_FUNCTION From 9e3ce53a369f6946bff33b0e28bd90f13c1ca985 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Thu, 30 May 2019 23:25:23 +1000 Subject: [PATCH 12/66] Remove implicit frame locals update --- Python/sysmodule.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index e50932dcb69709..d4e069f3da5e35 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -507,10 +507,6 @@ call_trampoline(PyObject* callback, PyObject *result; PyObject *stack[3]; - if (PyFrame_FastToLocalsWithError(frame) < 0) { - return NULL; - } - stack[0] = (PyObject *)frame; stack[1] = whatstrings[what]; stack[2] = (arg != NULL) ? arg : Py_None; From 8e886efb0d70689624fff8b10acd1425a313110a Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 31 May 2019 00:07:03 +1000 Subject: [PATCH 13/66] Avoid double DECREF on error --- Objects/frameobject.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 2907c913a4b170..efe08a7873e3ea 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1393,9 +1393,7 @@ _PyFastLocalsProxy_New(PyObject *frame) flp->frame = (PyFrameObject *) frame; fast_refs = _PyFrame_BuildFastRefs(flp->frame); if (fast_refs == NULL) { - Py_DECREF(flp); - Py_DECREF(mapping); - Py_DECREF(frame); + Py_DECREF(flp); // Also handles DECREF for mapping and frame return NULL; } flp->fast_refs = fast_refs; From 5b63e7cfe489c124ec4dcef3ba3fd1ca8027f099 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 31 May 2019 00:29:07 +1000 Subject: [PATCH 14/66] Attempt to make dealloc more robust under gc --- Objects/frameobject.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index efe08a7873e3ea..b4fcf8e05c3db9 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1325,11 +1325,17 @@ static PyMethodDef fastlocalsproxy_methods[] = { static void fastlocalsproxy_dealloc(fastlocalsproxyobject *flp) { - _PyObject_GC_UNTRACK(flp); + if (_PyObject_GC_IS_TRACKED(flp)) + _PyObject_GC_UNTRACK(flp); + + Py_TRASHCAN_SAFE_BEGIN(flp) + Py_CLEAR(flp->mapping); Py_CLEAR(flp->frame); Py_CLEAR(flp->fast_refs); PyObject_GC_Del(flp); + + Py_TRASHCAN_SAFE_END(flp) } static PyObject * From 1752b5436f8d85f6048b766404937aa9d2142789 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 29 Dec 2019 12:52:06 +1000 Subject: [PATCH 15/66] Fix post-merge compilation errors --- Include/frameobject.h | 5 ----- Include/internal/pycore_ceval.h | 3 ++- Include/internal/pycore_frameobject.h | 20 ++++++++++++++++++++ Python/sysmodule.c | 1 - 4 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 Include/internal/pycore_frameobject.h diff --git a/Include/frameobject.h b/Include/frameobject.h index 7e314892f9ca5c..f2d51a6d56339b 100644 --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -96,11 +96,6 @@ PyAPI_FUNC(PyObject *) PyFrame_GetPyLocals(PyFrameObject *); // = locals() PyAPI_FUNC(PyObject *) PyFrame_GetLocalsAttr(PyFrameObject *); // = frame.f_locals #endif -#ifdef Py_BUILD_CORE -PyObject *_PyFrame_BorrowPyLocals(PyFrameObject *f); /* For PyEval_GetLocals() */ -void _PyFrame_PostEvalCleanup(PyFrameObject *f); /* For _PyEval_EvalFrame() */ -#endif - /* This always raises RuntimeError now (use PyFrame_GetLocalsAttr() instead) */ PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int); diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 720ee76a14a705..3a7a7bed6373ad 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -13,7 +13,8 @@ struct pyruntimestate; struct _ceval_runtime_state; struct _frame; -#include "pycore_pystate.h" /* PyInterpreterState.eval_frame */ +#include "pycore_pystate.h" /* PyInterpreterState.eval_frame */ +#include "pycore_frameobject.h" /* _PyFrame_PostEvalCleanup */ PyAPI_FUNC(void) _Py_FinishPendingCalls(struct pyruntimestate *runtime); PyAPI_FUNC(void) _PyEval_Initialize(struct _ceval_runtime_state *); diff --git a/Include/internal/pycore_frameobject.h b/Include/internal/pycore_frameobject.h new file mode 100644 index 00000000000000..6da567238d3100 --- /dev/null +++ b/Include/internal/pycore_frameobject.h @@ -0,0 +1,20 @@ +#ifndef Py_INTERNAL_FRAMEOBJECT_H +#define Py_INTERNAL_FRAMEOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +struct _frame; +typedef struct _frame PyFrameObject; + +PyObject *_PyFrame_BorrowPyLocals(PyFrameObject *f); /* For PyEval_GetLocals() */ +void _PyFrame_PostEvalCleanup(PyFrameObject *f); /* For _PyEval_EvalFrame() */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_FRAMEOBJECT_H */ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index acde0c493d02e6..8305a7cd2f351d 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -878,7 +878,6 @@ static PyObject * call_trampoline(PyObject* callback, PyFrameObject *frame, int what, PyObject *arg) { - PyObject *result; PyObject *stack[3]; stack[0] = (PyObject *)frame; From 348a56d6e97b1142b1464cb0b7df9824f4670b1c Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 29 Dec 2019 14:20:09 +1000 Subject: [PATCH 16/66] Implement flp.pop() --- Objects/frameobject.c | 42 +++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 6ff5e4331f33fe..e1266c3bb40bff 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1468,24 +1468,40 @@ fastlocalsproxy_pop(PyObject *flp, PyObject *args, PyObject *kwargs) } static PyObject * -_fastlocalsproxy_popkey_hash(PyObject *flp, PyObject *key, PyObject *failobj, - Py_hash_t hash) +_fastlocalsproxy_popkey(PyObject *flp, PyObject *key, PyObject *failobj) { - PyObject *value = NULL; + // TODO: Similar to the odict implementation, the fast locals proxy + // could benefit from an internal API that accepts already calculated + // hashes, rather than recalculating the hash multiple times for the + // same key in a single operation (see _odict_popkey_hash) - PyErr_Format(PyExc_NotImplementedError, - "FastLocalsProxy does not yet implement pop()"); - return value; -} + PyObject *value = NULL; -static PyObject * -_fastlocalsproxy_popkey(PyObject *flp, PyObject *key, PyObject *failobj) -{ - Py_hash_t hash = PyObject_Hash(key); - if (hash == -1) + // Just implement naive lookup through the abstract C API for now + int exists = PySequence_Contains(flp, key); + if (exists < 0) return NULL; + if (exists) { + value = PyObject_GetItem(flp, key); + if (value != NULL) { + if (PyObject_DelItem(flp, key) == -1) { + Py_CLEAR(value); + } + } + } - return _fastlocalsproxy_popkey_hash(flp, key, failobj, hash); + /* Apply the fallback value, if necessary. */ + if (value == NULL && !PyErr_Occurred()) { + if (failobj) { + value = failobj; + Py_INCREF(failobj); + } + else { + PyErr_SetObject(PyExc_KeyError, key); + } + } + + return value; } /* popitem() */ From 39ec4d8c89205318bfc3cf1188efb29e48ae6301 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 29 Dec 2019 23:24:49 +1000 Subject: [PATCH 17/66] Refactor ref map creation --- Objects/frameobject.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index e1266c3bb40bff..d6dea91a7f6a8f 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1176,6 +1176,21 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear) ); } +static int +set_fast_ref(PyObject *fast_refs, PyObject *key, PyObject *value) +{ + // NOTE: Steals the "value" reference, so borrowed values need an INCREF + assert(PyUnicode_Check(key)); + // Don't rely on borrowed key reference while potentially running arbitrary code + // This shouldn't be necessary since all the keys are checked to be strings, + // but currently being paranoid while hunting an apparent refcounting bug + Py_INCREF(key); + int status = PyDict_SetItem(fast_refs, key, value); + Py_DECREF(key); + Py_DECREF(value); + return status; +} + static int add_local_refs(PyObject *map, Py_ssize_t nmap, PyObject *fast_refs) { @@ -1187,11 +1202,10 @@ add_local_refs(PyObject *map, Py_ssize_t nmap, PyObject *fast_refs) for (j = nmap; --j >= 0; ) { PyObject *key = PyTuple_GET_ITEM(map, j); PyObject *value = PyLong_FromSsize_t(j); - assert(PyUnicode_Check(key)); /* Values may be missing if the frame has been cleared */ if (value != NULL) { - if (PyDict_SetItem(fast_refs, key, value) != 0) { - return -1; + if (set_fast_ref(fast_refs, key, value) != 0) { + return -1; } } } @@ -1209,13 +1223,12 @@ add_nonlocal_refs(PyObject *map, Py_ssize_t nmap, PyObject *fast_refs, PyObject for (j = nmap; --j >= 0; ) { PyObject *key = PyTuple_GET_ITEM(map, j); PyObject *value = cells[j]; - assert(PyUnicode_Check(key)); /* Values may be missing if the frame has been cleared */ if (value != NULL) { assert(PyCell_Check(value)); - assert(PyUnicode_Check(key)); - if (PyDict_SetItem(fast_refs, key, value) != 0) { - return -1; + Py_INCREF(value); // set_fast_ref steals the value reference + if (set_fast_ref(fast_refs, key, value) != 0) { + return -1; } } } From 7078632520279fd19ec59b8faec5ddd27ea4b64c Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 29 Dec 2019 23:57:25 +1000 Subject: [PATCH 18/66] Correctly manage fast local refcounts --- Objects/frameobject.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d6dea91a7f6a8f..f472b10ca748ea 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1181,12 +1181,7 @@ set_fast_ref(PyObject *fast_refs, PyObject *key, PyObject *value) { // NOTE: Steals the "value" reference, so borrowed values need an INCREF assert(PyUnicode_Check(key)); - // Don't rely on borrowed key reference while potentially running arbitrary code - // This shouldn't be necessary since all the keys are checked to be strings, - // but currently being paranoid while hunting an apparent refcounting bug - Py_INCREF(key); int status = PyDict_SetItem(fast_refs, key, value); - Py_DECREF(key); Py_DECREF(value); return status; } @@ -1201,7 +1196,7 @@ add_local_refs(PyObject *map, Py_ssize_t nmap, PyObject *fast_refs) assert(PyTuple_Size(map) >= nmap); for (j = nmap; --j >= 0; ) { PyObject *key = PyTuple_GET_ITEM(map, j); - PyObject *value = PyLong_FromSsize_t(j); + PyObject *value = PyLong_FromSsize_t(j); // set_fast_ref steals the value /* Values may be missing if the frame has been cleared */ if (value != NULL) { if (set_fast_ref(fast_refs, key, value) != 0) { @@ -1226,7 +1221,7 @@ add_nonlocal_refs(PyObject *map, Py_ssize_t nmap, PyObject *fast_refs, PyObject /* Values may be missing if the frame has been cleared */ if (value != NULL) { assert(PyCell_Check(value)); - Py_INCREF(value); // set_fast_ref steals the value reference + Py_INCREF(value); // set_fast_ref steals the value if (set_fast_ref(fast_refs, key, value) != 0) { return -1; } @@ -1382,7 +1377,10 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje offset, max_offset); result = -1; } - flp->frame->f_localsplus[offset] = value; + if (result == 0) { + Py_INCREF(value); + Py_XSETREF(flp->frame->f_localsplus[offset], value); + } } } return result; From ed5f86ee989ec2826553684181d901a869a33ede Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 30 Dec 2019 00:55:27 +1000 Subject: [PATCH 19/66] Allow closure updates after frame termination --- Objects/frameobject.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index f472b10ca748ea..49dc49e889d1f1 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1355,16 +1355,17 @@ static int fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObject *value) { int result = 0; - if (flp->frame == NULL) { - // This indicates the frame has finished executing and the proxy's link - // back to the frame has been cleared to break the reference cycle - return 0; - } + assert(PyDict_Check(flp->fast_refs)); PyObject *fast_ref = PyDict_GetItem(flp->fast_refs, key); if (fast_ref != NULL) { - /* Key is also stored on the frame, so update that reference */ + /* Key is an actual Python variable, so update that reference */ if (PyCell_Check(fast_ref)) { + // Closure cells can be updated even after the frame terminates result = PyCell_Set(fast_ref, value); + } else if (flp->frame == NULL) { + // This indicates the frame has finished executing and the link + // back to the frame has been cleared to break the reference cycle + return 0; } else { /* Fast ref is a Python int mapping into the fast locals array */ Py_ssize_t offset = PyLong_AsSsize_t(fast_ref); @@ -1665,7 +1666,7 @@ _PyFastLocalsProxy_BreakReferenceCycle(PyObject *self) assert(_PyFastLocalsProxy_CheckExact(self)); fastlocalsproxyobject *flp = (fastlocalsproxyobject *) self; Py_CLEAR(flp->frame); - Py_CLEAR(flp->fast_refs); + // We keep flp->fast_refs alive to allow updating of closure variables } From a216747a0122f8754c414768415806ea5701976c Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 30 Dec 2019 01:44:24 +1000 Subject: [PATCH 20/66] Refactor frame post-eval cleanup --- Doc/c-api/reflection.rst | 1 + Objects/frameobject.c | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 1d86de66ed3eda..a25d2a90fcd356 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -16,6 +16,7 @@ Reflection Return a dictionary of the local variables in the current execution frame, or ``NULL`` if no frame is currently executing. + TODO: Clarify just how this relates to the Python level locals() builtin. .. c:function:: PyObject* PyEval_GetGlobals() diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 49dc49e889d1f1..7fa4bf24f3ba0f 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -57,14 +57,14 @@ _PyFrame_BorrowPyLocals(PyFrameObject *f) void _PyFrame_PostEvalCleanup(PyFrameObject *f) { - // This is called by PyEval_EvalFrameEx() to ensure that any reference - // cycle between the frame and f_locals gets broken when the frame finishes - // execution. - if (!f->f_locals) { + // Don't clean up still running coroutines and generators + if (f->f_executing) { return; } - if (_PyFastLocalsProxy_CheckExact(f->f_locals)) { + // Ensure that any reference cycle between the frame and f_locals gets + // broken when the frame finishes execution. + if (f->f_locals && _PyFastLocalsProxy_CheckExact(f->f_locals)) { _PyFastLocalsProxy_BreakReferenceCycle(f->f_locals); } } From a0dc787f3891fd58c133b50e9d18dbbb2a1a32ff Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 30 Dec 2019 02:12:41 +1000 Subject: [PATCH 21/66] Use full word in API name --- Include/frameobject.h | 4 ++-- Objects/frameobject.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Include/frameobject.h b/Include/frameobject.h index f2d51a6d56339b..cfff23dc40df60 100644 --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -93,11 +93,11 @@ PyTypeObject PyFastLocalsProxy_Type; /* Access the frame locals mapping */ PyAPI_FUNC(PyObject *) PyFrame_GetPyLocals(PyFrameObject *); // = locals() -PyAPI_FUNC(PyObject *) PyFrame_GetLocalsAttr(PyFrameObject *); // = frame.f_locals +PyAPI_FUNC(PyObject *) PyFrame_GetLocalsAttribute(PyFrameObject *); // = frame.f_locals #endif -/* This always raises RuntimeError now (use PyFrame_GetLocalsAttr() instead) */ +/* This always raises RuntimeError now (use PyFrame_GetLocalsAttribute() instead) */ PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int); #ifdef __cplusplus diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 7fa4bf24f3ba0f..b37ff3903a8095 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -36,7 +36,7 @@ _frame_get_updated_locals(PyFrameObject *f) } PyObject * -PyFrame_GetLocalsAttr(PyFrameObject *f) +PyFrame_GetLocalsAttribute(PyFrameObject *f) { PyObject *updated_locals =_frame_get_updated_locals(f); Py_INCREF(updated_locals); @@ -81,7 +81,7 @@ PyFrame_GetPyLocals(PyFrameObject *f) static PyObject * frame_getlocals(PyFrameObject *f, void *__unused) { - return PyFrame_GetLocalsAttr(f); + return PyFrame_GetLocalsAttribute(f); } int @@ -1172,7 +1172,7 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear) PyErr_SetString( PyExc_RuntimeError, "PyFrame_LocalsToFast is no longer supported. " - "Use PyFrame_GetPyLocals() instead." + "Use PyFrame_GetPyLocals() or PyFrame_GetLocalsAttribute() instead." ); } From 7b02bedd8cf2d87e43114995d925cdffd745da06 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 30 Dec 2019 02:58:42 +1000 Subject: [PATCH 22/66] Update to match latest PEP draft --- Doc/c-api/reflection.rst | 9 +++++++- Include/ceval.h | 4 ++++ Include/frameobject.h | 2 +- Objects/frameobject.c | 45 +++++++++++++++++++++++----------------- Python/bltinmodule.c | 15 +++++--------- Python/ceval.c | 13 ++++++++++++ 6 files changed, 57 insertions(+), 31 deletions(-) diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index a25d2a90fcd356..676be12874423d 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -11,12 +11,19 @@ Reflection or the interpreter of the thread state if no frame is currently executing. +.. c:function:: PyObject* PyEval_GetPyLocals() + + Return a dictionary of the local variables in the current execution frame, + or ``NULL`` if no frame is currently executing. + + Equivalent to calling the Python level ``locals()`` builtin. + .. c:function:: PyObject* PyEval_GetLocals() Return a dictionary of the local variables in the current execution frame, or ``NULL`` if no frame is currently executing. - TODO: Clarify just how this relates to the Python level locals() builtin. + TODO: Clarify just how this relates to PyEval_GetPyLocals(). .. c:function:: PyObject* PyEval_GetGlobals() diff --git a/Include/ceval.h b/Include/ceval.h index 62c6489ed1c351..1ce06d786eeb6c 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -38,6 +38,10 @@ PyAPI_FUNC(struct _frame *) PyEval_GetFrame(void); PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg); PyAPI_FUNC(int) Py_MakePendingCalls(void); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000 +PyAPI_FUNC(PyObject *) PyEval_GetPyLocals(void); +#endif + /* Protection against deeply nested recursive calls In Python 3.0, this protection has two levels: diff --git a/Include/frameobject.h b/Include/frameobject.h index cfff23dc40df60..ffd40a052e2c5b 100644 --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -85,7 +85,7 @@ PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03080000 +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000 /* Fast locals proxy for reliable write-through from trace functions */ PyTypeObject PyFastLocalsProxy_Type; #define _PyFastLocalsProxy_CheckExact(self) \ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index b37ff3903a8095..f187e2bf9bd88f 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -35,10 +35,24 @@ _frame_get_updated_locals(PyFrameObject *f) return f->f_locals; } +void _PyFrame_PostEvalCleanup(PyFrameObject *f) +{ + // Don't clean up still running coroutines and generators + if (f->f_executing) { + return; + } + + // Ensure that any reference cycle between the frame and f_locals gets + // broken when the frame finishes execution. + if (f->f_locals && _PyFastLocalsProxy_CheckExact(f->f_locals)) { + _PyFastLocalsProxy_BreakReferenceCycle(f->f_locals); + } +} + PyObject * PyFrame_GetLocalsAttribute(PyFrameObject *f) { - PyObject *updated_locals =_frame_get_updated_locals(f); + PyObject *updated_locals = _frame_get_updated_locals(f); Py_INCREF(updated_locals); return updated_locals; } @@ -48,33 +62,26 @@ _PyFrame_BorrowPyLocals(PyFrameObject *f) { // This is called by PyEval_GetLocals(), which has historically returned // a borrowed reference, so this does the same - PyObject *updated_locals =_frame_get_updated_locals(f); + PyObject *updated_locals = _frame_get_updated_locals(f); if (_PyFastLocalsProxy_CheckExact(updated_locals)) { updated_locals = _PyFastLocalsProxy_BorrowLocals(updated_locals); } return updated_locals; } -void _PyFrame_PostEvalCleanup(PyFrameObject *f) -{ - // Don't clean up still running coroutines and generators - if (f->f_executing) { - return; - } - - // Ensure that any reference cycle between the frame and f_locals gets - // broken when the frame finishes execution. - if (f->f_locals && _PyFastLocalsProxy_CheckExact(f->f_locals)) { - _PyFastLocalsProxy_BreakReferenceCycle(f->f_locals); - } -} - - PyObject * PyFrame_GetPyLocals(PyFrameObject *f) { - PyObject *updated_locals =_PyFrame_BorrowPyLocals(f); - Py_INCREF(updated_locals); + PyObject *updated_locals = _frame_get_updated_locals(f); + if (_PyFastLocalsProxy_CheckExact(updated_locals)) { + // Take a snapshot of optimised scopes to avoid quirky side effects + PyObject *d = _PyFastLocalsProxy_BorrowLocals(updated_locals); + updated_locals = PyDict_Copy(d); + } else { + // Share a direct locals reference for class and module scopes + Py_INCREF(updated_locals); + } + return updated_locals; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 34267685be2f1f..15ea360cd28a0c 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -907,6 +907,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, if (globals == Py_None) { globals = PyEval_GetGlobals(); if (locals == Py_None) { + // TODO: Consider if this should use PyEval_GetPyLocals() instead locals = PyEval_GetLocals(); if (locals == NULL) return NULL; @@ -986,6 +987,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, if (globals == Py_None) { globals = PyEval_GetGlobals(); if (locals == Py_None) { + // TODO: Consider if this should use PyEval_GetPyLocals() instead locals = PyEval_GetLocals(); if (locals == NULL) return NULL; @@ -1562,20 +1564,14 @@ locals as builtin_locals Return a dictionary containing the current scope's local variables. -NOTE: Whether or not updates to this dictionary will affect name lookups in -the local scope and vice-versa is *implementation dependent* and not -covered by any backwards compatibility guarantees. +TODO: Update the docstring with the gist of PEP 558 semantics. [clinic start generated code]*/ static PyObject * builtin_locals_impl(PyObject *module) /*[clinic end generated code: output=b46c94015ce11448 input=7874018d478d5c4b]*/ { - PyObject *d; - - d = PyEval_GetLocals(); - Py_XINCREF(d); - return d; + return PyEval_GetPyLocals(); } @@ -2249,8 +2245,7 @@ builtin_vars(PyObject *self, PyObject *args) if (!PyArg_UnpackTuple(args, "vars", 0, 1, &v)) return NULL; if (v == NULL) { - d = PyEval_GetLocals(); - Py_XINCREF(d); + d = PyEval_GetPyLocals(); } else { if (_PyObject_LookupAttrId(v, &PyId___dict__, &d) == 0) { diff --git a/Python/ceval.c b/Python/ceval.c index 27e7857ded2be2..5b5966f5431799 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4698,6 +4698,19 @@ PyEval_GetLocals(void) return _PyFrame_BorrowPyLocals(current_frame); } +PyObject * +PyEval_GetPyLocals(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyFrameObject *current_frame = _PyEval_GetFrame(tstate); + if (current_frame == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); + return NULL; + } + + return PyFrame_GetPyLocals(current_frame); +} + PyObject * PyEval_GetGlobals(void) { From e9876b5d5e4b012b3172417c828d5cf765a937cd Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 30 Dec 2019 03:25:50 +1000 Subject: [PATCH 23/66] Update test_scope for snapshot semantics --- Lib/test/test_scope.py | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 88cdb0ab77bea9..0893563c7db1fe 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -794,23 +794,43 @@ def test_locals_update_semantics_at_class_scope(self): self.assertEqual(local_ns["known_var"], "set_via_locals") self.assertEqual(local_ns["unknown_var"], "set_via_locals") - def test_locals_update_semantics_at_function_scope(self): + def test_locals_snapshot_semantics_at_function_scope(self): def function_local_semantics(): global_ns = globals() known_var = "original" - local_ns = locals() - local_ns["known_var"] = "set_via_locals" - local_ns["unknown_var"] = "set_via_locals" - return locals() + to_be_deleted = "not_yet_deleted" + local_ns1 = locals() + local_ns1["known_var"] = "set_via_ns1" + local_ns1["unknown_var"] = "set_via_ns1" + del to_be_deleted + local_ns2 = locals() + local_ns2["known_var"] = "set_via_ns2" + local_ns2["unknown_var"] = "set_via_ns2" + return dict(ns1=local_ns1, ns2=local_ns2) global_ns = globals() self.assertIsInstance(global_ns, dict) - local_ns = function_local_semantics() - self.assertIsInstance(local_ns, dict) - self.assertIs(local_ns["global_ns"], global_ns) - self.assertIs(local_ns["local_ns"], local_ns) - self.assertEqual(local_ns["known_var"], "original") - self.assertEqual(local_ns["unknown_var"], "set_via_locals") + local_namespaces = function_local_semantics() + # Check internal consistency of each snapshot + for name, local_ns in local_namespaces.items(): + with self.subTest(namespace=name): + self.assertIsInstance(local_ns, dict) + self.assertIs(local_ns["global_ns"], global_ns) + expected_text = "set_via_" + name + self.assertEqual(local_ns["known_var"], expected_text) + self.assertEqual(local_ns["unknown_var"], expected_text) + # Check independence of the snapshots + local_ns1 = local_namespaces["ns1"] + local_ns2 = local_namespaces["ns2"] + self.assertIsNot(local_ns1, local_ns2) + # Check that not yet set local variables are excluded from snapshot + self.assertNotIn("local_ns1", local_ns1) + self.assertNotIn("local_ns2", local_ns1) + self.assertIs(local_ns2["local_ns1"], local_ns1) + self.assertNotIn("local_ns2", local_ns2) + # Check that deleted variables are excluded from snapshot + self.assertEqual(local_ns1["to_be_deleted"], "not_yet_deleted") + self.assertNotIn("to_be_deleted", local_ns2) if __name__ == '__main__': From 0033c600591e028e7f9ed25dd0897f153087df36 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 30 Dec 2019 03:33:39 +1000 Subject: [PATCH 24/66] Update test_sys_settrace for snapshot semantics --- Lib/test/test_sys_settrace.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 9632506009e4df..c3a62153487203 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1510,10 +1510,10 @@ def nested_scope(): inner_snapshot["local_var"] = "modified_again" self.assertEqual(local_var, "modified") self.assertEqual(inner_snapshot["local_var"], "modified_again") - self.assertEqual(inner_proxy["local_var"], "modified_again") # Q: Is this desirable? + self.assertEqual(inner_proxy["local_var"], "modified") inner_proxy["local_var"] = "modified_yet_again" self.assertEqual(local_var, "modified_yet_again") - self.assertEqual(inner_snapshot["local_var"], "modified_yet_again") + self.assertEqual(inner_snapshot["local_var"], "modified_again") self.assertEqual(inner_proxy["local_var"], "modified_yet_again") def test_locals_writeback_complex_scenario(self): @@ -1543,8 +1543,8 @@ def inner(): a_local = 'original7' # We expect this to be retained another_local = 'original8' # Trace func will modify this ns = locals() - ns['a_local'] = 'modified7' # We expect this to be reverted - ns['a_nonlocal'] = 'modified5' # We expect this to be reverted + ns['a_local'] = 'modified7' # We expect this to be retained + ns['a_nonlocal'] = 'modified5' # We expect this to be retained ns['a_new_local'] = 'created9' # We expect this to be retained return a_local, another_local, ns outer_local = 'original10' # Trace func will modify this @@ -1607,12 +1607,12 @@ def tracefunc(frame, event, arg): "another_class_attribute": "modified4", "a_nonlocal": "original5", "a_nonlocal_via_ns": "original5", - "a_nonlocal_via_inner_ns": "original5", + "a_nonlocal_via_inner_ns": "modified5", "another_nonlocal": "modified6", "another_nonlocal_via_ns": "modified6", - "another_nonlocal_via_inner_ns": "modified6", + "another_nonlocal_via_inner_ns": "original6", "a_local": "original7", - "a_local_via_ns": "original7", + "a_local_via_ns": "modified7", "another_local": "modified8", "another_local_via_ns": "modified8", "a_new_local_via_ns": "created9", From a5a8b1971f1c9b74b15d57d0d4348f291a073ed2 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 30 Dec 2019 03:52:04 +1000 Subject: [PATCH 25/66] Fix pop/delete locals proxy bug --- Objects/frameobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index f187e2bf9bd88f..8cf73b1efd2fad 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1386,7 +1386,7 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje result = -1; } if (result == 0) { - Py_INCREF(value); + Py_XINCREF(value); Py_XSETREF(flp->frame->f_localsplus[offset], value); } } From 6c98f48c8ba1018acea3aa4c6ac2028dd6a82aaf Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 30 Dec 2019 17:31:55 +1000 Subject: [PATCH 26/66] Update argument clinic output --- Objects/clinic/frameobject.c.h | 12 ++++++++---- Python/bltinmodule.c | 2 +- Python/clinic/bltinmodule.c.h | 6 ++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Objects/clinic/frameobject.c.h b/Objects/clinic/frameobject.c.h index 10325dcb41ca65..43b3a587914887 100644 --- a/Objects/clinic/frameobject.c.h +++ b/Objects/clinic/frameobject.c.h @@ -10,16 +10,20 @@ fastlocalsproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; static const char * const _keywords[] = {"frame", NULL}; - static _PyArg_Parser _parser = {"O:fastlocalsproxy", _keywords, 0}; + static _PyArg_Parser _parser = {NULL, _keywords, "fastlocalsproxy", 0}; + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *frame; - if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, - &frame)) { + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + if (!fastargs) { goto exit; } + frame = fastargs[0]; return_value = fastlocalsproxy_new_impl(type, frame); exit: return return_value; } -/*[clinic end generated code: output=5fa72522109d3584 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=34617a00e21738f3 input=a9049054013a1b77]*/ diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 15ea360cd28a0c..312b08a8a623c5 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1569,7 +1569,7 @@ TODO: Update the docstring with the gist of PEP 558 semantics. static PyObject * builtin_locals_impl(PyObject *module) -/*[clinic end generated code: output=b46c94015ce11448 input=7874018d478d5c4b]*/ +/*[clinic end generated code: output=b46c94015ce11448 input=9869b08c278df34f]*/ { return PyEval_GetPyLocals(); } diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index d15af1f7f377c6..6507a1f829f157 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -570,9 +570,7 @@ PyDoc_STRVAR(builtin_locals__doc__, "\n" "Return a dictionary containing the current scope\'s local variables.\n" "\n" -"NOTE: Whether or not updates to this dictionary will affect name lookups in\n" -"the local scope and vice-versa is *implementation dependent* and not\n" -"covered by any backwards compatibility guarantees."); +"TODO: Update the docstring with the gist of PEP 558 semantics."); #define BUILTIN_LOCALS_METHODDEF \ {"locals", (PyCFunction)builtin_locals, METH_NOARGS, builtin_locals__doc__}, @@ -855,4 +853,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=29686a89b739d600 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fb6a24964077c99b input=a9049054013a1b77]*/ From 1fe964ed7380c3362cfe99f1d3ce68bcffeaeef3 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 2 Feb 2020 23:45:07 +1000 Subject: [PATCH 27/66] Migrate to revised public API design --- Doc/c-api/reflection.rst | 6 ++-- Include/ceval.h | 29 +++++++++++++++++-- Include/cpython/frameobject.h | 40 ++++++++++++++++++++++++++- Include/frameobject.h | 16 ----------- Include/internal/pycore_frameobject.h | 1 - Objects/frameobject.c | 19 +++++-------- Python/bltinmodule.c | 8 +++--- Python/ceval.c | 6 ++-- 8 files changed, 83 insertions(+), 42 deletions(-) diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 676be12874423d..7b80c52aa967a2 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -11,19 +11,21 @@ Reflection or the interpreter of the thread state if no frame is currently executing. -.. c:function:: PyObject* PyEval_GetPyLocals() +.. c:function:: PyObject* PyLocals_Get() Return a dictionary of the local variables in the current execution frame, or ``NULL`` if no frame is currently executing. Equivalent to calling the Python level ``locals()`` builtin. +.. TODO: cover the rest of the PEP 558 API here + .. c:function:: PyObject* PyEval_GetLocals() Return a dictionary of the local variables in the current execution frame, or ``NULL`` if no frame is currently executing. - TODO: Clarify just how this relates to PyEval_GetPyLocals(). + TODO: Clarify just how this relates to PyLocals_Get(). .. c:function:: PyObject* PyEval_GetGlobals() diff --git a/Include/ceval.h b/Include/ceval.h index 1ce06d786eeb6c..37b88a6090df06 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -35,13 +35,36 @@ PyAPI_FUNC(PyObject *) PyEval_GetGlobals(void); PyAPI_FUNC(PyObject *) PyEval_GetLocals(void); PyAPI_FUNC(struct _frame *) PyEval_GetFrame(void); -PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg); -PyAPI_FUNC(int) Py_MakePendingCalls(void); +// TODO: Update PyEval_GetLocals() documentation as described in +// https://discuss.python.org/t/pep-558-defined-semantics-for-locals/2936/11 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000 -PyAPI_FUNC(PyObject *) PyEval_GetPyLocals(void); +/* Access the frame locals mapping in an implementation independent way */ + +/* PyLocals_Get() is equivalent to the Python locals() builtin. + * It returns a read/write reference or a fresh snapshot depending on the scope + * of the active frame. + */ +// TODO: Add API tests for this +PyAPI_FUNC(PyObject *) PyLocals_Get(); + +/* PyLocals_GetSnaphot() returns a fresh snapshot of the active local namespace */ +// TODO: Implement this, and add API tests +PyAPI_FUNC(PyObject *) PyLocals_GetSnapshot(); + +/* PyLocals_GetView() returns a read-only proxy for the active local namespace */ +// TODO: Implement this, and add API tests +PyAPI_FUNC(PyObject *) PyLocals_GetView(); + +/* Returns true if PyLocals_Get() returns a snapshot in the active scope */ +// TODO: Implement this, and add API tests +PyAPI_FUNC(int) PyLocals_IsSnapshot(); #endif + +PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg); +PyAPI_FUNC(int) Py_MakePendingCalls(void); + /* Protection against deeply nested recursive calls In Python 3.0, this protection has two levels: diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 580cb29a9a703f..95e2cf6059dbb5 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -72,7 +72,7 @@ PyAPI_FUNC(PyTryBlock *) PyFrame_BlockPop(PyFrameObject *); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); -/* This always raises RuntimeError now (use PyFrame_GetLocalsAttribute() instead) */ +/* This always raises RuntimeError now (use the PyLocals_* API instead) */ PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int); /* Frame object memory management */ @@ -82,6 +82,44 @@ PyAPI_FUNC(void) _PyFrame_DebugMallocStats(FILE *out); /* Return the line of code the frame is currently executing. */ PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *); + +// TODO: Determine if there are any additional APIs needed beyond what's in +// https://discuss.python.org/t/pep-558-defined-semantics-for-locals/2936/11 + + +/* Fast locals proxy allows for reliable write-through from trace functions */ +// TODO: Perhaps this should be hidden, and API users told to query for +// PyFrame_LocalsIsSnapshot() instead. Having this available seems like +// a nice way to let folks write some useful debug assertions, though. +PyTypeObject PyFastLocalsProxy_Type; +#define _PyFastLocalsProxy_CheckExact(self) \ + (Py_TYPE(self) == &PyFastLocalsProxy_Type) + + +// Underlying implementation API supporting the stable PyLocals_*() APIs +// TODO: Add specific test cases for these (as any PyLocals_* tests won't cover +// checking the status of a frame other than the currently active one) +PyAPI_FUNC(PyObject *) PyFrame_GetLocals(PyFrameObject *); + +// TODO: Implement the rest of these, and add API tests +PyAPI_FUNC(PyObject *) PyFrame_GetLocalsSnapshot(PyFrameObject *); +PyAPI_FUNC(PyObject *) PyFrame_GetLocalsView(PyFrameObject *); +// TODO: Perhaps rename this to PyFrame_GetLocalsReturnsSnapshot()? +// The current name is ambiguous given the "f_locals" attribute. +// The stable ABI is different, as the association between PyLocals_Get() +// and PyLocals_IsSnapshot() is much less ambiguous +PyAPI_FUNC(int) PyFrame_LocalsIsSnapshot(PyFrameObject *); + +// Underlying API supporting PyEval_GetLocals() +PyAPI_FUNC(PyObject *) _PyFrame_BorrowLocals(PyFrameObject *); + +/* Force an update of any selectively updated views previously returned by + * PyFrame_GetLocalsView(frame). Currently also needed in CPython when + * accessing the f_locals attribute directly and it is not a plain dict + * instance (otherwise it may report stale information). + */ +PyAPI_FUNC(int) PyFrame_RefreshLocalsView(PyFrameObject *); + #ifdef __cplusplus } #endif diff --git a/Include/frameobject.h b/Include/frameobject.h index 32bda9706e40c8..1460e2210e3173 100644 --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -16,22 +16,6 @@ extern "C" { # undef Py_CPYTHON_FRAMEOBJECT_H #endif - -// TODO: Replace this with the revised API described in -// https://discuss.python.org/t/pep-558-defined-semantics-for-locals/2936/11 -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000 -/* Fast locals proxy for reliable write-through from trace functions */ -PyTypeObject PyFastLocalsProxy_Type; -#define _PyFastLocalsProxy_CheckExact(self) \ - (Py_TYPE(self) == &PyFastLocalsProxy_Type) - -/* Access the frame locals mapping */ -PyAPI_FUNC(PyObject *) PyFrame_GetPyLocals(PyFrameObject *); // = locals() -PyAPI_FUNC(PyObject *) PyFrame_GetLocalsAttribute(PyFrameObject *); // = frame.f_locals -#endif - - - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_frameobject.h b/Include/internal/pycore_frameobject.h index 6da567238d3100..93c9d6548684b3 100644 --- a/Include/internal/pycore_frameobject.h +++ b/Include/internal/pycore_frameobject.h @@ -11,7 +11,6 @@ extern "C" { struct _frame; typedef struct _frame PyFrameObject; -PyObject *_PyFrame_BorrowPyLocals(PyFrameObject *f); /* For PyEval_GetLocals() */ void _PyFrame_PostEvalCleanup(PyFrameObject *f); /* For _PyEval_EvalFrame() */ #ifdef __cplusplus diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 3a1a092d8c2829..6c48a96fb24b77 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -50,15 +50,7 @@ void _PyFrame_PostEvalCleanup(PyFrameObject *f) } PyObject * -PyFrame_GetLocalsAttribute(PyFrameObject *f) -{ - PyObject *updated_locals = _frame_get_updated_locals(f); - Py_INCREF(updated_locals); - return updated_locals; -} - -PyObject * -_PyFrame_BorrowPyLocals(PyFrameObject *f) +_PyFrame_BorrowLocals(PyFrameObject *f) { // This is called by PyEval_GetLocals(), which has historically returned // a borrowed reference, so this does the same @@ -70,7 +62,7 @@ _PyFrame_BorrowPyLocals(PyFrameObject *f) } PyObject * -PyFrame_GetPyLocals(PyFrameObject *f) +PyFrame_GetLocals(PyFrameObject *f) { PyObject *updated_locals = _frame_get_updated_locals(f); if (_PyFastLocalsProxy_CheckExact(updated_locals)) { @@ -88,7 +80,9 @@ PyFrame_GetPyLocals(PyFrameObject *f) static PyObject * frame_getlocals(PyFrameObject *f, void *__unused) { - return PyFrame_GetLocalsAttribute(f); + PyObject *updated_locals = _frame_get_updated_locals(f); + Py_INCREF(updated_locals); + return updated_locals; } int @@ -1179,7 +1173,8 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear) PyErr_SetString( PyExc_RuntimeError, "PyFrame_LocalsToFast is no longer supported. " - "Use PyFrame_GetPyLocals() or PyFrame_GetLocalsAttribute() instead." + "Use PyFrame_GetLocals(), PyFrame_GetLocalsSnapshot(), " + "or PyFrame_GetLocalsView() instead." ); } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 38d6eaf2bcff2c..bbad2986adb54f 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -907,7 +907,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, if (globals == Py_None) { globals = PyEval_GetGlobals(); if (locals == Py_None) { - // TODO: Consider if this should use PyEval_GetPyLocals() instead + // TODO: Consider if this should use PyLocals_Get() instead locals = PyEval_GetLocals(); if (locals == NULL) return NULL; @@ -987,7 +987,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, if (globals == Py_None) { globals = PyEval_GetGlobals(); if (locals == Py_None) { - // TODO: Consider if this should use PyEval_GetPyLocals() instead + // TODO: Consider if this should use PyLocals_Get() instead locals = PyEval_GetLocals(); if (locals == NULL) return NULL; @@ -1571,7 +1571,7 @@ static PyObject * builtin_locals_impl(PyObject *module) /*[clinic end generated code: output=b46c94015ce11448 input=9869b08c278df34f]*/ { - return PyEval_GetPyLocals(); + return PyLocals_Get(); } @@ -2250,7 +2250,7 @@ builtin_vars(PyObject *self, PyObject *args) if (!PyArg_UnpackTuple(args, "vars", 0, 1, &v)) return NULL; if (v == NULL) { - d = PyEval_GetPyLocals(); + d = PyLocals_Get(); } else { if (_PyObject_LookupAttrId(v, &PyId___dict__, &d) == 0) { diff --git a/Python/ceval.c b/Python/ceval.c index ceb586d2a8cfb8..05615a531aa9e8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4735,11 +4735,11 @@ PyEval_GetLocals(void) return NULL; } - return _PyFrame_BorrowPyLocals(current_frame); + return _PyFrame_BorrowLocals(current_frame); } PyObject * -PyEval_GetPyLocals(void) +PyLocals_Get(void) { PyThreadState *tstate = _PyThreadState_GET(); PyFrameObject *current_frame = _PyEval_GetFrame(tstate); @@ -4748,7 +4748,7 @@ PyEval_GetPyLocals(void) return NULL; } - return PyFrame_GetPyLocals(current_frame); + return PyFrame_GetLocals(current_frame); } PyObject * From b047ae4394cdf43f09b9e6db30c2d7c6781c786a Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 2 Feb 2020 23:47:46 +1000 Subject: [PATCH 28/66] Rename PyFrame_LocalsIsSnapshot to PyFrame_GetLocalsReturnsSnapshot --- Include/cpython/frameobject.h | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 95e2cf6059dbb5..c2aa04b74fafb2 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -89,8 +89,9 @@ PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *); /* Fast locals proxy allows for reliable write-through from trace functions */ // TODO: Perhaps this should be hidden, and API users told to query for -// PyFrame_LocalsIsSnapshot() instead. Having this available seems like -// a nice way to let folks write some useful debug assertions, though. +// PyFrame_GetLocalsReturnsSnapshot() instead. Having this available +// seems like a nice way to let folks write some useful debug assertions, +// though. PyTypeObject PyFastLocalsProxy_Type; #define _PyFastLocalsProxy_CheckExact(self) \ (Py_TYPE(self) == &PyFastLocalsProxy_Type) @@ -104,11 +105,7 @@ PyAPI_FUNC(PyObject *) PyFrame_GetLocals(PyFrameObject *); // TODO: Implement the rest of these, and add API tests PyAPI_FUNC(PyObject *) PyFrame_GetLocalsSnapshot(PyFrameObject *); PyAPI_FUNC(PyObject *) PyFrame_GetLocalsView(PyFrameObject *); -// TODO: Perhaps rename this to PyFrame_GetLocalsReturnsSnapshot()? -// The current name is ambiguous given the "f_locals" attribute. -// The stable ABI is different, as the association between PyLocals_Get() -// and PyLocals_IsSnapshot() is much less ambiguous -PyAPI_FUNC(int) PyFrame_LocalsIsSnapshot(PyFrameObject *); +PyAPI_FUNC(int) PyFrame_GetLocalsReturnsSnapshot(PyFrameObject *); // Underlying API supporting PyEval_GetLocals() PyAPI_FUNC(PyObject *) _PyFrame_BorrowLocals(PyFrameObject *); From d1a8420577543e551bddffce2c3887230715d8d2 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 3 Feb 2020 00:00:44 +1000 Subject: [PATCH 29/66] Mark fast locals proxy as an internal type --- Include/cpython/frameobject.h | 4 ++-- Objects/frameobject.c | 10 +++++----- Objects/object.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index c2aa04b74fafb2..05b64c753952c3 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -92,9 +92,9 @@ PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *); // PyFrame_GetLocalsReturnsSnapshot() instead. Having this available // seems like a nice way to let folks write some useful debug assertions, // though. -PyTypeObject PyFastLocalsProxy_Type; +PyTypeObject _PyFastLocalsProxy_Type; #define _PyFastLocalsProxy_CheckExact(self) \ - (Py_TYPE(self) == &PyFastLocalsProxy_Type) + (Py_TYPE(self) == &_PyFastLocalsProxy_Type) // Underlying implementation API supporting the stable PyLocals_*() APIs diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 6c48a96fb24b77..4dc03a0d666bb5 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1326,7 +1326,7 @@ _PyFrame_DebugMallocStats(FILE *out) numfree, sizeof(PyFrameObject)); } -/* PyFastLocalsProxy_Type +/* _PyFastLocalsProxy_Type * * Subclass of PyDict_Proxy (currently defined in descrobject.h/.c) * @@ -1336,9 +1336,9 @@ _PyFrame_DebugMallocStats(FILE *out) * frame. */ /*[clinic input] -class fastlocalsproxy "fastlocalsproxyobject *" "&PyFastLocalsProxy_Type" +class fastlocalsproxy "fastlocalsproxyobject *" "&_PyFastLocalsProxy_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b0e135835cface9f]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a2dd0ae6e1642243]*/ typedef struct { @@ -1634,7 +1634,7 @@ _PyFastLocalsProxy_New(PyObject *frame) if (fastlocalsproxy_check_frame(frame) == -1) return NULL; - flp = PyObject_GC_New(fastlocalsproxyobject, &PyFastLocalsProxy_Type); + flp = PyObject_GC_New(fastlocalsproxyobject, &_PyFastLocalsProxy_Type); if (flp == NULL) return NULL; mapping = PyDict_New(); @@ -1689,7 +1689,7 @@ fastlocalsproxy_new_impl(PyTypeObject *type, PyObject *frame) #include "clinic/frameobject.c.h" -PyTypeObject PyFastLocalsProxy_Type = { +PyTypeObject _PyFastLocalsProxy_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "fastlocalsproxy", /* tp_name */ sizeof(fastlocalsproxyobject), /* tp_basicsize */ diff --git a/Objects/object.c b/Objects/object.c index 0855158639707b..5457581b22f371 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1901,7 +1901,7 @@ _PyTypes_Init(void) INIT_TYPE(&PyMethod_Type, "method"); INIT_TYPE(&PyFunction_Type, "function"); INIT_TYPE(&PyDictProxy_Type, "dict proxy"); - INIT_TYPE(&PyFastLocalsProxy_Type, "fast locals proxy"); + INIT_TYPE(&_PyFastLocalsProxy_Type, "fast locals proxy"); INIT_TYPE(&PyGen_Type, "generator"); INIT_TYPE(&PyGetSetDescr_Type, "get-set descriptor"); INIT_TYPE(&PyWrapperDescr_Type, "wrapper"); From eccb1ea3b3ddca6c12747382718f297a12596149 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 16 Feb 2020 16:53:50 +1000 Subject: [PATCH 30/66] Update draft C API to match latest PEP text --- Include/ceval.h | 14 +++++++++----- Include/cpython/frameobject.h | 6 +++--- Objects/frameobject.c | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Include/ceval.h b/Include/ceval.h index 37b88a6090df06..d1053cbe4dacb1 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -42,23 +42,27 @@ PyAPI_FUNC(struct _frame *) PyEval_GetFrame(void); /* Access the frame locals mapping in an implementation independent way */ /* PyLocals_Get() is equivalent to the Python locals() builtin. - * It returns a read/write reference or a fresh snapshot depending on the scope + * It returns a read/write reference or a shallow copy depending on the scope * of the active frame. */ // TODO: Add API tests for this PyAPI_FUNC(PyObject *) PyLocals_Get(); -/* PyLocals_GetSnaphot() returns a fresh snapshot of the active local namespace */ +/* PyLocals_GetCopy() returns a fresh shallow copy of the active local namespace */ // TODO: Implement this, and add API tests -PyAPI_FUNC(PyObject *) PyLocals_GetSnapshot(); +PyAPI_FUNC(PyObject *) PyLocals_GetCopy(); /* PyLocals_GetView() returns a read-only proxy for the active local namespace */ // TODO: Implement this, and add API tests PyAPI_FUNC(PyObject *) PyLocals_GetView(); -/* Returns true if PyLocals_Get() returns a snapshot in the active scope */ +/* PyLocals_RefreshViews() updates previously created locals views */ // TODO: Implement this, and add API tests -PyAPI_FUNC(int) PyLocals_IsSnapshot(); +PyAPI_FUNC(int) PyLocals_RefreshViews(); + +/* Returns true if PyLocals_Get() returns a shallow copy in the active scope */ +// TODO: Implement this, and add API tests +PyAPI_FUNC(int) PyLocals_GetReturnsCopy(); #endif diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 05b64c753952c3..84e4cb89849c2b 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -89,7 +89,7 @@ PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *); /* Fast locals proxy allows for reliable write-through from trace functions */ // TODO: Perhaps this should be hidden, and API users told to query for -// PyFrame_GetLocalsReturnsSnapshot() instead. Having this available +// PyFrame_GetLocalsReturnsCopy() instead. Having this available // seems like a nice way to let folks write some useful debug assertions, // though. PyTypeObject _PyFastLocalsProxy_Type; @@ -103,9 +103,9 @@ PyTypeObject _PyFastLocalsProxy_Type; PyAPI_FUNC(PyObject *) PyFrame_GetLocals(PyFrameObject *); // TODO: Implement the rest of these, and add API tests -PyAPI_FUNC(PyObject *) PyFrame_GetLocalsSnapshot(PyFrameObject *); +PyAPI_FUNC(PyObject *) PyFrame_GetLocalsCopy(PyFrameObject *); PyAPI_FUNC(PyObject *) PyFrame_GetLocalsView(PyFrameObject *); -PyAPI_FUNC(int) PyFrame_GetLocalsReturnsSnapshot(PyFrameObject *); +PyAPI_FUNC(int) PyFrame_GetLocalsReturnsCopy(PyFrameObject *); // Underlying API supporting PyEval_GetLocals() PyAPI_FUNC(PyObject *) _PyFrame_BorrowLocals(PyFrameObject *); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 5ad39696d22377..460368fa0b768d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1173,7 +1173,7 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear) PyErr_SetString( PyExc_RuntimeError, "PyFrame_LocalsToFast is no longer supported. " - "Use PyFrame_GetLocals(), PyFrame_GetLocalsSnapshot(), " + "Use PyFrame_GetLocals(), PyFrame_GetLocalsCopy(), " "or PyFrame_GetLocalsView() instead." ); } From c1933e744f7d5b1d3489d3588267178b94851ab9 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 16 Feb 2020 17:25:58 +1000 Subject: [PATCH 31/66] Migrate exec() and eval() to PyLocals_Get() --- Python/bltinmodule.c | 63 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 13 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 1baaa2c74f6d8d..48db2b7ddcada2 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -892,9 +892,15 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals) /*[clinic end generated code: output=0a0824aa70093116 input=11ee718a8640e527]*/ { - PyObject *result, *source_copy; + PyObject *result, *source_copy = NULL, *locals_ref = NULL; const char *str; + // PEP 558: "locals_ref" is a quick fix to adapt this function to + // PyLocals_Get() returning a new reference, whereas PyEval_GetLocals() + // returned a borrowed reference. The proper fix will be to factor out + // a common set up function shared between eval() and exec() that means + // the main function body can always just call Py_DECREF(locals) at the end. + if (locals != Py_None && !PyMapping_Check(locals)) { PyErr_SetString(PyExc_TypeError, "locals must be a mapping"); return NULL; @@ -908,49 +914,61 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, if (globals == Py_None) { globals = PyEval_GetGlobals(); if (locals == Py_None) { - // TODO: Consider if this should use PyLocals_Get() instead - locals = PyEval_GetLocals(); + locals = PyLocals_Get(); if (locals == NULL) return NULL; + // This function owns the locals ref, need to decref on exit + locals_ref = locals; } } - else if (locals == Py_None) + else if (locals == Py_None) { locals = globals; + } if (globals == NULL || locals == NULL) { PyErr_SetString(PyExc_TypeError, "eval must be given globals and locals " "when called without a frame"); + Py_XDECREF(locals_ref); return NULL; } if (_PyDict_GetItemIdWithError(globals, &PyId___builtins__) == NULL) { if (_PyDict_SetItemId(globals, &PyId___builtins__, - PyEval_GetBuiltins()) != 0) + PyEval_GetBuiltins()) != 0) { + Py_XDECREF(locals_ref); return NULL; + } } else if (PyErr_Occurred()) { + Py_XDECREF(locals_ref); return NULL; } if (PyCode_Check(source)) { if (PySys_Audit("exec", "O", source) < 0) { + Py_XDECREF(locals_ref); return NULL; } if (PyCode_GetNumFree((PyCodeObject *)source) > 0) { PyErr_SetString(PyExc_TypeError, "code object passed to eval() may not contain free variables"); + Py_XDECREF(locals_ref); return NULL; } - return PyEval_EvalCode(source, globals, locals); + result = PyEval_EvalCode(source, globals, locals); + Py_XDECREF(locals_ref); + return result; } PyCompilerFlags cf = _PyCompilerFlags_INIT; cf.cf_flags = PyCF_SOURCE_IS_UTF8; str = _Py_SourceAsString(source, "eval", "string, bytes or code", &cf, &source_copy); - if (str == NULL) + if (str == NULL) { + Py_XDECREF(locals_ref); return NULL; + } while (*str == ' ' || *str == '\t') str++; @@ -958,6 +976,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, (void)PyEval_MergeCompilerFlags(&cf); result = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf); Py_XDECREF(source_copy); + Py_XDECREF(locals_ref); return result; } @@ -983,47 +1002,61 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals) /*[clinic end generated code: output=3c90efc6ab68ef5d input=01ca3e1c01692829]*/ { - PyObject *v; + PyObject *v, *locals_ref = NULL; + + // (ncoghlan) There is an annoying level of gratuitous differences between + // the exec setup code and the eval setup code, when ideally they would be + // sharing a common helper function at least as far as the call to + // PyCode_Check... if (globals == Py_None) { globals = PyEval_GetGlobals(); if (locals == Py_None) { - // TODO: Consider if this should use PyLocals_Get() instead - locals = PyEval_GetLocals(); + locals = PyLocals_Get(); if (locals == NULL) return NULL; + // This function owns the locals ref, need to decref on exit + locals_ref = locals; } if (!globals || !locals) { PyErr_SetString(PyExc_SystemError, "globals and locals cannot be NULL"); + Py_XDECREF(locals_ref); return NULL; } } - else if (locals == Py_None) + else if (locals == Py_None) { locals = globals; + } if (!PyDict_Check(globals)) { PyErr_Format(PyExc_TypeError, "exec() globals must be a dict, not %.100s", Py_TYPE(globals)->tp_name); + Py_XDECREF(locals_ref); return NULL; } if (!PyMapping_Check(locals)) { PyErr_Format(PyExc_TypeError, "locals must be a mapping or None, not %.100s", Py_TYPE(locals)->tp_name); + Py_XDECREF(locals_ref); return NULL; } if (_PyDict_GetItemIdWithError(globals, &PyId___builtins__) == NULL) { if (_PyDict_SetItemId(globals, &PyId___builtins__, - PyEval_GetBuiltins()) != 0) + PyEval_GetBuiltins()) != 0) { + Py_XDECREF(locals_ref); return NULL; + } } else if (PyErr_Occurred()) { + Py_XDECREF(locals_ref); return NULL; } if (PyCode_Check(source)) { if (PySys_Audit("exec", "O", source) < 0) { + Py_XDECREF(locals_ref); return NULL; } @@ -1031,6 +1064,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyErr_SetString(PyExc_TypeError, "code object passed to exec() may not " "contain free variables"); + Py_XDECREF(locals_ref); return NULL; } v = PyEval_EvalCode(source, globals, locals); @@ -1043,8 +1077,10 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, str = _Py_SourceAsString(source, "exec", "string, bytes or code", &cf, &source_copy); - if (str == NULL) + if (str == NULL) { + Py_XDECREF(locals_ref); return NULL; + } if (PyEval_MergeCompilerFlags(&cf)) v = PyRun_StringFlags(str, Py_file_input, globals, locals, &cf); @@ -1052,6 +1088,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, v = PyRun_String(str, Py_file_input, globals, locals); Py_XDECREF(source_copy); } + Py_XDECREF(locals_ref); if (v == NULL) return NULL; Py_DECREF(v); From 68f10ced11a24166269cef8416bb98715d9bd200 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 22 Feb 2020 13:57:19 +1000 Subject: [PATCH 32/66] Avoid circular reference between locals proxy and frame Mark Shannon pointed out that if every access to the Python level frame f_locals attributes creates a new write-through proxy instance, there's no need to actually store the proxy itself on the frame. Instead, we can continue to store only the shared mapping returned by PyEval_GetLocals(), and avoid creating a circular reference that we then need to break when frame evaluation ends. --- Include/internal/pycore_ceval.h | 5 +- Include/internal/pycore_frameobject.h | 2 - Objects/frameobject.c | 105 ++++++++++---------------- 3 files changed, 40 insertions(+), 72 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 78d5fb4067e13a..680ca6eb9949d6 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -14,7 +14,6 @@ struct _ceval_runtime_state; struct _frame; #include "pycore_pystate.h" /* PyInterpreterState.eval_frame */ -#include "pycore_frameobject.h" /* _PyFrame_PostEvalCleanup */ PyAPI_FUNC(void) _Py_FinishPendingCalls(PyThreadState *tstate); PyAPI_FUNC(void) _PyEval_Initialize(struct _ceval_runtime_state *); @@ -41,9 +40,7 @@ void _PyEval_Fini(void); static inline PyObject* _PyEval_EvalFrame(PyThreadState *tstate, struct _frame *f, int throwflag) { - PyObject * result = tstate->interp->eval_frame(f, throwflag); - _PyFrame_PostEvalCleanup(f); // Notify frame that execution has completed - return result; + return tstate->interp->eval_frame(f, throwflag); } extern PyObject *_PyEval_EvalCode( diff --git a/Include/internal/pycore_frameobject.h b/Include/internal/pycore_frameobject.h index 93c9d6548684b3..6725c290a8b746 100644 --- a/Include/internal/pycore_frameobject.h +++ b/Include/internal/pycore_frameobject.h @@ -11,8 +11,6 @@ extern "C" { struct _frame; typedef struct _frame PyFrameObject; -void _PyFrame_PostEvalCleanup(PyFrameObject *f); /* For _PyEval_EvalFrame() */ - #ifdef __cplusplus } #endif diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 460368fa0b768d..1d582f49001a8b 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -12,8 +12,6 @@ #define OFF(x) offsetof(PyFrameObject, x) static PyObject *_PyFastLocalsProxy_New(PyObject *frame); -static PyObject *_PyFastLocalsProxy_BorrowLocals(PyObject *flp); -static void _PyFastLocalsProxy_BreakReferenceCycle(PyObject *flp); static PyMemberDef frame_memberlist[] = { {"f_back", T_OBJECT, OFF(f_back), READONLY}, @@ -35,40 +33,23 @@ _frame_get_updated_locals(PyFrameObject *f) return f->f_locals; } -void _PyFrame_PostEvalCleanup(PyFrameObject *f) -{ - // Don't clean up still running coroutines and generators - if (f->f_executing) { - return; - } - - // Ensure that any reference cycle between the frame and f_locals gets - // broken when the frame finishes execution. - if (f->f_locals && _PyFastLocalsProxy_CheckExact(f->f_locals)) { - _PyFastLocalsProxy_BreakReferenceCycle(f->f_locals); - } -} - PyObject * _PyFrame_BorrowLocals(PyFrameObject *f) { // This is called by PyEval_GetLocals(), which has historically returned // a borrowed reference, so this does the same - PyObject *updated_locals = _frame_get_updated_locals(f); - if (_PyFastLocalsProxy_CheckExact(updated_locals)) { - updated_locals = _PyFastLocalsProxy_BorrowLocals(updated_locals); - } - return updated_locals; + return _frame_get_updated_locals(f); } PyObject * PyFrame_GetLocals(PyFrameObject *f) { + // This API implements the Python level locals() builtin PyObject *updated_locals = _frame_get_updated_locals(f); - if (_PyFastLocalsProxy_CheckExact(updated_locals)) { - // Take a snapshot of optimised scopes to avoid quirky side effects - PyObject *d = _PyFastLocalsProxy_BorrowLocals(updated_locals); - updated_locals = PyDict_Copy(d); + assert(f->f_code); + if (f->f_code->co_flags & CO_OPTIMIZED) { + // PEP 558: Make a copy of optimised scopes to avoid quirky side effects + updated_locals = PyDict_Copy(updated_locals); } else { // Share a direct locals reference for class and module scopes Py_INCREF(updated_locals); @@ -80,9 +61,21 @@ PyFrame_GetLocals(PyFrameObject *f) static PyObject * frame_getlocals(PyFrameObject *f, void *__unused) { - PyObject *updated_locals = _frame_get_updated_locals(f); - Py_INCREF(updated_locals); - return updated_locals; + // This API implements the Python level frame.f_locals descriptor + PyObject *f_locals_attr = NULL; + assert(f->f_code); + if (f->f_code->co_flags & CO_OPTIMIZED) { + /* PEP 558: If this is an optimized frame, ensure f_locals at the Python + * layer is a new fastlocalsproxy instance, while f_locals at the C + * layer still refers to the underlying shared namespace mapping. + */ + f_locals_attr = _PyFastLocalsProxy_New((PyObject *) f); + } else { + // Share a direct locals reference for class and module scopes + f_locals_attr = _frame_get_updated_locals(f); + Py_INCREF(f_locals_attr); + } + return f_locals_attr; } int @@ -465,6 +458,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore codetracker tracker; + assert(f->f_code); init_codetracker(&tracker, f->f_code); move_to_addr(&tracker, f->f_lasti); int current_line = tracker.line; @@ -702,6 +696,7 @@ frame_dealloc(PyFrameObject *f) Py_CLEAR(f->f_trace); co = f->f_code; + assert(co); if (co->co_zombieframe == NULL) co->co_zombieframe = f; else if (numfree < PyFrame_MAXFREELIST) { @@ -730,6 +725,7 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) Py_VISIT(f->f_trace); /* locals */ + assert(f->f_code); slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars); fastlocals = f->f_localsplus; for (i = slots; --i >= 0; ++fastlocals) @@ -800,6 +796,7 @@ frame_sizeof(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) { Py_ssize_t res, extras, ncells, nfrees; + assert(f->f_code); ncells = PyTuple_GET_SIZE(f->f_code->co_cellvars); nfrees = PyTuple_GET_SIZE(f->f_code->co_freevars); extras = f->f_code->co_stacksize + f->f_code->co_nlocals + @@ -1091,25 +1088,13 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f) return -1; } co = f->f_code; + assert(co); locals = f->f_locals; if (locals == NULL) { - if (co->co_flags & CO_OPTIMIZED) { - /* PEP 558: If this is an optimized frame, ensure f_locals is a - * fastlocalsproxy instance, while locals refers to the underlying mapping. - */ - PyObject *flp = _PyFastLocalsProxy_New((PyObject *) f); - if (flp == NULL) { - return -1; - } - f->f_locals = flp; - locals = _PyFastLocalsProxy_BorrowLocals(flp); - } else { - locals = f->f_locals = PyDict_New(); - if (locals == NULL) - return -1; + locals = f->f_locals = PyDict_New(); + if (locals == NULL) { + return -1; } - } else if (_PyFastLocalsProxy_CheckExact(locals)) { - locals = _PyFastLocalsProxy_BorrowLocals(locals); } map = co->co_varnames; @@ -1173,8 +1158,8 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear) PyErr_SetString( PyExc_RuntimeError, "PyFrame_LocalsToFast is no longer supported. " - "Use PyFrame_GetLocals(), PyFrame_GetLocalsCopy(), " - "or PyFrame_GetLocalsView() instead." + "Use PyObject_GetAttrString(frame, \"f_locals\") " + "to obtain a write-through mapping proxy instead." ); } @@ -1247,6 +1232,7 @@ _PyFrame_BuildFastRefs(PyFrameObject *f) * to the corresponding cell objects */ co = f->f_code; + assert(co); if (!(co->co_flags & CO_OPTIMIZED)) { PyErr_SetString(PyExc_SystemError, "attempted to build fast refs lookup table for non-optimized scope"); @@ -1348,6 +1334,8 @@ typedef struct { PyObject *fast_refs; /* Cell refs and local variable indices */ } fastlocalsproxyobject; +// TODO: Implement correct Python sizeof() support for fastlocalsproxyobject + /* Provide __setitem__() and __delitem__() implementations that not only * write to the namespace returned by locals(), but also write to the frame * storage directly (either the closure cells or the fast locals array) @@ -1631,20 +1619,22 @@ _PyFastLocalsProxy_New(PyObject *frame) fastlocalsproxyobject *flp; PyObject *mapping, *fast_refs; - if (fastlocalsproxy_check_frame(frame) == -1) + if (fastlocalsproxy_check_frame(frame) == -1) { return NULL; + } flp = PyObject_GC_New(fastlocalsproxyobject, &_PyFastLocalsProxy_Type); if (flp == NULL) return NULL; - mapping = PyDict_New(); + mapping = _frame_get_updated_locals((PyFrameObject *) frame); if (mapping == NULL) { Py_DECREF(flp); return NULL; } flp->mapping = mapping; - Py_INCREF(frame); + Py_INCREF(flp->mapping); flp->frame = (PyFrameObject *) frame; + Py_INCREF(flp->frame); fast_refs = _PyFrame_BuildFastRefs(flp->frame); if (fast_refs == NULL) { Py_DECREF(flp); // Also handles DECREF for mapping and frame @@ -1655,23 +1645,6 @@ _PyFastLocalsProxy_New(PyObject *frame) return (PyObject *)flp; } -static PyObject * -_PyFastLocalsProxy_BorrowLocals(PyObject *self) -{ - assert(_PyFastLocalsProxy_CheckExact(self)); - return ((fastlocalsproxyobject *) self)->mapping; -} - -static void -_PyFastLocalsProxy_BreakReferenceCycle(PyObject *self) -{ - assert(_PyFastLocalsProxy_CheckExact(self)); - fastlocalsproxyobject *flp = (fastlocalsproxyobject *) self; - Py_CLEAR(flp->frame); - // We keep flp->fast_refs alive to allow updating of closure variables -} - - /*[clinic input] @classmethod fastlocalsproxy.__new__ as fastlocalsproxy_new From 29ce344d55c502a08481987e5ff0cc749ead7946 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 22 Feb 2020 14:17:32 +1000 Subject: [PATCH 33/66] Add back implicit view refresh in Python trace hook --- Python/sysmodule.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index f6e8c6af764832..ebf6674f810b2e 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -878,6 +878,14 @@ static PyObject * call_trampoline(PyObject* callback, PyFrameObject *frame, int what, PyObject *arg) { + // Implicitly refresh frame namespace snapshot stored in f_locals, + // as even though the Python level f_locals descriptor now also + // refreshs the snapshot, trace functions may be calling other C APIs + // that expect the snapshot to have already been refreshed + if (PyFrame_FastToLocalsWithError(frame) < 0) { + return NULL; + } + PyObject *stack[3]; stack[0] = (PyObject *)frame; From ed6e53be7794ec94924b69cd46d5b009633c6307 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 22 Feb 2020 14:32:51 +1000 Subject: [PATCH 34/66] Attempt to tidy up Mac OS X compile warnings/errors --- Include/ceval.h | 10 +++++----- Objects/frameobject.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/ceval.h b/Include/ceval.h index d1053cbe4dacb1..4976610ff4e521 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -46,23 +46,23 @@ PyAPI_FUNC(struct _frame *) PyEval_GetFrame(void); * of the active frame. */ // TODO: Add API tests for this -PyAPI_FUNC(PyObject *) PyLocals_Get(); +PyAPI_FUNC(PyObject *) PyLocals_Get(void); /* PyLocals_GetCopy() returns a fresh shallow copy of the active local namespace */ // TODO: Implement this, and add API tests -PyAPI_FUNC(PyObject *) PyLocals_GetCopy(); +PyAPI_FUNC(PyObject *) PyLocals_GetCopy(void); /* PyLocals_GetView() returns a read-only proxy for the active local namespace */ // TODO: Implement this, and add API tests -PyAPI_FUNC(PyObject *) PyLocals_GetView(); +PyAPI_FUNC(PyObject *) PyLocals_GetView(void); /* PyLocals_RefreshViews() updates previously created locals views */ // TODO: Implement this, and add API tests -PyAPI_FUNC(int) PyLocals_RefreshViews(); +PyAPI_FUNC(int) PyLocals_RefreshViews(void); /* Returns true if PyLocals_Get() returns a shallow copy in the active scope */ // TODO: Implement this, and add API tests -PyAPI_FUNC(int) PyLocals_GetReturnsCopy(); +PyAPI_FUNC(int) PyLocals_GetReturnsCopy(void); #endif diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 1d582f49001a8b..d970a8e0b1b30f 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -59,7 +59,7 @@ PyFrame_GetLocals(PyFrameObject *f) } static PyObject * -frame_getlocals(PyFrameObject *f, void *__unused) +frame_getlocals(PyFrameObject *f, void *Py_UNUSED(ignored)) { // This API implements the Python level frame.f_locals descriptor PyObject *f_locals_attr = NULL; From a348d086c447a77860da8b17c79815bff6da2291 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 3 Jul 2021 17:05:28 +1000 Subject: [PATCH 35/66] Fix Argument Clinic checksum --- Python/clinic/bltinmodule.c.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 5ca244396f07df..1a14f698da87cc 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -875,4 +875,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=e1d8057298b5de61 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=01717291dd0bc202 input=a9049054013a1b77]*/ From 69c8f19742d8a8e083d11c568d512d6668c3b1a1 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 3 Jul 2021 17:14:22 +1000 Subject: [PATCH 36/66] Fix stable ABI minimum version --- Include/ceval.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/ceval.h b/Include/ceval.h index b96717e6fb114d..8f062e656643c8 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -36,7 +36,7 @@ PyAPI_FUNC(PyFrameObject *) PyEval_GetFrame(void); // TODO: Update PyEval_GetLocals() documentation as described in // https://discuss.python.org/t/pep-558-defined-semantics-for-locals/2936/11 -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000 +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030B0000 /* Access the frame locals mapping in an implementation independent way */ /* PyLocals_Get() is equivalent to the Python locals() builtin. From caeaf667a72e020b623dc1dc12564dc05be6b74b Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 10 Jul 2021 20:14:44 +1000 Subject: [PATCH 37/66] Bring implementation into line with latest PEP version --- Lib/test/test_frame.py | 4 +- Objects/descrobject.c | 3 +- Objects/frameobject.c | 634 +++++++++++++++++++++++++---------------- Python/ceval.c | 4 +- Python/sysmodule.c | 8 - 5 files changed, 400 insertions(+), 253 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index a715e725a7e45b..f6e456812a6b26 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -183,8 +183,8 @@ def test_clear_locals(self): def test_locals_clear_locals(self): # Test f_locals before and after clear() (to exercise caching) f, outer, inner = self.make_frames() - outer.f_locals - inner.f_locals + self.assertNotEqual(outer.f_locals, {}) + self.assertNotEqual(inner.f_locals, {}) outer.clear() inner.clear() self.assertEqual(outer.f_locals, {}) diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 9676653ddb1e39..d81705a0efcfa9 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1855,8 +1855,7 @@ PyTypeObject PyDictProxy_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_MAPPING | - Py_TPFLAGS_BASETYPE, /* tp_flags */ + Py_TPFLAGS_MAPPING, /* tp_flags */ 0, /* tp_doc */ mappingproxy_traverse, /* tp_traverse */ 0, /* tp_clear */ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 5d6b77b5be2a8a..e0d11a969a8e9c 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1016,7 +1016,7 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f) /* If the namespace is unoptimized, then one of the following cases applies: 1. It does not contain free variables, because it - uses import * or is a top-level namespace. + is a top-level namespace. 2. It is a class namespace. We don't want to accidentally copy free variables into the locals dict used by the class. @@ -1047,8 +1047,8 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f) } // (likely) Otherwise it it is an arg (kind & CO_FAST_LOCAL), // with the initial value set by _PyEval_MakeFrameVector()... - // (unlikely) ...or it was set to some initial value by - // an earlier call to PyFrame_LocalsToFast(). + // (unlikely) ...or it was set to some initial value via + // the frame locals proxy } } } @@ -1108,120 +1108,79 @@ set_fast_ref(PyObject *fast_refs, PyObject *key, PyObject *value) return status; } -static int -add_local_refs(PyObject *map, Py_ssize_t nmap, PyObject *fast_refs) -{ - /* Populate a lookup table from variable names to fast locals array indices */ - Py_ssize_t j; - assert(PyTuple_Check(map)); - assert(PyDict_Check(fast_refs)); - assert(PyTuple_Size(map) >= nmap); - for (j = nmap; --j >= 0; ) { - PyObject *key = PyTuple_GET_ITEM(map, j); - PyObject *value = PyLong_FromSsize_t(j); // set_fast_ref steals the value - /* Values may be missing if the frame has been cleared */ - if (value != NULL) { - if (set_fast_ref(fast_refs, key, value) != 0) { - return -1; - } - } - } - return 0; -} - -static int -add_nonlocal_refs(PyObject *map, Py_ssize_t nmap, PyObject *fast_refs, PyObject **cells) -{ - /* Populate a lookup table from variable names to closure cell references */ - Py_ssize_t j; - assert(PyTuple_Check(map)); - assert(PyDict_Check(fast_refs)); - assert(PyTuple_Size(map) >= nmap); - for (j = nmap; --j >= 0; ) { - PyObject *key = PyTuple_GET_ITEM(map, j); - PyObject *value = cells[j]; - /* Values may be missing if the frame has been cleared */ - if (value != NULL) { - assert(PyCell_Check(value)); - Py_INCREF(value); // set_fast_ref steals the value - if (set_fast_ref(fast_refs, key, value) != 0) { - return -1; - } - } - } - return 0; -} - - static PyObject * _PyFrame_BuildFastRefs(PyFrameObject *f) { - // PEP 558 TODO: This reverted to just doing LocalsToFast in the 3.11 dev sync - // PEP 558 fast locals proxy functionality needs to be restored - PyObject *fast_refs=NULL, *map=NULL; - - /* Merge locals into fast locals */ - PyObject *locals; - PyObject **fast; + /* Construct a combined mapping from local variable names to indices + * in the fast locals array, and from nonlocal variable names directly + * to the corresponding cell objects + */ + PyObject **fast_locals; PyCodeObject *co; - if (f == NULL || f->f_state == FRAME_CLEARED) { + PyObject *fast_refs; + + + if (f == NULL) { + PyErr_BadInternalCall(); return NULL; } - locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET]; - if (locals == NULL) { + co = _PyFrame_GetCode(f); + assert(co); + fast_locals = f->f_localsptr; + + if (fast_locals == NULL || !(co->co_flags & CO_OPTIMIZED)) { + PyErr_SetString(PyExc_SystemError, + "attempted to build fast refs lookup table for non-optimized scope"); return NULL; } - fast = f->f_localsptr; - co = _PyFrame_GetCode(f); - for (int i = 0; i < co->co_nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + fast_refs = PyDict_New(); + if (fast_refs == NULL) { + return NULL; + } - /* Same test as in PyFrame_FastToLocals() above. */ - if (kind & CO_FAST_FREE && !(co->co_flags & CO_OPTIMIZED)) { - continue; - } - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject *value = PyObject_GetItem(locals, name); - if (value == NULL) { - PyErr_Clear(); - continue; - } - PyObject *oldvalue = fast[i]; - PyObject *cell = NULL; - if (kind == CO_FAST_FREE) { - // The cell was set by _PyEval_MakeFrameVector() from - // the function's closure. - assert(oldvalue != NULL && PyCell_Check(oldvalue)); - cell = oldvalue; - } - else if (kind & CO_FAST_CELL && oldvalue != NULL) { - /* Same test as in PyFrame_FastToLocals() above. */ - if (PyCell_Check(oldvalue) && - _PyFrame_OpAlreadyRan(f, MAKE_CELL, i)) { - // (likely) MAKE_CELL must have executed already. - cell = oldvalue; + if (f->f_state != FRAME_CLEARED) { + for (int i = 0; i < co->co_nlocalsplus; i++) { + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyObject *target = NULL; + if (kind & CO_FAST_FREE) { + // Reference to closure cell, save it as the proxy target + target = fast_locals[i]; + assert(target != NULL && PyCell_Check(target)); + Py_INCREF(target); + } else if (kind & CO_FAST_CELL) { + // Closure cell referenced from nested scopes + // Save it as the proxy target if the cell already exists, + // otherwise save the index and fix it up later on access + target = fast_locals[i]; + if (target != NULL && PyCell_Check(target) && + _PyFrame_OpAlreadyRan(f, MAKE_CELL, i)) { + // MAKE_CELL built the cell, so use it as the proxy target + Py_INCREF(target); + } else { + // MAKE_CELL hasn't run yet, so just store the lookup index + // The proxy will check the kind on access, and switch over + // to using the cell once MAKE_CELL creates it + target = PyLong_FromSsize_t(i); + } + } else if (kind & CO_FAST_LOCAL) { + // Ordinary fast local variable. Save index as the proxy target + target = PyLong_FromSsize_t(i); + } else { + PyErr_SetString(PyExc_SystemError, + "unknown local variable kind while building fast refs lookup table"); } - // (unlikely) Otherwise, it must have been set to some - // initial value by an earlier call to PyFrame_LocalsToFast(). - } - if (cell != NULL) { - oldvalue = PyCell_GET(cell); - if (value != oldvalue) { - Py_XDECREF(oldvalue); - Py_XINCREF(value); - PyCell_SET(cell, value); + if (target == NULL) { + Py_DECREF(fast_refs); + return NULL; + } + if (set_fast_ref(fast_refs, name, target) != 0) { + return NULL; } } - else if (value != oldvalue) { - Py_XINCREF(value); - Py_XSETREF(fast[i], value); - } - Py_XDECREF(value); } - return fast_refs; - } /* Clear out the free list */ @@ -1299,12 +1258,7 @@ _PyEval_BuiltinsFromGlobals(PyThreadState *tstate, PyObject *globals) /* _PyFastLocalsProxy_Type * - * Subclass of PyDict_Proxy (currently defined in descrobject.h/.c) - * - * Mostly works just like PyDict_Proxy (backed by the frame locals), but - * supports setitem and delitem, with writes being delegated to both the - * referenced mapping *and* the fast locals and/or cell reference on the - * frame. + * Mapping object that provides name-based access to the fast locals on a frame */ /*[clinic input] class fastlocalsproxy "fastlocalsproxyobject *" "&_PyFastLocalsProxy_Type" @@ -1313,95 +1267,312 @@ class fastlocalsproxy "fastlocalsproxyobject *" "&_PyFastLocalsProxy_Type" typedef struct { - PyObject_HEAD /* Match mappingproxyobject in descrobject.c */ - PyObject *mapping; /* Match mappingproxyobject in descrobject.c */ + PyObject_HEAD PyFrameObject *frame; PyObject *fast_refs; /* Cell refs and local variable indices */ } fastlocalsproxyobject; -// TODO: Implement correct Python sizeof() support for fastlocalsproxyobject +// PEP 558 TODO: Implement correct Python sizeof() support for fastlocalsproxyobject + +static int +fastlocalsproxy_init_fast_refs(fastlocalsproxyobject *flp) +{ + // Build fast ref mapping if it hasn't been built yet + assert(flp); + if (flp->fast_refs != NULL) { + return 0; + } + PyObject *fast_refs = _PyFrame_BuildFastRefs(flp->frame); + if (fast_refs == NULL) { + return -1; + } + flp->fast_refs = fast_refs; + return 0; +} + +static Py_ssize_t +fastlocalsproxy_len(fastlocalsproxyobject *flp) +{ + assert(flp); + if (fastlocalsproxy_init_fast_refs(flp) != 0) { + return -1; + } + return PyObject_Size(flp->fast_refs); +} + +static PyObject * +fastlocalsproxy_getitem(fastlocalsproxyobject *flp, PyObject *key) +{ + // PEP 558 TODO: try to factor out the common get/set key lookup code + assert(flp); + if (fastlocalsproxy_init_fast_refs(flp) != 0) { + return NULL; + } + PyObject *fast_ref = PyDict_GetItem(flp->fast_refs, key); + if (fast_ref == NULL) { + // No such local variable, let KeyError escape + return NULL; + } + /* Key is a valid Python variable for the frame, so retrieve the value */ + if (PyCell_Check(fast_ref)) { + // Closure cells can be queried even after the frame terminates + return PyCell_Get(fast_ref); + } + PyFrameObject *f = flp->frame; + if (f->f_state == FRAME_CLEARED) { + PyErr_Format(PyExc_RuntimeError, + "Fast locals proxy attempted to read from cleared frame (%R)", f); + return NULL; + } + /* Fast ref is a Python int mapping into the fast locals array */ + assert(PyLong_CheckExact(fast_ref)); + Py_ssize_t offset = PyLong_AsSsize_t(fast_ref); + if (offset < 0) { + return NULL; + } + PyCodeObject *co = _PyFrame_GetCode(f); + assert(co); + Py_ssize_t max_offset = co->co_nlocalsplus - 1; + if (offset > max_offset) { + PyErr_Format(PyExc_SystemError, + "Fast locals ref (%zd) exceeds array bound (%zd)", + offset, max_offset); + return NULL; + } + PyObject **fast_locals = f->f_localsptr; + PyObject *value = fast_locals[offset]; + // Check if MAKE_CELL has been called since the proxy was created + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, offset); + if (kind & CO_FAST_CELL) { + // Value hadn't been converted to a cell yet when the proxy was created + // Update the proxy if MAKE_CELL has run since the last access, + // otherwise continue treating it as a regular local variable + PyObject *target = value; + if (target != NULL && PyCell_Check(target) && + _PyFrame_OpAlreadyRan(f, MAKE_CELL, offset)) { + // MAKE_CELL has built the cell, so use it as the proxy target + Py_INCREF(target); + if (set_fast_ref(flp->fast_refs, key, target) != 0) { + return NULL; + } + return PyCell_Get(target); + } + } + + // Local variable, or future cell variable that hasn't been converted yet + if (value == NULL && !PyErr_Occurred()) { + // Report KeyError if the variable hasn't been bound to a value yet + // (akin to getting an UnboundLocalError in running code) + PyErr_SetObject(PyExc_KeyError, key); + } else { + Py_XINCREF(value); + } + return value; +} -/* Provide __setitem__() and __delitem__() implementations that not only - * write to the namespace returned by locals(), but also write to the frame - * storage directly (either the closure cells or the fast locals array) - */ static int fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObject *value) { - int result = 0; - assert(PyDict_Check(flp->fast_refs)); + // PEP 558 TODO: try to factor out the common get/set key lookup code + assert(flp); + if (fastlocalsproxy_init_fast_refs(flp) != 0) { + return -1; + } PyObject *fast_ref = PyDict_GetItem(flp->fast_refs, key); - if (fast_ref != NULL) { - /* Key is an actual Python variable, so update that reference */ - if (PyCell_Check(fast_ref)) { - // Closure cells can be updated even after the frame terminates - result = PyCell_Set(fast_ref, value); - } else if (flp->frame == NULL) { - // This indicates the frame has finished executing and the link - // back to the frame has been cleared to break the reference cycle - return 0; - } else { - /* Fast ref is a Python int mapping into the fast locals array */ - Py_ssize_t offset = PyLong_AsSsize_t(fast_ref); - Py_ssize_t max_offset = _PyFrame_GetCode(flp->frame)->co_nlocals; - if (offset < 0) { - result = -1; - } else if (offset > max_offset) { - PyErr_Format(PyExc_SystemError, - "Fast locals ref (%zd) exceeds array bound (%zd)", - offset, max_offset); - result = -1; - } - if (result == 0) { - PyErr_SetString(PyExc_SystemError, - "PEP 558 TODO: Make locals proxy compatible with 3.11"); - result = -1; - // Py_XINCREF(value); - // Py_XSETREF(flp->frame->f_localsplus[offset], value); + if (fast_ref == NULL) { + // No such local variable, let KeyError escape + return -1; + } + /* Key is a valid Python variable for the frame, so update that reference */ + if (PyCell_Check(fast_ref)) { + // Closure cells can be updated even after the frame terminates + return PyCell_Set(fast_ref, value); + } + PyFrameObject *f = flp->frame; + if (f->f_state == FRAME_CLEARED) { + PyErr_Format(PyExc_RuntimeError, + "Fast locals proxy attempted to write to cleared frame (%R)", f); + return -1; + } + /* Fast ref is a Python int mapping into the fast locals array */ + assert(PyLong_CheckExact(fast_ref)); + Py_ssize_t offset = PyLong_AsSsize_t(fast_ref); + if (offset < 0) { + return -1; + } + PyCodeObject *co = _PyFrame_GetCode(f); + assert(co); + Py_ssize_t max_offset = co->co_nlocalsplus - 1; + if (offset > max_offset) { + PyErr_Format(PyExc_SystemError, + "Fast locals ref (%zd) exceeds array bound (%zd)", + offset, max_offset); + return -1; + } + PyObject **fast_locals = f->f_localsptr; + // Check if MAKE_CELL has been called since the proxy was created + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, offset); + if (kind & CO_FAST_CELL) { + // Value hadn't been converted to a cell yet when the proxy was created + // Update the proxy if MAKE_CELL has run since the last access, + // otherwise continue treating it as a regular local variable + PyObject *target = fast_locals[offset]; + if (target != NULL && PyCell_Check(target) && + _PyFrame_OpAlreadyRan(f, MAKE_CELL, offset)) { + // MAKE_CELL has built the cell, so use it as the proxy target + Py_INCREF(target); + if (set_fast_ref(flp->fast_refs, key, target) != 0) { + return -1; } + return PyCell_Set(target, value); } } - return result; + + // Local variable, or future cell variable that hasn't been converted yet + Py_XINCREF(value); + Py_XSETREF(fast_locals[offset], value); + return 0; } static int fastlocalsproxy_setitem(fastlocalsproxyobject *flp, PyObject *key, PyObject *value) { - int result; - result = PyDict_SetItem(flp->mapping, key, value); - if (result == 0) { - result = fastlocalsproxy_write_to_frame(flp, key, value); - } - return result; + return fastlocalsproxy_write_to_frame(flp, key, value); } static int fastlocalsproxy_delitem(fastlocalsproxyobject *flp, PyObject *key) { - int result; - result = PyDict_DelItem(flp->mapping, key); - if (result == 0) { - result = fastlocalsproxy_write_to_frame(flp, key, NULL); - } - return result; + return fastlocalsproxy_write_to_frame(flp, key, NULL); } -static int -fastlocalsproxy_mp_assign_subscript(PyObject *flp, PyObject *key, PyObject *value) +static PyMappingMethods fastlocalsproxy_as_mapping = { + (lenfunc)fastlocalsproxy_len, /* mp_length */ + (binaryfunc)fastlocalsproxy_getitem, /* mp_subscript */ + (objobjargproc)fastlocalsproxy_setitem, /* mp_ass_subscript */ +}; + +static PyObject * +fastlocalsproxy_or(PyObject *Py_UNUSED(left), PyObject *Py_UNUSED(right)) { - if (value == NULL) - return fastlocalsproxy_delitem((fastlocalsproxyobject *)flp, key); - else - return fastlocalsproxy_setitem((fastlocalsproxyobject *)flp, key, value); + // Delegate to the other operand to determine the return type + Py_RETURN_NOTIMPLEMENTED; } -static PyMappingMethods fastlocalsproxy_as_mapping = { - 0, /* mp_length */ - 0, /* mp_subscript */ - fastlocalsproxy_mp_assign_subscript, /* mp_ass_subscript */ +static PyObject * +fastlocalsproxy_ior(PyObject *self, PyObject *Py_UNUSED(other)) +{ + // PEP 558 TODO: Support |= to update from arbitrary mappings + // Check the latest mutablemapping_update code for __ior__ support + PyErr_Format(PyExc_NotImplementedError, + "FastLocalsProxy does not yet implement __ior__"); + return NULL; +} + +static PyNumberMethods fastlocalsproxy_as_number = { + .nb_or = fastlocalsproxy_or, + .nb_inplace_or = fastlocalsproxy_ior, }; +static int +fastlocalsproxy_contains(fastlocalsproxyobject *flp, PyObject *key) +{ + return PyDict_Contains(flp->fast_refs, key); +} + +static PySequenceMethods fastlocalsproxy_as_sequence = { + 0, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + 0, /* sq_item */ + 0, /* sq_slice */ + 0, /* sq_ass_item */ + 0, /* sq_ass_slice */ + (objobjproc)fastlocalsproxy_contains, /* sq_contains */ + 0, /* sq_inplace_concat */ + 0, /* sq_inplace_repeat */ +}; + +static PyObject * +fastlocalsproxy_get(fastlocalsproxyobject *flp, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *key = NULL; + PyObject *failobj = Py_None; + + if (!_PyArg_UnpackStack(args, nargs, "get", 1, 2, + &key, &failobj)) + { + return NULL; + } + + PyObject *value = fastlocalsproxy_getitem(flp, key); + if (value == NULL && PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + value = failobj; + Py_INCREF(value); + } + return value; +} + +static PyObject * +fastlocalsproxy_keys(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) +{ + assert(flp); + if (fastlocalsproxy_init_fast_refs(flp) != 0) { + return NULL; + } + return PyDict_Keys(flp->fast_refs); +} + +static PyObject * +fastlocalsproxy_values(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) +{ + PyObject *locals = _PyFrame_BorrowLocals(flp->frame); + return PyDict_Values(locals); +} + +static PyObject * +fastlocalsproxy_items(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) +{ + PyObject *locals = _PyFrame_BorrowLocals(flp->frame); + return PyDict_Items(locals); +} + +static PyObject * +fastlocalsproxy_copy(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) +{ + PyObject *locals = _PyFrame_BorrowLocals(flp->frame); + return PyDict_Copy(locals); +} + +static PyObject * +fastlocalsproxy_reversed(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) +{ + _Py_IDENTIFIER(__reversed__); + assert(flp); + if (fastlocalsproxy_init_fast_refs(flp) != 0) { + return NULL; + } + return _PyObject_CallMethodIdNoArgs(flp->fast_refs, &PyId___reversed__); +} + +static PyObject * +fastlocalsproxy_getiter(fastlocalsproxyobject *flp) +{ + assert(flp); + if (fastlocalsproxy_init_fast_refs(flp) != 0) { + return NULL; + } + return PyObject_GetIter(flp->fast_refs); +} + +static PyObject * +fastlocalsproxy_richcompare(fastlocalsproxyobject *flp, PyObject *w, int op) +{ + PyObject *locals = _PyFrame_BorrowLocals(flp->frame); + return PyObject_RichCompare(locals, w, op); +} /* setdefault() */ @@ -1410,8 +1581,6 @@ PyDoc_STRVAR(fastlocalsproxy_setdefault__doc__, is not in the dictionary.\n\n\ Return the value for key if key is in the dictionary, else default."); - - static PyObject * fastlocalsproxy_setdefault(PyObject *flp, PyObject *args, PyObject *kwargs) { @@ -1435,65 +1604,52 @@ fastlocalsproxy_setdefault(PyObject *flp, PyObject *args, PyObject *kwargs) /* pop() */ PyDoc_STRVAR(fastlocalsproxy_pop__doc__, -"flp.pop(k[,d]) -> v, remove specified key and return the corresponding\n\ +"flp.pop(k[,d]) -> v, unbind specified variable and return the corresponding\n\ value. If key is not found, d is returned if given, otherwise KeyError\n\ is raised."); -/* forward */ -static PyObject * _fastlocalsproxy_popkey(PyObject *, PyObject *, PyObject *); - static PyObject * -fastlocalsproxy_pop(PyObject *flp, PyObject *args, PyObject *kwargs) -{ - static char *kwlist[] = {"key", "default", 0}; - PyObject *key, *failobj = NULL; - - /* borrowed */ - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O:pop", kwlist, - &key, &failobj)) { - return NULL; - } - - return _fastlocalsproxy_popkey(flp, key, failobj); -} - -static PyObject * -_fastlocalsproxy_popkey(PyObject *flp, PyObject *key, PyObject *failobj) +_fastlocalsproxy_popkey(fastlocalsproxyobject *flp, PyObject *key, PyObject *failobj) { // TODO: Similar to the odict implementation, the fast locals proxy // could benefit from an internal API that accepts already calculated // hashes, rather than recalculating the hash multiple times for the // same key in a single operation (see _odict_popkey_hash) - - PyObject *value = NULL; - - // Just implement naive lookup through the abstract C API for now - int exists = PySequence_Contains(flp, key); - if (exists < 0) + assert(flp); + if (fastlocalsproxy_init_fast_refs(flp) != 0) { return NULL; - if (exists) { - value = PyObject_GetItem(flp, key); - if (value != NULL) { - if (PyObject_DelItem(flp, key) == -1) { - Py_CLEAR(value); - } - } } - /* Apply the fallback value, if necessary. */ - if (value == NULL && !PyErr_Occurred()) { - if (failobj) { - value = failobj; - Py_INCREF(failobj); - } - else { - PyErr_SetObject(PyExc_KeyError, key); + // Just implement naive lookup through the object based C API for now + PyObject *value = fastlocalsproxy_getitem(flp, key); + if (value != NULL) { + if (fastlocalsproxy_delitem(flp, key) != 0) { + Py_CLEAR(value); } + } else if (failobj != NULL && PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + value = failobj; + Py_INCREF(value); } return value; } +static PyObject * +fastlocalsproxy_pop(PyObject *flp, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"key", "default", 0}; + PyObject *key, *failobj = NULL; + + /* borrowed */ + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O:pop", kwlist, + &key, &failobj)) { + return NULL; + } + + return _fastlocalsproxy_popkey((fastlocalsproxyobject *)flp, key, failobj); +} + /* popitem() */ PyDoc_STRVAR(fastlocalsproxy_popitem__doc__, @@ -1521,7 +1677,7 @@ static PyObject * mutablemapping_update(PyObject *, PyObject *, PyObject *); /* clear() */ PyDoc_STRVAR(fastlocalsproxy_clear__doc__, - "flp.clear() -> None. Remove all items from snapshot and frame."); + "flp.clear() -> None. Unbind all variables in frame."); static PyObject * fastlocalsproxy_clear(register PyObject *flp, PyObject *Py_UNUSED(ignored)) @@ -1532,11 +1688,26 @@ fastlocalsproxy_clear(register PyObject *flp, PyObject *Py_UNUSED(ignored)) } static PyMethodDef fastlocalsproxy_methods[] = { + {"get", (PyCFunction)(void(*)(void))fastlocalsproxy_get, METH_FASTCALL, + PyDoc_STR("D.get(k[,d]) -> D[k] if k in D, else d." + " d defaults to None.")}, + {"keys", (PyCFunction)fastlocalsproxy_keys, METH_NOARGS, + PyDoc_STR("D.keys() -> virtual set of D's keys")}, + {"values", (PyCFunction)fastlocalsproxy_values, METH_NOARGS, + PyDoc_STR("D.values() -> virtual multiset of D's values")}, + {"items", (PyCFunction)fastlocalsproxy_items, METH_NOARGS, + PyDoc_STR("D.items() -> virtual set of D's (key, value) pairs, as 2-tuples")}, + {"copy", (PyCFunction)fastlocalsproxy_copy, METH_NOARGS, + PyDoc_STR("D.copy() -> a shallow copy of D as a regular dict")}, + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("See PEP 585")}, + {"__reversed__", (PyCFunction)fastlocalsproxy_reversed, METH_NOARGS, + PyDoc_STR("D.__reversed__() -> reverse iterator over D's keys")}, {"setdefault", (PyCFunction)(void(*)(void))fastlocalsproxy_setdefault, METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_setdefault__doc__}, {"pop", (PyCFunction)(void(*)(void))fastlocalsproxy_pop, METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_pop__doc__}, - {"clear", (PyCFunction)fastlocalsproxy_popitem, + {"popitem", (PyCFunction)fastlocalsproxy_popitem, METH_NOARGS, fastlocalsproxy_popitem__doc__}, {"update", (PyCFunction)(void(*)(void))fastlocalsproxy_update, METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_update__doc__}, @@ -1554,7 +1725,6 @@ fastlocalsproxy_dealloc(fastlocalsproxyobject *flp) Py_TRASHCAN_SAFE_BEGIN(flp) - Py_CLEAR(flp->mapping); Py_CLEAR(flp->frame); Py_CLEAR(flp->fast_refs); PyObject_GC_Del(flp); @@ -1565,14 +1735,13 @@ fastlocalsproxy_dealloc(fastlocalsproxyobject *flp) static PyObject * fastlocalsproxy_repr(fastlocalsproxyobject *flp) { - return PyUnicode_FromFormat("fastlocalsproxy(%R)", flp->mapping); + return PyUnicode_FromFormat("fastlocalsproxy(%R)", flp->frame); } static int fastlocalsproxy_traverse(PyObject *self, visitproc visit, void *arg) { fastlocalsproxyobject *flp = (fastlocalsproxyobject *)self; - Py_VISIT(flp->mapping); Py_VISIT(flp->frame); Py_VISIT(flp->fast_refs); return 0; @@ -1605,7 +1774,6 @@ static PyObject * _PyFastLocalsProxy_New(PyObject *frame) { fastlocalsproxyobject *flp; - PyObject *mapping, *fast_refs; if (fastlocalsproxy_check_frame(frame) == -1) { return NULL; @@ -1614,21 +1782,8 @@ _PyFastLocalsProxy_New(PyObject *frame) flp = PyObject_GC_New(fastlocalsproxyobject, &_PyFastLocalsProxy_Type); if (flp == NULL) return NULL; - mapping = _frame_get_updated_locals((PyFrameObject *) frame); - if (mapping == NULL) { - Py_DECREF(flp); - return NULL; - } - flp->mapping = mapping; - Py_INCREF(flp->mapping); flp->frame = (PyFrameObject *) frame; Py_INCREF(flp->frame); - fast_refs = _PyFrame_BuildFastRefs(flp->frame); - if (fast_refs == NULL) { - Py_DECREF(flp); // Also handles DECREF for mapping and frame - return NULL; - } - flp->fast_refs = fast_refs; _PyObject_GC_TRACK(flp); return (PyObject *)flp; } @@ -1661,27 +1816,29 @@ PyTypeObject _PyFastLocalsProxy_Type = { 0, /* tp_setattr */ 0, /* tp_reserved */ (reprfunc)fastlocalsproxy_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ + &fastlocalsproxy_as_number, /* tp_as_number */ + &fastlocalsproxy_as_sequence, /* tp_as_sequence */ &fastlocalsproxy_as_mapping, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ - 0, /* tp_getattro */ + PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + Py_TPFLAGS_DEFAULT | + Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_MAPPING, /* tp_flags */ 0, /* tp_doc */ (traverseproc)fastlocalsproxy_traverse, /* tp_traverse */ 0, /* tp_clear */ - 0, /* tp_richcompare */ + (richcmpfunc)fastlocalsproxy_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ + (getiterfunc)fastlocalsproxy_getiter, /* tp_iter */ 0, /* tp_iternext */ fastlocalsproxy_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ - &PyDictProxy_Type, /* tp_base */ + 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ @@ -1693,12 +1850,11 @@ PyTypeObject _PyFastLocalsProxy_Type = { }; - //========================================================================== // The rest of this file is currently DUPLICATED CODE from odictobject.c // -// TODO: move the duplicated code to abstract.c and expose it to the -// linker as a private API +// PEP 558 TODO: move the duplicated code to Objects/mutablemapping.c and +// expose it to the linker as a private API // //========================================================================== diff --git a/Python/ceval.c b/Python/ceval.c index 9fd1fb59e4b428..30871bd9b7615b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2923,7 +2923,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) case TARGET(MAKE_CELL): { // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). + // via a frame locals proxy before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { @@ -3506,7 +3506,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) case TARGET(IMPORT_STAR): { PyObject *from = POP(), *locals; int err; - /* TODO for PEP 558 + /* PEP 558 TODO: * Report an error here for CO_OPTIMIZED frames * The 3.x compiler treats wildcard imports as an error inside * functions, but they can still happen with independently diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 414984c30860c0..35a17c7f52aca4 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -950,14 +950,6 @@ static PyObject * call_trampoline(PyThreadState *tstate, PyObject* callback, PyFrameObject *frame, int what, PyObject *arg) { - // Implicitly refresh frame namespace snapshot stored in f_locals, - // as even though the Python level f_locals descriptor now also - // refreshs the snapshot, trace functions may be calling other C APIs - // that expect the snapshot to have already been refreshed - if (PyFrame_FastToLocalsWithError(frame) < 0) { - return NULL; - } - PyObject *stack[3]; stack[0] = (PyObject *)frame; From 7ec5d26622088840240f20276016f15d05a16050 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 10 Jul 2021 20:47:22 +1000 Subject: [PATCH 38/66] Register new stable ABI additions --- Include/ceval.h | 4 ---- Misc/stable_abi.txt | 16 ++++++++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Include/ceval.h b/Include/ceval.h index 8f062e656643c8..9a0aab61c5cd90 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -54,10 +54,6 @@ PyAPI_FUNC(PyObject *) PyLocals_GetCopy(void); // TODO: Implement this, and add API tests PyAPI_FUNC(PyObject *) PyLocals_GetView(void); -/* PyLocals_RefreshViews() updates previously created locals views */ -// TODO: Implement this, and add API tests -PyAPI_FUNC(int) PyLocals_RefreshViews(void); - /* Returns true if PyLocals_Get() returns a shallow copy in the active scope */ // TODO: Implement this, and add API tests PyAPI_FUNC(int) PyLocals_GetReturnsCopy(void); diff --git a/Misc/stable_abi.txt b/Misc/stable_abi.txt index f104f84e451da1..0f981de1d367fb 100644 --- a/Misc/stable_abi.txt +++ b/Misc/stable_abi.txt @@ -2133,5 +2133,17 @@ function PyGC_IsEnabled added 3.10 -# (Detailed comments aren't really needed for further entries: from here on -# we can use version control logs.) +# While detailed comments aren't technically needed for further entries +# (we can use version control logs for this file), they're still helpful +# when the full changelog isn't readily available + +# PEP 558: New locals() access functions + +function PyLocals_Get + added 3.11 +function PyLocals_GetCopy + added 3.11 +function PyLocals_GetReturnsCopy + added 3.11 +function PyLocals_GetView + added 3.11 From 7ddc3ebaf15f024e5ecef83c0ce6265b98bc7f1a Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 10 Jul 2021 22:02:17 +1000 Subject: [PATCH 39/66] Add FLP str(), fix various value lookup issues --- Doc/c-api/cell.rst | 9 ++-- Objects/frameobject.c | 97 ++++++++++++++++++++++++------------------- 2 files changed, 60 insertions(+), 46 deletions(-) diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index ac4ef5adc5cc20..69e2723f2d2184 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -39,13 +39,14 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: PyObject* PyCell_Get(PyObject *cell) - Return the contents of the cell *cell*. + Return a new reference to the contents of the cell *cell*. .. c:function:: PyObject* PyCell_GET(PyObject *cell) - Return the contents of the cell *cell*, but without checking that *cell* is - non-``NULL`` and a cell object. + Borrow a reference to the contents of the cell *cell*. No reference counts are + adjusted, and no checks are made for safety; *cell* must be non-``NULL`` and must + be a cell object. .. c:function:: int PyCell_Set(PyObject *cell, PyObject *value) @@ -58,6 +59,6 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: void PyCell_SET(PyObject *cell, PyObject *value) - Sets the value of the cell object *cell* to *value*. No reference counts are + Sets the value of the cell object *cell* to *value*. No reference counts are adjusted, and no checks are made for safety; *cell* must be non-``NULL`` and must be a cell object. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index e0d11a969a8e9c..f770b0fb40fa96 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1308,54 +1308,56 @@ fastlocalsproxy_getitem(fastlocalsproxyobject *flp, PyObject *key) if (fastlocalsproxy_init_fast_refs(flp) != 0) { return NULL; } - PyObject *fast_ref = PyDict_GetItem(flp->fast_refs, key); + PyObject *fast_ref = PyObject_GetItem(flp->fast_refs, key); if (fast_ref == NULL) { // No such local variable, let KeyError escape return NULL; } /* Key is a valid Python variable for the frame, so retrieve the value */ + PyObject *value = NULL; if (PyCell_Check(fast_ref)) { // Closure cells can be queried even after the frame terminates - return PyCell_Get(fast_ref); - } - PyFrameObject *f = flp->frame; - if (f->f_state == FRAME_CLEARED) { - PyErr_Format(PyExc_RuntimeError, - "Fast locals proxy attempted to read from cleared frame (%R)", f); - return NULL; - } - /* Fast ref is a Python int mapping into the fast locals array */ - assert(PyLong_CheckExact(fast_ref)); - Py_ssize_t offset = PyLong_AsSsize_t(fast_ref); - if (offset < 0) { - return NULL; - } - PyCodeObject *co = _PyFrame_GetCode(f); - assert(co); - Py_ssize_t max_offset = co->co_nlocalsplus - 1; - if (offset > max_offset) { - PyErr_Format(PyExc_SystemError, - "Fast locals ref (%zd) exceeds array bound (%zd)", - offset, max_offset); - return NULL; - } - PyObject **fast_locals = f->f_localsptr; - PyObject *value = fast_locals[offset]; - // Check if MAKE_CELL has been called since the proxy was created - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, offset); - if (kind & CO_FAST_CELL) { - // Value hadn't been converted to a cell yet when the proxy was created - // Update the proxy if MAKE_CELL has run since the last access, - // otherwise continue treating it as a regular local variable - PyObject *target = value; - if (target != NULL && PyCell_Check(target) && - _PyFrame_OpAlreadyRan(f, MAKE_CELL, offset)) { - // MAKE_CELL has built the cell, so use it as the proxy target - Py_INCREF(target); - if (set_fast_ref(flp->fast_refs, key, target) != 0) { - return NULL; + value = PyCell_GET(fast_ref); + } else { + PyFrameObject *f = flp->frame; + if (f->f_state == FRAME_CLEARED) { + PyErr_Format(PyExc_RuntimeError, + "Fast locals proxy attempted to read from cleared frame (%R)", f); + return NULL; + } + /* Fast ref is a Python int mapping into the fast locals array */ + assert(PyLong_CheckExact(fast_ref)); + Py_ssize_t offset = PyLong_AsSsize_t(fast_ref); + if (offset < 0) { + return NULL; + } + PyCodeObject *co = _PyFrame_GetCode(f); + assert(co); + Py_ssize_t max_offset = co->co_nlocalsplus - 1; + if (offset > max_offset) { + PyErr_Format(PyExc_SystemError, + "Fast locals ref (%zd) exceeds array bound (%zd)", + offset, max_offset); + return NULL; + } + PyObject **fast_locals = f->f_localsptr; + value = fast_locals[offset]; + // Check if MAKE_CELL has been called since the proxy was created + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, offset); + if (kind & CO_FAST_CELL) { + // Value hadn't been converted to a cell yet when the proxy was created + // Update the proxy if MAKE_CELL has run since the last access, + // otherwise continue treating it as a regular local variable + PyObject *target = value; + if (target != NULL && PyCell_Check(target) && + _PyFrame_OpAlreadyRan(f, MAKE_CELL, offset)) { + // MAKE_CELL has built the cell, so use it as the proxy target + Py_INCREF(target); + if (set_fast_ref(flp->fast_refs, key, target) != 0) { + return NULL; + } + value = PyCell_GET(target); } - return PyCell_Get(target); } } @@ -1379,7 +1381,7 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje if (fastlocalsproxy_init_fast_refs(flp) != 0) { return -1; } - PyObject *fast_ref = PyDict_GetItem(flp->fast_refs, key); + PyObject *fast_ref = PyObject_GetItem(flp->fast_refs, key); if (fast_ref == NULL) { // No such local variable, let KeyError escape return -1; @@ -1595,6 +1597,7 @@ fastlocalsproxy_setdefault(PyObject *flp, PyObject *args, PyObject *kwargs) PyObject *value = NULL; + // PEP 558 TODO: implement this PyErr_Format(PyExc_NotImplementedError, "FastLocalsProxy does not yet implement setdefault()"); return value; @@ -1659,6 +1662,7 @@ PyDoc_STRVAR(fastlocalsproxy_popitem__doc__, static PyObject * fastlocalsproxy_popitem(PyObject *flp, PyObject *Py_UNUSED(ignored)) { + // PEP 558 TODO: implement this PyErr_Format(PyExc_NotImplementedError, "FastLocalsProxy does not yet implement popitem()"); return NULL; @@ -1682,6 +1686,7 @@ PyDoc_STRVAR(fastlocalsproxy_clear__doc__, static PyObject * fastlocalsproxy_clear(register PyObject *flp, PyObject *Py_UNUSED(ignored)) { + // PEP 558 TODO: implement this PyErr_Format(PyExc_NotImplementedError, "FastLocalsProxy does not yet implement clear()"); return NULL; @@ -1703,6 +1708,7 @@ static PyMethodDef fastlocalsproxy_methods[] = { PyDoc_STR("See PEP 585")}, {"__reversed__", (PyCFunction)fastlocalsproxy_reversed, METH_NOARGS, PyDoc_STR("D.__reversed__() -> reverse iterator over D's keys")}, + // PEP 558 TODO: Convert these methods to METH_FASTCALL {"setdefault", (PyCFunction)(void(*)(void))fastlocalsproxy_setdefault, METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_setdefault__doc__}, {"pop", (PyCFunction)(void(*)(void))fastlocalsproxy_pop, @@ -1738,6 +1744,13 @@ fastlocalsproxy_repr(fastlocalsproxyobject *flp) return PyUnicode_FromFormat("fastlocalsproxy(%R)", flp->frame); } +static PyObject * +fastlocalsproxy_str(fastlocalsproxyobject *flp) +{ + PyObject *locals = _PyFrame_BorrowLocals(flp->frame); + return PyObject_Str(locals); +} + static int fastlocalsproxy_traverse(PyObject *self, visitproc visit, void *arg) { @@ -1821,7 +1834,7 @@ PyTypeObject _PyFastLocalsProxy_Type = { &fastlocalsproxy_as_mapping, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ - 0, /* tp_str */ + (reprfunc)fastlocalsproxy_str, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ From 7400a4608e3704c9e56f6b6c10cec3da15b05ee5 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 10 Jul 2021 22:12:54 +1000 Subject: [PATCH 40/66] Uninitialised fields will get you every time --- Objects/frameobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index f770b0fb40fa96..12985af64a24b2 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1797,6 +1797,7 @@ _PyFastLocalsProxy_New(PyObject *frame) return NULL; flp->frame = (PyFrameObject *) frame; Py_INCREF(flp->frame); + flp->fast_refs = NULL; _PyObject_GC_TRACK(flp); return (PyObject *)flp; } From 5eae0d52fd743c8e28613346fc76f0a895f1b26f Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 10 Jul 2021 22:50:18 +1000 Subject: [PATCH 41/66] Fix refcounting, bdb segfault, pdb functionality --- Objects/frameobject.c | 54 +++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 12985af64a24b2..828adfd57ba01c 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -29,6 +29,15 @@ get_frame_state(void) return &interp->frame; } +static PyObject * +_frame_get_locals_mapping(PyFrameObject *f) +{ + PyObject *locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET]; + if (locals == NULL) { + locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET] = PyDict_New(); + } + return locals; +} static PyObject * _frame_get_updated_locals(PyFrameObject *f) @@ -37,7 +46,7 @@ _frame_get_updated_locals(PyFrameObject *f) return NULL; PyObject *locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET]; assert(locals != NULL); - Py_INCREF(locals); + // Internal borrowed reference, caller increfs for external sharing return locals; } @@ -1001,12 +1010,9 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f) PyErr_BadInternalCall(); return -1; } - locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET]; + locals = _frame_get_locals_mapping(f); if (locals == NULL) { - locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET] = PyDict_New(); - if (locals == NULL) { - return -1; - } + return -1; } co = _PyFrame_GetCode(f); fast = f->f_localsptr; @@ -1308,10 +1314,15 @@ fastlocalsproxy_getitem(fastlocalsproxyobject *flp, PyObject *key) if (fastlocalsproxy_init_fast_refs(flp) != 0) { return NULL; } - PyObject *fast_ref = PyObject_GetItem(flp->fast_refs, key); + PyObject *fast_ref = PyDict_GetItem(flp->fast_refs, key); if (fast_ref == NULL) { - // No such local variable, let KeyError escape - return NULL; + // No such local variable, delegate the request to the f_locals mapping + // Used by pdb (at least) to access __return__ and __exception__ values + PyObject *locals = _frame_get_locals_mapping(flp->frame); + if (locals == NULL) { + return NULL; + } + return PyObject_GetItem(locals, key); } /* Key is a valid Python variable for the frame, so retrieve the value */ PyObject *value = NULL; @@ -1381,10 +1392,15 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje if (fastlocalsproxy_init_fast_refs(flp) != 0) { return -1; } - PyObject *fast_ref = PyObject_GetItem(flp->fast_refs, key); + PyObject *fast_ref = PyDict_GetItem(flp->fast_refs, key); if (fast_ref == NULL) { - // No such local variable, let KeyError escape - return -1; + // No such local variable, delegate the request to the f_locals mapping + // Used by pdb (at least) to store __return__ and __exception__ values + PyObject *locals = _frame_get_locals_mapping(flp->frame); + if (locals == NULL) { + return -1; + } + return PyObject_SetItem(locals, key, value); } /* Key is a valid Python variable for the frame, so update that reference */ if (PyCell_Check(fast_ref)) { @@ -1480,6 +1496,10 @@ static PyNumberMethods fastlocalsproxy_as_number = { static int fastlocalsproxy_contains(fastlocalsproxyobject *flp, PyObject *key) { + assert(flp); + if (fastlocalsproxy_init_fast_refs(flp) != 0) { + return -1; + } return PyDict_Contains(flp->fast_refs, key); } @@ -1530,21 +1550,21 @@ fastlocalsproxy_keys(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) static PyObject * fastlocalsproxy_values(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { - PyObject *locals = _PyFrame_BorrowLocals(flp->frame); + PyObject *locals = _frame_get_updated_locals(flp->frame); return PyDict_Values(locals); } static PyObject * fastlocalsproxy_items(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { - PyObject *locals = _PyFrame_BorrowLocals(flp->frame); + PyObject *locals = _frame_get_updated_locals(flp->frame); return PyDict_Items(locals); } static PyObject * fastlocalsproxy_copy(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { - PyObject *locals = _PyFrame_BorrowLocals(flp->frame); + PyObject *locals = _frame_get_updated_locals(flp->frame); return PyDict_Copy(locals); } @@ -1572,7 +1592,7 @@ fastlocalsproxy_getiter(fastlocalsproxyobject *flp) static PyObject * fastlocalsproxy_richcompare(fastlocalsproxyobject *flp, PyObject *w, int op) { - PyObject *locals = _PyFrame_BorrowLocals(flp->frame); + PyObject *locals = _frame_get_updated_locals(flp->frame); return PyObject_RichCompare(locals, w, op); } @@ -1747,7 +1767,7 @@ fastlocalsproxy_repr(fastlocalsproxyobject *flp) static PyObject * fastlocalsproxy_str(fastlocalsproxyobject *flp) { - PyObject *locals = _PyFrame_BorrowLocals(flp->frame); + PyObject *locals = _frame_get_updated_locals(flp->frame); return PyObject_Str(locals); } From 40db4e7d387261aaa9037b03546ece89401d5ee8 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 10 Jul 2021 23:24:50 +1000 Subject: [PATCH 42/66] Delegate more operations to the full dynamic snapshot --- Objects/frameobject.c | 70 ++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 828adfd57ba01c..66ee518173bd3e 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1299,11 +1299,27 @@ fastlocalsproxy_init_fast_refs(fastlocalsproxyobject *flp) static Py_ssize_t fastlocalsproxy_len(fastlocalsproxyobject *flp) { - assert(flp); - if (fastlocalsproxy_init_fast_refs(flp) != 0) { - return -1; - } - return PyObject_Size(flp->fast_refs); + // PEP 558 TODO: This is decidedly NOT an O(1) operation at the moment + // The challenge is that it's the only obviously correct implementation: + // * the fast_refs mapping includes not-yet-bound locals and omits extra + // keys added directly to the f_locals dict (e.g. by pdb) + // * the f_locals dict includes the extra keys, and omits the not yet + // bound locals, but is no longer implicitly updated on every instruction + // while tracing, so might be out of date + // * it might be possible to track the last executed instruction on the + // frame and only update when that changes, but doing that would miss + // any extra attributes injected into f_locals via either other proxies + // or PyEval_GetLocals() + // + // One potentially viable caching approach might be to skip the update if + // the frame execution hadn't advanced *and* ma_version_tag hadn't changed + // on the locals snapshot + + // Extra keys may have been added, and some variables may not have been + // bound yet, so use the dynamic snapshot on the frame rather than the + // keys in the fast locals reverse lookup mapping + PyObject *locals = _frame_get_updated_locals(flp->frame); + return PyObject_Size(locals); } static PyObject * @@ -1500,7 +1516,16 @@ fastlocalsproxy_contains(fastlocalsproxyobject *flp, PyObject *key) if (fastlocalsproxy_init_fast_refs(flp) != 0) { return -1; } - return PyDict_Contains(flp->fast_refs, key); + int result = PyDict_Contains(flp->fast_refs, key); + if (result) { + return result; + } + // Extra keys may have been stored directly in the frame locals + PyObject *locals = _frame_get_locals_mapping(flp->frame); + if (locals == NULL) { + return -1; + } + return PyDict_Contains(locals, key); } static PySequenceMethods fastlocalsproxy_as_sequence = { @@ -1540,16 +1565,17 @@ fastlocalsproxy_get(fastlocalsproxyobject *flp, PyObject *const *args, Py_ssize_ static PyObject * fastlocalsproxy_keys(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { - assert(flp); - if (fastlocalsproxy_init_fast_refs(flp) != 0) { - return NULL; - } - return PyDict_Keys(flp->fast_refs); + // Extra keys may have been added, and some variables may not have been + // bound yet, so use the dynamic snapshot on the frame rather than the + // keys in the fast locals reverse lookup mapping + PyObject *locals = _frame_get_updated_locals(flp->frame); + return PyDict_Keys(locals); } static PyObject * fastlocalsproxy_values(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { + // Need values, so use the dynamic snapshot on the frame PyObject *locals = _frame_get_updated_locals(flp->frame); return PyDict_Values(locals); } @@ -1557,6 +1583,7 @@ fastlocalsproxy_values(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) static PyObject * fastlocalsproxy_items(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { + // Need values, so use the dynamic snapshot on the frame PyObject *locals = _frame_get_updated_locals(flp->frame); return PyDict_Items(locals); } @@ -1564,6 +1591,7 @@ fastlocalsproxy_items(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) static PyObject * fastlocalsproxy_copy(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { + // Need values, so use the dynamic snapshot on the frame PyObject *locals = _frame_get_updated_locals(flp->frame); return PyDict_Copy(locals); } @@ -1572,21 +1600,21 @@ static PyObject * fastlocalsproxy_reversed(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { _Py_IDENTIFIER(__reversed__); - assert(flp); - if (fastlocalsproxy_init_fast_refs(flp) != 0) { - return NULL; - } - return _PyObject_CallMethodIdNoArgs(flp->fast_refs, &PyId___reversed__); + // Extra keys may have been added, and some variables may not have been + // bound yet, so use the dynamic snapshot on the frame rather than the + // keys in the fast locals reverse lookup mapping + PyObject *locals = _frame_get_updated_locals(flp->frame); + return _PyObject_CallMethodIdNoArgs(locals, &PyId___reversed__); } static PyObject * fastlocalsproxy_getiter(fastlocalsproxyobject *flp) { - assert(flp); - if (fastlocalsproxy_init_fast_refs(flp) != 0) { - return NULL; - } - return PyObject_GetIter(flp->fast_refs); + // Extra keys may have been added, and some variables may not have been + // bound yet, so use the dynamic snapshot on the frame rather than the + // keys in the fast locals reverse lookup mapping + PyObject *locals = _frame_get_updated_locals(flp->frame); + return PyObject_GetIter(locals); } static PyObject * From 74b97a3ef39a5e67c019858f8d60a033401dee64 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 11 Jul 2021 00:20:06 +1000 Subject: [PATCH 43/66] Add TODO item for false positives in containment checks --- Objects/frameobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 66ee518173bd3e..ed6b0f6c15528b 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1518,6 +1518,7 @@ fastlocalsproxy_contains(fastlocalsproxyobject *flp, PyObject *key) } int result = PyDict_Contains(flp->fast_refs, key); if (result) { + // PEP 558 TODO: This should return false if the name hasn't been bound yet return result; } // Extra keys may have been stored directly in the frame locals From 9f16513a919c3d2ad7e6ead6298d0cb590caf1a3 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 11 Jul 2021 00:43:52 +1000 Subject: [PATCH 44/66] Only ensure frame snapshot is up to date in O(n) proxy operations --- Objects/frameobject.c | 61 +++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index ed6b0f6c15528b..d654d8aff2acab 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -90,6 +90,9 @@ frame_getlocals(PyFrameObject *f, void *Py_UNUSED(ignored)) * layer is a new fastlocalsproxy instance, while f_locals at the C * layer still refers to the underlying shared namespace mapping. */ + if (PyFrame_FastToLocalsWithError(f) < 0) { + return NULL; + } f_locals_attr = _PyFastLocalsProxy_New((PyObject *) f); } else { // Share a direct locals reference for class and module scopes @@ -1299,26 +1302,11 @@ fastlocalsproxy_init_fast_refs(fastlocalsproxyobject *flp) static Py_ssize_t fastlocalsproxy_len(fastlocalsproxyobject *flp) { - // PEP 558 TODO: This is decidedly NOT an O(1) operation at the moment - // The challenge is that it's the only obviously correct implementation: - // * the fast_refs mapping includes not-yet-bound locals and omits extra - // keys added directly to the f_locals dict (e.g. by pdb) - // * the f_locals dict includes the extra keys, and omits the not yet - // bound locals, but is no longer implicitly updated on every instruction - // while tracing, so might be out of date - // * it might be possible to track the last executed instruction on the - // frame and only update when that changes, but doing that would miss - // any extra attributes injected into f_locals via either other proxies - // or PyEval_GetLocals() - // - // One potentially viable caching approach might be to skip the update if - // the frame execution hadn't advanced *and* ma_version_tag hadn't changed - // on the locals snapshot - // Extra keys may have been added, and some variables may not have been // bound yet, so use the dynamic snapshot on the frame rather than the // keys in the fast locals reverse lookup mapping - PyObject *locals = _frame_get_updated_locals(flp->frame); + // Assume f_locals snapshot is up to date (as actually checking is O(n)) + PyObject *locals = _frame_get_locals_mapping(flp->frame); return PyObject_Size(locals); } @@ -1569,7 +1557,8 @@ fastlocalsproxy_keys(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) // Extra keys may have been added, and some variables may not have been // bound yet, so use the dynamic snapshot on the frame rather than the // keys in the fast locals reverse lookup mapping - PyObject *locals = _frame_get_updated_locals(flp->frame); + // Assume f_locals snapshot is up to date (as actually checking is O(n)) + PyObject *locals = _frame_get_locals_mapping(flp->frame); return PyDict_Keys(locals); } @@ -1577,7 +1566,8 @@ static PyObject * fastlocalsproxy_values(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { // Need values, so use the dynamic snapshot on the frame - PyObject *locals = _frame_get_updated_locals(flp->frame); + // Assume f_locals snapshot is up to date (as actually checking is O(n)) + PyObject *locals = _frame_get_locals_mapping(flp->frame); return PyDict_Values(locals); } @@ -1585,7 +1575,8 @@ static PyObject * fastlocalsproxy_items(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { // Need values, so use the dynamic snapshot on the frame - PyObject *locals = _frame_get_updated_locals(flp->frame); + // Assume f_locals snapshot is up to date (as actually checking is O(n)) + PyObject *locals = _frame_get_locals_mapping(flp->frame); return PyDict_Items(locals); } @@ -1593,6 +1584,7 @@ static PyObject * fastlocalsproxy_copy(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { // Need values, so use the dynamic snapshot on the frame + // Ensure it is up to date, as checking is O(n) anyway PyObject *locals = _frame_get_updated_locals(flp->frame); return PyDict_Copy(locals); } @@ -1604,7 +1596,8 @@ fastlocalsproxy_reversed(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored // Extra keys may have been added, and some variables may not have been // bound yet, so use the dynamic snapshot on the frame rather than the // keys in the fast locals reverse lookup mapping - PyObject *locals = _frame_get_updated_locals(flp->frame); + // Assume f_locals snapshot is up to date (as actually checking is O(n)) + PyObject *locals = _frame_get_locals_mapping(flp->frame); return _PyObject_CallMethodIdNoArgs(locals, &PyId___reversed__); } @@ -1614,13 +1607,16 @@ fastlocalsproxy_getiter(fastlocalsproxyobject *flp) // Extra keys may have been added, and some variables may not have been // bound yet, so use the dynamic snapshot on the frame rather than the // keys in the fast locals reverse lookup mapping - PyObject *locals = _frame_get_updated_locals(flp->frame); + // Assume f_locals snapshot is up to date (as actually checking is O(n)) + PyObject *locals = _frame_get_locals_mapping(flp->frame); return PyObject_GetIter(locals); } static PyObject * fastlocalsproxy_richcompare(fastlocalsproxyobject *flp, PyObject *w, int op) { + // Need values, so use the dynamic snapshot on the frame + // Ensure it is up to date, as checking is O(n) anyway PyObject *locals = _frame_get_updated_locals(flp->frame); return PyObject_RichCompare(locals, w, op); } @@ -1741,6 +1737,22 @@ fastlocalsproxy_clear(register PyObject *flp, PyObject *Py_UNUSED(ignored)) return NULL; } + +PyDoc_STRVAR(fastlocalsproxy_sync__doc__, + "flp.sync() -> None. Ensure f_locals state is in sync with underlying frame."); + +static PyObject * +fastlocalsproxy_sync(register PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + fastlocalsproxyobject *flp = (fastlocalsproxyobject *)self; + if (PyFrame_FastToLocalsWithError(flp->frame) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + + static PyMethodDef fastlocalsproxy_methods[] = { {"get", (PyCFunction)(void(*)(void))fastlocalsproxy_get, METH_FASTCALL, PyDoc_STR("D.get(k[,d]) -> D[k] if k in D, else d." @@ -1768,7 +1780,8 @@ static PyMethodDef fastlocalsproxy_methods[] = { METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_update__doc__}, {"clear", (PyCFunction)fastlocalsproxy_clear, METH_NOARGS, fastlocalsproxy_clear__doc__}, - + {"sync", (PyCFunction)fastlocalsproxy_clear, + METH_NOARGS, fastlocalsproxy_clear__doc__}, {NULL, NULL} /* sentinel */ }; @@ -1796,6 +1809,8 @@ fastlocalsproxy_repr(fastlocalsproxyobject *flp) static PyObject * fastlocalsproxy_str(fastlocalsproxyobject *flp) { + // Need values, so use the dynamic snapshot on the frame + // Ensure it is up to date, as checking is O(n) anyway PyObject *locals = _frame_get_updated_locals(flp->frame); return PyObject_Str(locals); } From c477e2432c2d8c6c938340c230da7da520b31d8b Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 11 Jul 2021 17:34:35 +1000 Subject: [PATCH 45/66] Keep locals snapshot up to date when reading/writing individual keys --- Objects/frameobject.c | 68 +++++++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d654d8aff2acab..5bba9fc7e50df3 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1310,6 +1310,29 @@ fastlocalsproxy_len(fastlocalsproxyobject *flp) return PyObject_Size(locals); } +static int +fastlocalsproxy_set_snapshot_entry(fastlocalsproxyobject *flp, PyObject *key, PyObject *value) +{ + PyFrameObject *f = flp->frame; + if (f->f_state == FRAME_CLEARED) { + // Don't touch the locals cache on already cleared frames + return 0; + } + PyObject *locals = _frame_get_locals_mapping(f); + if (locals == NULL) { + return -1; + } + if (value == NULL) { + // Ensure key is absent from cache (deleting if necessary) + if (PyDict_Contains(locals, key)) { + return PyObject_DelItem(locals, key); + } + return 0; + } + // Set cached value for the given key + return PyObject_SetItem(locals, key, value); +} + static PyObject * fastlocalsproxy_getitem(fastlocalsproxyobject *flp, PyObject *key) { @@ -1376,7 +1399,14 @@ fastlocalsproxy_getitem(fastlocalsproxyobject *flp, PyObject *key) } } - // Local variable, or future cell variable that hasn't been converted yet + // Local variable, or cell variable that either hasn't been converted yet + // or was only just converted since the last cache sync + // Ensure the value cache is up to date if the frame is still live + if (!PyErr_Occurred()) { + if (fastlocalsproxy_set_snapshot_entry(flp, key, value) != 0) { + return NULL; + } + } if (value == NULL && !PyErr_Occurred()) { // Report KeyError if the variable hasn't been bound to a value yet // (akin to getting an UnboundLocalError in running code) @@ -1400,16 +1430,17 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje if (fast_ref == NULL) { // No such local variable, delegate the request to the f_locals mapping // Used by pdb (at least) to store __return__ and __exception__ values - PyObject *locals = _frame_get_locals_mapping(flp->frame); - if (locals == NULL) { - return -1; - } - return PyObject_SetItem(locals, key, value); + return fastlocalsproxy_set_snapshot_entry(flp, key, value); } /* Key is a valid Python variable for the frame, so update that reference */ if (PyCell_Check(fast_ref)) { // Closure cells can be updated even after the frame terminates - return PyCell_Set(fast_ref, value); + int result = PyCell_Set(fast_ref, value); + if (result == 0) { + // Ensure the value cache is up to date if the frame is still live + result = fastlocalsproxy_set_snapshot_entry(flp, key, value); + } + return result; } PyFrameObject *f = flp->frame; if (f->f_state == FRAME_CLEARED) { @@ -1447,14 +1478,20 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje if (set_fast_ref(flp->fast_refs, key, target) != 0) { return -1; } - return PyCell_Set(target, value); + int result = PyCell_Set(target, value); + if (result == 0) { + // Ensure the value cache is up to date if the frame is still live + result = fastlocalsproxy_set_snapshot_entry(flp, key, value); + } + return result; } } // Local variable, or future cell variable that hasn't been converted yet Py_XINCREF(value); Py_XSETREF(fast_locals[offset], value); - return 0; + // Ensure the value cache is up to date if the frame is still live + return fastlocalsproxy_set_snapshot_entry(flp, key, value); } static int @@ -1738,11 +1775,11 @@ fastlocalsproxy_clear(register PyObject *flp, PyObject *Py_UNUSED(ignored)) } -PyDoc_STRVAR(fastlocalsproxy_sync__doc__, - "flp.sync() -> None. Ensure f_locals state is in sync with underlying frame."); +PyDoc_STRVAR(fastlocalsproxy_sync_frame_cache__doc__, + "flp.sync_frame_cache() -> None. Ensure f_locals snapshot is in sync with underlying frame."); static PyObject * -fastlocalsproxy_sync(register PyObject *self, PyObject *Py_UNUSED(ignored)) +fastlocalsproxy_sync_frame_cache(register PyObject *self, PyObject *Py_UNUSED(ignored)) { fastlocalsproxyobject *flp = (fastlocalsproxyobject *)self; if (PyFrame_FastToLocalsWithError(flp->frame) < 0) { @@ -1752,7 +1789,6 @@ fastlocalsproxy_sync(register PyObject *self, PyObject *Py_UNUSED(ignored)) } - static PyMethodDef fastlocalsproxy_methods[] = { {"get", (PyCFunction)(void(*)(void))fastlocalsproxy_get, METH_FASTCALL, PyDoc_STR("D.get(k[,d]) -> D[k] if k in D, else d." @@ -1769,7 +1805,7 @@ static PyMethodDef fastlocalsproxy_methods[] = { PyDoc_STR("See PEP 585")}, {"__reversed__", (PyCFunction)fastlocalsproxy_reversed, METH_NOARGS, PyDoc_STR("D.__reversed__() -> reverse iterator over D's keys")}, - // PEP 558 TODO: Convert these methods to METH_FASTCALL + // PEP 558 TODO: Convert METH_VARARGS/METH_KEYWORDS methods to METH_FASTCALL {"setdefault", (PyCFunction)(void(*)(void))fastlocalsproxy_setdefault, METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_setdefault__doc__}, {"pop", (PyCFunction)(void(*)(void))fastlocalsproxy_pop, @@ -1780,8 +1816,8 @@ static PyMethodDef fastlocalsproxy_methods[] = { METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_update__doc__}, {"clear", (PyCFunction)fastlocalsproxy_clear, METH_NOARGS, fastlocalsproxy_clear__doc__}, - {"sync", (PyCFunction)fastlocalsproxy_clear, - METH_NOARGS, fastlocalsproxy_clear__doc__}, + {"sync_frame_cache", (PyCFunction)fastlocalsproxy_sync_frame_cache, + METH_NOARGS, fastlocalsproxy_sync_frame_cache__doc__}, {NULL, NULL} /* sentinel */ }; From dd946084ea5aaacc9961554bcddf475e9fe5a861 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 11 Jul 2021 17:42:06 +1000 Subject: [PATCH 46/66] Avoid false positives in FLP contains method --- Objects/frameobject.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 5bba9fc7e50df3..b708abcfdcf0b8 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1537,21 +1537,16 @@ static PyNumberMethods fastlocalsproxy_as_number = { static int fastlocalsproxy_contains(fastlocalsproxyobject *flp, PyObject *key) { - assert(flp); - if (fastlocalsproxy_init_fast_refs(flp) != 0) { - return -1; - } - int result = PyDict_Contains(flp->fast_refs, key); - if (result) { - // PEP 558 TODO: This should return false if the name hasn't been bound yet - return result; - } - // Extra keys may have been stored directly in the frame locals - PyObject *locals = _frame_get_locals_mapping(flp->frame); - if (locals == NULL) { - return -1; + // This runs a full key lookup so it will return false if the name + // hasn't been bound yet, but still runs in O(1) time without needing + // to rely on the f_locals cache already being up to date + PyObject *value = fastlocalsproxy_getitem(flp, key); + int found = (value != NULL); + if (!found && PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); } - return PyDict_Contains(locals, key); + Py_XDECREF(value); + return found; } static PySequenceMethods fastlocalsproxy_as_sequence = { From 1484c100ef9f5669f9e83b58c06a2dce4bfdec37 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 17 Jul 2021 17:32:03 +1000 Subject: [PATCH 47/66] Finish public C API, start dict API tests --- Include/cpython/frameobject.h | 9 --- Lib/test/test_frame.py | 144 ++++++++++++++++++++++++++++++++++ Objects/frameobject.c | 54 +++++++++++-- Python/ceval.c | 40 ++++++++++ 4 files changed, 232 insertions(+), 15 deletions(-) diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 1e117107be4afc..e72ce6a8fa791b 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -91,8 +91,6 @@ PyAPI_DATA(PyTypeObject) _PyFastLocalsProxy_Type; // TODO: Add specific test cases for these (as any PyLocals_* tests won't cover // checking the status of a frame other than the currently active one) PyAPI_FUNC(PyObject *) PyFrame_GetLocals(PyFrameObject *); - -// TODO: Implement the rest of these, and add API tests PyAPI_FUNC(PyObject *) PyFrame_GetLocalsCopy(PyFrameObject *); PyAPI_FUNC(PyObject *) PyFrame_GetLocalsView(PyFrameObject *); PyAPI_FUNC(int) PyFrame_GetLocalsReturnsCopy(PyFrameObject *); @@ -100,13 +98,6 @@ PyAPI_FUNC(int) PyFrame_GetLocalsReturnsCopy(PyFrameObject *); // Underlying API supporting PyEval_GetLocals() PyAPI_FUNC(PyObject *) _PyFrame_BorrowLocals(PyFrameObject *); -/* Force an update of any selectively updated views previously returned by - * PyFrame_GetLocalsView(frame). Currently also needed in CPython when - * accessing the f_locals attribute directly and it is not a plain dict - * instance (otherwise it may report stale information). - */ -PyAPI_FUNC(int) PyFrame_RefreshLocalsView(PyFrameObject *); - #ifdef __cplusplus } #endif diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index f6e456812a6b26..427534876483ae 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -5,6 +5,7 @@ import weakref from test import support +from test.support import import_helper class ClearTest(unittest.TestCase): @@ -195,6 +196,149 @@ def test_f_lineno_del_segfault(self): with self.assertRaises(AttributeError): del f.f_lineno +class FastLocalsProxyTest(unittest.TestCase): + + def check_proxy_contents(self, proxy, expected_contents): + # These checks should never implicitly resync the frame proxy's cache, + # even if the proxy is referenced as a local variable in the frame + self.assertEqual(len(proxy), len(expected_contents)) + self.assertCountEqual(proxy, expected_contents) + self.assertCountEqual(proxy.keys(), expected_contents.keys()) + self.assertCountEqual(proxy.values(), expected_contents.values()) + self.assertCountEqual(proxy.items(), expected_contents.items()) + + def test_dict_operations(self): + # No real iteration order guarantees for the locals proxy, as it + # depends on exactly how the compiler composes the frame locals array + proxy = sys._getframe().f_locals + # Not yet set values aren't visible in the proxy + self.check_proxy_contents(proxy, {"self": self}) + + # Ensuring copying the proxy produces a plain dict instance + dict_copy = proxy.copy() + self.assertIsInstance(dict_copy, dict) + self.assertEqual(dict_copy.keys(), {"proxy", "self"}) + # The proxy automatically updates its cache for O(n) operations like copying, + # but won't pick up new local variables until it is resync'ed with the frame + # or that particular key is accessed or queried + self.check_proxy_contents(proxy, dict_copy) + self.assertIn("dict_copy", proxy) # Implicitly updates cache for this key + dict_copy["dict_copy"] = dict_copy + self.check_proxy_contents(proxy, dict_copy) + + self.fail("Test not finished yet") + + def test_active_frame_c_apis(self): + # Use ctypes to access the C APIs under test + ctypes = import_helper.import_module('ctypes') + Py_IncRef = ctypes.pythonapi.Py_IncRef + PyEval_GetLocals = ctypes.pythonapi.PyEval_GetLocals + PyLocals_Get = ctypes.pythonapi.PyLocals_Get + PyLocals_GetReturnsCopy = ctypes.pythonapi.PyLocals_GetReturnsCopy + PyLocals_GetCopy = ctypes.pythonapi.PyLocals_GetCopy + PyLocals_GetView = ctypes.pythonapi.PyLocals_GetView + for capi_func in (Py_IncRef,): + capi_func.argtypes = (ctypes.py_object,) + for capi_func in (PyEval_GetLocals, + PyLocals_Get, PyLocals_GetCopy, PyLocals_GetView): + capi_func.restype = ctypes.py_object + + # PyEval_GetLocals() always accesses the running frame, + # so Py_IncRef has to be called inline (no helper function) + + # This test covers the retrieval APIs, the behavioural tests are covered + # elsewhere using the `frame.f_locals` attribute and the locals() builtin + + # Test retrieval API behaviour in an optimised scope + print("Retrieving C locals cache for frame") + c_locals_cache = PyEval_GetLocals() + Py_IncRef(c_locals_cache) # Make the borrowed reference a real one + Py_IncRef(c_locals_cache) # Account for next check's borrowed reference + self.assertIs(PyEval_GetLocals(), c_locals_cache) + self.assertTrue(PyLocals_GetReturnsCopy()) + locals_get = PyLocals_Get() + self.assertIsInstance(locals_get, dict) + self.assertIsNot(locals_get, c_locals_cache) + locals_copy = PyLocals_GetCopy() + self.assertIsInstance(locals_copy, dict) + self.assertIsNot(locals_copy, c_locals_cache) + locals_view = PyLocals_GetView() + self.assertIsInstance(locals_view, types.MappingProxyType) + + # Test API behaviour in an unoptimised scope + class ExecFrame: + c_locals_cache = PyEval_GetLocals() + Py_IncRef(c_locals_cache) # Make the borrowed reference a real one + Py_IncRef(c_locals_cache) # Account for next check's borrowed reference + self.assertIs(PyEval_GetLocals(), c_locals_cache) + self.assertFalse(PyLocals_GetReturnsCopy()) + locals_get = PyLocals_Get() + self.assertIs(locals_get, c_locals_cache) + locals_copy = PyLocals_GetCopy() + self.assertIsInstance(locals_copy, dict) + self.assertIsNot(locals_copy, c_locals_cache) + locals_view = PyLocals_GetView() + self.assertIsInstance(locals_view, types.MappingProxyType) + + def test_arbitrary_frame_c_apis(self): + # Use ctypes to access the C APIs under test + ctypes = import_helper.import_module('ctypes') + Py_IncRef = ctypes.pythonapi.Py_IncRef + _PyFrame_BorrowLocals = ctypes.pythonapi._PyFrame_BorrowLocals + PyFrame_GetLocals = ctypes.pythonapi.PyFrame_GetLocals + PyFrame_GetLocalsReturnsCopy = ctypes.pythonapi.PyFrame_GetLocalsReturnsCopy + PyFrame_GetLocalsCopy = ctypes.pythonapi.PyFrame_GetLocalsCopy + PyFrame_GetLocalsView = ctypes.pythonapi.PyFrame_GetLocalsView + for capi_func in (Py_IncRef, _PyFrame_BorrowLocals, + PyFrame_GetLocals, PyFrame_GetLocalsReturnsCopy, + PyFrame_GetLocalsCopy, PyFrame_GetLocalsView): + capi_func.argtypes = (ctypes.py_object,) + for capi_func in (_PyFrame_BorrowLocals, PyFrame_GetLocals, + PyFrame_GetLocalsCopy, PyFrame_GetLocalsView): + capi_func.restype = ctypes.py_object + + def get_c_locals(frame): + c_locals = _PyFrame_BorrowLocals(frame) + Py_IncRef(c_locals) # Make the borrowed reference a real one + return c_locals + + # This test covers the retrieval APIs, the behavioural tests are covered + # elsewhere using the `frame.f_locals` attribute and the locals() builtin + + # Test querying an optimised frame from an unoptimised scope + func_frame = sys._getframe() + cls_frame = None + def set_cls_frame(f): + nonlocal cls_frame + cls_frame = f + class ExecFrame: + c_locals_cache = get_c_locals(func_frame) + self.assertIs(get_c_locals(func_frame), c_locals_cache) + self.assertTrue(PyFrame_GetLocalsReturnsCopy(func_frame)) + locals_get = PyFrame_GetLocals(func_frame) + self.assertIsInstance(locals_get, dict) + self.assertIsNot(locals_get, c_locals_cache) + locals_copy = PyFrame_GetLocalsCopy(func_frame) + self.assertIsInstance(locals_copy, dict) + self.assertIsNot(locals_copy, c_locals_cache) + locals_view = PyFrame_GetLocalsView(func_frame) + self.assertIsInstance(locals_view, types.MappingProxyType) + + # Keep the class frame alive for the functions below to access + set_cls_frame(sys._getframe()) + + # Test querying an unoptimised frame from an optimised scope + c_locals_cache = get_c_locals(cls_frame) + self.assertIs(get_c_locals(cls_frame), c_locals_cache) + self.assertFalse(PyFrame_GetLocalsReturnsCopy(cls_frame)) + locals_get = PyFrame_GetLocals(cls_frame) + self.assertIs(locals_get, c_locals_cache) + locals_copy = PyFrame_GetLocalsCopy(cls_frame) + self.assertIsInstance(locals_copy, dict) + self.assertIsNot(locals_copy, c_locals_cache) + locals_view = PyFrame_GetLocalsView(cls_frame) + self.assertIsInstance(locals_view, types.MappingProxyType) + class ReprTest(unittest.TestCase): """ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index b708abcfdcf0b8..69a545023f100c 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -34,6 +34,7 @@ _frame_get_locals_mapping(PyFrameObject *f) { PyObject *locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET]; if (locals == NULL) { + printf("Allocating new frame locals cache\n"); locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET] = PyDict_New(); } return locals; @@ -53,16 +54,20 @@ _frame_get_updated_locals(PyFrameObject *f) PyObject * _PyFrame_BorrowLocals(PyFrameObject *f) { - // This is called by PyEval_GetLocals(), which has historically returned - // a borrowed reference, so this does the same + // This frame API supports the PyEval_GetLocals() stable API, which has + // historically returned a borrowed reference (so this does the same) return _frame_get_updated_locals(f); } PyObject * PyFrame_GetLocals(PyFrameObject *f) { - // This API implements the Python level locals() builtin + // This frame API implements the Python level locals() builtin + // and supports the PyLocals_Get() stable API PyObject *updated_locals = _frame_get_updated_locals(f); + if (updated_locals == NULL) { + return NULL; + } PyCodeObject *co = _PyFrame_GetCode(f); assert(co); @@ -77,6 +82,26 @@ PyFrame_GetLocals(PyFrameObject *f) return updated_locals; } +int +PyFrame_GetLocalsReturnsCopy(PyFrameObject *f) +{ + // This frame API supports the stable PyLocals_GetReturnsCopy() API + PyCodeObject *co = _PyFrame_GetCode(f); + assert(co); + return (co->co_flags & CO_OPTIMIZED); +} + +PyObject * +PyFrame_GetLocalsCopy(PyFrameObject *f) +{ + // This frame API supports the stable PyLocals_GetCopy() API + PyObject *updated_locals = _frame_get_updated_locals(f); + if (updated_locals == NULL) { + return NULL; + } + return PyDict_Copy(updated_locals); +} + static PyObject * frame_getlocals(PyFrameObject *f, void *Py_UNUSED(ignored)) { @@ -97,11 +122,25 @@ frame_getlocals(PyFrameObject *f, void *Py_UNUSED(ignored)) } else { // Share a direct locals reference for class and module scopes f_locals_attr = _frame_get_updated_locals(f); + if (f_locals_attr == NULL) { + return NULL; + } Py_INCREF(f_locals_attr); } return f_locals_attr; } +PyObject * +PyFrame_GetLocalsView(PyFrameObject *f) +{ + // This frame API supports the stable PyLocals_GetView() API + PyObject *rw_locals = frame_getlocals(f, NULL); + if (rw_locals == NULL) { + return NULL; + } + return PyDictProxy_New(rw_locals); +} + int PyFrame_GetLineNumber(PyFrameObject *f) { @@ -1018,6 +1057,7 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f) return -1; } co = _PyFrame_GetCode(f); + assert(co); fast = f->f_localsptr; for (int i = 0; i < co->co_nlocalsplus; i++) { _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); @@ -1648,8 +1688,10 @@ static PyObject * fastlocalsproxy_richcompare(fastlocalsproxyobject *flp, PyObject *w, int op) { // Need values, so use the dynamic snapshot on the frame - // Ensure it is up to date, as checking is O(n) anyway - PyObject *locals = _frame_get_updated_locals(flp->frame); + // Assume f_locals snapshot is up to date, as even though the worst + // case comparison is O(n) to determine equality, there are O(1) shortcuts + // for inequality checks (i.e. different sizes) + PyObject *locals = _frame_get_locals_mapping(flp->frame); return PyObject_RichCompare(locals, w, op); } @@ -1841,7 +1883,7 @@ static PyObject * fastlocalsproxy_str(fastlocalsproxyobject *flp) { // Need values, so use the dynamic snapshot on the frame - // Ensure it is up to date, as checking is O(n) anyway + // Ensure it is up to date, as displaying everything is O(n) anyway PyObject *locals = _frame_get_updated_locals(flp->frame); return PyObject_Str(locals); } diff --git a/Python/ceval.c b/Python/ceval.c index d224e0b04af087..d2135aec70f084 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5798,6 +5798,46 @@ PyLocals_Get(void) return PyFrame_GetLocals(current_frame); } +int +PyLocals_GetReturnsCopy(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyFrameObject *current_frame = tstate->frame; + if (current_frame == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); + return NULL; + } + + return PyFrame_GetLocalsReturnsCopy(current_frame); +} + +PyObject * +PyLocals_GetCopy(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyFrameObject *current_frame = tstate->frame; + if (current_frame == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); + return NULL; + } + + return PyFrame_GetLocalsCopy(current_frame); +} + +PyObject * +PyLocals_GetView(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyFrameObject *current_frame = tstate->frame; + if (current_frame == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); + return NULL; + } + + return PyFrame_GetLocalsView(current_frame); +} + + PyObject * PyEval_GetGlobals(void) { From 760ffa9e44aa02ebe3fc1f3d4ccb8f90df660a58 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 17 Jul 2021 19:42:15 +1000 Subject: [PATCH 48/66] Remove debugging print statement --- Objects/frameobject.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 69a545023f100c..959420206f285e 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -34,7 +34,6 @@ _frame_get_locals_mapping(PyFrameObject *f) { PyObject *locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET]; if (locals == NULL) { - printf("Allocating new frame locals cache\n"); locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET] = PyDict_New(); } return locals; From ae6b013787bb337759535b4fa4936f9912474c49 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 18 Jul 2021 16:06:15 +1000 Subject: [PATCH 49/66] Regenerated stable ABI files --- Doc/data/stable_abi.dat | 4 ++++ PC/python3dll.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index e373e2314a6517..0ddd3e08af346c 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -332,6 +332,10 @@ function,PyList_SetSlice,3.2, function,PyList_Size,3.2, function,PyList_Sort,3.2, var,PyList_Type,3.2, +function,PyLocals_Get,3.11, +function,PyLocals_GetCopy,3.11, +function,PyLocals_GetReturnsCopy,3.11, +function,PyLocals_GetView,3.11, type,PyLongObject,3.2, var,PyLongRangeIter_Type,3.2, function,PyLong_AsDouble,3.2, diff --git a/PC/python3dll.c b/PC/python3dll.c index 0ebb56efaecb2c..255d5b72bb7881 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -313,6 +313,10 @@ EXPORT_FUNC(PyList_SetItem) EXPORT_FUNC(PyList_SetSlice) EXPORT_FUNC(PyList_Size) EXPORT_FUNC(PyList_Sort) +EXPORT_FUNC(PyLocals_Get) +EXPORT_FUNC(PyLocals_GetCopy) +EXPORT_FUNC(PyLocals_GetReturnsCopy) +EXPORT_FUNC(PyLocals_GetView) EXPORT_FUNC(PyLong_AsDouble) EXPORT_FUNC(PyLong_AsLong) EXPORT_FUNC(PyLong_AsLongAndOverflow) From b03309bd8c454dca27c2f8bf248b0252445a0c04 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 21 Jul 2021 19:12:38 +1000 Subject: [PATCH 50/66] Rename _PyLocals_Kind APIs to avoid potential confusion --- Include/internal/pycore_code.h | 10 +++++----- Objects/codeobject.c | 14 +++++++------- Objects/frameobject.c | 8 ++++---- Objects/typeobject.c | 4 ++-- Python/compile.c | 2 +- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index bc469763670d4e..3a2fc398cc0b2c 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -164,19 +164,19 @@ extern Py_ssize_t _Py_QuickenedCount; #define CO_FAST_CELL 0x40 #define CO_FAST_FREE 0x80 -typedef unsigned char _PyLocals_Kind; +typedef unsigned char _PyLocal_VarKind; -static inline _PyLocals_Kind -_PyLocals_GetKind(PyObject *kinds, int i) +static inline _PyLocal_VarKind +_PyLocal_GetVarKind(PyObject *kinds, int i) { assert(PyBytes_Check(kinds)); assert(0 <= i && i < PyBytes_GET_SIZE(kinds)); char *ptr = PyBytes_AS_STRING(kinds); - return (_PyLocals_Kind)(ptr[i]); + return (_PyLocal_VarKind)(ptr[i]); } static inline void -_PyLocals_SetKind(PyObject *kinds, int i, _PyLocals_Kind kind) +_PyLocal_SetVarKind(PyObject *kinds, int i, _PyLocal_VarKind kind) { assert(PyBytes_Check(kinds)); assert(0 <= i && i < PyBytes_GET_SIZE(kinds)); diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 3dc9fd787f3859..c0206043278109 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -156,12 +156,12 @@ validate_and_copy_tuple(PyObject *tup) // This is also used in compile.c. void -_Py_set_localsplus_info(int offset, PyObject *name, _PyLocals_Kind kind, +_Py_set_localsplus_info(int offset, PyObject *name, _PyLocal_VarKind kind, PyObject *names, PyObject *kinds) { Py_INCREF(name); PyTuple_SET_ITEM(names, offset, name); - _PyLocals_SetKind(kinds, offset, kind); + _PyLocal_SetVarKind(kinds, offset, kind); } static void @@ -175,7 +175,7 @@ get_localsplus_counts(PyObject *names, PyObject *kinds, int nfreevars = 0; Py_ssize_t nlocalsplus = PyTuple_GET_SIZE(names); for (int i = 0; i < nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(kinds, i); + _PyLocal_VarKind kind = _PyLocal_GetVarKind(kinds, i); if (kind & CO_FAST_LOCAL) { nlocals += 1; if (kind & CO_FAST_CELL) { @@ -205,7 +205,7 @@ get_localsplus_counts(PyObject *names, PyObject *kinds, } static PyObject * -get_localsplus_names(PyCodeObject *co, _PyLocals_Kind kind, int num) +get_localsplus_names(PyCodeObject *co, _PyLocal_VarKind kind, int num) { PyObject *names = PyTuple_New(num); if (names == NULL) { @@ -213,7 +213,7 @@ get_localsplus_names(PyCodeObject *co, _PyLocals_Kind kind, int num) } int index = 0; for (int offset = 0; offset < co->co_nlocalsplus; offset++) { - _PyLocals_Kind k = _PyLocals_GetKind(co->co_localspluskinds, offset); + _PyLocal_VarKind k = _PyLocal_GetVarKind(co->co_localspluskinds, offset); if ((k & kind) == 0) { continue; } @@ -458,8 +458,8 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, // Merge the localsplus indices. nlocalsplus -= 1; offset -= 1; - _PyLocals_Kind kind = _PyLocals_GetKind(localspluskinds, argoffset); - _PyLocals_SetKind(localspluskinds, argoffset, kind | CO_FAST_CELL); + _PyLocal_VarKind kind = _PyLocal_GetVarKind(localspluskinds, argoffset); + _PyLocal_SetVarKind(localspluskinds, argoffset, kind | CO_FAST_CELL); continue; } _Py_set_localsplus_info(offset, name, CO_FAST_CELL, diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 959420206f285e..4bd08907916185 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1059,7 +1059,7 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f) assert(co); fast = f->f_localsptr; for (int i = 0; i < co->co_nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + _PyLocal_VarKind kind = _PyLocal_GetVarKind(co->co_localspluskinds, i); /* If the namespace is unoptimized, then one of the following cases applies: @@ -1189,7 +1189,7 @@ _PyFrame_BuildFastRefs(PyFrameObject *f) if (f->f_state != FRAME_CLEARED) { for (int i = 0; i < co->co_nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + _PyLocal_VarKind kind = _PyLocal_GetVarKind(co->co_localspluskinds, i); PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); PyObject *target = NULL; if (kind & CO_FAST_FREE) { @@ -1420,7 +1420,7 @@ fastlocalsproxy_getitem(fastlocalsproxyobject *flp, PyObject *key) PyObject **fast_locals = f->f_localsptr; value = fast_locals[offset]; // Check if MAKE_CELL has been called since the proxy was created - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, offset); + _PyLocal_VarKind kind = _PyLocal_GetVarKind(co->co_localspluskinds, offset); if (kind & CO_FAST_CELL) { // Value hadn't been converted to a cell yet when the proxy was created // Update the proxy if MAKE_CELL has run since the last access, @@ -1504,7 +1504,7 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje } PyObject **fast_locals = f->f_localsptr; // Check if MAKE_CELL has been called since the proxy was created - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, offset); + _PyLocal_VarKind kind = _PyLocal_GetVarKind(co->co_localspluskinds, offset); if (kind & CO_FAST_CELL) { // Value hadn't been converted to a cell yet when the proxy was created // Update the proxy if MAKE_CELL has run since the last access, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 3331fee4252dee..37ab467d541783 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8888,7 +8888,7 @@ super_init_without_args(PyFrameObject *f, PyCodeObject *co, PyObject *firstarg = f->f_localsptr[0]; // The first argument might be a cell. - if (firstarg != NULL && (_PyLocals_GetKind(co->co_localspluskinds, 0) & CO_FAST_CELL)) { + if (firstarg != NULL && (_PyLocal_GetVarKind(co->co_localspluskinds, 0) & CO_FAST_CELL)) { // "firstarg" is a cell here unless (very unlikely) super() // was called from the C-API before the first MAKE_CELL op. if (f->f_lasti >= 0) { @@ -8907,7 +8907,7 @@ super_init_without_args(PyFrameObject *f, PyCodeObject *co, PyTypeObject *type = NULL; int i = co->co_nlocals + co->co_nplaincellvars; for (; i < co->co_nlocalsplus; i++) { - assert((_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_FREE) != 0); + assert((_PyLocal_GetVarKind(co->co_localspluskinds, i) & CO_FAST_FREE) != 0); PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); assert(PyUnicode_Check(name)); if (_PyUnicode_EqualToASCIIId(name, &PyId___class__)) { diff --git a/Python/compile.c b/Python/compile.c index 3a20f6b57eb367..a13ad7121225c2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -7385,7 +7385,7 @@ compute_localsplus_info(struct compiler *c, int nlocalsplus, assert(offset >= 0); assert(offset < nlocalsplus); // For now we do not distinguish arg kinds. - _PyLocals_Kind kind = CO_FAST_LOCAL; + _PyLocal_VarKind kind = CO_FAST_LOCAL; if (PyDict_GetItem(c->u->u_cellvars, k) != NULL) { kind |= CO_FAST_CELL; } From 66d058c1eda1009c0590bcb0f36eafdba4df5dd5 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 21 Jul 2021 21:36:31 +1000 Subject: [PATCH 51/66] PyLocals_GetReturnsCopy -> PyLocals_GetKind() --- Doc/data/stable_abi.dat | 2 +- Include/ceval.h | 14 ++++++++------ Include/cpython/frameobject.h | 4 ++-- Lib/test/test_frame.py | 14 +++++++------- Misc/stable_abi.txt | 2 +- Objects/frameobject.c | 11 +++++++---- PC/python3dll.c | 2 +- Python/ceval.c | 8 ++++---- 8 files changed, 31 insertions(+), 26 deletions(-) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 0ddd3e08af346c..b7fd1bce20641b 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -334,7 +334,7 @@ function,PyList_Sort,3.2, var,PyList_Type,3.2, function,PyLocals_Get,3.11, function,PyLocals_GetCopy,3.11, -function,PyLocals_GetReturnsCopy,3.11, +function,PyLocals_Kind,3.11, function,PyLocals_GetView,3.11, type,PyLongObject,3.2, var,PyLongRangeIter_Type,3.2, diff --git a/Include/ceval.h b/Include/ceval.h index 9a0aab61c5cd90..7131707f8a28be 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -43,20 +43,22 @@ PyAPI_FUNC(PyFrameObject *) PyEval_GetFrame(void); * It returns a read/write reference or a shallow copy depending on the scope * of the active frame. */ -// TODO: Add API tests for this PyAPI_FUNC(PyObject *) PyLocals_Get(void); /* PyLocals_GetCopy() returns a fresh shallow copy of the active local namespace */ -// TODO: Implement this, and add API tests PyAPI_FUNC(PyObject *) PyLocals_GetCopy(void); /* PyLocals_GetView() returns a read-only proxy for the active local namespace */ -// TODO: Implement this, and add API tests PyAPI_FUNC(PyObject *) PyLocals_GetView(void); -/* Returns true if PyLocals_Get() returns a shallow copy in the active scope */ -// TODO: Implement this, and add API tests -PyAPI_FUNC(int) PyLocals_GetReturnsCopy(void); +/* PyLocals_GetKind()reports the behaviour of PyLocals_Get() in the active scope */ +typedef enum { + PyLocals_UNDEFINED = -1, // Indicates error (e.g. no thread state defined) + PyLocals_DIRECT_REFERENCE = 0, + PyLocals_SHALLOW_COPY = 1 +} PyLocals_Kind; + +PyAPI_FUNC(PyLocals_Kind) PyLocals_GetKind(void); #endif diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index e72ce6a8fa791b..438dfe0cb4ece4 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -79,7 +79,7 @@ PyAPI_FUNC(PyFrameObject *) PyFrame_GetBack(PyFrameObject *frame); /* Fast locals proxy allows for reliable write-through from trace functions */ // TODO: Perhaps this should be hidden, and API users told to query for -// PyFrame_GetLocalsReturnsCopy() instead. Having this available +// PyFrame_GetLocalsKind() instead. Having this available // seems like a nice way to let folks write some useful debug assertions, // though. PyAPI_DATA(PyTypeObject) _PyFastLocalsProxy_Type; @@ -93,7 +93,7 @@ PyAPI_DATA(PyTypeObject) _PyFastLocalsProxy_Type; PyAPI_FUNC(PyObject *) PyFrame_GetLocals(PyFrameObject *); PyAPI_FUNC(PyObject *) PyFrame_GetLocalsCopy(PyFrameObject *); PyAPI_FUNC(PyObject *) PyFrame_GetLocalsView(PyFrameObject *); -PyAPI_FUNC(int) PyFrame_GetLocalsReturnsCopy(PyFrameObject *); +PyAPI_FUNC(PyLocals_Kind) PyFrame_GetLocalsKind(PyFrameObject *); // Underlying API supporting PyEval_GetLocals() PyAPI_FUNC(PyObject *) _PyFrame_BorrowLocals(PyFrameObject *); diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 427534876483ae..b79bb30b261703 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -234,7 +234,7 @@ def test_active_frame_c_apis(self): Py_IncRef = ctypes.pythonapi.Py_IncRef PyEval_GetLocals = ctypes.pythonapi.PyEval_GetLocals PyLocals_Get = ctypes.pythonapi.PyLocals_Get - PyLocals_GetReturnsCopy = ctypes.pythonapi.PyLocals_GetReturnsCopy + PyLocals_GetKind = ctypes.pythonapi.PyLocals_GetKind PyLocals_GetCopy = ctypes.pythonapi.PyLocals_GetCopy PyLocals_GetView = ctypes.pythonapi.PyLocals_GetView for capi_func in (Py_IncRef,): @@ -255,7 +255,7 @@ def test_active_frame_c_apis(self): Py_IncRef(c_locals_cache) # Make the borrowed reference a real one Py_IncRef(c_locals_cache) # Account for next check's borrowed reference self.assertIs(PyEval_GetLocals(), c_locals_cache) - self.assertTrue(PyLocals_GetReturnsCopy()) + self.assertEqual(PyLocals_GetKind(), 1) # PyLocals_SHALLOW_COPY locals_get = PyLocals_Get() self.assertIsInstance(locals_get, dict) self.assertIsNot(locals_get, c_locals_cache) @@ -271,7 +271,7 @@ class ExecFrame: Py_IncRef(c_locals_cache) # Make the borrowed reference a real one Py_IncRef(c_locals_cache) # Account for next check's borrowed reference self.assertIs(PyEval_GetLocals(), c_locals_cache) - self.assertFalse(PyLocals_GetReturnsCopy()) + self.assertEqual(PyLocals_GetKind(), 0) # PyLocals_DIRECT_REFERENCE locals_get = PyLocals_Get() self.assertIs(locals_get, c_locals_cache) locals_copy = PyLocals_GetCopy() @@ -286,11 +286,11 @@ def test_arbitrary_frame_c_apis(self): Py_IncRef = ctypes.pythonapi.Py_IncRef _PyFrame_BorrowLocals = ctypes.pythonapi._PyFrame_BorrowLocals PyFrame_GetLocals = ctypes.pythonapi.PyFrame_GetLocals - PyFrame_GetLocalsReturnsCopy = ctypes.pythonapi.PyFrame_GetLocalsReturnsCopy + PyFrame_GetLocalsKind = ctypes.pythonapi.PyFrame_GetLocalsKind PyFrame_GetLocalsCopy = ctypes.pythonapi.PyFrame_GetLocalsCopy PyFrame_GetLocalsView = ctypes.pythonapi.PyFrame_GetLocalsView for capi_func in (Py_IncRef, _PyFrame_BorrowLocals, - PyFrame_GetLocals, PyFrame_GetLocalsReturnsCopy, + PyFrame_GetLocals, PyFrame_GetLocalsKind, PyFrame_GetLocalsCopy, PyFrame_GetLocalsView): capi_func.argtypes = (ctypes.py_object,) for capi_func in (_PyFrame_BorrowLocals, PyFrame_GetLocals, @@ -314,7 +314,7 @@ def set_cls_frame(f): class ExecFrame: c_locals_cache = get_c_locals(func_frame) self.assertIs(get_c_locals(func_frame), c_locals_cache) - self.assertTrue(PyFrame_GetLocalsReturnsCopy(func_frame)) + self.assertEqual(PyFrame_GetLocalsKind(func_frame), 1) # PyLocals_SHALLOW_COPY locals_get = PyFrame_GetLocals(func_frame) self.assertIsInstance(locals_get, dict) self.assertIsNot(locals_get, c_locals_cache) @@ -330,7 +330,7 @@ class ExecFrame: # Test querying an unoptimised frame from an optimised scope c_locals_cache = get_c_locals(cls_frame) self.assertIs(get_c_locals(cls_frame), c_locals_cache) - self.assertFalse(PyFrame_GetLocalsReturnsCopy(cls_frame)) + self.assertEqual(PyFrame_GetLocalsKind(cls_frame), 0) # PyLocals_DIRECT_REFERENCE locals_get = PyFrame_GetLocals(cls_frame) self.assertIs(locals_get, c_locals_cache) locals_copy = PyFrame_GetLocalsCopy(cls_frame) diff --git a/Misc/stable_abi.txt b/Misc/stable_abi.txt index 0f981de1d367fb..b858d50961070e 100644 --- a/Misc/stable_abi.txt +++ b/Misc/stable_abi.txt @@ -2143,7 +2143,7 @@ function PyLocals_Get added 3.11 function PyLocals_GetCopy added 3.11 -function PyLocals_GetReturnsCopy +function PyLocals_GetKind added 3.11 function PyLocals_GetView added 3.11 diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 4bd08907916185..8c5dbe0b5db878 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -81,13 +81,16 @@ PyFrame_GetLocals(PyFrameObject *f) return updated_locals; } -int -PyFrame_GetLocalsReturnsCopy(PyFrameObject *f) +PyLocals_Kind +PyFrame_GetLocalsKind(PyFrameObject *f) { - // This frame API supports the stable PyLocals_GetReturnsCopy() API + // This frame API supports the stable PyLocals_GetKind() API PyCodeObject *co = _PyFrame_GetCode(f); assert(co); - return (co->co_flags & CO_OPTIMIZED); + if (co->co_flags & CO_OPTIMIZED) { + return PyLocals_SHALLOW_COPY; + } + return PyLocals_DIRECT_REFERENCE; } PyObject * diff --git a/PC/python3dll.c b/PC/python3dll.c index 255d5b72bb7881..ac6d857d56a171 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -315,7 +315,7 @@ EXPORT_FUNC(PyList_Size) EXPORT_FUNC(PyList_Sort) EXPORT_FUNC(PyLocals_Get) EXPORT_FUNC(PyLocals_GetCopy) -EXPORT_FUNC(PyLocals_GetReturnsCopy) +EXPORT_FUNC(PyLocals_GetKind) EXPORT_FUNC(PyLocals_GetView) EXPORT_FUNC(PyLong_AsDouble) EXPORT_FUNC(PyLong_AsLong) diff --git a/Python/ceval.c b/Python/ceval.c index 33468e2c639728..35457176e3ad1d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5803,17 +5803,17 @@ PyLocals_Get(void) return PyFrame_GetLocals(current_frame); } -int -PyLocals_GetReturnsCopy(void) +PyLocals_Kind +PyLocals_GetKind(void) { PyThreadState *tstate = _PyThreadState_GET(); PyFrameObject *current_frame = tstate->frame; if (current_frame == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); - return NULL; + return PyLocals_UNDEFINED; } - return PyFrame_GetLocalsReturnsCopy(current_frame); + return PyFrame_GetLocalsKind(current_frame); } PyObject * From 67c3958c473923d548b6fdc9d7bb66fde7989e6d Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 21 Aug 2021 15:57:41 +1000 Subject: [PATCH 52/66] Share fast_refs mapping between proxy objects --- Include/cpython/frameobject.h | 1 + Objects/frameobject.c | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 438dfe0cb4ece4..2e3ec431fd6fe6 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -24,6 +24,7 @@ struct _frame { struct _frame *f_back; /* previous frame, or NULL */ PyObject **f_valuestack; /* points after the last local */ PyObject *f_trace; /* Trace function */ + PyObject *f_fast_refs; /* Name -> index-or-cell lookup for fast locals */ /* Borrowed reference to a generator, or NULL */ PyObject *f_gen; int f_stackdepth; /* Depth of value stack */ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 8c5dbe0b5db878..198a852ee197c9 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -55,6 +55,7 @@ _PyFrame_BorrowLocals(PyFrameObject *f) { // This frame API supports the PyEval_GetLocals() stable API, which has // historically returned a borrowed reference (so this does the same) + // It also ensures the cache is up to date return _frame_get_updated_locals(f); } @@ -770,6 +771,7 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) { Py_VISIT(f->f_back); Py_VISIT(f->f_trace); + Py_VISIT(f->f_fast_refs); /* locals */ PyObject **localsplus = f->f_localsptr; @@ -795,6 +797,7 @@ frame_tp_clear(PyFrameObject *f) f->f_state = FRAME_CLEARED; Py_CLEAR(f->f_trace); + Py_CLEAR(f->f_fast_refs); PyCodeObject *co = _PyFrame_GetCode(f); /* locals */ for (int i = 0; i < co->co_nlocalsplus; i++) { @@ -993,6 +996,7 @@ _PyFrame_New_NoTrack(PyThreadState *tstate, PyFrameConstructor *con, PyObject *l specials[FRAME_SPECIALS_GLOBALS_OFFSET] = Py_NewRef(con->fc_globals); specials[FRAME_SPECIALS_LOCALS_OFFSET] = Py_XNewRef(locals); f->f_trace = NULL; + f->f_fast_refs = NULL; f->f_stackdepth = 0; f->f_trace_lines = 1; f->f_trace_opcodes = 0; @@ -1320,7 +1324,7 @@ class fastlocalsproxy "fastlocalsproxyobject *" "&_PyFastLocalsProxy_Type" typedef struct { PyObject_HEAD PyFrameObject *frame; - PyObject *fast_refs; /* Cell refs and local variable indices */ + //int frame_cache_refreshed; /* Assume cache is out of date if this is not set */ } fastlocalsproxyobject; // PEP 558 TODO: Implement correct Python sizeof() support for fastlocalsproxyobject @@ -1330,14 +1334,15 @@ fastlocalsproxy_init_fast_refs(fastlocalsproxyobject *flp) { // Build fast ref mapping if it hasn't been built yet assert(flp); - if (flp->fast_refs != NULL) { + assert(flp->frame); + if (flp->frame->f_fast_refs != NULL) { return 0; } PyObject *fast_refs = _PyFrame_BuildFastRefs(flp->frame); if (fast_refs == NULL) { return -1; } - flp->fast_refs = fast_refs; + flp->frame->f_fast_refs = fast_refs; return 0; } @@ -1383,7 +1388,7 @@ fastlocalsproxy_getitem(fastlocalsproxyobject *flp, PyObject *key) if (fastlocalsproxy_init_fast_refs(flp) != 0) { return NULL; } - PyObject *fast_ref = PyDict_GetItem(flp->fast_refs, key); + PyObject *fast_ref = PyDict_GetItem(flp->frame->f_fast_refs, key); if (fast_ref == NULL) { // No such local variable, delegate the request to the f_locals mapping // Used by pdb (at least) to access __return__ and __exception__ values @@ -1433,7 +1438,7 @@ fastlocalsproxy_getitem(fastlocalsproxyobject *flp, PyObject *key) _PyFrame_OpAlreadyRan(f, MAKE_CELL, offset)) { // MAKE_CELL has built the cell, so use it as the proxy target Py_INCREF(target); - if (set_fast_ref(flp->fast_refs, key, target) != 0) { + if (set_fast_ref(flp->frame->f_fast_refs, key, target) != 0) { return NULL; } value = PyCell_GET(target); @@ -1468,7 +1473,7 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje if (fastlocalsproxy_init_fast_refs(flp) != 0) { return -1; } - PyObject *fast_ref = PyDict_GetItem(flp->fast_refs, key); + PyObject *fast_ref = PyDict_GetItem(flp->frame->f_fast_refs, key); if (fast_ref == NULL) { // No such local variable, delegate the request to the f_locals mapping // Used by pdb (at least) to store __return__ and __exception__ values @@ -1517,7 +1522,7 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje _PyFrame_OpAlreadyRan(f, MAKE_CELL, offset)) { // MAKE_CELL has built the cell, so use it as the proxy target Py_INCREF(target); - if (set_fast_ref(flp->fast_refs, key, target) != 0) { + if (set_fast_ref(flp->frame->f_fast_refs, key, target) != 0) { return -1; } int result = PyCell_Set(target, value); @@ -1869,7 +1874,6 @@ fastlocalsproxy_dealloc(fastlocalsproxyobject *flp) Py_TRASHCAN_SAFE_BEGIN(flp) Py_CLEAR(flp->frame); - Py_CLEAR(flp->fast_refs); PyObject_GC_Del(flp); Py_TRASHCAN_SAFE_END(flp) @@ -1895,7 +1899,6 @@ fastlocalsproxy_traverse(PyObject *self, visitproc visit, void *arg) { fastlocalsproxyobject *flp = (fastlocalsproxyobject *)self; Py_VISIT(flp->frame); - Py_VISIT(flp->fast_refs); return 0; } @@ -1936,7 +1939,6 @@ _PyFastLocalsProxy_New(PyObject *frame) return NULL; flp->frame = (PyFrameObject *) frame; Py_INCREF(flp->frame); - flp->fast_refs = NULL; _PyObject_GC_TRACK(flp); return (PyObject *)flp; } From 034345fd519784f98d46c216d301b238d3068c25 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 21 Aug 2021 16:02:57 +1000 Subject: [PATCH 53/66] Remove debugging print --- Lib/test/test_frame.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index b79bb30b261703..69c2b82faf519d 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -250,7 +250,6 @@ def test_active_frame_c_apis(self): # elsewhere using the `frame.f_locals` attribute and the locals() builtin # Test retrieval API behaviour in an optimised scope - print("Retrieving C locals cache for frame") c_locals_cache = PyEval_GetLocals() Py_IncRef(c_locals_cache) # Make the borrowed reference a real one Py_IncRef(c_locals_cache) # Account for next check's borrowed reference From 3c49ff822b8dbba18db1904caf319271d2b723ad Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 21 Aug 2021 17:21:03 +1000 Subject: [PATCH 54/66] Defer value cache refresh until needed, start fleshing out dict API tests --- Lib/test/test_frame.py | 101 ++++++++++++++++++++++++++++++---- Objects/frameobject.c | 121 ++++++++++++++++++++++++++--------------- 2 files changed, 167 insertions(+), 55 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 69c2b82faf519d..7fb63f4dffcb41 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -201,32 +201,109 @@ class FastLocalsProxyTest(unittest.TestCase): def check_proxy_contents(self, proxy, expected_contents): # These checks should never implicitly resync the frame proxy's cache, # even if the proxy is referenced as a local variable in the frame + # However, the first executed check may trigger the initial lazy sync self.assertEqual(len(proxy), len(expected_contents)) - self.assertCountEqual(proxy, expected_contents) - self.assertCountEqual(proxy.keys(), expected_contents.keys()) - self.assertCountEqual(proxy.values(), expected_contents.values()) self.assertCountEqual(proxy.items(), expected_contents.items()) - def test_dict_operations(self): - # No real iteration order guarantees for the locals proxy, as it - # depends on exactly how the compiler composes the frame locals array + def test_dict_query_operations(self): + # Check retrieval of individual keys via the proxy proxy = sys._getframe().f_locals - # Not yet set values aren't visible in the proxy - self.check_proxy_contents(proxy, {"self": self}) + self.assertIs(proxy["self"], self) + self.assertIs(proxy.get("self"), self) + self.assertIs(proxy.get("no-such-key"), None) + self.assertIs(proxy.get("no-such-key", Ellipsis), Ellipsis) + + # Proxy value cache is lazily refreshed on the first operation that cares + # about the full contents of the mapping (such as querying the length) + expected_proxy_contents = {"self": self, "proxy": proxy} + expected_proxy_contents["expected_proxy_contents"] = expected_proxy_contents + self.check_proxy_contents(proxy, expected_proxy_contents) # Ensuring copying the proxy produces a plain dict instance dict_copy = proxy.copy() self.assertIsInstance(dict_copy, dict) - self.assertEqual(dict_copy.keys(), {"proxy", "self"}) + self.check_proxy_contents(dict_copy, expected_proxy_contents) + # The proxy automatically updates its cache for O(n) operations like copying, # but won't pick up new local variables until it is resync'ed with the frame # or that particular key is accessed or queried self.check_proxy_contents(proxy, dict_copy) self.assertIn("dict_copy", proxy) # Implicitly updates cache for this key - dict_copy["dict_copy"] = dict_copy - self.check_proxy_contents(proxy, dict_copy) + expected_proxy_contents["dict_copy"] = dict_copy + self.check_proxy_contents(proxy, expected_proxy_contents) + + # Check forward iteration (order is abitrary, so only check overall contents) + # Note: len() and the items() method are covered by "check_proxy_contents" + self.assertCountEqual(proxy, expected_proxy_contents) + self.assertCountEqual(proxy.keys(), expected_proxy_contents.keys()) + self.assertCountEqual(proxy.values(), expected_proxy_contents.values()) + # Check reversed iteration (order should be reverse of forward iteration) + self.assertEqual(list(reversed(proxy)), list(reversed(list(proxy)))) - self.fail("Test not finished yet") + self.fail("PEP 558 TODO: Implement proxy '|' operator test") + + def test_dict_mutation_operations(self): + # Check mutation of local variables via proxy + proxy = sys._getframe().f_locals + if not len(proxy): # Trigger the initial implicit cache update + # This code block never actually runs + unbound_local = None + self.assertNotIn("unbound_local", proxy) + proxy["unbound_local"] = "set via proxy" + self.assertEqual(unbound_local, "set via proxy") + del proxy["unbound_local"] + with self.assertRaises(UnboundLocalError): + print(unbound_local) + # Check mutation of cell variables via proxy + cell_variable = None + proxy["cell_variable"] = "set via proxy" + self.assertEqual(cell_variable, "set via proxy") + def inner(): + return cell_variable + self.assertEqual(inner(), "set via proxy") + del proxy["cell_variable"] + with self.assertRaises(UnboundLocalError): + print(cell_variable) + with self.assertRaises(NameError): + inner() + # Check storage of additional variables in the frame value cache via proxy + proxy["extra_variable"] = "added via proxy" + self.assertEqual(proxy["extra_variable"], "added via proxy") + with self.assertRaises(NameError): + print(extra_variable) + del proxy["extra_variable"] + self.assertNotIn("extra_variable", proxy) + + # Check updating all 3 kinds of variable via update() + updated_keys = { + "unbound_local": "set via proxy.update()", + "cell_variable": "set via proxy.update()", + "extra_variable": "set via proxy.update()", + } + proxy.update(updated_keys) + self.assertEqual(unbound_local, "set via proxy.update()") + self.assertEqual(cell_variable, "set via proxy.update()") + self.assertEqual(proxy["extra_variable"], "set via proxy.update()") + + self.fail("PEP 558 TODO: Implement proxy '|=' operator test") + self.fail("PEP 558 TODO: Implement proxy update() test") + self.fail("PEP 558 TODO: Implement proxy setdefault() test") + self.fail("PEP 558 TODO: Implement proxy pop() test") + self.fail("PEP 558 TODO: Implement proxy popitem() test") + self.fail("PEP 558 TODO: Implement proxy clear() test") + + def test_sync_frame_cache(self): + proxy = sys._getframe().f_locals + self.assertEqual(len(proxy), 2) # Trigger the initial implicit cache update + new_variable = None + # No implicit value cache refresh + self.assertNotIn("new_variable", set(proxy)) + # But an explicit refresh adds the new key + proxy.sync_frame_cache() + self.assertIn("new_variable", set(proxy)) + + def test_proxy_sizeof(self): + self.fail("TODO: Implement proxy sys.getsizeof() test") def test_active_frame_c_apis(self): # Use ctypes to access the C APIs under test diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 198a852ee197c9..61d49c63cc81b6 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -117,14 +117,15 @@ frame_getlocals(PyFrameObject *f, void *Py_UNUSED(ignored)) /* PEP 558: If this is an optimized frame, ensure f_locals at the Python * layer is a new fastlocalsproxy instance, while f_locals at the C * layer still refers to the underlying shared namespace mapping. + * + * To minimise runtime overhead when the frame value cache isn't used, + * each new proxy instance postpones refreshing the cache until the + * first operation that assumes the value cache is up to date. */ - if (PyFrame_FastToLocalsWithError(f) < 0) { - return NULL; - } f_locals_attr = _PyFastLocalsProxy_New((PyObject *) f); } else { // Share a direct locals reference for class and module scopes - f_locals_attr = _frame_get_updated_locals(f); + f_locals_attr = _frame_get_locals_mapping(f); if (f_locals_attr == NULL) { return NULL; } @@ -1324,11 +1325,37 @@ class fastlocalsproxy "fastlocalsproxyobject *" "&_PyFastLocalsProxy_Type" typedef struct { PyObject_HEAD PyFrameObject *frame; - //int frame_cache_refreshed; /* Assume cache is out of date if this is not set */ + int frame_cache_updated; /* Assume cache is out of date if this is not set */ } fastlocalsproxyobject; // PEP 558 TODO: Implement correct Python sizeof() support for fastlocalsproxyobject +static PyObject * +fastlocalsproxy_get_updated_value_cache(fastlocalsproxyobject *flp) +{ + // Retrieve the locals value cache from an optimised frame, + // ensuring it is up to date with the current state of the frame + assert(flp); + assert(flp->frame); + flp->frame_cache_updated = 1; + return _frame_get_updated_locals(flp->frame); +} + +static PyObject * +fastlocalsproxy_get_value_cache(fastlocalsproxyobject *flp) +{ + // Retrieve the locals value cache from an optimised frame. + // If this proxy hasn't previously updated the locals value cache, + // assume the cache may be out of date and update it first + assert(flp); + if (flp->frame_cache_updated) { + assert(flp->frame); + return _frame_get_locals_mapping(flp->frame); + } + return fastlocalsproxy_get_updated_value_cache(flp); +} + + static int fastlocalsproxy_init_fast_refs(fastlocalsproxyobject *flp) { @@ -1350,15 +1377,16 @@ static Py_ssize_t fastlocalsproxy_len(fastlocalsproxyobject *flp) { // Extra keys may have been added, and some variables may not have been - // bound yet, so use the dynamic snapshot on the frame rather than the + // bound yet, so use the value cache on the frame rather than the // keys in the fast locals reverse lookup mapping - // Assume f_locals snapshot is up to date (as actually checking is O(n)) - PyObject *locals = _frame_get_locals_mapping(flp->frame); + // As long as it has been updated at least once, assume value cache + // is up to date (as actually checking is O(n)) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); return PyObject_Size(locals); } static int -fastlocalsproxy_set_snapshot_entry(fastlocalsproxyobject *flp, PyObject *key, PyObject *value) +fastlocalsproxy_set_value_cache_entry(fastlocalsproxyobject *flp, PyObject *key, PyObject *value) { PyFrameObject *f = flp->frame; if (f->f_state == FRAME_CLEARED) { @@ -1450,7 +1478,7 @@ fastlocalsproxy_getitem(fastlocalsproxyobject *flp, PyObject *key) // or was only just converted since the last cache sync // Ensure the value cache is up to date if the frame is still live if (!PyErr_Occurred()) { - if (fastlocalsproxy_set_snapshot_entry(flp, key, value) != 0) { + if (fastlocalsproxy_set_value_cache_entry(flp, key, value) != 0) { return NULL; } } @@ -1477,7 +1505,7 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje if (fast_ref == NULL) { // No such local variable, delegate the request to the f_locals mapping // Used by pdb (at least) to store __return__ and __exception__ values - return fastlocalsproxy_set_snapshot_entry(flp, key, value); + return fastlocalsproxy_set_value_cache_entry(flp, key, value); } /* Key is a valid Python variable for the frame, so update that reference */ if (PyCell_Check(fast_ref)) { @@ -1485,7 +1513,7 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje int result = PyCell_Set(fast_ref, value); if (result == 0) { // Ensure the value cache is up to date if the frame is still live - result = fastlocalsproxy_set_snapshot_entry(flp, key, value); + result = fastlocalsproxy_set_value_cache_entry(flp, key, value); } return result; } @@ -1528,7 +1556,7 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje int result = PyCell_Set(target, value); if (result == 0) { // Ensure the value cache is up to date if the frame is still live - result = fastlocalsproxy_set_snapshot_entry(flp, key, value); + result = fastlocalsproxy_set_value_cache_entry(flp, key, value); } return result; } @@ -1538,7 +1566,7 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje Py_XINCREF(value); Py_XSETREF(fast_locals[offset], value); // Ensure the value cache is up to date if the frame is still live - return fastlocalsproxy_set_snapshot_entry(flp, key, value); + return fastlocalsproxy_set_value_cache_entry(flp, key, value); } static int @@ -1634,37 +1662,40 @@ static PyObject * fastlocalsproxy_keys(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { // Extra keys may have been added, and some variables may not have been - // bound yet, so use the dynamic snapshot on the frame rather than the + // bound yet, so use the value cache on the frame rather than the // keys in the fast locals reverse lookup mapping - // Assume f_locals snapshot is up to date (as actually checking is O(n)) - PyObject *locals = _frame_get_locals_mapping(flp->frame); + // As long as it has been updated at least once, assume value cache + // is up to date (as actually checking is O(n)) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); return PyDict_Keys(locals); } static PyObject * fastlocalsproxy_values(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { - // Need values, so use the dynamic snapshot on the frame - // Assume f_locals snapshot is up to date (as actually checking is O(n)) - PyObject *locals = _frame_get_locals_mapping(flp->frame); + // Need values, so use the value cache on the frame + // As long as it has been updated at least once, assume value cache + // is up to date (as actually checking is O(n)) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); return PyDict_Values(locals); } static PyObject * fastlocalsproxy_items(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { - // Need values, so use the dynamic snapshot on the frame - // Assume f_locals snapshot is up to date (as actually checking is O(n)) - PyObject *locals = _frame_get_locals_mapping(flp->frame); + // Need values, so use the value cache on the frame + // As long as it has been updated at least once, assume value cache + // is up to date (as actually checking is O(n)) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); return PyDict_Items(locals); } static PyObject * fastlocalsproxy_copy(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { - // Need values, so use the dynamic snapshot on the frame - // Ensure it is up to date, as checking is O(n) anyway - PyObject *locals = _frame_get_updated_locals(flp->frame); + // Need values, so use the value cache on the frame + // Ensure it is up to date, as copying the entire mapping is already O(n) + PyObject *locals = fastlocalsproxy_get_updated_value_cache(flp); return PyDict_Copy(locals); } @@ -1673,10 +1704,11 @@ fastlocalsproxy_reversed(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored { _Py_IDENTIFIER(__reversed__); // Extra keys may have been added, and some variables may not have been - // bound yet, so use the dynamic snapshot on the frame rather than the + // bound yet, so use the value cache on the frame rather than the // keys in the fast locals reverse lookup mapping - // Assume f_locals snapshot is up to date (as actually checking is O(n)) - PyObject *locals = _frame_get_locals_mapping(flp->frame); + // As long as it has been updated at least once, assume value cache + // is up to date (as actually checking is O(n)) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); return _PyObject_CallMethodIdNoArgs(locals, &PyId___reversed__); } @@ -1684,21 +1716,23 @@ static PyObject * fastlocalsproxy_getiter(fastlocalsproxyobject *flp) { // Extra keys may have been added, and some variables may not have been - // bound yet, so use the dynamic snapshot on the frame rather than the + // bound yet, so use the value cache on the frame rather than the // keys in the fast locals reverse lookup mapping - // Assume f_locals snapshot is up to date (as actually checking is O(n)) - PyObject *locals = _frame_get_locals_mapping(flp->frame); + // As long as it has been updated at least once, assume value cache + // is up to date (as actually checking is O(n)) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); return PyObject_GetIter(locals); } static PyObject * fastlocalsproxy_richcompare(fastlocalsproxyobject *flp, PyObject *w, int op) { - // Need values, so use the dynamic snapshot on the frame - // Assume f_locals snapshot is up to date, as even though the worst - // case comparison is O(n) to determine equality, there are O(1) shortcuts - // for inequality checks (i.e. different sizes) - PyObject *locals = _frame_get_locals_mapping(flp->frame); + // Need values, so use the value cache on the frame + // As long as it has been updated at least once, assume value cache + // is up to date, as even though the worst case comparison is O(n) to + // determine equality, there are O(1) shortcuts for inequality checks + // (i.e. different sizes) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); return PyObject_RichCompare(locals, w, op); } @@ -1723,7 +1757,7 @@ fastlocalsproxy_setdefault(PyObject *flp, PyObject *args, PyObject *kwargs) PyObject *value = NULL; - // PEP 558 TODO: implement this + // PEP 558 TODO: implement setdefault() on proxy objects PyErr_Format(PyExc_NotImplementedError, "FastLocalsProxy does not yet implement setdefault()"); return value; @@ -1788,7 +1822,7 @@ PyDoc_STRVAR(fastlocalsproxy_popitem__doc__, static PyObject * fastlocalsproxy_popitem(PyObject *flp, PyObject *Py_UNUSED(ignored)) { - // PEP 558 TODO: implement this + // PEP 558 TODO: implement popitem() on proxy objects PyErr_Format(PyExc_NotImplementedError, "FastLocalsProxy does not yet implement popitem()"); return NULL; @@ -1812,7 +1846,7 @@ PyDoc_STRVAR(fastlocalsproxy_clear__doc__, static PyObject * fastlocalsproxy_clear(register PyObject *flp, PyObject *Py_UNUSED(ignored)) { - // PEP 558 TODO: implement this + // PEP 558 TODO: implement clear() on proxy objects PyErr_Format(PyExc_NotImplementedError, "FastLocalsProxy does not yet implement clear()"); return NULL; @@ -1888,9 +1922,9 @@ fastlocalsproxy_repr(fastlocalsproxyobject *flp) static PyObject * fastlocalsproxy_str(fastlocalsproxyobject *flp) { - // Need values, so use the dynamic snapshot on the frame - // Ensure it is up to date, as displaying everything is O(n) anyway - PyObject *locals = _frame_get_updated_locals(flp->frame); + // Need values, so use the value cache on the frame + // Ensure it is up to date, as rendering the entire mapping is already O(n) + PyObject *locals = fastlocalsproxy_get_updated_value_cache(flp); return PyObject_Str(locals); } @@ -1939,6 +1973,7 @@ _PyFastLocalsProxy_New(PyObject *frame) return NULL; flp->frame = (PyFrameObject *) frame; Py_INCREF(flp->frame); + flp->frame_cache_updated = 0; _PyObject_GC_TRACK(flp); return (PyObject *)flp; } From ea5f943e683c558049a81e9214dd874edd7be08b Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 21 Aug 2021 17:54:58 +1000 Subject: [PATCH 55/66] Add dict union operations to proxy --- Include/cpython/frameobject.h | 3 +- Lib/test/test_frame.py | 21 +++- Objects/frameobject.c | 180 ++++++++++++++++++---------------- 3 files changed, 115 insertions(+), 89 deletions(-) diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 2e3ec431fd6fe6..799f81e6443a0a 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -84,8 +84,7 @@ PyAPI_FUNC(PyFrameObject *) PyFrame_GetBack(PyFrameObject *frame); // seems like a nice way to let folks write some useful debug assertions, // though. PyAPI_DATA(PyTypeObject) _PyFastLocalsProxy_Type; -#define _PyFastLocalsProxy_CheckExact(self) \ - (Py_TYPE(self) == &_PyFastLocalsProxy_Type) +#define _PyFastLocalsProxy_CheckExact(op) Py_IS_TYPE(op, &_PyFastLocalsProxy_Type) // Underlying implementation API supporting the stable PyLocals_*() APIs diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 7fb63f4dffcb41..e3db1c7437675d 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -240,7 +240,13 @@ def test_dict_query_operations(self): # Check reversed iteration (order should be reverse of forward iteration) self.assertEqual(list(reversed(proxy)), list(reversed(list(proxy)))) - self.fail("PEP 558 TODO: Implement proxy '|' operator test") + # Check dict union operations (these implicitly refresh the value cache) + extra_contents = dict(a=1, b=2) + expected_proxy_contents["extra_contents"] = extra_contents + self.check_proxy_contents(proxy | proxy, expected_proxy_contents) + self.assertIsInstance(proxy | proxy, dict) + self.check_proxy_contents(proxy | extra_contents, expected_proxy_contents | extra_contents) + self.check_proxy_contents(extra_contents | proxy, expected_proxy_contents | extra_contents) def test_dict_mutation_operations(self): # Check mutation of local variables via proxy @@ -285,8 +291,17 @@ def inner(): self.assertEqual(cell_variable, "set via proxy.update()") self.assertEqual(proxy["extra_variable"], "set via proxy.update()") - self.fail("PEP 558 TODO: Implement proxy '|=' operator test") - self.fail("PEP 558 TODO: Implement proxy update() test") + # Check updating all 3 kinds of variable via an in-place dict union + updated_keys = { + "unbound_local": "set via proxy |=", + "cell_variable": "set via proxy |=", + "extra_variable": "set via proxy |=", + } + proxy |= updated_keys + self.assertEqual(unbound_local, "set via proxy |=") + self.assertEqual(cell_variable, "set via proxy |=") + self.assertEqual(proxy["extra_variable"], "set via proxy |=") + self.fail("PEP 558 TODO: Implement proxy setdefault() test") self.fail("PEP 558 TODO: Implement proxy pop() test") self.fail("PEP 558 TODO: Implement proxy popitem() test") diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 61d49c63cc81b6..f2dc6a0969e7ad 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1587,28 +1587,46 @@ static PyMappingMethods fastlocalsproxy_as_mapping = { (objobjargproc)fastlocalsproxy_setitem, /* mp_ass_subscript */ }; +static int mutablemapping_update_arg(PyObject*, PyObject*); + static PyObject * -fastlocalsproxy_or(PyObject *Py_UNUSED(left), PyObject *Py_UNUSED(right)) +fastlocalsproxy_or(PyObject *left, PyObject *right) { - // Delegate to the other operand to determine the return type - Py_RETURN_NOTIMPLEMENTED; + // Binary union operations are delegated to the frame value cache + // Ensure it is up to date, as the union operation is already O(m+n) + int repeated_operand = (left == right); + + if (_PyFastLocalsProxy_CheckExact(left)) { + left = fastlocalsproxy_get_updated_value_cache((fastlocalsproxyobject *)left); + } + if (_PyFastLocalsProxy_CheckExact(right)) { + if (repeated_operand) { + right = left; + } else { + right = fastlocalsproxy_get_updated_value_cache((fastlocalsproxyobject *)right); + } + } + return PyNumber_Or(left, right); } static PyObject * -fastlocalsproxy_ior(PyObject *self, PyObject *Py_UNUSED(other)) +fastlocalsproxy_ior(PyObject *self, PyObject *other) { - // PEP 558 TODO: Support |= to update from arbitrary mappings - // Check the latest mutablemapping_update code for __ior__ support - PyErr_Format(PyExc_NotImplementedError, - "FastLocalsProxy does not yet implement __ior__"); - return NULL; + // In-place union operations are equivalent to an update() method call + if (mutablemapping_update_arg(self, other) < 0) { + return NULL; + } + Py_INCREF(self); + return self; } +/* tp_as_number */ static PyNumberMethods fastlocalsproxy_as_number = { .nb_or = fastlocalsproxy_or, .nb_inplace_or = fastlocalsproxy_ior, }; + static int fastlocalsproxy_contains(fastlocalsproxyobject *flp, PyObject *key) { @@ -2110,17 +2128,78 @@ mutablemapping_add_pairs(PyObject *self, PyObject *pairs) return 0; } -static PyObject * -mutablemapping_update(PyObject *self, PyObject *args, PyObject *kwargs) +static int +mutablemapping_update_arg(PyObject *self, PyObject *arg) { int res = 0; - Py_ssize_t len; - _Py_IDENTIFIER(items); + if (PyDict_CheckExact(arg)) { + PyObject *items = PyDict_Items(arg); + if (items == NULL) { + return -1; + } + res = mutablemapping_add_pairs(self, items); + Py_DECREF(items); + return res; + } _Py_IDENTIFIER(keys); + PyObject *func; + if (_PyObject_LookupAttrId(arg, &PyId_keys, &func) < 0) { + return -1; + } + if (func != NULL) { + PyObject *keys = _PyObject_CallNoArg(func); + Py_DECREF(func); + if (keys == NULL) { + return -1; + } + PyObject *iterator = PyObject_GetIter(keys); + Py_DECREF(keys); + if (iterator == NULL) { + return -1; + } + PyObject *key; + while (res == 0 && (key = PyIter_Next(iterator))) { + PyObject *value = PyObject_GetItem(arg, key); + if (value != NULL) { + res = PyObject_SetItem(self, key, value); + Py_DECREF(value); + } + else { + res = -1; + } + Py_DECREF(key); + } + Py_DECREF(iterator); + if (res != 0 || PyErr_Occurred()) { + return -1; + } + return 0; + } + _Py_IDENTIFIER(items); + if (_PyObject_LookupAttrId(arg, &PyId_items, &func) < 0) { + return -1; + } + if (func != NULL) { + PyObject *items = _PyObject_CallNoArg(func); + Py_DECREF(func); + if (items == NULL) { + return -1; + } + res = mutablemapping_add_pairs(self, items); + Py_DECREF(items); + return res; + } + res = mutablemapping_add_pairs(self, arg); + return res; +} +static PyObject * +mutablemapping_update(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int res; /* first handle args, if any */ assert(args == NULL || PyTuple_Check(args)); - len = (args != NULL) ? PyTuple_GET_SIZE(args) : 0; + Py_ssize_t len = (args != NULL) ? PyTuple_GET_SIZE(args) : 0; if (len > 1) { const char *msg = "update() takes at most 1 positional argument (%zd given)"; PyErr_Format(PyExc_TypeError, msg, len); @@ -2128,83 +2207,16 @@ mutablemapping_update(PyObject *self, PyObject *args, PyObject *kwargs) } if (len) { - PyObject *func; PyObject *other = PyTuple_GET_ITEM(args, 0); /* borrowed reference */ assert(other != NULL); Py_INCREF(other); - if (PyDict_CheckExact(other)) { - PyObject *items = PyDict_Items(other); - Py_DECREF(other); - if (items == NULL) - return NULL; - res = mutablemapping_add_pairs(self, items); - Py_DECREF(items); - if (res == -1) - return NULL; - goto handle_kwargs; - } - - if (_PyObject_LookupAttrId(other, &PyId_keys, &func) < 0) { - Py_DECREF(other); - return NULL; - } - if (func != NULL) { - PyObject *keys, *iterator, *key; - keys = _PyObject_CallNoArg(func); - Py_DECREF(func); - if (keys == NULL) { - Py_DECREF(other); - return NULL; - } - iterator = PyObject_GetIter(keys); - Py_DECREF(keys); - if (iterator == NULL) { - Py_DECREF(other); - return NULL; - } - while (res == 0 && (key = PyIter_Next(iterator))) { - PyObject *value = PyObject_GetItem(other, key); - if (value != NULL) { - res = PyObject_SetItem(self, key, value); - Py_DECREF(value); - } - else { - res = -1; - } - Py_DECREF(key); - } - Py_DECREF(other); - Py_DECREF(iterator); - if (res != 0 || PyErr_Occurred()) - return NULL; - goto handle_kwargs; - } - - if (_PyObject_LookupAttrId(other, &PyId_items, &func) < 0) { - Py_DECREF(other); - return NULL; - } - if (func != NULL) { - PyObject *items; - Py_DECREF(other); - items = _PyObject_CallNoArg(func); - Py_DECREF(func); - if (items == NULL) - return NULL; - res = mutablemapping_add_pairs(self, items); - Py_DECREF(items); - if (res == -1) - return NULL; - goto handle_kwargs; - } - - res = mutablemapping_add_pairs(self, other); + res = mutablemapping_update_arg(self, other); Py_DECREF(other); - if (res != 0) + if (res < 0) { return NULL; + } } - handle_kwargs: /* now handle kwargs */ assert(kwargs == NULL || PyDict_Check(kwargs)); if (kwargs != NULL && PyDict_GET_SIZE(kwargs)) { From fcf99cac737d5a046b42ba515847e41697f3da9b Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 21 Aug 2021 20:25:15 +1000 Subject: [PATCH 56/66] Implement and test locals proxy clear() method --- Lib/test/test_frame.py | 41 ++++++++++++++-- Objects/frameobject.c | 105 ++++++++++++++++++++++++++++++++--------- 2 files changed, 121 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index e3db1c7437675d..8536df0b2e4ddd 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -259,7 +259,7 @@ def test_dict_mutation_operations(self): self.assertEqual(unbound_local, "set via proxy") del proxy["unbound_local"] with self.assertRaises(UnboundLocalError): - print(unbound_local) + unbound_local # Check mutation of cell variables via proxy cell_variable = None proxy["cell_variable"] = "set via proxy" @@ -269,7 +269,7 @@ def inner(): self.assertEqual(inner(), "set via proxy") del proxy["cell_variable"] with self.assertRaises(UnboundLocalError): - print(cell_variable) + cell_variable with self.assertRaises(NameError): inner() # Check storage of additional variables in the frame value cache via proxy @@ -302,10 +302,45 @@ def inner(): self.assertEqual(cell_variable, "set via proxy |=") self.assertEqual(proxy["extra_variable"], "set via proxy |=") + # Check clearing all variables via the proxy + # Use a nested generator to allow the test case reference to be + # restored even after the frame variables are cleared + def clear_frame_via_proxy(test_case_arg): + inner_proxy = sys._getframe().f_locals + inner_proxy["extra_variable"] = "added via inner_proxy" + test_case_arg.assertEqual(inner_proxy, { + "inner_proxy": inner_proxy, + "cell_variable": cell_variable, + "test_case_arg": test_case_arg, + "extra_variable": "added via inner_proxy", + }) + inner_proxy.clear() + test_case = yield None + with test_case.assertRaises(UnboundLocalError): + inner_proxy + with test_case.assertRaises(UnboundLocalError): + test_case_arg + with test_case.assertRaises(NameError): + cell_variable + inner_proxy = sys._getframe().f_locals + test_case.assertNotIn("extra_variable", inner_proxy) + # Clearing the inner frame even clears the cell in the outer frame + clear_iter = clear_frame_via_proxy(self) + next(clear_iter) + with self.assertRaises(UnboundLocalError): + cell_variable + # Run the final checks in the inner frame + try: + clear_iter.send(self) + self.fail("Inner proxy clearing iterator didn't stop") + except StopIteration: + pass + + + self.fail("PEP 558 TODO: Implement proxy setdefault() test") self.fail("PEP 558 TODO: Implement proxy pop() test") self.fail("PEP 558 TODO: Implement proxy popitem() test") - self.fail("PEP 558 TODO: Implement proxy clear() test") def test_sync_frame_cache(self): proxy = sys._getframe().f_locals diff --git a/Objects/frameobject.c b/Objects/frameobject.c index f2dc6a0969e7ad..29b8020f787630 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1426,19 +1426,20 @@ fastlocalsproxy_getitem(fastlocalsproxyobject *flp, PyObject *key) } return PyObject_GetItem(locals, key); } - /* Key is a valid Python variable for the frame, so retrieve the value */ + // If the frame has been cleared, disallow access to locals and cell variables + PyFrameObject *f = flp->frame; + if (f->f_state == FRAME_CLEARED) { + PyErr_Format(PyExc_RuntimeError, + "Fast locals proxy attempted to read from cleared frame (%R)", f); + return NULL; + } + // Key is a valid Python variable for the frame, so retrieve the value PyObject *value = NULL; if (PyCell_Check(fast_ref)) { - // Closure cells can be queried even after the frame terminates + // Closure cells are accessed directly via the fast refs mapping value = PyCell_GET(fast_ref); } else { - PyFrameObject *f = flp->frame; - if (f->f_state == FRAME_CLEARED) { - PyErr_Format(PyExc_RuntimeError, - "Fast locals proxy attempted to read from cleared frame (%R)", f); - return NULL; - } - /* Fast ref is a Python int mapping into the fast locals array */ + // Fast ref is a Python int mapping into the fast locals array assert(PyLong_CheckExact(fast_ref)); Py_ssize_t offset = PyLong_AsSsize_t(fast_ref); if (offset < 0) { @@ -1507,9 +1508,16 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje // Used by pdb (at least) to store __return__ and __exception__ values return fastlocalsproxy_set_value_cache_entry(flp, key, value); } - /* Key is a valid Python variable for the frame, so update that reference */ + // If the frame has been cleared, disallow access to locals and cell variables + PyFrameObject *f = flp->frame; + if (f->f_state == FRAME_CLEARED) { + PyErr_Format(PyExc_RuntimeError, + "Fast locals proxy attempted to write to cleared frame (%R)", f); + return -1; + } + // Key is a valid Python variable for the frame, so update that reference if (PyCell_Check(fast_ref)) { - // Closure cells can be updated even after the frame terminates + // Closure cells are accessed directly via the fast refs mapping int result = PyCell_Set(fast_ref, value); if (result == 0) { // Ensure the value cache is up to date if the frame is still live @@ -1517,13 +1525,7 @@ fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObje } return result; } - PyFrameObject *f = flp->frame; - if (f->f_state == FRAME_CLEARED) { - PyErr_Format(PyExc_RuntimeError, - "Fast locals proxy attempted to write to cleared frame (%R)", f); - return -1; - } - /* Fast ref is a Python int mapping into the fast locals array */ + // Fast ref is a Python int mapping into the fast locals array assert(PyLong_CheckExact(fast_ref)); Py_ssize_t offset = PyLong_AsSsize_t(fast_ref); if (offset < 0) { @@ -1864,10 +1866,69 @@ PyDoc_STRVAR(fastlocalsproxy_clear__doc__, static PyObject * fastlocalsproxy_clear(register PyObject *flp, PyObject *Py_UNUSED(ignored)) { - // PEP 558 TODO: implement clear() on proxy objects - PyErr_Format(PyExc_NotImplementedError, - "FastLocalsProxy does not yet implement clear()"); - return NULL; + /* Merge fast locals into f->f_locals */ + PyFrameObject *f; + PyObject *locals; + + assert(flp); + f = ((fastlocalsproxyobject *)flp)->frame; + if (f == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + // Clear any local and still referenced cell variables + // Nothing to do in this step if the frame itself has already been cleared + if (f->f_state != FRAME_CLEARED) { + PyCodeObject *co = _PyFrame_GetCode(f); + PyObject **fast = f->f_localsptr; + // Fast locals proxies only get created for optimised frames + assert(co); + assert(co->co_flags & CO_OPTIMIZED); + assert(fast); + for (int i = 0; i < co->co_nlocalsplus; i++) { + _PyLocal_VarKind kind = _PyLocal_GetVarKind(co->co_localspluskinds, i); + + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyObject *value = fast[i]; + if (kind & CO_FAST_FREE) { + // The cell was set by _PyEval_MakeFrameVector() from + // the function's closure. + assert(value != NULL && PyCell_Check(value)); + if (PyCell_Set(value, NULL)) { + return NULL; + } + } + else if (kind & CO_FAST_CELL) { + // If the cell has already been created, unbind its reference, + // otherwise clear its initial value (if any) + if (value != NULL) { + if (PyCell_Check(value) && + _PyFrame_OpAlreadyRan(f, MAKE_CELL, i)) { + // (likely) MAKE_CELL must have executed already. + if (PyCell_Set(value, NULL)) { + return NULL; + } + } else { + // Clear the initial value + Py_CLEAR(fast[i]); + } + } + } else if (value != NULL) { + // Clear local variable reference + Py_CLEAR(fast[i]); + } + } + } + + // Finally, clear the frame value cache (including any extra variables) + locals = _frame_get_locals_mapping(f); + if (locals == NULL) { + return NULL; + } + PyDict_Clear(locals); + + Py_RETURN_NONE; } From 16e05810d412317b8c53014b6fa0a0eae47eb90a Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 21 Aug 2021 20:27:58 +1000 Subject: [PATCH 57/66] Remove pointless print() call --- Lib/test/test_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 8536df0b2e4ddd..05b778027605d8 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -276,7 +276,7 @@ def inner(): proxy["extra_variable"] = "added via proxy" self.assertEqual(proxy["extra_variable"], "added via proxy") with self.assertRaises(NameError): - print(extra_variable) + extra_variable del proxy["extra_variable"] self.assertNotIn("extra_variable", proxy) From c35694981bc3c96b4dede707a476d2efd2958393 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 21 Aug 2021 21:19:46 +1000 Subject: [PATCH 58/66] Implement proxy pop() tests --- Lib/test/test_frame.py | 18 +++++++++++++++++- Objects/frameobject.c | 2 -- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 05b778027605d8..fde691a8c66eab 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -280,6 +280,23 @@ def inner(): del proxy["extra_variable"] self.assertNotIn("extra_variable", proxy) + # Check pop() on all 3 kinds of variable + unbound_local = "set directly" + self.assertEqual(proxy.pop("unbound_local"), "set directly") + self.assertIs(proxy.pop("unbound_local", None), None) + with self.assertRaises(KeyError): + proxy.pop("unbound_local") + cell_variable = "set directly" + self.assertEqual(proxy.pop("cell_variable"), "set directly") + self.assertIs(proxy.pop("cell_variable", None), None) + with self.assertRaises(KeyError): + proxy.pop("cell_variable") + proxy["extra_variable"] = "added via proxy" + self.assertEqual(proxy.pop("extra_variable"), "added via proxy") + self.assertIs(proxy.pop("extra_variable", None), None) + with self.assertRaises(KeyError): + proxy.pop("extra_variable") + # Check updating all 3 kinds of variable via update() updated_keys = { "unbound_local": "set via proxy.update()", @@ -339,7 +356,6 @@ def clear_frame_via_proxy(test_case_arg): self.fail("PEP 558 TODO: Implement proxy setdefault() test") - self.fail("PEP 558 TODO: Implement proxy pop() test") self.fail("PEP 558 TODO: Implement proxy popitem() test") def test_sync_frame_cache(self): diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 29b8020f787630..d5e36eff75f986 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1411,7 +1411,6 @@ fastlocalsproxy_set_value_cache_entry(fastlocalsproxyobject *flp, PyObject *key, static PyObject * fastlocalsproxy_getitem(fastlocalsproxyobject *flp, PyObject *key) { - // PEP 558 TODO: try to factor out the common get/set key lookup code assert(flp); if (fastlocalsproxy_init_fast_refs(flp) != 0) { return NULL; @@ -1497,7 +1496,6 @@ fastlocalsproxy_getitem(fastlocalsproxyobject *flp, PyObject *key) static int fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObject *value) { - // PEP 558 TODO: try to factor out the common get/set key lookup code assert(flp); if (fastlocalsproxy_init_fast_refs(flp) != 0) { return -1; From 8a4e7887cf38102cf0f3519cd1d1142c7217c0aa Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 21 Aug 2021 21:56:16 +1000 Subject: [PATCH 59/66] Implement and test proxy popitem() --- Lib/test/test_frame.py | 20 ++++++++++++++++++++ Objects/frameobject.c | 27 ++++++++++++++++++++++----- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index fde691a8c66eab..1bcb1b87db5daa 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -358,6 +358,26 @@ def clear_frame_via_proxy(test_case_arg): self.fail("PEP 558 TODO: Implement proxy setdefault() test") self.fail("PEP 558 TODO: Implement proxy popitem() test") + def test_popitem(self): + # Check popitem() in a controlled inner frame + # This is a separate test case so it can be skipped if the test case + # detects that something is injecting extra keys into the frame state + if len(sys._getframe().f_locals) != 1: + self.skipTest("Locals other than 'self' detected, test case will be unreliable") + + # With no local variables, trying to pop one should fail + def popitem_exception(): + return sys._getframe().f_locals.popitem() + with self.assertRaises(KeyError): + popitem_exception() + + # With exactly one local variable, it should be popped + def popitem_result(arg="only proxy entry"): + return sys._getframe().f_locals.popitem(), list(sys._getframe().f_locals) + popped_item, remaining_vars = popitem_result() + self.assertEqual(popped_item, ("arg", "only proxy entry")) + self.assertEqual(remaining_vars, []) + def test_sync_frame_cache(self): proxy = sys._getframe().f_locals self.assertEqual(len(proxy), 2) # Trigger the initial implicit cache update diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d5e36eff75f986..a157180d06a61c 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1835,15 +1835,32 @@ fastlocalsproxy_pop(PyObject *flp, PyObject *args, PyObject *kwargs) PyDoc_STRVAR(fastlocalsproxy_popitem__doc__, "flp.popitem() -> (k, v), remove and return some (key, value) pair as a\n\ - 2-tuple; but raise KeyError if D is empty."); + 2-tuple; but raise KeyError if proxy has no bound values."); static PyObject * fastlocalsproxy_popitem(PyObject *flp, PyObject *Py_UNUSED(ignored)) { - // PEP 558 TODO: implement popitem() on proxy objects - PyErr_Format(PyExc_NotImplementedError, - "FastLocalsProxy does not yet implement popitem()"); - return NULL; + _Py_IDENTIFIER(popitem); + // Need values, so use the value cache on the frame + // As long as it has been updated at least once, assume value cache + // is up to date (as actually checking is O(n)) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); + if (locals == NULL) { + return NULL; + } + PyObject *result = _PyObject_CallMethodIdNoArgs(locals, &PyId_popitem); + if (result != NULL) { + // We popped a key from the cache, so ensure it is also cleared on the frame + assert(PyTuple_CheckExact(result)); + assert(PyTuple_GET_SIZE(result) == 2); + PyObject *key = PyTuple_GET_ITEM(result, 0); + if (fastlocalsproxy_delitem(flp, key)) { + Py_DECREF(result); + return NULL; + } + } + + return result; } /* update() */ From 706eec4076ae902237fa508415e81182dc7883c0 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 21 Aug 2021 22:04:15 +1000 Subject: [PATCH 60/66] Test popitem with cells and extra variables --- Lib/test/test_frame.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 1bcb1b87db5daa..91fc0a93a3ef78 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -372,12 +372,31 @@ def popitem_exception(): popitem_exception() # With exactly one local variable, it should be popped - def popitem_result(arg="only proxy entry"): + def popitem_local(arg="only proxy entry"): return sys._getframe().f_locals.popitem(), list(sys._getframe().f_locals) - popped_item, remaining_vars = popitem_result() + popped_item, remaining_vars = popitem_local() self.assertEqual(popped_item, ("arg", "only proxy entry")) self.assertEqual(remaining_vars, []) + # With exactly one cell variable, it should be popped + cell_variable = initial_cell_ref = "only proxy entry" + def popitem_cell(): + return cell_variable, sys._getframe().f_locals.popitem(), list(sys._getframe().f_locals) + cell_ref, popped_item, remaining_vars = popitem_cell() + self.assertEqual(popped_item, ("cell_variable", "only proxy entry")) + self.assertEqual(remaining_vars, []) + self.assertIs(cell_ref, initial_cell_ref) + with self.assertRaises(UnboundLocalError): + cell_variable + + # With exactly one extra variable, it should be popped + def popitem_extra(): + sys._getframe().f_locals["extra_variable"] = "only proxy entry" + return sys._getframe().f_locals.popitem(), list(sys._getframe().f_locals) + popped_item, remaining_vars = popitem_extra() + self.assertEqual(popped_item, ("extra_variable", "only proxy entry")) + self.assertEqual(remaining_vars, []) + def test_sync_frame_cache(self): proxy = sys._getframe().f_locals self.assertEqual(len(proxy), 2) # Trigger the initial implicit cache update From 35a017cfdf7697af26212d156e0111e506092e89 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 21 Aug 2021 23:04:13 +1000 Subject: [PATCH 61/66] Implement and test setdefault() --- Lib/test/test_frame.py | 24 +++++++++++++++---- Objects/frameobject.c | 52 +++++++++++++++++++++++++++--------------- 2 files changed, 53 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 91fc0a93a3ef78..9260cc830f0f14 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -297,6 +297,25 @@ def inner(): with self.assertRaises(KeyError): proxy.pop("extra_variable") + # Check setdefault() on all 3 kinds of variable + expected_value = "set via setdefault()" + self.assertEqual(proxy.setdefault("unbound_local", expected_value), expected_value) + self.assertEqual(proxy.setdefault("unbound_local", "ignored"), expected_value) + del unbound_local + self.assertEqual(proxy.setdefault("unbound_local"), None) + self.assertIs(unbound_local, None) + self.assertEqual(proxy.setdefault("cell_variable", expected_value), expected_value) + self.assertEqual(proxy.setdefault("cell_variable", "ignored"), expected_value) + del cell_variable + self.assertEqual(proxy.setdefault("cell_variable"), None) + self.assertIs(cell_variable, None) + self.assertEqual(proxy.setdefault("extra_variable", expected_value), expected_value) + self.assertEqual(proxy.setdefault("extra_variable", "ignored"), expected_value) + del proxy["extra_variable"] + self.assertEqual(proxy.setdefault("extra_variable"), None) + self.assertIs(cell_variable, None) + + # Check updating all 3 kinds of variable via update() updated_keys = { "unbound_local": "set via proxy.update()", @@ -353,11 +372,6 @@ def clear_frame_via_proxy(test_case_arg): except StopIteration: pass - - - self.fail("PEP 558 TODO: Implement proxy setdefault() test") - self.fail("PEP 558 TODO: Implement proxy popitem() test") - def test_popitem(self): # Check popitem() in a controlled inner frame # This is a separate test case so it can be skipped if the test case diff --git a/Objects/frameobject.c b/Objects/frameobject.c index a157180d06a61c..41ee43149626fe 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1757,31 +1757,48 @@ fastlocalsproxy_richcompare(fastlocalsproxyobject *flp, PyObject *w, int op) /* setdefault() */ PyDoc_STRVAR(fastlocalsproxy_setdefault__doc__, -"flp.setdefault(k[, d=None]) -> v, Insert key with a value of default if key\n\ - is not in the dictionary.\n\n\ - Return the value for key if key is in the dictionary, else default."); +"flp.setdefault(k[, d=None]) -> v, Bind key to given default if key\n\ + is not already bound to a value.\n\n\ + Return the value for key if key is already bound, else default."); + + +static PyObject * +_fastlocalsproxy_setdefault_impl(fastlocalsproxyobject *flp, PyObject *key, PyObject *failobj) +{ + assert(flp); + if (fastlocalsproxy_init_fast_refs(flp) != 0) { + return NULL; + } + + PyObject *value = fastlocalsproxy_getitem(flp, key); + if (value == NULL && PyErr_ExceptionMatches(PyExc_KeyError)) { + // Given key is currently unbound, so bind it to the specified default + // and return that object + PyErr_Clear(); + value = failobj; + if (fastlocalsproxy_setitem(flp, key, value)) { + return NULL; + } + Py_INCREF(value); + } + return value; +} static PyObject * fastlocalsproxy_setdefault(PyObject *flp, PyObject *args, PyObject *kwargs) { static char *kwlist[] = {"key", "default", 0}; - PyObject *key, *failobj = NULL; + PyObject *key, *failobj = Py_None; /* borrowed */ - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O:pop", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O:setdefault", kwlist, &key, &failobj)) { return NULL; } - PyObject *value = NULL; - - // PEP 558 TODO: implement setdefault() on proxy objects - PyErr_Format(PyExc_NotImplementedError, - "FastLocalsProxy does not yet implement setdefault()"); - return value; + return _fastlocalsproxy_setdefault_impl((fastlocalsproxyobject *)flp, key, failobj); } - /* pop() */ PyDoc_STRVAR(fastlocalsproxy_pop__doc__, @@ -1790,7 +1807,7 @@ PyDoc_STRVAR(fastlocalsproxy_pop__doc__, is raised."); static PyObject * -_fastlocalsproxy_popkey(fastlocalsproxyobject *flp, PyObject *key, PyObject *failobj) +_fastlocalsproxy_pop_impl(fastlocalsproxyobject *flp, PyObject *key, PyObject *failobj) { // TODO: Similar to the odict implementation, the fast locals proxy // could benefit from an internal API that accepts already calculated @@ -1828,17 +1845,17 @@ fastlocalsproxy_pop(PyObject *flp, PyObject *args, PyObject *kwargs) return NULL; } - return _fastlocalsproxy_popkey((fastlocalsproxyobject *)flp, key, failobj); + return _fastlocalsproxy_pop_impl((fastlocalsproxyobject *)flp, key, failobj); } /* popitem() */ PyDoc_STRVAR(fastlocalsproxy_popitem__doc__, -"flp.popitem() -> (k, v), remove and return some (key, value) pair as a\n\ - 2-tuple; but raise KeyError if proxy has no bound values."); +"flp.popitem() -> (k, v), unbind and return some (key, value) pair as a\n\ + 2-tuple; but raise KeyError if underlying frame has no bound values."); static PyObject * -fastlocalsproxy_popitem(PyObject *flp, PyObject *Py_UNUSED(ignored)) +fastlocalsproxy_popitem(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) { _Py_IDENTIFIER(popitem); // Need values, so use the value cache on the frame @@ -1904,7 +1921,6 @@ fastlocalsproxy_clear(register PyObject *flp, PyObject *Py_UNUSED(ignored)) for (int i = 0; i < co->co_nlocalsplus; i++) { _PyLocal_VarKind kind = _PyLocal_GetVarKind(co->co_localspluskinds, i); - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); PyObject *value = fast[i]; if (kind & CO_FAST_FREE) { // The cell was set by _PyEval_MakeFrameVector() from From 06c406c9338e5bfe74de50a1307ba8bdebad3a6c Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 21 Aug 2021 23:04:37 +1000 Subject: [PATCH 62/66] Implement and test proxy __sizeof__() --- Lib/test/test_frame.py | 7 ++++++- Objects/frameobject.c | 22 +++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 9260cc830f0f14..d56ee86bb31ded 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -421,8 +421,13 @@ def test_sync_frame_cache(self): proxy.sync_frame_cache() self.assertIn("new_variable", set(proxy)) + @support.cpython_only def test_proxy_sizeof(self): - self.fail("TODO: Implement proxy sys.getsizeof() test") + # Proxy should only be storing a frame reference and the flag that + # indicates whether or not the proxy has refreshed the value cache + proxy = sys._getframe().f_locals + expected_size = support.calcobjsize("Pi") + support.check_sizeof(self, proxy, expected_size) def test_active_frame_c_apis(self): # Use ctypes to access the C APIs under test diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 41ee43149626fe..e234c1c8d424b8 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1328,7 +1328,13 @@ typedef struct { int frame_cache_updated; /* Assume cache is out of date if this is not set */ } fastlocalsproxyobject; -// PEP 558 TODO: Implement correct Python sizeof() support for fastlocalsproxyobject +static PyObject * +fastlocalsproxy_sizeof(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) +{ + Py_ssize_t res; + res = sizeof(fastlocalsproxyobject); + return PyLong_FromSsize_t(res); +} static PyObject * fastlocalsproxy_get_updated_value_cache(fastlocalsproxyobject *flp) @@ -1979,20 +1985,22 @@ fastlocalsproxy_sync_frame_cache(register PyObject *self, PyObject *Py_UNUSED(ig static PyMethodDef fastlocalsproxy_methods[] = { {"get", (PyCFunction)(void(*)(void))fastlocalsproxy_get, METH_FASTCALL, - PyDoc_STR("D.get(k[,d]) -> D[k] if k in D, else d." + PyDoc_STR("P.get(k[,d]) -> P[k] if k in P, else d." " d defaults to None.")}, {"keys", (PyCFunction)fastlocalsproxy_keys, METH_NOARGS, - PyDoc_STR("D.keys() -> virtual set of D's keys")}, + PyDoc_STR("P.keys() -> virtual set of proxy's bound keys")}, {"values", (PyCFunction)fastlocalsproxy_values, METH_NOARGS, - PyDoc_STR("D.values() -> virtual multiset of D's values")}, + PyDoc_STR("P.values() -> virtual multiset of proxy's bound values")}, {"items", (PyCFunction)fastlocalsproxy_items, METH_NOARGS, - PyDoc_STR("D.items() -> virtual set of D's (key, value) pairs, as 2-tuples")}, + PyDoc_STR("P.items() -> virtual set of P's (key, value) pairs, as 2-tuples")}, {"copy", (PyCFunction)fastlocalsproxy_copy, METH_NOARGS, - PyDoc_STR("D.copy() -> a shallow copy of D as a regular dict")}, + PyDoc_STR("P.copy() -> a shallow copy of P as a regular dict")}, {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {"__reversed__", (PyCFunction)fastlocalsproxy_reversed, METH_NOARGS, - PyDoc_STR("D.__reversed__() -> reverse iterator over D's keys")}, + PyDoc_STR("P.__reversed__() -> reverse iterator over P's keys")}, + {"__sizeof__", (PyCFunction)fastlocalsproxy_sizeof, METH_NOARGS, + PyDoc_STR("P.__sizeof__: size of P in memory, in bytes (excludes frame)")}, // PEP 558 TODO: Convert METH_VARARGS/METH_KEYWORDS methods to METH_FASTCALL {"setdefault", (PyCFunction)(void(*)(void))fastlocalsproxy_setdefault, METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_setdefault__doc__}, From 31493b98315d72850f92eb1935fcc4faa99773d2 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 21 Aug 2021 23:18:23 +1000 Subject: [PATCH 63/66] Add C API test for the LocalsToFast exception --- Lib/test/test_frame.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index d56ee86bb31ded..f41b24ddb6aebb 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -539,6 +539,21 @@ class ExecFrame: locals_view = PyFrame_GetLocalsView(cls_frame) self.assertIsInstance(locals_view, types.MappingProxyType) + def test_locals_to_fast_error(self): + # Use ctypes to access the C APIs under test + ctypes = import_helper.import_module('ctypes') + Py_IncRef = ctypes.pythonapi.Py_IncRef + PyFrame_LocalsToFast = ctypes.pythonapi.PyFrame_LocalsToFast + PyFrame_LocalsToFast.argtypes = (ctypes.py_object, ctypes.c_int) + frame = sys._getframe() + # Ensure the error message recommends the replacement API + replacement_api = r'PyObject_GetAttrString\(frame, "f_locals"\)' + with self.assertRaisesRegex(RuntimeError, replacement_api): + PyFrame_LocalsToFast(frame, 0) + # Ensure the error is still raised when the "clear" parameter is set + with self.assertRaisesRegex(RuntimeError, replacement_api): + PyFrame_LocalsToFast(frame, 1) + class ReprTest(unittest.TestCase): """ From b587a4170179d528d7f699c2a7315c458c807db9 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 24 Aug 2021 00:29:43 +1000 Subject: [PATCH 64/66] Force enum size --- Include/ceval.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Include/ceval.h b/Include/ceval.h index 7131707f8a28be..ab7c64b80afdc2 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -55,7 +55,8 @@ PyAPI_FUNC(PyObject *) PyLocals_GetView(void); typedef enum { PyLocals_UNDEFINED = -1, // Indicates error (e.g. no thread state defined) PyLocals_DIRECT_REFERENCE = 0, - PyLocals_SHALLOW_COPY = 1 + PyLocals_SHALLOW_COPY = 1, + _PyLocals_ENSURE_32BIT_ENUM = 2147483647 } PyLocals_Kind; PyAPI_FUNC(PyLocals_Kind) PyLocals_GetKind(void); From e1b505d9c4e8b9ed0f36eb8b2046f39c45c5a832 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 24 Aug 2021 00:29:53 +1000 Subject: [PATCH 65/66] Clarify code comment --- Objects/frameobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index e234c1c8d424b8..baab2014dff872 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -800,7 +800,7 @@ frame_tp_clear(PyFrameObject *f) Py_CLEAR(f->f_trace); Py_CLEAR(f->f_fast_refs); PyCodeObject *co = _PyFrame_GetCode(f); - /* locals */ + /* fast locals */ for (int i = 0; i < co->co_nlocalsplus; i++) { Py_CLEAR(f->f_localsptr[i]); } From 2b27389f10a282901b7c78b7758691501c7e7a2f Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Thu, 26 Aug 2021 18:55:58 +1000 Subject: [PATCH 66/66] Keep track of defined names even on cleared frames --- Objects/frameobject.c | 83 ++++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index baab2014dff872..3c2231834551c5 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -798,11 +798,17 @@ frame_tp_clear(PyFrameObject *f) f->f_state = FRAME_CLEARED; Py_CLEAR(f->f_trace); - Py_CLEAR(f->f_fast_refs); PyCodeObject *co = _PyFrame_GetCode(f); /* fast locals */ for (int i = 0; i < co->co_nlocalsplus; i++) { Py_CLEAR(f->f_localsptr[i]); + if (f->f_fast_refs != NULL) { + // Keep a record of the names defined on the code object, + // but don't keep any cells alive and discard the slot locations + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + assert(name); + PyDict_SetItem(f->f_fast_refs, name, Py_None); + } } /* stack */ @@ -1195,45 +1201,48 @@ _PyFrame_BuildFastRefs(PyFrameObject *f) return NULL; } - if (f->f_state != FRAME_CLEARED) { - for (int i = 0; i < co->co_nlocalsplus; i++) { - _PyLocal_VarKind kind = _PyLocal_GetVarKind(co->co_localspluskinds, i); - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject *target = NULL; - if (kind & CO_FAST_FREE) { - // Reference to closure cell, save it as the proxy target - target = fast_locals[i]; - assert(target != NULL && PyCell_Check(target)); + for (int i = 0; i < co->co_nlocalsplus; i++) { + _PyLocal_VarKind kind = _PyLocal_GetVarKind(co->co_localspluskinds, i); + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + assert(name); + PyObject *target = NULL; + if (f->f_state == FRAME_CLEARED) { + // Frame is cleared, map all names defined on the frame to None + target = Py_None; + Py_INCREF(target); + } else if (kind & CO_FAST_FREE) { + // Reference to closure cell, save it as the proxy target + target = fast_locals[i]; + assert(target != NULL && PyCell_Check(target)); + Py_INCREF(target); + } else if (kind & CO_FAST_CELL) { + // Closure cell referenced from nested scopes + // Save it as the proxy target if the cell already exists, + // otherwise save the index and fix it up later on access + target = fast_locals[i]; + if (target != NULL && PyCell_Check(target) && + _PyFrame_OpAlreadyRan(f, MAKE_CELL, i)) { + // MAKE_CELL built the cell, so use it as the proxy target Py_INCREF(target); - } else if (kind & CO_FAST_CELL) { - // Closure cell referenced from nested scopes - // Save it as the proxy target if the cell already exists, - // otherwise save the index and fix it up later on access - target = fast_locals[i]; - if (target != NULL && PyCell_Check(target) && - _PyFrame_OpAlreadyRan(f, MAKE_CELL, i)) { - // MAKE_CELL built the cell, so use it as the proxy target - Py_INCREF(target); - } else { - // MAKE_CELL hasn't run yet, so just store the lookup index - // The proxy will check the kind on access, and switch over - // to using the cell once MAKE_CELL creates it - target = PyLong_FromSsize_t(i); - } - } else if (kind & CO_FAST_LOCAL) { - // Ordinary fast local variable. Save index as the proxy target - target = PyLong_FromSsize_t(i); } else { - PyErr_SetString(PyExc_SystemError, - "unknown local variable kind while building fast refs lookup table"); - } - if (target == NULL) { - Py_DECREF(fast_refs); - return NULL; - } - if (set_fast_ref(fast_refs, name, target) != 0) { - return NULL; + // MAKE_CELL hasn't run yet, so just store the lookup index + // The proxy will check the kind on access, and switch over + // to using the cell once MAKE_CELL creates it + target = PyLong_FromSsize_t(i); } + } else if (kind & CO_FAST_LOCAL) { + // Ordinary fast local variable. Save index as the proxy target + target = PyLong_FromSsize_t(i); + } else { + PyErr_SetString(PyExc_SystemError, + "unknown local variable kind while building fast refs lookup table"); + } + if (target == NULL) { + Py_DECREF(fast_refs); + return NULL; + } + if (set_fast_ref(fast_refs, name, target) != 0) { + return NULL; } } return fast_refs;