bpo-37757: Disallow PEP 572 cases that expose implementation details

Doc/c-api/exceptions.rst

@@ -809,7 +809,6 @@ the variables:
    single: PyExc_SystemError
    single: PyExc_SystemExit
    single: PyExc_TabError
-   single: PyExc_TargetScopeError
    single: PyExc_TimeoutError
    single: PyExc_TypeError
    single: PyExc_UnboundLocalError
@@ -911,8 +910,6 @@ the variables:
 +-----------------------------------------+---------------------------------+----------+
 | :c:data:`PyExc_TabError`                | :exc:`TabError`                 |          |
 +-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_TargetScopeError`        | :exc:`TargetScopeError`         |          |
-+-----------------------------------------+---------------------------------+----------+
 | :c:data:`PyExc_TimeoutError`            | :exc:`TimeoutError`             |          |
 +-----------------------------------------+---------------------------------+----------+
 | :c:data:`PyExc_TypeError`               | :exc:`TypeError`                |          |

Include/pyerrors.h

@@ -97,7 +97,6 @@ PyAPI_DATA(PyObject *) PyExc_NotImplementedError;
 PyAPI_DATA(PyObject *) PyExc_SyntaxError;
 PyAPI_DATA(PyObject *) PyExc_IndentationError;
 PyAPI_DATA(PyObject *) PyExc_TabError;
-PyAPI_DATA(PyObject *) PyExc_TargetScopeError;
 PyAPI_DATA(PyObject *) PyExc_ReferenceError;
 PyAPI_DATA(PyObject *) PyExc_SystemError;
 PyAPI_DATA(PyObject *) PyExc_SystemExit;

Include/symtable.h

@@ -58,6 +58,8 @@ typedef struct _symtable_entry {
     unsigned ste_needs_class_closure : 1; /* for class scopes, true if a
                                              closure over __class__
                                              should be created */
+    unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */
+    int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */
     int ste_lineno;          /* first line of block */
     int ste_col_offset;      /* offset of first line of block */
     int ste_opt_lineno;      /* lineno of last exec or import * */
@@ -94,6 +96,7 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
 #define DEF_FREE_CLASS 2<<5    /* free variable from class's method */
 #define DEF_IMPORT 2<<6        /* assignment occurred via import */
 #define DEF_ANNOT 2<<7         /* this name is annotated */
+#define DEF_COMP_ITER 2<<8     /* this name is a comprehension iteration variable */
 
 #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)
 

Lib/_compat_pickle.py

@@ -128,7 +128,6 @@
     "SystemError",
     "SystemExit",
     "TabError",
-    "TargetScopeError",
     "TypeError",
     "UnboundLocalError",
     "UnicodeDecodeError",

Lib/test/exception_hierarchy.txt

@@ -42,7 +42,6 @@ BaseException
       |    +-- NotImplementedError
       |    +-- RecursionError
       +-- SyntaxError
-      |    +-- TargetScopeError
       |    +-- IndentationError
       |         +-- TabError
       +-- SystemError

Lib/test/test_named_expressions.py

@@ -104,15 +104,69 @@ def test_named_expression_invalid_17(self):
         with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
             exec(code, {}, {})
 
-    def test_named_expression_invalid_18(self):
+    def test_named_expression_invalid_in_class_body(self):
         code = """class Foo():
             [(42, 1 + ((( j := i )))) for i in range(5)]
         """
 
-        with self.assertRaisesRegex(TargetScopeError,
-            "named expression within a comprehension cannot be used in a class body"):
+        with self.assertRaisesRegex(SyntaxError,
+            "assignment expression within a comprehension cannot be used in a class body"):
             exec(code, {}, {})
 
+    def test_named_expression_invalid_rebinding_comprehension_iteration_variable(self):
+        cases = [
+            ("Local reuse", 'i', "[i := 0 for i in range(5)]"),
+            ("Nested reuse", 'j', "[[(j := 0) for i in range(5)] for j in range(5)]"),
+            ("Reuse inner loop target", 'j', "[(j := 0) for i in range(5) for j in range(5)]"),
+            ("Unpacking reuse", 'i', "[i := 0 for i, j in [(0, 1)]]"),
+            ("Reuse in loop condition", 'i', "[i+1 for i in range(5) if (i := 0)]"),
+            ("Unreachable reuse", 'i', "[False or (i:=0) for i in range(5)]"),
+            ("Unreachable nested reuse", 'i',
+                "[(i, j) for i in range(5) for j in range(5) if True or (i:=10)]"),
+        ]
+        for case, target, code in cases:
+            msg = f"assignment expression cannot rebind comprehension iteration variable '{target}'"
+            with self.subTest(case=case):
+                with self.assertRaisesRegex(SyntaxError, msg):
+                    exec(code, {}, {})
+
+    def test_named_expression_invalid_rebinding_comprehension_inner_loop(self):
+        cases = [
+            ("Inner reuse", 'j', "[i for i in range(5) if (j := 0) for j in range(5)]"),
+            ("Inner unpacking reuse", 'j', "[i for i in range(5) if (j := 0) for j, k in [(0, 1)]]"),
+        ]
+        for case, target, code in cases:
+            msg = f"comprehension inner loop cannot rebind assignment expression target '{target}'"
+            with self.subTest(case=case):
+                with self.assertRaisesRegex(SyntaxError, msg):
+                    exec(code, {}) # Module scope
+                with self.assertRaisesRegex(SyntaxError, msg):
+                    exec(code, {}, {}) # Class scope
+                with self.assertRaisesRegex(SyntaxError, msg):
+                    exec(f"lambda: {code}", {}) # Function scope
+
+    def test_named_expression_invalid_comprehension_iterable_expression(self):
+        cases = [
+            ("Top level", "[i for i in (i := range(5))]"),
+            ("Inside tuple", "[i for i in (2, 3, i := range(5))]"),
+            ("Inside list", "[i for i in [2, 3, i := range(5)]]"),
+            ("Different name", "[i for i in (j := range(5))]"),
+            ("Lambda expression", "[i for i in (lambda:(j := range(5)))()]"),
+            ("Inner loop", "[i for i in range(5) for j in (i := range(5))]"),
+            ("Nested comprehension", "[i for i in [j for j in (k := range(5))]]"),
+            ("Nested comprehension condition", "[i for i in [j for j in range(5) if (j := True)]]"),
+            ("Nested comprehension body", "[i for i in [(j := True) for j in range(5)]]"),
+        ]
+        msg = "assignment expression cannot be used in a comprehension iterable expression"
+        for case, code in cases:
+            with self.subTest(case=case):
+                with self.assertRaisesRegex(SyntaxError, msg):
+                    exec(code, {}) # Module scope
+                with self.assertRaisesRegex(SyntaxError, msg):
+                    exec(code, {}, {}) # Class scope
+                with self.assertRaisesRegex(SyntaxError, msg):
+                    exec(f"lambda: {code}", {}) # Function scope
+
 
 class NamedExpressionAssignmentTest(unittest.TestCase):
 
@@ -306,39 +360,6 @@ def test_named_expression_scope_11(self):
         self.assertEqual(res, [0, 1, 2, 3, 4])
         self.assertEqual(j, 4)
 
-    def test_named_expression_scope_12(self):
-        res = [i := i for i in range(5)]
-
-        self.assertEqual(res, [0, 1, 2, 3, 4])
-        self.assertEqual(i, 4)
-
-    def test_named_expression_scope_13(self):
-        res = [i := 0 for i, j in [(1, 2), (3, 4)]]
-
-        self.assertEqual(res, [0, 0])
-        self.assertEqual(i, 0)
-
-    def test_named_expression_scope_14(self):
-        res = [(i := 0, j := 1) for i, j in [(1, 2), (3, 4)]]
-
-        self.assertEqual(res, [(0, 1), (0, 1)])
-        self.assertEqual(i, 0)
-        self.assertEqual(j, 1)
-
-    def test_named_expression_scope_15(self):
-        res = [(i := i, j := j) for i, j in [(1, 2), (3, 4)]]
-
-        self.assertEqual(res, [(1, 2), (3, 4)])
-        self.assertEqual(i, 3)
-        self.assertEqual(j, 4)
-
-    def test_named_expression_scope_16(self):
-        res = [(i := j, j := i) for i, j in [(1, 2), (3, 4)]]
-
-        self.assertEqual(res, [(2, 2), (4, 4)])
-        self.assertEqual(i, 4)
-        self.assertEqual(j, 4)
-
     def test_named_expression_scope_17(self):
         b = 0
         res = [b := i + b for i in range(5)]
@@ -421,6 +442,34 @@ def spam():
 
         self.assertEqual(ns["a"], 20)
 
+    def test_named_expression_variable_reuse_in_comprehensions(self):
+        # The compiler is expected to raise syntax error for comprehension
+        # iteration variables, but should be fine with rebinding of other
+        # names (e.g. globals, nonlocals, other assignment expressions)
+
+        # The cases are all defined to produce the same expected result
+        # Each comprehension is checked at both function scope and module scope
+        rebinding = "[x := i for i in range(3) if (x := i) or not x]"
+        filter_ref = "[x := i for i in range(3) if x or not x]"
+        body_ref = "[x for i in range(3) if (x := i) or not x]"
+        nested_ref = "[j for i in range(3) if x or not x for j in range(3) if (x := i)][:-3]"
+        cases = [
+            ("Rebind global", f"x = 1; result = {rebinding}"),
+            ("Rebind nonlocal", f"result, x = (lambda x=1: ({rebinding}, x))()"),
+            ("Filter global", f"x = 1; result = {filter_ref}"),
+            ("Filter nonlocal", f"result, x = (lambda x=1: ({filter_ref}, x))()"),
+            ("Body global", f"x = 1; result = {body_ref}"),
+            ("Body nonlocal", f"result, x = (lambda x=1: ({body_ref}, x))()"),
+            ("Nested global", f"x = 1; result = {nested_ref}"),
+            ("Nested nonlocal", f"result, x = (lambda x=1: ({nested_ref}, x))()"),
+        ]
+        msg = "assignment expression cannot be used in a comprehension iterable expression"
+        for case, code in cases:
+            with self.subTest(case=case):
+                ns = {}
+                exec(code, ns)
+                self.assertEqual(ns["x"], 2)
+                self.assertEqual(ns["result"], [0, 1, 2])
 
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Core and Builtins/2019-08-05-14-22-59.bpo-37757.lRv5HX.rst

@@ -0,0 +1,7 @@
+:pep:`572`: As described in the PEP, assignment expressions now raise
+:exc:`SyntaxError` when their interaction with comprehension scoping results
+in an ambiguous target scope.
+
+The ``TargetScopeError`` subclass originally proposed by the PEP has been
+removed in favour of just raising regular syntax errors for the disallowed
+cases.

Objects/exceptions.c

@@ -1521,13 +1521,6 @@ MiddlingExtendsException(PyExc_SyntaxError, IndentationError, SyntaxError,
                          "Improper indentation.");
 
 
-/*
- *    TargetScopeError extends SyntaxError
- */
-MiddlingExtendsException(PyExc_SyntaxError, TargetScopeError, SyntaxError,
-                         "Improper scope target.");
-
-
 /*
  *    TabError extends IndentationError
  */
@@ -2539,7 +2532,6 @@ _PyExc_Init(void)
     PRE_INIT(AttributeError);
     PRE_INIT(SyntaxError);
     PRE_INIT(IndentationError);
-    PRE_INIT(TargetScopeError);
     PRE_INIT(TabError);
     PRE_INIT(LookupError);
     PRE_INIT(IndexError);
@@ -2680,7 +2672,6 @@ _PyBuiltins_AddExceptions(PyObject *bltinmod)
     POST_INIT(AttributeError);
     POST_INIT(SyntaxError);
     POST_INIT(IndentationError);
-    POST_INIT(TargetScopeError);
     POST_INIT(TabError);
     POST_INIT(LookupError);
     POST_INIT(IndexError);

PC/python3.def

@@ -235,7 +235,6 @@ EXPORTS
   PyExc_SystemError=python39.PyExc_SystemError DATA
   PyExc_SystemExit=python39.PyExc_SystemExit DATA
   PyExc_TabError=python39.PyExc_TabError DATA
-  PyExc_TargetScopeError=python39.PyExc_TargetScopeError DATA
   PyExc_TimeoutError=python39.PyExc_TimeoutError DATA
   PyExc_TypeError=python39.PyExc_TypeError DATA
   PyExc_UnboundLocalError=python39.PyExc_UnboundLocalError DATA