fix: respect braces better in f-string parsing (#4422)

CHANGES.md

@@ -39,6 +39,8 @@
 
 - Fix bug with Black incorrectly parsing empty lines with a backslash (#4343)
 
+- Fix bugs with Black's tokenizer not handling `\{` inside f-strings very well (#4422)
+
 - Fix incorrect line numbers in the tokenizer for certain tokens within f-strings
   (#4423)
 

src/blib2to3/pgen2/tokenize.py

@@ -136,12 +136,12 @@ def _combinations(*l: str) -> Set[str]:
 )
 
 # beginning of a single quoted f-string. must not end with `{{` or `\N{`
-SingleLbrace = r"(?:\\N{|\\.|{{|[^'\\{])*(?<!\\N){(?!{)"
-DoubleLbrace = r'(?:\\N{|\\.|{{|[^"\\{])*(?<!\\N){(?!{)'
+SingleLbrace = r"(?:\\N{|{{|\\'|[^\n'{])*(?<!\\N)({)(?!{)"
+DoubleLbrace = r'(?:\\N{|{{|\\"|[^\n"{])*(?<!\\N)({)(?!{)'
 
 # beginning of a triple quoted f-string. must not end with `{{` or `\N{`
-Single3Lbrace = r"(?:\\N{|\\[^{]|{{|'(?!'')|[^'{\\])*(?<!\\N){(?!{)"
-Double3Lbrace = r'(?:\\N{|\\[^{]|{{|"(?!"")|[^"{\\])*(?<!\\N){(?!{)'
+Single3Lbrace = r"(?:\\N{|{{|\\'|'(?!'')|[^'{])*(?<!\\N){(?!{)"
+Double3Lbrace = r'(?:\\N{|{{|\\"|"(?!"")|[^"{])*(?<!\\N){(?!{)'
 
 # ! format specifier inside an fstring brace, ensure it's not a `!=` token
 Bang = Whitespace + group("!") + r"(?!=)"
@@ -175,8 +175,8 @@ def _combinations(*l: str) -> Set[str]:
 _string_middle_double = r'(?:[^\n"\\]|\\.)*'
 
 # FSTRING_MIDDLE and LBRACE, must not end with a `{{` or `\N{`
-_fstring_middle_single = r"(?:\\N{|\\[^{]|{{|[^\n'{\\])*(?<!\\N)({)(?!{)"
-_fstring_middle_double = r'(?:\\N{|\\[^{]|{{|[^\n"{\\])*(?<!\\N)({)(?!{)'
+_fstring_middle_single = SingleLbrace
+_fstring_middle_double = DoubleLbrace
 
 # First (or only) line of ' or " string.
 ContStr = group(

tests/data/cases/pep_701.py

@@ -131,6 +131,11 @@
 f"{'\''}"
 f"{f'\''}"
 
+f'{1}\{{'
+f'{2} foo \{{[\}}'
+f'\{3}'
+rf"\{"a"}"
+
 # output
 
 x = f"foo"
@@ -264,3 +269,8 @@
 
 f"{'\''}"
 f"{f'\''}"
+
+f"{1}\{{"
+f"{2} foo \{{[\}}"
+f"\{3}"
+rf"\{"a"}"