Skip to content

Fix curly brace escape handling in f-strings #7331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dhruvmanila merged 22 commits into from
Sep 14, 2023
Merged

Fix curly brace escape handling in f-strings #7331

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
0cc8221
Add support for the new f-string tokens per PEP 701
dhruvmanila Aug 17, 2023
12a7c3a
Emit empty `FStringMiddle` token for special case
dhruvmanila Sep 4, 2023
72f9a21
Update comment from code review
dhruvmanila Sep 4, 2023
0159ae0
Avoid tracking parentheses nesting multiple times
dhruvmanila Sep 5, 2023
63f43dc
Add test for empty `FStringMiddle` tok in lambda expr
dhruvmanila Sep 5, 2023
bfa0296
Code review changes
dhruvmanila Sep 6, 2023
5db6b20
Emit empty token only in format spec, handle SingleRBrace error
dhruvmanila Sep 8, 2023
be4a3eb
Fix incorrect position to start the f-string token
dhruvmanila Sep 11, 2023
adb7a05
Update snapshots
dhruvmanila Sep 12, 2023
22cfd95
Pass in the source code to the parser
dhruvmanila Sep 1, 2023
408c58f
Add support for parsing f-strings as per PEP 701
dhruvmanila Sep 1, 2023
b70048a
Skip the empty `FStringMiddle` token emitted by lexer
dhruvmanila Sep 5, 2023
e8dade0
Remove unused error type, fix test case
dhruvmanila Sep 5, 2023
ab09524
Add some more f-string test cases
dhruvmanila Sep 5, 2023
cdc7b45
Code review changes
dhruvmanila Sep 6, 2023
19653a3
Pass arguments in the correct order
dhruvmanila Sep 11, 2023
9d8012d
Use narrow types for string parsing patterns
dhruvmanila Sep 7, 2023
18188d0
Remove single string literal pattern
dhruvmanila Sep 7, 2023
47c3215
Fix typo (`@L` -> `@R`)
dhruvmanila Sep 8, 2023
06ba358
Disallow non-parenthesized lambda expr in f-string
dhruvmanila Sep 11, 2023
44f3155
Fix curly brace escape handling for f-strings
dhruvmanila Sep 13, 2023
a8e4218
Merge branch 'dhruv/pep-701' into dhruv/fstring-parser-4
dhruvmanila Sep 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion crates/ruff_python_parser/src/lexer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -573,7 +573,8 @@ impl<'source> Lexer<'source> {
self.cursor.bump(); // '\'
if matches!(self.cursor.first(), '{' | '}') {
// Don't consume `{` or `}` as we want them to be emitted as tokens.
break;
// They will be handled in the next iteration.
continue;
} else if !fstring.is_raw_string() {
if self.cursor.eat_char2('N', '{') {
in_named_unicode = true;
Expand Down Expand Up @@ -1921,6 +1922,12 @@ def f(arg=%timeit a = b):
assert_debug_snapshot!(lex_source(source));
}

#[test]
fn test_fstring_escape_braces() {
let source = r"f'\{foo}' f'\\{foo}' f'\{{foo}}' f'\\{{foo}}'";
assert_debug_snapshot!(lex_source(source));
}

#[test]
fn test_fstring_escape_raw() {
let source = r#"rf"\{x:\"\{x}} \"\"\
Expand Down Expand Up @@ -2036,6 +2043,7 @@ f"{(lambda x:{x})}"
assert_eq!(lex_fstring_error(r"f'\u007b}'"), SingleRbrace);
assert_eq!(lex_fstring_error("f'{a:b}}'"), SingleRbrace);
assert_eq!(lex_fstring_error("f'{3:}}>10}'"), SingleRbrace);
assert_eq!(lex_fstring_error(r"f'\{foo}\}'"), SingleRbrace);

assert_eq!(lex_fstring_error("f'{'"), UnclosedLbrace);
assert_eq!(lex_fstring_error("f'{foo!r'"), UnclosedLbrace);
Expand Down
3 changes: 3 additions & 0 deletions crates/ruff_python_parser/src/parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1270,6 +1270,9 @@ f'{f"{3.1415=:.1f}":*^20}'
match foo:
case "foo " f"bar {x + y} " "baz":
pass

f"\{foo}\{bar:\}"
f"\\{{foo\\}}"
"#
.trim(),
"<test>",
Expand Down
98 changes: 98 additions & 0 deletions ..._python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_escape_braces.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
(
FStringStart,
0..2,
),
(
FStringMiddle {
value: "\\",
is_raw: false,
},
2..3,
),
(
Lbrace,
3..4,
),
(
Name {
name: "foo",
},
4..7,
),
(
Rbrace,
7..8,
),
(
FStringEnd,
8..9,
),
(
FStringStart,
10..12,
),
(
FStringMiddle {
value: "\\\\",
is_raw: false,
},
12..14,
),
(
Lbrace,
14..15,
),
(
Name {
name: "foo",
},
15..18,
),
(
Rbrace,
18..19,
),
(
FStringEnd,
19..20,
),
(
FStringStart,
21..23,
),
(
FStringMiddle {
value: "\\{foo}",
is_raw: false,
},
23..31,
),
(
FStringEnd,
31..32,
),
(
FStringStart,
33..35,
),
(
FStringMiddle {
value: "\\\\{foo}",
is_raw: false,
},
35..44,
),
(
FStringEnd,
44..45,
),
(
Newline,
45..45,
),
]
113 changes: 113 additions & 0 deletions crates/ruff_python_parser/src/snapshots/ruff_python_parser__parser__tests__fstrings.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -732,4 +732,117 @@ expression: parse_ast
],
},
),
Expr(
StmtExpr {
range: 271..288,
value: FString(
ExprFString {
range: 271..288,
values: [
Constant(
ExprConstant {
range: 273..274,
value: Str(
StringConstant {
value: "\\",
unicode: false,
implicit_concatenated: false,
},
),
},
),
FormattedValue(
ExprFormattedValue {
range: 274..279,
value: Name(
ExprName {
range: 275..278,
id: "foo",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
Constant(
ExprConstant {
range: 279..280,
value: Str(
StringConstant {
value: "\\",
unicode: false,
implicit_concatenated: false,
},
),
},
),
FormattedValue(
ExprFormattedValue {
range: 280..287,
value: Name(
ExprName {
range: 281..284,
id: "bar",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: Some(
FString(
ExprFString {
range: 285..286,
values: [
Constant(
ExprConstant {
range: 285..286,
value: Str(
StringConstant {
value: "\\",
unicode: false,
implicit_concatenated: false,
},
),
},
),
],
implicit_concatenated: false,
},
),
),
},
),
],
implicit_concatenated: false,
},
),
},
),
Expr(
StmtExpr {
range: 289..303,
value: FString(
ExprFString {
range: 289..303,
values: [
Constant(
ExprConstant {
range: 291..302,
value: Str(
StringConstant {
value: "\\{foo\\}",
unicode: false,
implicit_concatenated: false,
},
),
},
),
],
implicit_concatenated: false,
},
),
},
),
]
22 changes: 21 additions & 1 deletion crates/ruff_python_parser/src/string.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,27 @@ impl<'a> StringParser<'a> {
let start_location = self.get_pos();
while let Some(ch) = self.next_char() {
match ch {
'\\' if !self.kind.is_raw() => {
// We can encounter a `\` as the last character in a `FStringMiddle`
// token which is valid in this context. For example,
//
// ```python
// f"\{foo} \{bar:\}"
// # ^ ^^ ^
// ```
//
// Here, the `FStringMiddle` token content will be "\" and " \"
// which is invalid if we look at the content in isolation:
//
// ```python
// "\"
// ```
//
// However, the content is syntactically valid in the context of
// the f-string because it's a substring of the entire f-string.
// This is still an invalid escape sequence, but we don't want to
// raise a syntax error as is done by the CPython parser. It might
// be supported in the future, refer to point 3: https://peps.python.org/pep-0701/#rejected-ideas
'\\' if !self.kind.is_raw() && self.peek().is_some() => {
value.push_str(&self.parse_escaped_char()?);
}
// If there are any curly braces inside a `FStringMiddle` token,
Expand Down