From 95440ae592d2ffc476bc5c676f7b7796294ae1dc Mon Sep 17 00:00:00 2001 From: dylwil3 Date: Mon, 7 Jul 2025 09:43:37 -0500 Subject: [PATCH 1/8] add test fixtures --- .../resources/test/fixtures/ruff/RUF027_0.py | 5 ++++ crates/ruff_python_parser/src/parser/tests.rs | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/crates/ruff_linter/resources/test/fixtures/ruff/RUF027_0.py b/crates/ruff_linter/resources/test/fixtures/ruff/RUF027_0.py index e5bef3033f181d..c863347069ab41 100644 --- a/crates/ruff_linter/resources/test/fixtures/ruff/RUF027_0.py +++ b/crates/ruff_linter/resources/test/fixtures/ruff/RUF027_0.py @@ -79,3 +79,8 @@ def in_type_def(): from typing import cast a = 'int' cast('f"{a}"','11') + +# Regression test for parser bug +# https://github.com/astral-sh/ruff/issues/18860 +def fuzz_bug(): + c('{\t"i}') diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index 778637597cfe8c..dcb9ac16a0af58 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -134,3 +134,26 @@ foo.bar[0].baz[2].egg?? .unwrap(); insta::assert_debug_snapshot!(parsed.syntax()); } + +#[test] +fn test_fstring_expr_inner_line_continuation_and_t_string() { + let source = r#"f'{\t"i}'"#; + + let parsed = parse_expression(source); + + let error = parsed.unwrap_err(); + + insta::assert_debug_snapshot!(error); +} + +#[test] +fn test_fstring_expr_inner_line_continuation_newline_t_string() { + let source = r#"f'{\ +t"i}'"#; + + let parsed = parse_expression(source); + + let error = parsed.unwrap_err(); + + insta::assert_debug_snapshot!(error); +} From 5760ca8716af021639852ed8d30020181a3ca00e Mon Sep 17 00:00:00 2001 From: dylwil3 Date: Mon, 7 Jul 2025 09:45:00 -0500 Subject: [PATCH 2/8] replace unreachable with error --- crates/ruff_python_parser/src/parser/expression.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index e1f5f8c1245aae..d2fa98c786b962 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -1685,14 +1685,14 @@ impl<'src> Parser<'src> { return; } tok => { - // This should never happen because the list parsing will only - // call this closure for the above token kinds which are the same - // as in the FIRST set. - unreachable!( - "{}: unexpected token `{tok:?}` at {:?}", - string_kind, - parser.current_token_range() + parser.add_error( + ParseErrorType::OtherError(format!( + "{string_kind}: unexpected token `{tok:?}`" + )), + parser.current_token_range(), ); + parser.bump_any(); + return; } }; elements.push(element); From aeaee70cb49f258d429a3bfe79b8c78c1a62cbb4 Mon Sep 17 00:00:00 2001 From: dylwil3 Date: Mon, 7 Jul 2025 09:45:11 -0500 Subject: [PATCH 3/8] snapshots --- ...ng_expr_inner_line_continuation_and_t_string.snap | 10 ++++++++++ ...xpr_inner_line_continuation_newline_t_string.snap | 12 ++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__fstring_expr_inner_line_continuation_and_t_string.snap create mode 100644 crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__fstring_expr_inner_line_continuation_newline_t_string.snap diff --git a/crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__fstring_expr_inner_line_continuation_and_t_string.snap b/crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__fstring_expr_inner_line_continuation_and_t_string.snap new file mode 100644 index 00000000000000..490211dd4ea0ed --- /dev/null +++ b/crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__fstring_expr_inner_line_continuation_and_t_string.snap @@ -0,0 +1,10 @@ +--- +source: crates/ruff_python_parser/src/parser/tests.rs +expression: error +--- +ParseError { + error: Lexical( + LineContinuationError, + ), + location: 3..4, +} diff --git a/crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__fstring_expr_inner_line_continuation_newline_t_string.snap b/crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__fstring_expr_inner_line_continuation_newline_t_string.snap new file mode 100644 index 00000000000000..36cd036382569a --- /dev/null +++ b/crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__fstring_expr_inner_line_continuation_newline_t_string.snap @@ -0,0 +1,12 @@ +--- +source: crates/ruff_python_parser/src/parser/tests.rs +expression: error +--- +ParseError { + error: Lexical( + TStringError( + SingleRbrace, + ), + ), + location: 8..9, +} From ac24a9835c9a19126def57863d923bb8ce68d527 Mon Sep 17 00:00:00 2001 From: dylwil3 Date: Fri, 18 Jul 2025 16:46:00 -0500 Subject: [PATCH 4/8] add string kind to recovery context --- .../src/parser/expression.rs | 2 +- crates/ruff_python_parser/src/parser/mod.rs | 57 ++++++++++++------- crates/ruff_python_parser/src/string.rs | 2 +- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index d2fa98c786b962..60c704c5c8c9d9 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -1527,7 +1527,7 @@ impl<'src> Parser<'src> { self.bump(kind.start_token()); let elements = self.parse_interpolated_string_elements( flags, - InterpolatedStringElementsKind::Regular, + InterpolatedStringElementsKind::Regular(kind), kind, ); diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs index 904e92df95b8c3..1bd0c76e847934 100644 --- a/crates/ruff_python_parser/src/parser/mod.rs +++ b/crates/ruff_python_parser/src/parser/mod.rs @@ -8,6 +8,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::error::UnsupportedSyntaxError; use crate::parser::expression::ExpressionContext; use crate::parser::progress::{ParserProgress, TokenId}; +use crate::string::InterpolatedStringKind; use crate::token::TokenValue; use crate::token_set::TokenSet; use crate::token_source::{TokenSource, TokenSourceCheckpoint}; @@ -807,7 +808,7 @@ enum InterpolatedStringElementsKind { /// ```py /// f"hello {x:.2f} world" /// ``` - Regular, + Regular(InterpolatedStringKind), /// The f-string elements are part of the format specifier. /// @@ -819,15 +820,13 @@ enum InterpolatedStringElementsKind { } impl InterpolatedStringElementsKind { - const fn list_terminators(self) -> TokenSet { + const fn list_terminator(self) -> TokenKind { match self { - InterpolatedStringElementsKind::Regular => { - TokenSet::new([TokenKind::FStringEnd, TokenKind::TStringEnd]) - } + InterpolatedStringElementsKind::Regular(string_kind) => string_kind.end_token(), // test_ok fstring_format_spec_terminator // f"hello {x:} world" // f"hello {x:.3f} world" - InterpolatedStringElementsKind::FormatSpec => TokenSet::new([TokenKind::Rbrace]), + InterpolatedStringElementsKind::FormatSpec => TokenKind::Rbrace, } } } @@ -1121,7 +1120,7 @@ impl RecoveryContextKind { .then_some(ListTerminatorKind::Regular), }, RecoveryContextKind::InterpolatedStringElements(kind) => { - if p.at_ts(kind.list_terminators()) { + if p.at(kind.list_terminator()) { Some(ListTerminatorKind::Regular) } else { // test_err unterminated_fstring_newline_recovery @@ -1177,13 +1176,23 @@ impl RecoveryContextKind { ) || p.at_name_or_soft_keyword() } RecoveryContextKind::WithItems(_) => p.at_expr(), - RecoveryContextKind::InterpolatedStringElements(_) => matches!( - p.current_token_kind(), - // Literal element - TokenKind::FStringMiddle | TokenKind::TStringMiddle + RecoveryContextKind::InterpolatedStringElements(elements_kind) => { + match elements_kind { + InterpolatedStringElementsKind::Regular(interpolated_string_kind) => { + p.current_token_kind() == interpolated_string_kind.middle_token() + || p.current_token_kind() == TokenKind::Lbrace + } + InterpolatedStringElementsKind::FormatSpec => { + matches!( + p.current_token_kind(), + // Literal element + TokenKind::FStringMiddle | TokenKind::TStringMiddle // Expression element | TokenKind::Lbrace - ), + ) + } + } + } } } @@ -1272,8 +1281,8 @@ impl RecoveryContextKind { ), }, RecoveryContextKind::InterpolatedStringElements(kind) => match kind { - InterpolatedStringElementsKind::Regular => ParseErrorType::OtherError( - "Expected an f-string or t-string element or the end of the f-string or t-string".to_string(), + InterpolatedStringElementsKind::Regular(string_kind) => ParseErrorType::OtherError( + format!("Expected an element of or the end of the {string_kind}"), ), InterpolatedStringElementsKind::FormatSpec => ParseErrorType::OtherError( "Expected an f-string or t-string element or a '}'".to_string(), @@ -1316,8 +1325,9 @@ bitflags! { const WITH_ITEMS_PARENTHESIZED = 1 << 25; const WITH_ITEMS_PARENTHESIZED_EXPRESSION = 1 << 26; const WITH_ITEMS_UNPARENTHESIZED = 1 << 28; - const FT_STRING_ELEMENTS = 1 << 29; - const FT_STRING_ELEMENTS_IN_FORMAT_SPEC = 1 << 30; + const F_STRING_ELEMENTS = 1 << 29; + const T_STRING_ELEMENTS = 1 << 30; + const FT_STRING_ELEMENTS_IN_FORMAT_SPEC = 1 << 31; } } @@ -1371,7 +1381,13 @@ impl RecoveryContext { WithItemKind::Unparenthesized => RecoveryContext::WITH_ITEMS_UNPARENTHESIZED, }, RecoveryContextKind::InterpolatedStringElements(kind) => match kind { - InterpolatedStringElementsKind::Regular => RecoveryContext::FT_STRING_ELEMENTS, + InterpolatedStringElementsKind::Regular(InterpolatedStringKind::FString) => { + RecoveryContext::F_STRING_ELEMENTS + } + InterpolatedStringElementsKind::Regular(InterpolatedStringKind::TString) => { + RecoveryContext::T_STRING_ELEMENTS + } + InterpolatedStringElementsKind::FormatSpec => { RecoveryContext::FT_STRING_ELEMENTS_IN_FORMAT_SPEC } @@ -1442,8 +1458,11 @@ impl RecoveryContext { RecoveryContext::WITH_ITEMS_UNPARENTHESIZED => { RecoveryContextKind::WithItems(WithItemKind::Unparenthesized) } - RecoveryContext::FT_STRING_ELEMENTS => RecoveryContextKind::InterpolatedStringElements( - InterpolatedStringElementsKind::Regular, + RecoveryContext::F_STRING_ELEMENTS => RecoveryContextKind::InterpolatedStringElements( + InterpolatedStringElementsKind::Regular(InterpolatedStringKind::FString), + ), + RecoveryContext::T_STRING_ELEMENTS => RecoveryContextKind::InterpolatedStringElements( + InterpolatedStringElementsKind::Regular(InterpolatedStringKind::TString), ), RecoveryContext::FT_STRING_ELEMENTS_IN_FORMAT_SPEC => { RecoveryContextKind::InterpolatedStringElements( diff --git a/crates/ruff_python_parser/src/string.rs b/crates/ruff_python_parser/src/string.rs index 8dd9190b90057f..a3e7bee60edb3a 100644 --- a/crates/ruff_python_parser/src/string.rs +++ b/crates/ruff_python_parser/src/string.rs @@ -41,7 +41,7 @@ impl From for Expr { } } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq)] pub(crate) enum InterpolatedStringKind { FString, TString, From 0239b59db428ab8db496936e71b82199b2d96eef Mon Sep 17 00:00:00 2001 From: dylwil3 Date: Fri, 18 Jul 2025 16:46:34 -0500 Subject: [PATCH 5/8] mark arm as unreachable again --- crates/ruff_python_parser/src/parser/expression.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index 60c704c5c8c9d9..f953cc367c8c13 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -1685,14 +1685,14 @@ impl<'src> Parser<'src> { return; } tok => { - parser.add_error( - ParseErrorType::OtherError(format!( - "{string_kind}: unexpected token `{tok:?}`" - )), - parser.current_token_range(), + // This should never happen because the list parsing will only + // call this closure for the above token kinds which are the same + // as in the FIRST set. + unreachable!( + "{}: unexpected token `{tok:?}` at {:?}", + string_kind, + parser.current_token_range() ); - parser.bump_any(); - return; } }; elements.push(element); From 07cf8c056cbe55508c5e71e0d892ca3dbf1dba93 Mon Sep 17 00:00:00 2001 From: dylwil3 Date: Fri, 18 Jul 2025 16:47:02 -0500 Subject: [PATCH 6/8] remake snapshots --- .../invalid_syntax@f_string_lambda_without_parentheses.py.snap | 2 +- ...mplicitly_concatenated_unterminated_string_multiline.py.snap | 2 +- .../invalid_syntax@t_string_lambda_without_parentheses.py.snap | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_lambda_without_parentheses.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_lambda_without_parentheses.py.snap index 6f98814ce98316..3610b8d114a232 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_lambda_without_parentheses.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_lambda_without_parentheses.py.snap @@ -124,5 +124,5 @@ Module( | 1 | f"{lambda x: x}" - | ^ Syntax Error: Expected an f-string or t-string element or the end of the f-string or t-string + | ^ Syntax Error: Expected an element of or the end of the f-string | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@implicitly_concatenated_unterminated_string_multiline.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@implicitly_concatenated_unterminated_string_multiline.py.snap index da094ee09c49b3..59016cc425c23b 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@implicitly_concatenated_unterminated_string_multiline.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@implicitly_concatenated_unterminated_string_multiline.py.snap @@ -221,7 +221,7 @@ Module( 2 | 'hello' 3 | f'world {x} 4 | ) - | ^ Syntax Error: Expected an f-string or t-string element or the end of the f-string or t-string + | ^ Syntax Error: Expected an element of or the end of the f-string 5 | 1 + 1 6 | ( | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@t_string_lambda_without_parentheses.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@t_string_lambda_without_parentheses.py.snap index 121d002f283909..d87a72858f4b1d 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@t_string_lambda_without_parentheses.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@t_string_lambda_without_parentheses.py.snap @@ -128,5 +128,5 @@ Module( | 1 | # parse_options: {"target-version": "3.14"} 2 | t"{lambda x: x}" - | ^ Syntax Error: Expected an f-string or t-string element or the end of the f-string or t-string + | ^ Syntax Error: Expected an element of or the end of the t-string | From 6c2db9f99670f6ee5776104c8eec8bf8cbe81dbc Mon Sep 17 00:00:00 2001 From: dylwil3 Date: Sun, 20 Jul 2025 16:59:33 -0500 Subject: [PATCH 7/8] fix indentation --- crates/ruff_python_parser/src/parser/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs index 1bd0c76e847934..887d3f63aae20b 100644 --- a/crates/ruff_python_parser/src/parser/mod.rs +++ b/crates/ruff_python_parser/src/parser/mod.rs @@ -1187,8 +1187,8 @@ impl RecoveryContextKind { p.current_token_kind(), // Literal element TokenKind::FStringMiddle | TokenKind::TStringMiddle - // Expression element - | TokenKind::Lbrace + // Expression element + | TokenKind::Lbrace ) } } From 127d38930689248e0899902417060eac719baf76 Mon Sep 17 00:00:00 2001 From: dylwil3 Date: Sun, 20 Jul 2025 17:00:06 -0500 Subject: [PATCH 8/8] implement eq --- crates/ruff_python_parser/src/parser/mod.rs | 2 +- crates/ruff_python_parser/src/string.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs index 887d3f63aae20b..ccd44b2592925a 100644 --- a/crates/ruff_python_parser/src/parser/mod.rs +++ b/crates/ruff_python_parser/src/parser/mod.rs @@ -800,7 +800,7 @@ impl WithItemKind { } } -#[derive(Debug, PartialEq, Copy, Clone)] +#[derive(Debug, PartialEq, Eq, Copy, Clone)] enum InterpolatedStringElementsKind { /// The regular f-string elements. /// diff --git a/crates/ruff_python_parser/src/string.rs b/crates/ruff_python_parser/src/string.rs index a3e7bee60edb3a..a3fe1490de7d01 100644 --- a/crates/ruff_python_parser/src/string.rs +++ b/crates/ruff_python_parser/src/string.rs @@ -41,7 +41,7 @@ impl From for Expr { } } -#[derive(Debug, Clone, Copy, PartialEq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum InterpolatedStringKind { FString, TString,