diff --git a/crates/ruff_python_parser/src/lib.rs b/crates/ruff_python_parser/src/lib.rs index ce409200ae29f..efe89af9f6860 100644 --- a/crates/ruff_python_parser/src/lib.rs +++ b/crates/ruff_python_parser/src/lib.rs @@ -70,6 +70,7 @@ pub use crate::error::{ InterpolatedStringErrorType, LexicalErrorType, ParseError, ParseErrorType, UnsupportedSyntaxError, UnsupportedSyntaxErrorKind, }; +pub use crate::parenthesize::{parentheses_iterator_from_tokens, parenthesized_range_from_tokens}; pub use crate::parser::ParseOptions; pub use crate::token::{Token, TokenKind}; @@ -83,6 +84,7 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; mod error; pub mod lexer; +mod parenthesize; mod parser; pub mod semantic_errors; mod string; diff --git a/crates/ruff_python_parser/src/parenthesize.rs b/crates/ruff_python_parser/src/parenthesize.rs new file mode 100644 index 0000000000000..51c8cd252ebe0 --- /dev/null +++ b/crates/ruff_python_parser/src/parenthesize.rs @@ -0,0 +1,193 @@ +use ruff_python_ast::{AnyNodeRef, ExprRef}; +use ruff_text_size::{Ranged, TextLen, TextRange}; + +use crate::{Token, TokenKind}; + +const fn is_trivia(kind: TokenKind) -> bool { + matches!( + kind, + TokenKind::Comment + | TokenKind::NonLogicalNewline + | TokenKind::Newline + | TokenKind::Indent + | TokenKind::Dedent + ) +} + +/// Returns an iterator over the ranges of the optional parentheses surrounding an expression. +/// +/// E.g. for `((f()))` with `f()` as expression, the iterator returns the ranges (1, 6) and (0, 7). +/// +/// Note that without a parent the range can be inaccurate, e.g. `f(a)` we falsely return a set of +/// parentheses around `a` even if the parentheses actually belong to `f`. That is why you should +/// generally prefer [`parenthesized_range_from_tokens`]. +pub fn parentheses_iterator_from_tokens<'a>( + expr: ExprRef<'a>, + parent: Option, + tokens: &'a [Token], +) -> impl Iterator + 'a { + let exclusive_parent_end = if let Some(parent) = parent { + // If the parent is a node that brings its own parentheses, exclude the closing parenthesis + // from our search range. Otherwise, we risk matching on calls, like `func(x)`, for which + // the open and close parentheses are part of the `Arguments` node. + if parent.is_arguments() { + parent.end() - ")".text_len() + } else { + parent.end() + } + } else { + tokens.last().map(|t| t.end()).unwrap_or(expr.end()) + }; + + let after_expr_idx = tokens.partition_point(|token| token.end() <= expr.end()); + let tokens_after = &tokens[after_expr_idx..]; + + let before_expr_idx = tokens.partition_point(|token| token.start() < expr.start()); + let tokens_before = &tokens[..before_expr_idx]; + + let right_parens = tokens_after + .iter() + .filter(move |token| token.start() < exclusive_parent_end) + .filter(|token| !is_trivia(token.kind())) + .take_while(|token| token.kind() == TokenKind::Rpar); + + let left_parens = tokens_before + .iter() + .rev() + .filter(|token| !is_trivia(token.kind())) + .take_while(|token| token.kind() == TokenKind::Lpar); + + // Zip closing parenthesis with opening parenthesis. The order is intentional, as testing for + // closing parentheses is cheaper, and `zip` will avoid progressing the `left_parens` if + // the `right_parens` is exhausted. + right_parens + .zip(left_parens) + .map(|(right, left)| TextRange::new(left.start(), right.end())) +} + +/// Returns the [`TextRange`] of a given expression including parentheses, if the expression is +/// parenthesized; or `None`, if the expression is not parenthesized. +pub fn parenthesized_range_from_tokens( + expr: ExprRef, + parent: AnyNodeRef, + tokens: &[Token], +) -> Option { + parentheses_iterator_from_tokens(expr, Some(parent), tokens).last() +} + +#[cfg(test)] +mod tests { + use ruff_python_ast::{AnyNodeRef, Expr, Stmt}; + use ruff_text_size::TextRange; + + use crate::{parenthesized_range_from_tokens, parse_module}; + + #[test] + fn test_parenthesized_range() { + let source = "x = (1 + 2)"; + let parsed = parse_module(source).unwrap(); + let tokens = parsed.tokens(); + let stmt = parsed.suite().first().unwrap(); + + let Stmt::Assign(assign) = stmt else { + panic!("Expected Assign statement"); + }; + + let value = assign.value.as_ref(); + let range = + parenthesized_range_from_tokens(value.into(), AnyNodeRef::from(stmt), tokens.as_ref()); + + assert_eq!(range, Some(TextRange::new(4.into(), 11.into()))); + } + + #[test] + fn test_double_parenthesized_range() { + let source = "x = ((1 + 2))"; + let parsed = parse_module(source).unwrap(); + let tokens = parsed.tokens(); + let stmt = parsed.suite().first().unwrap(); + + let Stmt::Assign(assign) = stmt else { + panic!("Expected Assign statement"); + }; + + let value = assign.value.as_ref(); + let range = + parenthesized_range_from_tokens(value.into(), AnyNodeRef::from(stmt), tokens.as_ref()); + + // Should return the outermost parentheses + assert_eq!(range, Some(TextRange::new(4.into(), 13.into()))); + } + + #[test] + fn test_no_parentheses() { + let source = "x = 1 + 2"; + let parsed = parse_module(source).unwrap(); + let tokens = parsed.tokens(); + let stmt = parsed.suite().first().unwrap(); + + let Stmt::Assign(assign) = stmt else { + panic!("Expected Assign statement"); + }; + + let value = assign.value.as_ref(); + let range = + parenthesized_range_from_tokens(value.into(), AnyNodeRef::from(stmt), tokens.as_ref()); + + assert_eq!(range, None); + } + + #[test] + fn test_call_parentheses_not_included() { + let source = "f(a)"; + let parsed = parse_module(source).unwrap(); + let tokens = parsed.tokens(); + let stmt = parsed.suite().first().unwrap(); + + let Stmt::Expr(expr_stmt) = stmt else { + panic!("Expected Expr statement"); + }; + + let Expr::Call(call) = expr_stmt.value.as_ref() else { + panic!("Expected Call expression"); + }; + + // Get the argument `a` + let arg = call.arguments.args.first().unwrap(); + let range = parenthesized_range_from_tokens( + arg.into(), + AnyNodeRef::from(&call.arguments), + tokens.as_ref(), + ); + + // `a` is not parenthesized, the parens belong to the call + assert_eq!(range, None); + } + + #[test] + fn test_parenthesized_arg_in_call() { + let source = "f((a))"; + let parsed = parse_module(source).unwrap(); + let tokens = parsed.tokens(); + let stmt = parsed.suite().first().unwrap(); + + let Stmt::Expr(expr_stmt) = stmt else { + panic!("Expected Expr statement"); + }; + + let Expr::Call(call) = expr_stmt.value.as_ref() else { + panic!("Expected Call expression"); + }; + + // Get the argument `a` + let arg = call.arguments.args.first().unwrap(); + let range = parenthesized_range_from_tokens( + arg.into(), + AnyNodeRef::from(&call.arguments), + tokens.as_ref(), + ); + + // `a` is parenthesized within the call + assert_eq!(range, Some(TextRange::new(2.into(), 5.into()))); + } +} diff --git a/crates/ty/docs/rules.md b/crates/ty/docs/rules.md index 78f757a6bb291..0fddeff65f19d 100644 --- a/crates/ty/docs/rules.md +++ b/crates/ty/docs/rules.md @@ -39,7 +39,7 @@ def test(): -> "int": Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -63,7 +63,7 @@ Calling a non-callable object will raise a `TypeError` at runtime. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -95,7 +95,7 @@ f(int) # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -126,7 +126,7 @@ a = 1 Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -158,7 +158,7 @@ class C(A, B): ... Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -190,7 +190,7 @@ class B(A): ... Default level: error · Preview (since 1.0.0) · Related issues · -View source +View source @@ -218,7 +218,7 @@ type B = A Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -245,7 +245,7 @@ class B(A, A): ... Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -357,7 +357,7 @@ def test(): -> "Literal[5]": Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -387,7 +387,7 @@ class C(A, B): ... Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -413,7 +413,7 @@ t[3] # IndexError: tuple index out of range Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -502,7 +502,7 @@ an atypical memory layout. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -529,7 +529,7 @@ func("foo") # error: [invalid-argument-type] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -557,7 +557,7 @@ a: int = '' Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -591,7 +591,7 @@ C.instance_var = 3 # error: Cannot assign to instance variable Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -627,7 +627,7 @@ asyncio.run(main()) Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -651,7 +651,7 @@ class A(42): ... # error: [invalid-base] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -678,7 +678,7 @@ with 1: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -707,7 +707,7 @@ a: str Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -751,7 +751,7 @@ except ZeroDivisionError: Default level: error · Added in 0.0.1-alpha.28 · Related issues · -View source +View source @@ -793,7 +793,7 @@ class D(A): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -826,7 +826,7 @@ class C[U](Generic[T]): ... Default level: error · Added in 0.0.1-alpha.17 · Related issues · -View source +View source @@ -865,7 +865,7 @@ carol = Person(name="Carol", age=25) # typo! Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -900,7 +900,7 @@ def f(t: TypeVar("U")): ... Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -934,7 +934,7 @@ class B(metaclass=f): ... Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -1041,7 +1041,7 @@ Correct use of `@override` is enforced by ty's `invalid-explicit-override` rule. Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -1073,7 +1073,7 @@ TypeError: can only inherit from a NamedTuple type and Generic Default level: error · Preview (since 1.0.0) · Related issues · -View source +View source @@ -1103,7 +1103,7 @@ Baz = NewType("Baz", int | str) # error: invalid base for `typing.NewType` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1153,7 +1153,7 @@ def foo(x: int) -> int: ... Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1179,7 +1179,7 @@ def f(a: int = ''): ... Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1210,7 +1210,7 @@ P2 = ParamSpec("S2") # error: ParamSpec name must match the variable it's assig Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1244,7 +1244,7 @@ TypeError: Protocols can only inherit from other protocols, got Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1293,7 +1293,7 @@ def g(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1318,7 +1318,7 @@ def func() -> int: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1376,7 +1376,7 @@ TODO #14889 Default level: error · Added in 0.0.1-alpha.6 · Related issues · -View source +View source @@ -1403,7 +1403,7 @@ NewAlias = TypeAliasType(get_name(), int) # error: TypeAliasType name mus Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -1450,7 +1450,7 @@ Bar[int] # error: too few arguments Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1480,7 +1480,7 @@ TYPE_CHECKING = '' Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1510,7 +1510,7 @@ b: Annotated[int] # `Annotated` expects at least two arguments Default level: error · Added in 0.0.1-alpha.11 · Related issues · -View source +View source @@ -1544,7 +1544,7 @@ f(10) # Error Default level: error · Added in 0.0.1-alpha.11 · Related issues · -View source +View source @@ -1578,7 +1578,7 @@ class C: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1613,7 +1613,7 @@ T = TypeVar('T', bound=str) # valid bound TypeVar Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1638,7 +1638,7 @@ func() # TypeError: func() missing 1 required positional argument: 'x' Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -1671,7 +1671,7 @@ alice["age"] # KeyError Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1700,7 +1700,7 @@ func("string") # error: [no-matching-overload] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1724,7 +1724,7 @@ Subscripting an object that does not support it will raise a `TypeError` at runt Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1750,7 +1750,7 @@ for i in 34: # TypeError: 'int' object is not iterable Default level: error · Added in 0.0.1-alpha.29 · Related issues · -View source +View source @@ -1783,7 +1783,7 @@ class B(A): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1810,7 +1810,7 @@ f(1, x=2) # Error raised here Default level: error · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -1868,7 +1868,7 @@ def test(): -> "int": Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1898,7 +1898,7 @@ static_assert(int(2.0 * 3.0) == 6) # error: does not have a statically known tr Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1927,7 +1927,7 @@ class B(A): ... # Error raised here Default level: error · Preview (since 0.0.1-alpha.30) · Related issues · -View source +View source @@ -1961,7 +1961,7 @@ class F(NamedTuple): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1988,7 +1988,7 @@ f("foo") # Error raised here Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2016,7 +2016,7 @@ def _(x: int): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2062,7 +2062,7 @@ class A: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2089,7 +2089,7 @@ f(x=1, y=2) # Error raised here Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2117,7 +2117,7 @@ A().foo # AttributeError: 'A' object has no attribute 'foo' Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2142,7 +2142,7 @@ import foo # ModuleNotFoundError: No module named 'foo' Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2167,7 +2167,7 @@ print(x) # NameError: name 'x' is not defined Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2204,7 +2204,7 @@ b1 < b2 < b1 # exception raised here Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2232,7 +2232,7 @@ A() + A() # TypeError: unsupported operand type(s) for +: 'A' and 'A' Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2257,7 +2257,7 @@ l[1:10:0] # ValueError: slice step cannot be zero Default level: warn · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -2298,7 +2298,7 @@ class SubProto(BaseProto, Protocol): Default level: warn · Added in 0.0.1-alpha.16 · Related issues · -View source +View source @@ -2386,7 +2386,7 @@ a = 20 / 0 # type: ignore Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -2414,7 +2414,7 @@ A.c # AttributeError: type object 'A' has no attribute 'c' Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -2446,7 +2446,7 @@ A()[0] # TypeError: 'A' object is not subscriptable Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -2478,7 +2478,7 @@ from module import a # ImportError: cannot import name 'a' from 'module' Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2505,7 +2505,7 @@ cast(int, f()) # Redundant Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2529,7 +2529,7 @@ reveal_type(1) # NameError: name 'reveal_type' is not defined Default level: warn · Added in 0.0.1-alpha.15 · Related issues · -View source +View source @@ -2587,7 +2587,7 @@ def g(): Default level: warn · Added in 0.0.1-alpha.7 · Related issues · -View source +View source @@ -2626,7 +2626,7 @@ class D(C): ... # error: [unsupported-base] Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -2689,7 +2689,7 @@ def foo(x: int | str) -> int | str: Default level: ignore · Preview (since 0.0.1-alpha.1) · Related issues · -View source +View source @@ -2713,7 +2713,7 @@ Dividing by zero raises a `ZeroDivisionError` at runtime. Default level: ignore · Added in 0.0.1-alpha.1 · Related issues · -View source +View source diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index f4f62a0f0b158..f291237f81f3b 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -35,13 +35,11 @@ use itertools::Itertools; use ruff_db::{ diagnostic::{Annotation, Diagnostic, Span, SubDiagnostic, SubDiagnosticSeverity}, parsed::parsed_module, - source::source_text, }; use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::name::Name; -use ruff_python_ast::parenthesize::parentheses_iterator; use ruff_python_ast::{self as ast, AnyNodeRef, StringFlags}; -use ruff_python_trivia::CommentRanges; +use ruff_python_parser::parentheses_iterator_from_tokens; use ruff_text_size::{Ranged, TextRange}; use rustc_hash::FxHashSet; use std::fmt::{self, Formatter}; @@ -2399,9 +2397,7 @@ pub(super) fn report_invalid_assignment<'db>( // ) # ty: ignore <- or here // ``` - let comment_ranges = CommentRanges::from(context.module().tokens()); - let source = source_text(context.db(), context.file()); - parentheses_iterator(value_node.into(), None, &comment_ranges, &source) + parentheses_iterator_from_tokens(value_node.into(), None, context.module().tokens()) .last() .unwrap_or(value_node.range()) } else {