Remove BackwardsTokenizer based parenthesized_range references in ruff_linter

crates/ruff_linter/src/checkers/ast/mod.rs

@@ -437,6 +437,15 @@ impl<'a> Checker<'a> {
         }
     }
 
+    /// Returns the [`Tokens`] for the parsed source file.
+    ///
+    ///
+    /// Unlike [`Self::tokens`], this method always returns
+    /// the tokens for the current file, even when within a parsed type annotation.
+    pub(crate) fn source_tokens(&self) -> &'a Tokens {
+        self.parsed.tokens()
+    }
+
     /// The [`Locator`] for the current file, which enables extraction of source code from byte
     /// offsets.
     pub(crate) const fn locator(&self) -> &'a Locator<'a> {

crates/ruff_linter/src/fix/edits.rs

@@ -3,14 +3,13 @@
 use anyhow::{Context, Result};
 
 use ruff_python_ast::AnyNodeRef;
-use ruff_python_ast::parenthesize::parenthesized_range;
+use ruff_python_ast::token::{self, Tokens, parenthesized_range};
 use ruff_python_ast::{self as ast, Arguments, ExceptHandler, Expr, ExprList, Parameters, Stmt};
 use ruff_python_codegen::Stylist;
 use ruff_python_index::Indexer;
 use ruff_python_trivia::textwrap::dedent_to;
 use ruff_python_trivia::{
-    CommentRanges, PythonWhitespace, SimpleTokenKind, SimpleTokenizer, has_leading_content,
-    is_python_whitespace,
+    PythonWhitespace, SimpleTokenKind, SimpleTokenizer, has_leading_content, is_python_whitespace,
 };
 use ruff_source_file::{LineRanges, NewlineWithTrailingNewline, UniversalNewlines};
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
@@ -209,7 +208,7 @@ pub(crate) fn remove_argument<T: Ranged>(
     arguments: &Arguments,
     parentheses: Parentheses,
     source: &str,
-    comment_ranges: &CommentRanges,
+    tokens: &Tokens,
 ) -> Result<Edit> {
     // Partition into arguments before and after the argument to remove.
     let (before, after): (Vec<_>, Vec<_>) = arguments
@@ -224,7 +223,7 @@ pub(crate) fn remove_argument<T: Ranged>(
         .context("Unable to find argument")?;
 
     let parenthesized_range =
-        parenthesized_range(arg.value().into(), arguments.into(), comment_ranges, source)
+        token::parenthesized_range(arg.value().into(), arguments.into(), tokens)
             .unwrap_or(arg.range());
 
     if !after.is_empty() {
@@ -270,25 +269,14 @@ pub(crate) fn remove_argument<T: Ranged>(
 ///
 /// The new argument will be inserted before the first existing keyword argument in `arguments`, if
 /// there are any present. Otherwise, the new argument is added to the end of the argument list.
-pub(crate) fn add_argument(
-    argument: &str,
-    arguments: &Arguments,
-    comment_ranges: &CommentRanges,
-    source: &str,
-) -> Edit {
+pub(crate) fn add_argument(argument: &str, arguments: &Arguments, tokens: &Tokens) -> Edit {
     if let Some(ast::Keyword { range, value, .. }) = arguments.keywords.first() {
-        let keyword = parenthesized_range(value.into(), arguments.into(), comment_ranges, source)
-            .unwrap_or(*range);
+        let keyword = parenthesized_range(value.into(), arguments.into(), tokens).unwrap_or(*range);
         Edit::insertion(format!("{argument}, "), keyword.start())
     } else if let Some(last) = arguments.arguments_source_order().last() {
         // Case 1: existing arguments, so append after the last argument.
-        let last = parenthesized_range(
-            last.value().into(),
-            arguments.into(),
-            comment_ranges,
-            source,
-        )
-        .unwrap_or(last.range());
+        let last = parenthesized_range(last.value().into(), arguments.into(), tokens)
+            .unwrap_or(last.range());
         Edit::insertion(format!(", {argument}"), last.end())
     } else {
         // Case 2: no arguments. Add argument, without any trailing comma.

crates/ruff_linter/src/rules/fastapi/rules/fastapi_redundant_response_model.rs

@@ -91,8 +91,8 @@ pub(crate) fn fastapi_redundant_response_model(checker: &Checker, function_def:
                 response_model_arg,
                 &call.arguments,
                 Parentheses::Preserve,
-                checker.locator().contents(),
-                checker.comment_ranges(),
+                checker.source(),
+                checker.tokens(),
             )
             .map(Fix::unsafe_edit)
         });

crates/ruff_linter/src/rules/flake8_bugbear/rules/map_without_explicit_strict.rs

@@ -74,12 +74,7 @@ pub(crate) fn map_without_explicit_strict(checker: &Checker, call: &ast::ExprCal
         checker
             .report_diagnostic(MapWithoutExplicitStrict, call.range())
             .set_fix(Fix::applicable_edit(
-                add_argument(
-                    "strict=False",
-                    &call.arguments,
-                    checker.comment_ranges(),
-                    checker.locator().contents(),
-                ),
+                add_argument("strict=False", &call.arguments, checker.tokens()),
                 Applicability::Unsafe,
             ));
     }

crates/ruff_linter/src/rules/flake8_bugbear/rules/mutable_argument_default.rs

@@ -3,7 +3,7 @@ use std::fmt::Write;
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_python_ast::helpers::is_docstring_stmt;
 use ruff_python_ast::name::QualifiedName;
-use ruff_python_ast::parenthesize::parenthesized_range;
+use ruff_python_ast::token::parenthesized_range;
 use ruff_python_ast::{self as ast, Expr, ParameterWithDefault};
 use ruff_python_semantic::SemanticModel;
 use ruff_python_semantic::analyze::function_type::is_stub;
@@ -166,12 +166,7 @@ fn move_initialization(
         return None;
     }
 
-    let range = match parenthesized_range(
-        default.into(),
-        parameter.into(),
-        checker.comment_ranges(),
-        checker.source(),
-    ) {
+    let range = match parenthesized_range(default.into(), parameter.into(), checker.tokens()) {
         Some(range) => range,
         None => default.range(),
     };
@@ -194,13 +189,8 @@ fn move_initialization(
             "{} = {}",
             parameter.parameter.name(),
             locator.slice(
-                parenthesized_range(
-                    default.into(),
-                    parameter.into(),
-                    checker.comment_ranges(),
-                    checker.source()
-                )
-                .unwrap_or(default.range())
+                parenthesized_range(default.into(), parameter.into(), checker.tokens())
+                    .unwrap_or(default.range())
             )
         );
     } else {

crates/ruff_linter/src/rules/flake8_bugbear/rules/no_explicit_stacklevel.rs

@@ -92,12 +92,7 @@ pub(crate) fn no_explicit_stacklevel(checker: &Checker, call: &ast::ExprCall) {
     }
     let mut diagnostic = checker.report_diagnostic(NoExplicitStacklevel, call.func.range());
 
-    let edit = add_argument(
-        "stacklevel=2",
-        &call.arguments,
-        checker.comment_ranges(),
-        checker.locator().contents(),
-    );
+    let edit = add_argument("stacklevel=2", &call.arguments, checker.tokens());
 
     diagnostic.set_fix(Fix::unsafe_edit(edit));
 }

crates/ruff_linter/src/rules/flake8_bugbear/rules/zip_without_explicit_strict.rs

@@ -70,12 +70,7 @@ pub(crate) fn zip_without_explicit_strict(checker: &Checker, call: &ast::ExprCal
         checker
             .report_diagnostic(ZipWithoutExplicitStrict, call.range())
             .set_fix(Fix::applicable_edit(
-                add_argument(
-                    "strict=False",
-                    &call.arguments,
-                    checker.comment_ranges(),
-                    checker.locator().contents(),
-                ),
+                add_argument("strict=False", &call.arguments, checker.tokens()),
                 Applicability::Unsafe,
             ));
     }

crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_list.rs

@@ -2,8 +2,8 @@ use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_python_ast as ast;
 use ruff_python_ast::ExprGenerator;
 use ruff_python_ast::comparable::ComparableExpr;
-use ruff_python_ast::parenthesize::parenthesized_range;
 use ruff_python_ast::token::TokenKind;
+use ruff_python_ast::token::parenthesized_range;
 use ruff_text_size::{Ranged, TextRange, TextSize};
 
 use crate::checkers::ast::Checker;
@@ -142,13 +142,9 @@ pub(crate) fn unnecessary_generator_list(checker: &Checker, call: &ast::ExprCall
         if *parenthesized {
             // The generator's range will include the innermost parentheses, but it could be
             // surrounded by additional parentheses.
-            let range = parenthesized_range(
-                argument.into(),
-                (&call.arguments).into(),
-                checker.comment_ranges(),
-                checker.locator().contents(),
-            )
-            .unwrap_or(argument.range());
+            let range =
+                parenthesized_range(argument.into(), (&call.arguments).into(), checker.tokens())
+                    .unwrap_or(argument.range());
 
             // The generator always parenthesizes the expression; trim the parentheses.
             let generator = checker.generator().expr(argument);

crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_set.rs

@@ -2,8 +2,8 @@ use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_python_ast as ast;
 use ruff_python_ast::ExprGenerator;
 use ruff_python_ast::comparable::ComparableExpr;
-use ruff_python_ast::parenthesize::parenthesized_range;
 use ruff_python_ast::token::TokenKind;
+use ruff_python_ast::token::parenthesized_range;
 use ruff_text_size::{Ranged, TextRange, TextSize};
 
 use crate::checkers::ast::Checker;
@@ -147,13 +147,9 @@ pub(crate) fn unnecessary_generator_set(checker: &Checker, call: &ast::ExprCall)
         if *parenthesized {
             // The generator's range will include the innermost parentheses, but it could be
             // surrounded by additional parentheses.
-            let range = parenthesized_range(
-                argument.into(),
-                (&call.arguments).into(),
-                checker.comment_ranges(),
-                checker.locator().contents(),
-            )
-            .unwrap_or(argument.range());
+            let range =
+                parenthesized_range(argument.into(), (&call.arguments).into(), checker.tokens())
+                    .unwrap_or(argument.range());
 
             // The generator always parenthesizes the expression; trim the parentheses.
             let generator = checker.generator().expr(argument);

crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_list_comprehension_set.rs

@@ -1,7 +1,7 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_python_ast as ast;
-use ruff_python_ast::parenthesize::parenthesized_range;
 use ruff_python_ast::token::TokenKind;
+use ruff_python_ast::token::parenthesized_range;
 use ruff_text_size::{Ranged, TextRange, TextSize};
 
 use crate::checkers::ast::Checker;
@@ -89,13 +89,9 @@ pub(crate) fn unnecessary_list_comprehension_set(checker: &Checker, call: &ast::
 
     // If the list comprehension is parenthesized, remove the parentheses in addition to
     // removing the brackets.
-    let replacement_range = parenthesized_range(
-        argument.into(),
-        (&call.arguments).into(),
-        checker.comment_ranges(),
-        checker.locator().contents(),
-    )
-    .unwrap_or_else(|| argument.range());
+    let replacement_range =
+        parenthesized_range(argument.into(), (&call.arguments).into(), checker.tokens())
+            .unwrap_or_else(|| argument.range());
 
     let span = argument.range().add_start(one).sub_end(one);
     let replacement =

crates/ruff_linter/src/rules/flake8_implicit_str_concat/rules/explicit.rs

@@ -1,5 +1,5 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
-use ruff_python_ast::parenthesize::parenthesized_range;
+use ruff_python_ast::token::parenthesized_range;
 use ruff_python_ast::{self as ast, Expr, Operator};
 use ruff_python_trivia::is_python_whitespace;
 use ruff_source_file::LineRanges;
@@ -88,13 +88,7 @@ pub(crate) fn explicit(checker: &Checker, expr: &Expr) {
                     checker.report_diagnostic(ExplicitStringConcatenation, expr.range());
 
                 let is_parenthesized = |expr: &Expr| {
-                    parenthesized_range(
-                        expr.into(),
-                        bin_op.into(),
-                        checker.comment_ranges(),
-                        checker.source(),
-                    )
-                    .is_some()
+                    parenthesized_range(expr.into(), bin_op.into(), checker.tokens()).is_some()
                 };
                 // If either `left` or `right` is parenthesized, generating
                 // a fix would be too involved. Just report the diagnostic.

crates/ruff_linter/src/rules/flake8_logging/rules/exc_info_outside_except_handler.rs

@@ -111,7 +111,6 @@ pub(crate) fn exc_info_outside_except_handler(checker: &Checker, call: &ExprCall
     }
 
     let arguments = &call.arguments;
-    let source = checker.source();
 
     let mut diagnostic = checker.report_diagnostic(ExcInfoOutsideExceptHandler, exc_info.range);
 
@@ -120,8 +119,8 @@ pub(crate) fn exc_info_outside_except_handler(checker: &Checker, call: &ExprCall
             exc_info,
             arguments,
             Parentheses::Preserve,
-            source,
-            checker.comment_ranges(),
+            checker.source(),
+            checker.tokens(),
         )?;
         Ok(Fix::unsafe_edit(edit))
     });

crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_dict_kwargs.rs

@@ -2,7 +2,7 @@ use itertools::Itertools;
 use rustc_hash::{FxBuildHasher, FxHashSet};
 
 use ruff_macros::{ViolationMetadata, derive_message_formats};
-use ruff_python_ast::parenthesize::parenthesized_range;
+use ruff_python_ast::token::parenthesized_range;
 use ruff_python_ast::{self as ast, Expr};
 use ruff_python_stdlib::identifiers::is_identifier;
 use ruff_text_size::Ranged;
@@ -129,8 +129,8 @@ pub(crate) fn unnecessary_dict_kwargs(checker: &Checker, call: &ast::ExprCall) {
                     keyword,
                     &call.arguments,
                     Parentheses::Preserve,
-                    checker.locator().contents(),
-                    checker.comment_ranges(),
+                    checker.source(),
+                    checker.tokens(),
                 )
                 .map(Fix::safe_edit)
             });
@@ -158,8 +158,7 @@ pub(crate) fn unnecessary_dict_kwargs(checker: &Checker, call: &ast::ExprCall) {
                                         parenthesized_range(
                                             value.into(),
                                             dict.into(),
-                                            checker.comment_ranges(),
-                                            checker.locator().contents(),
+                                            checker.tokens()
                                         )
                                         .unwrap_or(value.range())
                                     )

crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_range_start.rs

@@ -73,11 +73,11 @@ pub(crate) fn unnecessary_range_start(checker: &Checker, call: &ast::ExprCall) {
     let mut diagnostic = checker.report_diagnostic(UnnecessaryRangeStart, start.range());
     diagnostic.try_set_fix(|| {
         remove_argument(
-            &start,
+            start,
             &call.arguments,
             Parentheses::Preserve,
-            checker.locator().contents(),
-            checker.comment_ranges(),
+            checker.source(),
+            checker.tokens(),
         )
         .map(Fix::safe_edit)
     });

crates/ruff_linter/src/rules/flake8_pyi/rules/generic_not_last_base_class.rs

@@ -160,20 +160,16 @@ fn generate_fix(
 ) -> anyhow::Result<Fix> {
     let locator = checker.locator();
     let source = locator.contents();
+    let tokens = checker.tokens();
 
     let deletion = remove_argument(
         generic_base,
         arguments,
         Parentheses::Preserve,
         source,
-        checker.comment_ranges(),
+        tokens,
     )?;
-    let insertion = add_argument(
-        locator.slice(generic_base),
-        arguments,
-        checker.comment_ranges(),
-        source,
-    );
+    let insertion = add_argument(locator.slice(generic_base), arguments, tokens);
 
     Ok(Fix::unsafe_edits(deletion, [insertion]))
 }