Skip to content

Reuse parser scratch buffers #26798

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
charliermarsh merged 10 commits into from
Jul 16, 2026
Merged

Reuse parser scratch buffers #26798

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
73fe3c4
Reuse parser scratch buffers
MichaReiser Jul 14, 2026
a8b26d1
Reuse parser scratch for call keywords
MichaReiser Jul 14, 2026
289fbd0
Consolidate parser scratch buffer handling
MichaReiser Jul 14, 2026
70767a6
Reuse parser scratch for parameters
MichaReiser Jul 14, 2026
e7986fc
Preallocate parser scratch buffers
MichaReiser Jul 14, 2026
ea98c47
Add parser scratch buffer debug safeguards
MichaReiser Jul 14, 2026
298e97c
Reuse parser scratch buffers in more statement paths
MichaReiser Jul 15, 2026
138e3f5
Simplify parser scratch buffer call sites
MichaReiser Jul 15, 2026
b3428f5
Make parser scratch buffers an explicit type
MichaReiser Jul 15, 2026
87174d6
Assert scratch buffers are empty on drop
MichaReiser Jul 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/ruff_python_parser/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ ruff_text_size = { workspace = true, features = ["get-size"] }
bitflags = { workspace = true }
bstr = { workspace = true }
compact_str = { workspace = true }
drop_bomb = { workspace = true }
get-size2 = { workspace = true }
memchr = { workspace = true }
rustc-hash = { workspace = true }
Expand Down
96 changes: 42 additions & 54 deletions crates/ruff_python_parser/src/parser/expression.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -815,8 +815,8 @@ impl<'src> Parser<'src> {
};
}

let mut args = vec![];
let mut keywords = vec![];
let args_snapshot = self.expr_scratch.snapshot();
let keywords_snapshot = self.keyword_scratch.snapshot();
let mut seen_keyword_argument = false; // foo = 1
let mut seen_keyword_unpacking = false; // **foo

Expand All @@ -826,7 +826,7 @@ impl<'src> Parser<'src> {
if parser.eat(TokenKind::DoubleStar) {
let value = parser.parse_conditional_expression_or_higher();

keywords.push(ast::Keyword {
parser.keyword_scratch.push(ast::Keyword {
arg: None,
value: value.expr,
range: parser.node_range(argument_start),
Expand Down Expand Up @@ -916,7 +916,7 @@ impl<'src> Parser<'src> {

let value = parser.parse_conditional_expression_or_higher();

keywords.push(ast::Keyword {
parser.keyword_scratch.push(ast::Keyword {
arg: Some(arg),
value: value.expr,
range: parser.node_range(argument_start),
Expand All @@ -936,23 +936,19 @@ impl<'src> Parser<'src> {
);
}
}
// Reserve exactly one slot for the first positional argument, while
// avoiding any allocation for keyword-only calls.
if args.is_empty() {
args.reserve_exact(1);
}
args.push(parsed_expr.expr);
parser.expr_scratch.push(parsed_expr.expr);
}
}
});

self.expect(TokenKind::Rpar);

let keywords = self.keyword_scratch.take_thin_vec(keywords_snapshot);
let arguments = ast::Arguments {
range: self.node_range(start),
node_index: AtomicNodeIndex::NONE,
args: args.into_boxed_slice(),
keywords: keywords.into(),
args: self.expr_scratch.take(args_snapshot),
keywords,
};

self.validate_arguments(&arguments, has_trailing_comma, context);
Expand Down Expand Up @@ -1005,16 +1001,16 @@ impl<'src> Parser<'src> {
// If there are more than one element in the slice, we need to create a tuple
// expression to represent it.
if self.eat(TokenKind::Comma) {
let mut slices = vec![slice];
let slices_snapshot = self.expr_scratch.snapshot();
self.expr_scratch.push(slice);

self.parse_comma_separated_list(RecoveryContextKind::Slices, |parser| {
slices.push(parser.parse_slice());
let slice = parser.parse_slice();
parser.expr_scratch.push(slice);
});

slices.shrink_to_fit();

slice = Expr::Tuple(ast::ExprTuple {
elts: slices,
elts: self.expr_scratch.take(slices_snapshot),
ctx: ExprContext::Load,
range: self.node_range(slice_start),
parenthesized: false,
Expand Down Expand Up @@ -1252,8 +1248,8 @@ impl<'src> Parser<'src> {
) -> ast::ExprBoolOp {
self.bump(TokenKind::from(op));

let mut values = Vec::with_capacity(2);
values.push(lhs);
let values_snapshot = self.expr_scratch.snapshot();
self.expr_scratch.push(lhs);
let mut progress = ParserProgress::default();

// Keep adding the expression to `values` until we see a different
Expand All @@ -1263,17 +1259,15 @@ impl<'src> Parser<'src> {

let parsed_expr =
self.parse_binary_expression_or_higher(OperatorPrecedence::from(op), context);
values.push(parsed_expr.expr);
self.expr_scratch.push(parsed_expr.expr);

if !self.eat(TokenKind::from(op)) {
break;
}
}

values.shrink_to_fit();

ast::ExprBoolOp {
values,
values: self.expr_scratch.take(values_snapshot),
op,
range: self.node_range(start),
node_index: AtomicNodeIndex::NONE,
Expand Down Expand Up @@ -1322,21 +1316,21 @@ impl<'src> Parser<'src> {
) -> ast::ExprCompare {
self.bump_cmp_op(op);

let mut comparators = vec![];
let comparators_snapshot = self.expr_scratch.snapshot();
let mut operators = vec![op];

let mut progress = ParserProgress::default();

loop {
progress.assert_progressing(self);

comparators.push(
self.parse_binary_expression_or_higher(
let comparator = self
.parse_binary_expression_or_higher(
OperatorPrecedence::ComparisonsMembershipIdentity,
context,
)
.expr,
);
.expr;
self.expr_scratch.push(comparator);

let next_token = self.current_token_kind();
if matches!(next_token, TokenKind::In) && context.is_in_excluded() {
Expand All @@ -1356,7 +1350,7 @@ impl<'src> Parser<'src> {
ast::ExprCompare {
left: Box::new(lhs),
ops: operators.into_boxed_slice(),
comparators: comparators.into_boxed_slice(),
comparators: self.expr_scratch.take(comparators_snapshot),
range: self.node_range(start),
node_index: AtomicNodeIndex::NONE,
}
Expand Down Expand Up @@ -2455,20 +2449,20 @@ impl<'src> Parser<'src> {
self.expect(TokenKind::Comma);
}

let mut elts = vec![first_element];
let elts_snapshot = self.expr_scratch.snapshot();
self.expr_scratch.push(first_element);

self.parse_comma_separated_list(RecoveryContextKind::TupleElements(parenthesized), |p| {
elts.push(parse_func(p).expr);
let element = parse_func(p).expr;
p.expr_scratch.push(element);
});

if parenthesized.is_yes() {
self.expect(TokenKind::Rpar);
}

elts.shrink_to_fit();

ast::ExprTuple {
elts,
elts: self.expr_scratch.take(elts_snapshot),
ctx: ExprContext::Load,
range: self.node_range(start),
node_index: AtomicNodeIndex::NONE,
Expand All @@ -2484,22 +2478,20 @@ impl<'src> Parser<'src> {
self.expect(TokenKind::Comma);
}

let mut elts = vec![first_element];
let elts_snapshot = self.expr_scratch.snapshot();
self.expr_scratch.push(first_element);

self.parse_comma_separated_list(RecoveryContextKind::ListElements, |parser| {
elts.push(
parser
.parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or())
.expr,
);
let element = parser
.parse_named_expression_or_higher(ExpressionContext::starred_bitwise_or())
.expr;
parser.expr_scratch.push(element);
});

self.expect(TokenKind::Rsqb);

elts.shrink_to_fit();

ast::ExprList {
elts,
elts: self.expr_scratch.take(elts_snapshot),
ctx: ExprContext::Load,
range: self.node_range(start),
node_index: AtomicNodeIndex::NONE,
Expand Down Expand Up @@ -2529,7 +2521,8 @@ impl<'src> Parser<'src> {
);
}

let mut elts = vec![first_element.expr];
let elts_snapshot = self.expr_scratch.snapshot();
self.expr_scratch.push(first_element.expr);

self.parse_comma_separated_list(RecoveryContextKind::SetElements, |parser| {
let parsed_expr =
Expand All @@ -2544,15 +2537,15 @@ impl<'src> Parser<'src> {
);
}

elts.push(parsed_expr.expr);
parser.expr_scratch.push(parsed_expr.expr);
});

self.expect(TokenKind::Rbrace);

ast::ExprSet {
range: self.node_range(start),
node_index: AtomicNodeIndex::NONE,
elts,
elts: self.expr_scratch.take(elts_snapshot),
}
}

Expand Down Expand Up @@ -2653,28 +2646,23 @@ impl<'src> Parser<'src> {
self.expect(TokenKind::In);
let iter = self.parse_simple_expression(ExpressionContext::default());

let mut ifs = Vec::new();
let ifs_snapshot = self.expr_scratch.snapshot();
let mut progress = ParserProgress::default();

while self.eat(TokenKind::If) {
progress.assert_progressing(self);

let parsed_expr = self.parse_simple_expression(ExpressionContext::default());

if ifs.is_empty() {
ifs.reserve_exact(1);
}
ifs.push(parsed_expr.expr);
self.expr_scratch.push(parsed_expr.expr);
}

ifs.shrink_to_fit();

ast::Comprehension {
range: self.node_range(start),
node_index: AtomicNodeIndex::NONE,
target: target.expr,
iter: iter.expr,
ifs,
ifs: self.expr_scratch.take(ifs_snapshot),
is_async,
}
}
Expand Down
30 changes: 28 additions & 2 deletions crates/ruff_python_parser/src/parser/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ use bitflags::bitflags;
use ruff_python_ast::name::Name;
use ruff_python_ast::token::TokenKind;
use ruff_python_ast::{
AtomicNodeIndex, Int, IpyEscapeKind, Mod, ModExpression, ModModule, StringFlags,
Alias, AtomicNodeIndex, ElifElseClause, Expr, Int, IpyEscapeKind, Keyword, Mod, ModExpression,
ModModule, ParameterWithDefault, Stmt, StringFlags,
};
use ruff_python_trivia::is_python_whitespace;
use ruff_text_size::{Ranged, TextRange, TextSize};
Expand All @@ -16,6 +17,7 @@ use unicode_normalization::UnicodeNormalization;
use crate::error::UnsupportedSyntaxError;
use crate::parser::expression::ExpressionContext;
use crate::parser::progress::{ParserProgress, TokenId};
use crate::parser::scratch_buffer::ScratchBuffer;
use crate::string::InterpolatedStringKind;
use crate::token_set::TokenSet;
use crate::token_source::{TokenSource, TokenSourceCheckpoint};
Expand All @@ -30,6 +32,7 @@ mod options;
mod pattern;
mod progress;
mod recovery;
mod scratch_buffer;
mod statement;
#[cfg(test)]
mod tests;
Expand Down Expand Up @@ -68,6 +71,24 @@ pub(crate) struct Parser<'src> {

/// Maximum lexer nesting depth before postfix calls and subscripts should stop recursing.
max_nesting_depth: u32,

/// Reusable, nesting-safe scratch storage for expression lists.
expr_scratch: ScratchBuffer<Expr>,

/// Reusable, nesting-safe scratch storage for call keywords.
keyword_scratch: ScratchBuffer<Keyword>,

/// Reusable, nesting-safe scratch storage for function and lambda parameters.
parameter_scratch: ScratchBuffer<ParameterWithDefault>,

/// Reusable, nesting-safe scratch storage for statement lists.
stmt_scratch: ScratchBuffer<Stmt>,

/// Reusable scratch storage for import aliases.
alias_scratch: ScratchBuffer<Alias>,

/// Reusable, nesting-safe scratch storage for `elif` and `else` clauses.
elif_else_scratch: ScratchBuffer<ElifElseClause>,
}

impl<'src> Parser<'src> {
Expand Down Expand Up @@ -98,6 +119,12 @@ impl<'src> Parser<'src> {
current_token_id: TokenId::default(),
depth_remaining,
max_nesting_depth,
expr_scratch: ScratchBuffer::with_capacity(16),
keyword_scratch: ScratchBuffer::new(),
parameter_scratch: ScratchBuffer::new(),
stmt_scratch: ScratchBuffer::with_capacity(32),
alias_scratch: ScratchBuffer::new(),
elif_else_scratch: ScratchBuffer::new(),
}
}

Expand Down Expand Up @@ -207,7 +234,6 @@ impl<'src> Parser<'src> {
TokenKind::EndOfFile,
"Parser should be at the end of the file."
);

// TODO consider re-integrating lexical error handling into the parser?
let parse_errors = self.errors;
let (tokens, lex_errors) = self.tokens.finish();
Expand Down
Loading
Loading