Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Translate expr scanner to table driven #1779

Merged
merged 5 commits into from
Nov 2, 2024
Merged

Translate expr scanner to table driven #1779

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7c102c3
Extract non-full expr scanner to module
dtolnay Nov 1, 2024
0132c44
Make scan_expr compilable from integration test
dtolnay Nov 1, 2024
8039cb3
Test that every expr can be scanned
dtolnay Nov 1, 2024
ca97c7d
Translate expr scanner to table driven
dtolnay Nov 1, 2024
0986a66
Ignore enum_glob_use pedantic clippy lint
dtolnay Nov 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 3 additions & 80 deletions src/data.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,8 @@ pub(crate) mod parsing {
use crate::parse::discouraged::Speculative as _;
use crate::parse::{Parse, ParseStream};
use crate::restriction::{FieldMutability, Visibility};
#[cfg(not(feature = "full"))]
use crate::scan_expr::scan_expr;
use crate::token;
use crate::ty::Type;
use crate::verbatim;
Expand Down Expand Up @@ -276,7 +278,7 @@ pub(crate) mod parsing {
let mut discriminant: Result<Expr> = ahead.parse();
if discriminant.is_ok() {
input.advance_to(&ahead);
} else if scan_lenient_discriminant(input).is_ok() {
} else if scan_expr(input).is_ok() {
discriminant = Ok(Expr::Verbatim(verbatim::between(&begin, input)));
}
discriminant?
Expand All @@ -294,85 +296,6 @@ pub(crate) mod parsing {
}
}

#[cfg(not(feature = "full"))]
pub(crate) fn scan_lenient_discriminant(input: ParseStream) -> Result<()> {
use crate::expr::Member;
use crate::lifetime::Lifetime;
use crate::lit::Lit;
use crate::lit::LitFloat;
use crate::op::{BinOp, UnOp};
use crate::path::{self, AngleBracketedGenericArguments};
use proc_macro2::Delimiter::{self, Brace, Bracket, Parenthesis};

let consume = |delimiter: Delimiter| {
Result::unwrap(input.step(|cursor| match cursor.group(delimiter) {
Some((_inside, _span, rest)) => Ok((true, rest)),
None => Ok((false, *cursor)),
}))
};

macro_rules! consume {
[$token:tt] => {
input.parse::<Option<Token![$token]>>().unwrap().is_some()
};
}

let mut initial = true;
let mut depth = 0usize;
loop {
if initial {
if consume![&] {
initial = consume![mut] || !consume![raw] || consume![const] || consume![mut];
} else if consume![if] || consume![match] || consume![while] {
depth += 1;
} else if input.parse::<Option<Lit>>()?.is_some()
|| (consume(Brace) || consume(Bracket) || consume(Parenthesis))
|| (consume![async] || consume![const] || consume![loop] || consume![unsafe])
&& (consume(Brace) || break)
{
initial = false;
} else if consume![let] {
while !consume![=] {
if !((consume![|] || consume![ref] || consume![mut] || consume![@])
|| (consume![!] || input.parse::<Option<Lit>>()?.is_some())
|| (consume![..=] || consume![..] || consume![&] || consume![_])
|| (consume(Brace) || consume(Bracket) || consume(Parenthesis)))
{
path::parsing::qpath(input, true)?;
}
}
} else if input.parse::<Option<Lifetime>>()?.is_some() && !consume![:] {
break;
} else if input.parse::<UnOp>().is_err() {
path::parsing::qpath(input, true)?;
initial = consume![!] || depth == 0 && input.peek(token::Brace);
}
} else if input.is_empty() || input.peek(Token![,]) {
return Ok(());
} else if depth > 0 && consume(Brace) {
if consume![else] && !consume(Brace) {
initial = consume![if] || break;
} else {
depth -= 1;
}
} else if input.parse::<BinOp>().is_ok() || (consume![..] | consume![=]) {
initial = true;
} else if consume![.] {
if input.parse::<Option<LitFloat>>()?.is_none()
&& (input.parse::<Member>()?.is_named() && consume![::])
{
AngleBracketedGenericArguments::do_parse(None, input)?;
}
} else if consume![as] {
input.parse::<Type>()?;
} else if !(consume(Brace) || consume(Bracket) || consume(Parenthesis)) {
break;
}
}

Err(input.error("unsupported expression"))
}

#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))]
impl Parse for FieldsNamed {
fn parse(input: ParseStream) -> Result<Self> {
Expand Down
6 changes: 6 additions & 0 deletions src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@
clippy::derivable_impls,
clippy::diverging_sub_expression,
clippy::doc_markdown,
clippy::enum_glob_use,
clippy::expl_impl_clone_on_copy,
clippy::explicit_auto_deref,
clippy::if_not_else,
Expand Down Expand Up @@ -307,6 +308,8 @@
clippy::wildcard_imports,
)]

extern crate self as syn;

#[cfg(feature = "proc-macro")]
extern crate proc_macro;

Expand Down Expand Up @@ -509,6 +512,9 @@ pub use crate::restriction::{FieldMutability, VisRestricted, Visibility};

mod sealed;

#[cfg(all(feature = "parsing", feature = "derive", not(feature = "full")))]
mod scan_expr;

mod span;

#[cfg(all(feature = "parsing", feature = "printing"))]
Expand Down
264 changes: 264 additions & 0 deletions src/scan_expr.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,264 @@
use self::{Action::*, Input::*};
use proc_macro2::{Delimiter, Ident, Spacing, TokenTree};
use syn::parse::{ParseStream, Result};
use syn::{AngleBracketedGenericArguments, BinOp, Expr, ExprPath, Lifetime, Lit, Token, Type};

enum Input {
Keyword(&'static str),
Punct(&'static str),
ConsumeAny,
ConsumeBinOp,
ConsumeBrace,
ConsumeDelimiter,
ConsumeIdent,
ConsumeLifetime,
ConsumeLiteral,
ConsumeNestedBrace,
ExpectPath,
ExpectTurbofish,
ExpectType,
CanBeginExpr,
Otherwise,
Empty,
}

enum Action {
SetState(&'static [(Input, Action)]),
IncDepth,
DecDepth,
Finish,
}

static INIT: [(Input, Action); 28] = [
(ConsumeDelimiter, SetState(&POSTFIX)),
(Keyword("async"), SetState(&ASYNC)),
(Keyword("break"), SetState(&BREAK_LABEL)),
(Keyword("const"), SetState(&CONST)),
(Keyword("continue"), SetState(&CONTINUE)),
(Keyword("for"), SetState(&FOR)),
(Keyword("if"), IncDepth),
(Keyword("let"), SetState(&PATTERN)),
(Keyword("loop"), SetState(&BLOCK)),
(Keyword("match"), IncDepth),
(Keyword("move"), SetState(&CLOSURE)),
(Keyword("return"), SetState(&RETURN)),
(Keyword("static"), SetState(&CLOSURE)),
(Keyword("unsafe"), SetState(&BLOCK)),
(Keyword("while"), IncDepth),
(Keyword("yield"), SetState(&RETURN)),
(Keyword("_"), SetState(&POSTFIX)),
(Punct("!"), SetState(&INIT)),
(Punct("#"), SetState(&[(ConsumeDelimiter, SetState(&INIT))])),
(Punct("&"), SetState(&REFERENCE)),
(Punct("*"), SetState(&INIT)),
(Punct("-"), SetState(&INIT)),
(Punct("..="), SetState(&INIT)),
(Punct(".."), SetState(&RANGE)),
(Punct("|"), SetState(&CLOSURE_ARGS)),
(ConsumeLifetime, SetState(&[(Punct(":"), SetState(&INIT))])),
(ConsumeLiteral, SetState(&POSTFIX)),
(ExpectPath, SetState(&PATH)),
];

static POSTFIX: [(Input, Action); 10] = [
(Keyword("as"), SetState(&[(ExpectType, SetState(&POSTFIX))])),
(Punct("..="), SetState(&INIT)),
(Punct(".."), SetState(&RANGE)),
(Punct("."), SetState(&DOT)),
(Punct("?"), SetState(&POSTFIX)),
(ConsumeBinOp, SetState(&INIT)),
(Punct("="), SetState(&INIT)),
(ConsumeNestedBrace, SetState(&IF_THEN)),
(ConsumeDelimiter, SetState(&POSTFIX)),
(Empty, Finish),
];

static ASYNC: [(Input, Action); 3] = [
(Keyword("move"), SetState(&ASYNC)),
(Punct("|"), SetState(&CLOSURE_ARGS)),
(ConsumeBrace, SetState(&POSTFIX)),
];

static BLOCK: [(Input, Action); 1] = [(ConsumeBrace, SetState(&POSTFIX))];

static BREAK_LABEL: [(Input, Action); 2] = [
(ConsumeLifetime, SetState(&BREAK_VALUE)),
(Otherwise, SetState(&BREAK_VALUE)),
];

static BREAK_VALUE: [(Input, Action); 3] = [
(ConsumeNestedBrace, SetState(&IF_THEN)),
(CanBeginExpr, SetState(&INIT)),
(Otherwise, SetState(&POSTFIX)),
];

static CLOSURE: [(Input, Action); 6] = [
(Keyword("async"), SetState(&CLOSURE)),
(Keyword("move"), SetState(&CLOSURE)),
(Punct(","), SetState(&CLOSURE)),
(Punct(">"), SetState(&CLOSURE)),
(Punct("|"), SetState(&CLOSURE_ARGS)),
(ConsumeLifetime, SetState(&CLOSURE)),
];

static CLOSURE_ARGS: [(Input, Action); 2] = [
(Punct("|"), SetState(&CLOSURE_RET)),
(ConsumeAny, SetState(&CLOSURE_ARGS)),
];

static CLOSURE_RET: [(Input, Action); 2] = [
(Punct("->"), SetState(&[(ExpectType, SetState(&BLOCK))])),
(Otherwise, SetState(&INIT)),
];

static CONST: [(Input, Action); 2] = [
(Punct("|"), SetState(&CLOSURE_ARGS)),
(ConsumeBrace, SetState(&POSTFIX)),
];

static CONTINUE: [(Input, Action); 2] = [
(ConsumeLifetime, SetState(&POSTFIX)),
(Otherwise, SetState(&POSTFIX)),
];

static DOT: [(Input, Action); 3] = [
(Keyword("await"), SetState(&POSTFIX)),
(ConsumeIdent, SetState(&METHOD)),
(ConsumeLiteral, SetState(&POSTFIX)),
];

static FOR: [(Input, Action); 2] = [
(Punct("<"), SetState(&CLOSURE)),
(Otherwise, SetState(&PATTERN)),
];

static IF_ELSE: [(Input, Action); 2] = [(Keyword("if"), SetState(&INIT)), (ConsumeBrace, DecDepth)];
static IF_THEN: [(Input, Action); 2] =
[(Keyword("else"), SetState(&IF_ELSE)), (Otherwise, DecDepth)];

static METHOD: [(Input, Action); 1] = [(ExpectTurbofish, SetState(&POSTFIX))];

static PATH: [(Input, Action); 4] = [
(Punct("!="), SetState(&INIT)),
(Punct("!"), SetState(&INIT)),
(ConsumeNestedBrace, SetState(&IF_THEN)),
(Otherwise, SetState(&POSTFIX)),
];

static PATTERN: [(Input, Action); 15] = [
(ConsumeDelimiter, SetState(&PATTERN)),
(Keyword("box"), SetState(&PATTERN)),
(Keyword("in"), IncDepth),
(Keyword("mut"), SetState(&PATTERN)),
(Keyword("ref"), SetState(&PATTERN)),
(Keyword("_"), SetState(&PATTERN)),
(Punct("!"), SetState(&PATTERN)),
(Punct("&"), SetState(&PATTERN)),
(Punct("..="), SetState(&PATTERN)),
(Punct(".."), SetState(&PATTERN)),
(Punct("="), SetState(&INIT)),
(Punct("@"), SetState(&PATTERN)),
(Punct("|"), SetState(&PATTERN)),
(ConsumeLiteral, SetState(&PATTERN)),
(ExpectPath, SetState(&PATTERN)),
];

static RANGE: [(Input, Action); 6] = [
(Punct("..="), SetState(&INIT)),
(Punct(".."), SetState(&RANGE)),
(Punct("."), SetState(&DOT)),
(ConsumeNestedBrace, SetState(&IF_THEN)),
(Empty, Finish),
(Otherwise, SetState(&INIT)),
];

static RAW: [(Input, Action); 3] = [
(Keyword("const"), SetState(&INIT)),
(Keyword("mut"), SetState(&INIT)),
(Otherwise, SetState(&POSTFIX)),
];

static REFERENCE: [(Input, Action); 3] = [
(Keyword("mut"), SetState(&INIT)),
(Keyword("raw"), SetState(&RAW)),
(Otherwise, SetState(&INIT)),
];

static RETURN: [(Input, Action); 2] = [
(CanBeginExpr, SetState(&INIT)),
(Otherwise, SetState(&POSTFIX)),
];

pub(crate) fn scan_expr(input: ParseStream) -> Result<()> {
let mut state = INIT.as_slice();
let mut depth = 0usize;
'table: loop {
for rule in state {
if match rule.0 {
Input::Keyword(expected) => input.step(|cursor| match cursor.ident() {
Some((ident, rest)) if ident == expected => Ok((true, rest)),
_ => Ok((false, *cursor)),
})?,
Input::Punct(expected) => input.step(|cursor| {
let begin = *cursor;
let mut cursor = begin;
for (i, ch) in expected.chars().enumerate() {
match cursor.punct() {
Some((punct, _)) if punct.as_char() != ch => break,
Some((_, rest)) if i == expected.len() - 1 => {
return Ok((true, rest));
}
Some((punct, rest)) if punct.spacing() == Spacing::Joint => {
cursor = rest;
}
_ => break,
}
}
Ok((false, begin))
})?,
Input::ConsumeAny => input.parse::<Option<TokenTree>>()?.is_some(),
Input::ConsumeBinOp => input.parse::<BinOp>().is_ok(),
Input::ConsumeBrace | Input::ConsumeNestedBrace => {
(matches!(rule.0, Input::ConsumeBrace) || depth > 0)
&& input.step(|cursor| match cursor.group(Delimiter::Brace) {
Some((_inside, _span, rest)) => Ok((true, rest)),
None => Ok((false, *cursor)),
})?
}
Input::ConsumeDelimiter => input.step(|cursor| match cursor.any_group() {
Some((_inside, _delimiter, _span, rest)) => Ok((true, rest)),
None => Ok((false, *cursor)),
})?,
Input::ConsumeIdent => input.parse::<Option<Ident>>()?.is_some(),
Input::ConsumeLifetime => input.parse::<Option<Lifetime>>()?.is_some(),
Input::ConsumeLiteral => input.parse::<Option<Lit>>()?.is_some(),
Input::ExpectPath => {
input.parse::<ExprPath>()?;
true
}
Input::ExpectTurbofish => {
if input.peek(Token![::]) {
input.parse::<AngleBracketedGenericArguments>()?;
}
true
}
Input::ExpectType => {
Type::without_plus(input)?;
true
}
Input::CanBeginExpr => Expr::peek(input),
Input::Otherwise => true,
Input::Empty => input.is_empty() || input.peek(Token![,]),
} {
state = match rule.1 {
Action::SetState(next) => next,
Action::IncDepth => (depth += 1, &INIT).1,
Action::DecDepth => (depth -= 1, &POSTFIX).1,
Action::Finish => return if depth == 0 { Ok(()) } else { break },
};
continue 'table;
}
}
return Err(input.error("unsupported expression"));
}
}
Loading
Loading