diff --git a/compiler/rustc_expand/src/mbe/metavar_expr.rs b/compiler/rustc_expand/src/mbe/metavar_expr.rs index 25958e03028f4..dbbd948fd7073 100644 --- a/compiler/rustc_expand/src/mbe/metavar_expr.rs +++ b/compiler/rustc_expand/src/mbe/metavar_expr.rs @@ -1,4 +1,4 @@ -use rustc_ast::token::{self, Delimiter, IdentIsRaw}; +use rustc_ast::token::{self, Delimiter, IdentIsRaw, Lit, Token, TokenKind}; use rustc_ast::tokenstream::{RefTokenTreeCursor, TokenStream, TokenTree}; use rustc_ast::{LitIntType, LitKind}; use rustc_ast_pretty::pprust; @@ -6,9 +6,10 @@ use rustc_errors::{Applicability, PResult}; use rustc_macros::{Decodable, Encodable}; use rustc_session::parse::ParseSess; use rustc_span::symbol::Ident; -use rustc_span::Span; +use rustc_span::{Span, Symbol}; pub(crate) const RAW_IDENT_ERR: &str = "`${concat(..)}` currently does not support raw identifiers"; +pub(crate) const UNSUPPORTED_CONCAT_ELEM_ERR: &str = "expected identifier or string literal"; /// A meta-variable expression, for expansions based on properties of meta-variables. #[derive(Debug, PartialEq, Encodable, Decodable)] @@ -51,11 +52,26 @@ impl MetaVarExpr { let mut result = Vec::new(); loop { let is_var = try_eat_dollar(&mut iter); - let element_ident = parse_ident(&mut iter, psess, outer_span)?; + let token = parse_token(&mut iter, psess, outer_span)?; let element = if is_var { - MetaVarExprConcatElem::Var(element_ident) + MetaVarExprConcatElem::Var(parse_ident_from_token(psess, token)?) + } else if let TokenKind::Literal(Lit { + kind: token::LitKind::Str, + symbol, + suffix: None, + }) = token.kind + { + MetaVarExprConcatElem::Literal(symbol) } else { - MetaVarExprConcatElem::Ident(element_ident) + match parse_ident_from_token(psess, token) { + Err(err) => { + err.cancel(); + return Err(psess + .dcx() + .struct_span_err(token.span, UNSUPPORTED_CONCAT_ELEM_ERR)); + } + Ok(elem) => MetaVarExprConcatElem::Ident(elem), + } }; result.push(element); if iter.look_ahead(0).is_none() { @@ -105,11 +121,13 @@ impl MetaVarExpr { #[derive(Debug, Decodable, Encodable, PartialEq)] pub(crate) enum MetaVarExprConcatElem { - /// There is NO preceding dollar sign, which means that this identifier should be interpreted - /// as a literal. + /// Identifier WITHOUT a preceding dollar sign, which means that this identifier should be + /// interpreted as a literal. Ident(Ident), - /// There is a preceding dollar sign, which means that this identifier should be expanded - /// and interpreted as a variable. + /// For example, a number or a string. + Literal(Symbol), + /// Identifier WITH a preceding dollar sign, which means that this identifier should be + /// expanded and interpreted as a variable. Var(Ident), } @@ -158,7 +176,7 @@ fn parse_depth<'psess>( span: Span, ) -> PResult<'psess, usize> { let Some(tt) = iter.next() else { return Ok(0) }; - let TokenTree::Token(token::Token { kind: token::TokenKind::Literal(lit), .. }, _) = tt else { + let TokenTree::Token(Token { kind: TokenKind::Literal(lit), .. }, _) = tt else { return Err(psess .dcx() .struct_span_err(span, "meta-variable expression depth must be a literal")); @@ -180,12 +198,14 @@ fn parse_ident<'psess>( psess: &'psess ParseSess, fallback_span: Span, ) -> PResult<'psess, Ident> { - let Some(tt) = iter.next() else { - return Err(psess.dcx().struct_span_err(fallback_span, "expected identifier")); - }; - let TokenTree::Token(token, _) = tt else { - return Err(psess.dcx().struct_span_err(tt.span(), "expected identifier")); - }; + let token = parse_token(iter, psess, fallback_span)?; + parse_ident_from_token(psess, token) +} + +fn parse_ident_from_token<'psess>( + psess: &'psess ParseSess, + token: &Token, +) -> PResult<'psess, Ident> { if let Some((elem, is_raw)) = token.ident() { if let IdentIsRaw::Yes = is_raw { return Err(psess.dcx().struct_span_err(elem.span, RAW_IDENT_ERR)); @@ -205,10 +225,24 @@ fn parse_ident<'psess>( Err(err) } +fn parse_token<'psess, 't>( + iter: &mut RefTokenTreeCursor<'t>, + psess: &'psess ParseSess, + fallback_span: Span, +) -> PResult<'psess, &'t Token> { + let Some(tt) = iter.next() else { + return Err(psess.dcx().struct_span_err(fallback_span, UNSUPPORTED_CONCAT_ELEM_ERR)); + }; + let TokenTree::Token(token, _) = tt else { + return Err(psess.dcx().struct_span_err(tt.span(), UNSUPPORTED_CONCAT_ELEM_ERR)); + }; + Ok(token) +} + /// Tries to move the iterator forward returning `true` if there is a comma. If not, then the /// iterator is not modified and the result is `false`. fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool { - if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) { + if let Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) { let _ = iter.next(); return true; } @@ -218,8 +252,7 @@ fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool { /// Tries to move the iterator forward returning `true` if there is a dollar sign. If not, then the /// iterator is not modified and the result is `false`. fn try_eat_dollar(iter: &mut RefTokenTreeCursor<'_>) -> bool { - if let Some(TokenTree::Token(token::Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0) - { + if let Some(TokenTree::Token(Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0) { let _ = iter.next(); return true; } @@ -232,8 +265,7 @@ fn eat_dollar<'psess>( psess: &'psess ParseSess, span: Span, ) -> PResult<'psess, ()> { - if let Some(TokenTree::Token(token::Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0) - { + if let Some(TokenTree::Token(Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0) { let _ = iter.next(); return Ok(()); } diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index f935f1b77e0b2..9b4dc13c703a1 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -11,11 +11,13 @@ use rustc_ast::token::{self, Delimiter, Token, TokenKind}; use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; use rustc_data_structures::fx::FxHashMap; use rustc_errors::{pluralize, Diag, DiagCtxtHandle, PResult}; +use rustc_parse::lexer::nfc_normalize; use rustc_parse::parser::ParseNtResult; use rustc_session::parse::ParseSess; +use rustc_session::parse::SymbolGallery; use rustc_span::hygiene::{LocalExpnId, Transparency}; use rustc_span::symbol::{sym, Ident, MacroRulesNormalizedIdent}; -use rustc_span::{with_metavar_spans, Span, Symbol, SyntaxContext}; +use rustc_span::{with_metavar_spans, Span, SyntaxContext}; use smallvec::{smallvec, SmallVec}; use std::mem; @@ -312,7 +314,16 @@ pub(super) fn transcribe<'a>( // Replace meta-variable expressions with the result of their expansion. mbe::TokenTree::MetaVarExpr(sp, expr) => { - transcribe_metavar_expr(dcx, expr, interp, &mut marker, &repeats, &mut result, sp)?; + transcribe_metavar_expr( + dcx, + expr, + interp, + &mut marker, + &repeats, + &mut result, + sp, + &psess.symbol_gallery, + )?; } // If we are entering a new delimiter, we push its contents to the `stack` to be @@ -669,6 +680,7 @@ fn transcribe_metavar_expr<'a>( repeats: &[(usize, usize)], result: &mut Vec, sp: &DelimSpan, + symbol_gallery: &SymbolGallery, ) -> PResult<'a, ()> { let mut visited_span = || { let mut span = sp.entire(); @@ -680,16 +692,26 @@ fn transcribe_metavar_expr<'a>( let mut concatenated = String::new(); for element in elements.into_iter() { let string = match element { - MetaVarExprConcatElem::Ident(ident) => ident.to_string(), - MetaVarExprConcatElem::Var(ident) => extract_ident(dcx, *ident, interp)?, + MetaVarExprConcatElem::Ident(elem) => elem.to_string(), + MetaVarExprConcatElem::Literal(elem) => elem.as_str().into(), + MetaVarExprConcatElem::Var(elem) => extract_ident(dcx, *elem, interp)?, }; concatenated.push_str(&string); } + let symbol = nfc_normalize(&concatenated); + let concatenated_span = visited_span(); + if !rustc_lexer::is_ident(symbol.as_str()) { + return Err(dcx.struct_span_err( + concatenated_span, + "`${concat(..)}` is not generating a valid identifier", + )); + } + symbol_gallery.insert(symbol, concatenated_span); // The current implementation marks the span as coming from the macro regardless of // contexts of the concatenated identifiers but this behavior may change in the // future. result.push(TokenTree::Token( - Token::from_ast_ident(Ident::new(Symbol::intern(&concatenated), visited_span())), + Token::from_ast_ident(Ident::new(symbol, concatenated_span)), Spacing::Alone, )); } diff --git a/tests/ui/macros/macro-metavar-expr-concat/allowed-operations.rs b/tests/ui/macros/macro-metavar-expr-concat/allowed-operations.rs index e44eeffb01be8..1acefa314aab8 100644 --- a/tests/ui/macros/macro-metavar-expr-concat/allowed-operations.rs +++ b/tests/ui/macros/macro-metavar-expr-concat/allowed-operations.rs @@ -37,6 +37,16 @@ macro_rules! without_dollar_sign_is_an_ident { }; } +macro_rules! literals { + ($ident:ident) => {{ + let ${concat(_a, "_b")}: () = (); + let ${concat("_b", _a)}: () = (); + + let ${concat($ident, "_b")}: () = (); + let ${concat("_b", $ident)}: () = (); + }}; +} + fn main() { create_things!(behold); behold_separated_idents_in_a_fn(); @@ -55,4 +65,6 @@ fn main() { without_dollar_sign_is_an_ident!(_123); assert_eq!(VARident, 1); assert_eq!(VAR_123, 2); + + literals!(_hello); } diff --git a/tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.rs b/tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.rs index f72b9baca89f8..b1cb2141cc42f 100644 --- a/tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.rs +++ b/tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.rs @@ -26,14 +26,14 @@ macro_rules! idents_11 { macro_rules! no_params { () => { let ${concat(r#abc, abc)}: () = (); - //~^ ERROR `${concat(..)}` currently does not support raw identifiers + //~^ ERROR expected identifier or string literal //~| ERROR expected pattern, found `$` let ${concat(abc, r#abc)}: () = (); - //~^ ERROR `${concat(..)}` currently does not support raw identifiers + //~^ ERROR expected identifier or string literal let ${concat(r#abc, r#abc)}: () = (); - //~^ ERROR `${concat(..)}` currently does not support raw identifiers + //~^ ERROR expected identifier or string literal }; } diff --git a/tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.stderr b/tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.stderr index dd525cf0801be..4e11e20acc551 100644 --- a/tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.stderr +++ b/tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.stderr @@ -1,16 +1,16 @@ -error: `${concat(..)}` currently does not support raw identifiers +error: expected identifier or string literal --> $DIR/raw-identifiers.rs:28:22 | LL | let ${concat(r#abc, abc)}: () = (); | ^^^^^ -error: `${concat(..)}` currently does not support raw identifiers +error: expected identifier or string literal --> $DIR/raw-identifiers.rs:32:27 | LL | let ${concat(abc, r#abc)}: () = (); | ^^^^^ -error: `${concat(..)}` currently does not support raw identifiers +error: expected identifier or string literal --> $DIR/raw-identifiers.rs:35:22 | LL | let ${concat(r#abc, r#abc)}: () = (); diff --git a/tests/ui/macros/macro-metavar-expr-concat/syntax-errors.rs b/tests/ui/macros/macro-metavar-expr-concat/syntax-errors.rs index bf47442ea76fb..b2845c8d1c1fc 100644 --- a/tests/ui/macros/macro-metavar-expr-concat/syntax-errors.rs +++ b/tests/ui/macros/macro-metavar-expr-concat/syntax-errors.rs @@ -11,9 +11,6 @@ macro_rules! wrong_concat_declarations { ${concat(aaaa,)} //~^ ERROR expected identifier - ${concat(aaaa, 1)} - //~^ ERROR expected identifier - ${concat(_, aaaa)} ${concat(aaaa aaaa)} @@ -30,9 +27,6 @@ macro_rules! wrong_concat_declarations { ${concat($ex, aaaa,)} //~^ ERROR expected identifier - - ${concat($ex, aaaa, 123)} - //~^ ERROR expected identifier }; } @@ -43,8 +37,80 @@ macro_rules! dollar_sign_without_referenced_ident { }; } +macro_rules! starting_number { + ($ident:ident) => {{ + let ${concat("1", $ident)}: () = (); + //~^ ERROR `${concat(..)}` is not generating a valid identifier + }}; +} + +macro_rules! starting_valid_unicode { + ($ident:ident) => {{ + let ${concat("Ý", $ident)}: () = (); + }}; +} + +macro_rules! starting_invalid_unicode { + ($ident:ident) => {{ + let ${concat("\u{00BD}", $ident)}: () = (); + //~^ ERROR `${concat(..)}` is not generating a valid identifier + }}; +} + +macro_rules! ending_number { + ($ident:ident) => {{ + let ${concat($ident, "1")}: () = (); + }}; +} + +macro_rules! ending_valid_unicode { + ($ident:ident) => {{ + let ${concat($ident, "Ý")}: () = (); + }}; +} + +macro_rules! ending_invalid_unicode { + ($ident:ident) => {{ + let ${concat($ident, "\u{00BD}")}: () = (); + //~^ ERROR `${concat(..)}` is not generating a valid identifier + }}; +} + +macro_rules! empty { + () => {{ + let ${concat("", "")}: () = (); + //~^ ERROR `${concat(..)}` is not generating a valid identifier + }}; +} + +macro_rules! unsupported_literals { + ($ident:ident) => {{ + let ${concat(_a, 'b')}: () = (); + //~^ ERROR expected identifier or string literal + //~| ERROR expected pattern + let ${concat(_a, 1)}: () = (); + //~^ ERROR expected identifier or string literal + + let ${concat($ident, 'b')}: () = (); + //~^ ERROR expected identifier or string literal + let ${concat($ident, 1)}: () = (); + //~^ ERROR expected identifier or string literal + }}; +} + fn main() { wrong_concat_declarations!(1); dollar_sign_without_referenced_ident!(VAR); + + starting_number!(_abc); + starting_valid_unicode!(_abc); + starting_invalid_unicode!(_abc); + + ending_number!(_abc); + ending_valid_unicode!(_abc); + ending_invalid_unicode!(_abc); + unsupported_literals!(_abc); + + empty!(); } diff --git a/tests/ui/macros/macro-metavar-expr-concat/syntax-errors.stderr b/tests/ui/macros/macro-metavar-expr-concat/syntax-errors.stderr index b216a86d59abe..2fe5842b39eb5 100644 --- a/tests/ui/macros/macro-metavar-expr-concat/syntax-errors.stderr +++ b/tests/ui/macros/macro-metavar-expr-concat/syntax-errors.stderr @@ -1,4 +1,4 @@ -error: expected identifier +error: expected identifier or string literal --> $DIR/syntax-errors.rs:5:10 | LL | ${concat()} @@ -10,59 +10,126 @@ error: `concat` must have at least two elements LL | ${concat(aaaa)} | ^^^^^^ -error: expected identifier +error: expected identifier or string literal --> $DIR/syntax-errors.rs:11:10 | LL | ${concat(aaaa,)} | ^^^^^^^^^^^^^^^ -error: expected identifier, found `1` - --> $DIR/syntax-errors.rs:14:24 - | -LL | ${concat(aaaa, 1)} - | ^ help: try removing `1` - error: expected comma - --> $DIR/syntax-errors.rs:19:10 + --> $DIR/syntax-errors.rs:16:10 | LL | ${concat(aaaa aaaa)} | ^^^^^^^^^^^^^^^^^^^ error: `concat` must have at least two elements - --> $DIR/syntax-errors.rs:22:11 + --> $DIR/syntax-errors.rs:19:11 | LL | ${concat($ex)} | ^^^^^^ error: expected comma - --> $DIR/syntax-errors.rs:28:10 + --> $DIR/syntax-errors.rs:25:10 | LL | ${concat($ex, aaaa 123)} | ^^^^^^^^^^^^^^^^^^^^^^^ -error: expected identifier - --> $DIR/syntax-errors.rs:31:10 +error: expected identifier or string literal + --> $DIR/syntax-errors.rs:28:10 | LL | ${concat($ex, aaaa,)} | ^^^^^^^^^^^^^^^^^^^^ -error: expected identifier, found `123` - --> $DIR/syntax-errors.rs:34:29 +error: expected identifier or string literal + --> $DIR/syntax-errors.rs:88:26 | -LL | ${concat($ex, aaaa, 123)} - | ^^^ help: try removing `123` +LL | let ${concat(_a, 'b')}: () = (); + | ^^^ + +error: expected identifier or string literal + --> $DIR/syntax-errors.rs:91:26 + | +LL | let ${concat(_a, 1)}: () = (); + | ^ + +error: expected identifier or string literal + --> $DIR/syntax-errors.rs:94:30 + | +LL | let ${concat($ident, 'b')}: () = (); + | ^^^ + +error: expected identifier or string literal + --> $DIR/syntax-errors.rs:96:30 + | +LL | let ${concat($ident, 1)}: () = (); + | ^ error: `${concat(..)}` currently only accepts identifiers or meta-variables as parameters - --> $DIR/syntax-errors.rs:25:19 + --> $DIR/syntax-errors.rs:22:19 | LL | ${concat($ex, aaaa)} | ^^ error: variable `foo` is not recognized in meta-variable expression - --> $DIR/syntax-errors.rs:41:30 + --> $DIR/syntax-errors.rs:35:30 | LL | const ${concat(FOO, $foo)}: i32 = 2; | ^^^ -error: aborting due to 11 previous errors +error: `${concat(..)}` is not generating a valid identifier + --> $DIR/syntax-errors.rs:42:14 + | +LL | let ${concat("1", $ident)}: () = (); + | ^^^^^^^^^^^^^^^^^^^^^ +... +LL | starting_number!(_abc); + | ---------------------- in this macro invocation + | + = note: this error originates in the macro `starting_number` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: `${concat(..)}` is not generating a valid identifier + --> $DIR/syntax-errors.rs:55:14 + | +LL | let ${concat("\u{00BD}", $ident)}: () = (); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +... +LL | starting_invalid_unicode!(_abc); + | ------------------------------- in this macro invocation + | + = note: this error originates in the macro `starting_invalid_unicode` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: `${concat(..)}` is not generating a valid identifier + --> $DIR/syntax-errors.rs:74:14 + | +LL | let ${concat($ident, "\u{00BD}")}: () = (); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +... +LL | ending_invalid_unicode!(_abc); + | ----------------------------- in this macro invocation + | + = note: this error originates in the macro `ending_invalid_unicode` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: expected pattern, found `$` + --> $DIR/syntax-errors.rs:88:13 + | +LL | let ${concat(_a, 'b')}: () = (); + | ^ expected pattern +... +LL | unsupported_literals!(_abc); + | --------------------------- in this macro invocation + | + = note: this error originates in the macro `unsupported_literals` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: `${concat(..)}` is not generating a valid identifier + --> $DIR/syntax-errors.rs:81:14 + | +LL | let ${concat("", "")}: () = (); + | ^^^^^^^^^^^^^^^^ +... +LL | empty!(); + | -------- in this macro invocation + | + = note: this error originates in the macro `empty` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: aborting due to 18 previous errors diff --git a/tests/ui/macros/macro-metavar-expr-concat/unicode-expansion.rs b/tests/ui/macros/macro-metavar-expr-concat/unicode-expansion.rs new file mode 100644 index 0000000000000..b2cfb211e2d1e --- /dev/null +++ b/tests/ui/macros/macro-metavar-expr-concat/unicode-expansion.rs @@ -0,0 +1,14 @@ +//@ run-pass + +#![feature(macro_metavar_expr_concat)] + +macro_rules! turn_to_page { + ($ident:ident) => { + const ${concat("Ḧ", $ident)}: i32 = 394; + }; +} + +fn main() { + turn_to_page!(P); + assert_eq!(ḦP, 394); +} diff --git a/tests/ui/macros/rfc-3086-metavar-expr/syntax-errors.stderr b/tests/ui/macros/rfc-3086-metavar-expr/syntax-errors.stderr index 8e4ba192d79f5..2c44ad2e0a4ad 100644 --- a/tests/ui/macros/rfc-3086-metavar-expr/syntax-errors.stderr +++ b/tests/ui/macros/rfc-3086-metavar-expr/syntax-errors.stderr @@ -190,7 +190,7 @@ error: unrecognized meta-variable expression LL | ( $( $i:ident ),* ) => { ${ aaaaaaaaaaaaaa(i) } }; | ^^^^^^^^^^^^^^ help: supported expressions are count, ignore, index and len -error: expected identifier +error: expected identifier or string literal --> $DIR/syntax-errors.rs:118:33 | LL | ( $( $i:ident ),* ) => { ${ {} } };