Rollup merge of rust-lang#126841 - c410-f3r:concat-again, r=petrochenkov

[`macro_metavar_expr_concat`] Add support for literals Adds support for things like `${concat($variable, 123)}` or `${concat("hello", "_world")}` . cc rust-lang#124225
jieyouxu · Jul 8, 2024 · 2c16d65 · 2c16d65
2 parents b1de36f c990e00
commit 2c16d65
Show file tree

Hide file tree

Showing 9 changed files with 272 additions and 59 deletions.
diff --git a/compiler/rustc_expand/src/mbe/metavar_expr.rs b/compiler/rustc_expand/src/mbe/metavar_expr.rs
@@ -1,14 1,15 @@
-use rustc_ast::token::{self, Delimiter, IdentIsRaw};
 use rustc_ast::token::{self, Delimiter, IdentIsRaw, Lit, Token, TokenKind};
 use rustc_ast::tokenstream::{RefTokenTreeCursor, TokenStream, TokenTree};
 use rustc_ast::{LitIntType, LitKind};
 use rustc_ast_pretty::pprust;
 use rustc_errors::{Applicability, PResult};
 use rustc_macros::{Decodable, Encodable};
 use rustc_session::parse::ParseSess;
 use rustc_span::symbol::Ident;
-use rustc_span::Span;
 use rustc_span::{Span, Symbol};
 
 pub(crate) const RAW_IDENT_ERR: &str = "`${concat(..)}` currently does not support raw identifiers";
 pub(crate) const UNSUPPORTED_CONCAT_ELEM_ERR: &str = "expected identifier or string literal";
 
 /// A meta-variable expression, for expansions based on properties of meta-variables.
 #[derive(Debug, PartialEq, Encodable, Decodable)]
@@ -51,11 52,26 @@ impl MetaVarExpr {
  let mut result = Vec::new();
  loop {
  let is_var = try_eat_dollar(&mut iter);
- let element_ident = parse_ident(&mut iter, psess, outer_span)?;
  let token = parse_token(&mut iter, psess, outer_span)?;
  let element = if is_var {
- MetaVarExprConcatElem::Var(element_ident)
  MetaVarExprConcatElem::Var(parse_ident_from_token(psess, token)?)
  } else if let TokenKind::Literal(Lit {
  kind: token::LitKind::Str,
  symbol,
  suffix: None,
  }) = token.kind
  {
  MetaVarExprConcatElem::Literal(symbol)
  } else {
- MetaVarExprConcatElem::Ident(element_ident)
  match parse_ident_from_token(psess, token) {
  Err(err) => {
  err.cancel();
  return Err(psess
  .dcx()
  .struct_span_err(token.span, UNSUPPORTED_CONCAT_ELEM_ERR));
  }
  Ok(elem) => MetaVarExprConcatElem::Ident(elem),
  }
  };
  result.push(element);
  if iter.look_ahead(0).is_none() {
@@ -105,11 121,13 @@ impl MetaVarExpr {
 
 #[derive(Debug, Decodable, Encodable, PartialEq)]
 pub(crate) enum MetaVarExprConcatElem {
- /// There is NO preceding dollar sign, which means that this identifier should be interpreted
- /// as a literal.
  /// Identifier WITHOUT a preceding dollar sign, which means that this identifier should be
  /// interpreted as a literal.
  Ident(Ident),
- /// There is a preceding dollar sign, which means that this identifier should be expanded
- /// and interpreted as a variable.
  /// For example, a number or a string.
  Literal(Symbol),
  /// Identifier WITH a preceding dollar sign, which means that this identifier should be
  /// expanded and interpreted as a variable.
  Var(Ident),
 }
 
@@ -158,7 176,7 @@ fn parse_depth<'psess>(
  span: Span,
 ) -> PResult<'psess, usize> {
  let Some(tt) = iter.next() else { return Ok(0) };
- let TokenTree::Token(token::Token { kind: token::TokenKind::Literal(lit), .. }, _) = tt else {
  let TokenTree::Token(Token { kind: TokenKind::Literal(lit), .. }, _) = tt else {
  return Err(psess
  .dcx()
  .struct_span_err(span, "meta-variable expression depth must be a literal"));
@@ -180,12 198,14 @@ fn parse_ident<'psess>(
  psess: &'psess ParseSess,
  fallback_span: Span,
 ) -> PResult<'psess, Ident> {
- let Some(tt) = iter.next() else {
- return Err(psess.dcx().struct_span_err(fallback_span, "expected identifier"));
- };
- let TokenTree::Token(token, _) = tt else {
- return Err(psess.dcx().struct_span_err(tt.span(), "expected identifier"));
- };
  let token = parse_token(iter, psess, fallback_span)?;
  parse_ident_from_token(psess, token)
 }
 
 fn parse_ident_from_token<'psess>(
  psess: &'psess ParseSess,
  token: &Token,
 ) -> PResult<'psess, Ident> {
  if let Some((elem, is_raw)) = token.ident() {
  if let IdentIsRaw::Yes = is_raw {
  return Err(psess.dcx().struct_span_err(elem.span, RAW_IDENT_ERR));
@@ -205,10 225,24 @@ fn parse_ident<'psess>(
  Err(err)
 }
 
 fn parse_token<'psess, 't>(
  iter: &mut RefTokenTreeCursor<'t>,
  psess: &'psess ParseSess,
  fallback_span: Span,
 ) -> PResult<'psess, &'t Token> {
  let Some(tt) = iter.next() else {
  return Err(psess.dcx().struct_span_err(fallback_span, UNSUPPORTED_CONCAT_ELEM_ERR));
  };
  let TokenTree::Token(token, _) = tt else {
  return Err(psess.dcx().struct_span_err(tt.span(), UNSUPPORTED_CONCAT_ELEM_ERR));
  };
  Ok(token)
 }
 
 /// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
 /// iterator is not modified and the result is `false`.
 fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
- if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) {
  if let Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) {
  let _ = iter.next();
  return true;
  }
@@ -218,8 252,7 @@ fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
 /// Tries to move the iterator forward returning `true` if there is a dollar sign. If not, then the
 /// iterator is not modified and the result is `false`.
 fn try_eat_dollar(iter: &mut RefTokenTreeCursor<'_>) -> bool {
- if let Some(TokenTree::Token(token::Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0)
- {
  if let Some(TokenTree::Token(Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0) {
  let _ = iter.next();
  return true;
  }
@@ -232,8 265,7 @@ fn eat_dollar<'psess>(
  psess: &'psess ParseSess,
  span: Span,
 ) -> PResult<'psess, ()> {
- if let Some(TokenTree::Token(token::Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0)
- {
  if let Some(TokenTree::Token(Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0) {
  let _ = iter.next();
  return Ok(());
  }

diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs
@@ -11,11 11,13 @@ use rustc_ast::token::{self, Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{pluralize, Diag, DiagCtxtHandle, PResult};
 use rustc_parse::lexer::nfc_normalize;
 use rustc_parse::parser::ParseNtResult;
 use rustc_session::parse::ParseSess;
 use rustc_session::parse::SymbolGallery;
 use rustc_span::hygiene::{LocalExpnId, Transparency};
 use rustc_span::symbol::{sym, Ident, MacroRulesNormalizedIdent};
-use rustc_span::{with_metavar_spans, Span, Symbol, SyntaxContext};
 use rustc_span::{with_metavar_spans, Span, SyntaxContext};
 use smallvec::{smallvec, SmallVec};
 use std::mem;
 
@@ -312,7 314,16 @@ pub(super) fn transcribe<'a>(
 
  // Replace meta-variable expressions with the result of their expansion.
  mbe::TokenTree::MetaVarExpr(sp, expr) => {
- transcribe_metavar_expr(dcx, expr, interp, &mut marker, &repeats, &mut result, sp)?;
  transcribe_metavar_expr(
  dcx,
  expr,
  interp,
  &mut marker,
  &repeats,
  &mut result,
  sp,
  &psess.symbol_gallery,
  )?;
  }
 
  // If we are entering a new delimiter, we push its contents to the `stack` to be
@@ -669,6 680,7 @@ fn transcribe_metavar_expr<'a>(
  repeats: &[(usize, usize)],
  result: &mut Vec<TokenTree>,
  sp: &DelimSpan,
  symbol_gallery: &SymbolGallery,
 ) -> PResult<'a, ()> {
  let mut visited_span = || {
  let mut span = sp.entire();
@@ -680,16 692,26 @@ fn transcribe_metavar_expr<'a>(
  let mut concatenated = String::new();
  for element in elements.into_iter() {
  let string = match element {
- MetaVarExprConcatElem::Ident(ident) => ident.to_string(),
- MetaVarExprConcatElem::Var(ident) => extract_ident(dcx, *ident, interp)?,
  MetaVarExprConcatElem::Ident(elem) => elem.to_string(),
  MetaVarExprConcatElem::Literal(elem) => elem.as_str().into(),
  MetaVarExprConcatElem::Var(elem) => extract_ident(dcx, *elem, interp)?,
  };
  concatenated.push_str(&string);
  }
  let symbol = nfc_normalize(&concatenated);
  let concatenated_span = visited_span();
  if !rustc_lexer::is_ident(symbol.as_str()) {
  return Err(dcx.struct_span_err(
  concatenated_span,
  "`${concat(..)}` is not generating a valid identifier",
  ));
  }
  symbol_gallery.insert(symbol, concatenated_span);
  // The current implementation marks the span as coming from the macro regardless of
  // contexts of the concatenated identifiers but this behavior may change in the
  // future.
  result.push(TokenTree::Token(
- Token::from_ast_ident(Ident::new(Symbol::intern(&concatenated), visited_span())),
  Token::from_ast_ident(Ident::new(symbol, concatenated_span)),
  Spacing::Alone,
  ));
  }

diff --git a/tests/ui/macros/macro-metavar-expr-concat/allowed-operations.rs b/tests/ui/macros/macro-metavar-expr-concat/allowed-operations.rs
@@ -37,6 37,16 @@ macro_rules! without_dollar_sign_is_an_ident {
  };
 }
 
 macro_rules! literals {
  ($ident:ident) => {{
  let ${concat(_a, "_b")}: () = ();
  let ${concat("_b", _a)}: () = ();
 
  let ${concat($ident, "_b")}: () = ();
  let ${concat("_b", $ident)}: () = ();
  }};
 }
 
 fn main() {
  create_things!(behold);
  behold_separated_idents_in_a_fn();
@@ -55,4 65,6 @@ fn main() {
  without_dollar_sign_is_an_ident!(_123);
  assert_eq!(VARident, 1);
  assert_eq!(VAR_123, 2);
 
  literals!(_hello);
 }
diff --git a/tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.rs b/tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.rs
@@ -26,14 26,14 @@ macro_rules! idents_11 {
 macro_rules! no_params {
  () => {
  let ${concat(r#abc, abc)}: () = ();
- //~^ ERROR `${concat(..)}` currently does not support raw identifiers
  //~^ ERROR expected identifier or string literal
  //~| ERROR expected pattern, found `$`
 
  let ${concat(abc, r#abc)}: () = ();
- //~^ ERROR `${concat(..)}` currently does not support raw identifiers
  //~^ ERROR expected identifier or string literal
 
  let ${concat(r#abc, r#abc)}: () = ();
- //~^ ERROR `${concat(..)}` currently does not support raw identifiers
  //~^ ERROR expected identifier or string literal
  };
 }
 

diff --git a/tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.stderr b/tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.stderr
@@ -1,16 1,16 @@
-error: `${concat(..)}` currently does not support raw identifiers
 error: expected identifier or string literal
  --> $DIR/raw-identifiers.rs:28:22
  |
 LL | let ${concat(r#abc, abc)}: () = ();
  | ^^^^^
 
-error: `${concat(..)}` currently does not support raw identifiers
 error: expected identifier or string literal
  --> $DIR/raw-identifiers.rs:32:27
  |
 LL | let ${concat(abc, r#abc)}: () = ();
  | ^^^^^
 
-error: `${concat(..)}` currently does not support raw identifiers
 error: expected identifier or string literal
  --> $DIR/raw-identifiers.rs:35:22
  |
 LL | let ${concat(r#abc, r#abc)}: () = ();

diff --git a/tests/ui/macros/macro-metavar-expr-concat/syntax-errors.rs b/tests/ui/macros/macro-metavar-expr-concat/syntax-errors.rs
@@ -11,9 11,6 @@ macro_rules! wrong_concat_declarations {
  ${concat(aaaa,)}
  //~^ ERROR expected identifier
 
- ${concat(aaaa, 1)}
- //~^ ERROR expected identifier
-
  ${concat(_, aaaa)}
 
  ${concat(aaaa aaaa)}
@@ -30,9 27,6 @@ macro_rules! wrong_concat_declarations {
 
  ${concat($ex, aaaa,)}
  //~^ ERROR expected identifier
-
- ${concat($ex, aaaa, 123)}
- //~^ ERROR expected identifier
  };
 }
 
@@ -43,8 37,80 @@ macro_rules! dollar_sign_without_referenced_ident {
  };
 }
 
 macro_rules! starting_number {
  ($ident:ident) => {{
  let ${concat("1", $ident)}: () = ();
  //~^ ERROR `${concat(..)}` is not generating a valid identifier
  }};
 }
 
 macro_rules! starting_valid_unicode {
  ($ident:ident) => {{
  let ${concat("Ý", $ident)}: () = ();
  }};
 }
 
 macro_rules! starting_invalid_unicode {
  ($ident:ident) => {{
  let ${concat("\u{00BD}", $ident)}: () = ();
  //~^ ERROR `${concat(..)}` is not generating a valid identifier
  }};
 }
 
 macro_rules! ending_number {
  ($ident:ident) => {{
  let ${concat($ident, "1")}: () = ();
  }};
 }
 
 macro_rules! ending_valid_unicode {
  ($ident:ident) => {{
  let ${concat($ident, "Ý")}: () = ();
  }};
 }
 
 macro_rules! ending_invalid_unicode {
  ($ident:ident) => {{
  let ${concat($ident, "\u{00BD}")}: () = ();
  //~^ ERROR `${concat(..)}` is not generating a valid identifier
  }};
 }
 
 macro_rules! empty {
  () => {{
  let ${concat("", "")}: () = ();
  //~^ ERROR `${concat(..)}` is not generating a valid identifier
  }};
 }
 
 macro_rules! unsupported_literals {
  ($ident:ident) => {{
  let ${concat(_a, 'b')}: () = ();
  //~^ ERROR expected identifier or string literal
  //~| ERROR expected pattern
  let ${concat(_a, 1)}: () = ();
  //~^ ERROR expected identifier or string literal
 
  let ${concat($ident, 'b')}: () = ();
  //~^ ERROR expected identifier or string literal
  let ${concat($ident, 1)}: () = ();
  //~^ ERROR expected identifier or string literal
  }};
 }
 
 fn main() {
  wrong_concat_declarations!(1);
 
  dollar_sign_without_referenced_ident!(VAR);
 
  starting_number!(_abc);
  starting_valid_unicode!(_abc);
  starting_invalid_unicode!(_abc);
 
  ending_number!(_abc);
  ending_valid_unicode!(_abc);
  ending_invalid_unicode!(_abc);
  unsupported_literals!(_abc);
 
  empty!();
 }