Skip to content

Commit

Permalink
Auto merge of rust-lang#133436 - nnethercote:rm-NtVis-NtTy, r=<try>
Browse files Browse the repository at this point in the history 
Remove `NtVis` and `NtTy`

The next part of rust-lang#124141. The first actual remove of `Nonterminal` variants. `NtVis` is a simple case that doesn't get much use, but `NtTy` is more complex.

r? `@petrochenkov`
  • Loading branch information
@bors
bors committed Nov 26, 2024
2 parents f2abf82 + 5552feb commit 4286768
Show file tree
Hide file tree
Showing 20 changed files with 197 additions and 60 deletions.
4 changes: 0 additions & 4 deletions compiler/rustc_ast/src/ast_traits.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,10 +203,8 @@ impl HasTokens for Nonterminal {
Nonterminal::NtStmt(stmt) => stmt.tokens(),
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
Nonterminal::NtPat(pat) => pat.tokens(),
Nonterminal::NtTy(ty) => ty.tokens(),
Nonterminal::NtMeta(attr_item) => attr_item.tokens(),
Nonterminal::NtPath(path) => path.tokens(),
Nonterminal::NtVis(vis) => vis.tokens(),
Nonterminal::NtBlock(block) => block.tokens(),
}
}
Expand All @@ -216,10 +214,8 @@ impl HasTokens for Nonterminal {
Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
Nonterminal::NtPat(pat) => pat.tokens_mut(),
Nonterminal::NtTy(ty) => ty.tokens_mut(),
Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
Nonterminal::NtPath(path) => path.tokens_mut(),
Nonterminal::NtVis(vis) => vis.tokens_mut(),
Nonterminal::NtBlock(block) => block.tokens_mut(),
}
}
Expand Down
2 changes: 0 additions & 2 deletions compiler/rustc_ast/src/mut_visit.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -884,7 +884,6 @@ fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
}),
token::NtPat(pat) => vis.visit_pat(pat),
token::NtExpr(expr) => vis.visit_expr(expr),
token::NtTy(ty) => vis.visit_ty(ty),
token::NtLiteral(expr) => vis.visit_expr(expr),
token::NtMeta(item) => {
let AttrItem { unsafety: _, path, args, tokens } = item.deref_mut();
Expand All @@ -893,7 +892,6 @@ fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
visit_lazy_tts(vis, tokens);
}
token::NtPath(path) => vis.visit_path(path),
token::NtVis(visib) => vis.visit_vis(visib),
}
}

Expand Down
20 changes: 10 additions & 10 deletions compiler/rustc_ast/src/token.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -660,7 +660,6 @@ impl Token {
| NtMeta(..)
| NtPat(..)
| NtPath(..)
| NtTy(..)
),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Expr { .. } |
Expand Down Expand Up @@ -689,7 +688,7 @@ impl Token {
Lifetime(..) | // lifetime bound in trait object
Lt | BinOp(Shl) | // associated path
PathSep => true, // global path
Interpolated(ref nt) => matches!(&**nt, NtTy(..) | NtPath(..)),
Interpolated(ref nt) => matches!(&**nt, NtPath(..)),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Ty |
MetaVarKind::Path
Expand Down Expand Up @@ -963,6 +962,15 @@ impl Token {
}
}

/// Is this an invisible open delimiter at the start of a token sequence
/// from an expanded metavar?
pub fn is_metavar_seq(&self) -> Option<MetaVarKind> {
match self.kind {
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => Some(kind),
_ => None,
}
}

pub fn glue(&self, joint: &Token) -> Option<Token> {
let kind = match self.kind {
Eq => match joint.kind {
Expand Down Expand Up @@ -1061,12 +1069,10 @@ pub enum Nonterminal {
NtStmt(P<ast::Stmt>),
NtPat(P<ast::Pat>),
NtExpr(P<ast::Expr>),
NtTy(P<ast::Ty>),
NtLiteral(P<ast::Expr>),
/// Stuff inside brackets for attributes
NtMeta(P<ast::AttrItem>),
NtPath(P<ast::Path>),
NtVis(P<ast::Visibility>),
}

#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
Expand Down Expand Up @@ -1160,10 +1166,8 @@ impl Nonterminal {
NtStmt(stmt) => stmt.span,
NtPat(pat) => pat.span,
NtExpr(expr) | NtLiteral(expr) => expr.span,
NtTy(ty) => ty.span,
NtMeta(attr_item) => attr_item.span(),
NtPath(path) => path.span,
NtVis(vis) => vis.span,
}
}

Expand All @@ -1175,10 +1179,8 @@ impl Nonterminal {
NtPat(..) => "pattern",
NtExpr(..) => "expression",
NtLiteral(..) => "literal",
NtTy(..) => "type",
NtMeta(..) => "attribute",
NtPath(..) => "path",
NtVis(..) => "visibility",
}
}
}
Expand All @@ -1201,11 +1203,9 @@ impl fmt::Debug for Nonterminal {
NtStmt(..) => f.pad("NtStmt(..)"),
NtPat(..) => f.pad("NtPat(..)"),
NtExpr(..) => f.pad("NtExpr(..)"),
NtTy(..) => f.pad("NtTy(..)"),
NtLiteral(..) => f.pad("NtLiteral(..)"),
NtMeta(..) => f.pad("NtMeta(..)"),
NtPath(..) => f.pad("NtPath(..)"),
NtVis(..) => f.pad("NtVis(..)"),
}
}
}
Expand Down
2 changes: 0 additions & 2 deletions compiler/rustc_ast/src/tokenstream.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -468,10 +468,8 @@ impl TokenStream {
}
Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
Nonterminal::NtPat(pat) => TokenStream::from_ast(pat),
Nonterminal::NtTy(ty) => TokenStream::from_ast(ty),
Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr),
Nonterminal::NtPath(path) => TokenStream::from_ast(path),
Nonterminal::NtVis(vis) => TokenStream::from_ast(vis),
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
}
}
Expand Down
39 changes: 37 additions & 2 deletions compiler/rustc_expand/src/mbe/transcribe.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ use std::mem;

use rustc_ast::ExprKind;
use rustc_ast::mut_visit::{self, MutVisitor};
use rustc_ast::token::{self, Delimiter, IdentIsRaw, Lit, LitKind, Nonterminal, Token, TokenKind};
use rustc_ast::token::{
self, Delimiter, IdentIsRaw, InvisibleOrigin, Lit, LitKind, MetaVarKind, Nonterminal, Token,
TokenKind,
};
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::Lrc;
Expand Down Expand Up @@ -253,7 +256,6 @@ pub(super) fn transcribe<'a>(
}
}

// Replace the meta-var with the matched token tree from the invocation.
mbe::TokenTree::MetaVar(mut sp, mut original_ident) => {
// Find the matched nonterminal from the macro invocation, and use it to replace
// the meta-var.
Expand All @@ -273,6 +275,33 @@ pub(super) fn transcribe<'a>(
// some of the unnecessary whitespace.
let ident = MacroRulesNormalizedIdent::new(original_ident);
if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) {
// We wrap the tokens in invisible delimiters, unless they are already wrapped
// in invisible delimiters with the same `MetaVarKind`. Because there is no
// point in having multiple layers of invisible delimiters of the same
// `MetaVarKind`. Indeed, some proc macros can't handle them.
let mut mk_delimited = |mv_kind, mut stream: TokenStream| {
if stream.len() == 1 {
let tree = stream.trees().next().unwrap();
if let TokenTree::Delimited(_, _, delim, inner) = tree
&& let Delimiter::Invisible(InvisibleOrigin::MetaVar(mvk)) = delim
&& mv_kind == *mvk
{
stream = inner.clone();
}
}

// Emit as a token stream within `Delimiter::Invisible` to maintain
// parsing priorities.
marker.visit_span(&mut sp);
// Both the open delim and close delim get the same span, which covers the
// `$foo` in the decl macro RHS.
TokenTree::Delimited(
DelimSpan::from_single(sp),
DelimSpacing::new(Spacing::Alone, Spacing::Alone),
Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)),
stream,
)
};
let tt = match cur_matched {
MatchedSingle(ParseNtResult::Tt(tt)) => {
// `tt`s are emitted into the output stream directly as "raw tokens",
Expand All @@ -289,6 +318,12 @@ pub(super) fn transcribe<'a>(
let kind = token::NtLifetime(*ident, *is_raw);
TokenTree::token_alone(kind, sp)
}
MatchedSingle(ParseNtResult::Ty(ty)) => {
mk_delimited(MetaVarKind::Ty, TokenStream::from_ast(ty))
}
MatchedSingle(ParseNtResult::Vis(vis)) => {
mk_delimited(MetaVarKind::Vis, TokenStream::from_ast(vis))
}
MatchedSingle(ParseNtResult::Nt(nt)) => {
// Other variables are emitted into the output stream as groups with
// `Delimiter::Invisible` to maintain parsing priorities.
Expand Down
79 changes: 72 additions & 7 deletions compiler/rustc_parse/src/parser/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,12 +116,16 @@ macro_rules! maybe_recover_from_interpolated_ty_qpath {
($self: expr, $allow_qpath_recovery: expr) => {
if $allow_qpath_recovery
&& $self.may_recover()
&& $self.look_ahead(1, |t| t == &token::PathSep)
&& let token::Interpolated(nt) = &$self.token.kind
&& let token::NtTy(ty) = &**nt
&& let Some(token::MetaVarKind::Ty) = $self.token.is_metavar_seq()
&& $self.check_noexpect_past_close_delim(&token::PathSep)
{
let ty = ty.clone();
$self.bump();
// Reparse the type, then move to recovery.
let ty = $self
.eat_metavar_seq(token::MetaVarKind::Ty, |this| {
this.parse_ty_no_question_mark_recover()
})
.expect("metavar seq ty");

return $self.maybe_recover_from_bad_qpath_stage_2($self.prev_token.span, ty);
}
};
Expand Down Expand Up @@ -611,6 +615,24 @@ impl<'a> Parser<'a> {
self.token == *tok
}

// Check the first token after the delimiter that closes the current
// delimited sequence. (Panics if used in the outermost token stream, which
// has no delimiters.) It uses a clone of the relevant tree cursor to skip
// past the entire `TokenTree::Delimited` in a single step, avoiding the
// need for unbounded token lookahead.
//
// Primarily used when `self.token` matches
// `OpenDelim(Delimiter::Invisible(_))`, to look ahead through the current
// metavar expansion.
fn check_noexpect_past_close_delim(&self, tok: &TokenKind) -> bool {
let mut tree_cursor = self.token_cursor.stack.last().unwrap().0.clone();
let tt = tree_cursor.next_ref();
matches!(
tt,
Some(ast::tokenstream::TokenTree::Token(token::Token { kind, .. }, _)) if kind == tok
)
}

/// Consumes a token 'tok' if it exists. Returns whether the given token was present.
///
/// the main purpose of this function is to reduce the cluttering of the suggestions list
Expand Down Expand Up @@ -720,6 +742,43 @@ impl<'a> Parser<'a> {
if !self.eat_keyword(kw) { self.unexpected() } else { Ok(()) }
}

/// Consume a sequence produced by a metavar expansion, if present.
fn eat_metavar_seq<T>(
&mut self,
mv_kind: MetaVarKind,
f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
) -> Option<T> {
self.eat_metavar_seq_with_matcher(|mvk| mvk == mv_kind, f)
}

/// A slightly more general form of `eat_metavar_seq`, for use with the
/// `MetaVarKind` variants that have parameters, where an exact match isn't
/// desired.
fn eat_metavar_seq_with_matcher<T>(
&mut self,
match_mv_kind: impl Fn(MetaVarKind) -> bool,
mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
) -> Option<T> {
if let token::OpenDelim(delim) = self.token.kind
&& let Delimiter::Invisible(token::InvisibleOrigin::MetaVar(mv_kind)) = delim
&& match_mv_kind(mv_kind)
{
self.bump();
let res = f(self).expect("failed to reparse {mv_kind:?}");
if let token::CloseDelim(delim) = self.token.kind
&& let Delimiter::Invisible(token::InvisibleOrigin::MetaVar(mv_kind)) = delim
&& match_mv_kind(mv_kind)
{
self.bump();
Some(res)
} else {
panic!("no close delim when reparsing {mv_kind:?}");
}
} else {
None
}
}

/// Is the given keyword `kw` followed by a non-reserved identifier?
fn is_kw_followed_by_ident(&self, kw: Symbol) -> bool {
self.token.is_keyword(kw) && self.look_ahead(1, |t| t.is_ident() && !t.is_reserved_ident())
Expand Down Expand Up @@ -1461,7 +1520,11 @@ impl<'a> Parser<'a> {
/// so emit a proper diagnostic.
// Public for rustfmt usage.
pub fn parse_visibility(&mut self, fbt: FollowedByType) -> PResult<'a, Visibility> {
maybe_whole!(self, NtVis, |vis| vis.into_inner());
if let Some(vis) = self
.eat_metavar_seq(MetaVarKind::Vis, |this| this.parse_visibility(FollowedByType::Yes))
{
return Ok(vis);
}

if !self.eat_keyword(kw::Pub) {
// We need a span for our `Spanned<VisibilityKind>`, but there's inherently no
Expand Down Expand Up @@ -1699,7 +1762,9 @@ pub enum ParseNtResult {
Tt(TokenTree),
Ident(Ident, IdentIsRaw),
Lifetime(Ident, IdentIsRaw),
Ty(P<ast::Ty>),
Vis(P<ast::Visibility>),

/// This case will eventually be removed, along with `Token::Interpolate`.
/// This variant will eventually be removed, along with `Token::Interpolate`.
Nt(Lrc<Nonterminal>),
}
18 changes: 9 additions & 9 deletions compiler/rustc_parse/src/parser/nonterminal.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,11 @@ impl<'a> Parser<'a> {
NtStmt(_)
| NtPat(_)
| NtExpr(_)
| NtTy(_)
| NtLiteral(_) // `true`, `false`
| NtMeta(_)
| NtPath(_) => true,

NtItem(_)
| NtBlock(_)
| NtVis(_) => false,
NtItem(_) | NtBlock(_) => false,
}
}

Expand Down Expand Up @@ -87,7 +84,7 @@ impl<'a> Parser<'a> {
NonterminalKind::Ident => get_macro_ident(token).is_some(),
NonterminalKind::Literal => token.can_begin_literal_maybe_minus(),
NonterminalKind::Vis => match token.kind {
// The follow-set of :vis + "priv" keyword + interpolated
// The follow-set of :vis + "priv" keyword + interpolated/metavar-expansion.
token::Comma
| token::Ident(..)
| token::NtIdent(..)
Expand All @@ -101,7 +98,7 @@ impl<'a> Parser<'a> {
token::NtLifetime(..) => true,
token::Interpolated(nt) => match &**nt {
NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) | NtVis(_) => false,
NtItem(_) | NtPat(_) | NtMeta(_) | NtPath(_) => false,
},
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
MetaVarKind::Block
Expand Down Expand Up @@ -188,7 +185,9 @@ impl<'a> Parser<'a> {
NtLiteral(self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus())?)
}
NonterminalKind::Ty => {
NtTy(self.collect_tokens_no_attrs(|this| this.parse_ty_no_question_mark_recover())?)
return Ok(ParseNtResult::Ty(
self.collect_tokens_no_attrs(|this| this.parse_ty_no_question_mark_recover())?,
));
}
// this could be handled like a token, since it is one
NonterminalKind::Ident => {
Expand All @@ -207,8 +206,9 @@ impl<'a> Parser<'a> {
}
NonterminalKind::Meta => NtMeta(P(self.parse_attr_item(ForceCollect::Yes)?)),
NonterminalKind::Vis => {
NtVis(P(self
.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?))
return Ok(ParseNtResult::Vis(P(self.collect_tokens_no_attrs(|this| {
this.parse_visibility(FollowedByType::Yes)
})?)));
}
NonterminalKind::Lifetime => {
// We want to keep `'keyword` parsing, just like `keyword` is still
Expand Down
17 changes: 9 additions & 8 deletions compiler/rustc_parse/src/parser/path.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use std::mem;

use ast::token::IdentIsRaw;
use rustc_ast::ptr::P;
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
use rustc_ast::token::{self, Delimiter, MetaVarKind, Token, TokenKind};
use rustc_ast::{
self as ast, AngleBracketedArg, AngleBracketedArgs, AnonConst, AssocItemConstraint,
AssocItemConstraintKind, BlockCheckMode, GenericArg, GenericArgs, Generics, ParenthesizedArgs,
Expand Down Expand Up @@ -198,13 +198,14 @@ impl<'a> Parser<'a> {

maybe_whole!(self, NtPath, |path| reject_generics_if_mod_style(self, path.into_inner()));

if let token::Interpolated(nt) = &self.token.kind {
if let token::NtTy(ty) = &**nt {
if let ast::TyKind::Path(None, path) = &ty.kind {
let path = path.clone();
self.bump();
return Ok(reject_generics_if_mod_style(self, path));
}
if let Some(MetaVarKind::Ty) = self.token.is_metavar_seq() {
let mut snapshot = self.create_snapshot_for_diagnostic();
let ty = snapshot
.eat_metavar_seq(MetaVarKind::Ty, |this| this.parse_ty_no_question_mark_recover())
.expect("metavar seq ty");
if let ast::TyKind::Path(None, path) = ty.into_inner().kind {
self.restore_snapshot(snapshot);
return Ok(reject_generics_if_mod_style(self, path));
}
}

Expand Down
Loading

0 comments on commit 4286768

Please sign in to comment.