Skip to content

Commit

Permalink
Remove NtIdent/NtLifetime/Nonterminal/Token::Interpolated.
Browse files Browse the repository at this point in the history 
Remnants of `Token::Interpolated` still exist in the form of the new
`token::IdentMv` and `token::LifetimeMv` tokens. I did them like that
because there's a lot of code that assumes an interpolated
ident/lifetime fits in a single token, and changing all that code to
work with invisible delimiters would have been a pain. (Maybe it could
be done in a follow-up.)

Fully kills off the "captured metavariables except for `:tt`, `:ident`
and `:lifetime` cannot be compared to other tokens" restriction.
  • Loading branch information
@nnethercote
nnethercote committed Aug 16, 2023
1 parent 5c1809a commit 67f4e3c
Show file tree
Hide file tree
Showing 16 changed files with 231 additions and 370 deletions.
14 changes: 0 additions & 14 deletions compiler/rustc_ast/src/ast_traits.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
//! The traits are not implemented exhaustively, only when actually necessary.
use crate::ptr::P;
use crate::token::Nonterminal;
use crate::tokenstream::LazyAttrTokenStream;
use crate::{Arm, Crate, ExprField, FieldDef, GenericParam, Param, PatField, Variant};
use crate::{AssocItem, Expr, ForeignItem, Item, NodeId};
Expand Down Expand Up @@ -228,19 +227,6 @@ impl HasTokens for Attribute {
}
}

impl HasTokens for Nonterminal {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
match self {
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
}
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
match self {
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
}
}
}

/// A trait for AST nodes having (or not having) attributes.
pub trait HasAttrs {
/// This is `true` if this `HasAttrs` might support 'custom' (proc-macro) inner
Expand Down
39 changes: 6 additions & 33 deletions compiler/rustc_ast/src/mut_visit.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -755,45 +755,18 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
*span = ident.span;
return; // Avoid visiting the span for the second time.
}
token::Interpolated(nt) => {
visit_nonterminal(Lrc::make_mut(nt), vis);
token::InterpolatedIdent(name, _, uninterpolated_span)
| token::InterpolatedLifetime(name, uninterpolated_span) => {
let mut ident = Ident::new(*name, *uninterpolated_span);
vis.visit_ident(&mut ident);
*name = ident.name;
*uninterpolated_span = ident.span;
}
_ => {}
}
vis.visit_span(span);
}

// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
/// Applies the visitor to elements of interpolated nodes.
//
// N.B., this can occur only when applying a visitor to partially expanded
// code, where parsed pieces have gotten implanted ito *other* macro
// invocations. This is relevant for macro hygiene, but possibly not elsewhere.
//
// One problem here occurs because the types for flat_map_item, flat_map_stmt,
// etc., allow the visitor to return *multiple* items; this is a problem for the
// nodes here, because they insist on having exactly one piece. One solution
// would be to mangle the MutVisitor trait to include one-to-many and
// one-to-one versions of these entry points, but that would probably confuse a
// lot of people and help very few. Instead, I'm just going to put in dynamic
// checks. I think the performance impact of this will be pretty much
// nonexistent. The danger is that someone will apply a `MutVisitor` to a
// partially expanded node, and will be confused by the fact that their
// `flat_map_item` or `flat_map_stmt` isn't getting called on `NtItem` or `NtStmt`
// nodes. Hopefully they'll wind up reading this comment, and doing something
// appropriate.
//
// BTW, design choice: I considered just changing the type of, e.g., `NtItem` to
// contain multiple items, but decided against it when I looked at
// `parse_item_or_view_item` and tried to figure out what I would do with
// multiple items there....
pub fn visit_nonterminal<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut T) {
match nt {
token::NtIdent(ident, _is_raw) => vis.visit_ident(ident),
token::NtLifetime(ident) => vis.visit_ident(ident),
}
}

// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
pub fn visit_defaultness<T: MutVisitor>(defaultness: &mut Defaultness, vis: &mut T) {
match defaultness {
Expand Down
184 changes: 75 additions & 109 deletions compiler/rustc_ast/src/token.rs
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
pub use BinOpToken::*;
pub use LitKind::*;
pub use Nonterminal::*;
pub use TokenKind::*;

use crate::ast;
use crate::util::case::Case;

use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_data_structures::sync::Lrc;
use rustc_macros::HashStable_Generic;
use rustc_span::symbol::{kw, sym};
#[allow(hidden_glob_reexports)]
Expand Down Expand Up @@ -262,9 +259,7 @@ fn ident_can_begin_type(name: Symbol, span: Span, is_raw: bool) -> bool {
.contains(&name)
}

// SAFETY: due to the `Clone` impl below, all fields of all variants other than
// `Interpolated` must impl `Copy`.
#[derive(PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
#[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
pub enum TokenKind {
/* Expression-operator symbols. */
Eq,
Expand Down Expand Up @@ -308,26 +303,23 @@ pub enum TokenKind {
Literal(Lit),

/// Identifier token.
/// Do not forget about `NtIdent` when you want to match on identifiers.
/// Do not forget about `InterpolatedIdent` when you want to match on identifiers.
/// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
/// treat regular and interpolated identifiers in the same way.
Ident(Symbol, /* is_raw */ bool),
/// This `Span` is the span of the original identifier passed to the
/// declarative macro. The span in the `Token` is the span of the `ident`
/// metavariable in the macro's RHS.
InterpolatedIdent(Symbol, /* is_raw */ bool, Span),
/// Lifetime identifier token.
/// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
/// Do not forget about `InterpolatedLIfetime` when you want to match on lifetime identifiers.
/// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
/// treat regular and interpolated lifetime identifiers in the same way.
Lifetime(Symbol),

/// An embedded AST node, as produced by a macro. This only exists for
/// historical reasons. We'd like to get rid of it, for multiple reasons.
/// - It's conceptually very strange. Saying a token can contain an AST
/// node is like saying, in natural language, that a word can contain a
/// sentence.
/// - It requires special handling in a bunch of places in the parser.
/// - It prevents `Token` from implementing `Copy`.
/// It adds complexity and likely slows things down. Please don't add new
/// occurrences of this token kind!
Interpolated(Lrc<Nonterminal>),
/// This `Span` is the span of the original lifetime passed to the
/// declarative macro. The span in the `Token` is the span of the
/// `lifetime` metavariable in the macro's RHS.
InterpolatedLifetime(Symbol, Span),

/// A doc comment token.
/// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc)
Expand All @@ -337,19 +329,6 @@ pub enum TokenKind {
Eof,
}

impl Clone for TokenKind {
fn clone(&self) -> Self {
// `TokenKind` would impl `Copy` if it weren't for `Interpolated`. So
// for all other variants, this implementation of `clone` is just like
// a copy. This is faster than the `derive(Clone)` version which has a
// separate path for every variant.
match self {
Interpolated(nt) => Interpolated(nt.clone()),
_ => unsafe { std::ptr::read(self) },
}
}
}

#[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
pub struct Token {
pub kind: TokenKind,
Expand Down Expand Up @@ -433,8 +412,12 @@ impl Token {
/// Note that keywords are also identifiers, so they should use this
/// if they keep spans or perform edition checks.
pub fn uninterpolated_span(&self) -> Span {
match &self.kind {
Interpolated(nt) => nt.span(),
match self.kind {
InterpolatedIdent(_, _, uninterpolated_span)
| InterpolatedLifetime(_, uninterpolated_span) => uninterpolated_span,
OpenDelim(Delimiter::Invisible(InvisibleSource::MetaVar(kind))) => {
panic!("njn: uninterpolated_span {kind:?}");
}
_ => self.span,
}
}
Expand All @@ -449,8 +432,15 @@ impl Token {
| BinOpEq(_) | At | Dot | DotDot | DotDotDot | DotDotEq | Comma | Semi | Colon
| ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question | SingleQuote => true,

OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
| Lifetime(..) | Interpolated(..) | Eof => false,
OpenDelim(..)
| CloseDelim(..)
| Literal(..)
| DocComment(..)
| Ident(..)
| InterpolatedIdent(..)
| Lifetime(..)
| InterpolatedLifetime(..)
| Eof => false,
}
}

Expand Down Expand Up @@ -612,13 +602,13 @@ impl Token {
/// into the regular identifier or lifetime token it refers to,
/// otherwise returns the original token.
pub fn uninterpolate(&self) -> Cow<'_, Token> {
match &self.kind {
Interpolated(nt) => match **nt {
NtIdent(ident, is_raw) => {
Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
}
NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
},
match self.kind {
InterpolatedIdent(name, is_raw, uninterpolated_span) => {
Cow::Owned(Token::new(Ident(name, is_raw), uninterpolated_span))
}
InterpolatedLifetime(name, uninterpolated_span) => {
Cow::Owned(Token::new(Lifetime(name), uninterpolated_span))
}
_ => Cow::Borrowed(self),
}
}
Expand All @@ -627,12 +617,11 @@ impl Token {
#[inline]
pub fn ident(&self) -> Option<(Ident, /* is_raw */ bool)> {
// We avoid using `Token::uninterpolate` here because it's slow.
match &self.kind {
&Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
Interpolated(nt) => match **nt {
NtIdent(ident, is_raw) => Some((ident, is_raw)),
_ => None,
},
match self.kind {
Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
InterpolatedIdent(name, is_raw, uninterpolated_span) => {
Some((Ident::new(name, uninterpolated_span), is_raw))
}
_ => None,
}
}
Expand All @@ -641,12 +630,11 @@ impl Token {
#[inline]
pub fn lifetime(&self) -> Option<Ident> {
// We avoid using `Token::uninterpolate` here because it's slow.
match &self.kind {
&Lifetime(name) => Some(Ident::new(name, self.span)),
Interpolated(nt) => match **nt {
NtLifetime(ident) => Some(ident),
_ => None,
},
match self.kind {
Lifetime(name) => Some(Ident::new(name, self.span)),
InterpolatedLifetime(name, uninterpolated_span) => {
Some(Ident::new(name, uninterpolated_span))
}
_ => None,
}
}
Expand Down Expand Up @@ -822,10 +810,35 @@ impl Token {
_ => return None,
},

Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
| DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
| Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None,
Le
| EqEq
| Ne
| Ge
| AndAnd
| OrOr
| Tilde
| BinOpEq(..)
| At
| DotDotDot
| DotDotEq
| Comma
| Semi
| ModSep
| RArrow
| LArrow
| FatArrow
| Pound
| Dollar
| Question
| OpenDelim(..)
| CloseDelim(..)
| Literal(..)
| Ident(..)
| InterpolatedIdent(..)
| Lifetime(..)
| InterpolatedLifetime(..)
| DocComment(..)
| Eof => return None,
};

Some(Token::new(kind, self.span.to(joint.span)))
Expand All @@ -839,13 +852,8 @@ impl PartialEq<TokenKind> for Token {
}
}

#[derive(Clone, Encodable, Decodable)]
/// For interpolation during macro expansion.
pub enum Nonterminal {
NtIdent(Ident, /* is_raw */ bool),
NtLifetime(Ident),
}

// njn: introduce cut-back version lacking Ident/Lifetime?
// - could that simplify the Pat cases too?
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
pub enum NonterminalKind {
Item,
Expand All @@ -859,6 +867,7 @@ pub enum NonterminalKind {
PatWithOr,
Expr,
Ty,
//njn: explain how these are never put in Invisible delims
Ident,
Lifetime,
Literal,
Expand Down Expand Up @@ -924,48 +933,6 @@ impl fmt::Display for NonterminalKind {
}
}

impl Nonterminal {
pub fn span(&self) -> Span {
match self {
NtIdent(ident, _) | NtLifetime(ident) => ident.span,
}
}
}

impl PartialEq for Nonterminal {
fn eq(&self, rhs: &Self) -> bool {
match (self, rhs) {
(NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => {
ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
}
(NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
// FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
// correctly based on data from AST. This will prevent them from matching each other
// in macros. The comparison will become possible only when each nonterminal has an
// attached token stream from which it was parsed.
_ => false,
}
}
}

impl fmt::Debug for Nonterminal {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
NtIdent(..) => f.pad("NtIdent(..)"),
NtLifetime(..) => f.pad("NtLifetime(..)"),
}
}
}

impl<CTX> HashStable<CTX> for Nonterminal
where
CTX: crate::HashStableContext,
{
fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) {
panic!("interpolated tokens should not be present in the HIR")
}
}

// Some types are used a lot. Make sure they don't unintentionally get bigger.
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
mod size_asserts {
Expand All @@ -974,7 +941,6 @@ mod size_asserts {
// tidy-alphabetical-start
static_assert_size!(Lit, 12);
static_assert_size!(LitKind, 2);
static_assert_size!(Nonterminal, 16);
static_assert_size!(Token, 24);
static_assert_size!(TokenKind, 16);
// tidy-alphabetical-end
Expand Down
Loading

0 comments on commit 67f4e3c

Please sign in to comment.