diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs index 940cc30c17e6e..090bcaf45c73d 100644 --- a/compiler/rustc_middle/src/query/erase.rs +++ b/compiler/rustc_middle/src/query/erase.rs @@ -1,3 +1,10 @@ +//! To improve compile times and code size for the compiler itself, query +//! values are "erased" in some contexts (e.g. inside in-memory cache types), +//! to reduce the number of generic instantiations created during codegen. +//! +//! See for some bootstrap-time +//! and performance benchmarks. + use std::ffi::OsStr; use std::intrinsics::transmute_unchecked; use std::mem::MaybeUninit; @@ -14,138 +21,169 @@ use crate::ty::adjustment::CoerceUnsizedInfo; use crate::ty::{self, Ty, TyCtxt}; use crate::{mir, traits}; +/// Internal implementation detail of [`Erased`]. #[derive(Copy, Clone)] -pub struct Erased { - // We use `MaybeUninit` here so we can store any value - // in `data` since we aren't actually storing a `T`. - data: MaybeUninit, -} - -pub trait EraseType: Copy { - type Result: Copy; -} - -// Allow `type_alias_bounds` since compilation will fail without `EraseType`. -#[allow(type_alias_bounds)] -pub type Erase = Erased; - +pub struct ErasedData { + /// We use `MaybeUninit` here to make sure it's legal to store a transmuted + /// value that isn't actually of type `Storage`. + data: MaybeUninit, +} + +/// Trait for types that can be erased into [`Erased`]. +/// +/// Erasing and unerasing values is performed by [`erase_val`] and [`restore_val`]. +/// +/// FIXME: This whole trait could potentially be replaced by `T: Copy` and the +/// storage type `[u8; size_of::()]` when support for that is more mature. +pub trait Erasable: Copy { + /// Storage type to used for erased values of this type. + /// Should be `[u8; N]`, where N is equal to `size_of::`. + /// + /// [`ErasedData`] wraps this storage type in `MaybeUninit` to ensure that + /// transmutes to/from erased storage are well-defined. + type Storage: Copy; +} + +/// A value of `T` that has been "erased" into some opaque storage type. +/// +/// This is helpful for reducing the number of concrete instantiations needed +/// during codegen when building the compiler. +/// +/// Using an opaque type alias allows the type checker to enforce that +/// `Erased` and `Erased` are still distinct types, while allowing +/// monomorphization to see that they might actually use the same storage type. +pub type Erased = ErasedData; + +/// Erases a value of type `T` into `Erased`. +/// +/// `Erased` and `Erased` are type-checked as distinct types, but codegen +/// can see whether they actually have the same storage type. +/// +/// FIXME: This might have soundness issues with erasable types that don't +/// implement the same auto-traits as `[u8; _]`; see +/// #[inline(always)] -#[define_opaque(Erase)] -pub fn erase(src: T) -> Erase { +#[define_opaque(Erased)] +pub fn erase_val(value: T) -> Erased { // Ensure the sizes match const { - if size_of::() != size_of::() { - panic!("size of T must match erased type T::Result") + if size_of::() != size_of::() { + panic!("size of T must match erased type ::Storage") } }; - Erased::<::Result> { + ErasedData::<::Storage> { // `transmute_unchecked` is needed here because it does not have `transmute`'s size check - // (and thus allows to transmute between `T` and `MaybeUninit`) (we do the size + // (and thus allows to transmute between `T` and `MaybeUninit`) (we do the size // check ourselves in the `const` block above). // // `transmute_copy` is also commonly used for this (and it would work here since - // `EraseType: Copy`), but `transmute_unchecked` better explains the intent. + // `Erasable: Copy`), but `transmute_unchecked` better explains the intent. // // SAFETY: It is safe to transmute to MaybeUninit for types with the same sizes. - data: unsafe { transmute_unchecked::>(src) }, + data: unsafe { transmute_unchecked::>(value) }, } } -/// Restores an erased value. +/// Restores an erased value to its real type. +/// +/// This relies on the fact that `Erased` and `Erased` are type-checked +/// as distinct types, even if they use the same storage type. #[inline(always)] -#[define_opaque(Erase)] -pub fn restore(value: Erase) -> T { - let value: Erased<::Result> = value; - // See comment in `erase` for why we use `transmute_unchecked`. +#[define_opaque(Erased)] +pub fn restore_val(erased_value: Erased) -> T { + let ErasedData { data }: ErasedData<::Storage> = erased_value; + // See comment in `erase_val` for why we use `transmute_unchecked`. // - // SAFETY: Due to the use of impl Trait in `Erase` the only way to safely create an instance - // of `Erase` is to call `erase`, so we know that `value.data` is a valid instance of `T` of - // the right size. - unsafe { transmute_unchecked::, T>(value.data) } + // SAFETY: Due to the use of impl Trait in `Erased` the only way to safely create an instance + // of `Erased` is to call `erase_val`, so we know that `erased_value.data` is a valid instance + // of `T` of the right size. + unsafe { transmute_unchecked::, T>(data) } } -impl EraseType for &'_ T { - type Result = [u8; size_of::<&'static ()>()]; +// FIXME(#151565): Using `T: ?Sized` here should let us remove the separate +// impls for fat reference types. +impl Erasable for &'_ T { + type Storage = [u8; size_of::<&'static ()>()]; } -impl EraseType for &'_ [T] { - type Result = [u8; size_of::<&'static [()]>()]; +impl Erasable for &'_ [T] { + type Storage = [u8; size_of::<&'static [()]>()]; } -impl EraseType for &'_ OsStr { - type Result = [u8; size_of::<&'static OsStr>()]; +impl Erasable for &'_ OsStr { + type Storage = [u8; size_of::<&'static OsStr>()]; } -impl EraseType for &'_ ty::List { - type Result = [u8; size_of::<&'static ty::List<()>>()]; +impl Erasable for &'_ ty::List { + type Storage = [u8; size_of::<&'static ty::List<()>>()]; } -impl EraseType for &'_ ty::ListWithCachedTypeInfo { - type Result = [u8; size_of::<&'static ty::ListWithCachedTypeInfo<()>>()]; +impl Erasable for &'_ ty::ListWithCachedTypeInfo { + type Storage = [u8; size_of::<&'static ty::ListWithCachedTypeInfo<()>>()]; } -impl EraseType for &'_ rustc_index::IndexSlice { - type Result = [u8; size_of::<&'static rustc_index::IndexSlice>()]; +impl Erasable for &'_ rustc_index::IndexSlice { + type Storage = [u8; size_of::<&'static rustc_index::IndexSlice>()]; } -impl EraseType for Result<&'_ T, traits::query::NoSolution> { - type Result = [u8; size_of::>()]; +impl Erasable for Result<&'_ T, traits::query::NoSolution> { + type Storage = [u8; size_of::>()]; } -impl EraseType for Result<&'_ [T], traits::query::NoSolution> { - type Result = [u8; size_of::>()]; +impl Erasable for Result<&'_ [T], traits::query::NoSolution> { + type Storage = [u8; size_of::>()]; } -impl EraseType for Result<&'_ T, rustc_errors::ErrorGuaranteed> { - type Result = [u8; size_of::>()]; +impl Erasable for Result<&'_ T, rustc_errors::ErrorGuaranteed> { + type Storage = [u8; size_of::>()]; } -impl EraseType for Result<&'_ [T], rustc_errors::ErrorGuaranteed> { - type Result = [u8; size_of::>()]; +impl Erasable for Result<&'_ [T], rustc_errors::ErrorGuaranteed> { + type Storage = [u8; size_of::>()]; } -impl EraseType for Result<&'_ T, traits::CodegenObligationError> { - type Result = [u8; size_of::>()]; +impl Erasable for Result<&'_ T, traits::CodegenObligationError> { + type Storage = [u8; size_of::>()]; } -impl EraseType for Result<&'_ T, &'_ ty::layout::FnAbiError<'_>> { - type Result = [u8; size_of::>>()]; +impl Erasable for Result<&'_ T, &'_ ty::layout::FnAbiError<'_>> { + type Storage = [u8; size_of::>>()]; } -impl EraseType for Result<(&'_ T, crate::thir::ExprId), rustc_errors::ErrorGuaranteed> { - type Result = [u8; size_of::< +impl Erasable for Result<(&'_ T, crate::thir::ExprId), rustc_errors::ErrorGuaranteed> { + type Storage = [u8; size_of::< Result<(&'static (), crate::thir::ExprId), rustc_errors::ErrorGuaranteed>, >()]; } -impl EraseType for Result>, rustc_errors::ErrorGuaranteed> { - type Result = +impl Erasable for Result>, rustc_errors::ErrorGuaranteed> { + type Storage = [u8; size_of::>, rustc_errors::ErrorGuaranteed>>()]; } -impl EraseType for Result { - type Result = [u8; size_of::>()]; +impl Erasable for Result { + type Storage = [u8; size_of::>()]; } -impl EraseType +impl Erasable for Result>>, rustc_errors::ErrorGuaranteed> { - type Result = [u8; size_of::< + type Storage = [u8; size_of::< Result>>, rustc_errors::ErrorGuaranteed>, >()]; } -impl EraseType for Result, traits::query::NoSolution> { - type Result = [u8; size_of::, traits::query::NoSolution>>()]; +impl Erasable for Result, traits::query::NoSolution> { + type Storage = [u8; size_of::, traits::query::NoSolution>>()]; } -impl EraseType for Result> { - type Result = [u8; size_of::>>()]; +impl Erasable for Result> { + type Storage = [u8; size_of::>>()]; } -impl EraseType for Result>, &ty::layout::LayoutError<'_>> { - type Result = [u8; size_of::< +impl Erasable for Result>, &ty::layout::LayoutError<'_>> { + type Storage = [u8; size_of::< Result< rustc_abi::TyAndLayout<'static, Ty<'static>>, &'static ty::layout::LayoutError<'static>, @@ -153,35 +191,36 @@ impl EraseType for Result>, &ty::layout::Layou >()]; } -impl EraseType for Result, mir::interpret::ErrorHandled> { - type Result = [u8; size_of::, mir::interpret::ErrorHandled>>()]; +impl Erasable for Result, mir::interpret::ErrorHandled> { + type Storage = + [u8; size_of::, mir::interpret::ErrorHandled>>()]; } -impl EraseType for Result { - type Result = [u8; size_of::>()]; +impl Erasable for Result { + type Storage = [u8; size_of::>()]; } -impl EraseType for Option<(mir::ConstValue, Ty<'_>)> { - type Result = [u8; size_of::)>>()]; +impl Erasable for Option<(mir::ConstValue, Ty<'_>)> { + type Storage = [u8; size_of::)>>()]; } -impl EraseType for EvalToValTreeResult<'_> { - type Result = [u8; size_of::>()]; +impl Erasable for EvalToValTreeResult<'_> { + type Storage = [u8; size_of::>()]; } -impl EraseType for Result<&'_ ty::List>, ty::util::AlwaysRequiresDrop> { - type Result = +impl Erasable for Result<&'_ ty::List>, ty::util::AlwaysRequiresDrop> { + type Storage = [u8; size_of::>, ty::util::AlwaysRequiresDrop>>()]; } -impl EraseType for Result>, CyclePlaceholder> { - type Result = [u8; size_of::>, CyclePlaceholder>>()]; +impl Erasable for Result>, CyclePlaceholder> { + type Storage = [u8; size_of::>, CyclePlaceholder>>()]; } -impl EraseType +impl Erasable for Result<(&'_ [Spanned>], &'_ [Spanned>]), NormalizationErrorInMono> { - type Result = [u8; size_of::< + type Storage = [u8; size_of::< Result< (&'static [Spanned>], &'static [Spanned>]), NormalizationErrorInMono, @@ -189,86 +228,89 @@ impl EraseType >()]; } -impl EraseType for Result<&'_ TokenStream, ()> { - type Result = [u8; size_of::>()]; +impl Erasable for Result<&'_ TokenStream, ()> { + type Storage = [u8; size_of::>()]; } -impl EraseType for Option<&'_ T> { - type Result = [u8; size_of::>()]; +impl Erasable for Option<&'_ T> { + type Storage = [u8; size_of::>()]; } -impl EraseType for Option<&'_ [T]> { - type Result = [u8; size_of::>()]; +impl Erasable for Option<&'_ [T]> { + type Storage = [u8; size_of::>()]; } -impl EraseType for Option<&'_ OsStr> { - type Result = [u8; size_of::>()]; +impl Erasable for Option<&'_ OsStr> { + type Storage = [u8; size_of::>()]; } -impl EraseType for Option> { - type Result = [u8; size_of::>>()]; +impl Erasable for Option> { + type Storage = [u8; size_of::>>()]; } -impl EraseType for ty::ImplTraitHeader<'_> { - type Result = [u8; size_of::>()]; +impl Erasable for ty::ImplTraitHeader<'_> { + type Storage = [u8; size_of::>()]; } -impl EraseType for Option>> { - type Result = [u8; size_of::>>>()]; +impl Erasable for Option>> { + type Storage = [u8; size_of::>>>()]; } -impl EraseType for rustc_hir::MaybeOwner<'_> { - type Result = [u8; size_of::>()]; +impl Erasable for rustc_hir::MaybeOwner<'_> { + type Storage = [u8; size_of::>()]; } -impl EraseType for ty::EarlyBinder<'_, T> { - type Result = T::Result; +impl Erasable for ty::EarlyBinder<'_, T> { + type Storage = T::Storage; } -impl EraseType for ty::Binder<'_, ty::FnSig<'_>> { - type Result = [u8; size_of::>>()]; +impl Erasable for ty::Binder<'_, ty::FnSig<'_>> { + type Storage = [u8; size_of::>>()]; } -impl EraseType for ty::Binder<'_, ty::CoroutineWitnessTypes>> { - type Result = +impl Erasable for ty::Binder<'_, ty::CoroutineWitnessTypes>> { + type Storage = [u8; size_of::>>>()]; } -impl EraseType for ty::Binder<'_, &'_ ty::List>> { - type Result = [u8; size_of::>>>()]; +impl Erasable for ty::Binder<'_, &'_ ty::List>> { + type Storage = [u8; size_of::>>>()]; } -impl EraseType for (&'_ T0, &'_ T1) { - type Result = [u8; size_of::<(&'static (), &'static ())>()]; +impl Erasable for (&'_ T0, &'_ T1) { + type Storage = [u8; size_of::<(&'static (), &'static ())>()]; } -impl EraseType for (solve::QueryResult<'_>, &'_ T0) { - type Result = [u8; size_of::<(solve::QueryResult<'static>, &'static ())>()]; +impl Erasable for (solve::QueryResult<'_>, &'_ T0) { + type Storage = [u8; size_of::<(solve::QueryResult<'static>, &'static ())>()]; } -impl EraseType for (&'_ T0, &'_ [T1]) { - type Result = [u8; size_of::<(&'static (), &'static [()])>()]; +impl Erasable for (&'_ T0, &'_ [T1]) { + type Storage = [u8; size_of::<(&'static (), &'static [()])>()]; } -impl EraseType for (&'_ [T0], &'_ [T1]) { - type Result = [u8; size_of::<(&'static [()], &'static [()])>()]; +impl Erasable for (&'_ [T0], &'_ [T1]) { + type Storage = [u8; size_of::<(&'static [()], &'static [()])>()]; } -impl EraseType for (&'_ T0, Result<(), ErrorGuaranteed>) { - type Result = [u8; size_of::<(&'static (), Result<(), ErrorGuaranteed>)>()]; +impl Erasable for (&'_ T0, Result<(), ErrorGuaranteed>) { + type Storage = [u8; size_of::<(&'static (), Result<(), ErrorGuaranteed>)>()]; } -macro_rules! trivial { +macro_rules! impl_erasable_for_simple_types { ($($ty:ty),+ $(,)?) => { $( - impl EraseType for $ty { - type Result = [u8; size_of::<$ty>()]; + impl Erasable for $ty { + type Storage = [u8; size_of::<$ty>()]; } )* } } -trivial! { +// For concrete types with no lifetimes, the erased storage for `Foo` is +// `[u8; size_of::()]`. +impl_erasable_for_simple_types! { + // FIXME(#151565): Add `tidy-alphabetical-{start,end}` and sort this. (), bool, Option<(rustc_span::def_id::DefId, rustc_session::config::EntryFnType)>, @@ -378,17 +420,23 @@ trivial! { usize, } -macro_rules! tcx_lifetime { +macro_rules! impl_erasable_for_single_lifetime_types { ($($($fake_path:ident)::+),+ $(,)?) => { $( - impl<'tcx> EraseType for $($fake_path)::+<'tcx> { - type Result = [u8; size_of::<$($fake_path)::+<'static>>()]; + impl<'tcx> Erasable for $($fake_path)::+<'tcx> { + type Storage = [u8; size_of::<$($fake_path)::+<'static>>()]; } )* } } -tcx_lifetime! { +// For types containing a single lifetime and no other generics, e.g. +// `Foo<'tcx>`, the erased storage is `[u8; size_of::>()]`. +// +// FIXME(#151565): Some of the hand-written impls above that only use one +// lifetime can probably be migrated here. +impl_erasable_for_single_lifetime_types! { + // FIXME(#151565): Add `tidy-alphabetical-{start,end}` and sort this. rustc_middle::middle::exported_symbols::ExportedSymbol, rustc_middle::mir::Const, rustc_middle::mir::DestructuredConstant, diff --git a/compiler/rustc_middle/src/query/inner.rs b/compiler/rustc_middle/src/query/inner.rs index ee828ae55f7a3..e41d23f82f127 100644 --- a/compiler/rustc_middle/src/query/inner.rs +++ b/compiler/rustc_middle/src/query/inner.rs @@ -11,7 +11,7 @@ use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span}; use crate::dep_graph; use crate::query::IntoQueryParam; -use crate::query::erase::{self, Erase, EraseType}; +use crate::query::erase::{self, Erasable, Erased}; use crate::ty::TyCtxt; /// Shared implementation of `tcx.$query(..)` and `tcx.at(span).$query(..)` @@ -63,15 +63,15 @@ pub(crate) fn query_ensure_error_guaranteed<'tcx, Cache, T>( check_cache: bool, ) -> Result<(), ErrorGuaranteed> where - Cache: QueryCache>>, - Result: EraseType, + Cache: QueryCache>>, + Result: Erasable, { let key = key.into_query_param(); if let Some(res) = try_get_cached(tcx, query_cache, &key) { - erase::restore(res).map(drop) + erase::restore_val(res).map(drop) } else { execute_query(tcx, DUMMY_SP, key, QueryMode::Ensure { check_cache }) - .map(erase::restore) + .map(erase::restore_val) .map(|res| res.map(drop)) // Either we actually executed the query, which means we got a full `Result`, // or we can just assume the query succeeded, because it was green in the @@ -90,17 +90,17 @@ pub(crate) fn query_feed<'tcx, Cache, Value>( hasher: Option, &Value) -> Fingerprint>, cache: &Cache, key: Cache::Key, - erased: Erase, + erased: Erased, ) where - Cache: QueryCache>, + Cache: QueryCache>, Cache::Key: DepNodeParams>, - Value: EraseType + Debug, + Value: Erasable + Debug, { - let value = erase::restore::(erased); + let value = erase::restore_val::(erased); match try_get_cached(tcx, cache, &key) { Some(old) => { - let old = erase::restore::(old); + let old = erase::restore_val::(old); if let Some(hasher) = hasher { let (value_hash, old_hash): (Fingerprint, Fingerprint) = tcx .with_stable_hashing_context(|mut hcx| { diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 901a023c4f308..ae94c154015ba 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -121,7 +121,6 @@ use crate::mir::interpret::{ use crate::mir::mono::{ CodegenUnit, CollectionMode, MonoItem, MonoItemPartitions, NormalizationErrorInMono, }; -use crate::query::erase::{Erase, erase, restore}; use crate::query::plumbing::CyclePlaceholder; use crate::traits::query::{ CanonicalAliasGoal, CanonicalDropckOutlivesGoal, CanonicalImpliedOutlivesBoundsGoal, diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 0e536352563f4..7b85dac41aeff 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -266,6 +266,7 @@ macro_rules! define_callbacks { pub mod queries { $(pub mod $name { use super::super::*; + use $crate::query::erase::{self, Erased}; pub type Key<'tcx> = $($K)*; pub type Value<'tcx> = $V; @@ -288,29 +289,33 @@ macro_rules! define_callbacks { #[inline(always)] pub fn provided_to_erased<'tcx>( _tcx: TyCtxt<'tcx>, - value: ProvidedValue<'tcx>, - ) -> Erase> { - erase(query_if_arena!([$($modifiers)*] + provided_value: ProvidedValue<'tcx>, + ) -> Erased> { + // Store the provided value in an arena and get a reference + // to it, for queries with `arena_cache`. + let value: Value<'tcx> = query_if_arena!([$($modifiers)*] { use $crate::query::arena_cached::ArenaCached; if mem::needs_drop::<<$V as ArenaCached<'tcx>>::Allocated>() { <$V as ArenaCached>::alloc_in_arena( |v| _tcx.query_system.arenas.$name.alloc(v), - value, + provided_value, ) } else { <$V as ArenaCached>::alloc_in_arena( |v| _tcx.arena.dropless.alloc(v), - value, + provided_value, ) } } - (value) - )) + // Otherwise, the provided value is the value. + (provided_value) + ); + erase::erase_val(value) } - pub type Storage<'tcx> = <$($K)* as keys::Key>::Cache>; + pub type Storage<'tcx> = <$($K)* as keys::Key>::Cache>; // Ensure that keys grow no larger than 88 bytes by accident. // Increase this limit if necessary, but do try to keep the size low if possible @@ -411,7 +416,9 @@ macro_rules! define_callbacks { #[inline(always)] pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V { - restore::<$V>(crate::query::inner::query_get_at( + use $crate::query::{erase, inner}; + + erase::restore_val::<$V>(inner::query_get_at( self.tcx, self.tcx.query_system.fns.engine.$name, &self.tcx.query_system.caches.$name, @@ -480,7 +487,7 @@ macro_rules! define_callbacks { Span, queries::$name::Key<'tcx>, QueryMode, - ) -> Option>,)* + ) -> Option<$crate::query::erase::Erased<$V>>,)* } }; } diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index d6310b62b2759..382b8750a1ce7 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -10,7 +10,6 @@ use rustc_data_structures::stable_hasher::HashStable; use rustc_data_structures::sync::AtomicU64; use rustc_middle::arena::Arena; use rustc_middle::dep_graph::{self, DepKind, DepKindVTable, DepNodeIndex}; -use rustc_middle::query::erase::{Erase, erase, restore}; use rustc_middle::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex, OnDiskCache}; use rustc_middle::query::plumbing::{QuerySystem, QuerySystemFns, QueryVTable}; use rustc_middle::query::{ diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 0223981fd55dc..c71352c3fd202 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -281,7 +281,10 @@ macro_rules! feedable { macro_rules! hash_result { ([][$V:ty]) => {{ - Some(|hcx, result| dep_graph::hash_result(hcx, &restore::<$V>(*result))) + Some(|hcx, result| { + let result = ::rustc_middle::query::erase::restore_val::<$V>(*result); + ::rustc_query_system::dep_graph::hash_result(hcx, &result) + }) }}; ([(no_hash) $($rest:tt)*][$V:ty]) => {{ None @@ -597,6 +600,7 @@ macro_rules! define_queries { pub(crate) mod query_impl { $(pub(crate) mod $name { use super::super::*; use std::marker::PhantomData; + use ::rustc_middle::query::erase::{self, Erased}; pub(crate) mod get_query_incr { use super::*; @@ -609,7 +613,7 @@ macro_rules! define_queries { span: Span, key: queries::$name::Key<'tcx>, mode: QueryMode, - ) -> Option>> { + ) -> Option>> { #[cfg(debug_assertions)] let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); get_query_incr( @@ -631,7 +635,7 @@ macro_rules! define_queries { span: Span, key: queries::$name::Key<'tcx>, __mode: QueryMode, - ) -> Option>> { + ) -> Option>> { Some(get_query_non_incr( QueryType::query_dispatcher(tcx), QueryCtxt::new(tcx), @@ -652,7 +656,7 @@ macro_rules! define_queries { query_state: std::mem::offset_of!(QueryStates<'tcx>, $name), query_cache: std::mem::offset_of!(QueryCaches<'tcx>, $name), cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key), - execute_query: |tcx, key| erase(tcx.$name(key)), + execute_query: |tcx, key| erase::erase_val(tcx.$name(key)), compute: |tcx, key| { #[cfg(debug_assertions)] let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); @@ -690,7 +694,7 @@ macro_rules! define_queries { }), value_from_cycle_error: |tcx, cycle, guar| { let result: queries::$name::Value<'tcx> = Value::from_cycle_error(tcx, cycle, guar); - erase(result) + erase::erase_val(result) }, loadable_from_disk: |_tcx, _key, _index| { should_ever_cache_on_disk!([$($modifiers)*] { @@ -701,7 +705,7 @@ macro_rules! define_queries { }) }, hash_result: hash_result!([$($modifiers)*][queries::$name::Value<'tcx>]), - format_value: |value| format!("{:?}", restore::>(*value)), + format_value: |value| format!("{:?}", erase::restore_val::>(*value)), } } @@ -731,7 +735,7 @@ macro_rules! define_queries { #[inline(always)] fn restore_val(value: ::Value) -> Self::UnerasedValue { - restore::>(value) + erase::restore_val::>(value) } }