diff options
Diffstat (limited to 'compiler/rustc_query_impl/src')
| -rw-r--r-- | compiler/rustc_query_impl/src/lib.rs | 205 | ||||
| -rw-r--r-- | compiler/rustc_query_impl/src/plumbing.rs | 289 | ||||
| -rw-r--r-- | compiler/rustc_query_impl/src/profiling_support.rs | 2 |
3 files changed, 325 insertions, 171 deletions
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 82b335f4b4b..b76734dd072 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -3,11 +3,12 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] // this shouldn't be necessary, but the check for `&mut _` is too naive and denies returning a function pointer that takes a mut ref #![feature(const_mut_refs)] +#![feature(const_refs_to_cell)] #![feature(min_specialization)] #![feature(never_type)] #![feature(rustc_attrs)] #![recursion_limit = "256"] -#![allow(rustc::potential_query_instability)] +#![allow(rustc::potential_query_instability, unused_parens)] #![deny(rustc::untranslatable_diagnostic)] #![deny(rustc::diagnostic_outside_of_impl)] @@ -15,16 +16,28 @@ extern crate rustc_middle; use crate::plumbing::{encode_all_query_results, try_mark_green}; +use field_offset::offset_of; +use rustc_data_structures::stable_hasher::HashStable; +use rustc_data_structures::sync::AtomicU64; use rustc_middle::arena::Arena; +use rustc_middle::dep_graph::DepNodeIndex; use rustc_middle::dep_graph::{self, DepKind, DepKindStruct}; use rustc_middle::query::erase::{erase, restore, Erase}; +use rustc_middle::query::on_disk_cache::OnDiskCache; +use rustc_middle::query::plumbing::{DynamicQuery, QuerySystem, QuerySystemFns}; use rustc_middle::query::AsLocalKey; -use rustc_middle::ty::query::{ +use rustc_middle::query::{ query_keys, query_provided, query_provided_to_value, query_storage, query_values, + DynamicQueries, ExternProviders, Providers, QueryCaches, QueryEngine, QueryStates, }; -use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine, QuerySystemFns}; use rustc_middle::ty::TyCtxt; use rustc_query_system::dep_graph::SerializedDepNodeIndex; +use rustc_query_system::ich::StableHashingContext; +use rustc_query_system::query::{ + get_query_incr, get_query_non_incr, HashResult, QueryCache, QueryConfig, QueryInfo, QueryMap, + QueryMode, QueryState, +}; +use rustc_query_system::HandleCycleError; use rustc_query_system::Value; use rustc_span::Span; @@ -32,31 +45,183 @@ use rustc_span::Span; mod plumbing; pub use crate::plumbing::QueryCtxt; -pub use rustc_query_system::query::QueryConfig; -use rustc_query_system::query::*; - mod profiling_support; pub use self::profiling_support::alloc_self_profile_query_strings; -/// This is implemented per query and restoring query values from their erased state. -trait QueryConfigRestored<'tcx>: QueryConfig<QueryCtxt<'tcx>> + Default { - type RestoredValue; +struct DynamicConfig< + 'tcx, + C: QueryCache, + const ANON: bool, + const DEPTH_LIMIT: bool, + const FEEDABLE: bool, +> { + dynamic: &'tcx DynamicQuery<'tcx, C>, +} - fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue; +impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Copy + for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE> +{ +} +impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Clone + for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE> +{ + fn clone(&self) -> Self { + DynamicConfig { dynamic: self.dynamic } + } } -rustc_query_append! { define_queries! } +impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> + QueryConfig<QueryCtxt<'tcx>> for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE> +where + for<'a> C::Key: HashStable<StableHashingContext<'a>>, +{ + type Key = C::Key; + type Value = C::Value; + type Cache = C; + + #[inline(always)] + fn name(self) -> &'static str { + self.dynamic.name + } + + #[inline(always)] + fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool { + (self.dynamic.cache_on_disk)(tcx, key) + } + + #[inline(always)] + fn query_state<'a>(self, qcx: QueryCtxt<'tcx>) -> &'a QueryState<Self::Key, DepKind> + where + QueryCtxt<'tcx>: 'a, + { + self.dynamic.query_state.apply(&qcx.tcx.query_system.states) + } + + #[inline(always)] + fn query_cache<'a>(self, qcx: QueryCtxt<'tcx>) -> &'a Self::Cache + where + 'tcx: 'a, + { + self.dynamic.query_cache.apply(&qcx.tcx.query_system.caches) + } + + #[inline(always)] + fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value { + (self.dynamic.execute_query)(tcx, key) + } + + #[inline(always)] + fn compute(self, qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value { + (self.dynamic.compute)(qcx.tcx, key) + } + + #[inline(always)] + fn try_load_from_disk( + self, + qcx: QueryCtxt<'tcx>, + key: &Self::Key, + prev_index: SerializedDepNodeIndex, + index: DepNodeIndex, + ) -> Option<Self::Value> { + if self.dynamic.can_load_from_disk { + (self.dynamic.try_load_from_disk)(qcx.tcx, key, prev_index, index) + } else { + None + } + } + + #[inline] + fn loadable_from_disk( + self, + qcx: QueryCtxt<'tcx>, + key: &Self::Key, + index: SerializedDepNodeIndex, + ) -> bool { + (self.dynamic.loadable_from_disk)(qcx.tcx, key, index) + } + + fn value_from_cycle_error( + self, + tcx: TyCtxt<'tcx>, + cycle: &[QueryInfo<DepKind>], + ) -> Self::Value { + (self.dynamic.value_from_cycle_error)(tcx, cycle) + } + + #[inline(always)] + fn format_value(self) -> fn(&Self::Value) -> String { + self.dynamic.format_value + } + + #[inline(always)] + fn anon(self) -> bool { + ANON + } + + #[inline(always)] + fn eval_always(self) -> bool { + self.dynamic.eval_always + } + + #[inline(always)] + fn depth_limit(self) -> bool { + DEPTH_LIMIT + } + + #[inline(always)] + fn feedable(self) -> bool { + FEEDABLE + } + + #[inline(always)] + fn dep_kind(self) -> DepKind { + self.dynamic.dep_kind + } + + #[inline(always)] + fn handle_cycle_error(self) -> HandleCycleError { + self.dynamic.handle_cycle_error + } + + #[inline(always)] + fn hash_result(self) -> HashResult<Self::Value> { + self.dynamic.hash_result + } +} -pub fn query_system_fns<'tcx>( +/// This is implemented per query. It allows restoring query values from their erased state +/// and constructing a QueryConfig. +trait QueryConfigRestored<'tcx> { + type RestoredValue; + type Config: QueryConfig<QueryCtxt<'tcx>>; + + fn config(tcx: TyCtxt<'tcx>) -> Self::Config; + fn restore(value: <Self::Config as QueryConfig<QueryCtxt<'tcx>>>::Value) + -> Self::RestoredValue; +} + +pub fn query_system<'tcx>( local_providers: Providers, extern_providers: ExternProviders, -) -> QuerySystemFns<'tcx> { - QuerySystemFns { - engine: engine(), - local_providers, - extern_providers, - query_structs: make_dep_kind_array!(query_structs).to_vec(), - encode_query_results: encode_all_query_results, - try_mark_green: try_mark_green, + on_disk_cache: Option<OnDiskCache<'tcx>>, + incremental: bool, +) -> QuerySystem<'tcx> { + QuerySystem { + states: Default::default(), + arenas: Default::default(), + caches: Default::default(), + dynamic_queries: dynamic_queries(), + on_disk_cache, + fns: QuerySystemFns { + engine: engine(incremental), + local_providers, + extern_providers, + query_structs: make_dep_kind_array!(query_structs).to_vec(), + encode_query_results: encode_all_query_results, + try_mark_green: try_mark_green, + }, + jobs: AtomicU64::new(1), } } + +rustc_query_append! { define_queries! } diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 9f8ac7ccd0b..79d8abc4b69 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -4,6 +4,7 @@ use crate::rustc_middle::dep_graph::DepContext; use crate::rustc_middle::ty::TyEncoder; +use crate::QueryConfigRestored; use rustc_data_structures::stable_hasher::{Hash64, HashStable, StableHasher}; use rustc_data_structures::sync::Lock; use rustc_errors::Diagnostic; @@ -265,14 +266,14 @@ macro_rules! hash_result { } macro_rules! call_provider { - ([][$qcx:expr, $name:ident, $key:expr]) => {{ - ($qcx.query_system.fns.local_providers.$name)($qcx, $key) + ([][$tcx:expr, $name:ident, $key:expr]) => {{ + ($tcx.query_system.fns.local_providers.$name)($tcx, $key) }}; - ([(separate_provide_extern) $($rest:tt)*][$qcx:expr, $name:ident, $key:expr]) => {{ + ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{ if let Some(key) = $key.as_local_key() { - ($qcx.query_system.fns.local_providers.$name)($qcx, key) + ($tcx.query_system.fns.local_providers.$name)($tcx, key) } else { - ($qcx.query_system.fns.extern_providers.$name)($qcx, $key) + ($tcx.query_system.fns.extern_providers.$name)($tcx, $key) } }}; ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => { @@ -341,7 +342,7 @@ pub(crate) fn create_query_frame< } pub(crate) fn encode_query_results<'a, 'tcx, Q>( - query: Q, + query: Q::Config, qcx: QueryCtxt<'tcx>, encoder: &mut CacheEncoder<'a, 'tcx>, query_result_index: &mut EncodedDepNodeIndex, @@ -392,12 +393,26 @@ pub(crate) fn loadable_from_disk<'tcx>(tcx: TyCtxt<'tcx>, id: SerializedDepNodeI pub(crate) fn try_load_from_disk<'tcx, V>( tcx: TyCtxt<'tcx>, - id: SerializedDepNodeIndex, + prev_index: SerializedDepNodeIndex, + index: DepNodeIndex, ) -> Option<V> where V: for<'a> Decodable<CacheDecoder<'a, 'tcx>>, { - tcx.query_system.on_disk_cache.as_ref()?.try_load_query_result(tcx, id) + let on_disk_cache = tcx.query_system.on_disk_cache.as_ref()?; + + let prof_timer = tcx.prof.incr_cache_loading(); + + // The call to `with_query_deserialization` enforces that no new `DepNodes` + // are created during deserialization. See the docs of that method for more + // details. + let value = tcx + .dep_graph + .with_query_deserialization(|| on_disk_cache.try_load_query_result(tcx, prev_index)); + + prof_timer.finish_with_query_invocation_id(index.into()); + + value } fn force_from_dep_node<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool @@ -434,10 +449,9 @@ where pub(crate) fn query_callback<'tcx, Q>(is_anon: bool, is_eval_always: bool) -> DepKindStruct<'tcx> where - Q: QueryConfig<QueryCtxt<'tcx>> + Default, - Q::Key: DepNodeParams<TyCtxt<'tcx>>, + Q: QueryConfigRestored<'tcx>, { - let fingerprint_style = Q::Key::fingerprint_style(); + let fingerprint_style = <Q::Config as QueryConfig<QueryCtxt<'tcx>>>::Key::fingerprint_style(); if is_anon || !fingerprint_style.reconstructible() { return DepKindStruct { @@ -453,9 +467,11 @@ where is_anon, is_eval_always, fingerprint_style, - force_from_dep_node: Some(|tcx, dep_node| force_from_dep_node(Q::default(), tcx, dep_node)), + force_from_dep_node: Some(|tcx, dep_node| { + force_from_dep_node(Q::config(tcx), tcx, dep_node) + }), try_load_from_on_disk_cache: Some(|tcx, dep_node| { - try_load_from_on_disk_cache(Q::default(), tcx, dep_node) + try_load_from_on_disk_cache(Q::config(tcx), tcx, dep_node) }), } } @@ -478,7 +494,7 @@ macro_rules! define_queries { ( $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => { - mod get_query { + mod get_query_incr { use super::*; $( @@ -490,8 +506,8 @@ macro_rules! define_queries { key: query_keys::$name<'tcx>, mode: QueryMode, ) -> Option<Erase<query_values::$name<'tcx>>> { - get_query( - queries::$name::default(), + get_query_incr( + queries::$name::config(tcx), QueryCtxt::new(tcx), span, key, @@ -501,9 +517,37 @@ macro_rules! define_queries { )* } - pub(crate) fn engine() -> QueryEngine { - QueryEngine { - $($name: get_query::$name,)* + mod get_query_non_incr { + use super::*; + + $( + #[inline(always)] + #[tracing::instrument(level = "trace", skip(tcx))] + pub(super) fn $name<'tcx>( + tcx: TyCtxt<'tcx>, + span: Span, + key: query_keys::$name<'tcx>, + __mode: QueryMode, + ) -> Option<Erase<query_values::$name<'tcx>>> { + Some(get_query_non_incr( + queries::$name::config(tcx), + QueryCtxt::new(tcx), + span, + key, + )) + } + )* + } + + pub(crate) fn engine(incremental: bool) -> QueryEngine { + if incremental { + QueryEngine { + $($name: get_query_incr::$name,)* + } + } else { + QueryEngine { + $($name: get_query_non_incr::$name,)* + } } } @@ -519,146 +563,91 @@ macro_rules! define_queries { )* } - $(impl<'tcx> QueryConfig<QueryCtxt<'tcx>> for queries::$name<'tcx> { - type Key = query_keys::$name<'tcx>; - type Value = Erase<query_values::$name<'tcx>>; - - #[inline(always)] - fn name(self) -> &'static str { - stringify!($name) - } - - #[inline] - fn format_value(self) -> fn(&Self::Value) -> String { - |value| format!("{:?}", restore::<query_values::$name<'tcx>>(*value)) - } - - #[inline] - fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool { - ::rustc_middle::query::cached::$name(tcx, key) - } - - type Cache = query_storage::$name<'tcx>; - - #[inline(always)] - fn query_state<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a QueryState<Self::Key, crate::dep_graph::DepKind> - where QueryCtxt<'tcx>: 'a - { - &tcx.query_system.states.$name - } - - #[inline(always)] - fn query_cache<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a Self::Cache - where 'tcx:'a - { - &tcx.query_system.caches.$name - } - - fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value { - erase(tcx.$name(key)) - } - - #[inline] - #[allow(unused_variables)] - fn compute(self, qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value { - query_provided_to_value::$name( - qcx.tcx, - call_provider!([$($modifiers)*][qcx.tcx, $name, key]) - ) - } + #[allow(nonstandard_style)] + mod dynamic_query { + use super::*; - #[inline] - fn try_load_from_disk( - self, - _qcx: QueryCtxt<'tcx>, - _key: &Self::Key - ) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self::Value> { - should_ever_cache_on_disk!([$($modifiers)*] { - if ::rustc_middle::query::cached::$name(_qcx.tcx, _key) { - Some(|qcx: QueryCtxt<'tcx>, dep_node| { - let value = $crate::plumbing::try_load_from_disk::<query_provided::$name<'tcx>>( - qcx.tcx, - dep_node - ); - value.map(|value| query_provided_to_value::$name(qcx.tcx, value)) - }) - } else { - None + $( + pub(super) fn $name<'tcx>() -> DynamicQuery<'tcx, query_storage::$name<'tcx>> { + DynamicQuery { + name: stringify!($name), + eval_always: is_eval_always!([$($modifiers)*]), + dep_kind: dep_graph::DepKind::$name, + handle_cycle_error: handle_cycle_error!([$($modifiers)*]), + query_state: offset_of!(QueryStates<'tcx> => $name), + query_cache: offset_of!(QueryCaches<'tcx> => $name), + cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key), + execute_query: |tcx, key| erase(tcx.$name(key)), + compute: |tcx, key| query_provided_to_value::$name( + tcx, + call_provider!([$($modifiers)*][tcx, $name, key]) + ), + can_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] true false), + try_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] { + |tcx, key, prev_index, index| { + if ::rustc_middle::query::cached::$name(tcx, key) { + let value = $crate::plumbing::try_load_from_disk::<query_provided::$name<'tcx>>( + tcx, + prev_index, + index, + ); + value.map(|value| query_provided_to_value::$name(tcx, value)) + } else { + None + } + } + } { + |_tcx, _key, _prev_index, _index| None + }), + value_from_cycle_error: |tcx, cycle| { + let result: query_values::$name<'tcx> = Value::from_cycle_error(tcx, cycle); + erase(result) + }, + loadable_from_disk: |_tcx, _key, _index| { + should_ever_cache_on_disk!([$($modifiers)*] { + ::rustc_middle::query::cached::$name(_tcx, _key) && + $crate::plumbing::loadable_from_disk(_tcx, _index) + } { + false + }) + }, + hash_result: hash_result!([$($modifiers)*][query_values::$name<'tcx>]), + format_value: |value| format!("{:?}", restore::<query_values::$name<'tcx>>(*value)), } - } { - None - }) - } - - #[inline] - fn loadable_from_disk( - self, - _qcx: QueryCtxt<'tcx>, - _key: &Self::Key, - _index: SerializedDepNodeIndex, - ) -> bool { - should_ever_cache_on_disk!([$($modifiers)*] { - self.cache_on_disk(_qcx.tcx, _key) && - $crate::plumbing::loadable_from_disk(_qcx.tcx, _index) - } { - false - }) - } - - #[inline] - fn value_from_cycle_error( - self, - tcx: TyCtxt<'tcx>, - cycle: &[QueryInfo<DepKind>], - ) -> Self::Value { - let result: query_values::$name<'tcx> = Value::from_cycle_error(tcx, cycle); - erase(result) - } - - #[inline(always)] - fn anon(self) -> bool { - is_anon!([$($modifiers)*]) - } - - #[inline(always)] - fn eval_always(self) -> bool { - is_eval_always!([$($modifiers)*]) - } - - #[inline(always)] - fn depth_limit(self) -> bool { - depth_limit!([$($modifiers)*]) - } - - #[inline(always)] - fn feedable(self) -> bool { - feedable!([$($modifiers)*]) - } + } + )* + } - #[inline(always)] - fn dep_kind(self) -> rustc_middle::dep_graph::DepKind { - dep_graph::DepKind::$name - } + $(impl<'tcx> QueryConfigRestored<'tcx> for queries::$name<'tcx> { + type RestoredValue = query_values::$name<'tcx>; + type Config = DynamicConfig< + 'tcx, + query_storage::$name<'tcx>, + { is_anon!([$($modifiers)*]) }, + { depth_limit!([$($modifiers)*]) }, + { feedable!([$($modifiers)*]) }, + >; #[inline(always)] - fn handle_cycle_error(self) -> rustc_query_system::HandleCycleError { - handle_cycle_error!([$($modifiers)*]) + fn config(tcx: TyCtxt<'tcx>) -> Self::Config { + DynamicConfig { + dynamic: &tcx.query_system.dynamic_queries.$name, + } } #[inline(always)] - fn hash_result(self) -> rustc_query_system::query::HashResult<Self::Value> { - hash_result!([$($modifiers)*][query_values::$name<'tcx>]) + fn restore(value: <Self::Config as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue { + restore::<query_values::$name<'tcx>>(value) } })* - $(impl<'tcx> QueryConfigRestored<'tcx> for queries::$name<'tcx> { - type RestoredValue = query_values::$name<'tcx>; - - #[inline(always)] - fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue { - restore::<query_values::$name<'tcx>>(value) + pub fn dynamic_queries<'tcx>() -> DynamicQueries<'tcx> { + DynamicQueries { + $( + $name: dynamic_query::$name(), + )* } - })* + } #[allow(nonstandard_style)] mod query_callbacks { @@ -727,9 +716,9 @@ macro_rules! define_queries { mod query_structs { use super::*; - use rustc_middle::ty::query::QueryStruct; - use rustc_middle::ty::query::QueryKeyStringCache; + use rustc_middle::query::plumbing::{QueryKeyStringCache, QueryStruct}; use rustc_middle::dep_graph::DepKind; + use crate::QueryConfigRestored; pub(super) const fn dummy_query_struct<'tcx>() -> QueryStruct<'tcx> { fn noop_try_collect_active_jobs(_: TyCtxt<'_>, _: &mut QueryMap<DepKind>) -> Option<()> { @@ -774,7 +763,7 @@ macro_rules! define_queries { }, encode_query_results: expand_if_cached!([$($modifiers)*], |tcx, encoder, query_result_index| $crate::plumbing::encode_query_results::<super::queries::$name<'tcx>>( - super::queries::$name::default(), + super::queries::$name::config(tcx), QueryCtxt::new(tcx), encoder, query_result_index, diff --git a/compiler/rustc_query_impl/src/profiling_support.rs b/compiler/rustc_query_impl/src/profiling_support.rs index 7d9306f8087..e042ee62dfe 100644 --- a/compiler/rustc_query_impl/src/profiling_support.rs +++ b/compiler/rustc_query_impl/src/profiling_support.rs @@ -2,7 +2,7 @@ use measureme::{StringComponent, StringId}; use rustc_data_structures::profiling::SelfProfiler; use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, LOCAL_CRATE}; use rustc_hir::definitions::DefPathData; -use rustc_middle::ty::query::QueryKeyStringCache; +use rustc_middle::query::plumbing::QueryKeyStringCache; use rustc_middle::ty::TyCtxt; use rustc_query_system::query::QueryCache; use std::fmt::Debug; |