From c556c196d38e267fbf21aa02cbc4958a90166383 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 18 Feb 2026 20:21:21 +1100 Subject: [PATCH 1/2] Remove `SemiDynamicQueryDispatcher` and `QueryFlags`. `SemiDynamicQueryDispatcher` is just a `QueryVTable` wrapper with an additional `const FLAGS: QueryFlags` generic parameter that contains three booleans. This arrangement exists as a performance optimization. But the performance effects are very small and it adds quite a bit of complexity to an already overly-complex part of the codebase. If it didn't exist and somebody proposed adding it and asked me to review, I almost certainly wouldn't approve it. This commit removes it. The three booleans in `QueryFlags` are moved into `QueryVTable` The non-trivial methods of `SemiDynamicQueryDispatcher` become methods of `QueryVTable`. --- compiler/rustc_middle/src/query/plumbing.rs | 91 ++++++++- .../rustc_query_impl/src/dep_kind_vtables.rs | 8 +- compiler/rustc_query_impl/src/execution.rs | 115 +++++------ compiler/rustc_query_impl/src/lib.rs | 192 +----------------- compiler/rustc_query_impl/src/plumbing.rs | 57 +++--- 5 files changed, 183 insertions(+), 280 deletions(-) diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 47b6aea077d17..7335c36bc5f34 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -1,4 +1,4 @@ -use std::fmt::Debug; +use std::fmt; use std::ops::Deref; use rustc_data_structures::fingerprint::Fingerprint; @@ -12,7 +12,7 @@ use rustc_span::{ErrorGuaranteed, Span}; pub use sealed::IntoQueryParam; use crate::dep_graph; -use crate::dep_graph::{DepKind, DepNodeIndex, SerializedDepNodeIndex}; +use crate::dep_graph::{DepKind, DepNode, DepNodeIndex, SerializedDepNodeIndex}; use crate::ich::StableHashingContext; use crate::queries::{ ExternProviders, PerQueryVTables, Providers, QueryArenas, QueryCaches, QueryEngine, QueryStates, @@ -104,7 +104,15 @@ pub enum QueryMode { /// Stores function pointers and other metadata for a particular query. pub struct QueryVTable<'tcx, C: QueryCache> { pub name: &'static str, + /// True if this query has the `anon` modifier. + pub anon: bool, + /// True if this query has the `eval_always` modifier. pub eval_always: bool, + /// True if this query has the `depth_limit` modifier. + pub depth_limit: bool, + /// True if this query has the `feedable` modifier. + pub feedable: bool, + pub dep_kind: DepKind, /// How this query deals with query cycle errors. pub cycle_error_handling: CycleErrorHandling, @@ -142,6 +150,85 @@ pub struct QueryVTable<'tcx, C: QueryCache> { pub description_fn: fn(TyCtxt<'tcx>, C::Key) -> String, } +impl<'tcx, C: QueryCache> fmt::Debug for QueryVTable<'tcx, C> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // When debug-printing a query vtable (e.g. for ICE or tracing), + // just print the query name to know what query we're dealing with. + // The other fields and flags are probably just unhelpful noise. + // + // If there is need for a more detailed dump of all flags and fields, + // consider writing a separate dump method and calling it explicitly. + f.write_str(self.name) + } +} + +impl<'tcx, C: QueryCache> QueryVTable<'tcx, C> { + #[inline(always)] + pub fn will_cache_on_disk_for_key(&self, tcx: TyCtxt<'tcx>, key: &C::Key) -> bool { + self.will_cache_on_disk_for_key_fn.map_or(false, |f| f(tcx, key)) + } + + // Don't use this method to access query results, instead use the methods on TyCtxt. + #[inline(always)] + pub fn query_state(&self, tcx: TyCtxt<'tcx>) -> &'tcx QueryState<'tcx, C::Key> { + // Safety: + // This is just manually doing the subfield referencing through pointer math. + unsafe { + &*(&tcx.query_system.states as *const QueryStates<'tcx>) + .byte_add(self.query_state) + .cast::>() + } + } + + // Don't use this method to access query results, instead use the methods on TyCtxt. + #[inline(always)] + pub fn query_cache(&self, tcx: TyCtxt<'tcx>) -> &'tcx C { + // Safety: + // This is just manually doing the subfield referencing through pointer math. + unsafe { + &*(&tcx.query_system.caches as *const QueryCaches<'tcx>) + .byte_add(self.query_cache) + .cast::() + } + } + + #[inline(always)] + pub fn try_load_from_disk( + &self, + tcx: TyCtxt<'tcx>, + key: &C::Key, + prev_index: SerializedDepNodeIndex, + index: DepNodeIndex, + ) -> Option { + // `?` will return None immediately for queries that never cache to disk. + self.try_load_from_disk_fn?(tcx, key, prev_index, index) + } + + #[inline] + pub fn is_loadable_from_disk( + &self, + tcx: TyCtxt<'tcx>, + key: &C::Key, + index: SerializedDepNodeIndex, + ) -> bool { + self.is_loadable_from_disk_fn.map_or(false, |f| f(tcx, key, index)) + } + + /// Synthesize an error value to let compilation continue after a cycle. + pub fn value_from_cycle_error( + &self, + tcx: TyCtxt<'tcx>, + cycle_error: &CycleError, + guar: ErrorGuaranteed, + ) -> C::Value { + (self.value_from_cycle_error)(tcx, cycle_error, guar) + } + + pub fn construct_dep_node(&self, tcx: TyCtxt<'tcx>, key: &C::Key) -> DepNode { + DepNode::construct(tcx, self.dep_kind, key) + } +} + pub struct QuerySystemFns { pub engine: QueryEngine, pub local_providers: Providers, diff --git a/compiler/rustc_query_impl/src/dep_kind_vtables.rs b/compiler/rustc_query_impl/src/dep_kind_vtables.rs index 39bd2569ef468..ea9af88fbfd3d 100644 --- a/compiler/rustc_query_impl/src/dep_kind_vtables.rs +++ b/compiler/rustc_query_impl/src/dep_kind_vtables.rs @@ -2,8 +2,8 @@ use rustc_middle::bug; use rustc_middle::dep_graph::{DepKindVTable, DepNodeKey, KeyFingerprintStyle}; use rustc_middle::query::QueryCache; +use crate::QueryDispatcherUnerased; use crate::plumbing::{force_from_dep_node_inner, try_load_from_on_disk_cache_inner}; -use crate::{QueryDispatcherUnerased, QueryFlags}; /// [`DepKindVTable`] constructors for special dep kinds that aren't queries. #[expect(non_snake_case, reason = "use non-snake case to avoid collision with query names")] @@ -110,14 +110,14 @@ mod non_query { /// Shared implementation of the [`DepKindVTable`] constructor for queries. /// Called from macro-generated code for each query. -pub(crate) fn make_dep_kind_vtable_for_query<'tcx, Q, Cache, const FLAGS: QueryFlags>( +pub(crate) fn make_dep_kind_vtable_for_query<'tcx, Q, Cache>( + is_anon: bool, is_eval_always: bool, ) -> DepKindVTable<'tcx> where - Q: QueryDispatcherUnerased<'tcx, Cache, FLAGS>, + Q: QueryDispatcherUnerased<'tcx, Cache>, Cache: QueryCache + 'tcx, { - let is_anon = FLAGS.is_anon; let key_fingerprint_style = if is_anon { KeyFingerprintStyle::Opaque } else { diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index 1f064599332c1..53afcacb63a6c 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -6,6 +6,7 @@ use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_data_structures::{outline, sharded, sync}; use rustc_errors::{Diag, FatalError, StashKey}; use rustc_middle::dep_graph::{DepGraphData, DepNodeKey}; +use rustc_middle::query::plumbing::QueryVTable; use rustc_middle::query::{ ActiveKeyStatus, CycleError, CycleErrorHandling, QueryCache, QueryJob, QueryJobId, QueryLatch, QueryMode, QueryStackDeferred, QueryStackFrame, QueryState, @@ -19,7 +20,6 @@ use crate::job::{QueryJobInfo, QueryJobMap, find_cycle_in_stack, report_cycle}; use crate::plumbing::{ collect_active_jobs_from_all_queries, current_query_job, next_job_id, start_query, }; -use crate::{QueryFlags, SemiDynamicQueryDispatcher}; #[inline] fn equivalent_key(k: &K) -> impl Fn(&(K, V)) -> bool + '_ { @@ -102,8 +102,8 @@ where #[cold] #[inline(never)] -fn mk_cycle<'tcx, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +fn mk_cycle<'tcx, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, cycle_error: CycleError, ) -> C::Value { @@ -111,13 +111,13 @@ fn mk_cycle<'tcx, C: QueryCache, const FLAGS: QueryFlags>( handle_cycle_error(query, tcx, &cycle_error, error) } -fn handle_cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +fn handle_cycle_error<'tcx, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, cycle_error: &CycleError, error: Diag<'_>, ) -> C::Value { - match query.cycle_error_handling() { + match query.cycle_error_handling { CycleErrorHandling::Error => { let guar = error.emit(); query.value_from_cycle_error(tcx, cycle_error, guar) @@ -208,8 +208,8 @@ where #[cold] #[inline(never)] -fn cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +fn cycle_error<'tcx, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, try_execute: QueryJobId, span: Span, @@ -225,8 +225,8 @@ fn cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>( } #[inline(always)] -fn wait_for_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +fn wait_for_query<'tcx, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, span: Span, key: C::Key, @@ -255,7 +255,7 @@ fn wait_for_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>( Some((_, ActiveKeyStatus::Poisoned)) => FatalError.raise(), _ => panic!( "query '{}' result must be in the cache or the query must be poisoned after a wait", - query.name() + query.name ), } }) @@ -271,8 +271,8 @@ fn wait_for_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>( } #[inline(never)] -fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, span: Span, key: C::Key, @@ -308,7 +308,7 @@ fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: b // Drop the lock before we start executing the query drop(state_lock); - execute_job::(query, tcx, state, key, key_hash, id, dep_node) + execute_job::(query, tcx, state, key, key_hash, id, dep_node) } Entry::Occupied(mut entry) => { match &mut entry.get_mut().1 { @@ -337,8 +337,8 @@ fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: b } #[inline(always)] -fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +fn execute_job<'tcx, C: QueryCache, const INCR: bool>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, state: &'tcx QueryState<'tcx, C::Key>, key: C::Key, @@ -360,25 +360,25 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>( }; let cache = query.query_cache(tcx); - if query.feedable() { + if query.feedable { // We should not compute queries that also got a value via feeding. // This can't happen, as query feeding adds the very dependencies to the fed query // as its feeding query had. So if the fed query is red, so is its feeder, which will // get evaluated first, and re-feed the query. if let Some((cached_result, _)) = cache.lookup(&key) { - let Some(hasher) = query.hash_result() else { + let Some(hasher) = query.hash_result else { panic!( "no_hash fed query later has its value computed.\n\ Remove `no_hash` modifier to allow recomputation.\n\ The already cached value: {}", - (query.format_value())(&cached_result) + (query.format_value)(&cached_result) ); }; let (old_hash, new_hash) = tcx.with_stable_hashing_context(|mut hcx| { (hasher(&mut hcx, &cached_result), hasher(&mut hcx, &result)) }); - let formatter = query.format_value(); + let formatter = query.format_value; if old_hash != new_hash { // We have an inconsistency. This can happen if one of the two // results is tainted by errors. @@ -386,7 +386,7 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>( tcx.dcx().has_errors().is_some(), "Computed query value for {:?}({:?}) is inconsistent with fed value,\n\ computed={:#?}\nfed={:#?}", - query.dep_kind(), + query.dep_kind, key, formatter(&result), formatter(&cached_result), @@ -403,8 +403,8 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>( // Fast path for when incr. comp. is off. #[inline(always)] -fn execute_job_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +fn execute_job_non_incr<'tcx, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, key: C::Key, job_id: QueryJobId, @@ -419,14 +419,15 @@ fn execute_job_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( let prof_timer = tcx.prof.query_provider(); // Call the query provider. - let result = start_query(tcx, job_id, query.depth_limit(), || query.invoke_provider(tcx, key)); + let result = + start_query(tcx, job_id, query.depth_limit, || (query.invoke_provider_fn)(tcx, key)); let dep_node_index = tcx.dep_graph.next_virtual_depnode_index(); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); // Similarly, fingerprint the result to assert that // it doesn't have anything not considered hashable. if cfg!(debug_assertions) - && let Some(hash_result) = query.hash_result() + && let Some(hash_result) = query.hash_result { tcx.with_stable_hashing_context(|mut hcx| { hash_result(&mut hcx, &result); @@ -437,8 +438,8 @@ fn execute_job_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( } #[inline(always)] -fn execute_job_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +fn execute_job_incr<'tcx, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, key: C::Key, mut dep_node_opt: Option, @@ -447,7 +448,7 @@ fn execute_job_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( let dep_graph_data = tcx.dep_graph.data().expect("should always be present in incremental mode"); - if !query.anon() && !query.eval_always() { + if !query.anon && !query.eval_always { // `to_dep_node` is expensive for some `DepKind`s. let dep_node = dep_node_opt.get_or_insert_with(|| query.construct_dep_node(tcx, &key)); @@ -462,11 +463,12 @@ fn execute_job_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( let prof_timer = tcx.prof.query_provider(); - let (result, dep_node_index) = start_query(tcx, job_id, query.depth_limit(), || { - if query.anon() { + let (result, dep_node_index) = start_query(tcx, job_id, query.depth_limit, || { + if query.anon { // Call the query provider inside an anon task. - return dep_graph_data - .with_anon_task_inner(tcx, query.dep_kind(), || query.invoke_provider(tcx, key)); + return dep_graph_data.with_anon_task_inner(tcx, query.dep_kind, || { + (query.invoke_provider_fn)(tcx, key) + }); } // `to_dep_node` is expensive for some `DepKind`s. @@ -477,8 +479,8 @@ fn execute_job_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( dep_node, tcx, (query, key), - |tcx, (query, key)| query.invoke_provider(tcx, key), - query.hash_result(), + |tcx, (query, key)| (query.invoke_provider_fn)(tcx, key), + query.hash_result, ) }); @@ -488,8 +490,8 @@ fn execute_job_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( } #[inline(always)] -fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, dep_graph_data: &DepGraphData, tcx: TyCtxt<'tcx>, key: &C::Key, @@ -526,8 +528,8 @@ fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: Quer dep_graph_data, &result, prev_dep_node_index, - query.hash_result(), - query.format_value(), + query.hash_result, + query.format_value, ); } @@ -555,7 +557,7 @@ fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: Quer // The dep-graph for this computation is already in-place. // Call the query provider. - let result = tcx.dep_graph.with_ignore(|| query.invoke_provider(tcx, *key)); + let result = tcx.dep_graph.with_ignore(|| (query.invoke_provider_fn)(tcx, *key)); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -573,8 +575,8 @@ fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: Quer dep_graph_data, &result, prev_dep_node_index, - query.hash_result(), - query.format_value(), + query.hash_result, + query.format_value, ); Some((result, dep_node_index)) @@ -589,18 +591,18 @@ fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: Quer /// /// Note: The optimization is only available during incr. comp. #[inline(never)] -fn ensure_must_run<'tcx, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +fn ensure_must_run<'tcx, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, key: &C::Key, check_cache: bool, ) -> (bool, Option) { - if query.eval_always() { + if query.eval_always { return (true, None); } // Ensuring an anonymous query makes no sense - assert!(!query.anon()); + assert!(!query.anon); let dep_node = query.construct_dep_node(tcx, key); @@ -632,20 +634,20 @@ fn ensure_must_run<'tcx, C: QueryCache, const FLAGS: QueryFlags>( } #[inline(always)] -pub(super) fn get_query_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +pub(super) fn get_query_non_incr<'tcx, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, span: Span, key: C::Key, ) -> C::Value { debug_assert!(!tcx.dep_graph.is_fully_enabled()); - ensure_sufficient_stack(|| try_execute_query::(query, tcx, span, key, None).0) + ensure_sufficient_stack(|| try_execute_query::(query, tcx, span, key, None).0) } #[inline(always)] -pub(super) fn get_query_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +pub(super) fn get_query_incr<'tcx, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, span: Span, key: C::Key, @@ -663,17 +665,16 @@ pub(super) fn get_query_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( None }; - let (result, dep_node_index) = ensure_sufficient_stack(|| { - try_execute_query::(query, tcx, span, key, dep_node) - }); + let (result, dep_node_index) = + ensure_sufficient_stack(|| try_execute_query::(query, tcx, span, key, dep_node)); if let Some(dep_node_index) = dep_node_index { tcx.dep_graph.read_index(dep_node_index) } Some(result) } -pub(crate) fn force_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +pub(crate) fn force_query<'tcx, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, key: C::Key, dep_node: DepNode, @@ -685,9 +686,9 @@ pub(crate) fn force_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>( return; } - debug_assert!(!query.anon()); + debug_assert!(!query.anon); ensure_sufficient_stack(|| { - try_execute_query::(query, tcx, DUMMY_SP, key, Some(dep_node)) + try_execute_query::(query, tcx, DUMMY_SP, key, Some(dep_node)) }); } diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index b4a64686728f1..79baae6c8326c 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -2,28 +2,22 @@ // tidy-alphabetical-start #![allow(internal_features)] -#![feature(adt_const_params)] #![feature(core_intrinsics)] #![feature(min_specialization)] #![feature(rustc_attrs)] #![feature(try_blocks)] // tidy-alphabetical-end -use std::fmt; -use std::marker::ConstParamTy; - use rustc_data_structures::sync::AtomicU64; -use rustc_middle::dep_graph::{self, DepKind, DepNode, DepNodeIndex, SerializedDepNodeIndex}; +use rustc_middle::dep_graph; use rustc_middle::queries::{ self, ExternProviders, Providers, QueryCaches, QueryEngine, QueryStates, }; use rustc_middle::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex, OnDiskCache}; -use rustc_middle::query::plumbing::{ - HashResult, QueryState, QuerySystem, QuerySystemFns, QueryVTable, -}; -use rustc_middle::query::{AsLocalKey, CycleError, CycleErrorHandling, QueryCache, QueryMode}; +use rustc_middle::query::plumbing::{QuerySystem, QuerySystemFns, QueryVTable}; +use rustc_middle::query::{AsLocalKey, QueryCache, QueryMode}; use rustc_middle::ty::TyCtxt; -use rustc_span::{ErrorGuaranteed, Span}; +use rustc_span::Span; pub use crate::dep_kind_vtables::make_dep_kind_vtables; pub use crate::job::{QueryJobMap, break_query_cycles, print_query_stack}; @@ -43,180 +37,6 @@ mod job; mod profiling_support; mod values; -#[derive(ConstParamTy)] // Allow this struct to be used for const-generic values. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct QueryFlags { - /// True if this query has the `anon` modifier. - is_anon: bool, - /// True if this query has the `depth_limit` modifier. - is_depth_limit: bool, - /// True if this query has the `feedable` modifier. - is_feedable: bool, -} - -/// Combines a [`QueryVTable`] with some additional compile-time booleans. -/// "Dispatcher" should be understood as a near-synonym of "vtable". -/// -/// Baking these boolean flags into the type gives a modest but measurable -/// improvement to compiler perf and compiler code size; see -/// . -struct SemiDynamicQueryDispatcher<'tcx, C: QueryCache, const FLAGS: QueryFlags> { - vtable: &'tcx QueryVTable<'tcx, C>, -} - -// Manually implement Copy/Clone, because deriving would put trait bounds on the cache type. -impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> Copy - for SemiDynamicQueryDispatcher<'tcx, C, FLAGS> -{ -} -impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> Clone - for SemiDynamicQueryDispatcher<'tcx, C, FLAGS> -{ - fn clone(&self) -> Self { - *self - } -} - -impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> fmt::Debug - for SemiDynamicQueryDispatcher<'tcx, C, FLAGS> -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // When debug-printing a query dispatcher (e.g. for ICE or tracing), - // just print the query name to know what query we're dealing with. - // The other fields and flags are probably just unhelpful noise. - // - // If there is need for a more detailed dump of all flags and fields, - // consider writing a separate dump method and calling it explicitly. - f.write_str(self.name()) - } -} - -impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> SemiDynamicQueryDispatcher<'tcx, C, FLAGS> { - #[inline(always)] - fn name(self) -> &'static str { - self.vtable.name - } - - #[inline(always)] - fn will_cache_on_disk_for_key(self, tcx: TyCtxt<'tcx>, key: &C::Key) -> bool { - self.vtable.will_cache_on_disk_for_key_fn.map_or(false, |f| f(tcx, key)) - } - - // Don't use this method to access query results, instead use the methods on TyCtxt. - #[inline(always)] - fn query_state(self, tcx: TyCtxt<'tcx>) -> &'tcx QueryState<'tcx, C::Key> { - // Safety: - // This is just manually doing the subfield referencing through pointer math. - unsafe { - &*(&tcx.query_system.states as *const QueryStates<'tcx>) - .byte_add(self.vtable.query_state) - .cast::>() - } - } - - // Don't use this method to access query results, instead use the methods on TyCtxt. - #[inline(always)] - fn query_cache(self, tcx: TyCtxt<'tcx>) -> &'tcx C { - // Safety: - // This is just manually doing the subfield referencing through pointer math. - unsafe { - &*(&tcx.query_system.caches as *const QueryCaches<'tcx>) - .byte_add(self.vtable.query_cache) - .cast::() - } - } - - /// Calls `tcx.$query(key)` for this query, and discards the returned value. - /// See [`QueryVTable::call_query_method_fn`] for details of this strange operation. - #[inline(always)] - fn call_query_method(self, tcx: TyCtxt<'tcx>, key: C::Key) { - (self.vtable.call_query_method_fn)(tcx, key) - } - - /// Calls the actual provider function for this query. - /// See [`QueryVTable::invoke_provider_fn`] for more details. - #[inline(always)] - fn invoke_provider(self, tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value { - (self.vtable.invoke_provider_fn)(tcx, key) - } - - #[inline(always)] - fn try_load_from_disk( - self, - tcx: TyCtxt<'tcx>, - key: &C::Key, - prev_index: SerializedDepNodeIndex, - index: DepNodeIndex, - ) -> Option { - // `?` will return None immediately for queries that never cache to disk. - self.vtable.try_load_from_disk_fn?(tcx, key, prev_index, index) - } - - #[inline] - fn is_loadable_from_disk( - self, - tcx: TyCtxt<'tcx>, - key: &C::Key, - index: SerializedDepNodeIndex, - ) -> bool { - self.vtable.is_loadable_from_disk_fn.map_or(false, |f| f(tcx, key, index)) - } - - /// Synthesize an error value to let compilation continue after a cycle. - fn value_from_cycle_error( - self, - tcx: TyCtxt<'tcx>, - cycle_error: &CycleError, - guar: ErrorGuaranteed, - ) -> C::Value { - (self.vtable.value_from_cycle_error)(tcx, cycle_error, guar) - } - - #[inline(always)] - fn format_value(self) -> fn(&C::Value) -> String { - self.vtable.format_value - } - - #[inline(always)] - fn anon(self) -> bool { - FLAGS.is_anon - } - - #[inline(always)] - fn eval_always(self) -> bool { - self.vtable.eval_always - } - - #[inline(always)] - fn depth_limit(self) -> bool { - FLAGS.is_depth_limit - } - - #[inline(always)] - fn feedable(self) -> bool { - FLAGS.is_feedable - } - - #[inline(always)] - fn dep_kind(self) -> DepKind { - self.vtable.dep_kind - } - - #[inline(always)] - fn cycle_error_handling(self) -> CycleErrorHandling { - self.vtable.cycle_error_handling - } - - #[inline(always)] - fn hash_result(self) -> HashResult { - self.vtable.hash_result - } - - fn construct_dep_node(self, tcx: TyCtxt<'tcx>, key: &C::Key) -> DepNode { - DepNode::construct(tcx, self.dep_kind(), key) - } -} - /// Provides access to vtable-like operations for a query /// (by creating a [`SemiDynamicQueryDispatcher`]), /// but also keeps track of the "unerased" value type of the query @@ -228,12 +48,12 @@ impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> SemiDynamicQueryDispatcher<'t /// /// There is one macro-generated implementation of this trait for each query, /// on the type `rustc_query_impl::query_impl::$name::QueryType`. -trait QueryDispatcherUnerased<'tcx, C: QueryCache, const FLAGS: QueryFlags> { +trait QueryDispatcherUnerased<'tcx, C: QueryCache> { type UnerasedValue; const NAME: &'static &'static str; - fn query_dispatcher(tcx: TyCtxt<'tcx>) -> SemiDynamicQueryDispatcher<'tcx, C, FLAGS>; + fn query_dispatcher(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, C>; fn restore_val(value: C::Value) -> Self::UnerasedValue; } diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 5981e7e0f5269..5680e650fe968 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -31,10 +31,10 @@ use rustc_middle::ty::{self, TyCtxt}; use rustc_serialize::{Decodable, Encodable}; use rustc_span::def_id::LOCAL_CRATE; +use crate::QueryDispatcherUnerased; use crate::error::{QueryOverflow, QueryOverflowNote}; use crate::execution::{all_inactive, force_query}; use crate::job::{QueryJobMap, find_dep_kind_root}; -use crate::{QueryDispatcherUnerased, QueryFlags, SemiDynamicQueryDispatcher}; fn depth_limit_error<'tcx>(tcx: TyCtxt<'tcx>, job: QueryJobId) { let job_map = @@ -189,7 +189,7 @@ macro_rules! is_eval_always { }; } -macro_rules! depth_limit { +macro_rules! is_depth_limit { ([]) => {{ false }}; @@ -197,11 +197,11 @@ macro_rules! depth_limit { true }}; ([$other:tt $($modifiers:tt)*]) => { - depth_limit!([$($modifiers)*]) + is_depth_limit!([$($modifiers)*]) }; } -macro_rules! feedable { +macro_rules! is_feedable { ([]) => {{ false }}; @@ -209,7 +209,7 @@ macro_rules! feedable { true }}; ([$other:tt $($modifiers:tt)*]) => { - feedable!([$($modifiers)*]) + is_feedable!([$($modifiers)*]) }; } @@ -335,16 +335,16 @@ where QueryStackFrame::new(info, kind, hash, def_id, def_id_for_ty_in_cycle) } -pub(crate) fn encode_query_results<'a, 'tcx, Q, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +pub(crate) fn encode_query_results<'a, 'tcx, Q, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, encoder: &mut CacheEncoder<'a, 'tcx>, query_result_index: &mut EncodedDepNodeIndex, ) where - Q: QueryDispatcherUnerased<'tcx, C, FLAGS>, + Q: QueryDispatcherUnerased<'tcx, C>, Q::UnerasedValue: Encodable>, { - let _timer = tcx.prof.generic_activity_with_arg("encode_query_results_for", query.name()); + let _timer = tcx.prof.generic_activity_with_arg("encode_query_results_for", query.name); assert!(all_inactive(query.query_state(tcx))); let cache = query.query_cache(tcx); @@ -362,16 +362,16 @@ pub(crate) fn encode_query_results<'a, 'tcx, Q, C: QueryCache, const FLAGS: Quer }); } -pub(crate) fn query_key_hash_verify<'tcx, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +pub(crate) fn query_key_hash_verify<'tcx, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, ) { - let _timer = tcx.prof.generic_activity_with_arg("query_key_hash_verify_for", query.name()); + let _timer = tcx.prof.generic_activity_with_arg("query_key_hash_verify_for", query.name); let cache = query.query_cache(tcx); let mut map = UnordMap::with_capacity(cache.len()); cache.iter(&mut |key, _, _| { - let node = DepNode::construct(tcx, query.dep_kind(), key); + let node = DepNode::construct(tcx, query.dep_kind, key); if let Some(other_key) = map.insert(node, *key) { bug!( "query key:\n\ @@ -389,8 +389,8 @@ pub(crate) fn query_key_hash_verify<'tcx, C: QueryCache, const FLAGS: QueryFlags } /// Implementation of [`DepKindVTable::try_load_from_on_disk_cache`] for queries. -pub(crate) fn try_load_from_on_disk_cache_inner<'tcx, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +pub(crate) fn try_load_from_on_disk_cache_inner<'tcx, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, dep_node: DepNode, ) { @@ -405,7 +405,7 @@ pub(crate) fn try_load_from_on_disk_cache_inner<'tcx, C: QueryCache, const FLAGS if query.will_cache_on_disk_for_key(tcx, &key) { // Call `tcx.$query(key)` for its side-effect of loading the disk-cached // value into memory. - query.call_query_method(tcx, key); + (query.call_query_method_fn)(tcx, key); } } @@ -442,8 +442,8 @@ where } /// Implementation of [`DepKindVTable::force_from_dep_node`] for queries. -pub(crate) fn force_from_dep_node_inner<'tcx, C: QueryCache, const FLAGS: QueryFlags>( - query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, +pub(crate) fn force_from_dep_node_inner<'tcx, C: QueryCache>( + query: &'tcx QueryVTable<'tcx, C>, tcx: TyCtxt<'tcx>, dep_node: DepNode, ) -> bool { @@ -566,7 +566,10 @@ macro_rules! define_queries { { QueryVTable { name: stringify!($name), + anon: is_anon!([$($modifiers)*]), eval_always: is_eval_always!([$($modifiers)*]), + depth_limit: is_depth_limit!([$($modifiers)*]), + feedable: is_feedable!([$($modifiers)*]), dep_kind: dep_graph::dep_kinds::$name, cycle_error_handling: cycle_error_handling!([$($modifiers)*]), query_state: std::mem::offset_of!(QueryStates<'tcx>, $name), @@ -621,13 +624,7 @@ macro_rules! define_queries { data: PhantomData<&'tcx ()> } - const FLAGS: QueryFlags = QueryFlags { - is_anon: is_anon!([$($modifiers)*]), - is_depth_limit: depth_limit!([$($modifiers)*]), - is_feedable: feedable!([$($modifiers)*]), - }; - - impl<'tcx> QueryDispatcherUnerased<'tcx, queries::$name::Storage<'tcx>, FLAGS> + impl<'tcx> QueryDispatcherUnerased<'tcx, queries::$name::Storage<'tcx>> for QueryType<'tcx> { type UnerasedValue = queries::$name::Value<'tcx>; @@ -636,11 +633,9 @@ macro_rules! define_queries { #[inline(always)] fn query_dispatcher(tcx: TyCtxt<'tcx>) - -> SemiDynamicQueryDispatcher<'tcx, queries::$name::Storage<'tcx>, FLAGS> + -> &'tcx QueryVTable<'tcx, queries::$name::Storage<'tcx>> { - SemiDynamicQueryDispatcher { - vtable: &tcx.query_system.query_vtables.$name, - } + &tcx.query_system.query_vtables.$name } #[inline(always)] @@ -704,7 +699,6 @@ macro_rules! define_queries { ) { $crate::plumbing::encode_query_results::< query_impl::$name::QueryType<'tcx>, - _, _ > ( query_impl::$name::QueryType::query_dispatcher(tcx), @@ -794,7 +788,8 @@ macro_rules! define_queries { /// `DepKindVTable` constructor for this query. pub(crate) fn $name<'tcx>() -> DepKindVTable<'tcx> { use $crate::query_impl::$name::QueryType; - make_dep_kind_vtable_for_query::, _, _>( + make_dep_kind_vtable_for_query::, _>( + is_anon!([$($modifiers)*]), is_eval_always!([$($modifiers)*]), ) } From e7fb201bbae9b192fb7ac102e367863e82ac1115 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 19 Feb 2026 15:36:53 +1100 Subject: [PATCH 2/2] Rename `query_dispatcher` and reduce its use. It's now `query_vtable` because its return type changed. And thanks to the previous commit it can be manually inlined in several places. (The only remaining calls to it are in `make_dep_kind_vtable_for_query`, which are more challenging to remove.) --- compiler/rustc_query_impl/src/dep_kind_vtables.rs | 4 ++-- compiler/rustc_query_impl/src/lib.rs | 9 ++++----- compiler/rustc_query_impl/src/plumbing.rs | 10 +++++----- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/compiler/rustc_query_impl/src/dep_kind_vtables.rs b/compiler/rustc_query_impl/src/dep_kind_vtables.rs index ea9af88fbfd3d..e186e13ccab07 100644 --- a/compiler/rustc_query_impl/src/dep_kind_vtables.rs +++ b/compiler/rustc_query_impl/src/dep_kind_vtables.rs @@ -140,10 +140,10 @@ where is_eval_always, key_fingerprint_style, force_from_dep_node: Some(|tcx, dep_node, _| { - force_from_dep_node_inner(Q::query_dispatcher(tcx), tcx, dep_node) + force_from_dep_node_inner(Q::query_vtable(tcx), tcx, dep_node) }), try_load_from_on_disk_cache: Some(|tcx, dep_node| { - try_load_from_on_disk_cache_inner(Q::query_dispatcher(tcx), tcx, dep_node) + try_load_from_on_disk_cache_inner(Q::query_vtable(tcx), tcx, dep_node) }), name: Q::NAME, } diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 79baae6c8326c..18325be71c2bb 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -37,10 +37,9 @@ mod job; mod profiling_support; mod values; -/// Provides access to vtable-like operations for a query -/// (by creating a [`SemiDynamicQueryDispatcher`]), -/// but also keeps track of the "unerased" value type of the query -/// (i.e. the actual result type in the query declaration). +/// Provides access to vtable-like operations for a query (by creating a +/// `QueryVTable`), but also keeps track of the "unerased" value type of the +/// query (i.e. the actual result type in the query declaration). /// /// This trait allows some per-query code to be defined in generic functions /// with a trait bound, instead of having to be defined inline within a macro @@ -53,7 +52,7 @@ trait QueryDispatcherUnerased<'tcx, C: QueryCache> { const NAME: &'static &'static str; - fn query_dispatcher(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, C>; + fn query_vtable(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, C>; fn restore_val(value: C::Value) -> Self::UnerasedValue; } diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 5680e650fe968..9535c65054c4b 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -503,7 +503,7 @@ macro_rules! define_queries { #[cfg(debug_assertions)] let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); execution::get_query_incr( - QueryType::query_dispatcher(tcx), + &tcx.query_system.query_vtables.$name, tcx, span, key, @@ -523,7 +523,7 @@ macro_rules! define_queries { __mode: QueryMode, ) -> Option>> { Some(execution::get_query_non_incr( - QueryType::query_dispatcher(tcx), + &tcx.query_system.query_vtables.$name, tcx, span, key, @@ -632,7 +632,7 @@ macro_rules! define_queries { const NAME: &'static &'static str = &stringify!($name); #[inline(always)] - fn query_dispatcher(tcx: TyCtxt<'tcx>) + fn query_vtable(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, queries::$name::Storage<'tcx>> { &tcx.query_system.query_vtables.$name @@ -701,7 +701,7 @@ macro_rules! define_queries { query_impl::$name::QueryType<'tcx>, _ > ( - query_impl::$name::QueryType::query_dispatcher(tcx), + &tcx.query_system.query_vtables.$name, tcx, encoder, query_result_index, @@ -711,7 +711,7 @@ macro_rules! define_queries { pub(crate) fn query_key_hash_verify<'tcx>(tcx: TyCtxt<'tcx>) { $crate::plumbing::query_key_hash_verify( - query_impl::$name::QueryType::query_dispatcher(tcx), + &tcx.query_system.query_vtables.$name, tcx, ) }