Overhaul rustc_query_impl

compiler/rustc_query_impl/src/dep_kind_vtables.rs

@@ -3,8 +3,8 @@ use rustc_middle::bug;
 use rustc_middle::dep_graph::{DepKindVTable, DepNodeKey, KeyFingerprintStyle};
 use rustc_middle::query::QueryCache;
 
-use crate::GetQueryVTable;
-use crate::plumbing::promote_from_disk_inner;
+use crate::incremental::promote_from_disk_inner;
+use crate::query_vtables::GetQueryVTable;
 
 /// [`DepKindVTable`] constructors for special dep kinds that aren't queries.
 #[expect(non_snake_case, reason = "use non-snake case to avoid collision with query names")]
@@ -166,7 +166,7 @@ macro_rules! define_dep_kind_vtables {
         let q_vtables: [DepKindVTable<'tcx>; _] = [
             $(
                 $crate::dep_kind_vtables::make_dep_kind_vtable_for_query::<
-                    $crate::query_impl::$name::VTableGetter,
+                    $crate::query_vtables::$name::VTableGetter,
                 >(
                     $cache_on_disk,
                     $eval_always,

compiler/rustc_query_impl/src/execution.rs

@@ -1,117 +1,35 @@
 use std::hash::Hash;
 use std::mem::ManuallyDrop;
+use std::num::NonZero;
 
-use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint};
-use rustc_data_structures::hash_table::{Entry, HashTable};
-use rustc_data_structures::sync::{DynSend, DynSync};
-use rustc_data_structures::{defer, outline, sharded, sync};
+use rustc_data_structures::hash_table::Entry;
+use rustc_data_structures::{Limit, defer, outline, sharded, sync};
 use rustc_errors::FatalError;
-use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, SerializedDepNodeIndex};
+use rustc_middle::dep_graph::{
+    DepGraphData, DepNode, DepNodeIndex, DepNodeKey, SerializedDepNodeIndex,
+};
 use rustc_middle::query::{
-    ActiveKeyStatus, Cycle, QueryCache, QueryJob, QueryJobId, QueryKey, QueryLatch, QueryMode,
-    QueryState, QueryVTable,
+    ActiveKeyStatus, Cycle, QueryCache, QueryJob, QueryJobId, QueryLatch, QueryMode, QueryState,
+    QueryVTable,
 };
 use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::tls::{self, ImplicitCtxt};
 use rustc_middle::verify_ich::incremental_verify_ich;
+use rustc_span::def_id::LOCAL_CRATE;
 use rustc_span::{DUMMY_SP, Span};
-use tracing::debug;
 
-use crate::dep_graph::{DepNode, DepNodeIndex};
+use crate::diagnostics::{QueryOverflow, QueryOverflowNote};
 use crate::handle_cycle_error;
-use crate::job::{QueryJobInfo, QueryJobMap, create_cycle_error, find_cycle_in_stack};
-use crate::plumbing::{current_query_job, next_job_id, start_query};
-use crate::query_impl::for_each_query_vtable;
+use crate::incremental::should_verify_loaded_value;
+use crate::job::{
+    CollectActiveJobsKind, collect_active_query_jobs, find_cycle_in_stack, find_dep_kind_root,
+};
 
 #[inline]
 fn equivalent_key<K: Eq, V>(k: K) -> impl Fn(&(K, V)) -> bool {
     move |x| x.0 == k
 }
 
-pub(crate) fn all_inactive<'tcx, K>(state: &QueryState<'tcx, K>) -> bool {
-    state.active.lock_shards().all(|shard| shard.is_empty())
-}
-
-#[derive(Clone, Copy)]
-pub enum CollectActiveJobsKind {
-    /// We need the full query job map, and we are willing to wait to obtain the query state
-    /// shard lock(s).
-    Full,
-
-    /// We need the full query job map, and we shouldn't need to wait to obtain the shard lock(s),
-    /// because we are in a place where nothing else could hold the shard lock(s).
-    FullNoContention,
-
-    /// We can get by without the full query job map, so we won't bother waiting to obtain the
-    /// shard lock(s) if they're not already unlocked.
-    PartialAllowed,
-}
-
-/// Returns a map of currently active query jobs, collected from all queries.
-pub fn collect_active_query_jobs<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    collect_kind: CollectActiveJobsKind,
-) -> QueryJobMap<'tcx> {
-    let mut job_map = QueryJobMap::default();
-
-    for_each_query_vtable!(ALL, tcx, |query| {
-        collect_active_query_jobs_inner(query, collect_kind, &mut job_map);
-    });
-
-    job_map
-}
-
-/// Internal plumbing for collecting the set of active jobs for this query.
-///
-/// Aborts if jobs can't be gathered as specified by `collect_kind`.
-fn collect_active_query_jobs_inner<'tcx, C>(
-    query: &'tcx QueryVTable<'tcx, C>,
-    collect_kind: CollectActiveJobsKind,
-    job_map: &mut QueryJobMap<'tcx>,
-) where
-    C: QueryCache<Key: QueryKey + DynSend + DynSync>,
-    QueryVTable<'tcx, C>: DynSync,
-{
-    let mut collect_shard_jobs = |shard: &HashTable<(C::Key, ActiveKeyStatus<'tcx>)>| {
-        for (key, status) in shard.iter() {
-            if let ActiveKeyStatus::Started(job) = status {
-                // It's fine to call `create_tagged_key` with the shard locked,
-                // because it's just a `TaggedQueryKey` variant constructor.
-                let tagged_key = (query.create_tagged_key)(*key);
-                job_map.insert(job.id, QueryJobInfo { tagged_key, job: job.clone() });
-            }
-        }
-    };
-
-    match collect_kind {
-        CollectActiveJobsKind::Full => {
-            for shard in query.state.active.lock_shards() {
-                collect_shard_jobs(&shard);
-            }
-        }
-        CollectActiveJobsKind::FullNoContention => {
-            for shard in query.state.active.try_lock_shards() {
-                match shard {
-                    Some(shard) => collect_shard_jobs(&shard),
-                    None => panic!("Failed to collect active jobs for query `{}`!", query.name),
-                }
-            }
-        }
-        CollectActiveJobsKind::PartialAllowed => {
-            for shard in query.state.active.try_lock_shards() {
-                match shard {
-                    Some(shard) => collect_shard_jobs(&shard),
-                    // This collection is best-effort (it is only used to print the query
-                    // stack on panic), so a contended shard is expected and fine to skip.
-                    // Emitting this at `warn!` would leak nondeterministically into the
-                    // panic output under the parallel front-end, where another thread may
-                    // still hold a shard lock, so keep it at `debug!`.
-                    None => debug!("Failed to collect active jobs for query `{}`!", query.name),
-                }
-            }
-        }
-    }
-}
-
 #[cold]
 #[inline(never)]
 fn handle_cycle<'tcx, C: QueryCache>(
@@ -135,7 +53,7 @@ fn handle_cycle<'tcx, C: QueryCache>(
     }
     let _guard = defer(|| *tcx.query_system.cycle_handler_nesting.lock() -= 1);
 
-    let error = create_cycle_error(tcx, &cycle, nested);
+    let error = handle_cycle_error::create_cycle_error(tcx, &cycle, nested);
 
     if nested {
         // Avoid custom handlers and only use the robust `create_cycle_error` for nested cycle errors
@@ -281,6 +199,19 @@ fn wait_for_query<'tcx, C: QueryCache>(
     }
 }
 
+#[inline]
+fn next_job_id<'tcx>(tcx: TyCtxt<'tcx>) -> QueryJobId {
+    QueryJobId(
+        NonZero::new(tcx.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed))
+            .unwrap(),
+    )
+}
+
+#[inline]
+fn current_query_job() -> Option<QueryJobId> {
+    tls::with_context(|icx| icx.query)
+}
+
 /// Shared main part of both [`execute_query_incr_inner`] and [`execute_query_non_incr_inner`].
 #[inline(never)]
 fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>(
@@ -407,6 +338,43 @@ fn check_feedable_consistency<'tcx, C: QueryCache>(
     }
 }
 
+fn depth_limit_error<'tcx>(tcx: TyCtxt<'tcx>, job: QueryJobId) {
+    let job_map = collect_active_query_jobs(tcx, CollectActiveJobsKind::Full);
+    let (span, desc, depth) = find_dep_kind_root(tcx, job, job_map);
+
+    let suggested_limit = match tcx.recursion_limit() {
+        Limit(0) => Limit(2),
+        limit => limit * 2,
+    };
+
+    tcx.dcx().emit_fatal(QueryOverflow {
+        span,
+        note: QueryOverflowNote { desc, depth },
+        suggested_limit,
+        crate_name: tcx.crate_name(LOCAL_CRATE),
+    });
+}
+
+/// Executes a job by changing the `ImplicitCtxt` to point to the new query job while it executes.
+#[inline(always)]
+fn start_query<R>(job_id: QueryJobId, depth_limit: bool, compute: impl FnOnce() -> R) -> R {
+    tls::with_context(move |icx| {
+        if depth_limit && !icx.tcx.recursion_limit().value_within_limit(icx.query_depth) {
+            depth_limit_error(icx.tcx, job_id);
+        }
+
+        // Update the `ImplicitCtxt` to point to our new query job.
+        let icx = ImplicitCtxt {
+            query: Some(job_id),
+            query_depth: icx.query_depth + if depth_limit { 1 } else { 0 },
+            ..*icx
+        };
+
+        // Use the `ImplicitCtxt` while we execute the query.
+        tls::enter_context(&icx, compute)
+    })
+}
+
 // Fast path for when incr. comp. is off.
 #[inline(always)]
 fn execute_job_non_incr<'tcx, C: QueryCache>(
@@ -484,29 +452,6 @@ fn execute_job_incr<'tcx, C: QueryCache>(
     (result, dep_node_index)
 }
 
-/// Whether a value loaded from the on-disk cache should have its fingerprint
-/// verified with `incremental_verify_ich`. If `-Zincremental-verify-ich` is
-/// specified, re-hash results from the cache and make sure that they have the
-/// expected fingerprint.
-///
-/// If not, we still verify a subset: re-hashing is too expensive to do for
-/// every value. The subset rotates with the session count, covering the whole
-/// cache every 32 sessions, and is deterministic so that a verification
-/// failure reproduces on retry.
-///
-/// `to_smaller_hash` mixes both fingerprint halves because neither half is
-/// evenly distributed on its own (`DefPathHash` keys share the
-/// `StableCrateId`, `HirId` keys contain a sequential id).
-pub(crate) fn should_verify_loaded_value(
-    tcx: TyCtxt<'_>,
-    dep_graph_data: &DepGraphData,
-    key_fingerprint: PackedFingerprint,
-) -> bool {
-    let hash = Fingerprint::from(key_fingerprint).to_smaller_hash().as_u64();
-    hash % 32 == dep_graph_data.session_count() % 32
-        || tcx.sess.opts.unstable_opts.incremental_verify_ich
-}
-
 /// Given that the dep node for this query+key is green, obtain a value for it
 /// by loading one from disk if possible, or by invoking its query provider if
 /// necessary.

compiler/rustc_query_impl/src/handle_cycle_error.rs

@@ -13,9 +13,7 @@ use rustc_middle::queries::TaggedQueryKey;
 use rustc_middle::query::Cycle;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_span::def_id::{DefId, LocalDefId};
-use rustc_span::{ErrorGuaranteed, Span};
-
-use crate::job::create_cycle_error;
+use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span};
 
 // Default cycle handler used for all queries that don't use the `handle_cycle_error` query
 // modifier.
@@ -343,3 +341,91 @@ fn find_item_ty_spans(
         _ => {}
     }
 }
+
+#[inline(never)]
+#[cold]
+pub(crate) fn create_cycle_error<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    Cycle { usage, frames }: &Cycle<'tcx>,
+    nested: bool,
+) -> Diag<'tcx> {
+    assert!(!frames.is_empty());
+
+    let span = frames[0].tagged_key.catch_default_span(tcx, frames[1 % frames.len()].span);
+
+    let mut cycle_stack = Vec::new();
+
+    use crate::diagnostics::StackCount;
+    let stack_bottom = frames[0].tagged_key.catch_description(tcx);
+    let stack_count = if frames.len() == 1 {
+        StackCount::Single { stack_bottom: stack_bottom.clone() }
+    } else {
+        StackCount::Multiple { stack_bottom: stack_bottom.clone() }
+    };
+
+    let mut prev = span;
+    for i in 1..frames.len() {
+        let frame = &frames[i];
+        let span = frame.tagged_key.catch_default_span(tcx, frames[(i + 1) % frames.len()].span);
+        cycle_stack.push(crate::diagnostics::CycleStack {
+            span: if span == prev { DUMMY_SP } else { span },
+            desc: frame.tagged_key.catch_description(tcx),
+        });
+        prev = span;
+    }
+
+    let cycle_usage = usage.as_ref().map(|usage| {
+        let cycle_span = usage.tagged_key.catch_default_span(tcx, usage.span);
+        crate::diagnostics::CycleUsage {
+            span: if cycle_span != span { cycle_span } else { DUMMY_SP },
+            usage: usage.tagged_key.catch_description(tcx),
+        }
+    });
+
+    let is_all_def_kind = |def_kind| {
+        // Trivial type alias and trait alias cycles consists of `type_of` and
+        // `explicit_implied_clauses_of` queries, so we just check just these here.
+        frames.iter().all(|frame| match frame.tagged_key {
+            TaggedQueryKey::type_of(def_id)
+            | TaggedQueryKey::explicit_implied_clauses_of(def_id)
+                if tcx.def_kind(def_id) == def_kind =>
+            {
+                true
+            }
+            _ => false,
+        })
+    };
+
+    let alias = if !nested {
+        if is_all_def_kind(DefKind::TyAlias) {
+            Some(crate::diagnostics::Alias::Ty)
+        } else if is_all_def_kind(DefKind::TraitAlias) {
+            Some(crate::diagnostics::Alias::Trait)
+        } else {
+            None
+        }
+    } else {
+        None
+    };
+
+    if nested {
+        tcx.sess.dcx().create_err(crate::diagnostics::NestedCycle {
+            span,
+            cycle_stack,
+            stack_bottom: crate::diagnostics::NestedCycleBottom { stack_bottom },
+            cycle_usage,
+            stack_count,
+            note_span: (),
+        })
+    } else {
+        tcx.sess.dcx().create_err(crate::diagnostics::Cycle {
+            span,
+            cycle_stack,
+            stack_bottom,
+            alias,
+            cycle_usage,
+            stack_count,
+            note_span: (),
+        })
+    }
+}