diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index a7f4e75e2143a..71cc8231dcc6d 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -1718,6 +1718,7 @@ rustc_queries! { desc { |tcx| "computing visibility of `{}`", tcx.def_path_str(def_id) } separate_provide_extern feedable + cache_on_disk_if { def_id.is_local() } } query inhabited_predicate_adt(key: DefId) -> ty::inhabitedness::InhabitedPredicate<'tcx> { diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index cc734e7157fa4..5d54438761024 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -55,8 +55,9 @@ use rustc_hir::lang_items::LangItem; use rustc_hir::{HirId, Node, TraitCandidate}; use rustc_index::IndexVec; use rustc_macros::HashStable; -use rustc_query_system::dep_graph::DepNodeIndex; +use rustc_query_system::dep_graph::{DepNodeIndex, TaskDepsRef}; use rustc_query_system::ich::StableHashingContext; +use rustc_query_system::query::DefIdInfo; use rustc_serialize::opaque::{FileEncodeResult, FileEncoder}; use rustc_session::config::CrateType; use rustc_session::cstore::{CrateStoreDyn, Untracked}; @@ -1087,9 +1088,28 @@ impl<'tcx> TyCtxt<'tcx> { // This function modifies `self.definitions` using a side-effect. // We need to ensure that these side effects are re-run by the incr. comp. engine. - // Depending on the forever-red node will tell the graph that the calling query - // needs to be re-evaluated. - self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE); + tls::with_context_opt(|icx| { + let icx = icx.unwrap(); + let Some(side_effects) = &icx.side_effects else { + assert!(matches!(icx.task_deps, TaskDepsRef::Ignore), "{:?}", icx.task_deps); + return; + }; + let info = DefIdInfo { parent, data }; + match icx.task_deps { + TaskDepsRef::Allow(_) + | TaskDepsRef::EvalAlways + | TaskDepsRef::Ignore + | TaskDepsRef::Forbid => { + side_effects.lock().definitions.push(DefIdInfo { parent, data }); + } + TaskDepsRef::Replay { prev_side_effects, created_def_ids } => { + let prev_info = &prev_side_effects.definitions + [created_def_ids.load(std::sync::atomic::Ordering::Relaxed)]; + created_def_ids.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + assert_eq!(*prev_info, info); + } + } + }); let feed = self.feed_local_def_id(def_id); feed.def_kind(def_kind); diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 06f00c7b29b48..aa4cd81b8dc80 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -21,8 +21,8 @@ use rustc_middle::ty::{self, print::with_no_queries, TyCtxt}; use rustc_query_system::dep_graph::{DepNodeParams, HasDepContext}; use rustc_query_system::ich::StableHashingContext; use rustc_query_system::query::{ - force_query, QueryCache, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffects, - QueryStackFrame, + force_query, DefIdInfo, QueryCache, QueryConfig, QueryContext, QueryJobId, QueryMap, + QuerySideEffects, QueryStackFrame, }; use rustc_query_system::{LayoutOfDepth, QueryOverflow}; use rustc_serialize::Decodable; @@ -176,11 +176,15 @@ impl QueryContext for QueryCtxt<'_> { #[tracing::instrument(level = "trace", skip(self))] fn apply_side_effects(self, side_effects: QuerySideEffects) { let dcx = self.dep_context().sess().dcx(); - let QuerySideEffects { diagnostics } = side_effects; + let QuerySideEffects { diagnostics, definitions } = side_effects; for diagnostic in diagnostics { dcx.emit_diagnostic(diagnostic); } + + for DefIdInfo { parent, data } in definitions { + self.tcx.untracked().definitions.write().create_def(parent, data); + } } } diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 6944db5bde0bd..0e9e78142e01c 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -1,10 +1,16 @@ +use super::query::DepGraphQuery; +use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex}; +use super::{DepContext, DepKind, DepNode, Deps, HasDepContext, WorkProductId}; +use crate::dep_graph::edges::EdgesVec; +use crate::ich::StableHashingContext; +use crate::query::{QueryContext, QuerySideEffects}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::profiling::{EventId, QueryInvocationId, SelfProfilerRef}; use rustc_data_structures::sharded::{self, Sharded}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::steal::Steal; -use rustc_data_structures::sync::{AtomicU32, AtomicU64, Lock, Lrc}; +use rustc_data_structures::sync::{AtomicU32, AtomicU64, AtomicUsize, Lock, Lrc}; use rustc_data_structures::unord::UnordMap; use rustc_index::IndexVec; use rustc_serialize::opaque::{FileEncodeResult, FileEncoder}; @@ -15,13 +21,6 @@ use std::hash::Hash; use std::marker::PhantomData; use std::sync::atomic::Ordering; -use super::query::DepGraphQuery; -use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex}; -use super::{DepContext, DepKind, DepNode, Deps, HasDepContext, WorkProductId}; -use crate::dep_graph::edges::EdgesVec; -use crate::ich::StableHashingContext; -use crate::query::{QueryContext, QuerySideEffects}; - #[cfg(debug_assertions)] use {super::debug::EdgeFilter, std::env}; @@ -218,6 +217,15 @@ impl DepGraph { D::with_deps(TaskDepsRef::Ignore, op) } + pub(crate) fn with_replay( + &self, + prev_side_effects: &QuerySideEffects, + created_def_ids: &AtomicUsize, + op: impl FnOnce() -> R, + ) -> R { + D::with_deps(TaskDepsRef::Replay { prev_side_effects, created_def_ids }, op) + } + /// Used to wrap the deserialization of a query result from disk, /// This method enforces that no new `DepNodes` are created during /// query result deserialization. @@ -272,6 +280,7 @@ impl DepGraph { } #[inline(always)] + /// A helper for `codegen_cranelift`. pub fn with_task, A: Debug, R>( &self, key: DepNode, @@ -468,6 +477,12 @@ impl DepGraph { return; } TaskDepsRef::Ignore => return, + // We don't need to record dependencies when rerunning a query + // because we have no disk cache entry to load. The dependencies + // are preserved. + // FIXME: assert that the dependencies don't change instead of + // recording them. + TaskDepsRef::Replay { .. } => return, TaskDepsRef::Forbid => { panic!("Illegal read of: {dep_node_index:?}") } @@ -573,6 +588,7 @@ impl DepGraph { edges.push(DepNodeIndex::FOREVER_RED_NODE); } TaskDepsRef::Ignore => {} + TaskDepsRef::Replay { .. } => {} TaskDepsRef::Forbid => { panic!("Cannot summarize when dependencies are not recorded.") } @@ -1323,6 +1339,16 @@ pub enum TaskDepsRef<'a> { /// to ensure that the decoding process doesn't itself /// require the execution of any queries. Forbid, + /// Side effects from the previous run made available to + /// queries when they are reexecuted because their result was not + /// available in the cache. The query removes entries from the + /// side effect table. The table must be empty + Replay { + prev_side_effects: &'a QuerySideEffects, + /// Every new `DefId` is pushed here so we can check + /// that they match the cached ones. + created_def_ids: &'a AtomicUsize, + }, } #[derive(Debug)] diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs index a908087fd45a0..66f7bf8fd53d0 100644 --- a/compiler/rustc_query_system/src/query/mod.rs +++ b/compiler/rustc_query_system/src/query/mod.rs @@ -23,7 +23,7 @@ use rustc_data_structures::stable_hasher::Hash64; use rustc_data_structures::sync::Lock; use rustc_errors::Diagnostic; use rustc_hir::def::DefKind; -use rustc_span::def_id::DefId; +use rustc_span::def_id::{DefId, LocalDefId}; use rustc_span::Span; use thin_vec::ThinVec; @@ -90,20 +90,30 @@ pub struct QuerySideEffects { /// These diagnostics will be re-emitted if we mark /// the query as green. pub diagnostics: ThinVec, + /// Stores any `DefId`s that were created during query execution. + /// These `DefId`s will be re-created when we mark the query as green. + pub definitions: ThinVec, +} + +#[derive(Debug, Clone, Encodable, Decodable, PartialEq)] +pub struct DefIdInfo { + pub parent: LocalDefId, + pub data: rustc_hir::definitions::DefPathData, } impl QuerySideEffects { /// Returns true if there might be side effects. #[inline] pub fn maybe_any(&self) -> bool { - let QuerySideEffects { diagnostics } = self; + let QuerySideEffects { diagnostics, definitions } = self; // Use `has_capacity` so that the destructor for `self.diagnostics` can be skipped // if `maybe_any` is known to be false. - diagnostics.has_capacity() + diagnostics.has_capacity() || definitions.has_capacity() } pub fn append(&mut self, other: QuerySideEffects) { - let QuerySideEffects { diagnostics } = self; + let QuerySideEffects { diagnostics, definitions } = self; diagnostics.extend(other.diagnostics); + definitions.extend(other.definitions); } } diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 7b23b4b2dd9db..c29e7676be944 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -16,7 +16,7 @@ use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sharded::Sharded; use rustc_data_structures::stack::ensure_sufficient_stack; -use rustc_data_structures::sync::Lock; +use rustc_data_structures::sync::{AtomicUsize, Lock}; #[cfg(parallel_compiler)] use rustc_data_structures::{outline, sync}; use rustc_errors::{DiagnosticBuilder, FatalError, StashKey}; @@ -502,7 +502,7 @@ where let dep_node = dep_node_opt.get_or_insert_with(|| query.construct_dep_node(*qcx.dep_context(), &key)); - // The diagnostics for this query will be promoted to the current session during + // The side_effects for this query will be promoted to the current session during // `try_mark_green()`, so we can ignore them here. if let Some(ret) = qcx.start_query(job_id, false, None, || { try_load_from_disk_and_cache_in_memory(query, dep_graph_data, qcx, &key, dep_node) @@ -620,8 +620,17 @@ where // recompute. let prof_timer = qcx.dep_context().profiler().query_provider(); + let prev_side_effects = qcx.load_side_effects(prev_dep_node_index); + let created_def_ids = AtomicUsize::new(0); // The dep-graph for this computation is already in-place. - let result = qcx.dep_context().dep_graph().with_ignore(|| query.compute(qcx, *key)); + let result = + qcx.dep_context() + .dep_graph() + .with_replay(&prev_side_effects, &created_def_ids, || query.compute(qcx, *key)); + + assert_eq!(created_def_ids.into_inner(), prev_side_effects.definitions.len()); + // We already checked at `DefId` creation time, that the created `DefId`s have the same parent and `DefPathData` + // as the cached ones. prof_timer.finish_with_query_invocation_id(dep_node_index.into());