chore: Adjust #[inline] annotations

src/active_query.rs

@@ -66,7 +66,6 @@ pub(crate) struct ActiveQuery {
 }
 
 impl ActiveQuery {
-    #[inline]
     pub(super) fn add_read(
         &mut self,
         input: DatabaseKeyIndex,
@@ -287,12 +286,14 @@ impl std::fmt::Debug for QueryStack {
 impl ops::Deref for QueryStack {
     type Target = [ActiveQuery];
 
+    #[inline(always)]
     fn deref(&self) -> &Self::Target {
         &self.stack[..self.len]
     }
 }
 
 impl ops::DerefMut for QueryStack {
+    #[inline(always)]
     fn deref_mut(&mut self) -> &mut Self::Target {
         &mut self.stack[..self.len]
     }

src/attach.rs

@@ -26,6 +26,7 @@ impl Attached {
         }
     }
 
+    #[inline]
     fn attach<Db, R>(&self, db: &Db, op: impl FnOnce() -> R) -> R
     where
         Db: ?Sized + Database,
@@ -35,31 +36,31 @@ impl Attached {
         }
 
         impl<'s> DbGuard<'s> {
+            #[inline]
             fn new(attached: &'s Attached, db: &dyn Database) -> Self {
-                if let Some(current_db) = attached.database.get() {
-                    let new_db = NonNull::from(db);
-
-                    // Already attached? Assert that the database has not changed.
-                    // NOTE: It's important to use `addr_eq` here because `NonNull::eq`
-                    // not only compares the address but also the type's metadata.
-                    if !std::ptr::addr_eq(current_db.as_ptr(), new_db.as_ptr()) {
-                        panic!(
-                            "Cannot change database mid-query. current: {current_db:?}, new: {new_db:?}",
-                        );
+                match attached.database.get() {
+                    Some(current_db) => {
+                        let new_db = NonNull::from(db);
+                        if !std::ptr::addr_eq(current_db.as_ptr(), new_db.as_ptr()) {
+                            panic!(
+                                                "Cannot change database mid-query. current: {current_db:?}, new: {new_db:?}",
+                                            );
+                        }
+                        Self { state: None }
                     }
-
-                    Self { state: None }
-                } else {
-                    // Otherwise, set the database.
-                    attached.database.set(Some(NonNull::from(db)));
-                    Self {
-                        state: Some(attached),
+                    None => {
+                        // Otherwise, set the database.
+                        attached.database.set(Some(NonNull::from(db)));
+                        Self {
+                            state: Some(attached),
+                        }
                     }
                 }
             }
         }
 
         impl Drop for DbGuard<'_> {
+            #[inline]
             fn drop(&mut self) {
                 // Reset database to null if we did anything in `DbGuard::new`.
                 if let Some(attached) = self.state {
@@ -74,6 +75,7 @@ impl Attached {
 
     /// Access the "attached" database. Returns `None` if no database is attached.
     /// Databases are attached with `attach_database`.
+    #[inline]
     fn with<R>(&self, op: impl FnOnce(&dyn Database) -> R) -> Option<R> {
         let db = self.database.get()?;
 
@@ -85,15 +87,23 @@ impl Attached {
 
 /// Attach the database to the current thread and execute `op`.
 /// Panics if a different database has already been attached.
+#[inline]
 pub fn attach<R, Db>(db: &Db, op: impl FnOnce() -> R) -> R
 where
     Db: ?Sized + Database,
 {
-    ATTACHED.with(|a| a.attach(db, op))
+    ATTACHED.with(
+        #[inline]
+        |a| a.attach(db, op),
+    )
 }
 
 /// Access the "attached" database. Returns `None` if no database is attached.
 /// Databases are attached with `attach_database`.
+#[inline]
 pub fn with_attached_database<R>(op: impl FnOnce(&dyn Database) -> R) -> Option<R> {
-    ATTACHED.with(|a| a.with(op))
+    ATTACHED.with(
+        #[inline]
+        |a| a.with(op),
+    )
 }

src/database.rs

@@ -81,6 +81,7 @@ pub trait Database: Send + AsDynDatabase + Any + ZalsaDatabase {
     }
 
     /// Execute `op` with the database in thread-local storage for debug print-outs.
+    #[inline(always)]
     fn attach<R>(&self, op: impl FnOnce(&Self) -> R) -> R
     where
         Self: Sized,
@@ -89,6 +90,7 @@ pub trait Database: Send + AsDynDatabase + Any + ZalsaDatabase {
     }
 
     #[doc(hidden)]
+    #[inline(always)]
     fn zalsa_register_downcaster(&self) {
         // The no-op downcaster is special cased in view caster construction.
     }
@@ -113,10 +115,12 @@ pub trait AsDynDatabase {
 }
 
 impl<T: Database> AsDynDatabase for T {
+    #[inline(always)]
     fn as_dyn_database(&self) -> &dyn Database {
         self
     }
 
+    #[inline(always)]
     fn as_dyn_database_mut(&mut self) -> &mut dyn Database {
         self
     }

src/database_impl.rs

@@ -21,17 +21,20 @@ impl DatabaseImpl {
 
 impl Database for DatabaseImpl {
     /// Default behavior: tracing debug log the event.
+    #[inline(always)]
     fn salsa_event(&self, event: &dyn Fn() -> Event) {
         tracing::debug!("salsa_event({:?})", event());
     }
 }
 
 // SAFETY: The `storage` and `storage_mut` fields return a reference to the same storage field owned by `self`.
 unsafe impl HasStorage for DatabaseImpl {
+    #[inline(always)]
     fn storage(&self) -> &Storage<Self> {
         &self.storage
     }
 
+    #[inline(always)]
     fn storage_mut(&mut self) -> &mut Storage<Self> {
         &mut self.storage
     }

src/function.rs

@@ -157,6 +157,7 @@ where
         }
     }
 
+    #[inline]
     pub fn database_key_index(&self, key: Id) -> DatabaseKeyIndex {
         DatabaseKeyIndex::new(self.index, key)
     }

src/function/accumulated.rs

@@ -96,7 +96,7 @@ where
         key: Id,
     ) -> (Option<&'db AccumulatedMap>, InputAccumulatedValues) {
         // NEXT STEP: stash and refactor `fetch` to return an `&Memo` so we can make this work
-        let memo = self.refresh_memo(db, key);
+        let memo = self.refresh_memo(db, db.zalsa(), key);
         (
             memo.revisions.accumulated.as_deref(),
             memo.revisions.accumulated_inputs.load(),

src/function/fetch.rs

@@ -3,7 +3,7 @@ use crate::function::{Configuration, IngredientImpl, VerifyResult};
 use crate::table::sync::ClaimResult;
 use crate::zalsa::{MemoIngredientIndex, Zalsa, ZalsaDatabase};
 use crate::zalsa_local::{ActiveQueryGuard, QueryRevisions, ZalsaLocal};
-use crate::{AsDynDatabase as _, DatabaseKeyIndex, Id};
+use crate::{DatabaseKeyIndex, Id};
 
 impl<C> IngredientImpl<C>
 where
@@ -13,7 +13,7 @@ where
         let (zalsa, zalsa_local) = db.zalsas();
         zalsa.unwind_if_revision_cancelled(db);
 
-        let memo = self.refresh_memo(db, id);
+        let memo = self.refresh_memo(db, zalsa, id);
         // SAFETY: We just refreshed the memo so it is guaranteed to contain a value now.
         let memo_value = unsafe { memo.value.as_ref().unwrap_unchecked() };
 
@@ -31,13 +31,13 @@ where
         memo_value
     }
 
-    #[inline]
+    #[inline(always)]
     pub(super) fn refresh_memo<'db>(
         &'db self,
         db: &'db C::DbView,
+        zalsa: &'db Zalsa,
         id: Id,
     ) -> &'db Memo<C::Output<'db>> {
-        let zalsa = db.zalsa();
         let memo_ingredient_index = self.memo_ingredient_index(zalsa, id);
         loop {
             if let Some(memo) = self
@@ -49,13 +49,7 @@ where
                 // any further (it could escape outside the cycle); we need to block on the other
                 // thread completing fixpoint iteration of the cycle, and then we can re-query for
                 // our no-longer-provisional memo.
-                if !(memo.may_be_provisional()
-                    && memo.provisional_retry(
-                        db.as_dyn_database(),
-                        zalsa,
-                        self.database_key_index(id),
-                    ))
-                {
+                if !memo.provisional_retry(db, zalsa, self.database_key_index(id)) {
                     return memo;
                 }
             }
@@ -89,6 +83,7 @@ where
         }
     }
 
+    #[inline(never)]
     fn fetch_cold<'db>(
         &'db self,
         zalsa: &'db Zalsa,

src/function/lru.rs

@@ -20,11 +20,9 @@ impl Lru {
 
     #[inline(always)]
     pub(super) fn record_use(&self, index: Id) {
-        if self.capacity.is_none() {
-            // LRU is disabled
-            return;
+        if self.capacity.is_some() {
+            self.insert(index);
         }
-        self.insert(index);
     }
 
     #[inline(never)]

src/function/maybe_changed_after.rs

@@ -92,6 +92,7 @@ where
         }
     }
 
+    #[inline(never)]
     fn maybe_changed_after_cold<'db>(
         &'db self,
         zalsa: &Zalsa,

src/function/memo.rs

@@ -162,51 +162,76 @@ impl<V> Memo<V> {
     ///
     /// Return `true` if the caller should retry, `false` if the caller should go ahead and return
     /// this memo to the caller.
+    #[inline(always)]
     pub(super) fn provisional_retry(
         &self,
-        db: &dyn crate::Database,
+        db: &(impl crate::Database + ?Sized),
         zalsa: &Zalsa,
         database_key_index: DatabaseKeyIndex,
     ) -> bool {
-        let mut retry = false;
-        let hit_cycle = self
-            .cycle_heads()
-            .into_iter()
-            .filter(|&head| head.database_key_index != database_key_index)
-            .any(|head| {
-                let head_index = head.database_key_index;
-                let ingredient = zalsa.lookup_ingredient(head_index.ingredient_index());
-                if !ingredient.is_provisional_cycle_head(db, head_index.key_index()) {
-                    // This cycle is already finalized, so we don't need to wait on it;
-                    // keep looping through cycle heads.
-                    retry = true;
-                    false
-                } else if ingredient.wait_for(db, head_index.key_index()) {
-                    // There's a new memo available for the cycle head; fetch our own
-                    // updated memo and see if it's still provisional or if the cycle
-                    // has resolved.
-                    retry = true;
-                    false
-                } else {
-                    // We hit a cycle blocking on the cycle head; this means it's in
-                    // our own active query stack and we are responsible to resolve the
-                    // cycle, so go ahead and return the provisional memo.
-                    true
-                }
-            });
-        if hit_cycle {
+        if !self.may_be_provisional() {
+            return false;
+        };
+        if self.revisions.cycle_heads.is_empty() {
             return false;
         }
-        // If `retry` is `true`, all our cycle heads (barring ourself) are complete; re-fetch
-        // and we should get a non-provisional memo. If we get here and `retry` is still
-        // `false`, we have no cycle heads other than ourself, so we are a provisional value of
-        // the cycle head (either initial value, or from a later iteration) and should be
-        // returned to caller to allow fixpoint iteration to proceed. (All cases in the loop
-        // above other than "cycle head is self" are either terminal or set `retry`.)
-        retry
+        return provisional_retry_cold(
+            db.as_dyn_database(),
+            zalsa,
+            database_key_index,
+            &self.revisions.cycle_heads,
+        );
+
+        #[inline(never)]
+        fn provisional_retry_cold(
+            db: &dyn crate::Database,
+            zalsa: &Zalsa,
+            database_key_index: DatabaseKeyIndex,
+            cycle_heads: &CycleHeads,
+        ) -> bool {
+            let mut retry = false;
+
+            let db = db.as_dyn_database();
+            let hit_cycle = cycle_heads
+                .into_iter()
+                .filter(|&head| head.database_key_index != database_key_index)
+                .any(|head| {
+                    let head_index = head.database_key_index;
+                    let ingredient = zalsa.lookup_ingredient(head_index.ingredient_index());
+                    if !ingredient.is_provisional_cycle_head(db, head_index.key_index()) {
+                        // This cycle is already finalized, so we don't need to wait on it;
+                        // keep looping through cycle heads.
+                        retry = true;
+                        false
+                    } else if ingredient.wait_for(db, head_index.key_index()) {
+                        // There's a new memo available for the cycle head; fetch our own
+                        // updated memo and see if it's still provisional or if the cycle
+                        // has resolved.
+                        retry = true;
+                        false
+                    } else {
+                        // We hit a cycle blocking on the cycle head; this means it's in
+                        // our own active query stack and we are responsible to resolve the
+                        // cycle, so go ahead and return the provisional memo.
+                        true
+                    }
+                });
+            // If `retry` is `true`, all our cycle heads (barring ourself) are complete; re-fetch
+            // and we should get a non-provisional memo. If we get here and `retry` is still
+            // `false`, we have no cycle heads other than ourself, so we are a provisional value of
+            // the cycle head (either initial value, or from a later iteration) and should be
+            // returned to caller to allow fixpoint iteration to proceed. (All cases in the loop
+            // above other than "cycle head is self" are either terminal or set `retry`.)
+            if hit_cycle {
+                false
+            } else {
+                retry
+            }
+        }
     }
 
     /// Cycle heads that should be propagated to dependent queries.
+    #[inline(always)]
     pub(super) fn cycle_heads(&self) -> &CycleHeads {
         if self.may_be_provisional() {
             &self.revisions.cycle_heads

src/input.rs

@@ -272,17 +272,17 @@ impl<C> Slot for Value<C>
 where
     C: Configuration,
 {
-    #[inline]
+    #[inline(always)]
     unsafe fn memos(&self, _current_revision: Revision) -> &crate::table::memo::MemoTable {
         &self.memos
     }
 
-    #[inline]
+    #[inline(always)]
     fn memos_mut(&mut self) -> &mut crate::table::memo::MemoTable {
         &mut self.memos
     }
 
-    #[inline]
+    #[inline(always)]
     unsafe fn syncs(&self, _current_revision: Revision) -> &SyncTable {
         &self.syncs
     }