Merge pull request diffblue#418 from diffblue/unload_lvsa_summaries_on_demand

SEC-395: Unload lvsa summaries on demand

Author: marek-trtik

src/taint-analysis/taint_summary.cpp

@@ -1726,12 +1726,61 @@ void taint_summarise_all_functions(
 {
   size_t processed = 0UL;
   size_t modelled = 0UL;
-  size_t skipped = 0UL;
+  std::unordered_set<irep_idt> skipped;
 
   const std::size_t topological_order_size =
     program->get_inverted_functions_topological_order().size();
 
   auto start_time=std::chrono::high_resolution_clock::now();
+
+  // In the map 'lvsadb_flush_ref_counts' there we store for each function
+  // in the inverted topological order two information:
+  //    - reference count (how many callers the function has)
+  //    - names of the callees
+  // where both information are considered w.r.t. the inverted topological
+  // order. Namely, considered are only recursive calls to predecessors in the
+  // topological order.
+  // We use the map for unloading not needed LVSA summaries from the memory
+  // (but they are saved on the disk). An LVSA summary of a given function
+  // is not needed at a given position in the inverted topological order of
+  // functions, if none of the remaining function (i.e. those after the position
+  // in the order) is a caller of the given function w.r.t the inverted
+  // topological order.
+  typedef std::pair<std::size_t, std::unordered_set<irep_idt>>
+          refcount_and_calleest;
+  std::unordered_map<irep_idt, refcount_and_calleest> lvsadb_flush_ref_counts;
+  for(const auto &fn_name :
+    program->get_inverted_functions_topological_order())
+  {
+    const auto self_it =
+      lvsadb_flush_ref_counts.insert({fn_name, {0UL, {}}}).first;
+    const auto node =
+      *program->get_directed_call_graph().get_node_index(fn_name);
+    for(const auto &succ_node :
+      program->get_directed_call_graph().get_successors(node))
+    {
+      const auto &callee_name =
+        program->get_directed_call_graph()[succ_node].function;
+      if(callee_name != fn_name) // Skip direct recursion.
+      {
+        auto it = lvsadb_flush_ref_counts.find(callee_name);
+        if(it != lvsadb_flush_ref_counts.end()) // Skip indirect recursion.
+        {
+          it->second.first += 1UL;
+          self_it->second.second.insert(callee_name);
+        }
+      }
+    }
+  }
+  for(auto &elem : lvsadb_flush_ref_counts)
+  {
+    if(elem.second.first == 0UL) // No caller -> unload yourself
+    {
+      elem.second.first = 1UL;
+      elem.second.second.insert(elem.first);
+    }
+  }
+
   for(const auto &fn_name :
     program->get_inverted_functions_topological_order())
   {
@@ -1743,7 +1792,7 @@ void taint_summarise_all_functions(
         << "["
         << std::fixed << std::setprecision(1) << std::setw(5)
         << (topological_order_size==0UL?100.0:
-              100.0*(double)(processed+skipped)/
+              100.0*(double)(processed+skipped.size())/
                     (double)topological_order_size)
         << "%] "
         << " TIMEOUT! ("
@@ -1765,7 +1814,7 @@ void taint_summarise_all_functions(
         << "["
         << std::fixed << std::setprecision(1) << std::setw(5)
         << (topological_order_size==0UL?100.0:
-              100.0*(double)(processed+skipped)/
+              100.0*(double)(processed+skipped.size())/
                     (double)topological_order_size)
         << "%] "
         << fn_name
@@ -1790,22 +1839,39 @@ void taint_summarise_all_functions(
         << "["
         << std::fixed << std::setprecision(1) << std::setw(5)
         << (topological_order_size==0UL?100.0:
-              100.0*(double)(processed+skipped)/
+              100.0*(double)(processed+skipped.size())/
                     (double)topological_order_size)
         << "%] Skipping"
         << (fn_it!=functions_map.cend() && !fn_it->second.body_available()
           ? " [function without a body]" : "")
         << ": "
         << fn_name
         << messaget::eom;
-      ++skipped;
+      skipped.insert(fn_name);
+    }
+
+    for(auto &succ : lvsadb_flush_ref_counts.at(fn_name).second)
+    {
+      auto &elem = lvsadb_flush_ref_counts.at(succ);
+      if(elem.first > 0UL)
+      {
+        --elem.first;
+        if(elem.first == 0UL)
+        {
+          const bool unloaded = lvsa_db.unload(succ);
+          INVARIANT(unloaded || skipped.count(succ) != 0UL,
+                    "Either the LVSA summary was unloaded or there was "
+                    "nothing to unload (the summary computation was "
+                    "skipped).");
+        }
+      }
     }
   }
   log->status()
     << "[100.0%] Summarisation of functions has finished successfully ("
     << processed << " processed, "
     << modelled << " modelled, "
-    << skipped << " skipped)."
+    << skipped.size() << " skipped)."
     << messaget::eom;
 }
 

src/util/cached_map.h

@@ -509,6 +509,35 @@ class cached_mapt final
     base_map.flush();
   }
 
+  /*******************************************************************\
+
+  Function: cached_mapt::unload(const key_type &)
+
+  Inputs:
+    key: A key whose corresponding value should be unloaded.
+
+  Outputs:
+   True, if the referenced element was unloaded, and false otherwise.
+
+  Purpose:
+    Unloads the element referenced by the passed key in the cache to allow the
+    backing store to be externally manipulated.
+
+  Remarks:
+    This function will not unload a locked element.
+
+  \*******************************************************************/
+  bool unload(const key_type &key)
+  {
+    if(std::find(unloadable.cbegin(), unloadable.cend(), key)
+       == unloadable.cend())
+    {
+      return false;
+    }
+    base_map.set(key, remove_cache(key));
+    return true;
+  }
+
 private:
   std::shared_ptr<mapped_type> get(
     const key_type &key, std::function<mapped_type()> load_value) const

unit/serialization/cached_map.cpp

@@ -122,6 +122,17 @@ void test_cached_map()
   test_assert(map.cache_size() == 4, "The unloaded element can be reloaded");
   test_assert(map.size() == 4, "Map size is correct after element is reloaded");
 
-  // Now f4, f3 and f1 will be unlocked as they go out of scope
-  // Now f4, f3 and f1 will be unloaded as map goes out of scope
+  test_assert(map.locked_count() == 4,
+              "All 4 elements must be locked before flush");
+  test_assert(!map.unload(get_F_key(f2)),
+              "Flush must fail as the element 7 is locked");
+  // Now we unlock elements 1 and 7 respectively.
+  f41 = nullptr;
+  f22 = nullptr;
+  test_assert(map.locked_count() == 2,
+              "After unlocking 1 and 7 there must remain only 2 locked");
+  test_assert(map.unload(get_F_key(f2)),
+              "Flush of 7 must succeed as it was unlocked");
+
+  // Now 5, 9 and will be unloaded as map goes out of scope
 }