Add StringDictionary::buildUnionTranslationMap.

Signed-off-by: ienkovich <[email protected]>

Author: ienkovich

omniscidb/ResultSet/RowSetMemoryOwner.h

@@ -218,8 +218,7 @@ class RowSetMemoryOwner final : public SimpleAllocator, boost::noncopyable {
     auto it = str_proxy_union_translation_maps_owned_.find(map_key);
     if (it == str_proxy_union_translation_maps_owned_.end()) {
       it = str_proxy_union_translation_maps_owned_
-               .emplace(map_key,
-                        source_proxy->buildUnionTranslationMapToOtherProxy(dest_proxy))
+               .emplace(map_key, source_proxy->buildUnionTranslationMap(dest_proxy))
                .first;
     }
     return &it->second;

omniscidb/StringDictionary/StringDictionary.cpp

@@ -1438,6 +1438,37 @@ std::vector<int32_t> StringDictionary::buildIntersectionTranslationMap(
   return translated_ids;
 }
 
+std::vector<int32_t> StringDictionary::buildUnionTranslationMap(
+    StringDictionary* dest) const {
+  auto dummy_callback = [](const std::string_view& source_string,
+                           const int32_t source_string_id) { return true; };
+  const size_t num_source_strings = entryCount();
+  const size_t num_dest_strings = dest->entryCount();
+  std::vector<int32_t> translated_ids(num_source_strings);
+  auto num_untranslated_strings =
+      StringDictionaryTranslator::buildDictionaryTranslationMap(this,
+                                                                dest,
+                                                                translated_ids.data(),
+                                                                num_source_strings,
+                                                                num_dest_strings,
+                                                                false,
+                                                                dummy_callback);
+  if (num_untranslated_strings > 0) {
+    // Todo (todd): Add call to fetch string_views (local) or strings (distributed)
+    // for all non-translated ids to avoid string-by-string fetch
+    for (int32_t source_string_id = 0;
+         source_string_id < static_cast<int32_t>(translated_ids.size());
+         ++source_string_id) {
+      if (translated_ids[source_string_id] == StringDictionary::INVALID_STR_ID) {
+        const auto source_string = getStringUnlocked(source_string_id);
+        const auto dest_string_id = dest->getOrAdd(source_string);
+        translated_ids[source_string_id] = dest_string_id;
+      }
+    }
+  }
+  return translated_ids;
+}
+
 }  // namespace legacy
 
 std::vector<int32_t> StringDictionaryTranslator::buildDictionaryTranslationMap(
@@ -1503,8 +1534,9 @@ size_t StringDictionaryTranslator::buildDictionaryTranslationMap(
     return num_source_strings;
   }
 
+  size_t base_num_strings_not_translated = 0;
   if (source_dict->base_dict_) {
-    buildDictionaryTranslationMap(
+    base_num_strings_not_translated = buildDictionaryTranslationMap(
         source_dict->base_dict_.get(),
         dest_dict,
         translated_ids,
@@ -1526,7 +1558,7 @@ size_t StringDictionaryTranslator::buildDictionaryTranslationMap(
       tbb::blocked_range<int32_t>(source_dict->base_generation_,
                                   num_source_strings,
                                   target_strings_per_task /* tbb grain_size */),
-      (size_t)0,
+      base_num_strings_not_translated,
       [&](const tbb::blocked_range<int32_t>& r, size_t num_strings_not_translated) {
         const int32_t start_idx = r.begin();
         const int32_t end_idx = r.end();

omniscidb/StringDictionary/StringDictionary.h

@@ -140,6 +140,7 @@ class StringDictionary {
 
   std::vector<int32_t> buildIntersectionTranslationMap(
       const StringDictionary* dest) const;
+  std::vector<int32_t> buildUnionTranslationMap(StringDictionary* dest) const;
 
   static constexpr int32_t INVALID_STR_ID = -1;
   static constexpr size_t MAX_STRLEN = (1 << 15) - 1;

omniscidb/StringDictionary/StringDictionaryProxy.cpp

@@ -292,7 +292,7 @@ std::vector<int32_t> StringDictionaryProxy::buildIntersectionTranslationMap(
                                                              num_strings_not_translated);
 }
 
-std::vector<int32_t> StringDictionaryProxy::buildUnionTranslationMapToOtherProxy(
+std::vector<int32_t> StringDictionaryProxy::buildUnionTranslationMap(
     StringDictionaryProxy* dest_proxy) const {
   auto timer = DEBUG_TIMER(__func__);
 

omniscidb/StringDictionary/StringDictionaryProxy.h

@@ -90,8 +90,7 @@ class StringDictionaryProxy {
   std::vector<int32_t> buildIntersectionTranslationMap(
       const StringDictionaryProxy* dest_proxy) const;
 
-  std::vector<int32_t> buildUnionTranslationMapToOtherProxy(
-      StringDictionaryProxy* dest_proxy) const;
+  std::vector<int32_t> buildUnionTranslationMap(StringDictionaryProxy* dest_proxy) const;
 
   /**
    * @brief Returns the number of transient string entries for this proxy,

omniscidb/Tests/StringDictionaryBenchmark.cpp

@@ -385,9 +385,7 @@ BENCHMARK_DEFINE_F(
       2, true, append_strings_10M_10M_10_randomized_truncated_8M);
   dest_proxy->getOrAddBulk(append_strings_10M_10M_10_randomized_truncated_100);
   for (auto _ : state) {
-    auto id_map = source_proxy->buildUnionTranslationMapToOtherProxy(dest_proxy.get());
-    const size_t num_expected_untranslated_strings =
-        num_elems - first_n_elems - last_n_elems;
+    auto id_map = source_proxy->buildUnionTranslationMap(dest_proxy.get());
   }
 }
 

omniscidb/Tests/StringDictionaryTest.cpp

@@ -1100,7 +1100,7 @@ TEST(NestedStringDictionary, BuildIntersectionTranslationMap) {
   }
 }
 
-TEST(StringDictionaryProxy, BuildUnionTranslationMapToEmptyProxy) {
+TEST(NestedStringDictionary, BuildUnionTranslationMap_Empty) {
   // Todo(todd): Migrate this and intersection translation tests to use
   // approach and methods in BuildUnionTranslationMapToPartialOverlapProxy
   const DictRef dict_ref1(-1, 1);
@@ -1123,15 +1123,14 @@ TEST(StringDictionaryProxy, BuildUnionTranslationMapToEmptyProxy) {
     // First try to union translate to empty dictionary.
     // All strings should end up as transient entries in
     // destination proxy
-    std::shared_ptr<StringDictionaryProxy> source_string_dict_proxy =
-        std::make_shared<StringDictionaryProxy>(source_string_dict,
-                                                source_string_dict->storageEntryCount());
-    std::shared_ptr<StringDictionaryProxy> dest_string_dict_proxy =
-        std::make_shared<StringDictionaryProxy>(dest_string_dict,
-                                                dest_string_dict->storageEntryCount());
+    std::shared_ptr<StringDictionary> source_string_dict_proxy =
+        std::make_shared<StringDictionary>(source_string_dict,
+                                           source_string_dict->storageEntryCount());
+    std::shared_ptr<StringDictionary> dest_string_dict_proxy =
+        std::make_shared<StringDictionary>(dest_string_dict,
+                                           dest_string_dict->storageEntryCount());
     const auto str_proxy_translation_map =
-        source_string_dict_proxy->buildUnionTranslationMapToOtherProxy(
-            dest_string_dict_proxy.get());
+        source_string_dict_proxy->buildUnionTranslationMap(dest_string_dict_proxy.get());
     ASSERT_FALSE(str_proxy_translation_map.empty());
     const size_t num_ids = str_proxy_translation_map.size();
     ASSERT_EQ(num_ids, source_string_dict_proxy->entryCount());
@@ -1161,7 +1160,7 @@ std::vector<std::string> add_strings_numeric_range(std::shared_ptr<StringDiction
   return strings;
 }
 
-std::vector<std::string> add_strings_numeric_range(StringDictionaryProxy& sdp,
+std::vector<std::string> add_strings_numeric_range(StringDictionary& sdp,
                                                    const size_t num_vals,
                                                    const int32_t start_val) {
   CHECK_GE(start_val, sdp.getBaseGeneration());
@@ -1174,8 +1173,8 @@ std::vector<std::string> add_strings_numeric_range(StringDictionaryProxy& sdp,
   return strings;
 }
 
-void verify_translation(const StringDictionaryProxy& source_proxy,
-                        const StringDictionaryProxy& dest_proxy,
+void verify_translation(const StringDictionary& source_proxy,
+                        const StringDictionary& dest_proxy,
                         const std::vector<int32_t>& id_map,
                         const std::vector<std::string>& persisted_source_strings,
                         const std::vector<std::string>& transient_source_strings,
@@ -1211,7 +1210,7 @@ void verify_translation(const StringDictionaryProxy& source_proxy,
   }
 }
 
-TEST(StringDictionaryProxy, BuildUnionTranslationMapToPartialOverlapProxy) {
+TEST(NestedStringDictionary, BuildUnionTranslationMap_PartialOverlap) {
   const DictRef dict_ref1(-1, 1);
   const DictRef dict_ref2(-1, 2);
   std::shared_ptr<StringDictionary> source_sd =
@@ -1236,23 +1235,25 @@ TEST(StringDictionaryProxy, BuildUnionTranslationMapToPartialOverlapProxy) {
       dest_sd, num_dest_persisted_entries, dest_persisted_start_val);
   ASSERT_EQ(dest_sd->storageEntryCount(), num_dest_persisted_entries);
 
-  StringDictionaryProxy source_sdp(source_sd, source_sd->storageEntryCount());
-  StringDictionaryProxy dest_sdp(dest_sd, dest_sd->storageEntryCount());
+  StringDictionary source_sdp(source_sd, source_sd->storageEntryCount());
+  StringDictionary dest_sdp(dest_sd, dest_sd->storageEntryCount());
   const auto transient_source_strings = add_strings_numeric_range(
       source_sdp, num_source_transient_entries, source_transient_start_val);
   ASSERT_EQ(source_sdp.getBaseDictionary()->getDictId(), 1);
   ASSERT_EQ(source_sdp.getBaseDictionary()->storageEntryCount(),
             num_source_persisted_entries);
-  ASSERT_EQ(source_sdp.transientEntryCount(), num_source_transient_entries);
+  ASSERT_EQ(source_sdp.entryCount() - source_sdp.getBaseGeneration(),
+            num_source_transient_entries);
 
   const auto transient_dest_strings = add_strings_numeric_range(
       dest_sdp, num_dest_transient_entries, dest_transient_start_val);
   ASSERT_EQ(dest_sdp.getBaseDictionary()->getDictId(), 2);
   ASSERT_EQ(dest_sdp.getBaseDictionary()->storageEntryCount(),
             num_dest_persisted_entries);
-  ASSERT_EQ(dest_sdp.transientEntryCount(), num_dest_transient_entries);
+  ASSERT_EQ(dest_sdp.entryCount() - dest_sdp.getBaseGeneration(),
+            num_dest_transient_entries);
 
-  const auto id_map = source_sdp.buildUnionTranslationMapToOtherProxy(&dest_sdp);
+  const auto id_map = source_sdp.buildUnionTranslationMap(&dest_sdp);
   ASSERT_EQ(id_map.size(), num_source_persisted_entries + num_source_transient_entries);
 
   verify_translation(source_sdp,