Fix cuckoo filter overflow issue

Author: kylebd99

src/CardinalityWithColors.jl

@@ -1,3 +1,4 @@
+using Probably
 using Probably: BloomFilter, constrain
 using DataStructures: counter, Dict, Set, Vector, inc!, Queue
 using AutoHashEquals

src/ColorSummary.jl

@@ -1,6 +1,4 @@
 using Graphs
-using Probably
-
 """
 The ColorSummary struct holds statistical information associated with the colored graph.
 It keeps detailed information about the number of edges between colors of a particular color and which land in
@@ -14,6 +12,7 @@ mutable struct ColorSummary{DS}
     color_label_cardinality::Dict{Color, Dict{Int, Int}} # color_label_cardinality[c][v] = num_vertices
     edge_deg::Dict{Int, Dict{Int, Dict{Color, Dict{Color, DS}}}} # edge_deg[e][v2][c1][c2] = degreestat
     color_filters::Dict{Color, SmallCuckoo} # color_filters[c] = filter
+    color_full::Set{Color} # Denotes if a color's filter is full
     cycle_probabilities::Dict{CyclePathAndColors, Float64} # cycle_probabilities[[c1, c2], path] = likelihood
     cycle_length_probabilities::Dict{Int, Float64} #cycle_probabilities[path_length] = likelihood
     max_cycle_size::Int
@@ -176,7 +175,7 @@ function generate_color_summary(g::DataGraph, params::ColorSummaryParams=ColorSu
     edge_stats_time = time() - edge_stats_time
     push!(timing_vec, edge_stats_time)
 
-    return ColorSummary{DS}(color_label_cardinality, edge_deg, color_filters,
+    return ColorSummary{DS}(color_label_cardinality, edge_deg, color_filters, Set{Color}(),
                 cycle_probabilities, cycle_length_probabilities, params.max_cycle_size,
                  ne(g.graph), nv(g.graph), num_colors, 0)
 end

src/QuasiStableCardinalityEstimator.jl

@@ -53,7 +53,7 @@ to the magnitude of the original weight of the output paths.
 - partial_paths::Matrix{Color} - the matrix of partial paths describing the current traversal over the lifted color graph.
 - partial_weights::Vector{W} - the list of weights for each partial path.
 - num_samples::Int - the number of paths to sample, determining the size of the output.
-- sampling_strategy::SAMPLING_STRATEGY - how to select samples from the partial paths. Will only check if this is set to "uniform" to sample uniformly. 
+- sampling_strategy::SAMPLING_STRATEGY - how to select samples from the partial paths. Will only check if this is set to "uniform" to sample uniformly.
                                          Otherwise, just selects samples by prioritizing those with higher weights.
 """
 function sample_paths(partial_paths::Matrix{Color}, partial_weights::Vector{W}, num_samples::Int, sampling_strategy::SAMPLING_STRATEGY) where W
@@ -403,7 +403,7 @@ function get_cardinality_bounds(query::QueryGraph, summary::ColorSummary{DS}; ma
                     data_label_is_in_color = true
                     continue
                 end
-                if data_label in summary.color_filters[color]
+                if data_label in summary.color_filters[color] || color in summary.color_full
                     data_label_is_in_color = true
                 end
             end
@@ -489,7 +489,7 @@ function get_cardinality_bounds(query::QueryGraph, summary::ColorSummary{DS}; ma
                             data_label_in_color = true
                             continue
                         end
-                        if data_label in summary.color_filters[new_color]
+                        if data_label in summary.color_filters[new_color] || new_color in summary.color_full
                             data_label_in_color = true
                         end
                     end

src/UpdateSummary.jl

@@ -42,21 +42,26 @@ function add_summary_node!(summary::ColorSummary{AvgDegStats}, node_labels, node
     color = choose_color(summary)
     # add to the bloom filter
     push!(summary.color_filters[color], data_label)
+    if Probably.loadfactor(summary.color_filters[color]) > .95
+        push!(summary.color_full, color)
+    end
     # for edge degrees, it decreases the average.
-    # we want to update all the avg out degrees where this is the landing node (c2 == color && v2 == label),
-    # and update all the avg in degrees where this is the starting node (c2 == color && v2 == label)
+    # we want to update all the avg out/in degrees where this is the starting node (c1 == color && v1 == label),
     for edge_label in keys(summary.edge_deg)
         for node_label in node_labels
             if !haskey(summary.edge_deg[edge_label], node_label)
                 summary.edge_deg[edge_label][node_label] = Dict()
             end
-            for other_color in keys(summary.edge_deg[edge_label][node_label])
-                current_ds = get(summary.edge_deg[edge_label][node_label][other_color], color, AvgDegStats(0, 0))
+            if !haskey(summary.edge_deg[edge_label][node_label], color)
+                summary.edge_deg[edge_label][node_label][color] = Dict()
+            end
+            for other_color in keys(summary.edge_deg[edge_label][node_label][color])
+                current_ds = get(summary.edge_deg[edge_label][node_label][color], other_color, AvgDegStats(0, 0))
                 current_cardinality = get(summary.color_label_cardinality[color], node_label, 0)
                 avg_in = current_ds.avg_in * (current_cardinality / (current_cardinality + 1))
                 avg_out = current_ds.avg_out * (current_cardinality / (current_cardinality + 1))
                 new_ds = AvgDegStats(avg_in, avg_out)
-                summary.edge_deg[edge_label][node_label][other_color][color] = new_ds
+                summary.edge_deg[edge_label][node_label][color][other_color] = new_ds
             end
         end
     end
@@ -79,17 +84,35 @@ Finds and returns the color of the node in the lifted graph summary.
 - node - the vertex ID of the node.
 """
 function get_node_summary_color(summary::ColorSummary, node)
-    possible_colors = []
+    possible_colors = collect(summary.color_full)
     for color in keys(summary.color_filters)
         filter = summary.color_filters[color]
         # in the data graph, the node's data label is just its id - 1
         if (node - 1) in filter
             push!(possible_colors, color)
         end
     end
+
     # Since Cuckoo filters are used, there is a chance that there will be false positive results.
-    # In that case, randomly select one of the colors that indicates it contains this node.
-    return length(possible_colors) == 0 ? rand(keys(summary.color_filters)) : rand(possible_colors)
+    # In that case, we select the largest color.
+    true_color = 0
+    if length(possible_colors) == 0
+        println("HERE")
+        true_color = get_largest_color(summary)
+    elseif length(possible_colors) == 1
+        true_color = possible_colors[1]
+    elseif length(possible_colors) > 1
+        min_size = Inf
+        for color in possible_colors
+            color_size = summary.color_label_cardinality[color][-1]
+            if color_size < min_size
+                true_color = color
+                min_size = color_size
+            end
+        end
+    end
+
+    return true_color
 end
 
 """
@@ -159,4 +182,4 @@ function add_summary_edge!(summary::ColorSummary{AvgDegStats}, start_node, end_n
         end
     end
     summary.total_added_edges += 1
-end
+end