Improved unique values filter to handle large numbers of values

YuriyIvon · Feb 11, 2024 · c89ad8b · c89ad8b
1 parent 5ccc2fa
commit c89ad8b
Show file tree

Hide file tree

Showing 6 changed files with 20 additions and 12 deletions.
diff --git a/src/DatabaseBenchmark/DatabaseBenchmark.csproj b/src/DatabaseBenchmark/DatabaseBenchmark.csproj
@@ -19,6 +19,7 @@
 
   <ItemGroup>
     <PackageReference Include="AWSSDK.DynamoDBv2" Version="3.7.301.9" />
+    <PackageReference Include="BloomFilter.NetCore" Version="2.2.0" />
     <PackageReference Include="Bogus" Version="35.4.0" />
     <PackageReference Include="CsvHelper" Version="31.0.0" />
     <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.38.1" />

diff --git a/src/DatabaseBenchmark/Generators/Options/FloatGeneratorOptions.cs b/src/DatabaseBenchmark/Generators/Options/FloatGeneratorOptions.cs
@@ -13,8 +13,5 @@ public class FloatGeneratorOptions : GeneratorOptionsBase
         public double Delta { get; set; } = 0;
 
         public bool RandomizeDelta { get; set; } = false;
-
-        //TODO: implement logic
-        public bool Unique { get; set; } = false;
     }
 }
diff --git a/src/DatabaseBenchmark/Generators/Options/IntegerGeneratorOptions.cs b/src/DatabaseBenchmark/Generators/Options/IntegerGeneratorOptions.cs
@@ -13,8 +13,5 @@ public class IntegerGeneratorOptions : GeneratorOptionsBase
         public int Delta { get; set; } = 0;
 
         public bool RandomizeDelta { get; set; } = false;
-
-        //TODO: implement logic
-        public bool Unique { get; set; } = false;
     }
 }
diff --git a/src/DatabaseBenchmark/Generators/Options/StringGeneratorOptions.cs b/src/DatabaseBenchmark/Generators/Options/StringGeneratorOptions.cs
@@ -9,8 +9,5 @@ public class StringGeneratorOptions : GeneratorOptionsBase
         public int MaxLength { get; set; } = 10;
 
         public string AllowedCharacters { get; set; } = "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
-
-        //TODO: implement logic
-        public bool Unique { get; set; } = false;
     }
 }
diff --git a/src/DatabaseBenchmark/Generators/Options/UniqueGeneratorOptions.cs b/src/DatabaseBenchmark/Generators/Options/UniqueGeneratorOptions.cs
@@ -9,5 +9,7 @@ public class UniqueGeneratorOptions : GeneratorOptionsBase
         public int AttemptCount { get; set; } = 100;
 
         public IGeneratorOptions SourceGeneratorOptions { get; set; }
+
+        public int MaxValues { get; set; } = 1000000;
     }
 }
diff --git a/src/DatabaseBenchmark/Generators/UniqueGenerator.cs b/src/DatabaseBenchmark/Generators/UniqueGenerator.cs
@@ -1,4 +1,5 @@
-using DatabaseBenchmark.Generators.Interfaces;
+using BloomFilter;
+using DatabaseBenchmark.Generators.Interfaces;
 using DatabaseBenchmark.Generators.Options;
 
 namespace DatabaseBenchmark.Generators
@@ -7,14 +8,15 @@ public class UniqueGenerator : IGenerator, IDisposable
     {
         private readonly UniqueGeneratorOptions _options;
         private readonly IGenerator _sourceGenerator;
-        private readonly HashSet<object> _existingValues = [];
+        private readonly IBloomFilter _bloomFilter;
 
         public object Current { get; private set; }
 
         public UniqueGenerator(UniqueGeneratorOptions options, IGenerator sourceGenerator) 
         {
             _options = options;
             _sourceGenerator = sourceGenerator;
+            _bloomFilter = FilterBuilder.Build(options.MaxValues, 0.01);
         }
 
         public bool Next()
@@ -23,7 +25,7 @@ public bool Next()
             {
                 _sourceGenerator.Next();
 
-                if (_existingValues.Add(_sourceGenerator.Current))
+                if (AppendFilter(_sourceGenerator.Current))
                 {
                     Current = _sourceGenerator.Current;
                     return true;
@@ -39,6 +41,18 @@ public void Dispose()
             {
                 disposable.Dispose();
             }
+
+            _bloomFilter.Dispose();
         }
+
+        public bool AppendFilter(object value) =>
+            value switch
+            {
+                int intValue => _bloomFilter.Add(intValue),
+                long longValue => _bloomFilter.Add(longValue),
+                double doubleValue => _bloomFilter.Add(doubleValue),
+                DateTime dateTimeValue => _bloomFilter.Add(dateTimeValue),
+                _ => _bloomFilter.Add(value.ToString())
+            };
     }
 }