From 46d9f60b50fb4293ae9a0f0ab51a997ad7fb928d Mon Sep 17 00:00:00 2001 From: Gabe Stocco <98900+gfs@users.noreply.github.com> Date: Wed, 11 Mar 2020 15:30:03 -0700 Subject: [PATCH] Allow journal mode configuration Fix pragmas to apply everytime a connection is opened per https://github.com/dotnet/efcore/issues/5024 --- AsaBenchmarks/InsertTestsWithoutTransactions.cs | 17 +++++++++++++---- AsaBenchmarks/Program.cs | 2 +- Lib/Objects/SqlConnectionHolder.cs | 14 +++++++++++++- Lib/Utils/DatabaseManager.cs | 17 ++++++++--------- 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/AsaBenchmarks/InsertTestsWithoutTransactions.cs b/AsaBenchmarks/InsertTestsWithoutTransactions.cs index 6d50ca057..78335c597 100644 --- a/AsaBenchmarks/InsertTestsWithoutTransactions.cs +++ b/AsaBenchmarks/InsertTestsWithoutTransactions.cs @@ -1,6 +1,7 @@ using AttackSurfaceAnalyzer.Objects; using AttackSurfaceAnalyzer.Utils; using BenchmarkDotNet.Attributes; +using System; using System.Collections.Concurrent; using System.Threading; using System.Threading.Tasks; @@ -13,7 +14,7 @@ public class InsertTestsWithoutTransactions { // The number of records to insert for the benchmark //[Params(25000,50000,100000)] - [Params(75000)] + [Params(10000)] public int N { get; set; } // The number of records to populate the database with before the benchmark @@ -31,6 +32,9 @@ public class InsertTestsWithoutTransactions [Params(1,2,3,4,5,6,7,8,9,10,11,12)] public int Shards { get; set; } + [Params("OFF","DELETE","WAL","MEMORY")] + public string JournalMode { get; set; } + // Bag of reusable objects to write to the database. private static readonly ConcurrentBag BagOfObjects = new ConcurrentBag(); @@ -80,7 +84,7 @@ public static FileSystemObject GetRandomObject(int ObjectPadding = 0) public void PopulateDatabases() { - DatabaseManager.Setup(filename: $"AsaBenchmark_{Shards}.sqlite", shardingFactor: Shards); + Setup(); DatabaseManager.BeginTransaction(); Insert_X_Objects(StartingSize,ObjectPadding,"PopulateDatabase"); @@ -98,17 +102,22 @@ public void GlobalSetup() [GlobalCleanup] public void GlobalCleanup() { - DatabaseManager.Setup(filename: $"AsaBenchmark_{Shards}.sqlite", shardingFactor: Shards); + Setup(); DatabaseManager.Destroy(); } [IterationSetup] public void IterationSetup() { - DatabaseManager.Setup(filename: $"AsaBenchmark_{Shards}.sqlite", shardingFactor: Shards); + Setup(); DatabaseManager.BeginTransaction(); } + private void Setup() + { + DatabaseManager.Setup(filename: $"AsaBenchmark_{Shards}.sqlite", shardingFactor: Shards, JournalMode: JournalMode); + } + [IterationCleanup] public void IterationCleanup() { diff --git a/AsaBenchmarks/Program.cs b/AsaBenchmarks/Program.cs index f8dac4546..7d84f084f 100644 --- a/AsaBenchmarks/Program.cs +++ b/AsaBenchmarks/Program.cs @@ -6,7 +6,7 @@ public class Program { public static void Main(string[] args) { - var summary = BenchmarkRunner.Run(); + var summary = BenchmarkRunner.Run(); } } } diff --git a/Lib/Objects/SqlConnectionHolder.cs b/Lib/Objects/SqlConnectionHolder.cs index 51deddbc7..2d2410b65 100644 --- a/Lib/Objects/SqlConnectionHolder.cs +++ b/Lib/Objects/SqlConnectionHolder.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Serilog; +using System.Globalization; namespace AttackSurfaceAnalyzer.Objects { @@ -18,10 +19,21 @@ public class SqlConnectionHolder private int RecordCount { get; set; } public int FlushCount { get; set; } = -1; - public SqlConnectionHolder(string databaseFilename, int flushCount = -1) + private const string PRAGMAS = "PRAGMA auto_vacuum = 0; PRAGMA synchronous = OFF"; + private const string JOURNAL_MODE = "PRAGMA journal_mode = {0};"; + + public SqlConnectionHolder(string databaseFilename, int flushCount = -1, string journalMode = "OFF") { Source = databaseFilename; Connection = new SqliteConnection($"Data source={Source}"); + Connection.Open(); + + using var cmd = new SqliteCommand(PRAGMAS, Connection); + cmd.ExecuteNonQuery(); + + cmd.CommandText = string.Format(CultureInfo.InvariantCulture,JOURNAL_MODE, journalMode); + cmd.ExecuteNonQuery(); + StartWriter(); FlushCount = flushCount; } diff --git a/Lib/Utils/DatabaseManager.cs b/Lib/Utils/DatabaseManager.cs index b6c3aa365..74e75635b 100644 --- a/Lib/Utils/DatabaseManager.cs +++ b/Lib/Utils/DatabaseManager.cs @@ -69,8 +69,6 @@ public static class DatabaseManager private const string SQL_INSERT = "insert into file_system_monitored (run_id, row_key, timestamp, change_type, path, old_path, name, old_name, extended_results, notify_filters, serialized) values (@run_id, @row_key, @timestamp, @change_type, @path, @old_path, @name, @old_name, @extended_results, @notify_filters, @serialized)"; - private const string PRAGMAS = "PRAGMA main.auto_vacuum = 0; PRAGMA main.synchronous = OFF; PRAGMA main.journal_mode = OFF;"; - private const string INSERT_RUN_INTO_RESULT_TABLE_SQL = "insert into results (base_run_id, compare_run_id, status) values (@base_run_id, @compare_run_id, @status);"; private const string UPDATE_RUN_IN_RESULT_TABLE = "update results set status = @status where (base_run_id = @base_run_id and compare_run_id = @compare_run_id)"; @@ -99,6 +97,8 @@ public static class DatabaseManager private static int SHARDING_FACTOR = 1; private static int FLUSH_COUNT = -1; + private static string JOURNAL_MODE; + public static SqlConnectionHolder MainConnection { @@ -116,9 +116,12 @@ public static SqlConnectionHolder MainConnection public static bool FirstRun { get; private set; } = true; - public static bool Setup(string filename = null, int shardingFactor = 1, int flushCount = -1) + public static bool Setup(string filename = null, int shardingFactor = 1, int flushCount = -1, string JournalMode = "OFF") { JsonSerializer.SetDefaultResolver(StandardResolver.ExcludeNull); + + JOURNAL_MODE = JournalMode; + if (filename != null) { if (SqliteFilename != filename) @@ -170,9 +173,6 @@ public static bool Setup(string filename = null, int shardingFactor = 1, int flu { try { - using var cmd = new SqliteCommand(PRAGMAS, MainConnection.Connection); - cmd.ExecuteNonQuery(); - BeginTransaction(); using var cmd2 = new SqliteCommand(SQL_CREATE_RUNS, MainConnection.Connection, MainConnection.Transaction); @@ -282,7 +282,6 @@ public static int PopulateConnections() for (int i = Connections.Count; i < SHARDING_FACTOR; i++) { Connections.Add(GenerateSqlConnection(i)); - Connections[i].Connection.Open(); connectionsCreated++; } return connectionsCreated; @@ -292,11 +291,11 @@ private static SqlConnectionHolder GenerateSqlConnection(int i) { if (i == 0) { - return new SqlConnectionHolder(SqliteFilename, flushCount:FLUSH_COUNT); + return new SqlConnectionHolder(SqliteFilename, flushCount:FLUSH_COUNT, journalMode:JOURNAL_MODE); } else { - return new SqlConnectionHolder($"{SqliteFilename}_{i}", flushCount: FLUSH_COUNT); + return new SqlConnectionHolder($"{SqliteFilename}_{i}", flushCount: FLUSH_COUNT, journalMode: JOURNAL_MODE); } }