diff --git a/src/Marten.ScaleTesting/Commands/DaemonLoadCommand.cs b/src/Marten.ScaleTesting/Commands/DaemonLoadCommand.cs new file mode 100644 index 0000000000..5edc8b92d0 --- /dev/null +++ b/src/Marten.ScaleTesting/Commands/DaemonLoadCommand.cs @@ -0,0 +1,421 @@ +using System.Diagnostics; +using System.Text.Json; +using JasperFx; +using JasperFx.CommandLine; +using JasperFx.Events; +using JasperFx.Events.Projections; +using Marten.Events; +using Marten.Events.Projections; +using Marten.ScaleTesting.Instrumentation; +using Marten.Storage; +using Npgsql; +using Spectre.Console; + +namespace Marten.ScaleTesting.Commands; + +/// +/// daemonload: the jasperfx#486 WS2 measurement scenario. Everything else in this harness +/// measures REBUILDS; this measures the steady-state RUNNING daemon — the deployment shape whose +/// database connection footprint at ~100 tenants is the WS2 concern. +/// +/// Flow: +/// +/// Build an isolated UseTenantPartitionedEvents store in its own schema with +/// --projections async projections and a dedicated Application Name so +/// pg_stat_activity can attribute every connection the store opens +/// Register --tenants tenants and start the projection daemon — +/// StartAllAsync fans out one subscription agent per (projection × tenant) +/// Append events continuously across every tenant for --duration-seconds while +/// sampling the store's connection count every --sample-seconds +/// Stop appending, wait for every tenant's agents to catch up to that tenant's own +/// high-water ceiling, and report peak/mean connections + catch-up coverage +/// +/// +/// The WS2 gate: connections should be O(databases), not O(tenant agents). Run this BEFORE and +/// AFTER the daemon command-batching work lands to quantify the win; --max-connections +/// turns the report into a pass/fail gate for regression runs. +/// +[Description("WS2 (jasperfx#486): run the async daemon over N partitioned tenants under continuous append load and sample pg_stat_activity for the store's connection footprint.")] +public sealed class DaemonLoadCommand: JasperFxAsyncCommand +{ + private const string Schema = "scaletest_daemonload"; + private const string ApplicationName = "scaletest-daemonload"; + + public override async Task Execute(DaemonLoadInput input) + { + var totalElapsed = Stopwatch.StartNew(); + + AnsiConsole.MarkupLine( + $"[blue]daemonload: schema=[yellow]{Schema}[/] tenants=[yellow]{input.TenantsFlag}[/] " + + $"projections=[yellow]{input.ProjectionsFlag}[/] duration=[yellow]{input.DurationSecondsFlag}s[/] " + + $"rate=[yellow]~{input.AppendRatePerSecondFlag}/s[/][/]"); + + if (input.WipeFlag) + { + await WipeSchemaAsync().ConfigureAwait(false); + } + + // Stamp our own Application Name so the sampler counts exactly the store's connections — + // daemon sessions, event loaders, appender sessions — and nothing else on the dev box. + var storeConnectionString = new NpgsqlConnectionStringBuilder(ConnectionSource.ConnectionString) + { + ApplicationName = ApplicationName + }.ConnectionString; + + var projectionNames = Enumerable.Range(0, Math.Max(1, input.ProjectionsFlag)) + .Select(i => $"LoadRollup{i}") + .ToArray(); + + using var store = Marten.DocumentStore.For(opts => + { + opts.Connection(storeConnectionString); + opts.DisableNpgsqlLogging = true; + opts.DatabaseSchemaName = Schema; + opts.Events.TenancyStyle = TenancyStyle.Conjoined; + opts.Events.UseTenantPartitionedEvents = true; + opts.Events.AppendMode = EventAppendMode.QuickWithServerTimestamps; + opts.Policies.AllDocumentsAreMultiTenanted(); + opts.Events.AddEventType(); + + foreach (var name in projectionNames) + { + opts.Projections.Add(new DaemonLoadRollupProjection(name), ProjectionLifecycle.Async, name); + } + }); + + await store.Storage.ApplyAllConfiguredChangesToDatabaseAsync().ConfigureAwait(false); + + var tenants = Enumerable.Range(0, input.TenantsFlag) + .Select(i => $"tenant_{i:0000}") + .ToArray(); + + AnsiConsole.MarkupLine($"[grey]Registering {tenants.Length} tenants (per-tenant partition DDL — this can take a bit)...[/]"); + await store.Advanced.AddMartenManagedTenantsAsync(CancellationToken.None, tenants).ConfigureAwait(false); + + // One seed event per tenant so every per-tenant sequence + partition exists before the + // daemon starts, and every (projection × tenant) agent has something to fan out for. + foreach (var tenant in tenants) + { + await using var session = store.LightweightSession(tenant); + session.Events.StartStream(Guid.NewGuid(), new DaemonLoadEvent(tenant, 0)); + await session.SaveChangesAsync().ConfigureAwait(false); + } + + using var cts = new CancellationTokenSource(); + await using var sampler = ConnectionSampler.Start( + ConnectionSource.ConnectionString, ApplicationName, + TimeSpan.FromSeconds(Math.Max(0.1, input.SampleSecondsFlag)), + string.IsNullOrWhiteSpace(input.TraceFlag) ? null : input.TraceFlag, + cts.Token); + + AnsiConsole.MarkupLine("[grey]Starting projection daemon (per-tenant agent fan-out)...[/]"); + using var daemon = await store.BuildProjectionDaemonAsync().ConfigureAwait(false); + await daemon.StartAllAsync().ConfigureAwait(false); + + // ---- Continuous append load ------------------------------------------ + + var appended = 0L; + var appendFailures = 0L; + var appendElapsed = Stopwatch.StartNew(); + var writers = Enumerable.Range(0, Math.Max(1, input.WritersFlag)) + .Select(w => Task.Run(() => AppendLoopAsync(store, tenants, w, input, cts.Token, + () => Interlocked.Increment(ref appended), + () => Interlocked.Increment(ref appendFailures)))) + .ToArray(); + + await Task.Delay(TimeSpan.FromSeconds(input.DurationSecondsFlag)).ConfigureAwait(false); + cts.Cancel(); + await Task.WhenAll(writers).ConfigureAwait(false); + appendElapsed.Stop(); + + // ---- Catch-up + verification ------------------------------------------ + + AnsiConsole.MarkupLine( + $"[grey]Appends stopped ({appended:N0} events). Waiting for per-tenant catch-up...[/]"); + var (caughtUpTenants, stalledTenants) = await WaitForCatchUpAsync( + tenants, projectionNames, TimeSpan.FromSeconds(input.CatchUpTimeoutSecondsFlag)) + .ConfigureAwait(false); + + var connections = sampler.Capture(); + await daemon.StopAllAsync().ConfigureAwait(false); + + totalElapsed.Stop(); + + // ---- Report ------------------------------------------------------------ + + var appendRate = appended / Math.Max(0.001, appendElapsed.Elapsed.TotalSeconds); + var agentCount = tenants.Length * projectionNames.Length; + + var table = new Table().AddColumn("Metric").AddColumn(new TableColumn("Value").RightAligned()); + table.AddRow("Tenants", tenants.Length.ToString("N0")); + table.AddRow("Async projections", projectionNames.Length.ToString("N0")); + table.AddRow("Tenant agents (projections × tenants)", agentCount.ToString("N0")); + table.AddRow("Events appended", appended.ToString("N0")); + table.AddRow("Append failures", appendFailures.ToString("N0")); + table.AddRow("Sustained append rate (events/sec)", appendRate.ToString("N0")); + table.AddRow("Connection samples", connections.SampleCount.ToString("N0")); + table.AddRow("[bold]Peak connections[/]", $"[bold]{connections.MaxTotal:N0}[/]"); + table.AddRow("Mean connections", connections.MeanTotal.ToString("N1")); + table.AddRow("Peak busy connections", connections.MaxBusy.ToString("N0")); + table.AddRow("Mean busy connections", connections.MeanBusy.ToString("N1")); + table.AddRow("Tenants caught up", $"{caughtUpTenants.Count:N0} / {tenants.Length:N0}"); + table.AddRow("Total elapsed", $"{totalElapsed.Elapsed.TotalSeconds:N1}s"); + AnsiConsole.Write(table); + + if (stalledTenants.Count > 0) + { + AnsiConsole.MarkupLine( + $"[red]{stalledTenants.Count} tenant(s) did not catch up within {input.CatchUpTimeoutSecondsFlag}s: " + + $"{string.Join(", ", stalledTenants.Take(10))}{(stalledTenants.Count > 10 ? ", ..." : "")}[/]"); + } + + await WriteMetricsAsync(input, tenants.Length, projectionNames.Length, appended, appendRate, + connections, caughtUpTenants.Count, stalledTenants).ConfigureAwait(false); + + var gatePassed = input.MaxConnectionsFlag <= 0 || connections.MaxTotal <= input.MaxConnectionsFlag; + if (!gatePassed) + { + AnsiConsole.MarkupLine( + $"[red]GATE FAILED: peak connections {connections.MaxTotal:N0} > --max-connections {input.MaxConnectionsFlag:N0}.[/]"); + } + + var healthy = appendFailures == 0 && stalledTenants.Count == 0; + if (!healthy) + { + AnsiConsole.MarkupLine("[red]Run unhealthy — see append failures / stalled tenants above.[/]"); + } + + return gatePassed && healthy; + } + + private static async Task AppendLoopAsync(IDocumentStore store, string[] tenants, int writerIndex, + DaemonLoadInput input, CancellationToken token, Action onAppended, Action onFailure) + { + // Each writer owns an interleaved slice of the tenant ring so all writers together cover + // every tenant. Rate control: each writer targets rate/writers events/sec, appending in + // small per-tenant batches with a delay computed from the batch size. + var writerCount = Math.Max(1, input.WritersFlag); + var perWriterRate = Math.Max(1.0, (double)input.AppendRatePerSecondFlag / writerCount); + const int batchSize = 5; + var delay = TimeSpan.FromSeconds(batchSize / perWriterRate); + + var sequence = 0; + var position = writerIndex; + while (!token.IsCancellationRequested) + { + var tenant = tenants[position % tenants.Length]; + position += writerCount; + + try + { + await using var session = store.LightweightSession(tenant); + var events = Enumerable.Range(0, batchSize) + .Select(_ => new DaemonLoadEvent(tenant, ++sequence)) + .Cast() + .ToArray(); + session.Events.StartStream(Guid.NewGuid(), events); + await session.SaveChangesAsync(token).ConfigureAwait(false); + + for (var i = 0; i < batchSize; i++) + { + onAppended(); + } + } + catch (OperationCanceledException) + { + break; + } + catch (Exception) + { + onFailure(); + } + + try + { + await Task.Delay(delay, token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + } + } + + /// + /// A tenant is caught up when, for every projection, its {projection}:All:{tenant} + /// progression row has reached that tenant's own sequence ceiling + /// (last_value of the per-tenant event sequence). + /// + private static async Task<(HashSet CaughtUp, List Stalled)> WaitForCatchUpAsync( + string[] tenants, string[] projectionNames, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + var stalled = new List(); + var caughtUp = new HashSet(); + + while (DateTime.UtcNow < deadline) + { + (caughtUp, stalled) = await CheckCatchUpOnceAsync(tenants, projectionNames).ConfigureAwait(false); + if (stalled.Count == 0) + { + return (caughtUp, stalled); + } + + await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false); + } + + return (caughtUp, stalled); + } + + private static async Task<(HashSet CaughtUp, List Stalled)> CheckCatchUpOnceAsync( + string[] tenants, string[] projectionNames) + { + // One round-trip: per tenant, min progression across the per-tenant projection rows vs the + // tenant's own sequence ceiling. Sequences are named mt_events_sequence_{tenant} by the + // per-tenant partitioning machinery. + const string sql = @" +SELECT s.tenant, coalesce(min(p.last_seq_id), 0) AS floor, max(s.ceiling) AS ceiling +FROM ( + SELECT replace(sequencename, 'mt_events_sequence_', '') AS tenant, + coalesce(last_value, 0) AS ceiling + FROM pg_sequences + WHERE schemaname = @schema AND sequencename LIKE 'mt_events_sequence_%' +) s +LEFT JOIN ( + SELECT split_part(name, ':', 3) AS tenant, name, last_seq_id + FROM {SCHEMA}.mt_event_progression + WHERE name LIKE '%:All:%' +) p ON p.tenant = s.tenant +GROUP BY s.tenant;"; + + var caughtUp = new HashSet(); + var stalled = new List(); + var expectedRows = projectionNames.Length; + + await using var conn = new NpgsqlConnection(ConnectionSource.ConnectionString); + await conn.OpenAsync().ConfigureAwait(false); + + // Also require the full per-tenant row COUNT so a tenant whose agents never started (zero + // progression rows) reads as stalled, not vacuously caught up. + var perTenantRows = new Dictionary(); + await using (var countCmd = new NpgsqlCommand( + $"select split_part(name, ':', 3), count(*) from {Schema}.mt_event_progression where name like '%:All:%' group by 1", + conn)) + await using (var countReader = await countCmd.ExecuteReaderAsync().ConfigureAwait(false)) + { + while (await countReader.ReadAsync().ConfigureAwait(false)) + { + perTenantRows[countReader.GetString(0)] = Convert.ToInt32(countReader.GetValue(1)); + } + } + + await using var cmd = new NpgsqlCommand(sql.Replace("{SCHEMA}", Schema), conn); + cmd.Parameters.AddWithValue("schema", Schema); + await using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + + var seen = new HashSet(); + while (await reader.ReadAsync().ConfigureAwait(false)) + { + var tenant = reader.GetString(0); + var floor = Convert.ToInt64(reader.GetValue(1)); + var ceiling = Convert.ToInt64(reader.GetValue(2)); + seen.Add(tenant); + + if (floor >= ceiling && perTenantRows.GetValueOrDefault(tenant) >= expectedRows) + { + caughtUp.Add(tenant); + } + else + { + stalled.Add(tenant); + } + } + + // A registered tenant with no per-tenant sequence at all is definitely stalled + stalled.AddRange(tenants.Where(t => !seen.Contains(t))); + + return (caughtUp, stalled); + } + + private static async Task WipeSchemaAsync() + { + await using var conn = new NpgsqlConnection(ConnectionSource.ConnectionString); + await conn.OpenAsync().ConfigureAwait(false); + await using var drop = new NpgsqlCommand($"drop schema if exists \"{Schema}\" cascade", conn); + await drop.ExecuteNonQueryAsync().ConfigureAwait(false); + } + + private static async Task WriteMetricsAsync(DaemonLoadInput input, int tenants, int projections, + long appended, double appendRate, ConnectionSampler.Snapshot connections, + int caughtUpTenants, List stalledTenants) + { + if (string.IsNullOrWhiteSpace(input.MetricsFlag)) + { + return; + } + + var doc = new + { + scenario = "daemonload", + tenants, + projections, + tenantAgents = tenants * projections, + durationSeconds = input.DurationSecondsFlag, + eventsAppended = appended, + appendRatePerSecond = appendRate, + connections = new + { + samples = connections.SampleCount, + maxTotal = connections.MaxTotal, + meanTotal = connections.MeanTotal, + maxBusy = connections.MaxBusy, + meanBusy = connections.MeanBusy + }, + caughtUpTenants, + stalledTenants + }; + + await File.WriteAllTextAsync(input.MetricsFlag, + JsonSerializer.Serialize(doc, new JsonSerializerOptions { WriteIndented = true })) + .ConfigureAwait(false); + AnsiConsole.MarkupLine($"[grey]Metrics written to {input.MetricsFlag}[/]"); + } +} + +public record DaemonLoadEvent(string Tenant, int Sequence); + +/// +/// Deliberately minimal async projection — one doc write per batch. The WS2 measurement targets +/// the DAEMON's connection behavior (event loaders, progression flushes, high-water polling), so +/// the projection body stays cheap to keep projection CPU out of the signal. +/// +public class DaemonLoadRollup +{ + public string Id { get; set; } = null!; + public long EventCount { get; set; } +} + +public class DaemonLoadRollupProjection: IProjection +{ + private readonly string _name; + + public DaemonLoadRollupProjection(string name) + { + _name = name; + } + + public async Task ApplyAsync(IDocumentOperations operations, IReadOnlyList events, + CancellationToken cancellation) + { + var count = events.Count(e => e.Data is DaemonLoadEvent); + if (count == 0) + { + return; + } + + var rollup = await operations.LoadAsync(_name, cancellation) + ?? new DaemonLoadRollup { Id = _name }; + rollup.EventCount += count; + operations.Store(rollup); + } +} diff --git a/src/Marten.ScaleTesting/Commands/DaemonLoadInput.cs b/src/Marten.ScaleTesting/Commands/DaemonLoadInput.cs new file mode 100644 index 0000000000..cb11412e18 --- /dev/null +++ b/src/Marten.ScaleTesting/Commands/DaemonLoadInput.cs @@ -0,0 +1,48 @@ +using JasperFx; +using JasperFx.CommandLine; + +namespace Marten.ScaleTesting.Commands; + +/// +/// Inputs for the daemonload subcommand (jasperfx#486 WS2 measurement). A RUNNING-daemon +/// scenario, as opposed to the rebuild-centric rest of the harness: N tenants under +/// UseTenantPartitionedEvents, per-tenant subscription agents fanned out by the daemon, +/// continuous appends across every tenant, and pg_stat_activity sampled throughout for +/// the store's connection footprint. The WS2 question this answers: do steady-state connections +/// scale O(databases) or O(tenant agents)? +/// +public sealed class DaemonLoadInput: NetCoreInput +{ + [Description("Number of tenants to register on the partitioned store. Default: 100.")] + public int TenantsFlag { get; set; } = 100; + + [Description("Number of async projections to register — each fans out one agent per tenant. Default: 2.")] + public int ProjectionsFlag { get; set; } = 2; + + [Description("Seconds to keep the daemon running under continuous append load. Default: 120.")] + public int DurationSecondsFlag { get; set; } = 120; + + [Description("Concurrent append writer tasks. Default: 4.")] + public int WritersFlag { get; set; } = 4; + + [Description("Approximate total events appended per second across all writers. Default: 200.")] + public int AppendRatePerSecondFlag { get; set; } = 200; + + [Description("pg_stat_activity sample interval in seconds. Default: 1.0.")] + public double SampleSecondsFlag { get; set; } = 1.0; + + [Description("Seconds to wait after appends stop for every tenant's agents to catch up to that tenant's ceiling. Default: 60.")] + public int CatchUpTimeoutSecondsFlag { get; set; } = 60; + + [Description("Fail (exit 1) if the store's peak concurrent connections exceed this. Default: 0 = report only.")] + public int MaxConnectionsFlag { get; set; } + + [Description("Drop + recreate the dedicated schema before the run. Default: false.")] + public bool WipeFlag { get; set; } + + [Description("Optional path for a JSON metrics file (peak/mean connections, throughput, catch-up).")] + public string? MetricsFlag { get; set; } + + [Description("Optional path for a per-sample CSV connection trace.")] + public string? TraceFlag { get; set; } +} diff --git a/src/Marten.ScaleTesting/Instrumentation/ConnectionSampler.cs b/src/Marten.ScaleTesting/Instrumentation/ConnectionSampler.cs new file mode 100644 index 0000000000..7718acea1a --- /dev/null +++ b/src/Marten.ScaleTesting/Instrumentation/ConnectionSampler.cs @@ -0,0 +1,154 @@ +using System.Globalization; +using Npgsql; + +namespace Marten.ScaleTesting.Instrumentation; + +/// +/// Background poll loop that snapshots pg_stat_activity for the connection footprint of a +/// single application name (the store under test stamps its own Application Name, so the +/// sampler's own session and unrelated dev-box traffic are excluded). This is the WS2 +/// (jasperfx#486) measurement primitive: peak/mean concurrent connections while the daemon runs. +/// Mirrors the sampling shape of . +/// +internal sealed class ConnectionSampler: IAsyncDisposable +{ + private const string SqlText = @" +SELECT + count(*) AS total, + count(*) FILTER (WHERE state <> 'idle') AS busy +FROM pg_stat_activity +WHERE datname = current_database() + AND pid <> pg_backend_pid() + AND application_name = @app;"; + + private readonly string _connectionString; + private readonly string _applicationName; + private readonly TimeSpan _interval; + private readonly CancellationTokenSource _cts; + private readonly Task _loop; + private readonly List _samples = new(); + private readonly StreamWriter? _traceWriter; + + private ConnectionSampler(string connectionString, string applicationName, TimeSpan interval, + string? tracePath, CancellationToken outerCancellation) + { + _connectionString = connectionString; + _applicationName = applicationName; + _interval = interval; + _cts = CancellationTokenSource.CreateLinkedTokenSource(outerCancellation); + if (tracePath != null) + { + // AutoFlush so a long-running scenario's trace can be tailed live + _traceWriter = new StreamWriter(tracePath, append: false) { AutoFlush = true }; + _traceWriter.WriteLine("timestamp,total_connections,busy_connections"); + } + + _loop = Task.Run(LoopAsync, CancellationToken.None); + } + + public static ConnectionSampler Start(string connectionString, string applicationName, + TimeSpan interval, string? tracePath, CancellationToken cancellation) + => new(connectionString, applicationName, interval, tracePath, cancellation); + + public sealed record ConnectionSample(DateTimeOffset Timestamp, int Total, int Busy); + + public sealed record Snapshot(int SampleCount, int MaxTotal, double MeanTotal, int MaxBusy, double MeanBusy); + + public Snapshot Capture() + { + lock (_samples) + { + if (_samples.Count == 0) + { + return new Snapshot(0, 0, 0, 0, 0); + } + + return new Snapshot( + _samples.Count, + _samples.Max(x => x.Total), + _samples.Average(x => x.Total), + _samples.Max(x => x.Busy), + _samples.Average(x => x.Busy)); + } + } + + private async Task LoopAsync() + { + while (!_cts.IsCancellationRequested) + { + try + { + await SampleOnceAsync().ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception) + { + // Transient sampling failure — skip the sample, keep the loop alive + } + + try + { + await Task.Delay(_interval, _cts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + } + } + + private async Task SampleOnceAsync() + { + await using var conn = new NpgsqlConnection(_connectionString); + await conn.OpenAsync(_cts.Token).ConfigureAwait(false); + await using var cmd = new NpgsqlCommand(SqlText, conn); + cmd.Parameters.AddWithValue("app", _applicationName); + + await using var reader = await cmd.ExecuteReaderAsync(_cts.Token).ConfigureAwait(false); + if (!await reader.ReadAsync(_cts.Token).ConfigureAwait(false)) + { + return; + } + + var sample = new ConnectionSample( + DateTimeOffset.UtcNow, + Convert.ToInt32(reader.GetValue(0)), + Convert.ToInt32(reader.GetValue(1))); + + lock (_samples) + { + _samples.Add(sample); + } + + if (_traceWriter != null) + { + await _traceWriter.WriteLineAsync(string.Create(CultureInfo.InvariantCulture, + $"{sample.Timestamp:O},{sample.Total},{sample.Busy}")) + .ConfigureAwait(false); + } + } + + public async ValueTask DisposeAsync() + { + _cts.Cancel(); + try + { + await _loop.ConfigureAwait(false); + } + catch + { + // Best-effort teardown + } + + if (_traceWriter != null) + { + await _traceWriter.FlushAsync().ConfigureAwait(false); + _traceWriter.Dispose(); + } + + _cts.Dispose(); + } +} diff --git a/src/Marten.ScaleTesting/README.md b/src/Marten.ScaleTesting/README.md index 177045290b..5ba10f1900 100644 --- a/src/Marten.ScaleTesting/README.md +++ b/src/Marten.ScaleTesting/README.md @@ -64,6 +64,12 @@ dotnet run --project src/Marten.ScaleTesting -- stress \ --shard-timeout-seconds 3600 --baseline scaletest-baseline.json \ --instrument --instrument-trace stress-trace.csv \ --instrument-lock-trace stress-lock-trace.csv + +# WS2 (jasperfx#486): RUNNING daemon over 100 partitioned tenants under continuous +# append load, sampling pg_stat_activity for the store's connection footprint +dotnet run --project src/Marten.ScaleTesting -- daemonload \ + --tenants 100 --projections 2 --duration-seconds 120 --wipe \ + --metrics daemonload-metrics.json --trace daemonload-connections.csv ``` `metrics.json` shape: @@ -94,6 +100,23 @@ dotnet run --project src/Marten.ScaleTesting -- stress \ Without `--instrument` the samplers / counter / activity are not constructed at all -- a measured rebuild has no perceptible overhead and the JSON section reads `"enabled": false` with zeroed totals. +## The `daemonload` scenario (jasperfx#486 WS2) + +Everything above measures **rebuilds**; `daemonload` measures the **steady-state running +daemon** — the deployment shape whose connection footprint at ~100 tenants is the WS2 +concern. It builds an isolated `UseTenantPartitionedEvents` store in the +`scaletest_daemonload` schema with `--projections` async projections, registers +`--tenants` tenants, starts the daemon (which fans out one subscription agent per +projection × tenant), appends continuously across every tenant for `--duration-seconds`, +and samples `pg_stat_activity` throughout. The store's connection string carries a +dedicated `Application Name` so the sampler counts exactly the store's connections. + +The run fails (exit 1) on any append failure or any tenant whose agents don't catch up to +that tenant's own per-tenant sequence ceiling within `--catch-up-timeout-seconds`. +`--max-connections N` additionally turns the peak-connection reading into a pass/fail +gate: the WS2 goal is connections O(databases), not O(tenant agents), so run this before +and after the daemon command-batching work to quantify the win and then pin it. + ## Non-goals * Not a microbenchmark — `src/MartenBenchmarks/` covers per-method timings.