Skip to content

Stop writing a periodic per-shard heartbeat nobody reads (#622) #628

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jeremydmiller merged 1 commit into from
Aug 4, 2026
Merged

Stop writing a periodic per-shard heartbeat nobody reads (#622) #628

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
184 changes: 184 additions & 0 deletions src/EventTests/Daemon/ExtendedProgressionHeartbeatIntervalTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using System.Threading;
using System.Threading.Tasks;
using EventTests.Projections;
using JasperFx;
using JasperFx.Events;
using JasperFx.Events.Daemon;
using JasperFx.Events.Daemon.HighWater;
using JasperFx.Events.Projections;
using NSubstitute;
using Shouldly;

namespace EventTests.Daemon;

/// <summary>
/// jasperfx#622 — the periodic per-shard extended progression heartbeat had no reader anywhere and
/// cost one pooled connection + one transaction per database per node every 5 seconds, with no
/// configuration path at all (the interval was private on the daemon and no DaemonSettings knob
/// reached it). It is off by default now, and DaemonSettings.ExtendedProgressionHeartbeatInterval is
/// the compatibility hatch.
/// </summary>
public class ExtendedProgressionHeartbeatIntervalTests : IDisposable
{
private readonly ShardStateTracker theTracker = new(new NulloLogger());
private readonly RecordingDatabase theDatabase;

private readonly List<JasperFxAsyncDaemon<FakeOperations, FakeSession, IJasperFxProjection<FakeOperations>>>
theDaemons = [];

public ExtendedProgressionHeartbeatIntervalTests()
{
theDatabase = new RecordingDatabase(theTracker);
}

public void Dispose()
{
foreach (var daemon in theDaemons) daemon.Dispose();
}

private async Task<JasperFxAsyncDaemon<FakeOperations, FakeSession, IJasperFxProjection<FakeOperations>>>
startedDaemon(TimeSpan? heartbeatInterval)
{
var store = Substitute.For<IEventStore<FakeOperations, FakeSession>>();
store.Meter.Returns(new Meter("tests"));
store.TimeProvider.Returns(TimeProvider.System);
store.ExtendedProgressionEnabled.Returns(true);
store.AutoCreateSchemaObjects.Returns(AutoCreate.None);

var graph = new FakeGraph { ExtendedProgressionHeartbeatInterval = heartbeatInterval };

var daemon = new JasperFxAsyncDaemon<FakeOperations, FakeSession, IJasperFxProjection<FakeOperations>>(
store, theDatabase, new NulloLogger(), new StubDetector(), graph);

theDaemons.Add(daemon);
await daemon.StartHighWaterDetectionAsync();
return daemon;
}

private static ShardState heartbeat() => new("Trip:All", 5)
{
Action = ShardAction.Updated, AgentStatus = "Running", LastHeartbeat = DateTimeOffset.UtcNow
};

private static ShardState transition() => new("Trip:All", 5)
{
Action = ShardAction.Paused, AgentStatus = "Paused", PauseReason = "boom"
};

private async Task<int> writeCountAfterPublishing(ShardState state, int expected)
{
await theTracker.PublishAsync(state);

for (var i = 0; i < 100 && theDatabase.Writes < expected; i++)
{
await Task.Delay(20, TestContext.Current.CancellationToken);
}

if (expected == 0)
{
await Task.Delay(250, TestContext.Current.CancellationToken);
}

return theDatabase.Writes;
}

[Fact]
public async Task no_periodic_heartbeat_write_by_default()
{
await startedDaemon(null);

(await writeCountAfterPublishing(heartbeat(), 0)).ShouldBe(0);
}

[Fact]
public async Task transitions_are_still_persisted_by_default()
{
await startedDaemon(null);

(await writeCountAfterPublishing(transition(), 1)).ShouldBe(1);
}

[Fact]
public async Task a_configured_interval_restores_the_periodic_beat()
{
await startedDaemon(TimeSpan.FromSeconds(5));

(await writeCountAfterPublishing(heartbeat(), 1)).ShouldBe(1);
}

[Fact]
public async Task a_non_positive_configured_interval_is_off()
{
await startedDaemon(TimeSpan.Zero);

(await writeCountAfterPublishing(heartbeat(), 0)).ShouldBe(0);
}

private sealed class RecordingDatabase : IEventDatabase
{
private int _writes;

public RecordingDatabase(ShardStateTracker tracker) => Tracker = tracker;

public int Writes => Volatile.Read(ref _writes);

public Task WriteExtendedProgressionAsync(IReadOnlyList<ShardState> states, CancellationToken token = default)
{
Interlocked.Increment(ref _writes);
return Task.CompletedTask;
}

public Task WriteExtendedProgressionAsync(ShardState state, CancellationToken token = default)
{
Interlocked.Increment(ref _writes);
return Task.CompletedTask;
}

public ShardStateTracker Tracker { get; }
public string Identifier => "db1";
public Uri DatabaseUri { get; } = new("fake://db1");
public string StorageIdentifier => "db1";

public Task StoreDeadLetterEventAsync(object storage, DeadLetterEvent deadLetterEvent, CancellationToken token)
=> Task.CompletedTask;

public Task EnsureStorageExistsAsync(Type storageType, CancellationToken token) => Task.CompletedTask;
public Task WaitForNonStaleProjectionDataAsync(TimeSpan timeout) => Task.CompletedTask;

public Task<long> ProjectionProgressFor(ShardName name, CancellationToken token = default)
=> Task.FromResult(0L);

public Task<long?> FindEventStoreFloorAtTimeAsync(DateTimeOffset timestamp, CancellationToken token)
=> Task.FromResult<long?>(null);

public Task<long> FetchHighestEventSequenceNumber(CancellationToken token) => Task.FromResult(0L);

public Task<IReadOnlyList<ShardState>> AllProjectionProgress(CancellationToken token = default)
=> Task.FromResult<IReadOnlyList<ShardState>>([]);
}

private sealed class StubDetector : IHighWaterDetector
{
public Uri DatabaseUri { get; } = new("fake://db1");

public Task<HighWaterStatistics> Detect(CancellationToken token)
=> Task.FromResult(new HighWaterStatistics());

public Task<HighWaterStatistics> DetectInSafeZone(CancellationToken token) => Detect(token);
}

private sealed class FakeGraph : ProjectionGraph<IJasperFxProjection<FakeOperations>, FakeOperations, FakeSession>
{
public FakeGraph() : base(Substitute.For<IEventRegistry>(), "tests")
{
}

protected override void onAddProjection(object projection)
{
// Nothing
}
}
}
25 changes: 15 additions & 10 deletions src/EventTests/Daemon/ExtendedProgressionSubscriptionLifecycleTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,13 @@ private JasperFxAsyncDaemon<FakeOperations, FakeSession, IJasperFxProjection<Fak
return daemon;
}

private static ShardState heartbeat(string shardName) =>
new(shardName, 5) { AgentStatus = "Running", LastHeartbeat = DateTimeOffset.UtcNow };
// jasperfx#622: only status transitions are persisted by default, so the probe publication that
// proves "this daemon is writing" has to be one
private static ShardState startedState(string shardName) =>
new(shardName, 5)
{
Action = ShardAction.Started, AgentStatus = "Running", LastHeartbeat = DateTimeOffset.UtcNow
};

// Publications are delivered asynchronously through the tracker's block, so both the positive and
// the negative assertion have to be given the same chance to happen
Expand Down Expand Up @@ -84,7 +89,7 @@ public async Task an_unstarted_daemon_does_not_write_extended_progression()
// every 15 seconds purely to read CurrentAgents()
var daemon = buildDaemon();

(await writeCountAfterPublishing(heartbeat("Trip:All"), 0)).ShouldBe(0);
(await writeCountAfterPublishing(startedState("Trip:All"), 0)).ShouldBe(0);
}

[Fact]
Expand All @@ -95,7 +100,7 @@ public async Task many_unstarted_daemons_on_one_shared_tracker_still_write_nothi
buildDaemon();
}

(await writeCountAfterPublishing(heartbeat("Trip:All"), 0)).ShouldBe(0);
(await writeCountAfterPublishing(startedState("Trip:All"), 0)).ShouldBe(0);
}

[Fact]
Expand All @@ -104,7 +109,7 @@ public async Task a_started_daemon_does_write_extended_progression()
var daemon = buildDaemon();
await daemon.StartHighWaterDetectionAsync();

(await writeCountAfterPublishing(heartbeat("Trip:All"), 1)).ShouldBe(1);
(await writeCountAfterPublishing(startedState("Trip:All"), 1)).ShouldBe(1);
}

[Fact]
Expand All @@ -119,7 +124,7 @@ public async Task only_the_started_daemon_writes_when_readers_share_the_tracker(

await owner.StartHighWaterDetectionAsync();

(await writeCountAfterPublishing(heartbeat("Trip:All"), 1)).ShouldBe(1);
(await writeCountAfterPublishing(startedState("Trip:All"), 1)).ShouldBe(1);
}

[Fact]
Expand All @@ -129,19 +134,19 @@ public async Task starting_twice_does_not_stack_a_second_subscription()
await daemon.StartHighWaterDetectionAsync();
await daemon.StartHighWaterDetectionAsync();

(await writeCountAfterPublishing(heartbeat("Trip:All"), 1)).ShouldBe(1);
(await writeCountAfterPublishing(startedState("Trip:All"), 1)).ShouldBe(1);
}

[Fact]
public async Task a_stopped_daemon_stops_writing()
{
var daemon = buildDaemon();
await daemon.StartHighWaterDetectionAsync();
(await writeCountAfterPublishing(heartbeat("Trip:All"), 1)).ShouldBe(1);
(await writeCountAfterPublishing(startedState("Trip:All"), 1)).ShouldBe(1);

await daemon.StopAllAsync();

(await writeCountAfterPublishing(heartbeat("Trip:All"), 2)).ShouldBe(1);
(await writeCountAfterPublishing(startedState("Trip:All"), 2)).ShouldBe(1);
}

[Fact]
Expand All @@ -154,7 +159,7 @@ public async Task a_restarted_daemon_resumes_writing()
await daemon.StopAllAsync();
await daemon.StartHighWaterDetectionAsync();

(await writeCountAfterPublishing(heartbeat("Trip:All"), 1)).ShouldBe(1);
(await writeCountAfterPublishing(startedState("Trip:All"), 1)).ShouldBe(1);
}

[Fact]
Expand Down
64 changes: 64 additions & 0 deletions src/EventTests/Daemon/ExtendedProgressionWriterTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@ public ExtendedProgressionWriterTests()
theWriter = new ExtendedProgressionWriter(theStore, theDatabase, theTime, NullLogger.Instance);
}

// jasperfx#622: the periodic per-shard beat is OFF by default now. The machinery is unchanged and
// still shipped behind DaemonSettings.ExtendedProgressionHeartbeatInterval, so the tests that pin
// its coalescing/flush behavior opt into it explicitly.
private void enablePeriodicHeartbeats()
{
theWriter.HeartbeatWriteInterval = TimeSpan.FromSeconds(5);
}

private static ShardState transition(ShardAction action, string status, string? pauseReason = null,
string shardName = "Counters:All")
{
Expand Down Expand Up @@ -100,6 +108,8 @@ public async Task skips_plain_progress_publications_with_no_agent_telemetry()
[Fact]
public async Task coalesces_heartbeats_into_one_batched_write_per_flush_interval()
{
enablePeriodicHeartbeats();

// The very first heartbeat flushes immediately (nothing to wait for)
theWriter.OnNext(heartbeat(sequence: 1));
var writes = await theDatabase.WaitForWrites(1);
Expand Down Expand Up @@ -127,6 +137,8 @@ public async Task coalesces_heartbeats_into_one_batched_write_per_flush_interval
[Fact]
public async Task a_transition_flushes_immediately_and_carries_the_pending_heartbeats_along()
{
enablePeriodicHeartbeats();

// Seed a flush so the interval throttle is active
theWriter.OnNext(heartbeat(sequence: 1));
await theDatabase.WaitForWrites(1);
Expand All @@ -148,6 +160,8 @@ public async Task a_transition_flushes_immediately_and_carries_the_pending_heart
[Fact]
public async Task a_transition_replaces_a_pending_heartbeat_for_the_same_shard()
{
enablePeriodicHeartbeats();

theWriter.OnNext(heartbeat(sequence: 1));
await theDatabase.WaitForWrites(1);

Expand Down Expand Up @@ -205,6 +219,8 @@ public async Task does_not_clobber_an_explicit_running_on_node()
[Fact]
public async Task disposing_flushes_the_pending_batch()
{
enablePeriodicHeartbeats();

theWriter.OnNext(heartbeat(sequence: 1));
await theDatabase.WaitForWrites(1);

Expand Down Expand Up @@ -242,6 +258,54 @@ public async Task dispose_awaits_the_in_flight_write_rather_than_draining_in_the
writes[0].AgentStatus.ShouldBe("Stopped");
}

// jasperfx#622
[Fact]
public async Task no_periodic_heartbeat_write_by_default()
{
// The whole point: a heartbeat nobody reads costs a connection + a transaction per database
// per node per interval (marten#5167). Off unless asked for.
theWriter.PeriodicHeartbeatsEnabled.ShouldBeFalse();

theWriter.OnNext(heartbeat(sequence: 1));
theWriter.OnNext(heartbeat(sequence: 2));
theWriter.OnNext(heartbeat("Others:All", sequence: 7));
theTime.Advance(TimeSpan.FromMinutes(5));
theWriter.OnNext(heartbeat(sequence: 3));

await theDatabase.AssertNoWrites();
}

// jasperfx#622: the transitional columns are exactly what survives -- rare writes, and the data
// the "durable across a crash" story was actually about
[Fact]
public async Task transitions_are_still_written_with_the_periodic_beat_off()
{
theWriter.OnNext(heartbeat(sequence: 1));
theWriter.OnNext(transition(ShardAction.Paused, "Paused", "boom"));

var writes = await theDatabase.WaitForWrites(1);
var batches = await theDatabase.Batches();

// ...and the dropped heartbeat does NOT ride along on the transition's write
batches.Count.ShouldBe(1);
batches[0].Length.ShouldBe(1);
writes[0].AgentStatus.ShouldBe("Paused");
writes[0].PauseReason.ShouldBe("boom");
}

// jasperfx#622: the compatibility hatch the interval never had
[Fact]
public async Task a_positive_interval_restores_the_periodic_beat()
{
enablePeriodicHeartbeats();
theWriter.PeriodicHeartbeatsEnabled.ShouldBeTrue();

theWriter.OnNext(heartbeat(sequence: 1));

var writes = await theDatabase.WaitForWrites(1);
writes[0].Sequence.ShouldBe(1);
}

[Fact]
public async Task the_default_batch_implementation_degrades_to_single_state_writes()
{
Expand Down
Loading
Loading