Skip to content
87 changes: 77 additions & 10 deletions src/Orleans.Core/Runtime/MembershipTableSnapshot.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public static MembershipTableSnapshot Update(MembershipTableSnapshot previousSna
var version = (table.Version.Version == 0 && table.Version.VersionEtag == "0")
? MembershipVersion.MinValue
: new MembershipVersion(table.Version.Version);
return Update(previousSnapshot, version, table.Members.Select(t => t.Item1));
return Update(previousSnapshot, version, table.Members.Select(t => t.Item1), preserveVersionedFields: true);
}

/// <summary>
Expand All @@ -63,7 +63,11 @@ public static MembershipTableSnapshot Update(MembershipTableSnapshot previousSna
return Update(previousSnapshot, updated.Version, updated.Entries.Values);
}

private static MembershipTableSnapshot Update(MembershipTableSnapshot previousSnapshot, MembershipVersion version, IEnumerable<MembershipEntry> updatedEntries)
private static MembershipTableSnapshot Update(
MembershipTableSnapshot previousSnapshot,
MembershipVersion version,
IEnumerable<MembershipEntry> updatedEntries,
bool preserveVersionedFields = false)
{
ArgumentNullException.ThrowIfNull(previousSnapshot);
ArgumentNullException.ThrowIfNull(updatedEntries);
Expand All @@ -72,6 +76,16 @@ private static MembershipTableSnapshot Update(MembershipTableSnapshot previousSn
foreach (var item in updatedEntries)
{
var entry = item;
if (preserveVersionedFields && version == previousSnapshot.Version
&& previousSnapshot.Entries.TryGetValue(entry.SiloAddress, out var previousEntry))
{
// Provider reads can round fields from our committed write. Only liveness advances
// at the same table version; retain the versioned fields we already accepted.
entry = entry.IAmAliveTime > previousEntry.IAmAliveTime
? previousEntry.WithIAmAliveTime(entry.IAmAliveTime)
: previousEntry;
}

entry = PreserveIAmAliveTime(previousSnapshot, entry);
entries.Add(entry.SiloAddress, entry);
}
Expand Down Expand Up @@ -170,25 +184,78 @@ public bool IsSuccessorTo(MembershipTableSnapshot other)
return false;
}

foreach (var entry in Entries)
var heartbeatAdvanced = false;
foreach (var (silo, entry) in Entries)
{
if (!other.Entries.TryGetValue(entry.Key, out var otherEntry))
if (!other.Entries.TryGetValue(silo, out var otherEntry)
|| !AreVersionedFieldsEqual(entry, otherEntry))
{
// Something is amiss.
// Membership changes require a table-version advance.
return false;
}

heartbeatAdvanced |= entry.EffectiveIAmAliveTime > otherEntry.EffectiveIAmAliveTime;
}

// This is a successor if any silo has a later EffectiveIAmAliveTime.
foreach (var entry in Entries)
if (Entries.Count == other.Entries.Count)
{
if (entry.Value.EffectiveIAmAliveTime > other.Entries[entry.Key].EffectiveIAmAliveTime)
return heartbeatAdvanced;
}

// Cleanup can remove inactive entries without advancing the table version or a heartbeat.
// Accept that inventory change while retaining every Active entry and the remaining statuses.
foreach (var (silo, previousEntry) in other.Entries)
{
if (previousEntry.Status == SiloStatus.Active && !Entries.ContainsKey(silo))
{
return true;
return false;
}
}

return false;
return true;
}

internal static bool AreVersionedFieldsEqual(MembershipEntry left, MembershipEntry right)
{
if (ReferenceEquals(left, right))
{
return true;
}

if (!left.SiloAddress.Equals(right.SiloAddress)
|| left.Status != right.Status
|| left.ProxyPort != right.ProxyPort
|| !string.Equals(left.HostName, right.HostName, StringComparison.Ordinal)
|| !string.Equals(left.SiloName, right.SiloName, StringComparison.Ordinal)
|| !string.Equals(left.RoleName, right.RoleName, StringComparison.Ordinal)
|| left.UpdateZone != right.UpdateZone
|| left.FaultZone != right.FaultZone
|| left.StartTime != right.StartTime)
{
return false;
}

var leftSuspects = left.SuspectTimes;
var rightSuspects = right.SuspectTimes;
if (ReferenceEquals(leftSuspects, rightSuspects))
{
return true;
}

if (leftSuspects is null || rightSuspects is null || leftSuspects.Count != rightSuspects.Count)
{
return false;
}

for (var index = 0; index < leftSuspects.Count; index++)
{
if (!leftSuspects[index].Equals(rightSuspects[index]))
{
return false;
}
}

return true;
}

public override string ToString()
Expand Down
6 changes: 5 additions & 1 deletion src/Orleans.Core/SystemTargetInterfaces/IMembershipTable.cs
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,12 @@ async Task DeleteMembershipTableEntriesAsync(string clusterId, CancellationToken
}

/// <summary>
/// Delete all dead silo entries older than <paramref name="beforeDate"/>
/// Cleans up defunct silo entries older than <paramref name="beforeDate"/>.
/// </summary>
/// <remarks>
/// Providers can remove expired inactive entries without advancing the table version.
/// Active entries are retained.
/// </remarks>
/// <param name="beforeDate">The exclusive upper bound for the last known update time of entries to delete.</param>
/// <returns>A task representing the cleanup operation.</returns>
[Obsolete("Use CleanupDefunctSiloEntriesAsync instead.")]
Expand Down
57 changes: 51 additions & 6 deletions src/Orleans.Runtime/MembershipService/MembershipTableManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,23 @@ public async Task RefreshFromSnapshot(MembershipTableSnapshot snapshot, Cancella

cancellationToken.ThrowIfCancellationRequested();

// Development clustering recreates its volatile table when the primary restarts.
// Peer snapshots have no table incarnation, so notifications refresh that authority.
if (this.membershipTableProvider is SystemTargetBasedMembershipTable)
{
try
{
await this.Refresh(cancellationToken: cancellationToken, requireFresh: true);
Comment thread
ReubenBond marked this conversation as resolved.
Resolved
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
throw;
}

return;
}

// Check if a refresh is underway
var pending = this.pendingRefresh;
if (pending != null && !pending.IsCompleted)
Expand All @@ -186,13 +203,14 @@ public async Task RefreshFromSnapshot(MembershipTableSnapshot snapshot, Cancella

LogInformationReceivedClusterMembershipSnapshot(this.log, snapshot);

cancellationToken.ThrowIfCancellationRequested();
this.TryProcessMembershipUpdate(MembershipTableSnapshot.Update, snapshot, nameof(RefreshFromSnapshot));
}

private async Task<bool> RefreshInternal(bool requireCleanup, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var table = await this.membershipTableProvider.ReadAllAsync(cancellationToken);
var table = await this.ReadMembershipTable(cancellationToken);

bool success;
try
Expand All @@ -214,6 +232,28 @@ private async Task<bool> RefreshInternal(bool requireCleanup, CancellationToken
return !requireCleanup || success;
}

private Task<MembershipTableData> ReadMembershipTable(CancellationToken cancellationToken) =>
this.membershipTableProvider is SystemTargetBasedMembershipTable
? ReadDevelopmentMembershipTable(cancellationToken)
: this.membershipTableProvider.ReadAllAsync(cancellationToken);

private async Task<MembershipTableData> ReadDevelopmentMembershipTable(CancellationToken cancellationToken)
{
// Capture before issuing the read: an older read completing after a newer one is not a reset.
var observedVersion = this.MembershipTableSnapshot.Version;
var table = await this.membershipTableProvider.ReadAllAsync(cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
if (table.Version.Version < observedVersion.Value)
{
var reason = $"The development membership table version decreased from {observedVersion} to {table.Version.Version}. "
+ "Its previous incarnation is no longer authoritative. Restart this silo against the recreated table.";
this.KillMyselfLocally(reason);
throw new OrleansException(reason);
}

return table;
}

private async Task Start(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
Expand Down Expand Up @@ -454,7 +494,7 @@ async Task<bool> UpdateMyStatusTask(int counter, CancellationToken token)
private async Task<bool> TryUpdateMyStatusGlobalOnce(SiloStatus newStatus, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var table = await membershipTableProvider.ReadAllAsync(cancellationToken);
var table = await ReadMembershipTable(cancellationToken);

LogDebugTryUpdateMyStatusGlobalOnce(this.log, newStatus.Equals(SiloStatus.Active) ? "All" : " my entry from", table.ToString());
LogMissedIAmAlives(table);
Expand Down Expand Up @@ -562,6 +602,11 @@ private MembershipTableSnapshot ProcessMembershipUpdate(
MembershipTableSnapshot previous,
MembershipTableSnapshot updated)
{
if (updated.Version < previous.Version)
{
return previous;
}

if (!previous.Entries.TryGetValue(this.myAddress, out var previousLocalSiloEntry)
|| previousLocalSiloEntry.Status == SiloStatus.Created)
{
Expand Down Expand Up @@ -909,7 +954,7 @@ private void PublishSuspectOrKillRequestCompletion(SuspectOrKillRequest request,
private async Task<bool> InnerTryKill(SiloAddress silo, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var table = await membershipTableProvider.ReadAllAsync(cancellationToken);
var table = await ReadMembershipTable(cancellationToken);

LogDebugTryKillReadMembershipTable(this.log, table.ToString());

Expand Down Expand Up @@ -956,7 +1001,7 @@ public async Task<bool> TryToSuspectOrKill(SiloAddress silo, SiloAddress? indire
private async Task<bool> InnerTryToSuspectOrKill(SiloAddress silo, SiloAddress? indirectProbingSilo, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var table = await membershipTableProvider.ReadAllAsync(cancellationToken);
var table = await ReadMembershipTable(cancellationToken);
var now = GetDateTimeUtcNow();

LogDebugTryToSuspectOrKillReadMembershipTable(this.log, table.ToString());
Expand Down Expand Up @@ -1031,7 +1076,7 @@ private async Task<bool> InnerTryToSuspectOrKill(SiloAddress silo, SiloAddress?
var ok = await membershipTableProvider.UpdateRowAsync(entry, eTag, table.Version.Next(), cancellationToken);
if (ok)
{
table = await membershipTableProvider.ReadAllAsync(cancellationToken);
table = await ReadMembershipTable(cancellationToken);
this.ProcessTableUpdate(table, "TrySuspectOrKill");

// Gossip using the local silo status, since this is just informational to propagate the suspicion vote.
Expand Down Expand Up @@ -1063,7 +1108,7 @@ private async Task<bool> DeclareDead(MembershipEntry entry, string etag, TableVe
{
LogDebugSuccessfullyUpdatedStatusToDead(this.log, entry.SiloAddress);

var table = await membershipTableProvider.ReadAllAsync(cancellationToken);
var table = await ReadMembershipTable(cancellationToken);
this.ProcessTableUpdate(table, "DeclareDead");
GossipToOthers(entry.SiloAddress, entry.Status, GossipTimeout, cancellationToken).Ignore();
return true;
Expand Down
Loading
Loading