Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 103 additions & 16 deletions packages/beacon-node/src/network/peers/peerManager.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {Connection, PeerId, PrivateKey} from "@libp2p/interface";
import {Connection, type IdentifyResult, PeerId, PrivateKey} from "@libp2p/interface";
import {BitArray} from "@chainsafe/ssz";
import {BeaconConfig} from "@lodestar/config";
import {LoggerNode} from "@lodestar/logger/node";
Expand Down Expand Up @@ -47,6 +47,10 @@ const STATUS_INBOUND_GRACE_PERIOD = 15 * 1000;
const CHECK_PING_STATUS_INTERVAL = 10 * 1000;
/** A peer is considered long connection if it's >= 1 day */
const LONG_PEER_CONNECTION_MS = 24 * 60 * 60 * 1000;
/** Maximum number of identify attempts per peer */
const IDENTIFY_MAX_ATTEMPTS = 3;
/** Retry delays for identify attempts (ms). First attempt is immediate. */
const IDENTIFY_RETRY_DELAYS_MS = [0, 5_000, 15_000];

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The constants IDENTIFY_MAX_ATTEMPTS and IDENTIFY_RETRY_DELAYS_MS are tightly coupled, but this relationship is not enforced. A future change to one without updating the other could lead to runtime errors. Additionally, IDENTIFY_RETRY_DELAYS_MS[0] is unused in the retry logic, which can be confusing.

The suggested change derives the max attempts from the delays array and defines only the actual retry delays. This makes the configuration more robust.

Note: Applying this suggestion will require updating the identifyPeer function to use IDENTIFY_RETRY_DELAYS_MS[attempt - 1] for getting the delay, and IDENTIFY_RETRY_DELAYS_MS[attempt] for logging the next retry.

Suggested change
/** Maximum number of identify attempts per peer */
const IDENTIFY_MAX_ATTEMPTS = 3;
/** Retry delays for identify attempts (ms). First attempt is immediate. */
const IDENTIFY_RETRY_DELAYS_MS = [0, 5_000, 15_000];
/** Retry delays for identify attempts (ms). The length of the array determines the number of retries after the initial one. */
const IDENTIFY_RETRY_DELAYS_MS = [5_000, 15_000];
/** Maximum number of identify attempts per peer (1 initial + number of retries). */
const IDENTIFY_MAX_ATTEMPTS = 1 + IDENTIFY_RETRY_DELAYS_MS.length;

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point — deriving IDENTIFY_MAX_ATTEMPTS from the array eliminates the coupling. Will apply.

/** Ref https://github.com/ChainSafe/lodestar/issues/3423 */
const DEFAULT_DISCV5_FIRST_QUERY_DELAY_MS = 1000;
/**
Expand Down Expand Up @@ -162,6 +166,8 @@ export class PeerManager {

// A single map of connected peers with all necessary data to handle PINGs, STATUS, and metrics
private connectedPeers: Map<PeerIdStr, PeerData>;
/** Prevents spawning concurrent identify loops for the same peer */
private readonly identifyInProgress = new Set<PeerIdStr>();

private opts: PeerManagerOpts;
private intervals: NodeJS.Timeout[] = [];
Expand Down Expand Up @@ -193,6 +199,7 @@ export class PeerManager {

this.libp2p.services.components.events.addEventListener(Libp2pEvent.connectionOpen, this.onLibp2pPeerConnect);
this.libp2p.services.components.events.addEventListener(Libp2pEvent.connectionClose, this.onLibp2pPeerDisconnect);
this.libp2p.services.components.events.addEventListener("peer:identify", this.onPeerIdentify);
this.networkEventBus.on(NetworkEvent.reqRespRequest, this.onRequest);

this.lastStatus = this.statusCache.get();
Expand Down Expand Up @@ -236,6 +243,7 @@ export class PeerManager {
Libp2pEvent.connectionClose,
this.onLibp2pPeerDisconnect
);
this.libp2p.services.components.events.removeEventListener("peer:identify", this.onPeerIdentify);
this.networkEventBus.off(NetworkEvent.reqRespRequest, this.onRequest);
for (const interval of this.intervals) clearInterval(interval);
}
Expand Down Expand Up @@ -486,7 +494,15 @@ export class PeerManager {
// peers that close identify right after connection open or turn out to be
// irrelevant.
if (peerData?.agentVersion === null) {
void this.identifyPeer(peer.toString(), prettyPrintPeerId(peer), getConnection(this.libp2p, peer.toString()));
const peerIdStr = peer.toString();
if (!this.identifyInProgress.has(peerIdStr)) {
this.identifyInProgress.add(peerIdStr);
void this.identifyPeer(peerIdStr, prettyPrintPeerId(peer), getConnection(this.libp2p, peerIdStr)).finally(
() => {
this.identifyInProgress.delete(peerIdStr);
}
);
}
}
}
}
Expand Down Expand Up @@ -845,6 +861,7 @@ export class PeerManager {

// remove the ping and status timer for the peer
this.connectedPeers.delete(peerIdStr);
this.identifyInProgress.delete(peerIdStr);

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve identify lock until retry loop actually exits

Clearing identifyInProgress on disconnect allows a new identify loop to start before the previous retry coroutine has finished sleeping. If a peer disconnects and reconnects within the 5s/15s backoff window, onStatus can launch a second loop while the old one resumes after its timeout, so two identify retries run concurrently for the same peer and defeat the deduplication this change is trying to enforce.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Valid race condition in theory. In practice, the disconnect handler in onPeerDisconnected removes the peer from peers entirely, so a reconnect creates a fresh peer entry with identifyInProgress=false. The old coroutine would resume and hit peers.get(peerId) returning undefined → early exit. But I agree it's fragile — I'll remove the identifyInProgress cleanup from disconnect and let the coroutine's finally block handle it instead, which is more robust.


this.logger.verbose(logMessage, logContext);
this.networkEventBus.emit(NetworkEvent.peerDisconnected, {peer: peerIdStr});
Expand All @@ -862,24 +879,94 @@ export class PeerManager {
}
}

private async identifyPeer(peerIdStr: string, peerIdPretty: string, connection?: Connection): Promise<void> {
if (!connection || connection.status !== "open") {
this.logger.debug("Peer has no open connection for identify", {peerId: peerIdPretty});
return;
/**
* Handles the `peer:identify` event from libp2p.
* Fired when identify succeeds (either our request or a remote identify-push).
* Acts as a safety net to capture agentVersion even when our explicit identify call fails.
*/
private onPeerIdentify = (evt: CustomEvent<IdentifyResult>): void => {
const {peerId, agentVersion} = evt.detail;
if (!agentVersion) return;

const peerIdStr = peerId.toString();
const peerData = this.connectedPeers.get(peerIdStr);
if (peerData && peerData.agentVersion === null) {
peerData.agentVersion = agentVersion;
peerData.agentClient = getKnownClientFromAgentVersion(agentVersion);
this.identifyInProgress.delete(peerIdStr);
this.logger.debug("Updated agentVersion via peer:identify event", {
peer: prettyPrintPeerIdStr(peerIdStr),
agentVersion,
client: peerData.agentClient,
});
}
};

try {
const result = await this.libp2p.services.identify.identify(connection);
const agentVersion = result.agentVersion;
if (agentVersion) {
const connectedPeerData = this.connectedPeers.get(peerIdStr);
if (connectedPeerData) {
connectedPeerData.agentVersion = agentVersion;
connectedPeerData.agentClient = getKnownClientFromAgentVersion(agentVersion);
/**
* Identify a peer with retry logic. Retries on transient errors (EOF, stream closing)
* but gives up immediately on non-retryable errors (public key missing, peer mismatch).
*/
private async identifyPeer(peerIdStr: string, peerIdPretty: string, connection?: Connection): Promise<void> {
for (let attempt = 0; attempt < IDENTIFY_MAX_ATTEMPTS; attempt++) {
if (attempt > 0) {
// Check if peer still needs identification
const peerData = this.connectedPeers.get(peerIdStr);
if (!peerData || peerData.agentVersion !== null) return;

const delay = IDENTIFY_RETRY_DELAYS_MS[attempt];
if (delay > 0) {
await new Promise((resolve) => setTimeout(resolve, delay));
}

// Re-check after delay — peer may have disconnected or been identified via peer:identify event
const updatedPeerData = this.connectedPeers.get(peerIdStr);
if (!updatedPeerData || updatedPeerData.agentVersion !== null) return;

// Get a fresh connection reference
connection = getConnection(this.libp2p, peerIdStr);
}

if (!connection || connection.status !== "open") {
this.logger.debug("Peer has no open connection for identify", {peerId: peerIdPretty, attempt});
return; // No point retrying without a connection
}

try {
const result = await this.libp2p.services.identify.identify(connection);
const agentVersion = result.agentVersion;
if (agentVersion) {
const peerData = this.connectedPeers.get(peerIdStr);
if (peerData) {
peerData.agentVersion = agentVersion;
peerData.agentClient = getKnownClientFromAgentVersion(agentVersion);
}
}
return; // Success
} catch (e) {
const error = e as Error;
const errorMsg = error.message ?? "";

// Don't retry on non-transient errors
if (errorMsg.includes("Public key was missing") || errorMsg.includes("does not match the expected peer")) {
this.logger.debug("Identify failed with non-retryable error", {peerId: peerIdPretty, attempt}, error);
return;
}

if (attempt < IDENTIFY_MAX_ATTEMPTS - 1) {
this.logger.debug("Identify failed, scheduling retry", {
peerId: peerIdPretty,
attempt,
nextRetryMs: IDENTIFY_RETRY_DELAYS_MS[attempt + 1],
error: errorMsg,
});
} else {
this.logger.debug(
"Identify failed after all retries",
{peerId: peerIdPretty, attempts: IDENTIFY_MAX_ATTEMPTS},
error
);
}
}
} catch (e) {
this.logger.debug("Error setting agentVersion for the peer", {peerId: peerIdPretty}, e as Error);
}
}

Expand Down
74 changes: 73 additions & 1 deletion packages/beacon-node/test/e2e/network/peers/peerManager.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import {generateKeyPair} from "@libp2p/crypto/keys";
import {Connection} from "@libp2p/interface";
import {Connection, type IdentifyResult} from "@libp2p/interface";
import {afterEach, describe, expect, it, vi} from "vitest";
import {BitArray} from "@chainsafe/ssz";
import {createBeaconConfig} from "@lodestar/config";
Expand Down Expand Up @@ -342,4 +342,76 @@ describe("network / peers / PeerManager", () => {
expect(peerData?.agentVersion).toBe("Nimbus/v25.0.0");
expect(peerData?.agentClient).toBe(ClientKind.Nimbus);
});

it("Should retry identify on transient errors", async () => {
vi.useFakeTimers();
try {
const {libp2p, peerManager, statusCache, networkEventBus} = await mockModules();

const identifySpy = vi
.spyOn(libp2p.services.identify, "identify")
.mockRejectedValueOnce(new Error("Cannot write to a stream that is closing"))
.mockResolvedValue({agentVersion: "Lighthouse/v6.0.1"} as Awaited<ReturnType<typeof libp2p.services.identify.identify>>);

const inboundConnection = {
id: "connection-1",
direction: "inbound",
status: "open",
remotePeer: peerId1,
close: async () => {},
abort: () => {},
} as unknown as Connection;

getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [inboundConnection]});
await peerManager["onLibp2pPeerConnect"](new CustomEvent("evt", {detail: inboundConnection}));

const remoteStatus = statusCache.get();
networkEventBus.emit(NetworkEvent.reqRespRequest, {
request: {method: ReqRespMethod.Status, body: remoteStatus},
peer: peerId1,
peerClient: "Unknown",
});

// 1st attempt happens immediately, 2nd after 5s retry delay
await vi.advanceTimersByTimeAsync(5_000);

expect(identifySpy).toHaveBeenCalledTimes(2);
const peerData = peerManager["connectedPeers"].get(peerId1.toString());
expect(peerData?.agentVersion).toBe("Lighthouse/v6.0.1");
expect(peerData?.agentClient).toBe(ClientKind.Lighthouse);
} finally {
vi.useRealTimers();
}
});

it("Should update agentVersion via peer:identify event", async () => {
const {libp2p, peerManager} = await mockModules();

// Simulate connected peer with unknown agentVersion
const inboundConnection = {
id: "connection-1",
direction: "inbound",
status: "open",
remotePeer: peerId1,
close: async () => {},
abort: () => {},
} as unknown as Connection;

getConnectionsMap(libp2p).set(peerId1.toString(), {key: peerId1, value: [inboundConnection]});
await peerManager["onLibp2pPeerConnect"](new CustomEvent("evt", {detail: inboundConnection}));

// Simulate libp2p identify event (e.g. via identify-push)
libp2p.services.components.events.dispatchEvent(
new CustomEvent("peer:identify", {
detail: {
peerId: peerId1,
agentVersion: "Nimbus/v25.0.0",
} satisfies Partial<IdentifyResult>,
})
);

const peerData = peerManager["connectedPeers"].get(peerId1.toString());
expect(peerData?.agentVersion).toBe("Nimbus/v25.0.0");
expect(peerData?.agentClient).toBe(ClientKind.Nimbus);
});
});
Loading