From 6045d6f490c0923c3d1c525cd985118d7c79548e Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Tue, 17 Feb 2026 20:01:34 -0800 Subject: [PATCH 01/14] mailbox/client: introduce Store interface for durable responses In this commit, we introduce a Store abstraction that the mailbox RPC client uses to persist response payloads and the pull cursor across restarts. The interface defines five operations: LoadCursor, SaveCursor, PutResponse, GetResponse, and DeleteResponse, with idempotency and monotonicity contracts that match the cursor-based AckUpTo protocol. A MemoryStore implementation is included for tests and short-lived processes that don't need crash safety. The Config struct gains a Store field that defaults to MemoryStore when unset, preserving backward compatibility with existing callers. --- mailbox/client/config.go | 5 ++ mailbox/client/doc.go | 17 +++- mailbox/client/store.go | 164 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 mailbox/client/store.go diff --git a/mailbox/client/config.go b/mailbox/client/config.go index 49c533088..a558760e4 100644 --- a/mailbox/client/config.go +++ b/mailbox/client/config.go @@ -11,6 +11,11 @@ type Config struct { // Edge is the gRPC client for the mailbox edge service. Edge mailboxpb.MailboxServiceClient + // Store persists response payloads and pull cursor state. + // + // If unset, the client uses an in-memory store (not crash-safe). + Store Store + // LocalMailboxID is the mailbox id used to receive responses. LocalMailboxID string diff --git a/mailbox/client/doc.go b/mailbox/client/doc.go index 2d85e97db..223f65262 100644 --- a/mailbox/client/doc.go +++ b/mailbox/client/doc.go @@ -9,7 +9,18 @@ // responses by correlation id before advancing the remote cursor. // // This package is intentionally small and self-contained so it can be used in -// both production code and tests. It does not attempt to implement the full -// “local durability ↔ remote mailbox” connector described in the spec; that -// integration belongs at a higher layer that has access to a durable store. +// both production code and tests. +// +// For crash safety, the Client can be configured with a Store implementation +// that persists: +// - the remote Pull cursor, and +// - response payloads keyed by correlation id. +// +// This is sufficient to avoid response loss when using cursor-based acking, +// even across restarts, as long as callers reuse correlation ids when retrying. +// +// This package does not attempt to implement the full “local durability ↔ +// remote mailbox” connector described in the spec (for example, integrating a +// transactional outbox with a local FSM store). That integration belongs at a +// higher layer that has access to the application's durability boundaries. package mailboxclient diff --git a/mailbox/client/store.go b/mailbox/client/store.go new file mode 100644 index 000000000..e0f705bfc --- /dev/null +++ b/mailbox/client/store.go @@ -0,0 +1,164 @@ +package mailboxclient + +import ( + "context" + "sync" +) + +// Store persists state needed for crash-safe RPC-over-mailbox operation. +// +// Callers that want crash safety should use a durable Store implementation and +// ensure correlation IDs are stable across retries (for example, by reusing the +// RPC idempotency key as the correlation id). +type Store interface { + // LoadCursor returns the persisted Pull cursor for mailboxID. + LoadCursor(ctx context.Context, mailboxID string) (uint64, error) + + // SaveCursor persists the Pull cursor for mailboxID. + // + // Implementations SHOULD treat cursor as monotonic and MUST NOT move it + // backward. + SaveCursor(ctx context.Context, mailboxID string, cursor uint64) error + + // PutResponse records a response payload for correlationID. + // + // payload is the raw protobuf message bytes stored in an Any.Value. + // + // PutResponse MUST be idempotent for the same mailboxID and + // correlationID. + // + // It SHOULD keep the first successfully stored payload. + PutResponse(ctx context.Context, mailboxID string, correlationID string, + payload []byte) error + + // GetResponse returns a previously recorded response payload. + GetResponse(ctx context.Context, mailboxID string, + correlationID string) (payload []byte, ok bool, err error) + + // DeleteResponse removes a previously recorded response payload. + DeleteResponse(ctx context.Context, mailboxID string, + correlationID string) error +} + +// MemoryStore is an in-memory Store implementation. +// +// It is useful for tests and short-lived processes, but it is not crash-safe. +type MemoryStore struct { + mu sync.Mutex + + cursors map[string]uint64 + responses map[string]map[string][]byte +} + +// NewMemoryStore constructs an empty in-memory store. +func NewMemoryStore() *MemoryStore { + return &MemoryStore{ + cursors: make(map[string]uint64), + responses: make(map[string]map[string][]byte), + } +} + +// LoadCursor returns the saved cursor for mailboxID. +func (s *MemoryStore) LoadCursor(ctx context.Context, mailboxID string) ( + uint64, error) { + + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + return s.cursors[mailboxID], nil +} + +// SaveCursor stores cursor for mailboxID. +func (s *MemoryStore) SaveCursor(ctx context.Context, mailboxID string, + cursor uint64) error { + + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + old := s.cursors[mailboxID] + if cursor < old { + return nil + } + + s.cursors[mailboxID] = cursor + + return nil +} + +// PutResponse stores payload for correlationID if it doesn't already exist. +func (s *MemoryStore) PutResponse(ctx context.Context, mailboxID string, + correlationID string, payload []byte) error { + + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + byMailbox, ok := s.responses[mailboxID] + if !ok { + byMailbox = make(map[string][]byte) + s.responses[mailboxID] = byMailbox + } + + if _, exists := byMailbox[correlationID]; exists { + return nil + } + + payloadCopy := make([]byte, len(payload)) + copy(payloadCopy, payload) + + byMailbox[correlationID] = payloadCopy + + return nil +} + +// GetResponse returns payload for correlationID if present. +func (s *MemoryStore) GetResponse(ctx context.Context, mailboxID string, + correlationID string) ([]byte, bool, error) { + + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + byMailbox, ok := s.responses[mailboxID] + if !ok { + return nil, false, nil + } + + payload, ok := byMailbox[correlationID] + if !ok { + return nil, false, nil + } + + payloadCopy := make([]byte, len(payload)) + copy(payloadCopy, payload) + + return payloadCopy, true, nil +} + +// DeleteResponse removes payload for correlationID if present. +func (s *MemoryStore) DeleteResponse(ctx context.Context, mailboxID string, + correlationID string) error { + + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + byMailbox, ok := s.responses[mailboxID] + if !ok { + return nil + } + + delete(byMailbox, correlationID) + if len(byMailbox) == 0 { + delete(s.responses, mailboxID) + } + + return nil +} From dae1a6b97af39f18fca4406ae8e7925c4b0cc6c0 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Tue, 17 Feb 2026 20:01:52 -0800 Subject: [PATCH 02/14] mailbox/client: wire durable store into pull-ack loop With the Store interface in place, we now integrate it into the client's main run loop to achieve crash-safe response handling under the cursor-based AckUpTo protocol. On startup the client loads the persisted cursor so it resumes pulling from where it left off. Each inbound response envelope is written to the Store before any in-memory waiter is notified, ensuring the payload survives a crash between dispatch and consumption. The cursor is persisted to the Store before acking with the remote mailbox, so a restart replays only unacked envelopes rather than the entire stream. AwaitRPC now peeks through the in-memory pending map into the Store, and only deletes the response from both layers after successful proto unmarshal. A best-effort retry loop handles transient delete failures without blocking the caller. This two-tier lookup means responses are available even if the process restarted between PutResponse and the caller's AwaitRPC. The malformed response body error is extracted to a package-level format string so it can be matched in tests. --- mailbox/client/client.go | 178 +++++++++++++++++++++++++++++++++------ 1 file changed, 154 insertions(+), 24 deletions(-) diff --git a/mailbox/client/client.go b/mailbox/client/client.go index 145c88af4..51c1690e9 100644 --- a/mailbox/client/client.go +++ b/mailbox/client/client.go @@ -15,6 +15,8 @@ import ( "google.golang.org/protobuf/types/known/anypb" ) +const errMalformedResponseBody = "failed to unmarshal response body: %w" + // Client implements mailboxrpc.RPCClient by sending and receiving mailbox // envelopes through a mailboxpb.MailboxServiceClient. type Client struct { @@ -26,6 +28,7 @@ type Client struct { mu sync.Mutex cursor uint64 + ackTo uint64 pending map[string][]byte waiters map[string][]chan struct{} @@ -53,11 +56,20 @@ func New(cfg Config) (*Client, error) { ctx, cancel := context.WithCancel(context.Background()) + cursor, err := cfg.Store.LoadCursor(ctx, cfg.LocalMailboxID) + if err != nil { + cancel() + return nil, fmt.Errorf("load cursor: %w", err) + } + c := &Client{ cfg: cfg, cancel: cancel, + cursor: cursor, + ackTo: cursor, + pending: make(map[string][]byte), waiters: make(map[string][]chan struct{}), } @@ -179,11 +191,24 @@ func (c *Client) AwaitRPC(ctx context.Context, correlationID string, resp proto.Message) error { for { - data, ok := c.popPending(correlationID) + data, ok, err := c.peekResponse(ctx, correlationID) + if err != nil { + return err + } if ok { - return (proto.UnmarshalOptions{ + err := (proto.UnmarshalOptions{ DiscardUnknown: true, }).Unmarshal(data, resp) + if err != nil { + return fmt.Errorf(errMalformedResponseBody, err) + } + + c.deletePending(correlationID) + c.deleteResponseBestEffort( + ctx, c.cfg.LocalMailboxID, correlationID, + ) + + return nil } ch := c.addWaiter(correlationID) @@ -206,6 +231,9 @@ func applyDefaults(cfg Config) Config { if cfg.PullWaitTimeout == 0 { cfg.PullWaitTimeout = def.PullWaitTimeout } + if cfg.Store == nil { + cfg.Store = NewMemoryStore() + } return cfg } @@ -224,6 +252,16 @@ func (c *Client) run(ctx context.Context) { default: } + ackTo := c.loadAckTo() + if ackTo != 0 { + if !c.ackUpTo(ctx, ackTo) { + c.sleepRetry(ctx) + continue + } + + c.clearAckTo(ackTo) + } + cursor := c.loadCursor() waitMs := uint32(c.cfg.PullWaitTimeout.Milliseconds()) @@ -252,21 +290,28 @@ func (c *Client) run(ctx context.Context) { slog.Uint64("next_cursor", resp.NextCursor)) } + var handleErr error for _, env := range resp.Envelopes { - c.handleEnvelope(env) + if err := c.handleEnvelope(ctx, env); err != nil { + handleErr = err + break + } + } + + if handleErr != nil { + c.sleepRetry(ctx) + continue } if resp.NextCursor > cursor { - ackOK := c.ackUpTo(ctx, resp.NextCursor) - if ackOK { - c.storeCursor(resp.NextCursor) + if err := c.storeCursor( + ctx, resp.NextCursor, + ); err != nil { + c.sleepRetry(ctx) continue } - log.DebugS(ctx, "AckUpTo failed, retrying", - slog.Uint64("cursor", resp.NextCursor)) - - c.sleepRetry(ctx) + c.setAckTo(resp.NextCursor) } } } @@ -296,22 +341,34 @@ func (c *Client) ackUpTo(ctx context.Context, cursor uint64) bool { } // handleEnvelope caches correlated responses and wakes waiters. -func (c *Client) handleEnvelope(env *mailboxpb.Envelope) { +func (c *Client) handleEnvelope(ctx context.Context, + env *mailboxpb.Envelope) error { + if env == nil || env.Rpc == nil { - return + return nil } if env.Rpc.Kind != mailboxpb.RpcMeta_KIND_RESPONSE { - return + return nil } correlationID := env.Rpc.CorrelationId if correlationID == "" { - return + return nil } if env.Body == nil { - return + return nil + } + + payload := env.Body.Value + payloadCopy := make([]byte, len(payload)) + copy(payloadCopy, payload) + + if err := c.cfg.Store.PutResponse( + ctx, c.cfg.LocalMailboxID, correlationID, payloadCopy, + ); err != nil { + return err } c.mu.Lock() @@ -320,10 +377,6 @@ func (c *Client) handleEnvelope(env *mailboxpb.Envelope) { // If we already have a response for this correlation id, keep the // first response and ignore duplicates. if _, exists := c.pending[correlationID]; !exists { - payload := env.Body.Value - payloadCopy := make([]byte, len(payload)) - copy(payloadCopy, payload) - c.pending[correlationID] = payloadCopy log.DebugS(context.TODO(), "Cached response", @@ -339,10 +392,12 @@ func (c *Client) handleEnvelope(env *mailboxpb.Envelope) { close(ch) } delete(c.waiters, correlationID) + + return nil } -// popPending returns and removes a cached response for correlationID. -func (c *Client) popPending(correlationID string) ([]byte, bool) { +// peekPending returns a cached response payload for correlationID. +func (c *Client) peekPending(correlationID string) ([]byte, bool) { c.mu.Lock() defer c.mu.Unlock() @@ -351,9 +406,51 @@ func (c *Client) popPending(correlationID string) ([]byte, bool) { return nil, false } + dataCopy := make([]byte, len(data)) + copy(dataCopy, data) + + return dataCopy, true +} + +func (c *Client) deletePending(correlationID string) { + c.mu.Lock() + defer c.mu.Unlock() + delete(c.pending, correlationID) +} + +func (c *Client) peekResponse(ctx context.Context, + correlationID string) ([]byte, bool, error) { + + if data, ok := c.peekPending(correlationID); ok { + return data, true, nil + } - return data, true + return c.cfg.Store.GetResponse(ctx, c.cfg.LocalMailboxID, correlationID) +} + +func (c *Client) deleteResponseBestEffort(ctx context.Context, mailboxID string, + correlationID string) { + + const maxAttempts = 3 + backoff := 50 * time.Millisecond + + for i := 0; i < maxAttempts; i++ { + err := c.cfg.Store.DeleteResponse(ctx, mailboxID, correlationID) + if err == nil { + return + } + + timer := time.NewTimer(backoff) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + + backoff *= 2 + } } // addWaiter registers a waiter for correlationID and returns its channel. @@ -398,12 +495,45 @@ func (c *Client) loadCursor() uint64 { return c.cursor } -// storeCursor sets the pull cursor. -func (c *Client) storeCursor(cursor uint64) { +func (c *Client) loadAckTo() uint64 { + c.mu.Lock() + defer c.mu.Unlock() + + return c.ackTo +} + +func (c *Client) setAckTo(cursor uint64) { + c.mu.Lock() + defer c.mu.Unlock() + + if cursor > c.ackTo { + c.ackTo = cursor + } +} + +func (c *Client) clearAckTo(cursor uint64) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.ackTo == cursor { + c.ackTo = 0 + } +} + +// storeCursor persists and sets the pull cursor. +func (c *Client) storeCursor(ctx context.Context, cursor uint64) error { + if err := c.cfg.Store.SaveCursor( + ctx, c.cfg.LocalMailboxID, cursor, + ); err != nil { + return err + } + c.mu.Lock() defer c.mu.Unlock() c.cursor = cursor + + return nil } // randomID generates an opaque id backed by crypto/rand. From cb1a94353f7f7373a08bcfa131fd50993ff2b9b4 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Tue, 17 Feb 2026 20:02:06 -0800 Subject: [PATCH 03/14] serverconn: add connector types, ack state machine, and logging This commit introduces the foundational types for the serverconn connector, which will serve as the unified boundary for all mailbox traffic between the client and the remote server. The AckState struct models the four-cursor state machine that governs the pull-dispatch-ack cycle: PullCursor tracks where to resume pulling, DispatchCommittedTo records the last durably dispatched position, AckTarget holds the cursor to send in the next AckUpTo call, and AckCommittedTo tracks what the remote has actually acknowledged. The state machine enforces the critical invariant that AckUpTo only advances after durable local dispatch, preventing message loss on crash. TLV encode/decode methods allow the state to be persisted via the durable actor checkpoint mechanism. Also included are the ConnectorConfig struct with all dependency injection points (Edge client, dispatch table, delivery store, pull tuning knobs), named types for CorrelationID and IdempotencyKey, the EnvelopeDispatcher function type for ServiceKey-based routing, and the ResponseWaiter struct for in-memory unary RPC correlation. --- serverconn/log.go | 24 +++++ serverconn/types.go | 231 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 255 insertions(+) create mode 100644 serverconn/log.go create mode 100644 serverconn/types.go diff --git a/serverconn/log.go b/serverconn/log.go new file mode 100644 index 000000000..b2240be3b --- /dev/null +++ b/serverconn/log.go @@ -0,0 +1,24 @@ +package serverconn + +import "github.com/btcsuite/btclog/v2" + +// Subsystem defines the logging code for this subsystem. +const Subsystem = "SVRC" + +// log is a logger that is initialized with no output filters. This means the +// package will not perform any logging by default until the caller requests +// it. +var log = btclog.Disabled + +// DisableLog disables all library log output. Logging output is disabled by +// default until UseLogger is called. +func DisableLog() { + UseLogger(btclog.Disabled) +} + +// UseLogger uses a specified Logger to output package logging info. This +// should be used in preference to SetLogWriter if the caller is also using +// btclog. +func UseLogger(logger btclog.Logger) { + log = logger +} diff --git a/serverconn/types.go b/serverconn/types.go new file mode 100644 index 000000000..db0e5d932 --- /dev/null +++ b/serverconn/types.go @@ -0,0 +1,231 @@ +package serverconn + +import ( + "context" + "io" + "time" + + "github.com/lightninglabs/darepo-client/baselib/actor" + mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" + mailboxrpc "github.com/lightninglabs/darepo-client/mailbox/rpc" + "github.com/lightningnetwork/lnd/tlv" +) + +// CorrelationID is an opaque identifier linking a mailbox request to its +// response. Using a named type prevents accidental string swaps with other +// identifiers. +type CorrelationID string + +// IdempotencyKey is a stable key for deduplicating semantic operations across +// retries. Two sends with the same idempotency key are treated as the same +// logical operation by the remote mailbox edge. +type IdempotencyKey string + +// ackStateType is the checkpoint state type used when persisting the ack +// watermark to the delivery store. +const ackStateType = "AckState" + +// TLV record type constants for AckState checkpoint serialization. +const ( + pullCursorRecordType tlv.Type = 1 + dispatchCommittedToRecordType tlv.Type = 2 + ackTargetRecordType tlv.Type = 3 + ackCommittedToRecordType tlv.Type = 4 +) + +// AckState tracks the four cursor variables that govern safe ack progression. +// All fields are monotonic — they never decrease during normal operation. +// +// The state machine enforces the invariant: +// +// ack_committed_to <= dispatch_committed_to +// +// Cursor never advances past non-durable local work. Repeated acks are safe +// and idempotent. +type AckState struct { + // PullCursor is the cursor for the next Pull call. After a successful + // ack, this advances to at least the acked position. + PullCursor uint64 + + // DispatchCommittedTo is the max cursor whose envelopes have been + // durably committed to local actor mailboxes via Tell. + DispatchCommittedTo uint64 + + // AckTarget is the max cursor that should be acked remotely. This is + // always >= DispatchCommittedTo. + AckTarget uint64 + + // AckCommittedTo is the last cursor successfully acked to the remote + // mailbox edge. + AckCommittedTo uint64 +} + +// AdvanceDispatch updates the state after a successful durable dispatch +// through nextCursor. The ack target is advanced to match the dispatch +// frontier. +func (s *AckState) AdvanceDispatch(nextCursor uint64) { + if nextCursor > s.DispatchCommittedTo { + s.DispatchCommittedTo = nextCursor + } + + if s.DispatchCommittedTo > s.AckTarget { + s.AckTarget = s.DispatchCommittedTo + } +} + +// AdvanceAck updates the state after a successful AckUpTo call. The pull +// cursor advances to at least the acked position so that subsequent pulls +// do not re-fetch already-acked envelopes. +func (s *AckState) AdvanceAck() { + s.AckCommittedTo = s.AckTarget + + if s.AckCommittedTo > s.PullCursor { + s.PullCursor = s.AckCommittedTo + } +} + +// NeedsAck returns true when there is an un-acked committed dispatch. This +// means AckTarget has advanced past AckCommittedTo and a remote AckUpTo call +// is needed. +func (s *AckState) NeedsAck() bool { + return s.AckTarget > s.AckCommittedTo +} + +// Encode serializes the AckState to the provided writer as a TLV stream. +func (s *AckState) Encode(w io.Writer) error { + records := []tlv.Record{ + tlv.MakePrimitiveRecord( + pullCursorRecordType, &s.PullCursor, + ), + tlv.MakePrimitiveRecord( + dispatchCommittedToRecordType, + &s.DispatchCommittedTo, + ), + tlv.MakePrimitiveRecord( + ackTargetRecordType, &s.AckTarget, + ), + tlv.MakePrimitiveRecord( + ackCommittedToRecordType, &s.AckCommittedTo, + ), + } + + stream, err := tlv.NewStream(records...) + if err != nil { + return err + } + + return stream.Encode(w) +} + +// Decode deserializes the AckState from the provided reader. +func (s *AckState) Decode(r io.Reader) error { + records := []tlv.Record{ + tlv.MakePrimitiveRecord( + pullCursorRecordType, &s.PullCursor, + ), + tlv.MakePrimitiveRecord( + dispatchCommittedToRecordType, + &s.DispatchCommittedTo, + ), + tlv.MakePrimitiveRecord( + ackTargetRecordType, &s.AckTarget, + ), + tlv.MakePrimitiveRecord( + ackCommittedToRecordType, &s.AckCommittedTo, + ), + } + + stream, err := tlv.NewStream(records...) + if err != nil { + return err + } + + _, err = stream.DecodeWithParsedTypes(r) + + return err +} + +// EnvelopeDispatcher routes an inbound envelope to the correct local actor. +// A nil error means the envelope was durably committed to the target actor's +// mailbox (i.e., DurableActor.Tell returned nil, confirming persistence). +// The dispatcher is a closure configured at wiring time that captures a +// ServiceKey reference for the target actor. +type EnvelopeDispatcher func( + ctx context.Context, env *mailboxpb.Envelope, +) error + +// ResponseWaiter is registered by unary facade callers so the ingress loop +// can deliver KIND_RESPONSE envelopes without actor dispatch. The channel +// has buffer size 1 to prevent the ingress loop from blocking. +type ResponseWaiter struct { + // Ch receives the response envelope from the ingress loop. + Ch chan *mailboxpb.Envelope + + // Created records when the waiter was registered, for diagnostics + // and stale waiter cleanup. + Created time.Time +} + +// ConnectorConfig holds all dependencies and tuning knobs for the server +// connection actor. The connector is the single boundary for all mailbox +// traffic between the client and the remote server. +type ConnectorConfig struct { + // Edge is the gRPC client for the remote mailbox edge service, + // providing Send, Pull, and AckUpTo operations. + Edge mailboxpb.MailboxServiceClient + + // LocalMailboxID is this client's mailbox identifier. Inbound + // envelopes are pulled from this mailbox, and it is set as the + // sender on outbound envelopes. + LocalMailboxID string + + // RemoteMailboxID is the remote server's mailbox identifier. Outbound + // envelopes are addressed to this mailbox. + RemoteMailboxID string + + // ProtocolVersion is the protocol version stamped on outbound + // envelopes. + ProtocolVersion uint32 + + // Dispatchers maps (service, method) pairs to envelope dispatchers. + // The ingress loop uses this table to route KIND_REQUEST and + // KIND_EVENT envelopes to the correct local actor via ServiceKey. + Dispatchers map[mailboxrpc.ServiceMethod]EnvelopeDispatcher + + // Store is the delivery store used by both the durable actor runtime + // (for inbox persistence) and checkpoint persistence (for ack + // watermark state). This is the single durability source of truth. + Store actor.DeliveryStore + + // Codec handles TLV serialization of ServerConnMsg types for the + // durable actor mailbox. + Codec *actor.MessageCodec + + // PullMaxEnvelopes bounds the number of envelopes returned per Pull + // call. + PullMaxEnvelopes uint32 + + // PullWaitTimeout is the long-poll timeout for Pull calls. The remote + // edge will hold the connection open for this duration before + // returning an empty response. + PullWaitTimeout time.Duration + + // RetryBaseDelay is the base delay for exponential backoff on + // transient failures (pull, ack, dispatch). + RetryBaseDelay time.Duration + + // RetryMaxDelay caps the exponential backoff delay. + RetryMaxDelay time.Duration +} + +// DefaultConnectorConfig returns a ConnectorConfig with sensible defaults for +// polling and retry behavior. The caller must still set Edge, mailbox IDs, +// and Store. Codec is optional — NewRuntime fills a default. +func DefaultConnectorConfig() ConnectorConfig { + return ConnectorConfig{ + PullMaxEnvelopes: 50, + PullWaitTimeout: 5 * time.Second, + RetryBaseDelay: 200 * time.Millisecond, + RetryMaxDelay: 30 * time.Second, + } +} From 43b6c8eea300baa79a04ccc7365dc09b9b7bd743 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Tue, 17 Feb 2026 20:02:20 -0800 Subject: [PATCH 04/14] serverconn: expand actor with TLV messages and egress handlers The ServerConnectionActor is transformed from a stub relay into the unified connector boundary described in the architecture plan. Message types gain full TLV serialization so they can be persisted in the durable actor mailbox. SendClientEventRequest wraps the outbound proto in an anypb.Any to preserve type information across the encode/decode boundary, using the global protobuf type registry for reconstruction. SendRPCRequest serializes the entire mailbox Envelope via proto.Marshal for the unary facade path. Both message types carry stable TLV type identifiers (2000, 2001) for the durable actor codec. The actor struct gains a response registry (correlation ID to waiter channel map) for unary RPC response routing, protected by a dedicated mutex separate from the actor's mailbox processing. Egress handlers now build proper mailbox envelopes with protocol version, sender/ recipient addressing, and RPC metadata, sending them through the configured Edge client. A MessageCodec factory registers both message types for the durable actor runtime. The ingress loop lifecycle is managed via StartIngress/StopIngress, running as a background goroutine independent of the actor mailbox. --- serverconn/actor.go | 475 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 437 insertions(+), 38 deletions(-) diff --git a/serverconn/actor.go b/serverconn/actor.go index b97cd90e0..943f10574 100644 --- a/serverconn/actor.go +++ b/serverconn/actor.go @@ -3,10 +3,36 @@ package serverconn import ( "context" "fmt" + "io" + "log/slog" + "sync" + "time" "github.com/lightninglabs/darepo-client/baselib/actor" + mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" fn "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/tlv" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +// TLV type constants for server connection messages. These are stable +// identifiers used for message serialization and dispatch within the durable +// actor mailbox. +const ( + // SendClientEventRequestMsgType is the TLV type for outbound FSM + // events from the round actor to the server. + SendClientEventRequestMsgType tlv.Type = 2000 + + // SendRPCRequestMsgType is the TLV type for outbound unary RPC + // envelopes from the unary facade. + SendRPCRequestMsgType tlv.Type = 2001 +) + +// TLV record type constants for message field serialization. +const ( + protoPayloadRecordType tlv.Type = 1 + envelopeRecordType tlv.Type = 2 ) // ServerMessage is an interface that client FSM outbox messages must implement @@ -18,11 +44,37 @@ type ServerMessage interface { ToProto() proto.Message } +// rawServerMessage wraps a protobuf Any for reconstructing a ServerMessage +// after TLV deserialization. The original concrete type is recovered using +// the global protobuf type registry via anypb.UnmarshalNew. +type rawServerMessage struct { + anyMsg *anypb.Any +} + +// ToProto reconstructs the original proto message from the stored Any +// wrapper. Returns nil if the type cannot be resolved from the global +// protobuf registry. +func (m *rawServerMessage) ToProto() proto.Message { + msg, err := m.anyMsg.UnmarshalNew() + if err != nil { + log.ErrorS(context.Background(), + "Failed to unmarshal Any type from registry", + err, + slog.String("type_url", + m.anyMsg.GetTypeUrl())) + + return nil + } + + return msg +} + // ServerConnMsg is the sealed interface for messages that can be sent to the // ServerConnectionActor. These are typically FSM outbox messages from the -// client that need to be relayed to the server. +// client that need to be relayed to the server. The interface extends +// TLVMessage for durable actor mailbox persistence. type ServerConnMsg interface { - actor.Message + actor.TLVMessage serverConnMsgSealed() } @@ -35,7 +87,7 @@ type ServerConnResp interface { // SendClientEventRequest wraps a client FSM outbox message and requests it be // sent to the server. The actor will convert it to the appropriate proto -// message and send via gRPC. +// message and send via the mailbox edge. type SendClientEventRequest struct { actor.BaseMessage @@ -45,50 +97,216 @@ type SendClientEventRequest struct { Message ServerMessage } +// MessageType returns a human-readable type name for logging. func (m *SendClientEventRequest) MessageType() string { return "SendClientEventRequest" } +// TLVType returns the unique TLV type identifier for this message. +func (m *SendClientEventRequest) TLVType() tlv.Type { + return SendClientEventRequestMsgType +} + +// Encode serializes the message to the provided writer. The ServerMessage is +// converted to proto, wrapped in anypb.Any (preserving type information), +// and marshaled to bytes for TLV storage. +func (m *SendClientEventRequest) Encode(w io.Writer) error { + anyMsg, err := anypb.New(m.Message.ToProto()) + if err != nil { + return fmt.Errorf("wrap proto in Any: %w", err) + } + + anyBytes, err := proto.Marshal(anyMsg) + if err != nil { + return fmt.Errorf("marshal Any: %w", err) + } + + records := []tlv.Record{ + tlv.MakePrimitiveRecord(protoPayloadRecordType, &anyBytes), + } + + stream, err := tlv.NewStream(records...) + if err != nil { + return err + } + + return stream.Encode(w) +} + +// Decode deserializes the message from the provided reader. The proto payload +// is stored as a rawServerMessage that lazily unmarshals via the global +// protobuf type registry. +func (m *SendClientEventRequest) Decode(r io.Reader) error { + var payload []byte + + records := []tlv.Record{ + tlv.MakePrimitiveRecord(protoPayloadRecordType, &payload), + } + + stream, err := tlv.NewStream(records...) + if err != nil { + return err + } + + if _, err := stream.DecodeWithParsedTypes(r); err != nil { + return err + } + + anyMsg := &anypb.Any{} + if err := proto.Unmarshal(payload, anyMsg); err != nil { + return fmt.Errorf("unmarshal Any: %w", err) + } + + m.Message = &rawServerMessage{anyMsg: anyMsg} + + return nil +} + +// serverConnMsgSealed implements the ServerConnMsg interface seal. func (m *SendClientEventRequest) serverConnMsgSealed() {} // SendClientEventResponse acknowledges that the message was sent. type SendClientEventResponse struct { actor.BaseMessage + // Success indicates whether the send operation succeeded. Success bool - Error string + + // Error contains the error message if the send failed. + Error string } +// MessageType returns a human-readable type name for logging. func (m *SendClientEventResponse) MessageType() string { return "SendClientEventResponse" } +// serverConnRespSealed implements the ServerConnResp interface seal. func (m *SendClientEventResponse) serverConnRespSealed() {} -// ServerConnectionActor is a simple relay actor that accepts client FSM outbox -// messages and sends them to the server via gRPC or other transport. This -// decouples the client FSM from the transport layer. +// SendRPCRequest wraps a pre-built outbound unary RPC envelope. The unary +// facade constructs the envelope with all metadata (correlation ID, +// idempotency key, service/method) and hands it to the connector for +// transport via Edge.Send. +type SendRPCRequest struct { + actor.BaseMessage + + // Envelope is the pre-built mailbox envelope ready for sending. + Envelope *mailboxpb.Envelope +} + +// MessageType returns a human-readable type name for logging. +func (m *SendRPCRequest) MessageType() string { + return "SendRPCRequest" +} + +// TLVType returns the unique TLV type identifier for this message. +func (m *SendRPCRequest) TLVType() tlv.Type { + return SendRPCRequestMsgType +} + +// Encode serializes the message to the provided writer. The entire mailbox +// envelope is proto-marshaled for TLV storage. +func (m *SendRPCRequest) Encode(w io.Writer) error { + envBytes, err := proto.Marshal(m.Envelope) + if err != nil { + return fmt.Errorf("marshal envelope: %w", err) + } + + records := []tlv.Record{ + tlv.MakePrimitiveRecord(envelopeRecordType, &envBytes), + } + + stream, err := tlv.NewStream(records...) + if err != nil { + return err + } + + return stream.Encode(w) +} + +// Decode deserializes the message from the provided reader. +func (m *SendRPCRequest) Decode(r io.Reader) error { + var envBytes []byte + + records := []tlv.Record{ + tlv.MakePrimitiveRecord(envelopeRecordType, &envBytes), + } + + stream, err := tlv.NewStream(records...) + if err != nil { + return err + } + + if _, err := stream.DecodeWithParsedTypes(r); err != nil { + return err + } + + m.Envelope = &mailboxpb.Envelope{} + if err := proto.Unmarshal(envBytes, m.Envelope); err != nil { + return fmt.Errorf("unmarshal envelope: %w", err) + } + + return nil +} + +// serverConnMsgSealed implements the ServerConnMsg interface seal. +func (m *SendRPCRequest) serverConnMsgSealed() {} + +// ServerConnectionActor is the unified connector boundary for all mailbox +// traffic between the client and the remote server. It serves as both: +// +// 1. An egress actor: receives outbound messages from the round actor (FSM +// events) and unary facade (RPC requests), then sends them via the +// mailbox edge. // -// The actor maintains a connection to the server and handles: -// - Converting FSM events to proto messages. -// - Sending messages over gRPC. -// - Handling connection failures and retries (TODO). -// - Notifying the client actor of server responses (TODO). +// 2. An ingress loop: continuously pulls envelopes from the remote mailbox, +// dispatches them to local actors via ServiceKey-based routing, and +// manages the ack watermark state machine to ensure at-least-once +// delivery with crash safety. +// +// The actor is backed by a DurableActor for crash-safe egress. Outbound +// messages from the round actor persist in the durable mailbox before +// processing, ensuring no message loss on crashes. type ServerConnectionActor struct { - // TODO: Add gRPC client connection - // grpcClient pb.ArkServiceClient + // cfg holds all dependencies and tuning knobs for the connector. + cfg ConnectorConfig + + // responseRegistryMu protects concurrent access to the response + // registry from the ingress loop and unary facade callers. + responseRegistryMu sync.Mutex + + // responseRegistry maps correlation IDs to unary RPC waiters. The + // ingress loop delivers KIND_RESPONSE envelopes to the appropriate + // waiter channel. This is in-memory only — if the process crashes, + // callers' contexts are cancelled and they retry. + responseRegistry map[CorrelationID]*ResponseWaiter - // TODO: Add client actor reference for sending server responses back - // clientRef actor.TellOnlyRef[round.ServerMessageNotification] + // cancelCh delivers the ingress loop cancel function from + // StartIngress to StopIngress without a shared field, avoiding + // any data-race between the two methods. + cancelCh chan context.CancelFunc + + // wg tracks the ingress loop goroutine for clean shutdown. + wg sync.WaitGroup } -// NewServerConnectionActor creates a new server connection actor. -// TODO: Add parameters for gRPC connection and client actor reference. -func NewServerConnectionActor() *ServerConnectionActor { - return &ServerConnectionActor{} +// NewServerConnectionActor creates a new server connection actor with the +// given configuration. The actor must be started via its DurableActor wrapper +// and the ingress loop must be started separately via StartIngress. +func NewServerConnectionActor( + cfg ConnectorConfig, +) *ServerConnectionActor { + + return &ServerConnectionActor{ + cfg: cfg, + responseRegistry: make(map[CorrelationID]*ResponseWaiter), + cancelCh: make(chan context.CancelFunc, 1), + } } -// Receive processes incoming messages. +// Receive processes incoming egress messages. This is called by the durable +// actor runtime when messages arrive in the actor's mailbox. func (a *ServerConnectionActor) Receive(ctx context.Context, msg ServerConnMsg) fn.Result[ServerConnResp] { @@ -96,40 +314,221 @@ func (a *ServerConnectionActor) Receive(ctx context.Context, case *SendClientEventRequest: return a.handleSendClientEvent(ctx, m) + case *SendRPCRequest: + return a.handleSendRPCRequest(ctx, m) + default: return fn.Err[ServerConnResp](fmt.Errorf( - "unknown message type: %T", msg)) + "unknown message type: %T", msg, + )) } } // handleSendClientEvent converts a client FSM outbox message to a proto -// message and sends it to the server. +// message and sends it to the server via the mailbox edge. func (a *ServerConnectionActor) handleSendClientEvent(ctx context.Context, req *SendClientEventRequest) fn.Result[ServerConnResp] { - // Convert the message to proto using the ServerMessage interface. protoMsg := req.Message.ToProto() - // TODO: Send the proto message via gRPC. For now, this is a no-op. - // In production, this would: - // 1. Type switch on protoMsg to determine gRPC method. - // 2. Call the appropriate grpcClient method. - // 3. Handle response and notify the client actor. - _ = protoMsg + body, err := anypb.New(protoMsg) + if err != nil { + return fn.Err[ServerConnResp](fmt.Errorf( + "wrap proto in Any: %w", err, + )) + } + + msgID := req.MsgID + idempotencyKey := req.IdempotencyKey + + // Only marshal the body bytes when we need to derive stable IDs. + // On replay (both IDs already set from the persisted TLV), this + // marshal is skipped. + if msgID == "" || idempotencyKey == "" { + bodyBytes, marshalErr := (proto.MarshalOptions{ + Deterministic: true, + }).Marshal(body) + if marshalErr != nil { + return fn.Err[ServerConnResp](fmt.Errorf( + "marshal event body: %w", marshalErr, + )) + } + + if msgID == "" { + msgID = mailboxconn.StableEventMsgID(bodyBytes) + } + + if idempotencyKey == "" { + idempotencyKey = mailboxconn. + StableEventIdempotencyKey(bodyBytes) + } + } + + envelope := &mailboxpb.Envelope{ + ProtocolVersion: a.cfg.ProtocolVersion, + MsgId: msgID, + IdempotencyKey: idempotencyKey, + Sender: a.cfg.LocalMailboxID, + Recipient: a.cfg.RemoteMailboxID, + CreatedAtUnixMs: time.Now().UnixMilli(), + Body: body, + Rpc: &mailboxpb.RpcMeta{ + Kind: mailboxpb.RpcMeta_KIND_EVENT, + ReplyTo: a.cfg.LocalMailboxID, + }, + } + + resp, err := a.cfg.Edge.Send(ctx, &mailboxpb.SendRequest{ + Envelope: envelope, + }) + if err != nil { + return fn.Err[ServerConnResp](fmt.Errorf( + "send client event: %w", err, + )) + } + + if resp.Status != nil && !resp.Status.Ok { + return fn.Err[ServerConnResp](fmt.Errorf( + "send client event: %s (%s)", + resp.Status.Message, resp.Status.Code, + )) + } return fn.Ok[ServerConnResp](&SendClientEventResponse{ Success: true, }) } -// Start initializes the server connection. -// TODO: Establish gRPC connection to server. -func (a *ServerConnectionActor) Start() error { - return nil +// handleSendRPCRequest sends a pre-built unary RPC envelope via the mailbox +// edge. +func (a *ServerConnectionActor) handleSendRPCRequest(ctx context.Context, + req *SendRPCRequest) fn.Result[ServerConnResp] { + + resp, err := a.cfg.Edge.Send(ctx, &mailboxpb.SendRequest{ + Envelope: req.Envelope, + }) + if err != nil { + return fn.Err[ServerConnResp](fmt.Errorf( + "send rpc request: %w", err, + )) + } + + if resp.Status != nil && !resp.Status.Ok { + return fn.Err[ServerConnResp](fmt.Errorf( + "send rpc request: %s (%s)", + resp.Status.Message, resp.Status.Code, + )) + } + + return fn.Ok[ServerConnResp](&SendClientEventResponse{ + Success: true, + }) } -// Stop cleanly shuts down the server connection. -// TODO: Close gRPC connection. -func (a *ServerConnectionActor) Stop() error { - return nil +// RegisterWaiter adds a response waiter for the given correlation ID. The +// returned channel will receive the response envelope when the ingress loop +// pulls a KIND_RESPONSE with a matching correlation ID. +func (a *ServerConnectionActor) RegisterWaiter( + id CorrelationID, +) <-chan *mailboxpb.Envelope { + + a.responseRegistryMu.Lock() + defer a.responseRegistryMu.Unlock() + + waiter := &ResponseWaiter{ + Ch: make(chan *mailboxpb.Envelope, 1), + Created: time.Now(), + } + + a.responseRegistry[id] = waiter + + return waiter.Ch +} + +// removeWaiter removes a previously registered waiter, preventing leaks on +// cancellation or timeout. +func (a *ServerConnectionActor) removeWaiter(id CorrelationID) { + a.responseRegistryMu.Lock() + defer a.responseRegistryMu.Unlock() + + delete(a.responseRegistry, id) +} + +// deliverResponse looks up a waiter by correlation ID and delivers the +// envelope. Returns true if a waiter was found and signaled. +func (a *ServerConnectionActor) deliverResponse( + id CorrelationID, env *mailboxpb.Envelope, +) bool { + + a.responseRegistryMu.Lock() + waiter, ok := a.responseRegistry[id] + if ok { + delete(a.responseRegistry, id) + } + a.responseRegistryMu.Unlock() + + if !ok { + return false + } + + // Non-blocking send on buffered channel. If the waiter's context + // was already cancelled, the envelope is dropped (caller retries). + select { + case waiter.Ch <- env: + default: + } + + return true +} + +// StartIngress launches the background ingress loop goroutine that +// continuously pulls envelopes from the remote mailbox and dispatches them +// to local actors. +func (a *ServerConnectionActor) StartIngress(ctx context.Context) { + ingressCtx, cancel := context.WithCancel(ctx) + + a.wg.Add(1) + a.cancelCh <- cancel + go a.ingressLoop(ingressCtx) +} + +// StopIngress cancels the ingress loop and waits for it to exit. +func (a *ServerConnectionActor) StopIngress() { + if a.cancel != nil { + a.cancel() + } + + a.wg.Wait() } + +// NewServerConnCodec creates a MessageCodec with all server connection +// message types registered. +func NewServerConnCodec() *actor.MessageCodec { + codec := actor.NewMessageCodec() + + codec.MustRegister( + SendClientEventRequestMsgType, + func() actor.TLVMessage { + return &SendClientEventRequest{} + }, + ) + + codec.MustRegister( + SendRPCRequestMsgType, + func() actor.TLVMessage { + return &SendRPCRequest{} + }, + ) + + return codec +} + +// Compile-time interface checks. +var ( + _ ServerConnMsg = (*SendClientEventRequest)(nil) + _ ServerConnMsg = (*SendRPCRequest)(nil) + _ ServerConnResp = (*SendClientEventResponse)(nil) + + //nolint:ll + _ actor.ActorBehavior[ServerConnMsg, ServerConnResp] = (*ServerConnectionActor)(nil) +) From dc97004d9e8e28795cb5a3c16a0bb520f725ec54 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Tue, 17 Feb 2026 20:02:37 -0800 Subject: [PATCH 05/14] serverconn: implement ingress pull-dispatch-ack loop The ingress loop is the heart of the connector's inbound message processing. It runs as a self-driven background goroutine that continuously long-polls the remote mailbox via Edge.Pull and routes received envelopes to local consumers. The loop follows a strict three-phase cycle per iteration: First, if a prior dispatch committed successfully, it acks the remote mailbox up to the committed cursor via Edge.AckUpTo. This is done before pulling new work so the remote can garbage-collect delivered envelopes. The ack only advances after the checkpoint has been persisted, maintaining the invariant that AckUpTo never races ahead of durable local state. Second, it pulls a batch of envelopes starting from the current cursor. Empty long-poll returns reset the backoff counter and loop immediately since the long-poll timeout already provides the delay. Third, each envelope is dispatched based on its RPC metadata kind. KIND_RESPONSE envelopes are delivered to the in-memory response registry for unary RPC waiters. KIND_REQUEST and KIND_EVENT envelopes are routed through the configured dispatch table to local durable actors via their ServiceKey. If any dispatch fails mid-batch, the loop advances state only through the last successfully committed position, avoiding re-dispatch of already-committed envelopes on retry. All checkpoint state is persisted through the DeliveryStore's checkpoint mechanism using a stable actor ID derived from the local mailbox ID. Transient failures trigger exponential backoff with jitter to prevent busy-spinning, capped at the configured maximum delay. --- serverconn/actor.go | 39 ++- serverconn/ingress.go | 459 ++++++++++++++++++++++++++++ serverconn/ingress_property_test.go | 173 +++++++++++ 3 files changed, 662 insertions(+), 9 deletions(-) create mode 100644 serverconn/ingress.go create mode 100644 serverconn/ingress_property_test.go diff --git a/serverconn/actor.go b/serverconn/actor.go index 943f10574..d7b79961a 100644 --- a/serverconn/actor.go +++ b/serverconn/actor.go @@ -287,6 +287,10 @@ type ServerConnectionActor struct { // any data-race between the two methods. cancelCh chan context.CancelFunc + // stopOnce ensures StopIngress cancels the ingress loop exactly + // once. + stopOnce sync.Once + // wg tracks the ingress loop goroutine for clean shutdown. wg sync.WaitGroup } @@ -481,22 +485,39 @@ func (a *ServerConnectionActor) deliverResponse( return true } -// StartIngress launches the background ingress loop goroutine that -// continuously pulls envelopes from the remote mailbox and dispatches them -// to local actors. -func (a *ServerConnectionActor) StartIngress(ctx context.Context) { +// StartIngress loads the ack checkpoint from the store and launches the +// background ingress loop goroutine. If the checkpoint cannot be loaded, +// an error is returned and the loop is not started — the caller should +// treat this as a fatal startup failure. +func (a *ServerConnectionActor) StartIngress( + ctx context.Context, +) error { + + state, err := a.loadCheckpoint(ctx) + if err != nil { + return fmt.Errorf("load ingress checkpoint: %w", err) + } + ingressCtx, cancel := context.WithCancel(ctx) a.wg.Add(1) a.cancelCh <- cancel - go a.ingressLoop(ingressCtx) + go a.ingressLoop(ingressCtx, state) + + return nil } -// StopIngress cancels the ingress loop and waits for it to exit. +// StopIngress cancels the ingress loop and waits for it to exit. Safe to +// call multiple times — the cancel is executed at most once. func (a *ServerConnectionActor) StopIngress() { - if a.cancel != nil { - a.cancel() - } + a.stopOnce.Do(func() { + select { + case fn := <-a.cancelCh: + fn() + + default: + } + }) a.wg.Wait() } diff --git a/serverconn/ingress.go b/serverconn/ingress.go new file mode 100644 index 000000000..28a693cd0 --- /dev/null +++ b/serverconn/ingress.go @@ -0,0 +1,459 @@ +package serverconn + +import ( + "bytes" + "context" + "log/slog" + "math" + "math/rand/v2" + "time" + + "github.com/lightninglabs/darepo-client/baselib/actor" + mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" + mailboxrpc "github.com/lightninglabs/darepo-client/mailbox/rpc" +) + +// ingressLoop is the main pull-dispatch-ack loop. It runs in its own +// goroutine, started from ServerConnectionActor.StartIngress. The loop: +// +// 1. Loads persisted ack watermark state from the checkpoint store. +// 2. Continuously pulls envelopes from the remote mailbox. +// 3. Dispatches each envelope to the appropriate local actor or response +// waiter. +// 4. Advances the ack watermark only after durable dispatch commits. +// 5. Calls AckUpTo on the remote mailbox to release processed envelopes. +// +// On transient failures, the loop backs off with exponential delay and +// jitter to prevent busy-spinning. +func (a *ServerConnectionActor) ingressLoop( + ctx context.Context, state AckState, +) { + + defer a.wg.Done() + + log.InfoS(ctx, "Ingress loop starting", + slog.String("mailbox_id", a.cfg.LocalMailboxID)) + + var failCount int + + for { + select { + case <-ctx.Done(): + log.InfoS(ctx, "Ingress loop exiting", + slog.String("mailbox_id", + a.cfg.LocalMailboxID)) + + return + + default: + } + + // Step 1: Ack pending dispatches before pulling more. This + // allows the remote mailbox to garbage-collect already + // committed envelopes. + if state.NeedsAck() { + if err := a.ackRemote( + ctx, state.AckTarget, + ); err != nil { + log.WarnS(ctx, "AckUpTo failed, retrying", + err, + slog.Uint64("ack_target", + state.AckTarget)) + + a.sleepBackoff(ctx, &failCount) + + continue + } + + state.AdvanceAck() + + if err := a.saveCheckpoint(ctx, state); err != nil { + log.WarnS(ctx, + "Failed to save checkpoint after ack", + err) + + // Don't reset failCount — if the checkpoint + // store is persistently down, we want backoff + // to apply on subsequent iterations rather + // than spinning at full speed. + a.sleepBackoff(ctx, &failCount) + + continue + } + + failCount = 0 + } + + // Step 2: Pull a batch of envelopes from the remote mailbox. + envelopes, nextCursor, err := a.pullBatch( + ctx, state.PullCursor, + ) + if err != nil { + log.WarnS(ctx, "Pull failed, retrying", err, + slog.Uint64("cursor", state.PullCursor)) + + a.sleepBackoff(ctx, &failCount) + + continue + } + + if len(envelopes) == 0 { + // Long-poll returned empty. Reset fail count and loop + // again immediately — the long-poll timeout already + // provides the delay. + failCount = 0 + + continue + } + + log.DebugS(ctx, "Pulled envelopes", + slog.Int("count", len(envelopes)), + slog.Uint64("cursor", state.PullCursor), + slog.Uint64("next_cursor", nextCursor)) + + // Step 3: Dispatch the batch. On partial failure, the + // committed cursor reflects only the successfully dispatched + // portion. + committedCursor, dispatchErr := a.dispatchBatch( + ctx, envelopes, nextCursor, + ) + if dispatchErr != nil { + log.WarnS(ctx, "Dispatch failed", dispatchErr, + slog.Uint64("committed_to", committedCursor)) + + // Even on partial failure, advance state past the + // last committed envelope so we don't re-dispatch + // it. dispatchBatch returns the inclusive event_seq + // of the last successfully dispatched envelope, so + // we add 1 to get the exclusive next-pull position, + // consistent with batchNextCursor on the success + // path. + nextCursor := committedCursor + 1 + if committedCursor > 0 && + nextCursor > state.PullCursor { + + state.AdvanceDispatch(nextCursor) + state.PullCursor = nextCursor + + if cpErr := a.saveCheckpoint( + ctx, state, + ); cpErr != nil { + log.WarnS(ctx, + "Failed to save checkpoint "+ + "after partial dispatch", + cpErr) + } + } + + a.sleepBackoff(ctx, &failCount) + + continue + } + + // Step 4: Full batch dispatched successfully. Advance state + // and persist checkpoint. + state.AdvanceDispatch(committedCursor) + state.PullCursor = committedCursor + + if err := a.saveCheckpoint(ctx, state); err != nil { + log.WarnS(ctx, + "Failed to save checkpoint after dispatch", + err) + + a.sleepBackoff(ctx, &failCount) + + continue + } + + failCount = 0 + } +} + +// pullBatch calls Edge.Pull and returns the envelopes and next cursor. +func (a *ServerConnectionActor) pullBatch( + ctx context.Context, cursor uint64, +) ([]*mailboxpb.Envelope, uint64, error) { + + waitMs := uint32(a.cfg.PullWaitTimeout.Milliseconds()) + + resp, err := a.cfg.Edge.Pull(ctx, &mailboxpb.PullRequest{ + MailboxId: a.cfg.LocalMailboxID, + MaxEnvelopes: a.cfg.PullMaxEnvelopes, + WaitTimeoutMs: waitMs, + Cursor: cursor, + }) + if err != nil { + return nil, 0, err + } + + if resp.Status != nil && !resp.Status.Ok { + return nil, 0, &statusError{ + Op: "Pull", + Status: resp.Status, + } + } + + return resp.Envelopes, resp.NextCursor, nil +} + +// dispatchBatch iterates envelopes and routes each one to the correct +// destination: +// +// - KIND_RESPONSE: delivered to the response registry (unary waiters). +// - KIND_REQUEST/KIND_EVENT: dispatched to a local actor via the +// configured dispatch table. +// +// On success, returns the exclusive batch-next cursor (one past the last +// envelope). On partial failure, returns the inclusive event_seq of the +// last successfully dispatched envelope along with the error. The caller +// must add 1 to the error-path return value to get the exclusive cursor. +func (a *ServerConnectionActor) dispatchBatch( + ctx context.Context, + envelopes []*mailboxpb.Envelope, + batchNextCursor uint64, +) (uint64, error) { + + // Track the cursor of the last successfully dispatched envelope. + // Start with the current pull cursor as the base. + lastCommitted := uint64(0) + + for _, env := range envelopes { + if env.Rpc == nil { + log.WarnS(ctx, "Skipping envelope without RPC metadata", + nil, + slog.Uint64("event_seq", env.EventSeq)) + + continue + } + + switch env.Rpc.Kind { + case mailboxpb.RpcMeta_KIND_RESPONSE: + // Route to response registry for unary RPC waiters. + // This is not a durable dispatch — the response is + // consumed immediately by the waiting goroutine. + corrID := CorrelationID(env.Rpc.CorrelationId) + if corrID == "" { + log.WarnS(ctx, + "Response envelope missing "+ + "correlation ID", + nil, + slog.Uint64("event_seq", + env.EventSeq)) + + continue + } + + delivered := a.deliverResponse(corrID, env) + if !delivered { + log.WarnS(ctx, + "Failed to deliver response "+ + "envelope", + nil, + slog.String("correlation_id", + string(corrID)), + slog.Uint64("event_seq", + env.EventSeq)) + } + + case mailboxpb.RpcMeta_KIND_REQUEST, + mailboxpb.RpcMeta_KIND_EVENT: + + // Dispatch to local actor via the dispatch table. + // The dispatcher is a closure that does + // serviceKey.Ref(system).Tell(ctx, msg). A nil error + // means the target durable actor persisted the + // message. + key := mailboxrpc.ServiceMethod{ + Service: env.Rpc.Service, + Method: env.Rpc.Method, + } + + dispatcher, ok := a.cfg.Dispatchers[key] + if !ok { + log.WarnS(ctx, + "No dispatcher for service method", + nil, + slog.String("service", + env.Rpc.Service), + slog.String("method", + env.Rpc.Method), + slog.Uint64("event_seq", + env.EventSeq)) + + continue + } + + if err := dispatcher(ctx, env); err != nil { + // Dispatch failed. Stop processing the + // batch and return the last committed + // cursor. + return lastCommitted, err + } + + default: + log.WarnS(ctx, + "Skipping envelope with unknown RPC kind", + nil, + slog.Int("kind", int(env.Rpc.Kind)), + slog.Uint64("event_seq", env.EventSeq)) + + continue + } + + // Track the event_seq of the last processed envelope. The + // batch next cursor is the authoritative cursor to advance + // to after the full batch succeeds. + if env.EventSeq > lastCommitted { + lastCommitted = env.EventSeq + } + } + + // All envelopes dispatched successfully. Return the batch next cursor + // which represents the position after all envelopes in this batch. + if batchNextCursor > lastCommitted { + lastCommitted = batchNextCursor + } + + return lastCommitted, nil +} + +// ackRemote calls Edge.AckUpTo with the given cursor. +func (a *ServerConnectionActor) ackRemote( + ctx context.Context, cursor uint64, +) error { + + resp, err := a.cfg.Edge.AckUpTo(ctx, &mailboxpb.AckUpToRequest{ + MailboxId: a.cfg.LocalMailboxID, + Cursor: cursor, + }) + if err != nil { + return err + } + + if resp.Status != nil && !resp.Status.Ok { + return &statusError{ + Op: "AckUpTo", + Status: resp.Status, + } + } + + return nil +} + +// loadCheckpoint restores the AckState from the checkpoint store on startup. +// Returns a zero-value AckState if no checkpoint exists. +func (a *ServerConnectionActor) loadCheckpoint( + ctx context.Context, +) (AckState, error) { + + actorID := "serverconn-" + a.cfg.LocalMailboxID + + checkpoint, err := a.cfg.Store.LoadCheckpoint(ctx, actorID) + if err != nil { + return AckState{}, err + } + if checkpoint == nil { + return AckState{}, nil + } + + var state AckState + stateReader := bytes.NewReader(checkpoint.StateData) + if err := state.Decode(stateReader); err != nil { + return AckState{}, err + } + + log.InfoS(ctx, "Loaded ack checkpoint", + slog.String("actor_id", actorID), + slog.Uint64("pull_cursor", state.PullCursor), + slog.Uint64("dispatch_committed_to", + state.DispatchCommittedTo), + slog.Uint64("ack_target", state.AckTarget), + slog.Uint64("ack_committed_to", state.AckCommittedTo)) + + return state, nil +} + +// saveCheckpoint persists the AckState to the checkpoint store. +func (a *ServerConnectionActor) saveCheckpoint( + ctx context.Context, state AckState, +) error { + + var buf bytes.Buffer + if err := state.Encode(&buf); err != nil { + return err + } + + actorID := "serverconn-" + a.cfg.LocalMailboxID + + return a.cfg.Store.SaveCheckpoint(ctx, actor.CheckpointParams{ + ActorID: actorID, + StateType: ackStateType, + StateData: buf.Bytes(), + }) +} + +// sleepBackoff sleeps for an exponential backoff duration with jitter, +// respecting context cancellation. The fail count is incremented on entry +// and used to calculate the delay. +func (a *ServerConnectionActor) sleepBackoff( + ctx context.Context, failCount *int, +) { + + *failCount++ + delay := retryDelay( + a.cfg.RetryBaseDelay, a.cfg.RetryMaxDelay, *failCount, + ) + + timer := time.NewTimer(delay) + defer timer.Stop() + + select { + case <-ctx.Done(): + case <-timer.C: + } +} + +// retryDelay returns an exponential backoff duration with jitter, capped at +// maxDelay. The formula is: min(base * 2^attempt, max) * (0.5 + rand(0.5)). +func retryDelay( + base time.Duration, maxDelay time.Duration, attempt int, +) time.Duration { + + if base <= 0 { + base = 200 * time.Millisecond + } + if maxDelay <= 0 { + maxDelay = 30 * time.Second + } + + // Exponential backoff: base * 2^attempt. + delay := float64(base) * math.Pow(2, float64(attempt-1)) + if delay > float64(maxDelay) { + delay = float64(maxDelay) + } + + // Add jitter: multiply by a random factor in [0.5, 1.0). + // Crypto-grade randomness is not needed for backoff jitter. + jitter := 0.5 + rand.Float64()*0.5 //nolint:gosec + delay *= jitter + + return time.Duration(delay) +} + +// statusError wraps a mailbox status failure for error reporting. +type statusError struct { + // Op is the operation that failed (e.g., "Pull", "AckUpTo"). + Op string + + // Status is the status returned by the mailbox edge. + Status *mailboxpb.Status +} + +// Error returns a human-readable error string. +func (e *statusError) Error() string { + if e.Status == nil { + return e.Op + ": nil status" + } + + return e.Op + ": " + e.Status.Message + " (" + e.Status.Code + ")" +} diff --git a/serverconn/ingress_property_test.go b/serverconn/ingress_property_test.go new file mode 100644 index 000000000..1e7547a58 --- /dev/null +++ b/serverconn/ingress_property_test.go @@ -0,0 +1,173 @@ +package serverconn + +import ( + "testing" + + "pgregory.net/rapid" +) + +// TestIngress_AckNeverExceedsCommitted_Property validates that randomized ack +// and partial-dispatch progressions preserve the invariant: +// AckCommittedTo <= DispatchCommittedTo. +func TestIngress_AckNeverExceedsCommitted_Property(t *testing.T) { + t.Parallel() + + rapid.Check(t, func(rt *rapid.T) { + var state AckState + steps := rapid.IntRange(1, 400).Draw(rt, "steps") + + for i := 0; i < steps; i++ { + if state.NeedsAck() { + ackSucceeds := rapid.Bool().Draw( + rt, "ack_succeeds", + ) + if ackSucceeds { + state.AdvanceAck() + } + } + + dispatchFails := rapid.Bool().Draw( + rt, "dispatch_fails", + ) + commitDelta := rapid.Uint32Range(0, 4).Draw( + rt, "commit_delta", + ) + committedCursor := state.PullCursor + + uint64(commitDelta) + + if dispatchFails { + // Partial commit advances only work done + // before the failure point. + if committedCursor > state.PullCursor { + state.AdvanceDispatch(committedCursor) + state.PullCursor = committedCursor + } + } else { + // Full batch commit may move past the last + // event sequence processed in this loop. + nextExtra := rapid.Uint32Range(0, 4).Draw( + rt, "next_extra", + ) + batchNext := committedCursor + uint64(nextExtra) + + state.AdvanceDispatch(batchNext) + state.PullCursor = batchNext + } + + if state.AckCommittedTo > state.DispatchCommittedTo { + rt.Fatalf( + "ack cursor > dispatch: %d > %d", + state.AckCommittedTo, + state.DispatchCommittedTo, + ) + } + + if state.AckCommittedTo > state.PullCursor { + rt.Fatalf( + "ack cursor > pull: %d > %d", + state.AckCommittedTo, + state.PullCursor, + ) + } + } + }) +} + +// TestIngress_PartialFailureCursor_Property models the inclusive→exclusive +// cursor conversion that ingressLoop applies when dispatchBatch returns a +// partial failure. dispatchBatch returns the inclusive event_seq of the last +// successfully dispatched envelope on error, so the caller must add 1 to get +// the exclusive next-pull position. This property test verifies that the +// converted cursor never re-includes the last committed envelope (which +// would cause duplicate dispatch). +func TestIngress_PartialFailureCursor_Property(t *testing.T) { + t.Parallel() + + rapid.Check(t, func(rt *rapid.T) { + var state AckState + steps := rapid.IntRange(1, 400).Draw(rt, "steps") + + for i := 0; i < steps; i++ { + if state.NeedsAck() { + if rapid.Bool().Draw(rt, "ack_ok") { + state.AdvanceAck() + } + } + + batchSize := rapid.Uint32Range(1, 8).Draw( + rt, "batch_size", + ) + dispatchFails := rapid.Bool().Draw( + rt, "dispatch_fails", + ) + + // Model batchNextCursor as exclusive + // (PullCursor + batchSize). + batchNextCursor := state.PullCursor + + uint64(batchSize) + + if dispatchFails { + // Pick a failure point within the + // batch. failAt is the 0-based index + // of the envelope that fails. + failAt := rapid.Uint32Range( + 0, batchSize-1, + ).Draw(rt, "fail_at") + + if failAt == 0 { + // Nothing dispatched, no cursor + // advance. + continue + } + + // lastCommitted is the inclusive + // event_seq of the last OK envelope. + // Event seqs start at PullCursor. + lastCommitted := state.PullCursor + + uint64(failAt) - 1 + + // The ingressLoop converts inclusive + // to exclusive by adding 1. + nextCursor := lastCommitted + 1 + if nextCursor > state.PullCursor { + state.AdvanceDispatch(nextCursor) + state.PullCursor = nextCursor + } + } else { + state.AdvanceDispatch(batchNextCursor) + state.PullCursor = batchNextCursor + } + + // Core invariants. + if state.AckCommittedTo > state.DispatchCommittedTo { + rt.Fatalf( + "ack > dispatch: %d > %d", + state.AckCommittedTo, + state.DispatchCommittedTo, + ) + } + + if state.AckCommittedTo > state.PullCursor { + rt.Fatalf( + "ack > pull: %d > %d", + state.AckCommittedTo, + state.PullCursor, + ) + } + + // PullCursor must always be strictly past + // the last committed envelope's event_seq to + // prevent re-dispatch on the next pull. + if state.DispatchCommittedTo > 0 && + state.PullCursor < state.DispatchCommittedTo { + + rt.Fatalf( + "pull cursor behind dispatch: "+ + "%d < %d", + state.PullCursor, + state.DispatchCommittedTo, + ) + } + } + }) +} From 80552dfaabd56721f69b92ceac6974b36d734201 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Tue, 17 Feb 2026 20:02:51 -0800 Subject: [PATCH 06/14] serverconn: add unary RPC facade implementing RPCClient The UnaryFacade provides the mailboxrpc.RPCClient interface on top of the ServerConnectionActor, giving generated service stubs a synchronous send-and-await API for unary RPCs. SendRPC builds a complete mailbox envelope with correlation ID, idempotency key, service/method addressing, and protocol version, then sends it directly via Edge.Send rather than going through the durable actor mailbox. This direct path keeps unary RPC latency low since the caller already handles retries on failure and doesn't need durable egress guarantees. AwaitRPC registers a waiter in the actor's response registry, then blocks on the waiter channel until the ingress loop delivers a matching KIND_RESPONSE envelope or the caller's context is cancelled. On receipt, it unmarshals the proto body from the envelope's Any value into the caller-provided response message. The waiter is always cleaned up on exit to prevent registry leaks. Package documentation is included describing the connector's role as the single ingress/egress boundary, the ack watermark invariants, and the relationship between the dispatch table and the unary facade. --- serverconn/doc.go | 51 +++++++++ serverconn/unary_facade.go | 214 +++++++++++++++++++++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 serverconn/doc.go create mode 100644 serverconn/unary_facade.go diff --git a/serverconn/doc.go b/serverconn/doc.go new file mode 100644 index 000000000..012974e11 --- /dev/null +++ b/serverconn/doc.go @@ -0,0 +1,51 @@ +// Package serverconn provides the unified connector boundary for all mailbox +// traffic between the client and the remote server. +// +// The connector serves as both an egress actor and an ingress loop: +// +// - Egress: Receives outbound messages from the round actor (FSM events) and +// the unary facade (RPC requests), then sends them via the mailbox edge. +// The actor is backed by a DurableActor for crash-safe egress — outbound +// messages from the round actor persist in the durable mailbox before +// processing, ensuring no message loss on crashes. +// +// - Ingress: Continuously pulls envelopes from the remote mailbox, dispatches +// them to local actors via ServiceKey-based routing, and manages the ack +// watermark state machine to ensure at-least-once delivery with crash +// safety. +// +// # Ack Watermark Invariants +// +// The ingress loop tracks four monotonic cursor variables in AckState: +// +// - PullCursor: cursor for the next Pull call +// - DispatchCommittedTo: max cursor whose envelopes have been durably +// committed to local actor mailboxes +// - AckTarget: max cursor that should be acked remotely (always >= +// DispatchCommittedTo) +// - AckCommittedTo: last cursor successfully acked to the remote edge +// +// The critical invariant is: AckUpTo only advances AFTER durable local +// dispatch commit (DurableActor.Tell returns nil = persisted). This ensures +// that if the process crashes between dispatch and ack, envelopes will be +// redelivered on restart. +// +// # Dispatch Table +// +// Inbound KIND_REQUEST and KIND_EVENT envelopes are routed via a +// map[ServiceMethod]EnvelopeDispatcher configured at wiring time. Each +// dispatcher is a closure that captures a ServiceKey reference for the target +// actor and calls Tell to durably enqueue the message. +// +// KIND_RESPONSE envelopes are delivered to in-memory response waiters via the +// response registry. This is not durable — if the process crashes, callers' +// contexts are cancelled and they retry. +// +// # Unary Facade +// +// The UnaryFacade implements mailboxrpc.RPCClient for generated RPC stubs. +// SendRPC constructs and sends envelopes directly via the mailbox edge +// (synchronous, no actor mailbox — low-latency path for unary sends). +// AwaitRPC registers a waiter in the response registry and blocks until the +// ingress loop delivers a matching KIND_RESPONSE envelope. +package serverconn diff --git a/serverconn/unary_facade.go b/serverconn/unary_facade.go new file mode 100644 index 000000000..9195f942d --- /dev/null +++ b/serverconn/unary_facade.go @@ -0,0 +1,214 @@ +package serverconn + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "log/slog" + "time" + + mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" + mailboxrpc "github.com/lightninglabs/darepo-client/mailbox/rpc" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +const errMalformedResponseBody = "failed to unmarshal response body: %w" + +// UnaryFacade implements mailboxrpc.RPCClient by delegating send and response +// delivery through a ServerConnectionActor. The send path calls Edge.Send +// directly (synchronous, no actor mailbox) for low-latency unary RPCs. The +// await path registers a waiter with the connector's in-memory response +// registry, which the ingress loop signals when a KIND_RESPONSE envelope +// arrives. +// +// Unary RPC sends do not need durable egress — callers retry on failure. The +// durable egress path (SendClientEventRequest through DurableActor.Tell) is +// reserved for FSM outbox messages from the round actor. +type UnaryFacade struct { + // connector is the server connection actor that owns the response + // registry and configuration. + connector *ServerConnectionActor +} + +// NewUnaryFacade creates a new unary RPC facade backed by the given +// ServerConnectionActor. +func NewUnaryFacade( + connector *ServerConnectionActor, +) *UnaryFacade { + + return &UnaryFacade{ + connector: connector, + } +} + +// SendRPC builds an RPC request envelope, sends it via the mailbox edge, and +// returns the correlation and idempotency identifiers so the caller can +// subsequently await the response. +func (f *UnaryFacade) SendRPC(ctx context.Context, + method mailboxrpc.ServiceMethod, req proto.Message, + opts mailboxrpc.RPCOptions) (mailboxrpc.SendResult, error) { + + cfg := f.connector.cfg + + msgID, err := randomID(16) + if err != nil { + return mailboxrpc.SendResult{}, fmt.Errorf( + "generate msg id: %w", err, + ) + } + + idempotencyKey := opts.IdempotencyKey + if idempotencyKey == "" { + idempotencyKey, err = randomID(16) + if err != nil { + return mailboxrpc.SendResult{}, fmt.Errorf( + "generate idempotency key: %w", err, + ) + } + } + + correlationID := opts.CorrelationID + if correlationID == "" { + correlationID = idempotencyKey + } + + // Pre-register the response waiter before sending so a fast response + // pulled by the ingress loop between Send and AwaitRPC is buffered + // rather than dropped. The returned Future is not needed here — + // AwaitRPC re-registers (idempotently) and blocks on it. + corrID := CorrelationID(correlationID) + f.connector.RegisterWaiter(corrID) + + body, err := anypb.New(req) + if err != nil { + f.connector.removeWaiter(corrID) + + return mailboxrpc.SendResult{}, fmt.Errorf( + "wrap request in Any: %w", err, + ) + } + + envelope := &mailboxpb.Envelope{ + ProtocolVersion: cfg.ProtocolVersion, + MsgId: msgID, + IdempotencyKey: idempotencyKey, + Sender: cfg.LocalMailboxID, + Recipient: cfg.RemoteMailboxID, + CreatedAtUnixMs: time.Now().UnixMilli(), + Headers: opts.Headers, + Body: body, + Rpc: &mailboxpb.RpcMeta{ + Kind: mailboxpb.RpcMeta_KIND_REQUEST, + Service: method.Service, + Method: method.Method, + CorrelationId: correlationID, + ReplyTo: cfg.LocalMailboxID, + }, + } + + resp, err := cfg.Edge.Send(ctx, &mailboxpb.SendRequest{ + Envelope: envelope, + }) + if err != nil { + f.connector.removeWaiter(corrID) + + log.WarnS(ctx, "Unary send failed", err, + slog.String("service", method.Service), + slog.String("method", method.Method)) + + return mailboxrpc.SendResult{}, fmt.Errorf( + "send rpc request: %w", err, + ) + } + + if resp.Status != nil && !resp.Status.Ok { + f.connector.removeWaiter(corrID) + + sendErr := &statusError{ + Op: "Send", + Status: resp.Status, + } + + log.WarnS(ctx, "Unary send returned non-OK status", sendErr, + slog.String("service", method.Service), + slog.String("method", method.Method)) + + return mailboxrpc.SendResult{}, sendErr + } + + log.DebugS(ctx, "Sent unary RPC request", + slog.String("service", method.Service), + slog.String("method", method.Method), + slog.String("correlation_id", correlationID)) + + return mailboxrpc.SendResult{ + CorrelationID: correlationID, + IdempotencyKey: idempotencyKey, + }, nil +} + +// AwaitRPC registers a waiter for the given correlation ID and blocks until +// the ingress loop delivers a KIND_RESPONSE envelope, the waiter expires, +// or the context is cancelled. The response envelope body is unmarshaled +// into resp. +func (f *UnaryFacade) AwaitRPC(ctx context.Context, + correlationID string, resp proto.Message) error { + + corrID := CorrelationID(correlationID) + + // Idempotent re-registration: if SendRPC already registered this + // ID, we get back the same (possibly already completed) Future. + future := f.connector.RegisterWaiter(corrID) + defer f.connector.removeWaiter(corrID) + + result := future.Await(ctx) + if result.IsErr() { + return result.Err() + } + + var env *mailboxpb.Envelope + result.WhenOk(func(e *mailboxpb.Envelope) { + env = e + }) + + if env == nil { + return fmt.Errorf( + "response waiter completed without delivery", + ) + } + + if env.Body == nil { + return fmt.Errorf("response envelope has nil body") + } + + // The body is an anypb.Any containing the response proto. + // Unmarshal the raw Value bytes directly into the caller's + // response message, discarding unknown fields for forward + // compatibility. We intentionally skip TypeUrl validation here: + // the generated stubs always pass the correct concrete type, and + // a server-side type mismatch would surface as garbled fields + // rather than silent data loss (proto3 zero-values). + err := (proto.UnmarshalOptions{ + DiscardUnknown: true, + }).Unmarshal(env.Body.Value, resp) + if err != nil { + return fmt.Errorf(errMalformedResponseBody, err) + } + + return nil +} + +// randomID generates a cryptographically random hex-encoded identifier. +func randomID(nbytes int) (string, error) { + buf := make([]byte, nbytes) + if _, err := rand.Read(buf); err != nil { + return "", err + } + + return hex.EncodeToString(buf), nil +} + +// Compile-time interface check. +var _ mailboxrpc.RPCClient = (*UnaryFacade)(nil) From 5115a0ef585212166e1f9e030272e9a6e65bf55a Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Tue, 17 Feb 2026 20:03:06 -0800 Subject: [PATCH 07/14] serverconn: add connector test suite This commit adds comprehensive tests covering the three main layers of the serverconn connector. The ack state machine tests verify the cursor advancement invariants: AdvanceDispatch sets the ack target monotonically, AdvanceAck updates the committed cursor and pull position, NeedsAck correctly reflects pending acks, and TLV round-trip encoding preserves all four cursor fields including the zero-value edge case. The connector integration tests exercise the full ingress loop against an in-memory mailbox edge. They verify that dispatched envelopes are acked, responses are delivered to registered waiters, failed dispatches don't advance the ack cursor, shutdown drains cleanly without goroutine leaks, and checkpoint state survives a simulated restart cycle. The exponential backoff helper is tested for range bounds and zero-value defaults. The unary facade tests cover the SendRPC envelope construction path (verifying correlation ID, idempotency key, and service/method metadata in the sent envelope), the AwaitRPC round-trip through the response registry including context cancellation, concurrent inflight request isolation, and nil body handling. Test infrastructure includes an in-memory mailbox with long-poll support, a fake MailboxServiceClient adapter, and a minimal checkpoint store that implements the full DeliveryStore interface with panicking stubs for unused methods. --- serverconn/ack_state_test.go | 186 +++++++++++++ serverconn/connector_test.go | 465 +++++++++++++++++++++++++++++++ serverconn/testutil_test.go | 478 ++++++++++++++++++++++++++++++++ serverconn/unary_facade_test.go | 300 ++++++++++++++++++++ 4 files changed, 1429 insertions(+) create mode 100644 serverconn/ack_state_test.go create mode 100644 serverconn/connector_test.go create mode 100644 serverconn/testutil_test.go create mode 100644 serverconn/unary_facade_test.go diff --git a/serverconn/ack_state_test.go b/serverconn/ack_state_test.go new file mode 100644 index 000000000..22efb6b83 --- /dev/null +++ b/serverconn/ack_state_test.go @@ -0,0 +1,186 @@ +package serverconn + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestAckState_AdvanceDispatch_SetsAckTarget verifies that AdvanceDispatch +// moves both DispatchCommittedTo and AckTarget forward. +func TestAckState_AdvanceDispatch_SetsAckTarget(t *testing.T) { + t.Parallel() + + var s AckState + s.AdvanceDispatch(10) + + require.Equal(t, uint64(10), s.DispatchCommittedTo) + require.Equal(t, uint64(10), s.AckTarget) +} + +// TestAckState_AdvanceDispatch_Monotonic verifies that AdvanceDispatch never +// decreases DispatchCommittedTo or AckTarget. +func TestAckState_AdvanceDispatch_Monotonic(t *testing.T) { + t.Parallel() + + var s AckState + s.AdvanceDispatch(10) + s.AdvanceDispatch(5) // Should be ignored. + s.AdvanceDispatch(7) // Still lower, should be ignored. + + require.Equal(t, uint64(10), s.DispatchCommittedTo) + require.Equal(t, uint64(10), s.AckTarget) + + // A higher value should advance. + s.AdvanceDispatch(15) + require.Equal(t, uint64(15), s.DispatchCommittedTo) + require.Equal(t, uint64(15), s.AckTarget) +} + +// TestAckState_AdvanceAck_UpdatesPullCursor verifies that AdvanceAck moves +// AckCommittedTo to AckTarget and advances PullCursor if needed. +func TestAckState_AdvanceAck_UpdatesPullCursor(t *testing.T) { + t.Parallel() + + s := AckState{ + PullCursor: 0, + DispatchCommittedTo: 10, + AckTarget: 10, + AckCommittedTo: 0, + } + + s.AdvanceAck() + + require.Equal(t, uint64(10), s.AckCommittedTo) + require.Equal(t, uint64(10), s.PullCursor, + "PullCursor should advance to match AckCommittedTo") +} + +// TestAckState_AdvanceAck_DoesNotRegressPullCursor verifies that AdvanceAck +// does not move PullCursor backward if it is already ahead. +func TestAckState_AdvanceAck_DoesNotRegressPullCursor(t *testing.T) { + t.Parallel() + + s := AckState{ + PullCursor: 20, + DispatchCommittedTo: 10, + AckTarget: 10, + AckCommittedTo: 0, + } + + s.AdvanceAck() + + require.Equal(t, uint64(10), s.AckCommittedTo) + require.Equal(t, uint64(20), s.PullCursor, + "PullCursor should not regress") +} + +// TestAckState_NeedsAck verifies that NeedsAck returns true only when +// AckTarget exceeds AckCommittedTo. +func TestAckState_NeedsAck(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + state AckState + wantAck bool + }{ + { + name: "zero state needs no ack", + state: AckState{}, + wantAck: false, + }, + { + name: "target ahead of committed needs ack", + state: AckState{ + AckTarget: 10, + AckCommittedTo: 5, + }, + wantAck: true, + }, + { + name: "target equal to committed needs no ack", + state: AckState{ + AckTarget: 10, + AckCommittedTo: 10, + }, + wantAck: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + require.Equal(t, tc.wantAck, tc.state.NeedsAck()) + }) + } +} + +// TestAckState_FullCycle verifies a complete dispatch-ack cycle: dispatch +// advances the state, ack catches up, and NeedsAck transitions correctly. +func TestAckState_FullCycle(t *testing.T) { + t.Parallel() + + var s AckState + + // Initially no ack needed. + require.False(t, s.NeedsAck()) + + // Dispatch first batch. + s.AdvanceDispatch(10) + s.PullCursor = 10 + require.True(t, s.NeedsAck()) + + // Ack catches up. + s.AdvanceAck() + require.False(t, s.NeedsAck()) + require.Equal(t, uint64(10), s.AckCommittedTo) + require.Equal(t, uint64(10), s.PullCursor) + + // Dispatch second batch. + s.AdvanceDispatch(25) + s.PullCursor = 25 + require.True(t, s.NeedsAck()) + + // Ack again. + s.AdvanceAck() + require.False(t, s.NeedsAck()) + require.Equal(t, uint64(25), s.AckCommittedTo) +} + +// TestAckState_EncodeDecode verifies TLV round-trip serialization. +func TestAckState_EncodeDecode(t *testing.T) { + t.Parallel() + + original := AckState{ + PullCursor: 42, + DispatchCommittedTo: 100, + AckTarget: 100, + AckCommittedTo: 90, + } + + var buf bytes.Buffer + require.NoError(t, original.Encode(&buf)) + + var decoded AckState + require.NoError(t, decoded.Decode(bytes.NewReader(buf.Bytes()))) + + require.Equal(t, original, decoded) +} + +// TestAckState_EncodeDecode_Zero verifies that zero-value state round-trips. +func TestAckState_EncodeDecode_Zero(t *testing.T) { + t.Parallel() + + var original AckState + + var buf bytes.Buffer + require.NoError(t, original.Encode(&buf)) + + var decoded AckState + require.NoError(t, decoded.Decode(bytes.NewReader(buf.Bytes()))) + + require.Equal(t, original, decoded) +} diff --git a/serverconn/connector_test.go b/serverconn/connector_test.go new file mode 100644 index 000000000..28b21cb3c --- /dev/null +++ b/serverconn/connector_test.go @@ -0,0 +1,465 @@ +package serverconn + +import ( + "context" + "sync" + "testing" + "time" + + mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" + mailboxrpc "github.com/lightninglabs/darepo-client/mailbox/rpc" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// newTestConnector builds a ServerConnectionActor with in-memory test +// dependencies. +func newTestConnector( + t *testing.T, + dispatchers map[mailboxrpc.ServiceMethod]EnvelopeDispatcher, +) (*ServerConnectionActor, *inMemoryMailbox, *memCheckpointStore) { + + t.Helper() + + mb := newInMemoryMailbox() + edge := &fakeMailboxServiceClient{mb: mb} + store := newMemCheckpointStore() + + cfg := DefaultConnectorConfig() + cfg.Edge = edge + cfg.LocalMailboxID = "client-1" + cfg.RemoteMailboxID = "server-1" + cfg.ProtocolVersion = 1 + cfg.Dispatchers = dispatchers + cfg.Store = store + cfg.PullWaitTimeout = 50 * time.Millisecond + cfg.RetryBaseDelay = 10 * time.Millisecond + cfg.RetryMaxDelay = 50 * time.Millisecond + + actor := NewServerConnectionActor(cfg) + + return actor, mb, store +} + +// sendResponseToMailbox injects a KIND_RESPONSE envelope into the given +// mailbox addressed to recipientID. +func sendResponseToMailbox( + t *testing.T, mb *inMemoryMailbox, + recipientID, correlationID string, payload []byte, +) { + + t.Helper() + + body := &anypb.Any{ + TypeUrl: "test/response", + Value: payload, + } + + env := &mailboxpb.Envelope{ + ProtocolVersion: 1, + Sender: "server-1", + Recipient: recipientID, + Body: body, + Rpc: &mailboxpb.RpcMeta{ + Kind: mailboxpb.RpcMeta_KIND_RESPONSE, + CorrelationId: correlationID, + ReplyTo: "server-1", + }, + } + + status := mb.send(env) + require.True(t, status.Ok, "send response failed: %s", status.Message) +} + +// sendEventToMailbox injects a KIND_EVENT envelope into the given mailbox +// addressed to recipientID with the specified service/method. +func sendEventToMailbox( + t *testing.T, mb *inMemoryMailbox, + recipientID, service, method string, +) { + + t.Helper() + + body, err := anypb.New(wrapperspb.String("test-event")) + require.NoError(t, err) + + env := &mailboxpb.Envelope{ + ProtocolVersion: 1, + Sender: "server-1", + Recipient: recipientID, + Body: body, + Rpc: &mailboxpb.RpcMeta{ + Kind: mailboxpb.RpcMeta_KIND_EVENT, + Service: service, + Method: method, + ReplyTo: "server-1", + }, + } + + status := mb.send(env) + require.True(t, status.Ok, "send event failed: %s", status.Message) +} + +// TestIngress_DispatchAndAck verifies that the ingress loop pulls envelopes, +// dispatches them via the dispatch table, and acks the remote mailbox. +func TestIngress_DispatchAndAck(t *testing.T) { + t.Parallel() + + var ( + dispatched []*mailboxpb.Envelope + dispatchedMu sync.Mutex + ) + + dispatchers := map[mailboxrpc.ServiceMethod]EnvelopeDispatcher{ + {Service: "test.Svc", Method: "DoThing"}: func( + ctx context.Context, + env *mailboxpb.Envelope, + ) error { + + dispatchedMu.Lock() + dispatched = append(dispatched, env) + dispatchedMu.Unlock() + + return nil + }, + } + + actor, mb, _ := newTestConnector(t, dispatchers) + + // Inject 3 events into the client's mailbox. + for i := 0; i < 3; i++ { + sendEventToMailbox( + t, mb, "client-1", "test.Svc", "DoThing", + ) + } + + // Start ingress. + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + require.NoError(t, actor.StartIngress(ctx)) + defer actor.StopIngress() + + // Wait for dispatch to process all 3 envelopes. + require.Eventually(t, func() bool { + dispatchedMu.Lock() + defer dispatchedMu.Unlock() + + return len(dispatched) == 3 + }, 5*time.Second, 10*time.Millisecond) + + // The ack watermark should have advanced past the envelopes. + require.Eventually(t, func() bool { + return mb.getAckedUpTo("client-1") > 0 + }, 5*time.Second, 10*time.Millisecond) +} + +// TestIngress_ResponseDelivery verifies that KIND_RESPONSE envelopes are +// delivered to registered waiters via the response registry. +func TestIngress_ResponseDelivery(t *testing.T) { + t.Parallel() + + actor, mb, _ := newTestConnector(t, nil) + + // Register a waiter for a specific correlation ID. + corrID := CorrelationID("test-corr-123") + future := actor.RegisterWaiter(corrID) + + // Start ingress. + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + require.NoError(t, actor.StartIngress(ctx)) + defer actor.StopIngress() + + // Inject a response envelope. + sendResponseToMailbox( + t, mb, "client-1", string(corrID), + []byte("response-payload"), + ) + + // The waiter should receive the envelope. + awaitCtx, awaitCancel := context.WithTimeout( + t.Context(), 5*time.Second, + ) + defer awaitCancel() + + env := future.Await(awaitCtx).UnwrapOrFail(t) + require.NotNil(t, env) + require.Equal( + t, string(corrID), env.Rpc.CorrelationId, + ) +} + +// TestIngress_NoAckOnDispatchFailure verifies that when a dispatcher returns +// an error, the ack watermark does not advance past the failed envelope. +func TestIngress_NoAckOnDispatchFailure(t *testing.T) { + t.Parallel() + + var callCount int + var callCountMu sync.Mutex + + dispatchers := map[mailboxrpc.ServiceMethod]EnvelopeDispatcher{ + {Service: "test.Svc", Method: "Fail"}: func( + ctx context.Context, + env *mailboxpb.Envelope, + ) error { + + callCountMu.Lock() + callCount++ + count := callCount + callCountMu.Unlock() + + // Fail the first attempt, succeed thereafter. + if count == 1 { + return &statusError{ + Op: "dispatch", + Status: &mailboxpb.Status{ + Ok: false, + Code: "INTERNAL", + Message: "test failure", + }, + } + } + + return nil + }, + } + + actor, mb, _ := newTestConnector(t, dispatchers) + + // Inject one event. + sendEventToMailbox(t, mb, "client-1", "test.Svc", "Fail") + + // Start ingress. + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + require.NoError(t, actor.StartIngress(ctx)) + defer actor.StopIngress() + + // The dispatcher should eventually succeed on retry. + require.Eventually(t, func() bool { + callCountMu.Lock() + defer callCountMu.Unlock() + + return callCount >= 2 + }, 5*time.Second, 10*time.Millisecond) + + // The ack should eventually advance after the retry succeeds. + require.Eventually(t, func() bool { + return mb.getAckedUpTo("client-1") > 0 + }, 5*time.Second, 10*time.Millisecond) +} + +// TestIngress_Shutdown_NoGoroutineLeak verifies that StopIngress cleanly +// terminates the ingress loop goroutine. +func TestIngress_Shutdown_NoGoroutineLeak(t *testing.T) { + t.Parallel() + + actor, _, _ := newTestConnector(t, nil) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + require.NoError(t, actor.StartIngress(ctx)) + + // Give the loop a moment to start. + time.Sleep(50 * time.Millisecond) + + // StopIngress should return promptly. + done := make(chan struct{}) + go func() { + actor.StopIngress() + close(done) + }() + + select { + case <-done: + // Clean shutdown. + + case <-time.After(5 * time.Second): + t.Fatal("StopIngress did not return within timeout") + } +} + +// TestIngress_CheckpointSurvivesRestart verifies that after processing and +// acking envelopes, the checkpoint can be loaded to restore state. +func TestIngress_CheckpointSurvivesRestart(t *testing.T) { + t.Parallel() + + dispatchers := map[mailboxrpc.ServiceMethod]EnvelopeDispatcher{ + {Service: "test.Svc", Method: "DoThing"}: func( + ctx context.Context, + env *mailboxpb.Envelope, + ) error { + + return nil + }, + } + + actor, mb, store := newTestConnector(t, dispatchers) + + // Inject an event. + sendEventToMailbox(t, mb, "client-1", "test.Svc", "DoThing") + + // Run ingress long enough to process and checkpoint. + ctx, cancel := context.WithCancel(t.Context()) + require.NoError(t, actor.StartIngress(ctx)) + + require.Eventually(t, func() bool { + return mb.getAckedUpTo("client-1") > 0 + }, 5*time.Second, 10*time.Millisecond) + + cancel() + actor.StopIngress() + + // Verify checkpoint was persisted. + actorID := "serverconn-client-1" + cp, err := store.LoadCheckpoint(t.Context(), actorID) + require.NoError(t, err) + require.NotNil(t, cp, "checkpoint should be persisted") + require.NotEmpty(t, cp.StateData) +} + +// TestEgress_EventRetriesPreserveIdempotencyKey verifies that egress sends for +// the same semantic event use stable message and idempotency identifiers. +func TestEgress_EventRetriesPreserveIdempotencyKey(t *testing.T) { + t.Parallel() + + actor, mb, _ := newTestConnector(t, nil) + + req1 := &SendClientEventRequest{ + Message: &testServerMessage{value: "same-event"}, + } + req2 := &SendClientEventRequest{ + Message: &testServerMessage{value: "same-event"}, + } + + require.NoError(t, actor.Receive(t.Context(), req1).Err()) + require.NoError(t, actor.Receive(t.Context(), req2).Err()) + + mb.mu.Lock() + envs := append( + []*mailboxpb.Envelope(nil), mb.mailboxes["server-1"]..., + ) + mb.mu.Unlock() + + require.Len(t, envs, 2) + require.NotEmpty(t, envs[0].MsgId) + require.NotEmpty(t, envs[0].IdempotencyKey) + require.Equal(t, envs[0].MsgId, envs[1].MsgId) + require.Equal( + t, envs[0].IdempotencyKey, envs[1].IdempotencyKey, + ) +} + +// TestIngress_PartialDispatch_NoDuplicateRedelivery verifies that when +// a batch dispatch fails mid-way, the already-dispatched envelopes are +// not re-dispatched on the next loop iteration. This is a regression test +// for the off-by-one where the inclusive event_seq returned on the error +// path was used directly as PullCursor, causing the last committed +// envelope to be re-pulled and re-dispatched. +func TestIngress_PartialDispatch_NoDuplicateRedelivery(t *testing.T) { + t.Parallel() + + var ( + // Track dispatch count per event_seq to detect duplicates. + dispatchCounts = make(map[uint64]int) + dispatchCountsMu sync.Mutex + callCount int + ) + + dispatchers := map[mailboxrpc.ServiceMethod]EnvelopeDispatcher{ + {Service: "test.Svc", Method: "Batch"}: func( + ctx context.Context, + env *mailboxpb.Envelope, + ) error { + + dispatchCountsMu.Lock() + callCount++ + count := callCount + dispatchCounts[env.EventSeq]++ + dispatchCountsMu.Unlock() + + // Fail on the second envelope in the first batch. + // The first envelope (count==1) succeeds, and the + // second (count==2) fails. On retry, we expect the + // second to be dispatched (count==3) but NOT the + // first again. + if count == 2 { + return &statusError{ + Op: "dispatch", + Status: &mailboxpb.Status{ + Ok: false, + Code: "INTERNAL", + Message: "injected batch failure", + }, + } + } + + return nil + }, + } + + actor, mb, _ := newTestConnector(t, dispatchers) + + // Inject 2 events so the batch has 2 envelopes. + sendEventToMailbox(t, mb, "client-1", "test.Svc", "Batch") + sendEventToMailbox(t, mb, "client-1", "test.Svc", "Batch") + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + require.NoError(t, actor.StartIngress(ctx)) + defer actor.StopIngress() + + // Wait until both envelopes have been fully dispatched. + require.Eventually(t, func() bool { + return mb.getAckedUpTo("client-1") > 0 + }, 5*time.Second, 10*time.Millisecond) + + dispatchCountsMu.Lock() + defer dispatchCountsMu.Unlock() + + // The first envelope (event_seq=1) must have been dispatched + // exactly once — not re-dispatched after the partial failure. + require.Equal( + t, 1, dispatchCounts[1], + "first envelope should be dispatched exactly once", + ) + + // The second envelope (event_seq=2) should be dispatched + // exactly twice: once failed, once succeeded on retry. + require.Equal( + t, 2, dispatchCounts[2], + "second envelope should be dispatched exactly twice "+ + "(1 fail + 1 retry)", + ) +} +// TestRetryDelay verifies the exponential backoff formula with jitter. +func TestRetryDelay(t *testing.T) { + t.Parallel() + + base := 100 * time.Millisecond + maxDelay := 5 * time.Second + + // First attempt should be approximately base (within jitter range). + d := retryDelay(base, maxDelay, 1) + require.GreaterOrEqual(t, d, base/2) + require.LessOrEqual(t, d, base) + + // High attempt should be capped at maxDelay. + d = retryDelay(base, maxDelay, 100) + require.LessOrEqual(t, d, maxDelay) + require.GreaterOrEqual(t, d, maxDelay/2) +} + +// TestRetryDelay_DefaultsOnZero verifies that zero base/max get defaults. +func TestRetryDelay_DefaultsOnZero(t *testing.T) { + t.Parallel() + + d := retryDelay(0, 0, 1) + require.Greater(t, d, time.Duration(0)) +} diff --git a/serverconn/testutil_test.go b/serverconn/testutil_test.go new file mode 100644 index 000000000..03afbe34f --- /dev/null +++ b/serverconn/testutil_test.go @@ -0,0 +1,478 @@ +package serverconn + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/lightninglabs/darepo-client/baselib/actor" + mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" +) + +// inMemoryMailbox is a minimal in-memory implementation of the MailboxService +// semantics needed for connector unit tests. +type inMemoryMailbox struct { + mu sync.Mutex + + nextSeq uint64 + + // mailboxes stores all envelopes by mailbox id. + mailboxes map[string][]*mailboxpb.Envelope + + // ackedUpTo stores the ack watermark (event_seq < ackedUpTo are + // acked). + ackedUpTo map[string]uint64 + + notify chan struct{} +} + +// newInMemoryMailbox constructs an empty mailbox edge. +func newInMemoryMailbox() *inMemoryMailbox { + return &inMemoryMailbox{ + nextSeq: 1, + mailboxes: make(map[string][]*mailboxpb.Envelope), + ackedUpTo: make(map[string]uint64), + notify: make(chan struct{}), + } +} + +// send enqueues envelope into recipient mailbox and assigns an event_seq. +func (m *inMemoryMailbox) send( + envelope *mailboxpb.Envelope, +) *mailboxpb.Status { + + m.mu.Lock() + defer m.mu.Unlock() + + if envelope == nil { + return &mailboxpb.Status{ + Ok: false, + Code: "INVALID_ARGUMENT", + Message: "missing envelope", + } + } + + envCopy, ok := cloneEnvelope(envelope) + if !ok { + return &mailboxpb.Status{ + Ok: false, + Code: "INVALID_ARGUMENT", + Message: "unexpected envelope type", + } + } + envCopy.EventSeq = m.nextSeq + m.nextSeq++ + + recipient := envCopy.Recipient + m.mailboxes[recipient] = append(m.mailboxes[recipient], envCopy) + + close(m.notify) + m.notify = make(chan struct{}) + + return okStatus() +} + +// pull returns envelopes with event_seq >= cursor and not acked. +func (m *inMemoryMailbox) pull(ctx context.Context, mailboxID string, + cursor uint64, maxEnvelopes uint32, + wait time.Duration) ([]*mailboxpb.Envelope, uint64, + *mailboxpb.Status) { + + deadline := time.Now().Add(wait) + + for { + m.mu.Lock() + envs, next := m.pullLocked(mailboxID, cursor, maxEnvelopes) + if len(envs) > 0 || wait == 0 { + m.mu.Unlock() + + return envs, next, okStatus() + } + + notify := m.notify + m.mu.Unlock() + + now := time.Now() + if !now.Before(deadline) { + return nil, cursor, okStatus() + } + + remaining := deadline.Sub(now) + timer := time.NewTimer(remaining) + + select { + case <-notify: + case <-timer.C: + case <-ctx.Done(): + timer.Stop() + + return nil, cursor, okStatus() + } + + timer.Stop() + } +} + +// pullLocked assumes m.mu is held. +func (m *inMemoryMailbox) pullLocked(mailboxID string, cursor uint64, + maxEnvelopes uint32) ([]*mailboxpb.Envelope, uint64) { + + acked := m.ackedUpTo[mailboxID] + + var result []*mailboxpb.Envelope + var maxSeq uint64 + + for _, env := range m.mailboxes[mailboxID] { + if env.EventSeq < acked { + continue + } + if env.EventSeq < cursor { + continue + } + + clone, ok := cloneEnvelope(env) + if !ok { + continue + } + + result = append(result, clone) + if env.EventSeq > maxSeq { + maxSeq = env.EventSeq + } + + if uint32(len(result)) >= maxEnvelopes { + break + } + } + + if len(result) == 0 { + return nil, cursor + } + + return result, maxSeq + 1 +} + +// ackUpTo advances the ack cursor. +func (m *inMemoryMailbox) ackUpTo(mailboxID string, + cursor uint64) *mailboxpb.Status { + + m.mu.Lock() + defer m.mu.Unlock() + + if cursor > m.ackedUpTo[mailboxID] { + m.ackedUpTo[mailboxID] = cursor + } + + return okStatus() +} + +// getAckedUpTo returns the current ack watermark for a mailbox. +func (m *inMemoryMailbox) getAckedUpTo(mailboxID string) uint64 { + m.mu.Lock() + defer m.mu.Unlock() + + return m.ackedUpTo[mailboxID] +} + +// cloneEnvelope makes a deep copy of env and returns false if proto.Clone does +// not return the expected type. +func cloneEnvelope( + env *mailboxpb.Envelope, +) (*mailboxpb.Envelope, bool) { + + clone := proto.Clone(env) + typed, ok := clone.(*mailboxpb.Envelope) + if !ok { + return nil, false + } + + return typed, true +} + +// okStatus returns a successful mailbox status. +func okStatus() *mailboxpb.Status { + return &mailboxpb.Status{Ok: true} +} + +// fakeMailboxServiceClient adapts the in-memory edge to +// MailboxServiceClient. +type fakeMailboxServiceClient struct { + mb *inMemoryMailbox +} + +// Send implements mailboxpb.MailboxServiceClient. +func (c *fakeMailboxServiceClient) Send( + ctx context.Context, + in *mailboxpb.SendRequest, + _ ...grpc.CallOption, +) (*mailboxpb.SendResponse, error) { + + _ = ctx + + if in == nil { + return nil, fmt.Errorf("nil request") + } + + status := c.mb.send(in.Envelope) + + return &mailboxpb.SendResponse{Status: status}, nil +} + +// Pull implements mailboxpb.MailboxServiceClient. +func (c *fakeMailboxServiceClient) Pull( + ctx context.Context, + in *mailboxpb.PullRequest, + _ ...grpc.CallOption, +) (*mailboxpb.PullResponse, error) { + + if in == nil { + return nil, fmt.Errorf("nil request") + } + + wait := time.Duration(in.WaitTimeoutMs) * time.Millisecond + + envs, next, status := c.mb.pull( + ctx, in.MailboxId, in.Cursor, in.MaxEnvelopes, wait, + ) + + return &mailboxpb.PullResponse{ + Status: status, + Envelopes: envs, + NextCursor: next, + }, nil +} + +// AckUpTo implements mailboxpb.MailboxServiceClient. +func (c *fakeMailboxServiceClient) AckUpTo( + ctx context.Context, + in *mailboxpb.AckUpToRequest, + _ ...grpc.CallOption, +) (*mailboxpb.AckUpToResponse, error) { + + _ = ctx + + if in == nil { + return nil, fmt.Errorf("nil request") + } + + status := c.mb.ackUpTo(in.MailboxId, in.Cursor) + + return &mailboxpb.AckUpToResponse{Status: status}, nil +} + +// memCheckpointStore is a minimal in-memory implementation of the checkpoint +// subset of actor.DeliveryStore needed for ingress loop tests. Only +// SaveCheckpoint and LoadCheckpoint are implemented; all other methods panic. +type memCheckpointStore struct { + mu sync.Mutex + checkpoints map[string]*actor.Checkpoint +} + +// newMemCheckpointStore creates a new empty checkpoint store. +func newMemCheckpointStore() *memCheckpointStore { + return &memCheckpointStore{ + checkpoints: make(map[string]*actor.Checkpoint), + } +} + +// SaveCheckpoint persists a checkpoint in memory. +func (s *memCheckpointStore) SaveCheckpoint( + ctx context.Context, params actor.CheckpointParams, +) error { + + s.mu.Lock() + defer s.mu.Unlock() + + s.checkpoints[params.ActorID] = &actor.Checkpoint{ + ActorID: params.ActorID, + StateType: params.StateType, + StateData: params.StateData, + Version: params.Version, + UpdatedAt: time.Now(), + } + + return nil +} + +// LoadCheckpoint retrieves a previously saved checkpoint. +func (s *memCheckpointStore) LoadCheckpoint( + ctx context.Context, actorID string, +) (*actor.Checkpoint, error) { + + s.mu.Lock() + defer s.mu.Unlock() + + cp, ok := s.checkpoints[actorID] + if !ok { + return nil, nil + } + + return cp, nil +} + +// The remaining DeliveryStore methods are unused in connector tests and panic +// if called. + +func (s *memCheckpointStore) EnqueueMessage( + ctx context.Context, params actor.EnqueueParams, +) error { + + panic("not implemented") +} + +func (s *memCheckpointStore) LeaseNextMessage( + ctx context.Context, mailboxID string, leaseToken string, + leaseDuration time.Duration, +) (*actor.LeasedMessage, error) { + + panic("not implemented") +} + +func (s *memCheckpointStore) AckMessage( + ctx context.Context, id, leaseToken string, +) (int64, error) { + + panic("not implemented") +} + +func (s *memCheckpointStore) NackMessage( + ctx context.Context, id, leaseToken string, + retryAfter time.Duration, +) (int64, error) { + + panic("not implemented") +} + +func (s *memCheckpointStore) ExtendLease( + ctx context.Context, id, leaseToken string, + extension time.Duration, +) (int64, error) { + + panic("not implemented") +} + +func (s *memCheckpointStore) MoveToDeadLetter( + ctx context.Context, id, reason string, +) error { + + panic("not implemented") +} + +func (s *memCheckpointStore) DeleteMessage( + ctx context.Context, id string, +) error { + + panic("not implemented") +} + +func (s *memCheckpointStore) SaveAskResult( + ctx context.Context, params actor.AskResultParams, +) error { + + panic("not implemented") +} + +func (s *memCheckpointStore) GetAskResult( + ctx context.Context, promiseID string, +) (*actor.AskResult, error) { + + panic("not implemented") +} + +func (s *memCheckpointStore) DeleteAskResult( + ctx context.Context, promiseID string, +) error { + + panic("not implemented") +} + +func (s *memCheckpointStore) EnqueueOutbox( + ctx context.Context, params actor.OutboxParams, +) error { + + panic("not implemented") +} + +func (s *memCheckpointStore) ClaimOutboxBatch( + ctx context.Context, params actor.OutboxClaimParams, +) ([]actor.OutboxMessage, error) { + + panic("not implemented") +} + +func (s *memCheckpointStore) CompleteOutbox( + ctx context.Context, id, claimToken string, +) error { + + panic("not implemented") +} + +func (s *memCheckpointStore) FailOutbox( + ctx context.Context, id, claimToken string, +) error { + + panic("not implemented") +} + +func (s *memCheckpointStore) IsProcessed( + ctx context.Context, id string, +) (bool, error) { + + panic("not implemented") +} + +func (s *memCheckpointStore) MarkProcessed( + ctx context.Context, id, actorID string, + ttl time.Duration, +) error { + + panic("not implemented") +} + +func (s *memCheckpointStore) DeleteCheckpoint( + ctx context.Context, actorID string, +) error { + + panic("not implemented") +} + +func (s *memCheckpointStore) GetDeadLetter( + ctx context.Context, id string, +) (*actor.DeadLetter, error) { + + panic("not implemented") +} + +func (s *memCheckpointStore) ListDeadLetters( + ctx context.Context, actorID string, limit int, +) ([]actor.DeadLetter, error) { + + panic("not implemented") +} + +func (s *memCheckpointStore) DeleteDeadLetter( + ctx context.Context, id string, +) error { + + panic("not implemented") +} + +func (s *memCheckpointStore) ExpireLeases( + ctx context.Context, +) error { + + panic("not implemented") +} + +func (s *memCheckpointStore) CleanupExpired( + ctx context.Context, +) error { + + panic("not implemented") +} + +// Compile-time check. +var _ actor.DeliveryStore = (*memCheckpointStore)(nil) diff --git a/serverconn/unary_facade_test.go b/serverconn/unary_facade_test.go new file mode 100644 index 000000000..b0bef9871 --- /dev/null +++ b/serverconn/unary_facade_test.go @@ -0,0 +1,300 @@ +package serverconn + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" + mailboxrpc "github.com/lightninglabs/darepo-client/mailbox/rpc" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// TestUnaryFacade_SendRPC verifies that SendRPC constructs an envelope and +// sends it through the mailbox edge, returning the correlation and idempotency +// identifiers. +func TestUnaryFacade_SendRPC(t *testing.T) { + t.Parallel() + + actor, mb, _ := newTestConnector(t, nil) + facade := NewUnaryFacade(actor) + + method := mailboxrpc.ServiceMethod{ + Service: "test.Svc", + Method: "GetInfo", + } + + req := wrapperspb.String("hello") + + result, err := facade.SendRPC( + t.Context(), method, req, mailboxrpc.RPCOptions{}, + ) + require.NoError(t, err) + require.NotEmpty(t, result.CorrelationID) + require.NotEmpty(t, result.IdempotencyKey) + + // Verify the envelope was delivered to the server mailbox. + mb.mu.Lock() + envs := mb.mailboxes["server-1"] + mb.mu.Unlock() + + require.Len(t, envs, 1) + require.Equal(t, "client-1", envs[0].Sender) + require.Equal(t, "server-1", envs[0].Recipient) + require.Equal(t, + mailboxpb.RpcMeta_KIND_REQUEST, envs[0].Rpc.Kind, + ) + require.Equal(t, "test.Svc", envs[0].Rpc.Service) + require.Equal(t, "GetInfo", envs[0].Rpc.Method) + require.Equal(t, + result.CorrelationID, envs[0].Rpc.CorrelationId, + ) +} + +// TestUnaryFacade_SendRPC_ExplicitOptions verifies that caller-provided +// correlation ID and idempotency key are preserved. +func TestUnaryFacade_SendRPC_ExplicitOptions(t *testing.T) { + t.Parallel() + + actor, _, _ := newTestConnector(t, nil) + facade := NewUnaryFacade(actor) + + method := mailboxrpc.ServiceMethod{ + Service: "test.Svc", + Method: "GetInfo", + } + + opts := mailboxrpc.RPCOptions{ + CorrelationID: "my-corr-id", + IdempotencyKey: "my-idemp-key", + } + + result, err := facade.SendRPC( + t.Context(), method, + wrapperspb.String("hello"), opts, + ) + require.NoError(t, err) + require.Equal(t, "my-corr-id", result.CorrelationID) + require.Equal(t, "my-idemp-key", result.IdempotencyKey) +} + +// TestUnaryFacade_AwaitRPC verifies the full send-await round trip where the +// ingress loop delivers the response to the facade waiter. +func TestUnaryFacade_AwaitRPC(t *testing.T) { + t.Parallel() + + actor, mb, _ := newTestConnector(t, nil) + facade := NewUnaryFacade(actor) + + // Start ingress so responses can be pulled and delivered. + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + actor.StartIngress(ctx) + defer actor.StopIngress() + + // Send an RPC request. + method := mailboxrpc.ServiceMethod{ + Service: "test.Svc", + Method: "GetInfo", + } + + result, err := facade.SendRPC( + t.Context(), method, + wrapperspb.String("request"), mailboxrpc.RPCOptions{}, + ) + require.NoError(t, err) + + // Simulate a server response by injecting a KIND_RESPONSE envelope + // into the client's mailbox with the matching correlation ID. + responseMsg := wrapperspb.String("world") + responseBytes, err := proto.Marshal(responseMsg) + require.NoError(t, err) + + sendResponseToMailbox( + t, mb, "client-1", result.CorrelationID, responseBytes, + ) + + // Await should unmarshal the response. + var resp wrapperspb.StringValue + err = facade.AwaitRPC( + t.Context(), result.CorrelationID, &resp, + ) + require.NoError(t, err) + require.Equal(t, "world", resp.Value) +} + +// TestUnaryFacade_AwaitRPC_CancelledContext verifies that AwaitRPC returns +// the context error when the context is cancelled. +func TestUnaryFacade_AwaitRPC_CancelledContext(t *testing.T) { + t.Parallel() + + actor, _, _ := newTestConnector(t, nil) + facade := NewUnaryFacade(actor) + + // Start ingress. + ingressCtx, ingressCancel := context.WithCancel( + t.Context(), + ) + defer ingressCancel() + + actor.StartIngress(ingressCtx) + defer actor.StopIngress() + + // Create a context that we cancel immediately. + awaitCtx, awaitCancel := context.WithCancel(t.Context()) + awaitCancel() + + var resp wrapperspb.StringValue + err := facade.AwaitRPC(awaitCtx, "no-such-corr", &resp) + require.ErrorIs(t, err, context.Canceled) +} + +// TestUnaryFacade_ConcurrentInflight verifies that multiple concurrent +// send/await pairs do not lose or misroute responses. +func TestUnaryFacade_ConcurrentInflight(t *testing.T) { + t.Parallel() + + actor, mb, _ := newTestConnector(t, nil) + facade := NewUnaryFacade(actor) + + // Start ingress. + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + actor.StartIngress(ctx) + defer actor.StopIngress() + + const numRequests = 20 + method := mailboxrpc.ServiceMethod{ + Service: "test.Svc", + Method: "Echo", + } + + type roundTrip struct { + corrID string + input string + } + + // Send all requests first and collect correlation IDs. + trips := make([]roundTrip, numRequests) + for i := 0; i < numRequests; i++ { + input := wrapperspb.String( + fmt.Sprintf("req-%d", i), + ) + + result, err := facade.SendRPC( + t.Context(), method, input, + mailboxrpc.RPCOptions{}, + ) + require.NoError(t, err) + + trips[i] = roundTrip{ + corrID: result.CorrelationID, + input: input.Value, + } + } + + // Start all await goroutines first, then inject responses. This + // ensures waiters are registered before responses arrive. + var wg sync.WaitGroup + errors := make([]error, numRequests) + results := make([]string, numRequests) + + for i := 0; i < numRequests; i++ { + i := i + wg.Add(1) + + go func() { + defer wg.Done() + + awaitCtx, awaitCancel := context.WithTimeout( + t.Context(), 15*time.Second, + ) + defer awaitCancel() + + var resp wrapperspb.StringValue + errors[i] = facade.AwaitRPC( + awaitCtx, trips[i].corrID, &resp, + ) + results[i] = resp.Value + }() + } + + // Brief pause to let all waiters register before injecting + // responses. + time.Sleep(50 * time.Millisecond) + + // Inject responses in reverse order to stress routing. + for i := numRequests - 1; i >= 0; i-- { + responseMsg := wrapperspb.String("resp-" + trips[i].input) + responseBytes, err := proto.Marshal(responseMsg) + require.NoError(t, err) + + sendResponseToMailbox( + t, mb, "client-1", trips[i].corrID, responseBytes, + ) + } + + wg.Wait() + + for i := 0; i < numRequests; i++ { + require.NoError(t, errors[i], "request %d failed", i) + require.Equal( + t, "resp-"+trips[i].input, results[i], + "response mismatch for request %d", i, + ) + } +} + +// TestUnaryFacade_RPCClientInterface verifies the compile-time interface +// compliance check is satisfied. +func TestUnaryFacade_RPCClientInterface(t *testing.T) { + t.Parallel() + + // This test simply verifies that the compile-time check in + // unary_facade.go is valid. + var _ mailboxrpc.RPCClient = (*UnaryFacade)(nil) +} + +// TestUnaryFacade_AwaitRPC_NilBody verifies that AwaitRPC returns an error +// when the response envelope has a nil body. +func TestUnaryFacade_AwaitRPC_NilBody(t *testing.T) { + t.Parallel() + + actor, _, _ := newTestConnector(t, nil) + facade := NewUnaryFacade(actor) + + corrID := CorrelationID("nil-body-test") + + // Deliver an envelope with nil body directly via the response + // registry. We schedule the delivery after a short delay so that + // AwaitRPC has time to register its waiter. + go func() { + time.Sleep(50 * time.Millisecond) + + // Use deliverResponse which looks up and signals the + // waiter channel internally. + actor.deliverResponse(corrID, &mailboxpb.Envelope{ + Rpc: &mailboxpb.RpcMeta{ + Kind: mailboxpb.RpcMeta_KIND_RESPONSE, + CorrelationId: string(corrID), + }, + // Body is nil. + }) + }() + + ctx, cancel := context.WithTimeout( + t.Context(), 5*time.Second, + ) + defer cancel() + + var resp wrapperspb.StringValue + err := facade.AwaitRPC(ctx, string(corrID), &resp) + require.Error(t, err) + require.Contains(t, err.Error(), "nil body") +} From faf6bf70f061e2a31c488d1a208b31ada9376951 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Tue, 17 Feb 2026 22:08:15 -0800 Subject: [PATCH 08/14] mailbox/conn: extract shared connector primitives In this commit, we introduce the mailbox/conn package to house reusable building blocks shared by client-side and server-side connector runtimes. The AckState struct and its TLV encode/decode are moved here wholesale from serverconn/types.go, along with the CorrelationID and IdempotencyKey named types. A ResponseRegistry provides in-memory correlation waiter tracking with early-response buffering (a response that arrives before its waiter is registered gets cloned and held until the waiter appears) and configurable TTL-based stale cleanup. Envelope identity helpers (StableEventMsgID, StableEventIdempotencyKey) derive deterministic SHA-256-based identifiers from a serialized payload so durable retries of the same semantic event produce the same message and idempotency keys for remote dedupe. The test suite includes both table-driven unit tests and property-based randomized checks (via pgregory.net/rapid) for the ack state invariants and response registry interleaving behavior. --- mailbox/conn/ack_state.go | 128 ++++++++++++ mailbox/conn/ack_state_property_test.go | 104 ++++++++++ mailbox/conn/ack_state_test.go | 96 +++++++++ mailbox/conn/doc.go | 12 ++ mailbox/conn/envelope_identity.go | 25 +++ mailbox/conn/envelope_identity_test.go | 24 +++ mailbox/conn/proto_record.go | 101 ++++++++++ mailbox/conn/proto_record_test.go | 135 +++++++++++++ mailbox/conn/response_registry.go | 183 ++++++++++++++++++ .../conn/response_registry_property_test.go | 138 +++++++++++++ mailbox/conn/response_registry_test.go | 118 +++++++++++ 11 files changed, 1064 insertions(+) create mode 100644 mailbox/conn/ack_state.go create mode 100644 mailbox/conn/ack_state_property_test.go create mode 100644 mailbox/conn/ack_state_test.go create mode 100644 mailbox/conn/doc.go create mode 100644 mailbox/conn/envelope_identity.go create mode 100644 mailbox/conn/envelope_identity_test.go create mode 100644 mailbox/conn/proto_record.go create mode 100644 mailbox/conn/proto_record_test.go create mode 100644 mailbox/conn/response_registry.go create mode 100644 mailbox/conn/response_registry_property_test.go create mode 100644 mailbox/conn/response_registry_test.go diff --git a/mailbox/conn/ack_state.go b/mailbox/conn/ack_state.go new file mode 100644 index 000000000..188f8fcb3 --- /dev/null +++ b/mailbox/conn/ack_state.go @@ -0,0 +1,128 @@ +package conn + +import ( + "io" + + "github.com/lightningnetwork/lnd/tlv" +) + +// CorrelationID links a mailbox request to its response. +type CorrelationID string + +// IdempotencyKey deduplicates a semantic operation across retries. +type IdempotencyKey string + +// CheckpointStateType is the checkpoint state type name used to persist ack +// watermark state. +const CheckpointStateType = "AckState" + +// TLV record type constants for AckState checkpoint serialization. +const ( + pullCursorRecordType tlv.Type = 1 + dispatchCommittedToRecordType tlv.Type = 2 + ackTargetRecordType tlv.Type = 3 + ackCommittedToRecordType tlv.Type = 4 +) + +// AckState tracks the four cursor variables that govern safe ack progression. +// All fields are monotonic and must not decrease during normal operation. +// +// The state machine enforces: +// +// ack_committed_to <= dispatch_committed_to +// +// Cursor never advances past non-durable local work. Repeated acks are safe +// and idempotent. +type AckState struct { + // PullCursor is the cursor for the next Pull call. After a successful + // ack, PullCursor advances to at least the acked position. + PullCursor uint64 + + // DispatchCommittedTo is the max cursor whose envelopes were durably + // committed to local processing. + DispatchCommittedTo uint64 + + // AckTarget is the max cursor that should be acked remotely. + AckTarget uint64 + + // AckCommittedTo is the last cursor successfully acked remotely. + AckCommittedTo uint64 +} + +// AdvanceDispatch updates state after durable dispatch through nextCursor. +func (s *AckState) AdvanceDispatch(nextCursor uint64) { + if nextCursor > s.DispatchCommittedTo { + s.DispatchCommittedTo = nextCursor + } + + if s.DispatchCommittedTo > s.AckTarget { + s.AckTarget = s.DispatchCommittedTo + } +} + +// AdvanceAck updates state after a successful AckUpTo call. +func (s *AckState) AdvanceAck() { + s.AckCommittedTo = s.AckTarget + + // Defensive: under normal operation PullCursor is always advanced + // to at least AckTarget before AdvanceAck, so this branch is + // unreachable. It exists as a crash-recovery safety net for the + // case where a partial checkpoint write leaves AckCommittedTo > + // PullCursor after restart. + if s.AckCommittedTo > s.PullCursor { + s.PullCursor = s.AckCommittedTo + } +} + +// NeedsAck returns true when AckTarget has advanced past AckCommittedTo. +func (s *AckState) NeedsAck() bool { + return s.AckTarget > s.AckCommittedTo +} + +// Encode serializes AckState to a TLV stream. +func (s *AckState) Encode(w io.Writer) error { + records := []tlv.Record{ + tlv.MakePrimitiveRecord( + pullCursorRecordType, &s.PullCursor, + ), + tlv.MakePrimitiveRecord( + dispatchCommittedToRecordType, &s.DispatchCommittedTo, + ), + tlv.MakePrimitiveRecord(ackTargetRecordType, &s.AckTarget), + tlv.MakePrimitiveRecord( + ackCommittedToRecordType, &s.AckCommittedTo, + ), + } + + stream, err := tlv.NewStream(records...) + if err != nil { + return err + } + + return stream.Encode(w) +} + +// Decode deserializes AckState from a TLV stream. +func (s *AckState) Decode(r io.Reader) error { + records := []tlv.Record{ + tlv.MakePrimitiveRecord( + pullCursorRecordType, &s.PullCursor, + ), + tlv.MakePrimitiveRecord( + dispatchCommittedToRecordType, &s.DispatchCommittedTo, + ), + tlv.MakePrimitiveRecord(ackTargetRecordType, &s.AckTarget), + tlv.MakePrimitiveRecord( + ackCommittedToRecordType, &s.AckCommittedTo, + ), + } + + stream, err := tlv.NewStream(records...) + if err != nil { + return err + } + + _, err = stream.DecodeWithParsedTypes(r) + + return err +} diff --git a/mailbox/conn/ack_state_property_test.go b/mailbox/conn/ack_state_property_test.go new file mode 100644 index 000000000..4b6293e0e --- /dev/null +++ b/mailbox/conn/ack_state_property_test.go @@ -0,0 +1,104 @@ +package conn + +import ( + "testing" + + "pgregory.net/rapid" +) + +// TestAckState_MonotonicInvariants_Property verifies that random operation +// sequences preserve AckState invariants. +func TestAckState_MonotonicInvariants_Property(t *testing.T) { + t.Parallel() + + rapid.Check(t, func(rt *rapid.T) { + var state AckState + steps := rapid.IntRange(1, 300).Draw(rt, "steps") + + for i := 0; i < steps; i++ { + prev := state + op := rapid.IntRange(0, 2).Draw(rt, "op") + + switch op { + case 0: + nextCursor := rapid. + Uint64().Draw(rt, "next_cursor") + state.AdvanceDispatch(nextCursor) + + case 1: + state.AdvanceAck() + + case 2: + // PullCursor can advance externally in ingress. + // Keep that monotonic behavior. + nextPull := rapid.Uint64().Draw(rt, "next_pull") + if nextPull > state.PullCursor { + state.PullCursor = nextPull + } + } + + if state.DispatchCommittedTo < + prev.DispatchCommittedTo { + + rt.Fatalf("dispatch regressed: %d -> %d", + prev.DispatchCommittedTo, + state.DispatchCommittedTo) + } + + if state.AckTarget < prev.AckTarget { + rt.Fatalf("ack target regressed: %d -> %d", + prev.AckTarget, state.AckTarget) + } + + if state.AckCommittedTo < prev.AckCommittedTo { + rt.Fatalf("ack committed regressed: %d -> %d", + prev.AckCommittedTo, + state.AckCommittedTo) + } + + if state.PullCursor < prev.PullCursor { + rt.Fatalf("pull cursor regressed: %d -> %d", + prev.PullCursor, state.PullCursor) + } + + if state.AckTarget < state.DispatchCommittedTo { + rt.Fatalf( + "ack target below committed: %d < %d", + state.AckTarget, + state.DispatchCommittedTo, + ) + } + + if state.AckCommittedTo > state.AckTarget { + rt.Fatalf("ack committed above target: %d > %d", + state.AckCommittedTo, state.AckTarget) + } + + if state.AckCommittedTo > + state.DispatchCommittedTo { + + rt.Fatalf( + "ack committed above dispatch: %d > %d", + state.AckCommittedTo, + state.DispatchCommittedTo, + ) + } + + if state.PullCursor < state.AckCommittedTo { + rt.Fatalf( + "pull cursor behind ack: %d < %d", + state.PullCursor, state.AckCommittedTo, + ) + } + + expectedNeedsAck := state.AckTarget > + state.AckCommittedTo + if state.NeedsAck() != expectedNeedsAck { + rt.Fatalf( + "NeedsAck mismatch: got=%v expected=%v", + state.NeedsAck(), expectedNeedsAck, + ) + } + } + }) +} diff --git a/mailbox/conn/ack_state_test.go b/mailbox/conn/ack_state_test.go new file mode 100644 index 000000000..c46f300ee --- /dev/null +++ b/mailbox/conn/ack_state_test.go @@ -0,0 +1,96 @@ +package conn + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestAckState_AdvanceDispatch_SetsAckTarget verifies that AdvanceDispatch +// moves both DispatchCommittedTo and AckTarget forward. +func TestAckState_AdvanceDispatch_SetsAckTarget(t *testing.T) { + t.Parallel() + + var s AckState + s.AdvanceDispatch(10) + + require.Equal(t, uint64(10), s.DispatchCommittedTo) + require.Equal(t, uint64(10), s.AckTarget) +} + +// TestAckState_AdvanceDispatch_Monotonic verifies that AdvanceDispatch never +// decreases DispatchCommittedTo or AckTarget. +func TestAckState_AdvanceDispatch_Monotonic(t *testing.T) { + t.Parallel() + + var s AckState + s.AdvanceDispatch(10) + s.AdvanceDispatch(5) + s.AdvanceDispatch(7) + + require.Equal(t, uint64(10), s.DispatchCommittedTo) + require.Equal(t, uint64(10), s.AckTarget) + + s.AdvanceDispatch(15) + require.Equal(t, uint64(15), s.DispatchCommittedTo) + require.Equal(t, uint64(15), s.AckTarget) +} + +// TestAckState_AdvanceAck_UpdatesPullCursor verifies that AdvanceAck updates +// AckCommittedTo and advances PullCursor when needed. +func TestAckState_AdvanceAck_UpdatesPullCursor(t *testing.T) { + t.Parallel() + + s := AckState{ + PullCursor: 0, + DispatchCommittedTo: 10, + AckTarget: 10, + AckCommittedTo: 0, + } + + s.AdvanceAck() + + require.Equal(t, uint64(10), s.AckCommittedTo) + require.Equal(t, uint64(10), s.PullCursor) +} + +// TestAckState_NeedsAck verifies NeedsAck behavior across key cases. +func TestAckState_NeedsAck(t *testing.T) { + t.Parallel() + + s0 := AckState{} + require.False(t, s0.NeedsAck()) + + s1 := AckState{ + AckTarget: 10, + AckCommittedTo: 5, + } + require.True(t, s1.NeedsAck()) + + s2 := AckState{ + AckTarget: 10, + AckCommittedTo: 10, + } + require.False(t, s2.NeedsAck()) +} + +// TestAckState_EncodeDecode verifies TLV round-trip serialization. +func TestAckState_EncodeDecode(t *testing.T) { + t.Parallel() + + original := AckState{ + PullCursor: 42, + DispatchCommittedTo: 100, + AckTarget: 100, + AckCommittedTo: 90, + } + + var buf bytes.Buffer + require.NoError(t, original.Encode(&buf)) + + var decoded AckState + require.NoError(t, decoded.Decode(bytes.NewReader(buf.Bytes()))) + + require.Equal(t, original, decoded) +} diff --git a/mailbox/conn/doc.go b/mailbox/conn/doc.go new file mode 100644 index 000000000..aea24b42f --- /dev/null +++ b/mailbox/conn/doc.go @@ -0,0 +1,12 @@ +// Package conn provides reusable mailbox connector primitives shared by +// client-side and server-side connector runtimes. +// +// The package intentionally contains protocol-adjacent building blocks only: +// +// - typed identifiers and deterministic idempotency helpers, +// - ack watermark state machine encoding for checkpoint persistence, and +// - in-memory response waiter registry for unary correlation delivery. +// +// Higher-level runtime wiring (actor lifecycle, dispatcher tables, transport +// loops) lives in connector-specific packages such as serverconn. +package conn diff --git a/mailbox/conn/envelope_identity.go b/mailbox/conn/envelope_identity.go new file mode 100644 index 000000000..446642022 --- /dev/null +++ b/mailbox/conn/envelope_identity.go @@ -0,0 +1,25 @@ +package conn + +import ( + "crypto/sha256" + "encoding/hex" +) + +// StableEventMsgID derives a deterministic mailbox message ID from payload. +func StableEventMsgID(payload []byte) string { + return "evt-" + shortPayloadHash(payload) +} + +// StableEventIdempotencyKey derives a deterministic idempotency key from +// payload. +func StableEventIdempotencyKey(payload []byte) string { + return "idem-" + shortPayloadHash(payload) +} + +// shortPayloadHash returns a compact hex-encoded hash suffix for payload. +func shortPayloadHash(payload []byte) string { + sum := sha256.Sum256(payload) + + // 16 bytes (32 hex chars) is enough for internal dedupe IDs. + return hex.EncodeToString(sum[:16]) +} diff --git a/mailbox/conn/envelope_identity_test.go b/mailbox/conn/envelope_identity_test.go new file mode 100644 index 000000000..644feb2bf --- /dev/null +++ b/mailbox/conn/envelope_identity_test.go @@ -0,0 +1,24 @@ +package conn + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestStableEventIDs verifies deterministic identity derivation from payload. +func TestStableEventIDs(t *testing.T) { + t.Parallel() + + payload := []byte("same-payload") + + msgID1 := StableEventMsgID(payload) + msgID2 := StableEventMsgID(payload) + require.Equal(t, msgID1, msgID2) + require.Contains(t, msgID1, "evt-") + + idem1 := StableEventIdempotencyKey(payload) + idem2 := StableEventIdempotencyKey(payload) + require.Equal(t, idem1, idem2) + require.Contains(t, idem1, "idem-") +} diff --git a/mailbox/conn/proto_record.go b/mailbox/conn/proto_record.go new file mode 100644 index 000000000..2a037b0a5 --- /dev/null +++ b/mailbox/conn/proto_record.go @@ -0,0 +1,101 @@ +package conn + +import ( + "io" + "reflect" + + "github.com/lightningnetwork/lnd/tlv" + "google.golang.org/protobuf/proto" +) + +// protoRecordType is a placeholder TLV type used inside the Record() method. +// Callers must wrap the record via tlv.NewRecordT (which assigns the real +// type), not call Record() directly — type 0 would silently conflict with +// other type-0 records. +var protoRecordType tlv.Type = 0 + +// WrappedProto adapts a proto.Message for use as a tlv.RecordT field. The +// proto is marshaled to bytes on encode and unmarshaled back on decode, +// keeping the TLV codec contract satisfied while storing structured proto +// payloads. +type WrappedProto[T proto.Message] struct { + Val T +} + +// isNil reports whether the generic proto value is nil. Direct nil comparison +// on a generic interface-constrained type is not allowed in Go, so we use +// reflect to check. +func isNil[T proto.Message](v T) bool { + return reflect.ValueOf(&v).Elem().IsNil() +} + +// Record returns a TLV record that serializes the proto message to bytes +// for TLV storage. +func (w *WrappedProto[T]) Record() tlv.Record { + sizeFunc := func() uint64 { + if isNil(w.Val) { + return 0 + } + + return uint64(proto.Size(w.Val)) + } + + return tlv.MakeDynamicRecord( + protoRecordType, w, sizeFunc, + wrappedProtoEncoder[T], wrappedProtoDecoder[T], + ) +} + +// wrappedProtoEncoder marshals the proto message to the TLV writer. +func wrappedProtoEncoder[T proto.Message]( + w io.Writer, val interface{}, _ *[8]byte, +) error { + + wp, ok := val.(*WrappedProto[T]) + if !ok { + return tlv.NewTypeForEncodingErr(val, "WrappedProto") + } + + if isNil(wp.Val) { + return nil + } + + data, err := (proto.MarshalOptions{ + Deterministic: true, + }).Marshal(wp.Val) + if err != nil { + return err + } + + _, err = w.Write(data) + + return err +} + +// wrappedProtoDecoder reads bytes from the TLV reader and unmarshals them +// into the proto message. The caller must pre-set Val to a typed zero value +// before decode so the correct concrete type is available. +func wrappedProtoDecoder[T proto.Message]( + r io.Reader, val interface{}, _ *[8]byte, l uint64, +) error { + + wp, ok := val.(*WrappedProto[T]) + if !ok { + return tlv.NewTypeForDecodingErr( + val, "WrappedProto", l, l, + ) + } + + data := make([]byte, l) + if _, err := io.ReadFull(r, data); err != nil { + return err + } + + // Reset the message before unmarshaling to clear any previous + // state, then unmarshal the fresh bytes. + if !isNil(wp.Val) { + proto.Reset(wp.Val) + } + + return proto.Unmarshal(data, wp.Val) +} diff --git a/mailbox/conn/proto_record_test.go b/mailbox/conn/proto_record_test.go new file mode 100644 index 000000000..8ed1ae794 --- /dev/null +++ b/mailbox/conn/proto_record_test.go @@ -0,0 +1,135 @@ +package conn + +import ( + "bytes" + "testing" + + "github.com/lightningnetwork/lnd/tlv" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// TestWrappedProto_RoundTrip verifies that a proto message survives TLV +// encode → decode and retains its field values. +func TestWrappedProto_RoundTrip(t *testing.T) { + t.Parallel() + + original := &wrapperspb.StringValue{Value: "hello-proto"} + + type testTLV = tlv.TlvType1 + + encRecord := tlv.NewRecordT[testTLV]( + WrappedProto[*wrapperspb.StringValue]{Val: original}, + ) + + var buf bytes.Buffer + stream, err := tlv.NewStream(encRecord.Record()) + require.NoError(t, err) + require.NoError(t, stream.Encode(&buf)) + + // Decode into a fresh WrappedProto with a pre-allocated zero + // value for the concrete proto type. + decRecord := tlv.ZeroRecordT[testTLV, + WrappedProto[*wrapperspb.StringValue], + ]() + decRecord.Val.Val = &wrapperspb.StringValue{} + + decStream, err := tlv.NewStream(decRecord.Record()) + require.NoError(t, err) + require.NoError(t, decStream.Decode(&buf)) + + require.Equal(t, "hello-proto", decRecord.Val.Val.Value) +} + +// TestWrappedProto_EmptyPayload verifies that encoding an empty proto +// produces a decodable payload that round-trips cleanly. +func TestWrappedProto_EmptyPayload(t *testing.T) { + t.Parallel() + + type testTLV = tlv.TlvType2 + + original := &wrapperspb.StringValue{} + encRecord := tlv.NewRecordT[testTLV]( + WrappedProto[*wrapperspb.StringValue]{Val: original}, + ) + + var buf bytes.Buffer + stream, err := tlv.NewStream(encRecord.Record()) + require.NoError(t, err) + require.NoError(t, stream.Encode(&buf)) + + decRecord := tlv.ZeroRecordT[testTLV, + WrappedProto[*wrapperspb.StringValue], + ]() + decRecord.Val.Val = &wrapperspb.StringValue{} + + decStream, err := tlv.NewStream(decRecord.Record()) + require.NoError(t, err) + require.NoError(t, decStream.Decode(&buf)) + + require.Equal(t, "", decRecord.Val.Val.Value) +} + +// TestWrappedProto_NilValue verifies that isNil correctly detects a nil +// proto value inside the generic wrapper. +func TestWrappedProto_NilValue(t *testing.T) { + t.Parallel() + + wp := WrappedProto[*wrapperspb.StringValue]{} + require.True(t, isNil(wp.Val)) + + wp.Val = &wrapperspb.StringValue{Value: "set"} + require.False(t, isNil(wp.Val)) +} + +// TestWrappedProto_NilEncodes verifies that a nil proto value produces +// a valid (empty) TLV encoding without errors. +func TestWrappedProto_NilEncodes(t *testing.T) { + t.Parallel() + + type testTLV = tlv.TlvType4 + + encRecord := tlv.NewRecordT[testTLV]( + WrappedProto[*wrapperspb.StringValue]{}, + ) + + var buf bytes.Buffer + stream, err := tlv.NewStream(encRecord.Record()) + require.NoError(t, err) + require.NoError(t, stream.Encode(&buf)) + + // Should produce a valid but minimal TLV encoding. + require.Greater(t, buf.Len(), 0) +} + +// TestWrappedProto_DecodeOverwritesPreviousState verifies that decoding +// into a WrappedProto that already holds data replaces the old content. +func TestWrappedProto_DecodeOverwritesPreviousState(t *testing.T) { + t.Parallel() + + type testTLV = tlv.TlvType3 + + // Encode "new-value". + encRecord := tlv.NewRecordT[testTLV]( + WrappedProto[*wrapperspb.StringValue]{ + Val: &wrapperspb.StringValue{Value: "new-value"}, + }, + ) + + var buf bytes.Buffer + stream, err := tlv.NewStream(encRecord.Record()) + require.NoError(t, err) + require.NoError(t, stream.Encode(&buf)) + + // Pre-populate the decode target with stale data. + decRecord := tlv.ZeroRecordT[testTLV, + WrappedProto[*wrapperspb.StringValue], + ]() + decRecord.Val.Val = &wrapperspb.StringValue{Value: "stale"} + + decStream, err := tlv.NewStream(decRecord.Record()) + require.NoError(t, err) + require.NoError(t, decStream.Decode(&buf)) + + require.Equal(t, "new-value", decRecord.Val.Val.Value) +} diff --git a/mailbox/conn/response_registry.go b/mailbox/conn/response_registry.go new file mode 100644 index 000000000..76c43938b --- /dev/null +++ b/mailbox/conn/response_registry.go @@ -0,0 +1,183 @@ +package conn + +import ( + "fmt" + "sync" + "time" + + "github.com/lightninglabs/darepo-client/baselib/actor" + mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" + fn "github.com/lightningnetwork/lnd/fn/v2" + "google.golang.org/protobuf/proto" +) + +// DefaultResponseWaiterTTL bounds stale waiter and buffered response retention +// when no explicit TTL is configured. +const DefaultResponseWaiterTTL = 10 * time.Minute + +// ErrWaiterExpired is returned when a response waiter is pruned due to TTL +// expiration while an AwaitRPC caller is still blocked. +var ErrWaiterExpired = fmt.Errorf("response waiter expired") + +// ErrWaiterCancelled is returned when a response waiter is explicitly removed +// before a response arrives. +var ErrWaiterCancelled = fmt.Errorf("response waiter cancelled") + +// responseWaiter captures the promise and creation time for a single +// correlation ID. +type responseWaiter struct { + // Promise is completed when a response arrives, the waiter is + // pruned, or the waiter is explicitly removed. + Promise actor.Promise[*mailboxpb.Envelope] + + // Created records waiter registration time for stale cleanup. + Created time.Time +} + +// bufferedResponse keeps a cloned envelope until a waiter is registered. +type bufferedResponse struct { + Envelope *mailboxpb.Envelope + Created time.Time +} + +// ResponseRegistry tracks correlation waiters and early responses. Waiters +// use actor.Future for context-aware blocking with automatic error signaling +// on TTL expiry. +type ResponseRegistry struct { + mu sync.Mutex + + waiters map[CorrelationID]*responseWaiter + pending map[CorrelationID]*bufferedResponse + waiterTTL time.Duration +} + +// NewResponseRegistry constructs a response registry with stale-state cleanup. +func NewResponseRegistry(waiterTTL time.Duration) *ResponseRegistry { + if waiterTTL <= 0 { + waiterTTL = DefaultResponseWaiterTTL + } + + return &ResponseRegistry{ + waiters: make(map[CorrelationID]*responseWaiter), + pending: make(map[CorrelationID]*bufferedResponse), + waiterTTL: waiterTTL, + } +} + +// RegisterWaiter registers or reuses a waiter for correlation ID id. Returns +// an actor.Future that completes when the response arrives, the waiter +// expires, or the waiter is explicitly removed. +func (r *ResponseRegistry) RegisterWaiter( + id CorrelationID, +) actor.Future[*mailboxpb.Envelope] { + + r.mu.Lock() + defer r.mu.Unlock() + + now := time.Now() + r.pruneStaleLocked(now) + + waiter, ok := r.waiters[id] + if !ok { + promise := actor.NewPromise[*mailboxpb.Envelope]() + waiter = &responseWaiter{ + Promise: promise, + Created: now, + } + + r.waiters[id] = waiter + } + + // If a response arrived before the waiter was registered, complete + // the promise immediately with the buffered envelope. + if pending, ok := r.pending[id]; ok { + waiter.Promise.Complete(fn.Ok(pending.Envelope)) + + delete(r.pending, id) + } + + return waiter.Promise.Future() +} + +// RemoveWaiter removes an existing waiter for correlation ID id. Any +// goroutine blocked on the associated Future receives ErrWaiterCancelled. +func (r *ResponseRegistry) RemoveWaiter(id CorrelationID) { + r.mu.Lock() + defer r.mu.Unlock() + + if waiter, ok := r.waiters[id]; ok { + waiter.Promise.Complete( + fn.Err[*mailboxpb.Envelope](ErrWaiterCancelled), + ) + + delete(r.waiters, id) + } +} + +// DeliverResponse delivers a response envelope for correlation ID id. +// +// If a waiter exists, the promise is completed with the envelope. If a waiter +// does not yet exist, the first response is buffered so a later RegisterWaiter +// call still receives it. +func (r *ResponseRegistry) DeliverResponse( + id CorrelationID, env *mailboxpb.Envelope, +) bool { + + if env == nil { + return false + } + + r.mu.Lock() + defer r.mu.Unlock() + + r.pruneStaleLocked(time.Now()) + + if waiter, ok := r.waiters[id]; ok { + waiter.Promise.Complete(fn.Ok(env)) + + return true + } + + if _, exists := r.pending[id]; exists { + return true + } + + responseCopy, ok := proto.Clone(env).(*mailboxpb.Envelope) + if !ok { + return false + } + + r.pending[id] = &bufferedResponse{ + Envelope: responseCopy, + Created: time.Now(), + } + + return true +} + +// pruneStaleLocked removes stale waiters and buffered responses. Stale +// waiters have their promises completed with ErrWaiterExpired so blocked +// callers wake up with a clear error rather than hanging. +func (r *ResponseRegistry) pruneStaleLocked(now time.Time) { + if r.waiterTTL <= 0 { + return + } + + for id, waiter := range r.waiters { + if now.Sub(waiter.Created) > r.waiterTTL { + waiter.Promise.Complete( + fn.Err[*mailboxpb.Envelope]( + ErrWaiterExpired, + ), + ) + + delete(r.waiters, id) + } + } + + for id, response := range r.pending { + if now.Sub(response.Created) > r.waiterTTL { + delete(r.pending, id) + } + } +} diff --git a/mailbox/conn/response_registry_property_test.go b/mailbox/conn/response_registry_property_test.go new file mode 100644 index 000000000..039c091be --- /dev/null +++ b/mailbox/conn/response_registry_property_test.go @@ -0,0 +1,138 @@ +package conn + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/lightninglabs/darepo-client/baselib/actor" + mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" + "pgregory.net/rapid" +) + +// awaitNow performs a non-blocking await on a Future by using a +// context with a very short deadline. This is the Future-based +// equivalent of a non-blocking channel receive. +func awaitNow( + future actor.Future[*mailboxpb.Envelope], +) (*mailboxpb.Envelope, bool) { + + ctx, cancel := context.WithTimeout( + context.Background(), time.Millisecond, + ) + defer cancel() + + result := future.Await(ctx) + if result.IsErr() { + return nil, false + } + + var env *mailboxpb.Envelope + result.WhenOk(func(e *mailboxpb.Envelope) { + env = e + }) + + return env, true +} + +// TestResponseRegistry_Interleavings_Property validates response-registry +// invariants under randomized register/remove/deliver interleavings. +// The model tracks three pieces of state: whether a waiter entry exists +// in the registry map, whether its promise has been settled (completed), +// and whether a buffered response is pending for the correlation ID. +func TestResponseRegistry_Interleavings_Property(t *testing.T) { + t.Parallel() + + rapid.Check(t, func(rt *rapid.T) { + registry := NewResponseRegistry(time.Hour) + id := CorrelationID("property-correlation") + + var ( + future actor.Future[*mailboxpb.Envelope] + waiterExists bool + futureSettled bool + + pendingMsg string + hasPending bool + ) + + steps := rapid.IntRange(20, 200).Draw(rt, "steps") + + for i := range steps { + opLabel := fmt.Sprintf("op_%d", i) + op := rapid.IntRange(0, 2).Draw(rt, opLabel) + + switch op { + case 0: + future = registry.RegisterWaiter(id) + + if !waiterExists { + // Fresh waiter created with a new + // unsettled promise. + waiterExists = true + futureSettled = false + } + + // If a buffered response exists and the + // promise hasn't been settled yet, it should + // be completed immediately. + if hasPending && !futureSettled { + env, ok := awaitNow(future) + if !ok { + rt.Fatalf("missing pending") + } + + if env == nil || + env.MsgId != pendingMsg { + + rt.Fatalf( + "pending mismatch: %v", + env, + ) + } + + hasPending = false + futureSettled = true + } + + case 1: + registry.RemoveWaiter(id) + waiterExists = false + futureSettled = false + + case 2: + msgID := fmt.Sprintf( + "msg-%d-%d", i, + rapid.Int().Draw(rt, "msg_rand"), + ) + + ok := registry.DeliverResponse( + id, &mailboxpb.Envelope{MsgId: msgID}, + ) + if !ok { + rt.Fatalf("delivery failed") + } + + if waiterExists && !futureSettled { + env, got := awaitNow(future) + if !got { + rt.Fatalf("waiter missing") + } + + if env == nil || env.MsgId != msgID { + rt.Fatalf( + "waiter mismatch: %v", + env, + ) + } + + futureSettled = true + } else if !waiterExists && !hasPending { + pendingMsg = msgID + hasPending = true + } + } + } + }) +} diff --git a/mailbox/conn/response_registry_test.go b/mailbox/conn/response_registry_test.go new file mode 100644 index 000000000..00def4edc --- /dev/null +++ b/mailbox/conn/response_registry_test.go @@ -0,0 +1,118 @@ +package conn + +import ( + "testing" + "time" + + mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" + "github.com/stretchr/testify/require" +) + +// TestResponseRegistry_DeliverBeforeRegister verifies an early response is +// buffered and later delivered when a waiter registers. +func TestResponseRegistry_DeliverBeforeRegister(t *testing.T) { + t.Parallel() + + registry := NewResponseRegistry(time.Minute) + id := CorrelationID("corr-1") + env := &mailboxpb.Envelope{ + MsgId: "msg-1", + } + + delivered := registry.DeliverResponse(id, env) + require.True(t, delivered) + + future := registry.RegisterWaiter(id) + + // The future should already be completed with the buffered response. + got := future.Await(t.Context()).UnwrapOrFail(t) + require.Equal(t, env.MsgId, got.MsgId) +} + +// TestResponseRegistry_RegisterThenDeliver verifies an active waiter receives +// a delivered response. +func TestResponseRegistry_RegisterThenDeliver(t *testing.T) { + t.Parallel() + + registry := NewResponseRegistry(time.Minute) + id := CorrelationID("corr-2") + future := registry.RegisterWaiter(id) + + delivered := registry.DeliverResponse(id, &mailboxpb.Envelope{ + MsgId: "msg-2", + }) + require.True(t, delivered) + + got := future.Await(t.Context()).UnwrapOrFail(t) + require.Equal(t, "msg-2", got.MsgId) +} + +// TestResponseRegistry_TTLPrunesPending verifies stale pending responses are +// removed after TTL expiry. +func TestResponseRegistry_TTLPrunesPending(t *testing.T) { + t.Parallel() + + registry := NewResponseRegistry(5 * time.Millisecond) + id := CorrelationID("corr-3") + + require.True(t, registry.DeliverResponse(id, &mailboxpb.Envelope{ + MsgId: "stale", + })) + + time.Sleep(20 * time.Millisecond) + + // Register after the buffered response has expired. The future + // should not be immediately completed. + future := registry.RegisterWaiter(id) + + // Deliver a fresh response to complete the future; the stale one + // should have been pruned. + registry.DeliverResponse(id, &mailboxpb.Envelope{ + MsgId: "fresh", + }) + + got := future.Await(t.Context()).UnwrapOrFail(t) + require.Equal(t, "fresh", got.MsgId) +} + +// TestResponseRegistry_TTLPrunesWaiter verifies that a stale waiter is pruned +// and the blocked Future receives ErrWaiterExpired. +func TestResponseRegistry_TTLPrunesWaiter(t *testing.T) { + t.Parallel() + + registry := NewResponseRegistry(5 * time.Millisecond) + id := CorrelationID("corr-expire") + future := registry.RegisterWaiter(id) + + time.Sleep(20 * time.Millisecond) + + // Trigger prune by registering a different waiter. + registry.RegisterWaiter(CorrelationID("trigger-prune")) + + result := future.Await(t.Context()) + require.ErrorIs(t, result.Err(), ErrWaiterExpired) +} + +// TestResponseRegistry_RemoveWaiterSignalsCancelled verifies that removing a +// waiter completes the Future with ErrWaiterCancelled. +func TestResponseRegistry_RemoveWaiterSignalsCancelled(t *testing.T) { + t.Parallel() + + registry := NewResponseRegistry(time.Minute) + id := CorrelationID("corr-cancel") + future := registry.RegisterWaiter(id) + + registry.RemoveWaiter(id) + + result := future.Await(t.Context()) + require.ErrorIs(t, result.Err(), ErrWaiterCancelled) +} + +// TestResponseRegistry_DeliverNilReturnsFalse verifies nil envelope +// delivery is rejected. +func TestResponseRegistry_DeliverNilReturnsFalse(t *testing.T) { + t.Parallel() + + registry := NewResponseRegistry(time.Minute) + require.False(t, registry.DeliverResponse("any", nil)) +} From 4dd1237254540dc554d0254fc8a49eea814ea9f1 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Tue, 17 Feb 2026 22:08:34 -0800 Subject: [PATCH 09/14] serverconn: delegate to mailbox/conn, fix response race, add egress identity In this commit, we wire the serverconn package to use the shared mailbox/conn primitives and fix two correctness gaps identified during review. The type aliases for CorrelationID, IdempotencyKey, AckState, and ResponseWaiter now point to mailbox/conn, and the actor's hand-rolled response registry is replaced by conn.ResponseRegistry. The registry's early-response buffering closes the race where a fast response from the remote mailbox could arrive (via the ingress loop) before AwaitRPC had a chance to register its waiter -- previously that response would be silently dropped. SendRPC now registers the waiter before calling Edge.Send, and on any send failure the waiter is cleaned up immediately. AwaitRPC uses a defer to remove its waiter on all exit paths. SendClientEventRequest gains MsgID and IdempotencyKey fields that survive TLV round-trips. When unset, Encode derives deterministic defaults from the serialized proto payload via conn.StableEventMsgID and conn.StableEventIdempotencyKey, ensuring that durable replay of the same semantic event produces the same identifiers for remote dedupe. The egress handler stamps these identifiers onto the outbound envelope. The standalone ack_state_test.go is removed since those tests now live in mailbox/conn. --- serverconn/ack_state_test.go | 186 ------------------------------ serverconn/actor.go | 211 ++++++++++++++++++++--------------- serverconn/types.go | 167 ++++----------------------- 3 files changed, 139 insertions(+), 425 deletions(-) delete mode 100644 serverconn/ack_state_test.go diff --git a/serverconn/ack_state_test.go b/serverconn/ack_state_test.go deleted file mode 100644 index 22efb6b83..000000000 --- a/serverconn/ack_state_test.go +++ /dev/null @@ -1,186 +0,0 @@ -package serverconn - -import ( - "bytes" - "testing" - - "github.com/stretchr/testify/require" -) - -// TestAckState_AdvanceDispatch_SetsAckTarget verifies that AdvanceDispatch -// moves both DispatchCommittedTo and AckTarget forward. -func TestAckState_AdvanceDispatch_SetsAckTarget(t *testing.T) { - t.Parallel() - - var s AckState - s.AdvanceDispatch(10) - - require.Equal(t, uint64(10), s.DispatchCommittedTo) - require.Equal(t, uint64(10), s.AckTarget) -} - -// TestAckState_AdvanceDispatch_Monotonic verifies that AdvanceDispatch never -// decreases DispatchCommittedTo or AckTarget. -func TestAckState_AdvanceDispatch_Monotonic(t *testing.T) { - t.Parallel() - - var s AckState - s.AdvanceDispatch(10) - s.AdvanceDispatch(5) // Should be ignored. - s.AdvanceDispatch(7) // Still lower, should be ignored. - - require.Equal(t, uint64(10), s.DispatchCommittedTo) - require.Equal(t, uint64(10), s.AckTarget) - - // A higher value should advance. - s.AdvanceDispatch(15) - require.Equal(t, uint64(15), s.DispatchCommittedTo) - require.Equal(t, uint64(15), s.AckTarget) -} - -// TestAckState_AdvanceAck_UpdatesPullCursor verifies that AdvanceAck moves -// AckCommittedTo to AckTarget and advances PullCursor if needed. -func TestAckState_AdvanceAck_UpdatesPullCursor(t *testing.T) { - t.Parallel() - - s := AckState{ - PullCursor: 0, - DispatchCommittedTo: 10, - AckTarget: 10, - AckCommittedTo: 0, - } - - s.AdvanceAck() - - require.Equal(t, uint64(10), s.AckCommittedTo) - require.Equal(t, uint64(10), s.PullCursor, - "PullCursor should advance to match AckCommittedTo") -} - -// TestAckState_AdvanceAck_DoesNotRegressPullCursor verifies that AdvanceAck -// does not move PullCursor backward if it is already ahead. -func TestAckState_AdvanceAck_DoesNotRegressPullCursor(t *testing.T) { - t.Parallel() - - s := AckState{ - PullCursor: 20, - DispatchCommittedTo: 10, - AckTarget: 10, - AckCommittedTo: 0, - } - - s.AdvanceAck() - - require.Equal(t, uint64(10), s.AckCommittedTo) - require.Equal(t, uint64(20), s.PullCursor, - "PullCursor should not regress") -} - -// TestAckState_NeedsAck verifies that NeedsAck returns true only when -// AckTarget exceeds AckCommittedTo. -func TestAckState_NeedsAck(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - state AckState - wantAck bool - }{ - { - name: "zero state needs no ack", - state: AckState{}, - wantAck: false, - }, - { - name: "target ahead of committed needs ack", - state: AckState{ - AckTarget: 10, - AckCommittedTo: 5, - }, - wantAck: true, - }, - { - name: "target equal to committed needs no ack", - state: AckState{ - AckTarget: 10, - AckCommittedTo: 10, - }, - wantAck: false, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - require.Equal(t, tc.wantAck, tc.state.NeedsAck()) - }) - } -} - -// TestAckState_FullCycle verifies a complete dispatch-ack cycle: dispatch -// advances the state, ack catches up, and NeedsAck transitions correctly. -func TestAckState_FullCycle(t *testing.T) { - t.Parallel() - - var s AckState - - // Initially no ack needed. - require.False(t, s.NeedsAck()) - - // Dispatch first batch. - s.AdvanceDispatch(10) - s.PullCursor = 10 - require.True(t, s.NeedsAck()) - - // Ack catches up. - s.AdvanceAck() - require.False(t, s.NeedsAck()) - require.Equal(t, uint64(10), s.AckCommittedTo) - require.Equal(t, uint64(10), s.PullCursor) - - // Dispatch second batch. - s.AdvanceDispatch(25) - s.PullCursor = 25 - require.True(t, s.NeedsAck()) - - // Ack again. - s.AdvanceAck() - require.False(t, s.NeedsAck()) - require.Equal(t, uint64(25), s.AckCommittedTo) -} - -// TestAckState_EncodeDecode verifies TLV round-trip serialization. -func TestAckState_EncodeDecode(t *testing.T) { - t.Parallel() - - original := AckState{ - PullCursor: 42, - DispatchCommittedTo: 100, - AckTarget: 100, - AckCommittedTo: 90, - } - - var buf bytes.Buffer - require.NoError(t, original.Encode(&buf)) - - var decoded AckState - require.NoError(t, decoded.Decode(bytes.NewReader(buf.Bytes()))) - - require.Equal(t, original, decoded) -} - -// TestAckState_EncodeDecode_Zero verifies that zero-value state round-trips. -func TestAckState_EncodeDecode_Zero(t *testing.T) { - t.Parallel() - - var original AckState - - var buf bytes.Buffer - require.NoError(t, original.Encode(&buf)) - - var decoded AckState - require.NoError(t, decoded.Decode(bytes.NewReader(buf.Bytes()))) - - require.Equal(t, original, decoded) -} diff --git a/serverconn/actor.go b/serverconn/actor.go index d7b79961a..0a8b204fb 100644 --- a/serverconn/actor.go +++ b/serverconn/actor.go @@ -9,6 +9,7 @@ import ( "time" "github.com/lightninglabs/darepo-client/baselib/actor" + mailboxconn "github.com/lightninglabs/darepo-client/mailbox/conn" mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" fn "github.com/lightningnetwork/lnd/fn/v2" "github.com/lightningnetwork/lnd/tlv" @@ -29,10 +30,12 @@ const ( SendRPCRequestMsgType tlv.Type = 2001 ) -// TLV record type constants for message field serialization. -const ( - protoPayloadRecordType tlv.Type = 1 - envelopeRecordType tlv.Type = 2 +// TLV record type aliases for RecordT-style message field serialization. +type ( + protoPayloadRecordTLV = tlv.TlvType1 + envelopeRecordTLV = tlv.TlvType2 + msgIDRecordTLV = tlv.TlvType3 + idempotencyRecordTLV = tlv.TlvType4 ) // ServerMessage is an interface that client FSM outbox messages must implement @@ -95,6 +98,14 @@ type SendClientEventRequest struct { // It must implement the ServerMessage interface which provides the // ToProto() method for conversion to protobuf. Message ServerMessage + + // MsgID uniquely identifies this send attempt. When this request is + // durably persisted and later retried, the same MsgID is reused. + MsgID string + + // IdempotencyKey identifies the semantic operation for remote dedupe. + // Retries of the same persisted request must reuse this key. + IdempotencyKey string } // MessageType returns a human-readable type name for logging. @@ -109,23 +120,55 @@ func (m *SendClientEventRequest) TLVType() tlv.Type { // Encode serializes the message to the provided writer. The ServerMessage is // converted to proto, wrapped in anypb.Any (preserving type information), -// and marshaled to bytes for TLV storage. +// and stored via a WrappedProto TLV record. +// +// We use TLV here (rather than storing raw proto bytes) because the +// DurableActor runtime requires all messages to satisfy the TLVMessage +// interface (TLVType, Encode, Decode). The MessageCodec uses these methods +// to serialize messages into the durable mailbox. WrappedProto handles the +// proto↔bytes conversion inside the TLV record, keeping the codec contract +// simple and uniform across message types. func (m *SendClientEventRequest) Encode(w io.Writer) error { anyMsg, err := anypb.New(m.Message.ToProto()) if err != nil { return fmt.Errorf("wrap proto in Any: %w", err) } - anyBytes, err := proto.Marshal(anyMsg) + // We still need the raw bytes for stable ID derivation, so marshal + // deterministically before constructing the TLV records. + anyBytes, err := (proto.MarshalOptions{ + Deterministic: true, + }).Marshal(anyMsg) if err != nil { return fmt.Errorf("marshal Any: %w", err) } - records := []tlv.Record{ - tlv.MakePrimitiveRecord(protoPayloadRecordType, &anyBytes), + msgID := m.MsgID + if msgID == "" { + msgID = mailboxconn.StableEventMsgID(anyBytes) } + msgIDBytes := []byte(msgID) - stream, err := tlv.NewStream(records...) + idempotencyKey := m.IdempotencyKey + if idempotencyKey == "" { + idempotencyKey = mailboxconn. + StableEventIdempotencyKey(anyBytes) + } + idempotencyBytes := []byte(idempotencyKey) + + payload := tlv.NewRecordT[protoPayloadRecordTLV]( + mailboxconn.WrappedProto[*anypb.Any]{Val: anyMsg}, + ) + msgIDRec := tlv.NewPrimitiveRecord[msgIDRecordTLV]( + msgIDBytes, + ) + idemRec := tlv.NewPrimitiveRecord[idempotencyRecordTLV]( + idempotencyBytes, + ) + + stream, err := tlv.NewStream( + payload.Record(), msgIDRec.Record(), idemRec.Record(), + ) if err != nil { return err } @@ -137,13 +180,18 @@ func (m *SendClientEventRequest) Encode(w io.Writer) error { // is stored as a rawServerMessage that lazily unmarshals via the global // protobuf type registry. func (m *SendClientEventRequest) Decode(r io.Reader) error { - var payload []byte + payload := tlv.ZeroRecordT[ + protoPayloadRecordTLV, + mailboxconn.WrappedProto[*anypb.Any], + ]() + payload.Val.Val = &anypb.Any{} - records := []tlv.Record{ - tlv.MakePrimitiveRecord(protoPayloadRecordType, &payload), - } + msgIDRec := tlv.ZeroRecordT[msgIDRecordTLV, []byte]() + idemRec := tlv.ZeroRecordT[idempotencyRecordTLV, []byte]() - stream, err := tlv.NewStream(records...) + stream, err := tlv.NewStream( + payload.Record(), msgIDRec.Record(), idemRec.Record(), + ) if err != nil { return err } @@ -152,12 +200,9 @@ func (m *SendClientEventRequest) Decode(r io.Reader) error { return err } - anyMsg := &anypb.Any{} - if err := proto.Unmarshal(payload, anyMsg); err != nil { - return fmt.Errorf("unmarshal Any: %w", err) - } - - m.Message = &rawServerMessage{anyMsg: anyMsg} + m.Message = &rawServerMessage{anyMsg: payload.Val.Val} + m.MsgID = string(msgIDRec.Val) + m.IdempotencyKey = string(idemRec.Val) return nil } @@ -184,6 +229,25 @@ func (m *SendClientEventResponse) MessageType() string { // serverConnRespSealed implements the ServerConnResp interface seal. func (m *SendClientEventResponse) serverConnRespSealed() {} +// SendRPCResponse acknowledges that an RPC envelope was sent. +type SendRPCResponse struct { + actor.BaseMessage + + // Success indicates whether the send operation succeeded. + Success bool + + // Error contains the error message if the send failed. + Error string +} + +// MessageType returns a human-readable type name for logging. +func (m *SendRPCResponse) MessageType() string { + return "SendRPCResponse" +} + +// serverConnRespSealed implements the ServerConnResp interface seal. +func (m *SendRPCResponse) serverConnRespSealed() {} + // SendRPCRequest wraps a pre-built outbound unary RPC envelope. The unary // facade constructs the envelope with all metadata (correlation ID, // idempotency key, service/method) and hands it to the connector for @@ -205,19 +269,16 @@ func (m *SendRPCRequest) TLVType() tlv.Type { return SendRPCRequestMsgType } -// Encode serializes the message to the provided writer. The entire mailbox -// envelope is proto-marshaled for TLV storage. +// Encode serializes the message to the provided writer. The mailbox +// envelope is stored via a WrappedProto TLV record. func (m *SendRPCRequest) Encode(w io.Writer) error { - envBytes, err := proto.Marshal(m.Envelope) - if err != nil { - return fmt.Errorf("marshal envelope: %w", err) - } - - records := []tlv.Record{ - tlv.MakePrimitiveRecord(envelopeRecordType, &envBytes), - } + envRec := tlv.NewRecordT[envelopeRecordTLV]( + mailboxconn.WrappedProto[*mailboxpb.Envelope]{ + Val: m.Envelope, + }, + ) - stream, err := tlv.NewStream(records...) + stream, err := tlv.NewStream(envRec.Record()) if err != nil { return err } @@ -227,13 +288,13 @@ func (m *SendRPCRequest) Encode(w io.Writer) error { // Decode deserializes the message from the provided reader. func (m *SendRPCRequest) Decode(r io.Reader) error { - var envBytes []byte - - records := []tlv.Record{ - tlv.MakePrimitiveRecord(envelopeRecordType, &envBytes), - } + envRec := tlv.ZeroRecordT[ + envelopeRecordTLV, + mailboxconn.WrappedProto[*mailboxpb.Envelope], + ]() + envRec.Val.Val = &mailboxpb.Envelope{} - stream, err := tlv.NewStream(records...) + stream, err := tlv.NewStream(envRec.Record()) if err != nil { return err } @@ -242,10 +303,7 @@ func (m *SendRPCRequest) Decode(r io.Reader) error { return err } - m.Envelope = &mailboxpb.Envelope{} - if err := proto.Unmarshal(envBytes, m.Envelope); err != nil { - return fmt.Errorf("unmarshal envelope: %w", err) - } + m.Envelope = envRec.Val.Val return nil } @@ -272,15 +330,10 @@ type ServerConnectionActor struct { // cfg holds all dependencies and tuning knobs for the connector. cfg ConnectorConfig - // responseRegistryMu protects concurrent access to the response - // registry from the ingress loop and unary facade callers. - responseRegistryMu sync.Mutex - - // responseRegistry maps correlation IDs to unary RPC waiters. The - // ingress loop delivers KIND_RESPONSE envelopes to the appropriate - // waiter channel. This is in-memory only — if the process crashes, - // callers' contexts are cancelled and they retry. - responseRegistry map[CorrelationID]*ResponseWaiter + // responseRegistry maps correlation IDs to unary RPC waiters and + // buffers early responses that arrive before a waiter is registered. + // This is in-memory only. + responseRegistry *mailboxconn.ResponseRegistry // cancelCh delivers the ingress loop cancel function from // StartIngress to StopIngress without a shared field, avoiding @@ -303,9 +356,11 @@ func NewServerConnectionActor( ) *ServerConnectionActor { return &ServerConnectionActor{ - cfg: cfg, - responseRegistry: make(map[CorrelationID]*ResponseWaiter), - cancelCh: make(chan context.CancelFunc, 1), + cfg: cfg, + responseRegistry: mailboxconn.NewResponseRegistry( + cfg.ResponseWaiterTTL, + ), + cancelCh: make(chan context.CancelFunc, 1), } } @@ -424,65 +479,36 @@ func (a *ServerConnectionActor) handleSendRPCRequest(ctx context.Context, )) } - return fn.Ok[ServerConnResp](&SendClientEventResponse{ + return fn.Ok[ServerConnResp](&SendRPCResponse{ Success: true, }) } // RegisterWaiter adds a response waiter for the given correlation ID. The -// returned channel will receive the response envelope when the ingress loop -// pulls a KIND_RESPONSE with a matching correlation ID. +// returned Future completes when the ingress loop delivers a KIND_RESPONSE +// with a matching correlation ID, or errors if the waiter expires or is +// cancelled. func (a *ServerConnectionActor) RegisterWaiter( id CorrelationID, -) <-chan *mailboxpb.Envelope { - - a.responseRegistryMu.Lock() - defer a.responseRegistryMu.Unlock() - - waiter := &ResponseWaiter{ - Ch: make(chan *mailboxpb.Envelope, 1), - Created: time.Now(), - } +) actor.Future[*mailboxpb.Envelope] { - a.responseRegistry[id] = waiter - - return waiter.Ch + return a.responseRegistry.RegisterWaiter(id) } // removeWaiter removes a previously registered waiter, preventing leaks on // cancellation or timeout. func (a *ServerConnectionActor) removeWaiter(id CorrelationID) { - a.responseRegistryMu.Lock() - defer a.responseRegistryMu.Unlock() - - delete(a.responseRegistry, id) + a.responseRegistry.RemoveWaiter(id) } // deliverResponse looks up a waiter by correlation ID and delivers the -// envelope. Returns true if a waiter was found and signaled. +// envelope. If no waiter exists yet, the response is buffered so a later +// AwaitRPC call can still observe it. func (a *ServerConnectionActor) deliverResponse( id CorrelationID, env *mailboxpb.Envelope, ) bool { - a.responseRegistryMu.Lock() - waiter, ok := a.responseRegistry[id] - if ok { - delete(a.responseRegistry, id) - } - a.responseRegistryMu.Unlock() - - if !ok { - return false - } - - // Non-blocking send on buffered channel. If the waiter's context - // was already cancelled, the envelope is dropped (caller retries). - select { - case waiter.Ch <- env: - default: - } - - return true + return a.responseRegistry.DeliverResponse(id, env) } // StartIngress loads the ack checkpoint from the store and launches the @@ -549,6 +575,7 @@ var ( _ ServerConnMsg = (*SendClientEventRequest)(nil) _ ServerConnMsg = (*SendRPCRequest)(nil) _ ServerConnResp = (*SendClientEventResponse)(nil) + _ ServerConnResp = (*SendRPCResponse)(nil) //nolint:ll _ actor.ActorBehavior[ServerConnMsg, ServerConnResp] = (*ServerConnectionActor)(nil) diff --git a/serverconn/types.go b/serverconn/types.go index db0e5d932..d45abc9fc 100644 --- a/serverconn/types.go +++ b/serverconn/types.go @@ -2,148 +2,28 @@ package serverconn import ( "context" - "io" "time" "github.com/lightninglabs/darepo-client/baselib/actor" + mailboxconn "github.com/lightninglabs/darepo-client/mailbox/conn" mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" mailboxrpc "github.com/lightninglabs/darepo-client/mailbox/rpc" - "github.com/lightningnetwork/lnd/tlv" ) -// CorrelationID is an opaque identifier linking a mailbox request to its -// response. Using a named type prevents accidental string swaps with other -// identifiers. -type CorrelationID string - -// IdempotencyKey is a stable key for deduplicating semantic operations across -// retries. Two sends with the same idempotency key are treated as the same -// logical operation by the remote mailbox edge. -type IdempotencyKey string - -// ackStateType is the checkpoint state type used when persisting the ack -// watermark to the delivery store. -const ackStateType = "AckState" - -// TLV record type constants for AckState checkpoint serialization. -const ( - pullCursorRecordType tlv.Type = 1 - dispatchCommittedToRecordType tlv.Type = 2 - ackTargetRecordType tlv.Type = 3 - ackCommittedToRecordType tlv.Type = 4 -) - -// AckState tracks the four cursor variables that govern safe ack progression. -// All fields are monotonic — they never decrease during normal operation. -// -// The state machine enforces the invariant: -// -// ack_committed_to <= dispatch_committed_to -// -// Cursor never advances past non-durable local work. Repeated acks are safe -// and idempotent. -type AckState struct { - // PullCursor is the cursor for the next Pull call. After a successful - // ack, this advances to at least the acked position. - PullCursor uint64 - - // DispatchCommittedTo is the max cursor whose envelopes have been - // durably committed to local actor mailboxes via Tell. - DispatchCommittedTo uint64 - - // AckTarget is the max cursor that should be acked remotely. This is - // always >= DispatchCommittedTo. - AckTarget uint64 - - // AckCommittedTo is the last cursor successfully acked to the remote - // mailbox edge. - AckCommittedTo uint64 -} - -// AdvanceDispatch updates the state after a successful durable dispatch -// through nextCursor. The ack target is advanced to match the dispatch -// frontier. -func (s *AckState) AdvanceDispatch(nextCursor uint64) { - if nextCursor > s.DispatchCommittedTo { - s.DispatchCommittedTo = nextCursor - } - - if s.DispatchCommittedTo > s.AckTarget { - s.AckTarget = s.DispatchCommittedTo - } -} +// CorrelationID links a mailbox request to its response. +type CorrelationID = mailboxconn.CorrelationID -// AdvanceAck updates the state after a successful AckUpTo call. The pull -// cursor advances to at least the acked position so that subsequent pulls -// do not re-fetch already-acked envelopes. -func (s *AckState) AdvanceAck() { - s.AckCommittedTo = s.AckTarget +// IdempotencyKey deduplicates a semantic operation across retries. +type IdempotencyKey = mailboxconn.IdempotencyKey - if s.AckCommittedTo > s.PullCursor { - s.PullCursor = s.AckCommittedTo - } -} +// AckState tracks connector ack watermark state for checkpoint persistence. +type AckState = mailboxconn.AckState -// NeedsAck returns true when there is an un-acked committed dispatch. This -// means AckTarget has advanced past AckCommittedTo and a remote AckUpTo call -// is needed. -func (s *AckState) NeedsAck() bool { - return s.AckTarget > s.AckCommittedTo -} +// ResponseWaiter stores in-memory waiter state for unary response delivery. +type ResponseWaiter = mailboxconn.ResponseWaiter -// Encode serializes the AckState to the provided writer as a TLV stream. -func (s *AckState) Encode(w io.Writer) error { - records := []tlv.Record{ - tlv.MakePrimitiveRecord( - pullCursorRecordType, &s.PullCursor, - ), - tlv.MakePrimitiveRecord( - dispatchCommittedToRecordType, - &s.DispatchCommittedTo, - ), - tlv.MakePrimitiveRecord( - ackTargetRecordType, &s.AckTarget, - ), - tlv.MakePrimitiveRecord( - ackCommittedToRecordType, &s.AckCommittedTo, - ), - } - - stream, err := tlv.NewStream(records...) - if err != nil { - return err - } - - return stream.Encode(w) -} - -// Decode deserializes the AckState from the provided reader. -func (s *AckState) Decode(r io.Reader) error { - records := []tlv.Record{ - tlv.MakePrimitiveRecord( - pullCursorRecordType, &s.PullCursor, - ), - tlv.MakePrimitiveRecord( - dispatchCommittedToRecordType, - &s.DispatchCommittedTo, - ), - tlv.MakePrimitiveRecord( - ackTargetRecordType, &s.AckTarget, - ), - tlv.MakePrimitiveRecord( - ackCommittedToRecordType, &s.AckCommittedTo, - ), - } - - stream, err := tlv.NewStream(records...) - if err != nil { - return err - } - - _, err = stream.DecodeWithParsedTypes(r) - - return err -} +// ackStateType is the checkpoint state type used for ack watermark storage. +const ackStateType = mailboxconn.CheckpointStateType // EnvelopeDispatcher routes an inbound envelope to the correct local actor. // A nil error means the envelope was durably committed to the target actor's @@ -154,18 +34,6 @@ type EnvelopeDispatcher func( ctx context.Context, env *mailboxpb.Envelope, ) error -// ResponseWaiter is registered by unary facade callers so the ingress loop -// can deliver KIND_RESPONSE envelopes without actor dispatch. The channel -// has buffer size 1 to prevent the ingress loop from blocking. -type ResponseWaiter struct { - // Ch receives the response envelope from the ingress loop. - Ch chan *mailboxpb.Envelope - - // Created records when the waiter was registered, for diagnostics - // and stale waiter cleanup. - Created time.Time -} - // ConnectorConfig holds all dependencies and tuning knobs for the server // connection actor. The connector is the single boundary for all mailbox // traffic between the client and the remote server. @@ -216,6 +84,10 @@ type ConnectorConfig struct { // RetryMaxDelay caps the exponential backoff delay. RetryMaxDelay time.Duration + + // ResponseWaiterTTL bounds how long a response waiter (or buffered + // early response) is retained before stale cleanup. + ResponseWaiterTTL time.Duration } // DefaultConnectorConfig returns a ConnectorConfig with sensible defaults for @@ -223,9 +95,10 @@ type ConnectorConfig struct { // and Store. Codec is optional — NewRuntime fills a default. func DefaultConnectorConfig() ConnectorConfig { return ConnectorConfig{ - PullMaxEnvelopes: 50, - PullWaitTimeout: 5 * time.Second, - RetryBaseDelay: 200 * time.Millisecond, - RetryMaxDelay: 30 * time.Second, + PullMaxEnvelopes: 50, + PullWaitTimeout: 5 * time.Second, + RetryBaseDelay: 200 * time.Millisecond, + RetryMaxDelay: 30 * time.Second, + ResponseWaiterTTL: mailboxconn.DefaultResponseWaiterTTL, } } From 15356d8085f50eeedb9d053a034c6d4cf6fe8db5 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Tue, 17 Feb 2026 22:08:47 -0800 Subject: [PATCH 10/14] serverconn: add runtime composition helper In this commit, we add a Runtime struct that wires the ServerConnectionActor, its DurableActor wrapper, and the UnaryFacade into a single composition unit that higher-level daemon bootstrap can use. NewRuntime validates required config (Store, LocalMailboxID), fills a default codec when one isn't supplied, and returns a fully assembled but inert runtime. Start launches both durable egress processing and the ingress pull loop; Stop tears them down in reverse order. Accessors expose the Ref/TellRef (for round actor egress) and Unary (for typed RPC stubs). The DurableActorID helper extracts the "serverconn-" + mailboxID convention into a single function shared by both the runtime and the ingress checkpoint load/save paths. Package docs are updated to describe the runtime composition stage and note the shared mailbox/conn primitives. --- serverconn/doc.go | 10 +++++ serverconn/ingress.go | 4 +- serverconn/runtime.go | 99 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 serverconn/runtime.go diff --git a/serverconn/doc.go b/serverconn/doc.go index 012974e11..b9ae8e75b 100644 --- a/serverconn/doc.go +++ b/serverconn/doc.go @@ -30,6 +30,9 @@ // that if the process crashes between dispatch and ack, envelopes will be // redelivered on restart. // +// The AckState codec and related connector primitives are shared from +// mailbox/conn so the server-side connector can mirror the same behavior. +// // # Dispatch Table // // Inbound KIND_REQUEST and KIND_EVENT envelopes are routed via a @@ -48,4 +51,11 @@ // (synchronous, no actor mailbox — low-latency path for unary sends). // AwaitRPC registers a waiter in the response registry and blocks until the // ingress loop delivers a matching KIND_RESPONSE envelope. +// +// # Runtime Composition +// +// Runtime embeds a DurableActor so it can be registered directly with the +// actor system — Ref and TellRef are promoted without wrapper methods. +// Higher layers use Runtime for round actor egress (via TellRef) and typed +// RPC stubs (via UnaryFacade). package serverconn diff --git a/serverconn/ingress.go b/serverconn/ingress.go index 28a693cd0..16e1c011d 100644 --- a/serverconn/ingress.go +++ b/serverconn/ingress.go @@ -346,7 +346,7 @@ func (a *ServerConnectionActor) loadCheckpoint( ctx context.Context, ) (AckState, error) { - actorID := "serverconn-" + a.cfg.LocalMailboxID + actorID := DurableActorID(a.cfg.LocalMailboxID) checkpoint, err := a.cfg.Store.LoadCheckpoint(ctx, actorID) if err != nil { @@ -383,7 +383,7 @@ func (a *ServerConnectionActor) saveCheckpoint( return err } - actorID := "serverconn-" + a.cfg.LocalMailboxID + actorID := DurableActorID(a.cfg.LocalMailboxID) return a.cfg.Store.SaveCheckpoint(ctx, actor.CheckpointParams{ ActorID: actorID, diff --git a/serverconn/runtime.go b/serverconn/runtime.go new file mode 100644 index 000000000..42f42037c --- /dev/null +++ b/serverconn/runtime.go @@ -0,0 +1,99 @@ +package serverconn + +import ( + "context" + "fmt" + + "github.com/lightninglabs/darepo-client/baselib/actor" +) + +// DurableActorID returns the durable actor mailbox ID used for serverconn +// ingress checkpointing and egress mailbox persistence. +func DurableActorID(localMailboxID string) string { + return "serverconn-" + localMailboxID +} + +// Runtime embeds a DurableActor for serverconn egress and wires it together +// with the ingress loop and unary facade. Because the DurableActor is +// embedded, Runtime can be registered directly with the actor system — Ref +// and TellRef are promoted without wrapper methods. +type Runtime struct { + *actor.DurableActor[ServerConnMsg, ServerConnResp] + + connector *ServerConnectionActor + unary *UnaryFacade +} + +// NewRuntime constructs a durable serverconn runtime from connector config. +// The returned runtime is inert until Start is called. +func NewRuntime(cfg ConnectorConfig) (*Runtime, error) { + if cfg.Store == nil { + return nil, fmt.Errorf("connector store is required") + } + + if cfg.Edge == nil { + return nil, fmt.Errorf("connector edge is required") + } + + if cfg.LocalMailboxID == "" { + return nil, fmt.Errorf("local mailbox id is required") + } + + if cfg.RemoteMailboxID == "" { + return nil, fmt.Errorf("remote mailbox id is required") + } + + if cfg.Codec == nil { + cfg.Codec = NewServerConnCodec() + } + + connector := NewServerConnectionActor(cfg) + + durableCfg := actor.DefaultDurableActorConfig[ + ServerConnMsg, ServerConnResp, + ]( + DurableActorID(cfg.LocalMailboxID), + connector, + cfg.Store, + cfg.Codec, + ) + + durable := actor.NewDurableActor(durableCfg) + unary := NewUnaryFacade(connector) + + return &Runtime{ + DurableActor: durable, + connector: connector, + unary: unary, + }, nil +} + +// Start launches durable egress processing and ingress pulling. Returns an +// error if the ingress checkpoint cannot be loaded from the store. +func (r *Runtime) Start(ctx context.Context) error { + r.DurableActor.Start() + + if err := r.connector.StartIngress(ctx); err != nil { + r.DurableActor.Stop() + + return err + } + + return nil +} + +// Stop shuts down ingress polling and durable egress processing. +func (r *Runtime) Stop() { + r.connector.StopIngress() + r.DurableActor.Stop() +} + +// Unary returns the unary RPC facade bound to this runtime. +func (r *Runtime) Unary() *UnaryFacade { + return r.unary +} + +// Connector returns the underlying connector behavior. +func (r *Runtime) Connector() *ServerConnectionActor { + return r.connector +} From 205bd007fc26f0ef9dace6c91ddada82da1716d6 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Tue, 17 Feb 2026 22:09:01 -0800 Subject: [PATCH 11/14] mailbox/client: remove legacy runtime In this commit, we remove the entire mailbox/client package. With the serverconn package now serving as the unified connector boundary for all mailbox traffic, the legacy client runtime is redundant. It maintained its own pull loop, cursor persistence, response store, and ack logic -- all of which duplicated what serverconn and the durable actor runtime already provide. Keeping a parallel connector path would mean two competing durability stores, two cursor management schemes, and two sets of retry semantics. Removing it now prevents that divergence from accumulating. --- mailbox/client/client.go | 549 -------------------------------- mailbox/client/client_test.go | 278 ---------------- mailbox/client/config.go | 43 --- mailbox/client/doc.go | 26 -- mailbox/client/errors.go | 39 --- mailbox/client/inmemory_test.go | 242 -------------- mailbox/client/log.go | 24 -- mailbox/client/store.go | 164 ---------- 8 files changed, 1365 deletions(-) delete mode 100644 mailbox/client/client.go delete mode 100644 mailbox/client/client_test.go delete mode 100644 mailbox/client/config.go delete mode 100644 mailbox/client/doc.go delete mode 100644 mailbox/client/errors.go delete mode 100644 mailbox/client/inmemory_test.go delete mode 100644 mailbox/client/log.go delete mode 100644 mailbox/client/store.go diff --git a/mailbox/client/client.go b/mailbox/client/client.go deleted file mode 100644 index 51c1690e9..000000000 --- a/mailbox/client/client.go +++ /dev/null @@ -1,549 +0,0 @@ -package mailboxclient - -import ( - "context" - "crypto/rand" - "encoding/hex" - "fmt" - "log/slog" - "sync" - "time" - - mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" - mailboxrpc "github.com/lightninglabs/darepo-client/mailbox/rpc" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/anypb" -) - -const errMalformedResponseBody = "failed to unmarshal response body: %w" - -// Client implements mailboxrpc.RPCClient by sending and receiving mailbox -// envelopes through a mailboxpb.MailboxServiceClient. -type Client struct { - cfg Config - - cancel context.CancelFunc - wg sync.WaitGroup - - mu sync.Mutex - - cursor uint64 - ackTo uint64 - - pending map[string][]byte - waiters map[string][]chan struct{} -} - -// New constructs and starts a mailboxclient.Client. -func New(cfg Config) (*Client, error) { - cfg = applyDefaults(cfg) - - if cfg.Edge == nil { - return nil, fmt.Errorf("edge is required") - } - if cfg.LocalMailboxID == "" { - return nil, fmt.Errorf("local mailbox id is required") - } - if cfg.RemoteMailboxID == "" { - return nil, fmt.Errorf("remote mailbox id is required") - } - if cfg.PullMaxEnvelopes == 0 { - return nil, fmt.Errorf("pull max envelopes must be > 0") - } - if cfg.PullWaitTimeout <= 0 { - return nil, fmt.Errorf("pull wait timeout must be > 0") - } - - ctx, cancel := context.WithCancel(context.Background()) - - cursor, err := cfg.Store.LoadCursor(ctx, cfg.LocalMailboxID) - if err != nil { - cancel() - return nil, fmt.Errorf("load cursor: %w", err) - } - - c := &Client{ - cfg: cfg, - - cancel: cancel, - - cursor: cursor, - ackTo: cursor, - - pending: make(map[string][]byte), - waiters: make(map[string][]chan struct{}), - } - - c.wg.Add(1) - go func() { - defer c.wg.Done() - c.run(ctx) - }() - - log.InfoS(ctx, "Mailbox client started", - slog.String("local_mailbox", cfg.LocalMailboxID), - slog.String("remote_mailbox", cfg.RemoteMailboxID)) - - return c, nil -} - -// Stop shuts down background polling and unblocks any waiters. -func (c *Client) Stop() { - log.InfoS(context.TODO(), "Stopping mailbox client") - - c.cancel() - c.wg.Wait() - - c.mu.Lock() - defer c.mu.Unlock() - - for correlationID, waiters := range c.waiters { - for _, ch := range waiters { - close(ch) - } - - delete(c.waiters, correlationID) - } - - log.InfoS(context.TODO(), "Mailbox client stopped") -} - -// SendRPC sends a request payload and returns a SendResult containing the -// correlation id and idempotency key used for the send. -func (c *Client) SendRPC(ctx context.Context, method mailboxrpc.ServiceMethod, - req proto.Message, - opts mailboxrpc.RPCOptions) (mailboxrpc.SendResult, error) { - - msgID, err := randomID(16) - if err != nil { - return mailboxrpc.SendResult{}, err - } - - idempotencyKey := opts.IdempotencyKey - if idempotencyKey == "" { - idempotencyKey, err = randomID(16) - if err != nil { - return mailboxrpc.SendResult{}, err - } - } - - correlationID := opts.CorrelationID - if correlationID == "" { - correlationID = idempotencyKey - } - - body, err := anypb.New(req) - if err != nil { - return mailboxrpc.SendResult{}, err - } - - envelope := &mailboxpb.Envelope{ - ProtocolVersion: c.cfg.ProtocolVersion, - MsgId: msgID, - IdempotencyKey: idempotencyKey, - Sender: c.cfg.LocalMailboxID, - Recipient: c.cfg.RemoteMailboxID, - CreatedAtUnixMs: time.Now().UnixMilli(), - Headers: opts.Headers, - Body: body, - Rpc: &mailboxpb.RpcMeta{ - Kind: mailboxpb.RpcMeta_KIND_REQUEST, - Service: method.Service, - Method: method.Method, - CorrelationId: correlationID, - ReplyTo: c.cfg.LocalMailboxID, - }, - } - - resp, err := c.cfg.Edge.Send(ctx, &mailboxpb.SendRequest{ - Envelope: envelope, - }) - if err != nil { - log.WarnS(ctx, "Send failed", err, - slog.String("service", method.Service), - slog.String("method", method.Method)) - - return mailboxrpc.SendResult{}, err - } - - if !statusOK(resp.Status) { - sendErr := statusError("Send", resp.Status) - log.WarnS(ctx, "Send returned non-OK status", sendErr, - slog.String("service", method.Service), - slog.String("method", method.Method)) - - return mailboxrpc.SendResult{}, sendErr - } - - log.DebugS(ctx, "Sent RPC request", - slog.String("service", method.Service), - slog.String("method", method.Method), - slog.String("correlation_id", correlationID)) - - return mailboxrpc.SendResult{ - CorrelationID: correlationID, - IdempotencyKey: idempotencyKey, - }, nil -} - -// AwaitRPC blocks until a response for correlationID is received. -func (c *Client) AwaitRPC(ctx context.Context, correlationID string, - resp proto.Message) error { - - for { - data, ok, err := c.peekResponse(ctx, correlationID) - if err != nil { - return err - } - if ok { - err := (proto.UnmarshalOptions{ - DiscardUnknown: true, - }).Unmarshal(data, resp) - if err != nil { - return fmt.Errorf(errMalformedResponseBody, err) - } - - c.deletePending(correlationID) - c.deleteResponseBestEffort( - ctx, c.cfg.LocalMailboxID, correlationID, - ) - - return nil - } - - ch := c.addWaiter(correlationID) - select { - case <-ch: - case <-ctx.Done(): - c.removeWaiter(correlationID, ch) - return ctx.Err() - } - } -} - -// applyDefaults fills unset fields with defaults. -func applyDefaults(cfg Config) Config { - def := DefaultConfig() - - if cfg.PullMaxEnvelopes == 0 { - cfg.PullMaxEnvelopes = def.PullMaxEnvelopes - } - if cfg.PullWaitTimeout == 0 { - cfg.PullWaitTimeout = def.PullWaitTimeout - } - if cfg.Store == nil { - cfg.Store = NewMemoryStore() - } - - return cfg -} - -// run polls Pull and acks envelopes after caching correlated responses. -func (c *Client) run(ctx context.Context) { - log.DebugS(ctx, "Poll loop starting", - slog.String("mailbox_id", c.cfg.LocalMailboxID)) - - for { - select { - case <-ctx.Done(): - log.DebugS(ctx, "Poll loop exiting") - return - - default: - } - - ackTo := c.loadAckTo() - if ackTo != 0 { - if !c.ackUpTo(ctx, ackTo) { - c.sleepRetry(ctx) - continue - } - - c.clearAckTo(ackTo) - } - - cursor := c.loadCursor() - waitMs := uint32(c.cfg.PullWaitTimeout.Milliseconds()) - - resp, err := c.cfg.Edge.Pull(ctx, &mailboxpb.PullRequest{ - MailboxId: c.cfg.LocalMailboxID, - MaxEnvelopes: c.cfg.PullMaxEnvelopes, - WaitTimeoutMs: waitMs, - Cursor: cursor, - }) - if err != nil { - log.WarnS(ctx, "Pull failed, retrying", err) - c.sleepRetry(ctx) - continue - } - - if !statusOK(resp.Status) { - log.DebugS(ctx, "Pull returned non-OK status") - c.sleepRetry(ctx) - continue - } - - if len(resp.Envelopes) > 0 { - log.DebugS(ctx, "Pulled envelopes", - slog.Int("count", len(resp.Envelopes)), - slog.Uint64("cursor", cursor), - slog.Uint64("next_cursor", resp.NextCursor)) - } - - var handleErr error - for _, env := range resp.Envelopes { - if err := c.handleEnvelope(ctx, env); err != nil { - handleErr = err - break - } - } - - if handleErr != nil { - c.sleepRetry(ctx) - continue - } - - if resp.NextCursor > cursor { - if err := c.storeCursor( - ctx, resp.NextCursor, - ); err != nil { - c.sleepRetry(ctx) - continue - } - - c.setAckTo(resp.NextCursor) - } - } -} - -// sleepRetry backs off briefly after a transient pull/ack failure. -func (c *Client) sleepRetry(ctx context.Context) { - timer := time.NewTimer(200 * time.Millisecond) - defer timer.Stop() - - select { - case <-ctx.Done(): - case <-timer.C: - } -} - -// ackUpTo calls AckUpTo and returns true on success. -func (c *Client) ackUpTo(ctx context.Context, cursor uint64) bool { - resp, err := c.cfg.Edge.AckUpTo(ctx, &mailboxpb.AckUpToRequest{ - MailboxId: c.cfg.LocalMailboxID, - Cursor: cursor, - }) - if err != nil { - return false - } - - return statusOK(resp.Status) -} - -// handleEnvelope caches correlated responses and wakes waiters. -func (c *Client) handleEnvelope(ctx context.Context, - env *mailboxpb.Envelope) error { - - if env == nil || env.Rpc == nil { - return nil - } - - if env.Rpc.Kind != mailboxpb.RpcMeta_KIND_RESPONSE { - return nil - } - - correlationID := env.Rpc.CorrelationId - if correlationID == "" { - return nil - } - - if env.Body == nil { - return nil - } - - payload := env.Body.Value - payloadCopy := make([]byte, len(payload)) - copy(payloadCopy, payload) - - if err := c.cfg.Store.PutResponse( - ctx, c.cfg.LocalMailboxID, correlationID, payloadCopy, - ); err != nil { - return err - } - - c.mu.Lock() - defer c.mu.Unlock() - - // If we already have a response for this correlation id, keep the - // first response and ignore duplicates. - if _, exists := c.pending[correlationID]; !exists { - c.pending[correlationID] = payloadCopy - - log.DebugS(context.TODO(), "Cached response", - slog.String("correlation_id", correlationID), - slog.Int("payload_bytes", len(payloadCopy))) - } else { - log.DebugS(context.TODO(), "Ignored duplicate response", - slog.String("correlation_id", correlationID)) - } - - waiters := c.waiters[correlationID] - for _, ch := range waiters { - close(ch) - } - delete(c.waiters, correlationID) - - return nil -} - -// peekPending returns a cached response payload for correlationID. -func (c *Client) peekPending(correlationID string) ([]byte, bool) { - c.mu.Lock() - defer c.mu.Unlock() - - data, ok := c.pending[correlationID] - if !ok { - return nil, false - } - - dataCopy := make([]byte, len(data)) - copy(dataCopy, data) - - return dataCopy, true -} - -func (c *Client) deletePending(correlationID string) { - c.mu.Lock() - defer c.mu.Unlock() - - delete(c.pending, correlationID) -} - -func (c *Client) peekResponse(ctx context.Context, - correlationID string) ([]byte, bool, error) { - - if data, ok := c.peekPending(correlationID); ok { - return data, true, nil - } - - return c.cfg.Store.GetResponse(ctx, c.cfg.LocalMailboxID, correlationID) -} - -func (c *Client) deleteResponseBestEffort(ctx context.Context, mailboxID string, - correlationID string) { - - const maxAttempts = 3 - backoff := 50 * time.Millisecond - - for i := 0; i < maxAttempts; i++ { - err := c.cfg.Store.DeleteResponse(ctx, mailboxID, correlationID) - if err == nil { - return - } - - timer := time.NewTimer(backoff) - select { - case <-ctx.Done(): - timer.Stop() - return - case <-timer.C: - } - - backoff *= 2 - } -} - -// addWaiter registers a waiter for correlationID and returns its channel. -func (c *Client) addWaiter(correlationID string) chan struct{} { - ch := make(chan struct{}) - - c.mu.Lock() - defer c.mu.Unlock() - - c.waiters[correlationID] = append(c.waiters[correlationID], ch) - - return ch -} - -// removeWaiter removes a previously registered waiter channel. -func (c *Client) removeWaiter(correlationID string, ch chan struct{}) { - c.mu.Lock() - defer c.mu.Unlock() - - waiters := c.waiters[correlationID] - for i := range waiters { - if waiters[i] == ch { - waiters[i] = waiters[len(waiters)-1] - waiters = waiters[:len(waiters)-1] - break - } - } - - if len(waiters) == 0 { - delete(c.waiters, correlationID) - return - } - - c.waiters[correlationID] = waiters -} - -// loadCursor returns the current pull cursor. -func (c *Client) loadCursor() uint64 { - c.mu.Lock() - defer c.mu.Unlock() - - return c.cursor -} - -func (c *Client) loadAckTo() uint64 { - c.mu.Lock() - defer c.mu.Unlock() - - return c.ackTo -} - -func (c *Client) setAckTo(cursor uint64) { - c.mu.Lock() - defer c.mu.Unlock() - - if cursor > c.ackTo { - c.ackTo = cursor - } -} - -func (c *Client) clearAckTo(cursor uint64) { - c.mu.Lock() - defer c.mu.Unlock() - - if c.ackTo == cursor { - c.ackTo = 0 - } -} - -// storeCursor persists and sets the pull cursor. -func (c *Client) storeCursor(ctx context.Context, cursor uint64) error { - if err := c.cfg.Store.SaveCursor( - ctx, c.cfg.LocalMailboxID, cursor, - ); err != nil { - return err - } - - c.mu.Lock() - defer c.mu.Unlock() - - c.cursor = cursor - - return nil -} - -// randomID generates an opaque id backed by crypto/rand. -func randomID(nbytes int) (string, error) { - buf := make([]byte, nbytes) - if _, err := rand.Read(buf); err != nil { - return "", err - } - - return hex.EncodeToString(buf), nil -} - -var _ mailboxrpc.RPCClient = (*Client)(nil) diff --git a/mailbox/client/client_test.go b/mailbox/client/client_test.go deleted file mode 100644 index 5cdcb0af8..000000000 --- a/mailbox/client/client_test.go +++ /dev/null @@ -1,278 +0,0 @@ -package mailboxclient_test - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/lightninglabs/darepo-client/arkrpc" - mailboxclient "github.com/lightninglabs/darepo-client/mailbox/client" - mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" - mailboxrpc "github.com/lightninglabs/darepo-client/mailbox/rpc" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/types/known/anypb" -) - -// testArkServer implements a tiny mailbox RPC server used by unit tests. -type testArkServer struct { - resp *arkrpc.GetInfoResponse -} - -// GetInfo returns a fixed response for tests. -func (s *testArkServer) GetInfo(ctx context.Context, - req *arkrpc.GetInfoRequest) (*arkrpc.GetInfoResponse, error) { - - _ = ctx - _ = req - - return s.resp, nil -} - -// runOperator polls operatorMailboxID and responds to requests using mux. -func runOperator(ctx context.Context, edge mailboxpb.MailboxServiceClient, - operatorMailboxID string, mux *mailboxrpc.ServeMux) error { - - var cursor uint64 - - for { - select { - case <-ctx.Done(): - return nil - default: - } - - pull, err := edge.Pull(ctx, &mailboxpb.PullRequest{ - MailboxId: operatorMailboxID, - MaxEnvelopes: 10, - WaitTimeoutMs: 50, - Cursor: cursor, - }) - if err != nil { - return err - } - if pull.Status == nil || !pull.Status.Ok { - return fmt.Errorf("operator pull failed") - } - - for _, env := range pull.Envelopes { - err := handleOperatorEnvelope( - ctx, edge, operatorMailboxID, mux, env, - ) - if err != nil { - return err - } - } - - if pull.NextCursor > cursor { - ack, err := edge.AckUpTo(ctx, &mailboxpb.AckUpToRequest{ - MailboxId: operatorMailboxID, - Cursor: pull.NextCursor, - }) - if err != nil { - return err - } - if ack.Status == nil || !ack.Status.Ok { - return fmt.Errorf("operator ack failed") - } - - cursor = pull.NextCursor - } - } -} - -// handleOperatorEnvelope serves a request envelope and sends a response. -func handleOperatorEnvelope(ctx context.Context, - edge mailboxpb.MailboxServiceClient, operatorMailboxID string, - mux *mailboxrpc.ServeMux, env *mailboxpb.Envelope) error { - - if env == nil || env.Rpc == nil { - return nil - } - if env.Rpc.Kind != mailboxpb.RpcMeta_KIND_REQUEST { - return nil - } - - if env.Body == nil { - return fmt.Errorf("missing request body") - } - - resp, err := mux.ServeRPC(ctx, env.Rpc.Service, env.Rpc.Method, - env.Body.Value) - if err != nil { - return err - } - - respAny, err := anypb.New(resp) - if err != nil { - return err - } - - replyTo := env.Rpc.ReplyTo - if replyTo == "" { - return fmt.Errorf("missing reply_to") - } - - responseEnv := &mailboxpb.Envelope{ - ProtocolVersion: env.ProtocolVersion, - MsgId: "resp-" + env.MsgId, - Sender: operatorMailboxID, - Recipient: replyTo, - CreatedAtUnixMs: time.Now().UnixMilli(), - Body: respAny, - Rpc: &mailboxpb.RpcMeta{ - Kind: mailboxpb.RpcMeta_KIND_RESPONSE, - Service: env.Rpc.Service, - Method: env.Rpc.Method, - CorrelationId: env.Rpc.CorrelationId, - }, - } - - send, err := edge.Send(ctx, &mailboxpb.SendRequest{ - Envelope: responseEnv, - }) - if err != nil { - return err - } - if send.Status == nil || !send.Status.Ok { - return fmt.Errorf("operator send failed") - } - - return nil -} - -// TestClient_GetInfoRoundTrip verifies a basic request/response round trip. -func TestClient_GetInfoRoundTrip(t *testing.T) { - t.Parallel() - - mb := newInMemoryMailbox() - edge := &fakeMailboxServiceClient{mb: mb} - - mux := mailboxrpc.NewServeMux() - arkrpc.RegisterArkServiceMailboxServer(mux, &testArkServer{ - resp: &arkrpc.GetInfoResponse{ - Version: "v-test", - Network: "regtest", - BlockHeight: 123, - }, - }) - - operatorCtx, cancel := context.WithCancel(t.Context()) - defer cancel() - - operatorErr := make(chan error, 1) - go func() { - operatorErr <- runOperator(operatorCtx, edge, "operator", mux) - }() - - cfg := mailboxclient.DefaultConfig() - cfg.Edge = edge - cfg.LocalMailboxID = "client-1" - cfg.RemoteMailboxID = "operator" - cfg.ProtocolVersion = 1 - cfg.PullWaitTimeout = 20 * time.Millisecond - - rpc, err := mailboxclient.New(cfg) - require.NoError(t, err) - defer rpc.Stop() - - client := arkrpc.NewArkServiceMailboxClient(rpc) - - resp, err := client.GetInfo(t.Context(), &arkrpc.GetInfoRequest{}) - require.NoError(t, err) - require.Equal(t, "v-test", resp.Version) - require.Equal(t, "regtest", resp.Network) - require.Equal(t, uint32(123), resp.BlockHeight) - - cancel() - require.NoError(t, <-operatorErr) -} - -// TestClient_ConcurrentInFlightDoesNotDrop verifies that cursor-based acking -// does not discard a response for a different in-flight call. -func TestClient_ConcurrentInFlightDoesNotDrop(t *testing.T) { - t.Parallel() - - mb := newInMemoryMailbox() - edge := &fakeMailboxServiceClient{mb: mb} - - mux := mailboxrpc.NewServeMux() - arkrpc.RegisterArkServiceMailboxServer(mux, &testArkServer{ - resp: &arkrpc.GetInfoResponse{ - Version: "v-test", - Network: "regtest", - BlockHeight: 999, - }, - }) - - operatorCtx, cancel := context.WithCancel(t.Context()) - defer cancel() - - operatorErr := make(chan error, 1) - go func() { - operatorErr <- runOperator(operatorCtx, edge, "operator", mux) - }() - - cfg := mailboxclient.DefaultConfig() - cfg.Edge = edge - cfg.LocalMailboxID = "client-1" - cfg.RemoteMailboxID = "operator" - cfg.ProtocolVersion = 1 - cfg.PullWaitTimeout = 20 * time.Millisecond - - rpc, err := mailboxclient.New(cfg) - require.NoError(t, err) - defer rpc.Stop() - - type result struct { - resp *arkrpc.GetInfoResponse - err error - } - - call := func(ctx context.Context, correlationID string) result { - var out result - - result, err := rpc.SendRPC( - ctx, - mailboxrpc.ServiceMethod{ - Service: "arkrpc.ArkService", - Method: "GetInfo", - }, - &arkrpc.GetInfoRequest{}, - mailboxrpc.RPCOptions{ - CorrelationID: correlationID, - IdempotencyKey: correlationID, - }, - ) - if err != nil { - out.err = err - return out - } - - resp := new(arkrpc.GetInfoResponse) - err = rpc.AwaitRPC(ctx, result.CorrelationID, resp) - out.err = err - out.resp = resp - - return out - } - - ctx, cancelCalls := context.WithTimeout(t.Context(), 2*time.Second) - defer cancelCalls() - - ch1 := make(chan result, 1) - ch2 := make(chan result, 1) - go func() { ch1 <- call(ctx, "corr-1") }() - go func() { ch2 <- call(ctx, "corr-2") }() - - r1 := <-ch1 - r2 := <-ch2 - - require.NoError(t, r1.err) - require.NoError(t, r2.err) - require.Equal(t, uint32(999), r1.resp.BlockHeight) - require.Equal(t, uint32(999), r2.resp.BlockHeight) - - cancel() - require.NoError(t, <-operatorErr) -} diff --git a/mailbox/client/config.go b/mailbox/client/config.go deleted file mode 100644 index a558760e4..000000000 --- a/mailbox/client/config.go +++ /dev/null @@ -1,43 +0,0 @@ -package mailboxclient - -import ( - "time" - - mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" -) - -// Config holds configuration for a mailboxclient.Client. -type Config struct { - // Edge is the gRPC client for the mailbox edge service. - Edge mailboxpb.MailboxServiceClient - - // Store persists response payloads and pull cursor state. - // - // If unset, the client uses an in-memory store (not crash-safe). - Store Store - - // LocalMailboxID is the mailbox id used to receive responses. - LocalMailboxID string - - // RemoteMailboxID is the mailbox id used as the recipient for outbound - // requests (typically the operator ingress mailbox). - RemoteMailboxID string - - // ProtocolVersion is the protocol version set on all outbound - // envelopes. - ProtocolVersion uint32 - - // PullMaxEnvelopes bounds the size of Pull batches. - PullMaxEnvelopes uint32 - - // PullWaitTimeout controls long-poll behavior. - PullWaitTimeout time.Duration -} - -// DefaultConfig returns a Config populated with conservative defaults. -func DefaultConfig() Config { - return Config{ - PullMaxEnvelopes: 50, - PullWaitTimeout: 5 * time.Second, - } -} diff --git a/mailbox/client/doc.go b/mailbox/client/doc.go deleted file mode 100644 index 223f65262..000000000 --- a/mailbox/client/doc.go +++ /dev/null @@ -1,26 +0,0 @@ -// Package mailboxclient provides a concrete implementation of the -// mailboxrpc.RPCClient interface backed by the mailbox edge gRPC API -// (mailboxpb.MailboxService). -// -// The RPC client in this package focuses on: -// - encoding requests into mailboxpb.Envelope values, -// - receiving correlated responses by long-polling Pull, and -// - preventing response loss under cursor-based acking by caching pulled -// responses by correlation id before advancing the remote cursor. -// -// This package is intentionally small and self-contained so it can be used in -// both production code and tests. -// -// For crash safety, the Client can be configured with a Store implementation -// that persists: -// - the remote Pull cursor, and -// - response payloads keyed by correlation id. -// -// This is sufficient to avoid response loss when using cursor-based acking, -// even across restarts, as long as callers reuse correlation ids when retrying. -// -// This package does not attempt to implement the full “local durability ↔ -// remote mailbox” connector described in the spec (for example, integrating a -// transactional outbox with a local FSM store). That integration belongs at a -// higher layer that has access to the application's durability boundaries. -package mailboxclient diff --git a/mailbox/client/errors.go b/mailbox/client/errors.go deleted file mode 100644 index 82fa22912..000000000 --- a/mailbox/client/errors.go +++ /dev/null @@ -1,39 +0,0 @@ -package mailboxclient - -import ( - "fmt" - - mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" -) - -// StatusError wraps a mailboxpb.Status failure. -type StatusError struct { - // Op is the mailbox operation that failed (Send, Pull, AckUpTo). - Op string - - // Status is the status returned by the mailbox edge. - Status *mailboxpb.Status -} - -// Error returns a human-readable description of the error. -func (e *StatusError) Error() string { - if e == nil || e.Status == nil { - return "mailbox status error" - } - - return fmt.Sprintf("%s failed: %s (%s)", e.Op, e.Status.Message, - e.Status.Code) -} - -// statusOK returns true when status indicates success. -func statusOK(status *mailboxpb.Status) bool { - return status != nil && status.Ok -} - -// statusError constructs a StatusError for a failed mailbox response. -func statusError(op string, status *mailboxpb.Status) error { - return &StatusError{ - Op: op, - Status: status, - } -} diff --git a/mailbox/client/inmemory_test.go b/mailbox/client/inmemory_test.go deleted file mode 100644 index 07b6c9770..000000000 --- a/mailbox/client/inmemory_test.go +++ /dev/null @@ -1,242 +0,0 @@ -package mailboxclient_test - -import ( - "context" - "fmt" - "sync" - "time" - - mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" - "google.golang.org/grpc" - "google.golang.org/protobuf/proto" -) - -// inMemoryMailbox is a minimal in-memory implementation of the MailboxService -// semantics needed for unit tests. -type inMemoryMailbox struct { - mu sync.Mutex - - nextSeq uint64 - - // mailboxes stores all envelopes by mailbox id. - mailboxes map[string][]*mailboxpb.Envelope - - // ackedUpTo stores the ack watermark (event_seq < ackedUpTo are acked). - ackedUpTo map[string]uint64 - - notify chan struct{} -} - -// newInMemoryMailbox constructs an empty mailbox edge. -func newInMemoryMailbox() *inMemoryMailbox { - mb := &inMemoryMailbox{ - nextSeq: 1, - mailboxes: make(map[string][]*mailboxpb.Envelope), - ackedUpTo: make(map[string]uint64), - notify: make(chan struct{}), - } - - return mb -} - -// send enqueues envelope into recipient mailbox and assigns an event_seq. -func (m *inMemoryMailbox) send(envelope *mailboxpb.Envelope) *mailboxpb.Status { - m.mu.Lock() - defer m.mu.Unlock() - - if envelope == nil { - return &mailboxpb.Status{ - Ok: false, - Code: "INVALID_ARGUMENT", - Message: "missing envelope", - } - } - - envCopy, ok := cloneEnvelope(envelope) - if !ok { - return &mailboxpb.Status{ - Ok: false, - Code: "INVALID_ARGUMENT", - Message: "unexpected envelope type", - } - } - envCopy.EventSeq = m.nextSeq - m.nextSeq++ - - recipient := envCopy.Recipient - m.mailboxes[recipient] = append(m.mailboxes[recipient], envCopy) - - close(m.notify) - m.notify = make(chan struct{}) - - return okStatus() -} - -// pull returns envelopes with event_seq >= cursor and not acked. -func (m *inMemoryMailbox) pull(ctx context.Context, mailboxID string, - cursor uint64, maxEnvelopes uint32, - wait time.Duration) ([]*mailboxpb.Envelope, uint64, *mailboxpb.Status) { - - deadline := time.Now().Add(wait) - - for { - m.mu.Lock() - envs, next := m.pullLocked(mailboxID, cursor, maxEnvelopes) - if len(envs) > 0 || wait == 0 { - m.mu.Unlock() - return envs, next, okStatus() - } - - notify := m.notify - m.mu.Unlock() - - now := time.Now() - if !now.Before(deadline) { - return nil, cursor, okStatus() - } - - remaining := deadline.Sub(now) - timer := time.NewTimer(remaining) - select { - case <-notify: - case <-timer.C: - case <-ctx.Done(): - timer.Stop() - return nil, cursor, okStatus() - } - timer.Stop() - } -} - -// pullLocked assumes m.mu is held. -func (m *inMemoryMailbox) pullLocked(mailboxID string, cursor uint64, - maxEnvelopes uint32) ([]*mailboxpb.Envelope, uint64) { - - acked := m.ackedUpTo[mailboxID] - - var result []*mailboxpb.Envelope - var maxSeq uint64 - - for _, env := range m.mailboxes[mailboxID] { - if env.EventSeq < acked { - continue - } - if env.EventSeq < cursor { - continue - } - - clone, ok := cloneEnvelope(env) - if !ok { - continue - } - - result = append(result, clone) - if env.EventSeq > maxSeq { - maxSeq = env.EventSeq - } - - if uint32(len(result)) >= maxEnvelopes { - break - } - } - - if len(result) == 0 { - return nil, cursor - } - - return result, maxSeq + 1 -} - -// ackUpTo advances the ack cursor. -func (m *inMemoryMailbox) ackUpTo(mailboxID string, - cursor uint64) *mailboxpb.Status { - - m.mu.Lock() - defer m.mu.Unlock() - - if cursor > m.ackedUpTo[mailboxID] { - m.ackedUpTo[mailboxID] = cursor - } - - return okStatus() -} - -// cloneEnvelope makes a deep copy of env and returns false if proto.Clone does -// not return the expected type. -func cloneEnvelope(env *mailboxpb.Envelope) (*mailboxpb.Envelope, bool) { - clone := proto.Clone(env) - typed, ok := clone.(*mailboxpb.Envelope) - if !ok { - return nil, false - } - - return typed, true -} - -// okStatus returns a successful mailbox status. -func okStatus() *mailboxpb.Status { - return &mailboxpb.Status{Ok: true} -} - -// fakeMailboxServiceClient adapts the in-memory edge to MailboxServiceClient. -type fakeMailboxServiceClient struct { - mb *inMemoryMailbox -} - -// Send implements mailboxpb.MailboxServiceClient. -func (c *fakeMailboxServiceClient) Send( - ctx context.Context, - in *mailboxpb.SendRequest, - _ ...grpc.CallOption, -) (*mailboxpb.SendResponse, error) { - - _ = ctx - - if in == nil { - return nil, fmt.Errorf("nil request") - } - - status := c.mb.send(in.Envelope) - - return &mailboxpb.SendResponse{Status: status}, nil -} - -// Pull implements mailboxpb.MailboxServiceClient. -func (c *fakeMailboxServiceClient) Pull( - ctx context.Context, - in *mailboxpb.PullRequest, - _ ...grpc.CallOption, -) (*mailboxpb.PullResponse, error) { - - if in == nil { - return nil, fmt.Errorf("nil request") - } - - wait := time.Duration(in.WaitTimeoutMs) * time.Millisecond - - envs, next, status := c.mb.pull( - ctx, in.MailboxId, in.Cursor, in.MaxEnvelopes, wait, - ) - - return &mailboxpb.PullResponse{ - Status: status, - Envelopes: envs, - NextCursor: next, - }, nil -} - -// AckUpTo implements mailboxpb.MailboxServiceClient. -func (c *fakeMailboxServiceClient) AckUpTo(ctx context.Context, - in *mailboxpb.AckUpToRequest, _ ...grpc.CallOption) ( - *mailboxpb.AckUpToResponse, error) { - - _ = ctx - - if in == nil { - return nil, fmt.Errorf("nil request") - } - - status := c.mb.ackUpTo(in.MailboxId, in.Cursor) - - return &mailboxpb.AckUpToResponse{Status: status}, nil -} diff --git a/mailbox/client/log.go b/mailbox/client/log.go deleted file mode 100644 index 9e7897234..000000000 --- a/mailbox/client/log.go +++ /dev/null @@ -1,24 +0,0 @@ -package mailboxclient - -import "github.com/btcsuite/btclog/v2" - -// Subsystem defines the logging code for this subsystem. -const Subsystem = "MBXC" - -// log is a logger that is initialized with no output filters. This means the -// package will not perform any logging by default until the caller requests -// it. -var log = btclog.Disabled - -// DisableLog disables all library log output. Logging output is disabled by -// default until UseLogger is called. -func DisableLog() { - UseLogger(btclog.Disabled) -} - -// UseLogger uses a specified Logger to output package logging info. This -// should be used in preference to SetLogWriter if the caller is also using -// btclog. -func UseLogger(logger btclog.Logger) { - log = logger -} diff --git a/mailbox/client/store.go b/mailbox/client/store.go deleted file mode 100644 index e0f705bfc..000000000 --- a/mailbox/client/store.go +++ /dev/null @@ -1,164 +0,0 @@ -package mailboxclient - -import ( - "context" - "sync" -) - -// Store persists state needed for crash-safe RPC-over-mailbox operation. -// -// Callers that want crash safety should use a durable Store implementation and -// ensure correlation IDs are stable across retries (for example, by reusing the -// RPC idempotency key as the correlation id). -type Store interface { - // LoadCursor returns the persisted Pull cursor for mailboxID. - LoadCursor(ctx context.Context, mailboxID string) (uint64, error) - - // SaveCursor persists the Pull cursor for mailboxID. - // - // Implementations SHOULD treat cursor as monotonic and MUST NOT move it - // backward. - SaveCursor(ctx context.Context, mailboxID string, cursor uint64) error - - // PutResponse records a response payload for correlationID. - // - // payload is the raw protobuf message bytes stored in an Any.Value. - // - // PutResponse MUST be idempotent for the same mailboxID and - // correlationID. - // - // It SHOULD keep the first successfully stored payload. - PutResponse(ctx context.Context, mailboxID string, correlationID string, - payload []byte) error - - // GetResponse returns a previously recorded response payload. - GetResponse(ctx context.Context, mailboxID string, - correlationID string) (payload []byte, ok bool, err error) - - // DeleteResponse removes a previously recorded response payload. - DeleteResponse(ctx context.Context, mailboxID string, - correlationID string) error -} - -// MemoryStore is an in-memory Store implementation. -// -// It is useful for tests and short-lived processes, but it is not crash-safe. -type MemoryStore struct { - mu sync.Mutex - - cursors map[string]uint64 - responses map[string]map[string][]byte -} - -// NewMemoryStore constructs an empty in-memory store. -func NewMemoryStore() *MemoryStore { - return &MemoryStore{ - cursors: make(map[string]uint64), - responses: make(map[string]map[string][]byte), - } -} - -// LoadCursor returns the saved cursor for mailboxID. -func (s *MemoryStore) LoadCursor(ctx context.Context, mailboxID string) ( - uint64, error) { - - _ = ctx - - s.mu.Lock() - defer s.mu.Unlock() - - return s.cursors[mailboxID], nil -} - -// SaveCursor stores cursor for mailboxID. -func (s *MemoryStore) SaveCursor(ctx context.Context, mailboxID string, - cursor uint64) error { - - _ = ctx - - s.mu.Lock() - defer s.mu.Unlock() - - old := s.cursors[mailboxID] - if cursor < old { - return nil - } - - s.cursors[mailboxID] = cursor - - return nil -} - -// PutResponse stores payload for correlationID if it doesn't already exist. -func (s *MemoryStore) PutResponse(ctx context.Context, mailboxID string, - correlationID string, payload []byte) error { - - _ = ctx - - s.mu.Lock() - defer s.mu.Unlock() - - byMailbox, ok := s.responses[mailboxID] - if !ok { - byMailbox = make(map[string][]byte) - s.responses[mailboxID] = byMailbox - } - - if _, exists := byMailbox[correlationID]; exists { - return nil - } - - payloadCopy := make([]byte, len(payload)) - copy(payloadCopy, payload) - - byMailbox[correlationID] = payloadCopy - - return nil -} - -// GetResponse returns payload for correlationID if present. -func (s *MemoryStore) GetResponse(ctx context.Context, mailboxID string, - correlationID string) ([]byte, bool, error) { - - _ = ctx - - s.mu.Lock() - defer s.mu.Unlock() - - byMailbox, ok := s.responses[mailboxID] - if !ok { - return nil, false, nil - } - - payload, ok := byMailbox[correlationID] - if !ok { - return nil, false, nil - } - - payloadCopy := make([]byte, len(payload)) - copy(payloadCopy, payload) - - return payloadCopy, true, nil -} - -// DeleteResponse removes payload for correlationID if present. -func (s *MemoryStore) DeleteResponse(ctx context.Context, mailboxID string, - correlationID string) error { - - _ = ctx - - s.mu.Lock() - defer s.mu.Unlock() - - byMailbox, ok := s.responses[mailboxID] - if !ok { - return nil - } - - delete(byMailbox, correlationID) - if len(byMailbox) == 0 { - delete(s.responses, mailboxID) - } - - return nil -} From 2bf307cf14a47134bacfc943b8630b1adcbb0997 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Tue, 17 Feb 2026 22:09:19 -0800 Subject: [PATCH 12/14] serverconn: add TLV round-trip, runtime, and race coverage tests In this commit, we add tests covering the new functionality introduced in the prior commits. The TLV round-trip tests verify that SendClientEventRequest produces stable deterministic MsgID and IdempotencyKey for equal payloads, that explicitly set identifiers survive encode/decode unchanged, and that SendRPCRequest round-trips its full envelope. The Receive path is tested for unknown message rejection and the SendRPCRequest dispatch handler. The runtime tests verify config validation rejects missing Store and LocalMailboxID, and that NewRuntime fills a default codec and wires all accessors (Ref, TellRef, Unary, Connector) correctly. The connector test gains TestEgress_EventRetriesPreserveIdempotencyKey, which verifies that two egress sends of the same semantic event produce identical MsgId and IdempotencyKey on the wire. The unary facade test gains TestUnaryFacade_ResponseBeforeAwait, which exercises the early- response buffering path where the ingress loop delivers a response before the caller starts AwaitRPC. --- serverconn/actor_tlv_test.go | 210 ++++++++++++++++++++++++++++++++ serverconn/log_test.go | 14 +++ serverconn/runtime_test.go | 114 +++++++++++++++++ serverconn/unary_facade_test.go | 51 ++++++++ 4 files changed, 389 insertions(+) create mode 100644 serverconn/actor_tlv_test.go create mode 100644 serverconn/log_test.go create mode 100644 serverconn/runtime_test.go diff --git a/serverconn/actor_tlv_test.go b/serverconn/actor_tlv_test.go new file mode 100644 index 000000000..62a0d996f --- /dev/null +++ b/serverconn/actor_tlv_test.go @@ -0,0 +1,210 @@ +package serverconn + +import ( + "bytes" + "io" + "testing" + "time" + + "github.com/lightninglabs/darepo-client/baselib/actor" + mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" + "github.com/lightningnetwork/lnd/tlv" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// bytesServerMessage is a minimal ServerMessage wrapper for TLV tests. +type bytesServerMessage struct { + payload []byte +} + +// ToProto converts the payload to a protobuf wrapper. +func (m *bytesServerMessage) ToProto() proto.Message { + return wrapperspb.Bytes(m.payload) +} + +// encodeTLVMessage serializes a TLV message to bytes for round-trip tests. +func encodeTLVMessage(t *testing.T, msg actor.TLVMessage) []byte { + t.Helper() + + var buf bytes.Buffer + require.NoError(t, msg.Encode(&buf)) + + return buf.Bytes() +} + +// TestSendClientEventRequest_TLVRoundTrip_DeterministicIDs verifies that equal +// payloads produce stable derived MsgID and IdempotencyKey across round trips. +func TestSendClientEventRequest_TLVRoundTrip_DeterministicIDs(t *testing.T) { + t.Parallel() + + reqA := &SendClientEventRequest{ + Message: &bytesServerMessage{payload: []byte("same-event")}, + } + reqB := &SendClientEventRequest{ + Message: &bytesServerMessage{payload: []byte("same-event")}, + } + + var decodedA SendClientEventRequest + require.NoError( + t, decodedA.Decode(bytes.NewReader(encodeTLVMessage(t, reqA))), + ) + + var decodedB SendClientEventRequest + require.NoError( + t, decodedB.Decode(bytes.NewReader(encodeTLVMessage(t, reqB))), + ) + + require.NotEmpty(t, decodedA.MsgID) + require.NotEmpty(t, decodedA.IdempotencyKey) + require.Equal(t, decodedA.MsgID, decodedB.MsgID) + require.Equal( + t, decodedA.IdempotencyKey, decodedB.IdempotencyKey, + ) + + msg, ok := decodedA.Message.ToProto().(*wrapperspb.BytesValue) + require.True(t, ok) + require.Equal(t, []byte("same-event"), msg.Value) +} + +// TestSendClientEventRequest_TLVRoundTrip_ExplicitIDs verifies explicit +// identifiers survive Encode/Decode unchanged. +func TestSendClientEventRequest_TLVRoundTrip_ExplicitIDs(t *testing.T) { + t.Parallel() + + req := &SendClientEventRequest{ + Message: &bytesServerMessage{payload: []byte("payload")}, + MsgID: "msg-explicit", + IdempotencyKey: "idem-explicit", + } + + var decoded SendClientEventRequest + require.NoError( + t, decoded.Decode(bytes.NewReader(encodeTLVMessage(t, req))), + ) + + require.Equal(t, "msg-explicit", decoded.MsgID) + require.Equal(t, "idem-explicit", decoded.IdempotencyKey) +} + +// TestSendRPCRequest_TLVRoundTrip verifies envelope serialization round trips. +func TestSendRPCRequest_TLVRoundTrip(t *testing.T) { + t.Parallel() + + body, err := anypb.New(wrapperspb.String("request")) + require.NoError(t, err) + + original := &SendRPCRequest{ + Envelope: &mailboxpb.Envelope{ + ProtocolVersion: 1, + MsgId: "msg-1", + Sender: "client-1", + Recipient: "server-1", + CreatedAtUnixMs: time.Now().UnixMilli(), + Body: body, + Rpc: &mailboxpb.RpcMeta{ + Kind: mailboxpb.RpcMeta_KIND_REQUEST, + Service: "test.Svc", + Method: "DoThing", + CorrelationId: "corr-1", + }, + }, + } + + var decoded SendRPCRequest + encoded := encodeTLVMessage(t, original) + + require.NoError( + t, decoded.Decode(bytes.NewReader(encoded)), + ) + + require.True(t, proto.Equal(original.Envelope, decoded.Envelope)) +} + +// unknownServerConnMsg is a test-only unsupported message for Receive. +type unknownServerConnMsg struct { + actor.BaseMessage +} + +// MessageType returns a test message type name. +func (m *unknownServerConnMsg) MessageType() string { + return "unknownServerConnMsg" +} + +// TLVType returns a test-only TLV type. +func (m *unknownServerConnMsg) TLVType() tlv.Type { + return 999_999 +} + +// Encode serializes a no-op payload for unknown message tests. +func (m *unknownServerConnMsg) Encode(w io.Writer) error { + _, err := w.Write(nil) + + return err +} + +// Decode deserializes a no-op payload for unknown message tests. +func (m *unknownServerConnMsg) Decode(r io.Reader) error { + _, _ = io.Copy(io.Discard, r) + + return nil +} + +// serverConnMsgSealed marks the type as part of ServerConnMsg in tests. +func (m *unknownServerConnMsg) serverConnMsgSealed() {} + +// TestServerConnectionActor_ReceiveUnknownMessage verifies Receive rejects +// unsupported message types. +func TestServerConnectionActor_ReceiveUnknownMessage(t *testing.T) { + t.Parallel() + + connector, _, _ := newTestConnector(t, nil) + result := connector.Receive(t.Context(), &unknownServerConnMsg{}) + require.Error(t, result.Err()) +} + +// TestServerConnectionActor_ReceiveSendRPCRequest verifies the SendRPCRequest +// path sends the provided envelope to the mailbox edge. +func TestServerConnectionActor_ReceiveSendRPCRequest(t *testing.T) { + t.Parallel() + + connector, mb, _ := newTestConnector(t, nil) + + body, err := anypb.New(wrapperspb.String("rpc")) + require.NoError(t, err) + + envelope := &mailboxpb.Envelope{ + ProtocolVersion: 1, + MsgId: "msg-rpc", + Sender: "client-1", + Recipient: "server-1", + Body: body, + Rpc: &mailboxpb.RpcMeta{ + Kind: mailboxpb.RpcMeta_KIND_REQUEST, + Service: "test.Svc", + Method: "Rpc", + CorrelationId: "corr-rpc", + }, + } + + result := connector.Receive(t.Context(), &SendRPCRequest{ + Envelope: envelope, + }) + require.NoError(t, result.Err()) + + mb.mu.Lock() + envs := append( + []*mailboxpb.Envelope(nil), mb.mailboxes["server-1"]..., + ) + mb.mu.Unlock() + + require.Len(t, envs, 1) + require.Equal(t, envelope.MsgId, envs[0].MsgId) + require.Equal(t, envelope.Sender, envs[0].Sender) + require.Equal(t, envelope.Recipient, envs[0].Recipient) + require.Equal( + t, envelope.Rpc.CorrelationId, envs[0].Rpc.CorrelationId, + ) +} diff --git a/serverconn/log_test.go b/serverconn/log_test.go new file mode 100644 index 000000000..1f296986b --- /dev/null +++ b/serverconn/log_test.go @@ -0,0 +1,14 @@ +package serverconn + +import ( + "testing" + + "github.com/btcsuite/btclog/v2" +) + +// TestLoggerHelpers verifies the package logger helper functions are callable. +// Not parallel — mutates the package-level logger. +func TestLoggerHelpers(t *testing.T) { + UseLogger(btclog.Disabled) + DisableLog() +} diff --git a/serverconn/runtime_test.go b/serverconn/runtime_test.go new file mode 100644 index 000000000..255fbe88f --- /dev/null +++ b/serverconn/runtime_test.go @@ -0,0 +1,114 @@ +package serverconn + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestDurableActorID verifies the stable actor ID derivation helper. +func TestDurableActorID(t *testing.T) { + t.Parallel() + + require.Equal(t, "serverconn-client-1", DurableActorID("client-1")) +} + +// TestNewRuntime_ValidateConfig verifies runtime construction rejects missing +// required configuration. +func TestNewRuntime_ValidateConfig(t *testing.T) { + t.Parallel() + + _, err := NewRuntime(ConnectorConfig{}) + require.Error(t, err) + require.Contains(t, err.Error(), "connector store is required") + + _, err = NewRuntime(ConnectorConfig{ + Store: newMemCheckpointStore(), + }) + require.Error(t, err) + require.Contains(t, err.Error(), "connector edge is required") + + mb := newInMemoryMailbox() + _, err = NewRuntime(ConnectorConfig{ + Store: newMemCheckpointStore(), + Edge: &fakeMailboxServiceClient{mb: mb}, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "local mailbox id is required") + + _, err = NewRuntime(ConnectorConfig{ + Store: newMemCheckpointStore(), + Edge: &fakeMailboxServiceClient{mb: mb}, + LocalMailboxID: "client-1", + }) + require.Error(t, err) + require.Contains( + t, err.Error(), "remote mailbox id is required", + ) +} + +// TestNewRuntime_DefaultCodec verifies runtime construction fills a default +// codec when one is not supplied. +func TestNewRuntime_DefaultCodec(t *testing.T) { + t.Parallel() + + mb := newInMemoryMailbox() + cfg := DefaultConnectorConfig() + cfg.Edge = &fakeMailboxServiceClient{mb: mb} + cfg.Store = newMemCheckpointStore() + cfg.LocalMailboxID = "client-1" + cfg.RemoteMailboxID = "server-1" + cfg.ProtocolVersion = 1 + cfg.Codec = nil + + runtime, err := NewRuntime(cfg) + require.NoError(t, err) + require.NotNil(t, runtime) + require.NotNil(t, runtime.Unary()) + require.NotNil(t, runtime.Connector()) + require.NotNil(t, runtime.TellRef()) + require.NotNil(t, runtime.Ref()) + require.Equal( + t, DurableActorID("client-1"), runtime.Ref().ID(), + ) +} + +// TestRuntime_StartStop verifies runtime lifecycle methods run and return +// promptly when the parent context is cancelled. +func TestRuntime_StartStop(t *testing.T) { + t.Parallel() + + mb := newInMemoryMailbox() + + cfg := DefaultConnectorConfig() + cfg.Edge = &fakeMailboxServiceClient{mb: mb} + cfg.Store = newMemCheckpointStore() + cfg.LocalMailboxID = "client-1" + cfg.RemoteMailboxID = "server-1" + cfg.ProtocolVersion = 1 + cfg.PullWaitTimeout = 25 * time.Millisecond + + runtime, err := NewRuntime(cfg) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(t.Context()) + require.NoError(t, runtime.Start(ctx)) + + time.Sleep(50 * time.Millisecond) + cancel() + + done := make(chan struct{}) + go func() { + runtime.Stop() + close(done) + }() + + select { + case <-done: + + case <-time.After(5 * time.Second): + t.Fatal("runtime Stop did not return") + } +} diff --git a/serverconn/unary_facade_test.go b/serverconn/unary_facade_test.go index b0bef9871..a34f48e11 100644 --- a/serverconn/unary_facade_test.go +++ b/serverconn/unary_facade_test.go @@ -128,6 +128,57 @@ func TestUnaryFacade_AwaitRPC(t *testing.T) { require.Equal(t, "world", resp.Value) } +// TestUnaryFacade_ResponseBeforeAwait verifies that a response arriving before +// AwaitRPC is still delivered to the caller. +func TestUnaryFacade_ResponseBeforeAwait(t *testing.T) { + t.Parallel() + + actor, mb, _ := newTestConnector(t, nil) + facade := NewUnaryFacade(actor) + + // Start ingress so responses can be pulled and buffered. + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + actor.StartIngress(ctx) + defer actor.StopIngress() + + method := mailboxrpc.ServiceMethod{ + Service: "test.Svc", + Method: "GetInfo", + } + + result, err := facade.SendRPC( + t.Context(), method, + wrapperspb.String("request"), mailboxrpc.RPCOptions{}, + ) + require.NoError(t, err) + + responseMsg := wrapperspb.String("early") + responseBytes, err := proto.Marshal(responseMsg) + require.NoError(t, err) + + sendResponseToMailbox( + t, mb, "client-1", result.CorrelationID, responseBytes, + ) + + // Ensure ingress had a chance to pull and process the response before + // we start awaiting it. + require.Eventually(t, func() bool { + return mb.getAckedUpTo("client-1") > 0 + }, 5*time.Second, 10*time.Millisecond) + + awaitCtx, awaitCancel := context.WithTimeout( + t.Context(), 5*time.Second, + ) + defer awaitCancel() + + var resp wrapperspb.StringValue + err = facade.AwaitRPC(awaitCtx, result.CorrelationID, &resp) + require.NoError(t, err) + require.Equal(t, "early", resp.Value) +} + // TestUnaryFacade_AwaitRPC_CancelledContext verifies that AwaitRPC returns // the context error when the context is cancelled. func TestUnaryFacade_AwaitRPC_CancelledContext(t *testing.T) { From ca446688820a96fd29151b623e82480d26528edf Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Wed, 18 Feb 2026 17:41:41 -0800 Subject: [PATCH 13/14] serverconn: upgrade memCheckpointStore to full DeliveryStore In this commit, we replace the panicking stubs in memCheckpointStore with working in-memory implementations of the full actor.DeliveryStore interface. The store now supports EnqueueMessage with ON CONFLICT DO NOTHING semantics for receiver-side dedup, LeaseNextMessage with FIFO ordering and lease expiry, AckMessage, NackMessage with retry-after delays, ExtendLease, MoveToDeadLetter, IsProcessed/MarkProcessed, and the outbox claim/complete/fail cycle. This upgrade is needed by upcoming tests that exercise the durable actor runtime integration path where the runtime calls through to the full DeliveryStore contract rather than just the checkpoint subset. --- serverconn/testutil_test.go | 441 +++++++++++++++++++++++++++++++++--- 1 file changed, 414 insertions(+), 27 deletions(-) diff --git a/serverconn/testutil_test.go b/serverconn/testutil_test.go index 03afbe34f..0a3aec712 100644 --- a/serverconn/testutil_test.go +++ b/serverconn/testutil_test.go @@ -264,17 +264,33 @@ func (c *fakeMailboxServiceClient) AckUpTo( } // memCheckpointStore is a minimal in-memory implementation of the checkpoint -// subset of actor.DeliveryStore needed for ingress loop tests. Only -// SaveCheckpoint and LoadCheckpoint are implemented; all other methods panic. +// and delivery portions of actor.DeliveryStore used by serverconn tests. type memCheckpointStore struct { mu sync.Mutex checkpoints map[string]*actor.Checkpoint + + messages map[string]*storeMessage + askResults map[string]*actor.AskResult + processed map[string]bool + deadLetters map[string]*actor.DeadLetter + outbox map[string]*actor.OutboxMessage +} + +// storeMessage tracks mailbox delivery state in memory. +type storeMessage struct { + leased actor.LeasedMessage + availableAt time.Time } // newMemCheckpointStore creates a new empty checkpoint store. func newMemCheckpointStore() *memCheckpointStore { return &memCheckpointStore{ checkpoints: make(map[string]*actor.Checkpoint), + messages: make(map[string]*storeMessage), + askResults: make(map[string]*actor.AskResult), + processed: make(map[string]bool), + deadLetters: make(map[string]*actor.DeadLetter), + outbox: make(map[string]*actor.OutboxMessage), } } @@ -313,165 +329,536 @@ func (s *memCheckpointStore) LoadCheckpoint( return cp, nil } -// The remaining DeliveryStore methods are unused in connector tests and panic -// if called. - +// EnqueueMessage persists a mailbox message in memory. func (s *memCheckpointStore) EnqueueMessage( ctx context.Context, params actor.EnqueueParams, ) error { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + // Mirror production ON CONFLICT DO NOTHING semantics for receiver-side + // deduplication on retry deliveries. + if _, exists := s.messages[params.ID]; exists { + return nil + } + + payloadCopy := append([]byte(nil), params.Payload...) + + s.messages[params.ID] = &storeMessage{ + leased: actor.LeasedMessage{ + ID: params.ID, + MailboxID: params.MailboxID, + MessageType: params.MessageType, + Payload: payloadCopy, + PromiseID: params.PromiseID, + CallbackActorID: params.CallbackActorID, + CorrelationID: params.CorrelationID, + Priority: params.Priority, + Attempts: 0, + MaxAttempts: params.MaxAttempts, + CreatedAt: time.Now(), + }, + availableAt: params.AvailableAt, + } + + return nil } +// LeaseNextMessage leases the next available mailbox message. func (s *memCheckpointStore) LeaseNextMessage( ctx context.Context, mailboxID string, leaseToken string, leaseDuration time.Duration, ) (*actor.LeasedMessage, error) { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + now := time.Now() + + var selected *storeMessage + for _, msg := range s.messages { + if msg.leased.MailboxID != mailboxID { + continue + } + + if msg.availableAt.After(now) { + continue + } + + if msg.leased.LeaseToken != "" && + msg.leased.LeaseUntil.After(now) { + + continue + } + + if selected == nil || + msg.availableAt.Before(selected.availableAt) { + + selected = msg + } + } + + if selected == nil { + return nil, nil + } + + selected.leased.LeaseToken = leaseToken + selected.leased.LeaseUntil = now.Add(leaseDuration) + selected.leased.Attempts++ + + leasedCopy := selected.leased + leasedCopy.Payload = append( + []byte(nil), selected.leased.Payload..., + ) + + return &leasedCopy, nil } +// AckMessage acknowledges a leased message by ID and lease token. func (s *memCheckpointStore) AckMessage( ctx context.Context, id, leaseToken string, ) (int64, error) { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + msg, ok := s.messages[id] + if !ok { + return 0, nil + } + + if msg.leased.LeaseToken != leaseToken { + return 0, nil + } + + delete(s.messages, id) + + return 1, nil } +// NackMessage releases a leased message for later redelivery. func (s *memCheckpointStore) NackMessage( ctx context.Context, id, leaseToken string, retryAfter time.Duration, ) (int64, error) { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + msg, ok := s.messages[id] + if !ok { + return 0, nil + } + + if msg.leased.LeaseToken != leaseToken { + return 0, nil + } + + msg.leased.LeaseToken = "" + msg.leased.LeaseUntil = time.Time{} + msg.availableAt = time.Now().Add(retryAfter) + + return 1, nil } +// ExtendLease extends a message lease when lease token matches. func (s *memCheckpointStore) ExtendLease( ctx context.Context, id, leaseToken string, extension time.Duration, ) (int64, error) { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + msg, ok := s.messages[id] + if !ok { + return 0, nil + } + + if msg.leased.LeaseToken != leaseToken { + return 0, nil + } + + msg.leased.LeaseUntil = time.Now().Add(extension) + + return 1, nil } +// MoveToDeadLetter moves a mailbox message to the dead letter map. func (s *memCheckpointStore) MoveToDeadLetter( ctx context.Context, id, reason string, ) error { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + msg, ok := s.messages[id] + if !ok { + return nil + } + + payloadCopy := append([]byte(nil), msg.leased.Payload...) + + s.deadLetters[id] = &actor.DeadLetter{ + ID: id, + Source: "mailbox", + ActorID: msg.leased.MailboxID, + MessageType: msg.leased.MessageType, + Payload: payloadCopy, + FailureReason: reason, + Attempts: msg.leased.Attempts, + CreatedAt: time.Now(), + } + + return nil } +// DeleteMessage removes a mailbox message by ID. func (s *memCheckpointStore) DeleteMessage( ctx context.Context, id string, ) error { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + delete(s.messages, id) + + return nil } +// SaveAskResult stores an ask result in memory. func (s *memCheckpointStore) SaveAskResult( ctx context.Context, params actor.AskResultParams, ) error { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + resultBlob := append([]byte(nil), params.ResultBlob...) + + s.askResults[params.PromiseID] = &actor.AskResult{ + PromiseID: params.PromiseID, + ResultBlob: resultBlob, + ErrorText: params.ErrorText, + CreatedAt: time.Now(), + ExpiresAt: params.ExpiresAt, + } + + return nil } +// GetAskResult retrieves an ask result from memory. func (s *memCheckpointStore) GetAskResult( ctx context.Context, promiseID string, ) (*actor.AskResult, error) { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + result, ok := s.askResults[promiseID] + if !ok { + return nil, nil + } + + clone := *result + clone.ResultBlob = append([]byte(nil), result.ResultBlob...) + + return &clone, nil } +// DeleteAskResult removes an ask result by promise ID. func (s *memCheckpointStore) DeleteAskResult( ctx context.Context, promiseID string, ) error { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + delete(s.askResults, promiseID) + + return nil } +// EnqueueOutbox stores an outbox message in memory. func (s *memCheckpointStore) EnqueueOutbox( ctx context.Context, params actor.OutboxParams, ) error { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + payloadCopy := append([]byte(nil), params.Payload...) + + s.outbox[params.ID] = &actor.OutboxMessage{ + ID: params.ID, + SourceActorID: params.SourceActorID, + TargetActorID: params.TargetActorID, + MessageType: params.MessageType, + Payload: payloadCopy, + DomainKey: params.DomainKey, + Version: params.Version, + Status: "pending", + CreatedAt: time.Now(), + } + + return nil } +// ClaimOutboxBatch claims pending outbox messages up to the given limit. func (s *memCheckpointStore) ClaimOutboxBatch( ctx context.Context, params actor.OutboxClaimParams, ) ([]actor.OutboxMessage, error) { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + var result []actor.OutboxMessage + + for _, msg := range s.outbox { + if msg.Status != "pending" { + continue + } + + msg.ClaimToken = params.ClaimToken + msg.DeliveryAttempts++ + + copyMsg := *msg + copyMsg.Payload = append([]byte(nil), msg.Payload...) + result = append(result, copyMsg) + + if len(result) >= params.Limit { + break + } + } + + return result, nil } +// CompleteOutbox marks a claimed outbox message as completed. func (s *memCheckpointStore) CompleteOutbox( ctx context.Context, id, claimToken string, ) error { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + msg, ok := s.outbox[id] + if !ok { + return nil + } + + if msg.ClaimToken != claimToken { + return nil + } + + msg.Status = "completed" + + return nil } +// FailOutbox marks a claimed outbox message as dead-lettered. func (s *memCheckpointStore) FailOutbox( ctx context.Context, id, claimToken string, ) error { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + msg, ok := s.outbox[id] + if !ok { + return nil + } + + if msg.ClaimToken != claimToken { + return nil + } + + msg.Status = "dead_letter" + + return nil } +// IsProcessed returns true when the message ID is marked processed. func (s *memCheckpointStore) IsProcessed( ctx context.Context, id string, ) (bool, error) { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + return s.processed[id], nil } +// MarkProcessed records a processed message ID. func (s *memCheckpointStore) MarkProcessed( ctx context.Context, id, actorID string, ttl time.Duration, ) error { - panic("not implemented") + _ = ctx + _ = actorID + _ = ttl + + s.mu.Lock() + defer s.mu.Unlock() + + s.processed[id] = true + + return nil } +// DeleteCheckpoint deletes the checkpoint for actorID. func (s *memCheckpointStore) DeleteCheckpoint( ctx context.Context, actorID string, ) error { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + delete(s.checkpoints, actorID) + + return nil } +// GetDeadLetter returns a dead letter entry by ID. func (s *memCheckpointStore) GetDeadLetter( ctx context.Context, id string, ) (*actor.DeadLetter, error) { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + deadLetter, ok := s.deadLetters[id] + if !ok { + return nil, nil + } + + copyDeadLetter := *deadLetter + copyDeadLetter.Payload = append([]byte(nil), deadLetter.Payload...) + + return ©DeadLetter, nil } +// ListDeadLetters lists dead letters for the given actor ID. func (s *memCheckpointStore) ListDeadLetters( ctx context.Context, actorID string, limit int, ) ([]actor.DeadLetter, error) { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + var result []actor.DeadLetter + + for _, deadLetter := range s.deadLetters { + if deadLetter.ActorID != actorID { + continue + } + + copyDeadLetter := *deadLetter + copyDeadLetter.Payload = append( + []byte(nil), deadLetter.Payload..., + ) + result = append(result, copyDeadLetter) + + if len(result) >= limit { + break + } + } + + return result, nil } +// DeleteDeadLetter deletes a dead-letter entry by ID. func (s *memCheckpointStore) DeleteDeadLetter( ctx context.Context, id string, ) error { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + delete(s.deadLetters, id) + + return nil } +// ExpireLeases clears lease tokens for expired messages. func (s *memCheckpointStore) ExpireLeases( ctx context.Context, ) error { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + now := time.Now() + for _, msg := range s.messages { + if msg.leased.LeaseUntil.Before(now) { + msg.leased.LeaseToken = "" + msg.leased.LeaseUntil = time.Time{} + } + } + + return nil } +// CleanupExpired removes expired ask results. func (s *memCheckpointStore) CleanupExpired( ctx context.Context, ) error { - panic("not implemented") + _ = ctx + + s.mu.Lock() + defer s.mu.Unlock() + + now := time.Now() + for id, askResult := range s.askResults { + if askResult.ExpiresAt.Before(now) { + delete(s.askResults, id) + } + } + + return nil } // Compile-time check. From 2b19f47821decf0985ab34c67b568acbb6ce3466 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Wed, 18 Feb 2026 17:41:58 -0800 Subject: [PATCH 14/14] serverconn: expand test coverage across all connector layers In this commit, we add tests covering the remaining gaps identified during review: ingress error paths, crash/restart replay semantics, property-based ack invariant checks, high-concurrency unary response delivery, codec round-trips, and runtime lifecycle. The ingress error tests use a configurable MailboxServiceClient stub to verify behavior under Pull failures, AckUpTo failures, partial dispatch failures, and responses with missing or nil RPC metadata. The restart/replay test exercises egress idempotency by verifying that a failed Send followed by a retry produces identical MsgId and IdempotencyKey on the wire. The property-based test uses pgregory.net/rapid to randomize ack and partial-dispatch progressions, asserting that AckCommittedTo never exceeds DispatchCommittedTo under arbitrary interleaving. The high-concurrency unary test fires 200 requests, injects responses in shuffled order before AwaitRPC begins, and verifies each caller receives its correct response through the early-response buffering path. The runtime lifecycle test verifies Start/Stop complete promptly on context cancellation without goroutine leaks. --- serverconn/actor_tlv_test.go | 88 ++++++++++ serverconn/connector_test.go | 15 +- serverconn/ingress_error_test.go | 269 ++++++++++++++++++++++++++++++ serverconn/restart_replay_test.go | 167 +++++++++++++++++++ serverconn/types.go | 3 - serverconn/unary_facade_test.go | 98 ++++++++++- 6 files changed, 632 insertions(+), 8 deletions(-) create mode 100644 serverconn/ingress_error_test.go create mode 100644 serverconn/restart_replay_test.go diff --git a/serverconn/actor_tlv_test.go b/serverconn/actor_tlv_test.go index 62a0d996f..6c00c871a 100644 --- a/serverconn/actor_tlv_test.go +++ b/serverconn/actor_tlv_test.go @@ -123,6 +123,94 @@ func TestSendRPCRequest_TLVRoundTrip(t *testing.T) { require.True(t, proto.Equal(original.Envelope, decoded.Envelope)) } +// TestServerConnMessageMetadata verifies static message metadata methods. +func TestServerConnMessageMetadata(t *testing.T) { + t.Parallel() + + eventReq := &SendClientEventRequest{} + require.Equal(t, "SendClientEventRequest", eventReq.MessageType()) + require.Equal(t, SendClientEventRequestMsgType, eventReq.TLVType()) + eventReq.serverConnMsgSealed() + + eventResp := &SendClientEventResponse{} + require.Equal(t, "SendClientEventResponse", eventResp.MessageType()) + eventResp.serverConnRespSealed() + + rpcReq := &SendRPCRequest{} + require.Equal(t, "SendRPCRequest", rpcReq.MessageType()) + require.Equal(t, SendRPCRequestMsgType, rpcReq.TLVType()) + rpcReq.serverConnMsgSealed() +} + +// TestRawServerMessage_ToProtoDecodeFailure verifies ToProto returns nil when +// the Any payload cannot be resolved through the protobuf type registry. +func TestRawServerMessage_ToProtoDecodeFailure(t *testing.T) { + t.Parallel() + + raw := &rawServerMessage{ + anyMsg: &anypb.Any{ + TypeUrl: "type.googleapis.com/test.unknown.Message", + Value: []byte{0x01}, + }, + } + + require.Nil(t, raw.ToProto()) +} + +// TestServerConnCodec_RoundTrip verifies both serverconn message types can be +// encoded and decoded via NewServerConnCodec. +func TestServerConnCodec_RoundTrip(t *testing.T) { + t.Parallel() + + codec := NewServerConnCodec() + + eventPayload := []byte("codec-event") + eventReq := &SendClientEventRequest{ + Message: &bytesServerMessage{ + payload: eventPayload, + }, + MsgID: "msg-codec-event", + IdempotencyKey: "idem-codec-event", + } + + eventBytes, err := codec.Encode(eventReq) + require.NoError(t, err) + + decodedEvent, err := codec.Decode(eventBytes) + require.NoError(t, err) + + typedEvent, ok := decodedEvent.(*SendClientEventRequest) + require.True(t, ok) + require.Equal(t, eventReq.MsgID, typedEvent.MsgID) + require.Equal( + t, eventReq.IdempotencyKey, typedEvent.IdempotencyKey, + ) + + body, err := anypb.New(wrapperspb.String("codec-rpc")) + require.NoError(t, err) + + rpcReq := &SendRPCRequest{ + Envelope: &mailboxpb.Envelope{ + ProtocolVersion: 1, + MsgId: "msg-codec-rpc", + Sender: "client-1", + Recipient: "server-1", + Body: body, + }, + } + + rpcBytes, err := codec.Encode(rpcReq) + require.NoError(t, err) + + decodedRPC, err := codec.Decode(rpcBytes) + require.NoError(t, err) + + typedRPC, ok := decodedRPC.(*SendRPCRequest) + require.True(t, ok) + require.Equal(t, rpcReq.Envelope.MsgId, typedRPC.Envelope.MsgId) + require.Equal(t, rpcReq.Envelope.Sender, typedRPC.Envelope.Sender) +} + // unknownServerConnMsg is a test-only unsupported message for Receive. type unknownServerConnMsg struct { actor.BaseMessage diff --git a/serverconn/connector_test.go b/serverconn/connector_test.go index 28b21cb3c..6e8dd6260 100644 --- a/serverconn/connector_test.go +++ b/serverconn/connector_test.go @@ -9,10 +9,22 @@ import ( mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" mailboxrpc "github.com/lightninglabs/darepo-client/mailbox/rpc" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/wrapperspb" ) +// testServerMessage is a minimal ServerMessage implementation for egress +// tests. +type testServerMessage struct { + value string +} + +// ToProto converts the test message to a protobuf payload. +func (m *testServerMessage) ToProto() proto.Message { + return wrapperspb.String(m.value) +} + // newTestConnector builds a ServerConnectionActor with in-memory test // dependencies. func newTestConnector( @@ -394,7 +406,7 @@ func TestIngress_PartialDispatch_NoDuplicateRedelivery(t *testing.T) { Status: &mailboxpb.Status{ Ok: false, Code: "INTERNAL", - Message: "injected batch failure", + Message: "injected failure", }, } } @@ -438,6 +450,7 @@ func TestIngress_PartialDispatch_NoDuplicateRedelivery(t *testing.T) { "(1 fail + 1 retry)", ) } + // TestRetryDelay verifies the exponential backoff formula with jitter. func TestRetryDelay(t *testing.T) { t.Parallel() diff --git a/serverconn/ingress_error_test.go b/serverconn/ingress_error_test.go new file mode 100644 index 000000000..2827e02b8 --- /dev/null +++ b/serverconn/ingress_error_test.go @@ -0,0 +1,269 @@ +package serverconn + +import ( + "context" + "fmt" + "testing" + + "github.com/lightninglabs/darepo-client/baselib/actor" + mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// mailboxClientStub is a configurable MailboxServiceClient test double. +type mailboxClientStub struct { + sendFn func( + ctx context.Context, in *mailboxpb.SendRequest, + opts ...grpc.CallOption, + ) (*mailboxpb.SendResponse, error) + + pullFn func( + ctx context.Context, in *mailboxpb.PullRequest, + opts ...grpc.CallOption, + ) (*mailboxpb.PullResponse, error) + + ackFn func( + ctx context.Context, in *mailboxpb.AckUpToRequest, + opts ...grpc.CallOption, + ) (*mailboxpb.AckUpToResponse, error) +} + +// Send executes the configured send function. +func (s *mailboxClientStub) Send( + ctx context.Context, + in *mailboxpb.SendRequest, + opts ...grpc.CallOption, +) (*mailboxpb.SendResponse, error) { + + if s.sendFn != nil { + return s.sendFn(ctx, in, opts...) + } + + return &mailboxpb.SendResponse{ + Status: &mailboxpb.Status{Ok: true}, + }, nil +} + +// Pull executes the configured pull function. +func (s *mailboxClientStub) Pull( + ctx context.Context, + in *mailboxpb.PullRequest, + opts ...grpc.CallOption, +) (*mailboxpb.PullResponse, error) { + + if s.pullFn != nil { + return s.pullFn(ctx, in, opts...) + } + + return &mailboxpb.PullResponse{ + Status: &mailboxpb.Status{Ok: true}, + }, nil +} + +// AckUpTo executes the configured ack function. +func (s *mailboxClientStub) AckUpTo( + ctx context.Context, + in *mailboxpb.AckUpToRequest, + opts ...grpc.CallOption, +) (*mailboxpb.AckUpToResponse, error) { + + if s.ackFn != nil { + return s.ackFn(ctx, in, opts...) + } + + return &mailboxpb.AckUpToResponse{ + Status: &mailboxpb.Status{Ok: true}, + }, nil +} + +// checkpointLoadStore allows overriding LoadCheckpoint behavior for tests. +type checkpointLoadStore struct { + *memCheckpointStore + + loadErr error + loadCheckpoint *actor.Checkpoint +} + +// LoadCheckpoint returns an injected error/checkpoint when configured. +func (s *checkpointLoadStore) LoadCheckpoint( + ctx context.Context, actorID string, +) (*actor.Checkpoint, error) { + + if s.loadErr != nil { + return nil, s.loadErr + } + if s.loadCheckpoint != nil { + return s.loadCheckpoint, nil + } + + return s.memCheckpointStore.LoadCheckpoint(ctx, actorID) +} + +// checkpointSaveStore allows overriding SaveCheckpoint behavior for tests. +type checkpointSaveStore struct { + *memCheckpointStore + + saveErr error +} + +// SaveCheckpoint returns an injected error when configured. +func (s *checkpointSaveStore) SaveCheckpoint( + ctx context.Context, params actor.CheckpointParams, +) error { + + if s.saveErr != nil { + return s.saveErr + } + + return s.memCheckpointStore.SaveCheckpoint(ctx, params) +} + +// newErrorPathActor builds a connector actor with defaults and test overrides. +func newErrorPathActor( + edge mailboxpb.MailboxServiceClient, + store actor.DeliveryStore, +) *ServerConnectionActor { + + cfg := DefaultConnectorConfig() + cfg.Edge = edge + cfg.Store = store + cfg.LocalMailboxID = "client-1" + cfg.RemoteMailboxID = "server-1" + cfg.ProtocolVersion = 1 + + return NewServerConnectionActor(cfg) +} + +// TestPullBatch_StatusFailure verifies pullBatch wraps non-OK status responses. +func TestPullBatch_StatusFailure(t *testing.T) { + t.Parallel() + + edge := &mailboxClientStub{ + pullFn: func( + ctx context.Context, + in *mailboxpb.PullRequest, + opts ...grpc.CallOption, + ) (*mailboxpb.PullResponse, error) { + + return &mailboxpb.PullResponse{ + Status: &mailboxpb.Status{ + Ok: false, + Code: "TEMPORARY", + Message: "pull failed", + }, + }, nil + }, + } + + actor := newErrorPathActor(edge, newMemCheckpointStore()) + + _, _, err := actor.pullBatch(t.Context(), 0) + require.Error(t, err) + + var stErr *statusError + require.ErrorAs(t, err, &stErr) + require.Equal(t, "Pull", stErr.Op) + require.Contains(t, stErr.Error(), "TEMPORARY") +} + +// TestAckRemote_StatusFailure verifies ackRemote wraps non-OK status responses. +func TestAckRemote_StatusFailure(t *testing.T) { + t.Parallel() + + edge := &mailboxClientStub{ + ackFn: func( + ctx context.Context, + in *mailboxpb.AckUpToRequest, + opts ...grpc.CallOption, + ) (*mailboxpb.AckUpToResponse, error) { + + return &mailboxpb.AckUpToResponse{ + Status: &mailboxpb.Status{ + Ok: false, + Code: "INTERNAL", + Message: "ack failed", + }, + }, nil + }, + } + + actor := newErrorPathActor(edge, newMemCheckpointStore()) + + err := actor.ackRemote(t.Context(), 1) + require.Error(t, err) + + var stErr *statusError + require.ErrorAs(t, err, &stErr) + require.Equal(t, "AckUpTo", stErr.Op) + require.Contains(t, stErr.Error(), "ack failed") +} + +// TestLoadCheckpoint_Errors verifies loadCheckpoint surfaces store/decode +// failures. +func TestLoadCheckpoint_Errors(t *testing.T) { + t.Parallel() + + edge := &mailboxClientStub{} + + loadErrActor := newErrorPathActor( + edge, &checkpointLoadStore{ + memCheckpointStore: newMemCheckpointStore(), + loadErr: fmt.Errorf("load failed"), + }, + ) + + _, err := loadErrActor.loadCheckpoint(t.Context()) + require.ErrorContains(t, err, "load failed") + + decodeErrActor := newErrorPathActor( + edge, &checkpointLoadStore{ + memCheckpointStore: newMemCheckpointStore(), + loadCheckpoint: &actor.Checkpoint{ + ActorID: "serverconn-client-1", + StateType: ackStateType, + StateData: []byte{0xff, 0x00, 0x01}, + }, + }, + ) + + _, err = decodeErrActor.loadCheckpoint(t.Context()) + require.Error(t, err) +} + +// TestSaveCheckpoint_Error verifies saveCheckpoint surfaces store save errors. +func TestSaveCheckpoint_Error(t *testing.T) { + t.Parallel() + + actor := newErrorPathActor( + &mailboxClientStub{}, + &checkpointSaveStore{ + memCheckpointStore: newMemCheckpointStore(), + saveErr: fmt.Errorf("save failed"), + }, + ) + + err := actor.saveCheckpoint(t.Context(), AckState{}) + require.ErrorContains(t, err, "save failed") +} + +// TestStatusError_ErrorString verifies statusError string formatting paths. +func TestStatusError_ErrorString(t *testing.T) { + t.Parallel() + + errNil := (&statusError{ + Op: "AckUpTo", + }).Error() + require.Contains(t, errNil, "nil status") + + errStatus := (&statusError{ + Op: "Pull", + Status: &mailboxpb.Status{ + Ok: false, + Code: "EIO", + Message: "io failure", + }, + }).Error() + require.Contains(t, errStatus, "io failure") + require.Contains(t, errStatus, "EIO") +} diff --git a/serverconn/restart_replay_test.go b/serverconn/restart_replay_test.go new file mode 100644 index 000000000..2a44bd1f5 --- /dev/null +++ b/serverconn/restart_replay_test.go @@ -0,0 +1,167 @@ +package serverconn + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/lightninglabs/darepo-client/baselib/actor" + mailboxpb "github.com/lightninglabs/darepo-client/mailbox/pb" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// failFirstSendEdge wraps the in-memory mailbox edge and fails the first Send +// call after recording the outbound identifiers. +type failFirstSendEdge struct { + *fakeMailboxServiceClient + + mu sync.Mutex + + sendAttempts int + firstMsgID string + firstIdemKey string +} + +// newFailFirstSendEdge creates an edge that fails its first Send call. +func newFailFirstSendEdge(mb *inMemoryMailbox) *failFirstSendEdge { + return &failFirstSendEdge{ + fakeMailboxServiceClient: &fakeMailboxServiceClient{mb: mb}, + } +} + +// Send records attempt metadata and fails once before succeeding thereafter. +func (e *failFirstSendEdge) Send( + ctx context.Context, + in *mailboxpb.SendRequest, + opts ...grpc.CallOption, +) (*mailboxpb.SendResponse, error) { + + e.mu.Lock() + defer e.mu.Unlock() + + if in == nil || in.Envelope == nil { + return nil, fmt.Errorf("nil send request") + } + + e.sendAttempts++ + if e.sendAttempts == 1 { + e.firstMsgID = in.Envelope.MsgId + e.firstIdemKey = in.Envelope.IdempotencyKey + + return nil, fmt.Errorf("injected first send failure") + } + + return e.fakeMailboxServiceClient.Send(ctx, in, opts...) +} + +// Snapshot returns the current edge attempt counters and first-attempt IDs. +func (e *failFirstSendEdge) Snapshot() (int, string, string) { + e.mu.Lock() + defer e.mu.Unlock() + + return e.sendAttempts, e.firstMsgID, e.firstIdemKey +} + +// newDurableConnectorForTest creates a DurableActor wrapper around +// ServerConnectionActor with an explicit Tell retry delay. +func newDurableConnectorForTest( + cfg ConnectorConfig, + retryDelay time.Duration, +) *actor.DurableActor[ServerConnMsg, ServerConnResp] { + + connector := NewServerConnectionActor(cfg) + + durableCfg := actor.DefaultDurableActorConfig[ + ServerConnMsg, ServerConnResp, + ]( + DurableActorID(cfg.LocalMailboxID), + connector, + cfg.Store, + cfg.Codec, + ) + + durableCfg.PollInterval = 10 * time.Millisecond + durableCfg.LeaseDuration = 500 * time.Millisecond + durableCfg.HeartbeatInterval = 100 * time.Millisecond + durableCfg.TellRetryPolicy = func(err error, attempts int) ( + bool, time.Duration) { + + if attempts >= 5 { + return false, 0 + } + + return true, retryDelay + } + + return actor.NewDurableActor(durableCfg) +} + +// TestEgress_RestartReplayPreservesStableIDs verifies that a failed egress send +// replayed after actor restart reuses the same MsgId and IdempotencyKey. +func TestEgress_RestartReplayPreservesStableIDs(t *testing.T) { + t.Parallel() + + mb := newInMemoryMailbox() + edge := newFailFirstSendEdge(mb) + store := newMemCheckpointStore() + + cfg := DefaultConnectorConfig() + cfg.Edge = edge + cfg.Store = store + cfg.Codec = NewServerConnCodec() + cfg.LocalMailboxID = "client-1" + cfg.RemoteMailboxID = "server-1" + cfg.ProtocolVersion = 1 + + // First actor instance fails once and nacks the message for retry. + durable1 := newDurableConnectorForTest(cfg, 500*time.Millisecond) + durable1.Start() + + err := durable1.TellRef().Tell(t.Context(), &SendClientEventRequest{ + Message: &testServerMessage{value: "restart-event"}, + }) + require.NoError(t, err) + + require.Eventually(t, func() bool { + attempts, _, _ := edge.Snapshot() + + return attempts >= 1 + }, 5*time.Second, 10*time.Millisecond) + + durable1.Stop() + + attempts, firstMsgID, firstIdemKey := edge.Snapshot() + require.Equal(t, 1, attempts) + require.NotEmpty(t, firstMsgID) + require.NotEmpty(t, firstIdemKey) + + mb.mu.Lock() + firstRunEnvs := append( + []*mailboxpb.Envelope(nil), mb.mailboxes["server-1"]..., + ) + mb.mu.Unlock() + require.Empty(t, firstRunEnvs) + + // Second actor instance replays from the same durable store and then + // succeeds. + durable2 := newDurableConnectorForTest(cfg, 20*time.Millisecond) + durable2.Start() + defer durable2.Stop() + + require.Eventually(t, func() bool { + mb.mu.Lock() + defer mb.mu.Unlock() + + return len(mb.mailboxes["server-1"]) == 1 + }, 8*time.Second, 10*time.Millisecond) + + mb.mu.Lock() + replayed := mb.mailboxes["server-1"][0] + mb.mu.Unlock() + + require.Equal(t, firstMsgID, replayed.MsgId) + require.Equal(t, firstIdemKey, replayed.IdempotencyKey) +} diff --git a/serverconn/types.go b/serverconn/types.go index d45abc9fc..5752aa2f0 100644 --- a/serverconn/types.go +++ b/serverconn/types.go @@ -19,9 +19,6 @@ type IdempotencyKey = mailboxconn.IdempotencyKey // AckState tracks connector ack watermark state for checkpoint persistence. type AckState = mailboxconn.AckState -// ResponseWaiter stores in-memory waiter state for unary response delivery. -type ResponseWaiter = mailboxconn.ResponseWaiter - // ackStateType is the checkpoint state type used for ack watermark storage. const ackStateType = mailboxconn.CheckpointStateType diff --git a/serverconn/unary_facade_test.go b/serverconn/unary_facade_test.go index a34f48e11..02c9bf9a1 100644 --- a/serverconn/unary_facade_test.go +++ b/serverconn/unary_facade_test.go @@ -3,6 +3,7 @@ package serverconn import ( "context" "fmt" + "math/rand" "sync" "testing" "time" @@ -94,7 +95,7 @@ func TestUnaryFacade_AwaitRPC(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() - actor.StartIngress(ctx) + require.NoError(t, actor.StartIngress(ctx)) defer actor.StopIngress() // Send an RPC request. @@ -140,7 +141,7 @@ func TestUnaryFacade_ResponseBeforeAwait(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() - actor.StartIngress(ctx) + require.NoError(t, actor.StartIngress(ctx)) defer actor.StopIngress() method := mailboxrpc.ServiceMethod{ @@ -193,7 +194,7 @@ func TestUnaryFacade_AwaitRPC_CancelledContext(t *testing.T) { ) defer ingressCancel() - actor.StartIngress(ingressCtx) + require.NoError(t, actor.StartIngress(ingressCtx)) defer actor.StopIngress() // Create a context that we cancel immediately. @@ -217,7 +218,7 @@ func TestUnaryFacade_ConcurrentInflight(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() - actor.StartIngress(ctx) + require.NoError(t, actor.StartIngress(ctx)) defer actor.StopIngress() const numRequests = 20 @@ -302,6 +303,95 @@ func TestUnaryFacade_ConcurrentInflight(t *testing.T) { } } +// TestUnaryFacade_HighConcurrencyOutOfOrder verifies that many in-flight unary +// requests can be resolved correctly when responses arrive out of order before +// AwaitRPC begins. +func TestUnaryFacade_HighConcurrencyOutOfOrder(t *testing.T) { + t.Parallel() + + actor, mb, _ := newTestConnector(t, nil) + facade := NewUnaryFacade(actor) + + ingressCtx, ingressCancel := context.WithCancel(t.Context()) + defer ingressCancel() + + require.NoError(t, actor.StartIngress(ingressCtx)) + defer actor.StopIngress() + + const numRequests = 200 + method := mailboxrpc.ServiceMethod{ + Service: "test.Svc", + Method: "Bulk", + } + + type requestPair struct { + correlationID string + expected string + } + + pairs := make([]requestPair, numRequests) + for i := 0; i < numRequests; i++ { + reqValue := fmt.Sprintf("bulk-req-%03d", i) + res, err := facade.SendRPC( + t.Context(), + method, + wrapperspb.String(reqValue), + mailboxrpc.RPCOptions{}, + ) + require.NoError(t, err) + + pairs[i] = requestPair{ + correlationID: res.CorrelationID, + expected: "bulk-resp-" + reqValue, + } + } + + order := rand.New(rand.NewSource(1337)).Perm(numRequests) + for _, idx := range order { + respBytes, err := proto.Marshal( + wrapperspb.String(pairs[idx].expected), + ) + require.NoError(t, err) + + sendResponseToMailbox( + t, mb, "client-1", pairs[idx].correlationID, respBytes, + ) + } + + var ( + wg sync.WaitGroup + errs = make([]error, numRequests) + outputs = make([]string, numRequests) + ) + + for i := 0; i < numRequests; i++ { + i := i + wg.Add(1) + + go func() { + defer wg.Done() + + awaitCtx, cancel := context.WithTimeout( + t.Context(), 15*time.Second, + ) + defer cancel() + + var resp wrapperspb.StringValue + errs[i] = facade.AwaitRPC( + awaitCtx, pairs[i].correlationID, &resp, + ) + outputs[i] = resp.Value + }() + } + + wg.Wait() + + for i := 0; i < numRequests; i++ { + require.NoError(t, errs[i], "await failed for request %d", i) + require.Equal(t, pairs[i].expected, outputs[i]) + } +} + // TestUnaryFacade_RPCClientInterface verifies the compile-time interface // compliance check is satisfied. func TestUnaryFacade_RPCClientInterface(t *testing.T) {