Skip to content

Commit 04d753d

Browse files
rpc, server: add DRPC support for node join RPC
Epic: CRDB-51459 Informs: CRDB-52248 Release note: None
1 parent 9d2b6d9 commit 04d753d

File tree

11 files changed

+231
-105
lines changed

11 files changed

+231
-105
lines changed

DEPS.bzl

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10890,10 +10890,10 @@ def go_deps():
1089010890
name = "io_storj_drpc",
1089110891
build_file_proto_mode = "disable_global",
1089210892
importpath = "storj.io/drpc",
10893-
sha256 = "98b44a51f82873f93f77da80230212ab40f35044e8d38645cb1392ae03462f0b",
10894-
strip_prefix = "github.com/cockroachdb/[email protected]20250924114114-78d4e121902a",
10893+
sha256 = "c3861ad4fc0e814d1a9648176f96f4f7fecc385a1a88b439c7205007aa2f36ae",
10894+
strip_prefix = "github.com/cthumuluru-crdb/[email protected]20251118112210-c686eec55198",
1089510895
urls = [
10896-
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/drpc/com_github_cockroachdb_drpc-v0.0.0-20250924114114-78d4e121902a.zip",
10896+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cthumuluru-crdb/drpc/com_github_cthumuluru_crdb_drpc-v0.0.0-20251118112210-c686eec55198.zip",
1089710897
],
1089810898
)
1089910899
go_repository(

build/bazelutil/distdir_files.bzl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,6 @@ DISTDIR_FILES = {
344344
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlfmt/com_github_cockroachdb_crlfmt-v0.0.0-20221214225007-b2fc5c302548.zip": "fedc01bdd6d964da0425d5eaac8efadc951e78e13f102292cc0774197f09ab63",
345345
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlib/com_github_cockroachdb_crlib-v0.0.0-20251024155502-a2e0a212ef05.zip": "a9a4b8810d52a2d18dddbbc359d060f5db854e328b4d75f12b7c5081ac25b2a2",
346346
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/datadriven/com_github_cockroachdb_datadriven-v1.0.3-0.20251006155849-f84f9e519edd.zip": "a7ffcef0b264d9c28c36b2f9b737ff739542f472d7614938ae507e2da269f6c2",
347-
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/drpc/com_github_cockroachdb_drpc-v0.0.0-20250924114114-78d4e121902a.zip": "98b44a51f82873f93f77da80230212ab40f35044e8d38645cb1392ae03462f0b",
348347
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/errors/com_github_cockroachdb_errors-v1.12.1-0.20251010171200-64801262cd6f.zip": "4df66cc44791d4290071696abf179dc6df7b94b4cb5d29a20f39c6bf522c60ee",
349348
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/go-test-teamcity/com_github_cockroachdb_go_test_teamcity-v0.0.0-20191211140407-cff980ad0a55.zip": "bac30148e525b79d004da84d16453ddd2d5cd20528e9187f1d7dac708335674b",
350349
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gogoproto/com_github_cockroachdb_gogoproto-v1.3.3-0.20241216150617-2358cdb156a1.zip": "bf052c9a7f9e23fb3ec7e9f3b7201cfc264c18ed6da0d662952d276dbc339003",
@@ -396,6 +395,7 @@ DISTDIR_FILES = {
396395
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cpuguy83/go-md2man/v2/com_github_cpuguy83_go_md2man_v2-v2.0.2.zip": "70a7e609809cf2a92c5535104db5eb82d75c54bfcfed2d224e87dd2fd9729f62",
397396
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/creack/pty/com_github_creack_pty-v1.1.11.zip": "d6594fd4844c242a5c7d6e9b25516182460cffa820e47e8ffb8eea625991986c",
398397
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/crossdock/crossdock-go/com_github_crossdock_crossdock_go-v0.0.0-20160816171116-049aabb0122b.zip": "f8a2ed6cd39e4f3e8108b8987f72bf6746276ada6fd3fcc62015bdbdd097f1a3",
398+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cthumuluru-crdb/drpc/com_github_cthumuluru_crdb_drpc-v0.0.0-20251118112210-c686eec55198.zip": "c3861ad4fc0e814d1a9648176f96f4f7fecc385a1a88b439c7205007aa2f36ae",
399399
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cyberdelia/templates/com_github_cyberdelia_templates-v0.0.0-20141128023046-ca7fffd4298c.zip": "a0ed6b8037d36222f63128f6064ed5b0e461fa9798c3592440a08875154d6c72",
400400
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cyphar/filepath-securejoin/com_github_cyphar_filepath_securejoin-v0.2.3.zip": "1e38690899f84b347ddc67cb8c6395812aea795e735b2208d680163278a3e3ba",
401401
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/d2g/dhcp4/com_github_d2g_dhcp4-v0.0.0-20170904100407-a1d1b6c41b1c.zip": "15df9468cf548a626e1319e92d550432512c4319cf555bf278ea9215de3504e3",

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -526,7 +526,7 @@ replace github.com/docker/docker => github.com/moby/moby v24.0.6+incompatible
526526

527527
replace github.com/gogo/protobuf => github.com/cockroachdb/gogoproto v1.3.3-0.20241216150617-2358cdb156a1
528528

529-
replace storj.io/drpc => github.com/cockroachdb/drpc v0.0.0-20250924114114-78d4e121902a
529+
replace storj.io/drpc => github.com/cthumuluru-crdb/drpc v0.0.0-20251118112210-c686eec55198
530530

531531
// Note: This forked dependency adds a commit that opens up some
532532
// private APIs to enable us to make some perf improvements to

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -556,8 +556,6 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z
556556
github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
557557
github.com/cockroachdb/datadriven v1.0.3-0.20251006155849-f84f9e519edd h1:vpWCe7VvdQbQ/9wGtlH3i+Oj+9OggKci3lsASL1ydvg=
558558
github.com/cockroachdb/datadriven v1.0.3-0.20251006155849-f84f9e519edd/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g=
559-
github.com/cockroachdb/drpc v0.0.0-20250924114114-78d4e121902a h1:zXCfk52Hpu2IoejmDm4Bkxmb5Nh9vxwaYOCiqA6f3YA=
560-
github.com/cockroachdb/drpc v0.0.0-20250924114114-78d4e121902a/go.mod h1:Ag2/Yfl22WZ8ywFUasRQ2brdltpX5QvY63jnYTZ3N5U=
561559
github.com/cockroachdb/errors v1.12.1-0.20251010171200-64801262cd6f h1:lUmJxzb2/ukuRIvKTaNkvuj5LwlX4u/KxnI3zmx1SSw=
562560
github.com/cockroachdb/errors v1.12.1-0.20251010171200-64801262cd6f/go.mod h1:SvzfYNNBshAVbZ8wzNc/UPK3w1vf0dKDUP41ucAIf7g=
563561
github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55 h1:YqzBA7tf8Gv8Oz0BbBsPenqkyjiohS7EUIwi7p1QJCU=
@@ -713,6 +711,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
713711
github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
714712
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
715713
github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b/go.mod h1:v9FBN7gdVTpiD/+LZ7Po0UKvROyT87uLVxTHVky/dlQ=
714+
github.com/cthumuluru-crdb/drpc v0.0.0-20251118112210-c686eec55198 h1:w0whIZa/Koi059vXP9NemhrxVoBK/KBYpLJXozaNHlc=
715+
github.com/cthumuluru-crdb/drpc v0.0.0-20251118112210-c686eec55198/go.mod h1:Ag2/Yfl22WZ8ywFUasRQ2brdltpX5QvY63jnYTZ3N5U=
716716
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
717717
github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
718718
github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=

pkg/rpc/BUILD.bazel

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,15 +81,13 @@ go_library(
8181
"@io_opentelemetry_go_otel//attribute",
8282
"@io_storj_drpc//:drpc",
8383
"@io_storj_drpc//drpcclient",
84-
"@io_storj_drpc//drpcconn",
8584
"@io_storj_drpc//drpcctx",
8685
"@io_storj_drpc//drpcmanager",
8786
"@io_storj_drpc//drpcmetadata",
8887
"@io_storj_drpc//drpcmigrate",
8988
"@io_storj_drpc//drpcmux",
9089
"@io_storj_drpc//drpcpool",
9190
"@io_storj_drpc//drpcserver",
92-
"@io_storj_drpc//drpcstream",
9391
"@io_storj_drpc//drpcwire",
9492
"@org_golang_google_grpc//:grpc",
9593
"@org_golang_google_grpc//backoff",

pkg/rpc/context.go

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ import (
5555
"google.golang.org/grpc/stats"
5656
"storj.io/drpc"
5757
"storj.io/drpc/drpcclient"
58+
"storj.io/drpc/drpcmigrate"
5859
)
5960

6061
// NewServer sets up an RPC server. Depending on the ServerOptions, the Server
@@ -1432,6 +1433,17 @@ func (rpcCtx *Context) GRPCDialOptions(
14321433
return rpcCtx.grpcDialOptionsInternal(ctx, target, class, transport, onNetworkDial)
14331434
}
14341435

1436+
// DRPCDialOptions is same as GRPCDialOptions but for drpc connections.
1437+
func (rpcCtx *Context) DRPCDialOptions(
1438+
ctx context.Context, target string, class rpcbase.ConnectionClass,
1439+
) ([]drpcclient.DialOption, error) {
1440+
transport := tcpTransport
1441+
if rpcCtx.ContextOptions.AdvertiseAddr == target && rpcCtx.canLoopbackDial() {
1442+
transport = loopbackTransport
1443+
}
1444+
return rpcCtx.drpcDialOptionsInternal(ctx, target, class, transport)
1445+
}
1446+
14351447
// grpcDialOptions produces dial options suitable for connecting to the given target and class.
14361448
func (rpcCtx *Context) grpcDialOptionsInternal(
14371449
ctx context.Context,
@@ -1465,6 +1477,39 @@ func (rpcCtx *Context) grpcDialOptionsInternal(
14651477
return dialOpts, nil
14661478
}
14671479

1480+
// drpcDialOptionsInternal is similar to grpcDialOptionsInternal but for
1481+
// drpc connections.
1482+
func (rpcCtx *Context) drpcDialOptionsInternal(
1483+
ctx context.Context,
1484+
target string,
1485+
class rpcbase.ConnectionClass,
1486+
transport transportType,
1487+
) ([]drpcclient.DialOption, error) {
1488+
drpcDialOpts, err := rpcCtx.drpcDialOptsCommon(ctx, target, class)
1489+
if err != nil {
1490+
return nil, err
1491+
}
1492+
1493+
switch transport {
1494+
case tcpTransport:
1495+
netOpts, err := rpcCtx.drpcDialOptsNetwork(ctx, target, class)
1496+
if err != nil {
1497+
return nil, err
1498+
}
1499+
drpcDialOpts = append(drpcDialOpts, netOpts...)
1500+
case loopbackTransport:
1501+
localOpts, err := rpcCtx.drpcDialOptsLocal()
1502+
if err != nil {
1503+
return nil, err
1504+
}
1505+
drpcDialOpts = append(drpcDialOpts, localOpts...)
1506+
default:
1507+
// This panic in case the type is ever changed to include more values.
1508+
panic(errors.AssertionFailedf("unhandled: %v", transport))
1509+
}
1510+
return drpcDialOpts, nil
1511+
}
1512+
14681513
// dialOptsLocal computes options used only for loopback connections.
14691514
func (rpcCtx *Context) dialOptsLocal() ([]grpc.DialOption, error) {
14701515
// We need to include a TLS overlay even for loopback connections,
@@ -1489,6 +1534,22 @@ func (rpcCtx *Context) dialOptsLocal() ([]grpc.DialOption, error) {
14891534
return dialOpts, err
14901535
}
14911536

1537+
// drpcDialOptsLocal is simialar to dialOptsLocal but for drpc connections.
1538+
func (rpcCtx *Context) drpcDialOptsLocal() ([]drpcclient.DialOption, error) {
1539+
drpcDialOpts, err := rpcCtx.drpcDialOptsNetworkCredentials()
1540+
if err != nil {
1541+
return nil, err
1542+
}
1543+
1544+
drpcDialOpts = append(drpcDialOpts, drpcclient.WithContextDialer(
1545+
func(ctx context.Context, target string) (net.Conn, error) {
1546+
return rpcCtx.loopbackDRPCDialFn(ctx)
1547+
},
1548+
))
1549+
1550+
return drpcDialOpts, err
1551+
}
1552+
14921553
// GetBreakerForAddr looks up a breaker for the matching (NodeID,Class,Addr).
14931554
// If it exists, it is unique.
14941555
//
@@ -1566,6 +1627,20 @@ func (rpcCtx *Context) dialOptsNetworkCredentials() ([]grpc.DialOption, error) {
15661627
return dialOpts, nil
15671628
}
15681629

1630+
// drpcDialOptsNetworkCredentials is same as dialOptsNetworkCredentials but for drpc connections.
1631+
func (rpcCtx *Context) drpcDialOptsNetworkCredentials() ([]drpcclient.DialOption, error) {
1632+
drpcDialOpts := []drpcclient.DialOption{}
1633+
if !rpcCtx.ContextOptions.Insecure {
1634+
tlsConfig, err := rpcCtx.GetClientTLSConfig()
1635+
if err != nil {
1636+
return nil, err
1637+
}
1638+
drpcDialOpts = append(drpcDialOpts, drpcclient.WithTLSConfig(tlsConfig))
1639+
}
1640+
1641+
return drpcDialOpts, nil
1642+
}
1643+
15691644
type statsTracker struct {
15701645
m localityMetrics
15711646
}
@@ -1713,6 +1788,38 @@ func (rpcCtx *Context) dialOptsNetwork(
17131788
return dialOpts, nil
17141789
}
17151790

1791+
// drpcDialOptsNetwork is same as dialOptsNetwork but for drpc connections.
1792+
func (rpcCtx *Context) drpcDialOptsNetwork(
1793+
ctx context.Context, target string, class rpcbase.ConnectionClass,
1794+
) ([]drpcclient.DialOption, error) {
1795+
// TODO(server): add compression support to drpc.
1796+
// TODO(server): add support for dial timeout.
1797+
// TODO(server): check if onlyOnceDialer is needed for drpc.
1798+
1799+
drpcDialOpts, err := rpcCtx.drpcDialOptsNetworkCredentials()
1800+
if err != nil {
1801+
return nil, err
1802+
}
1803+
1804+
dialerFunc := func(ctx context.Context, target string) (net.Conn, error) {
1805+
return drpcmigrate.DialWithHeader(ctx, "tcp", target, drpcmigrate.DRPCHeader)
1806+
}
1807+
if rpcCtx.Knobs.InjectedLatencyOracle != nil {
1808+
latency := rpcCtx.Knobs.InjectedLatencyOracle.GetLatency(target)
1809+
log.VEventf(ctx, 1, "connecting with simulated latency %dms",
1810+
latency)
1811+
dialer := artificialLatencyDialer{
1812+
dialerFunc: dialerFunc,
1813+
latency: latency,
1814+
enabled: rpcCtx.Knobs.InjectedLatencyEnabled,
1815+
}
1816+
dialerFunc = dialer.dial
1817+
}
1818+
drpcDialOpts = append(drpcDialOpts, drpcclient.WithContextDialer(dialerFunc))
1819+
1820+
return drpcDialOpts, nil
1821+
}
1822+
17161823
// dialOptsCommon computes options used for both in-memory and
17171824
// over-the-network RPC connections.
17181825
func (rpcCtx *Context) dialOptsCommon(
@@ -1762,6 +1869,37 @@ func (rpcCtx *Context) dialOptsCommon(
17621869
return dialOpts, nil
17631870
}
17641871

1872+
// drpcDialOptsCommon is same as dialOptsCommon but for drpc connections.
1873+
func (rpcCtx *Context) drpcDialOptsCommon(
1874+
ctx context.Context, target string, class rpcbase.ConnectionClass,
1875+
) ([]drpcclient.DialOption, error) {
1876+
drpcDialOpts := []drpcclient.DialOption{}
1877+
if !rpcCtx.TenantID.IsSystem() {
1878+
key, value := newPerRPCTIDMetdata(rpcCtx.TenantID)
1879+
drpcDialOpts = append(drpcDialOpts, drpcclient.WithPerRPCMetadata(map[string]string{key: value}))
1880+
}
1881+
1882+
unaryInterceptors := rpcCtx.clientUnaryInterceptorsDRPC
1883+
if rpcCtx.Knobs.UnaryClientInterceptorDRPC != nil {
1884+
interceptor := rpcCtx.Knobs.UnaryClientInterceptorDRPC(target, rpcbase.DefaultClass)
1885+
if interceptor != nil {
1886+
unaryInterceptors = append(unaryInterceptors, interceptor)
1887+
}
1888+
}
1889+
drpcDialOpts = append(drpcDialOpts, drpcclient.WithChainUnaryInterceptor(unaryInterceptors...))
1890+
1891+
streamInterceptors := rpcCtx.clientStreamInterceptorsDRPC
1892+
if rpcCtx.Knobs.StreamClientInterceptorDRPC != nil {
1893+
interceptor := rpcCtx.Knobs.StreamClientInterceptorDRPC(target, rpcbase.DefaultClass)
1894+
if interceptor != nil {
1895+
streamInterceptors = append(streamInterceptors, interceptor)
1896+
}
1897+
}
1898+
drpcDialOpts = append(drpcDialOpts, drpcclient.WithChainStreamInterceptor(streamInterceptors...))
1899+
1900+
return drpcDialOpts, nil
1901+
}
1902+
17651903
// ClientInterceptors returns the client interceptors that the Context uses on
17661904
// RPC calls. They are exposed so that RPC calls that bypass the Context (i.e.
17671905
// the ones done locally through the internalClientAdapater) can use the same
@@ -2065,6 +2203,32 @@ func (rpcCtx *Context) grpcDialRaw(
20652203
return grpc.DialContext(ctx, target, dialOpts...)
20662204
}
20672205

2206+
// drpcDialRaw is similar to grpcDialRaw but for drpc connections.
2207+
func (rpcCtx *Context) drpcDialRaw(
2208+
ctx context.Context,
2209+
target string,
2210+
class rpcbase.ConnectionClass,
2211+
additionalOpts ...drpcclient.DialOption,
2212+
) (*drpcclient.ClientConn, error) {
2213+
transport := tcpTransport
2214+
if rpcCtx.ContextOptions.AdvertiseAddr == target && rpcCtx.canLoopbackDial() {
2215+
transport = loopbackTransport
2216+
}
2217+
drpcDialOpts, err := rpcCtx.drpcDialOptionsInternal(ctx, target, class, transport)
2218+
if err != nil {
2219+
return nil, err
2220+
}
2221+
2222+
drpcDialOpts = append(drpcDialOpts, additionalOpts...)
2223+
2224+
drpcConn, err := drpcclient.DialContext(ctx, target, drpcDialOpts...)
2225+
if err != nil {
2226+
return nil, err
2227+
}
2228+
2229+
return drpcclient.NewClientConnWithOptions(ctx, drpcConn, drpcDialOpts...)
2230+
}
2231+
20682232
// GRPCUnvalidatedDial uses GRPCDialNode and disables validation of the
20692233
// node ID between client and server. This function should only be
20702234
// used with the gossip client and CLI commands which can talk to any

0 commit comments

Comments
 (0)