Skip to content

Commit 4cf29a0

Browse files
rpc, server: add DRPC support for node join RPC
Epic: CRDB-51459 Informs: CRDB-52248 Release note: None
1 parent b76a4de commit 4cf29a0

File tree

11 files changed

+231
-105
lines changed

11 files changed

+231
-105
lines changed

DEPS.bzl

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10890,10 +10890,10 @@ def go_deps():
1089010890
name = "io_storj_drpc",
1089110891
build_file_proto_mode = "disable_global",
1089210892
importpath = "storj.io/drpc",
10893-
sha256 = "98b44a51f82873f93f77da80230212ab40f35044e8d38645cb1392ae03462f0b",
10894-
strip_prefix = "github.com/cockroachdb/[email protected]20250924114114-78d4e121902a",
10893+
sha256 = "c3861ad4fc0e814d1a9648176f96f4f7fecc385a1a88b439c7205007aa2f36ae",
10894+
strip_prefix = "github.com/cthumuluru-crdb/[email protected]20251118112210-c686eec55198",
1089510895
urls = [
10896-
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/drpc/com_github_cockroachdb_drpc-v0.0.0-20250924114114-78d4e121902a.zip",
10896+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cthumuluru-crdb/drpc/com_github_cthumuluru_crdb_drpc-v0.0.0-20251118112210-c686eec55198.zip",
1089710897
],
1089810898
)
1089910899
go_repository(

build/bazelutil/distdir_files.bzl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,6 @@ DISTDIR_FILES = {
344344
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlfmt/com_github_cockroachdb_crlfmt-v0.0.0-20221214225007-b2fc5c302548.zip": "fedc01bdd6d964da0425d5eaac8efadc951e78e13f102292cc0774197f09ab63",
345345
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlib/com_github_cockroachdb_crlib-v0.0.0-20251024155502-a2e0a212ef05.zip": "a9a4b8810d52a2d18dddbbc359d060f5db854e328b4d75f12b7c5081ac25b2a2",
346346
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/datadriven/com_github_cockroachdb_datadriven-v1.0.3-0.20251006155849-f84f9e519edd.zip": "a7ffcef0b264d9c28c36b2f9b737ff739542f472d7614938ae507e2da269f6c2",
347-
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/drpc/com_github_cockroachdb_drpc-v0.0.0-20250924114114-78d4e121902a.zip": "98b44a51f82873f93f77da80230212ab40f35044e8d38645cb1392ae03462f0b",
348347
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/errors/com_github_cockroachdb_errors-v1.12.1-0.20251010171200-64801262cd6f.zip": "4df66cc44791d4290071696abf179dc6df7b94b4cb5d29a20f39c6bf522c60ee",
349348
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/go-test-teamcity/com_github_cockroachdb_go_test_teamcity-v0.0.0-20191211140407-cff980ad0a55.zip": "bac30148e525b79d004da84d16453ddd2d5cd20528e9187f1d7dac708335674b",
350349
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gogoproto/com_github_cockroachdb_gogoproto-v1.3.3-0.20241216150617-2358cdb156a1.zip": "bf052c9a7f9e23fb3ec7e9f3b7201cfc264c18ed6da0d662952d276dbc339003",
@@ -396,6 +395,7 @@ DISTDIR_FILES = {
396395
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cpuguy83/go-md2man/v2/com_github_cpuguy83_go_md2man_v2-v2.0.2.zip": "70a7e609809cf2a92c5535104db5eb82d75c54bfcfed2d224e87dd2fd9729f62",
397396
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/creack/pty/com_github_creack_pty-v1.1.11.zip": "d6594fd4844c242a5c7d6e9b25516182460cffa820e47e8ffb8eea625991986c",
398397
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/crossdock/crossdock-go/com_github_crossdock_crossdock_go-v0.0.0-20160816171116-049aabb0122b.zip": "f8a2ed6cd39e4f3e8108b8987f72bf6746276ada6fd3fcc62015bdbdd097f1a3",
398+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cthumuluru-crdb/drpc/com_github_cthumuluru_crdb_drpc-v0.0.0-20251118112210-c686eec55198.zip": "c3861ad4fc0e814d1a9648176f96f4f7fecc385a1a88b439c7205007aa2f36ae",
399399
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cyberdelia/templates/com_github_cyberdelia_templates-v0.0.0-20141128023046-ca7fffd4298c.zip": "a0ed6b8037d36222f63128f6064ed5b0e461fa9798c3592440a08875154d6c72",
400400
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cyphar/filepath-securejoin/com_github_cyphar_filepath_securejoin-v0.2.3.zip": "1e38690899f84b347ddc67cb8c6395812aea795e735b2208d680163278a3e3ba",
401401
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/d2g/dhcp4/com_github_d2g_dhcp4-v0.0.0-20170904100407-a1d1b6c41b1c.zip": "15df9468cf548a626e1319e92d550432512c4319cf555bf278ea9215de3504e3",

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -526,7 +526,7 @@ replace github.com/docker/docker => github.com/moby/moby v24.0.6+incompatible
526526

527527
replace github.com/gogo/protobuf => github.com/cockroachdb/gogoproto v1.3.3-0.20241216150617-2358cdb156a1
528528

529-
replace storj.io/drpc => github.com/cockroachdb/drpc v0.0.0-20250924114114-78d4e121902a
529+
replace storj.io/drpc => github.com/cthumuluru-crdb/drpc v0.0.0-20251118112210-c686eec55198
530530

531531
// Note: This forked dependency adds a commit that opens up some
532532
// private APIs to enable us to make some perf improvements to

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -556,8 +556,6 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z
556556
github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
557557
github.com/cockroachdb/datadriven v1.0.3-0.20251006155849-f84f9e519edd h1:vpWCe7VvdQbQ/9wGtlH3i+Oj+9OggKci3lsASL1ydvg=
558558
github.com/cockroachdb/datadriven v1.0.3-0.20251006155849-f84f9e519edd/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g=
559-
github.com/cockroachdb/drpc v0.0.0-20250924114114-78d4e121902a h1:zXCfk52Hpu2IoejmDm4Bkxmb5Nh9vxwaYOCiqA6f3YA=
560-
github.com/cockroachdb/drpc v0.0.0-20250924114114-78d4e121902a/go.mod h1:Ag2/Yfl22WZ8ywFUasRQ2brdltpX5QvY63jnYTZ3N5U=
561559
github.com/cockroachdb/errors v1.12.1-0.20251010171200-64801262cd6f h1:lUmJxzb2/ukuRIvKTaNkvuj5LwlX4u/KxnI3zmx1SSw=
562560
github.com/cockroachdb/errors v1.12.1-0.20251010171200-64801262cd6f/go.mod h1:SvzfYNNBshAVbZ8wzNc/UPK3w1vf0dKDUP41ucAIf7g=
563561
github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55 h1:YqzBA7tf8Gv8Oz0BbBsPenqkyjiohS7EUIwi7p1QJCU=
@@ -713,6 +711,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
713711
github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
714712
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
715713
github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b/go.mod h1:v9FBN7gdVTpiD/+LZ7Po0UKvROyT87uLVxTHVky/dlQ=
714+
github.com/cthumuluru-crdb/drpc v0.0.0-20251118112210-c686eec55198 h1:w0whIZa/Koi059vXP9NemhrxVoBK/KBYpLJXozaNHlc=
715+
github.com/cthumuluru-crdb/drpc v0.0.0-20251118112210-c686eec55198/go.mod h1:Ag2/Yfl22WZ8ywFUasRQ2brdltpX5QvY63jnYTZ3N5U=
716716
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
717717
github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
718718
github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=

pkg/rpc/BUILD.bazel

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,15 +81,13 @@ go_library(
8181
"@io_opentelemetry_go_otel//attribute",
8282
"@io_storj_drpc//:drpc",
8383
"@io_storj_drpc//drpcclient",
84-
"@io_storj_drpc//drpcconn",
8584
"@io_storj_drpc//drpcctx",
8685
"@io_storj_drpc//drpcmanager",
8786
"@io_storj_drpc//drpcmetadata",
8887
"@io_storj_drpc//drpcmigrate",
8988
"@io_storj_drpc//drpcmux",
9089
"@io_storj_drpc//drpcpool",
9190
"@io_storj_drpc//drpcserver",
92-
"@io_storj_drpc//drpcstream",
9391
"@io_storj_drpc//drpcwire",
9492
"@org_golang_google_grpc//:grpc",
9593
"@org_golang_google_grpc//backoff",

pkg/rpc/context.go

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ import (
5555
"google.golang.org/grpc/stats"
5656
"storj.io/drpc"
5757
"storj.io/drpc/drpcclient"
58+
"storj.io/drpc/drpcmigrate"
5859
)
5960

6061
// NewServer sets up an RPC server. Depending on the ServerOptions, the Server
@@ -1436,6 +1437,17 @@ func (rpcCtx *Context) GRPCDialOptions(
14361437
return rpcCtx.grpcDialOptionsInternal(ctx, target, class, transport, onNetworkDial)
14371438
}
14381439

1440+
// DRPCDialOptions is same as GRPCDialOptions but for drpc connections.
1441+
func (rpcCtx *Context) DRPCDialOptions(
1442+
ctx context.Context, target string, class rpcbase.ConnectionClass,
1443+
) ([]drpcclient.DialOption, error) {
1444+
transport := tcpTransport
1445+
if rpcCtx.ContextOptions.AdvertiseAddr == target && rpcCtx.canLoopbackDial() {
1446+
transport = loopbackTransport
1447+
}
1448+
return rpcCtx.drpcDialOptionsInternal(ctx, target, class, transport)
1449+
}
1450+
14391451
// grpcDialOptions produces dial options suitable for connecting to the given target and class.
14401452
func (rpcCtx *Context) grpcDialOptionsInternal(
14411453
ctx context.Context,
@@ -1469,6 +1481,39 @@ func (rpcCtx *Context) grpcDialOptionsInternal(
14691481
return dialOpts, nil
14701482
}
14711483

1484+
// drpcDialOptionsInternal is similar to grpcDialOptionsInternal but for
1485+
// drpc connections.
1486+
func (rpcCtx *Context) drpcDialOptionsInternal(
1487+
ctx context.Context,
1488+
target string,
1489+
class rpcbase.ConnectionClass,
1490+
transport transportType,
1491+
) ([]drpcclient.DialOption, error) {
1492+
drpcDialOpts, err := rpcCtx.drpcDialOptsCommon(ctx, target, class)
1493+
if err != nil {
1494+
return nil, err
1495+
}
1496+
1497+
switch transport {
1498+
case tcpTransport:
1499+
netOpts, err := rpcCtx.drpcDialOptsNetwork(ctx, target, class)
1500+
if err != nil {
1501+
return nil, err
1502+
}
1503+
drpcDialOpts = append(drpcDialOpts, netOpts...)
1504+
case loopbackTransport:
1505+
localOpts, err := rpcCtx.drpcDialOptsLocal()
1506+
if err != nil {
1507+
return nil, err
1508+
}
1509+
drpcDialOpts = append(drpcDialOpts, localOpts...)
1510+
default:
1511+
// This panic in case the type is ever changed to include more values.
1512+
panic(errors.AssertionFailedf("unhandled: %v", transport))
1513+
}
1514+
return drpcDialOpts, nil
1515+
}
1516+
14721517
// dialOptsLocal computes options used only for loopback connections.
14731518
func (rpcCtx *Context) dialOptsLocal() ([]grpc.DialOption, error) {
14741519
// We need to include a TLS overlay even for loopback connections,
@@ -1493,6 +1538,22 @@ func (rpcCtx *Context) dialOptsLocal() ([]grpc.DialOption, error) {
14931538
return dialOpts, err
14941539
}
14951540

1541+
// drpcDialOptsLocal is simialar to dialOptsLocal but for drpc connections.
1542+
func (rpcCtx *Context) drpcDialOptsLocal() ([]drpcclient.DialOption, error) {
1543+
drpcDialOpts, err := rpcCtx.drpcDialOptsNetworkCredentials()
1544+
if err != nil {
1545+
return nil, err
1546+
}
1547+
1548+
drpcDialOpts = append(drpcDialOpts, drpcclient.WithContextDialer(
1549+
func(ctx context.Context, target string) (net.Conn, error) {
1550+
return rpcCtx.loopbackDRPCDialFn(ctx)
1551+
},
1552+
))
1553+
1554+
return drpcDialOpts, err
1555+
}
1556+
14961557
// GetBreakerForAddr looks up a breaker for the matching (NodeID,Class,Addr).
14971558
// If it exists, it is unique.
14981559
//
@@ -1570,6 +1631,20 @@ func (rpcCtx *Context) dialOptsNetworkCredentials() ([]grpc.DialOption, error) {
15701631
return dialOpts, nil
15711632
}
15721633

1634+
// drpcDialOptsNetworkCredentials is same as dialOptsNetworkCredentials but for drpc connections.
1635+
func (rpcCtx *Context) drpcDialOptsNetworkCredentials() ([]drpcclient.DialOption, error) {
1636+
drpcDialOpts := []drpcclient.DialOption{}
1637+
if !rpcCtx.ContextOptions.Insecure {
1638+
tlsConfig, err := rpcCtx.GetClientTLSConfig()
1639+
if err != nil {
1640+
return nil, err
1641+
}
1642+
drpcDialOpts = append(drpcDialOpts, drpcclient.WithTLSConfig(tlsConfig))
1643+
}
1644+
1645+
return drpcDialOpts, nil
1646+
}
1647+
15731648
type statsTracker struct {
15741649
m localityMetrics
15751650
}
@@ -1717,6 +1792,38 @@ func (rpcCtx *Context) dialOptsNetwork(
17171792
return dialOpts, nil
17181793
}
17191794

1795+
// drpcDialOptsNetwork is same as dialOptsNetwork but for drpc connections.
1796+
func (rpcCtx *Context) drpcDialOptsNetwork(
1797+
ctx context.Context, target string, class rpcbase.ConnectionClass,
1798+
) ([]drpcclient.DialOption, error) {
1799+
// TODO(server): add compression support to drpc.
1800+
// TODO(server): add support for dial timeout.
1801+
// TODO(server): check if onlyOnceDialer is needed for drpc.
1802+
1803+
drpcDialOpts, err := rpcCtx.drpcDialOptsNetworkCredentials()
1804+
if err != nil {
1805+
return nil, err
1806+
}
1807+
1808+
dialerFunc := func(ctx context.Context, target string) (net.Conn, error) {
1809+
return drpcmigrate.DialWithHeader(ctx, "tcp", target, drpcmigrate.DRPCHeader)
1810+
}
1811+
if rpcCtx.Knobs.InjectedLatencyOracle != nil {
1812+
latency := rpcCtx.Knobs.InjectedLatencyOracle.GetLatency(target)
1813+
log.VEventf(ctx, 1, "connecting with simulated latency %dms",
1814+
latency)
1815+
dialer := artificialLatencyDialer{
1816+
dialerFunc: dialerFunc,
1817+
latency: latency,
1818+
enabled: rpcCtx.Knobs.InjectedLatencyEnabled,
1819+
}
1820+
dialerFunc = dialer.dial
1821+
}
1822+
drpcDialOpts = append(drpcDialOpts, drpcclient.WithContextDialer(dialerFunc))
1823+
1824+
return drpcDialOpts, nil
1825+
}
1826+
17201827
// dialOptsCommon computes options used for both in-memory and
17211828
// over-the-network RPC connections.
17221829
func (rpcCtx *Context) dialOptsCommon(
@@ -1766,6 +1873,37 @@ func (rpcCtx *Context) dialOptsCommon(
17661873
return dialOpts, nil
17671874
}
17681875

1876+
// drpcDialOptsCommon is same as dialOptsCommon but for drpc connections.
1877+
func (rpcCtx *Context) drpcDialOptsCommon(
1878+
ctx context.Context, target string, class rpcbase.ConnectionClass,
1879+
) ([]drpcclient.DialOption, error) {
1880+
drpcDialOpts := []drpcclient.DialOption{}
1881+
if !rpcCtx.TenantID.IsSystem() {
1882+
key, value := newPerRPCTIDMetdata(rpcCtx.TenantID)
1883+
drpcDialOpts = append(drpcDialOpts, drpcclient.WithPerRPCMetadata(map[string]string{key: value}))
1884+
}
1885+
1886+
unaryInterceptors := rpcCtx.clientUnaryInterceptorsDRPC
1887+
if rpcCtx.Knobs.UnaryClientInterceptorDRPC != nil {
1888+
interceptor := rpcCtx.Knobs.UnaryClientInterceptorDRPC(target, rpcbase.DefaultClass)
1889+
if interceptor != nil {
1890+
unaryInterceptors = append(unaryInterceptors, interceptor)
1891+
}
1892+
}
1893+
drpcDialOpts = append(drpcDialOpts, drpcclient.WithChainUnaryInterceptor(unaryInterceptors...))
1894+
1895+
streamInterceptors := rpcCtx.clientStreamInterceptorsDRPC
1896+
if rpcCtx.Knobs.StreamClientInterceptorDRPC != nil {
1897+
interceptor := rpcCtx.Knobs.StreamClientInterceptorDRPC(target, rpcbase.DefaultClass)
1898+
if interceptor != nil {
1899+
streamInterceptors = append(streamInterceptors, interceptor)
1900+
}
1901+
}
1902+
drpcDialOpts = append(drpcDialOpts, drpcclient.WithChainStreamInterceptor(streamInterceptors...))
1903+
1904+
return drpcDialOpts, nil
1905+
}
1906+
17691907
// ClientInterceptors returns the client interceptors that the Context uses on
17701908
// RPC calls. They are exposed so that RPC calls that bypass the Context (i.e.
17711909
// the ones done locally through the internalClientAdapater) can use the same
@@ -2069,6 +2207,32 @@ func (rpcCtx *Context) grpcDialRaw(
20692207
return grpc.DialContext(ctx, target, dialOpts...)
20702208
}
20712209

2210+
// drpcDialRaw is similar to grpcDialRaw but for drpc connections.
2211+
func (rpcCtx *Context) drpcDialRaw(
2212+
ctx context.Context,
2213+
target string,
2214+
class rpcbase.ConnectionClass,
2215+
additionalOpts ...drpcclient.DialOption,
2216+
) (*drpcclient.ClientConn, error) {
2217+
transport := tcpTransport
2218+
if rpcCtx.ContextOptions.AdvertiseAddr == target && rpcCtx.canLoopbackDial() {
2219+
transport = loopbackTransport
2220+
}
2221+
drpcDialOpts, err := rpcCtx.drpcDialOptionsInternal(ctx, target, class, transport)
2222+
if err != nil {
2223+
return nil, err
2224+
}
2225+
2226+
drpcDialOpts = append(drpcDialOpts, additionalOpts...)
2227+
2228+
drpcConn, err := drpcclient.DialContext(ctx, target, drpcDialOpts...)
2229+
if err != nil {
2230+
return nil, err
2231+
}
2232+
2233+
return drpcclient.NewClientConnWithOptions(ctx, drpcConn, drpcDialOpts...)
2234+
}
2235+
20722236
// GRPCUnvalidatedDial uses GRPCDialNode and disables validation of the
20732237
// node ID between client and server. This function should only be
20742238
// used with the gossip client and CLI commands which can talk to any

0 commit comments

Comments
 (0)