Add TELEPORT_UNSTABLE_GRPC_RECV_SIZE env var option

api/client/client.go

@@ -112,6 +112,7 @@ import (
 	"github.com/gravitational/teleport/api/types/events"
 	"github.com/gravitational/teleport/api/types/wrappers"
 	"github.com/gravitational/teleport/api/utils"
+	grpcutils "github.com/gravitational/teleport/api/utils/grpc"
 	"github.com/gravitational/teleport/api/utils/grpc/interceptors"
 )
 
@@ -503,7 +504,6 @@ func (c *Client) dialGRPC(ctx context.Context, addr string) error {
 	if err != nil {
 		return trace.Wrap(err)
 	}
-
 	var dialOpts []grpc.DialOption
 	dialOpts = append(dialOpts, grpc.WithContextDialer(c.grpcDialer()))
 	dialOpts = append(dialOpts,
@@ -519,6 +519,9 @@ func (c *Client) dialGRPC(ctx context.Context, addr string) error {
 			interceptors.GRPCClientStreamErrorInterceptor,
 			breaker.StreamClientInterceptor(cb),
 		),
+		grpc.WithDefaultCallOptions(
+			grpc.MaxCallRecvMsgSize(grpcutils.MaxClientRecvMsgSize()),
+		),
 	)
 	// Only set transportCredentials if tlsConfig is set. This makes it possible
 	// to explicitly provide grpc.WithTransportCredentials(insecure.NewCredentials())

api/client/proxy/client.go

@@ -38,6 +38,7 @@ import (
 	"github.com/gravitational/teleport/api/defaults"
 	transportv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/transport/v1"
 	"github.com/gravitational/teleport/api/metadata"
+	grpcutils "github.com/gravitational/teleport/api/utils/grpc"
 	"github.com/gravitational/teleport/api/utils/grpc/interceptors"
 )
 
@@ -315,6 +316,9 @@ func newGRPCClient(ctx context.Context, cfg *ClientConfig) (_ *Client, err error
 					interceptors.GRPCClientStreamErrorInterceptor,
 				)...,
 			),
+			grpc.WithDefaultCallOptions(
+				grpc.MaxCallRecvMsgSize(grpcutils.MaxClientRecvMsgSize()),
+			),
 		}, cfg.DialOpts...)...,
 	)
 	if err != nil {

api/utils/grpc/size.go

@@ -0,0 +1,116 @@
+// Copyright 2025 Gravitational, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package grpc
+
+import (
+	"fmt"
+	"math"
+	"os"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+// defaultClientRecvSize is the grpc client default for received message sizes
+const defaultClientRecvSize = 4 * 1024 * 1024 // 4MB
+
+// parseBytes takes human represtation of bytes such as '24mb' and returns number of bytes as an integer
+//
+// Only support a subset of SI prefixes up to gi/gb.
+func parseBytes(s string) (int, error) {
+	const (
+		Byte = 1 << (iota * 10)
+		KiByte
+		MiByte
+		GiByte
+		IByte = 1
+		KByte = IByte * 1000
+		MByte = KByte * 1000
+		GByte = MByte * 1000
+	)
+
+	var bytesSizeTable = map[string]int{
+		"b":   Byte,
+		"kib": KiByte,
+		"kb":  KByte,
+		"mib": MiByte,
+		"mb":  MByte,
+		"gib": GiByte,
+		"gb":  GByte,
+		"":    Byte,
+		"ki":  KiByte,
+		"k":   KByte,
+		"mi":  MiByte,
+		"m":   MByte,
+		"gi":  GiByte,
+		"g":   GByte,
+	}
+
+	lastDigit := 0
+	for _, r := range s {
+		if !unicode.IsDigit(r) && r != '.' {
+			break
+		}
+		lastDigit++
+	}
+
+	num := s[:lastDigit]
+
+	var value float32
+	if f, err := strconv.ParseFloat(num, 32); err == nil {
+		value = float32(f)
+	} else {
+		return 0, err
+
+	}
+
+	extra := strings.ToLower(strings.TrimSpace(s[lastDigit:]))
+	if m, ok := bytesSizeTable[extra]; ok {
+		value *= float32(m)
+		if value >= math.MaxInt32 {
+			return 0, fmt.Errorf("too large: %v", s)
+		}
+		return int(value), nil
+	}
+
+	return 0, fmt.Errorf("unhandled size name: %v", extra)
+}
+
+// MaxClientRecvMsgSize returns maximum message size in bytes the client can receive.
+//
+// By default 4MB is returned, to overwrite this, set `TELEPORT_UNSTABLE_GRPC_RECV_SIZE` envriroment
+// variable. If the value cannot be parsed or exceeds int32 limits, the default value is returned.
+//
+// The result of this call can be passed directly into `grpc.MaxCallRecvMsgSize`, example:
+//
+//	conn, err := grpc.DialContext(ctx, target,
+//		grpc.WithDefaultCallOptions(
+//			grpc.MaxCallRecvMsgSize(grpcutils.MaxClientRecvMsgSize()),
+//		),
+//	)
+func MaxClientRecvMsgSize() int {
+
+	val := os.Getenv("TELEPORT_UNSTABLE_GRPC_RECV_SIZE")
+	if val == "" {
+		return defaultClientRecvSize
+	}
+
+	size, err := parseBytes(val)
+	if err != nil {
+		return defaultClientRecvSize
+	}
+
+	return size
+}

api/utils/grpc/size_test.go

@@ -0,0 +1,98 @@
+// Copyright 2025 Gravitational, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package grpc
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMaxClientRecvMsgSize(t *testing.T) {
+
+	testCases := []struct {
+		desc  string
+		size  string
+		bytes int
+	}{
+		{
+			desc:  "Decimal",
+			size:  "1234",
+			bytes: 1234,
+		},
+		{
+			desc:  "Unset",
+			size:  "",
+			bytes: defaultClientRecvSize,
+		},
+		{
+			desc:  "Unhandled units",
+			size:  "4TB",
+			bytes: defaultClientRecvSize,
+		},
+		{
+			desc:  "Too large",
+			size:  "20GB",
+			bytes: defaultClientRecvSize,
+		},
+		{
+			desc:  "Rubbish",
+			size:  "foobar",
+			bytes: defaultClientRecvSize,
+		},
+		{
+			desc:  "Human mib",
+			size:  "8mib",
+			bytes: 8 * 1024 * 1024,
+		},
+		{
+			desc:  "Human kib",
+			size:  "8kib",
+			bytes: 8 * 1024,
+		},
+		{
+			desc:  "Human mb",
+			size:  "8mb",
+			bytes: 8 * 1000 * 1000,
+		},
+		{
+			desc:  "Floats",
+			size:  "2.5kb",
+			bytes: 2500,
+		},
+		{
+			desc:  "Human kb",
+			size:  "8kb",
+			bytes: 8 * 1000,
+		},
+		{
+			desc:  "Human m",
+			size:  "8m",
+			bytes: 8 * 1000 * 1000,
+		},
+		{
+			desc:  "Human k",
+			size:  "8k",
+			bytes: 8 * 1000,
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.desc, func(t *testing.T) {
+			t.Setenv("TELEPORT_UNSTABLE_GRPC_RECV_SIZE", tt.size)
+			assert.Equal(t, tt.bytes, MaxClientRecvMsgSize())
+		})
+	}
+}