Retry "use of closed network connection" in s3 backup handle

go/vt/mysqlctl/s3backupstorage/retryer.go

@@ -0,0 +1,49 @@
+package s3backupstorage
+
+import (
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+// ClosedConnectionRetryer implements the aws request.Retryer interface
+// and is used to retry closed connection errors during MultipartUpload
+// operations.
+type ClosedConnectionRetryer struct {
+	awsRetryer request.Retryer
+}
+
+// RetryRules is part of the Retryer interface. It defers to the underlying
+// aws Retryer to compute backoff rules.
+func (retryer *ClosedConnectionRetryer) RetryRules(r *request.Request) time.Duration {
+	return retryer.awsRetryer.RetryRules(r)
+}
+
+// ShouldRetry is part of the Retryer interface. It retries on errors that occur
+// due to a closed network connection, and then falls back to the underlying aws
+// Retryer for checking additional retry conditions.
+func (retryer *ClosedConnectionRetryer) ShouldRetry(r *request.Request) bool {
+	if retryer.MaxRetries() == 0 {
+		return false
+	}
+
+	if r.Retryable != nil {
+		return *r.Retryable
+	}
+
+	if r.Error != nil {
+		if awsErr, ok := r.Error.(awserr.Error); ok {
+			return strings.Contains(awsErr.OrigErr().Error(), "use of closed network connection")
+		}
+	}
+
+	return retryer.awsRetryer.ShouldRetry(r)
+}
+
+// MaxRetries is part of the Retryer interface. It defers to the
+// underlying aws Retryer for the max number of retries.
+func (retryer *ClosedConnectionRetryer) MaxRetries() int {
+	return retryer.awsRetryer.MaxRetries()
+}

go/vt/mysqlctl/s3backupstorage/s3.go

@@ -37,6 +37,8 @@ import (
 	"vitess.io/vitess/go/vt/log"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
@@ -353,7 +355,11 @@ func (bs *S3BackupStorage) client() (*s3.S3, error) {
 		}
 
 		if *retryCount >= 0 {
-			awsConfig.WithMaxRetries(*retryCount)
+			awsConfig = *request.WithRetryer(&awsConfig, &ClosedConnectionRetryer{
+				awsRetryer: &client.DefaultRetryer{
+					NumMaxRetries: *retryCount,
+				},
+			})
 		}
 
 		bs._client = s3.New(session, &awsConfig)