From 8d43431bb3efbb83e343259e14964bc772e65a58 Mon Sep 17 00:00:00 2001 From: Simon Mudd Date: Wed, 4 Jul 2018 15:36:55 +0200 Subject: [PATCH 1/3] Add S3 support for appliance: Cloudian HyperStore * Add logging when using s3 for: StartBackup / ListBackups / RemoveBackup * Added options to enable usage of an S3 appliance: Cloudian HyperStore: -s3_backup_aws_endpoint (port is required) -s3_backup_force_path_style=true/false By default the s3 client will try to connect to ..amazonaws.com. Adjusting the endpoint will allow this to be changed. Given the way the FQDN is configured the TLS certificate may not match the server's "base" (.) due to the leading so setting -s3_backup_force_path_style=true will force the s3 client to connect to . and then make a request using the full path within the http calls. Signed-off-by: Simon Mudd --- go/vt/mysqlctl/s3backupstorage/README.txt | 17 ++++++++ go/vt/mysqlctl/s3backupstorage/s3.go | 48 ++++++++++++++++++++++- 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 go/vt/mysqlctl/s3backupstorage/README.txt diff --git a/go/vt/mysqlctl/s3backupstorage/README.txt b/go/vt/mysqlctl/s3backupstorage/README.txt new file mode 100644 index 00000000000..52b58f89bfe --- /dev/null +++ b/go/vt/mysqlctl/s3backupstorage/README.txt @@ -0,0 +1,17 @@ +Recently added options to enable usage of an S3 appliance: Cloudian +HyperStore: + -s3_backup_aws_endpoint (port is required) + -s3_backup_force_path_style=true/false + -s3_backup_log_level can be one of: LogOff, LogDebug, LogDebugWithSigning, LogDebugWithHTTPBody, LogDebugWithRequestRetries, LogDebugWithRequestErrors. Default: LogOff + +By default the s3 client will try to connect to +..amazonaws.com. Adjusting the endpoint will allow this +to be changed. + +Given the way the FQDN is configured the TLS certificate may not match the +server's "base" (.) due to the leading +so setting -s3_backup_force_path_style=true will force the s3 client to +connect to . and then make a request using the full +path within the http calls. + +-s3backup_log_level enables more verbose logging of the S3 calls. diff --git a/go/vt/mysqlctl/s3backupstorage/s3.go b/go/vt/mysqlctl/s3backupstorage/s3.go index 03bdd41b72b..32fd5522de7 100644 --- a/go/vt/mysqlctl/s3backupstorage/s3.go +++ b/go/vt/mysqlctl/s3backupstorage/s3.go @@ -32,6 +32,8 @@ import ( "strings" "sync" + "vitess.io/vitess/go/vt/log" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" @@ -46,12 +48,21 @@ var ( // AWS API region region = flag.String("s3_backup_aws_region", "us-east-1", "AWS region to use") + // AWS endpoint, defaults to amazonaws.com but appliances may use a different location + endpoint = flag.String("s3_backup_aws_endpoint", "amazonaws.com", "endpoint of the S3 backend (region must be provided)") + // bucket is where the backups will go. bucket = flag.String("s3_backup_storage_bucket", "", "S3 bucket to use for backups") // root is a prefix added to all object names. root = flag.String("s3_backup_storage_root", "", "root prefix for all backup-related object names") + // forcePath is used to ensure that the certificate and path used match the endpoint + region + forcePath = flag.Bool("s3_backup_force_path_style", false, "force the s3 path style") + + // verboseLogging provides more verbose logging of AWS actions + requiredLogLevel = flag.String("s3_backup_log_level", "LogOff", "determine the S3 loglevel to use from LogOff, LogDebug, LogDebugWithSigning, LogDebugWithHTTPBody, LogDebugWithRequestRetries, LogDebugWithRequestErrors") + // sse is the server-side encryption algorithm used when storing this object in S3 sse = flag.String("s3_backup_server_side_encryption", "", "server-side encryption algorithm (e.g., AES256, aws:kms)") @@ -59,6 +70,10 @@ var ( delimiter = "/" ) +type logNameToLogLevel map[string]aws.LogLevelType + +var logNameMap logNameToLogLevel + // S3BackupHandle implements the backupstorage.BackupHandle interface. type S3BackupHandle struct { client *s3.S3 @@ -169,6 +184,7 @@ type S3BackupStorage struct { // ListBackups is part of the backupstorage.BackupStorage interface. func (bs *S3BackupStorage) ListBackups(ctx context.Context, dir string) ([]backupstorage.BackupHandle, error) { + log.Infof("ListBackups: [s3] dir: %v, bucket: %v", dir, bucket) c, err := bs.client() if err != nil { return nil, err @@ -217,6 +233,7 @@ func (bs *S3BackupStorage) ListBackups(ctx context.Context, dir string) ([]backu // StartBackup is part of the backupstorage.BackupStorage interface. func (bs *S3BackupStorage) StartBackup(ctx context.Context, dir, name string) (backupstorage.BackupHandle, error) { + log.Infof("StartBackup: [s3] dir: %v, name: %v, bucket: %v", dir, name, bucket) c, err := bs.client() if err != nil { return nil, err @@ -233,6 +250,8 @@ func (bs *S3BackupStorage) StartBackup(ctx context.Context, dir, name string) (b // RemoveBackup is part of the backupstorage.BackupStorage interface. func (bs *S3BackupStorage) RemoveBackup(ctx context.Context, dir, name string) error { + log.Infof("RemoveBackup: [s3] dir: %v, name: %v, bucket: %v", dir, name, bucket) + c, err := bs.client() if err != nil { return err @@ -293,11 +312,29 @@ func (bs *S3BackupStorage) Close() error { var _ backupstorage.BackupStorage = (*S3BackupStorage)(nil) +// getLogLevel converts the string loglevel to an aws.LogLevelType +func getLogLevel() *aws.LogLevelType { + l := new(aws.LogLevelType) + *l = aws.LogOff // default setting + if level, found := logNameMap[*requiredLogLevel]; found { + *l = level // adjust as required + } + return l +} + func (bs *S3BackupStorage) client() (*s3.S3, error) { bs.mu.Lock() defer bs.mu.Unlock() if bs._client == nil { - bs._client = s3.New(session.New(), &aws.Config{Region: aws.String(*region)}) + logLevel := getLogLevel() + + bs._client = s3.New(session.New(), + &aws.Config{ + LogLevel: logLevel, + Endpoint: aws.String(*endpoint), + Region: aws.String(*region), + S3ForcePathStyle: aws.Bool(*forcePath), + }) if len(*bucket) == 0 { return nil, fmt.Errorf("-s3_backup_storage_bucket required") @@ -321,4 +358,13 @@ func objName(parts ...string) *string { func init() { backupstorage.BackupStorageMap["s3"] = &S3BackupStorage{} + + logNameMap = logNameToLogLevel{ + "LogOff": aws.LogOff, + "LogDebug": aws.LogDebug, + "LogDebugWithSigning": aws.LogDebugWithSigning, + "LogDebugWithHTTPBody": aws.LogDebugWithHTTPBody, + "LogDebugWithRequestRetries": aws.LogDebugWithRequestRetries, + "LogDebugWithRequestErrors": aws.LogDebugWithRequestErrors, + } } From 68630f3f6ed6c39775b92055eb701777e90f40bd Mon Sep 17 00:00:00 2001 From: Scott Lanning Date: Tue, 18 Sep 2018 12:01:56 +0200 Subject: [PATCH 2/3] dereference bucket, which is a string pointer Signed-off-by: Scott Lanning --- go/vt/mysqlctl/s3backupstorage/s3.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/vt/mysqlctl/s3backupstorage/s3.go b/go/vt/mysqlctl/s3backupstorage/s3.go index 32fd5522de7..35e4c6b6548 100644 --- a/go/vt/mysqlctl/s3backupstorage/s3.go +++ b/go/vt/mysqlctl/s3backupstorage/s3.go @@ -184,7 +184,7 @@ type S3BackupStorage struct { // ListBackups is part of the backupstorage.BackupStorage interface. func (bs *S3BackupStorage) ListBackups(ctx context.Context, dir string) ([]backupstorage.BackupHandle, error) { - log.Infof("ListBackups: [s3] dir: %v, bucket: %v", dir, bucket) + log.Infof("ListBackups: [s3] dir: %v, bucket: %v", dir, *bucket) c, err := bs.client() if err != nil { return nil, err @@ -233,7 +233,7 @@ func (bs *S3BackupStorage) ListBackups(ctx context.Context, dir string) ([]backu // StartBackup is part of the backupstorage.BackupStorage interface. func (bs *S3BackupStorage) StartBackup(ctx context.Context, dir, name string) (backupstorage.BackupHandle, error) { - log.Infof("StartBackup: [s3] dir: %v, name: %v, bucket: %v", dir, name, bucket) + log.Infof("StartBackup: [s3] dir: %v, name: %v, bucket: %v", dir, name, *bucket) c, err := bs.client() if err != nil { return nil, err @@ -250,7 +250,7 @@ func (bs *S3BackupStorage) StartBackup(ctx context.Context, dir, name string) (b // RemoveBackup is part of the backupstorage.BackupStorage interface. func (bs *S3BackupStorage) RemoveBackup(ctx context.Context, dir, name string) error { - log.Infof("RemoveBackup: [s3] dir: %v, name: %v, bucket: %v", dir, name, bucket) + log.Infof("RemoveBackup: [s3] dir: %v, name: %v, bucket: %v", dir, name, *bucket) c, err := bs.client() if err != nil { From 5940b272d612baa9b5e4b8707b66edb03a5fc194 Mon Sep 17 00:00:00 2001 From: Scott Lanning Date: Wed, 19 Sep 2018 14:39:40 +0200 Subject: [PATCH 3/3] add -s3_backup_tls_skip_verify_cert flag In case for some reason the SSL certificate is invalid... this flag allows disabling the validity check. Signed-off-by: Scott Lanning --- go/vt/mysqlctl/s3backupstorage/s3.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/go/vt/mysqlctl/s3backupstorage/s3.go b/go/vt/mysqlctl/s3backupstorage/s3.go index 35e4c6b6548..f8947eef413 100644 --- a/go/vt/mysqlctl/s3backupstorage/s3.go +++ b/go/vt/mysqlctl/s3backupstorage/s3.go @@ -24,10 +24,12 @@ limitations under the License. package s3backupstorage import ( + "crypto/tls" "flag" "fmt" "io" "math" + "net/http" "sort" "strings" "sync" @@ -60,6 +62,8 @@ var ( // forcePath is used to ensure that the certificate and path used match the endpoint + region forcePath = flag.Bool("s3_backup_force_path_style", false, "force the s3 path style") + tlsSkipVerifyCert = flag.Bool("s3_backup_tls_skip_verify_cert", false, "skip the 'certificate is valid' check for SSL connections") + // verboseLogging provides more verbose logging of AWS actions requiredLogLevel = flag.String("s3_backup_log_level", "LogOff", "determine the S3 loglevel to use from LogOff, LogDebug, LogDebugWithSigning, LogDebugWithHTTPBody, LogDebugWithRequestRetries, LogDebugWithRequestErrors") @@ -328,8 +332,13 @@ func (bs *S3BackupStorage) client() (*s3.S3, error) { if bs._client == nil { logLevel := getLogLevel() + tlsClientConf := &tls.Config{InsecureSkipVerify: *tlsSkipVerifyCert} + httpTransport := &http.Transport{TLSClientConfig: tlsClientConf} + httpClient := &http.Client{Transport: httpTransport} + bs._client = s3.New(session.New(), &aws.Config{ + HTTPClient: httpClient, LogLevel: logLevel, Endpoint: aws.String(*endpoint), Region: aws.String(*region),