From d8620da1a4a07639ce2e00455cd31d4a6fb80d33 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Wed, 21 Dec 2022 09:21:23 -0600 Subject: [PATCH 1/3] util/s3: enable more credential options --- util/s3/s3Helper.go | 92 +++++----------------------------------- util/s3/s3Helper_test.go | 34 +++------------ 2 files changed, 17 insertions(+), 109 deletions(-) diff --git a/util/s3/s3Helper.go b/util/s3/s3Helper.go index 43f017443a..2edfc6193d 100644 --- a/util/s3/s3Helper.go +++ b/util/s3/s3Helper.go @@ -28,13 +28,9 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - - "github.com/algorand/go-algorand/util" - "github.com/algorand/go-algorand/util/codecs" ) const ( @@ -45,9 +41,6 @@ const ( s3DefaultReleaseBucket = "algorand-releases" s3DefaultUploadBucket = "algorand-uploads" s3DefaultRegion = "us-east-1" - - downloadAction = "download" - uploadAction = "upload" ) // Helper encapsulates the s3 session state for interactive with our default S3 bucket with appropriate credentials @@ -84,20 +77,12 @@ func getS3Region() (region string) { // MakeS3SessionForUploadWithBucket upload to bucket func MakeS3SessionForUploadWithBucket(awsBucket string) (helper Helper, err error) { - creds, err := getCredentials(uploadAction, awsBucket) - if err != nil { - return - } - return makeS3Session(creds, awsBucket) + return makeS3Session(awsBucket) } // MakeS3SessionForDownloadWithBucket download from bucket func MakeS3SessionForDownloadWithBucket(awsBucket string) (helper Helper, err error) { - creds, err := getCredentials(downloadAction, awsBucket) - if err != nil { - return - } - return makeS3Session(creds, awsBucket) + return makeS3Session(awsBucket) } // UploadFileStream sends file as stream to s3 @@ -114,58 +99,6 @@ func (helper *Helper) UploadFileStream(targetFile string, reader io.Reader) erro return nil } -type s3Keys struct { - ID string - Secret string -} - -func getCredentials(action string, awsBucket string) (creds *credentials.Credentials, err error) { - awsID, awsKey := getAWSCredentials() - credentailsRequired := checkCredentialsRequired(action, awsBucket) - if !credentailsRequired && (awsID == "" || awsKey == "") { - return credentials.AnonymousCredentials, nil - } - err = validateS3Credentials(awsID, awsKey) - if err != nil { - return - } - creds = credentials.NewStaticCredentials(awsID, awsKey, "") - return - -} - -func loadS3KeysFromFile(keyFile string) (keys s3Keys, err error) { - err = codecs.LoadObjectFromFile(keyFile, &keys) - return -} - -func getAWSCredentials() (awsID string, awsKey string) { - awsID, _ = os.LookupEnv("AWS_ACCESS_KEY_ID") - awsKey, _ = os.LookupEnv("AWS_SECRET_ACCESS_KEY") - - // If not in environment, try to load from s3.json file in bin dir - if awsID == "" || awsKey == "" { - baseDir, err := util.ExeDir() - if err == nil { - keyFile := filepath.Join(baseDir, "s3.json") - keys, err := loadS3KeysFromFile(keyFile) - if err == nil { - awsID = keys.ID - awsKey = keys.Secret - } - } - } - return -} - -func validateS3Credentials(awsID string, awsKey string) (err error) { - if awsID == "" || awsKey == "" { - err = fmt.Errorf("AWS credentials must be specified in environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY") - return - } - return -} - func validateS3Bucket(awsBucket string) (err error) { if awsBucket == "" { err = fmt.Errorf("bucket name is empty") @@ -174,21 +107,18 @@ func validateS3Bucket(awsBucket string) (err error) { return } -func checkCredentialsRequired(action string, bucketName string) (required bool) { - required = true - if action == downloadAction && bucketName == s3DefaultReleaseBucket { - required = false - } - return -} - -func makeS3Session(credentials *credentials.Credentials, bucket string) (helper Helper, err error) { +func makeS3Session(bucket string) (helper Helper, err error) { err = validateS3Bucket(bucket) if err != nil { return } - sess, err := session.NewSession(&aws.Config{Region: aws.String(getS3Region()), - Credentials: credentials}) + sess, err := session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + Config: aws.Config{ + CredentialsChainVerboseErrors: aws.Bool(true), + Region: aws.String(getS3Region()), + }, + }) if err != nil { return } @@ -294,7 +224,7 @@ func (helper *Helper) GetPackageFilesVersion(channel string, pkgFiles string, sp func GetVersionFromName(name string) (version uint64, err error) { re := regexp.MustCompile(`_(\d*)\.(\d*)\.(\d*)`) submatchAll := re.FindAllStringSubmatch(name, -1) - if submatchAll == nil || len(submatchAll) == 0 || len(submatchAll[0]) != 4 { + if len(submatchAll) == 0 || len(submatchAll[0]) != 4 { err = errors.New("unable to parse version from filename " + name) return } diff --git a/util/s3/s3Helper_test.go b/util/s3/s3Helper_test.go index 6c46095015..9af4f5ff14 100644 --- a/util/s3/s3Helper_test.go +++ b/util/s3/s3Helper_test.go @@ -114,8 +114,6 @@ func TestMakeS3SessionForUploadWithBucket(t *testing.T) { const emptyBucket = "" type args struct { awsBucket string - awsID string - awsSecret string } tests := []struct { name string @@ -123,21 +121,12 @@ func TestMakeS3SessionForUploadWithBucket(t *testing.T) { wantHelper Helper wantErr bool }{ - {name: "test1", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: false}, - {name: "test2", args: args{awsBucket: emptyBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true}, - {name: "test3", args: args{awsBucket: bucket1, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: true}, - {name: "test4", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true}, - {name: "test5", args: args{awsBucket: bucket1, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true}, - // public upload bucket requires AWS credentials for uploads - {name: "test6", args: args{awsBucket: publicUploadBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: false}, - {name: "test7", args: args{awsBucket: publicUploadBucket, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true}, - {name: "test8", args: args{awsBucket: publicUploadBucket, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true}, - {name: "test9", args: args{awsBucket: publicUploadBucket, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: true}, + {name: "test1", args: args{awsBucket: bucket1}, wantHelper: Helper{bucket: bucket1}, wantErr: false}, + {name: "test2", args: args{awsBucket: emptyBucket}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true}, + {name: "test6", args: args{awsBucket: publicUploadBucket}, wantHelper: Helper{bucket: publicUploadBucket}, wantErr: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - os.Setenv("AWS_ACCESS_KEY_ID", tt.args.awsID) - os.Setenv("AWS_SECRET_ACCESS_KEY", tt.args.awsSecret) gotHelper, err := MakeS3SessionForUploadWithBucket(tt.args.awsBucket) if (err != nil) != tt.wantErr { t.Errorf("MakeS3SessionForUploadWithBucket() error = %v, wantErr %v", err, tt.wantErr) @@ -158,8 +147,6 @@ func TestMakeS3SessionForDownloadWithBucket(t *testing.T) { const emptyBucket = "" type args struct { awsBucket string - awsID string - awsSecret string } tests := []struct { name string @@ -167,21 +154,12 @@ func TestMakeS3SessionForDownloadWithBucket(t *testing.T) { wantHelper Helper wantErr bool }{ - {name: "test1", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: false}, - {name: "test2", args: args{awsBucket: emptyBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true}, - {name: "test3", args: args{awsBucket: bucket1, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: bucket1}, wantErr: true}, - {name: "test4", args: args{awsBucket: bucket1, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true}, - {name: "test5", args: args{awsBucket: bucket1, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: bucket1}, wantErr: true}, - // public release bucket does not require AWS credentials for downloads - {name: "test6", args: args{awsBucket: publicReleaseBucket, awsID: "AWS_ID", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false}, - {name: "test7", args: args{awsBucket: publicReleaseBucket, awsID: "", awsSecret: "AWS_SECRET"}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false}, - {name: "test8", args: args{awsBucket: publicReleaseBucket, awsID: "AWS_ID", awsSecret: ""}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false}, - {name: "test9", args: args{awsBucket: publicReleaseBucket, awsID: "", awsSecret: ""}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false}, + {name: "test1", args: args{awsBucket: bucket1}, wantHelper: Helper{bucket: bucket1}, wantErr: false}, + {name: "test2", args: args{awsBucket: emptyBucket}, wantHelper: Helper{bucket: emptyBucket}, wantErr: true}, + {name: "test6", args: args{awsBucket: publicReleaseBucket}, wantHelper: Helper{bucket: publicReleaseBucket}, wantErr: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - os.Setenv("AWS_ACCESS_KEY_ID", tt.args.awsID) - os.Setenv("AWS_SECRET_ACCESS_KEY", tt.args.awsSecret) gotHelper, err := MakeS3SessionForDownloadWithBucket(tt.args.awsBucket) if (err != nil) != tt.wantErr { t.Errorf("MakeS3SessionForDownloadWithBucket() error = %v, wantErr %v", err, tt.wantErr) From 5b9cb3b5dc71b04692fc2eb3923b3d0756883152 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Wed, 21 Dec 2022 11:33:10 -0600 Subject: [PATCH 2/3] util/s3: use AnonymousCredentials for algorand-releases --- util/s3/s3Helper.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/util/s3/s3Helper.go b/util/s3/s3Helper.go index 2edfc6193d..a04a768118 100644 --- a/util/s3/s3Helper.go +++ b/util/s3/s3Helper.go @@ -28,6 +28,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" @@ -112,13 +113,21 @@ func makeS3Session(bucket string) (helper Helper, err error) { if err != nil { return } + + awsConfig := &aws.Config{ + CredentialsChainVerboseErrors: aws.Bool(true), + Region: aws.String(getS3Region()), + } + + if bucket == s3DefaultReleaseBucket { + awsConfig.Credentials = credentials.AnonymousCredentials + } + sess, err := session.NewSessionWithOptions(session.Options{ SharedConfigState: session.SharedConfigEnable, - Config: aws.Config{ - CredentialsChainVerboseErrors: aws.Bool(true), - Region: aws.String(getS3Region()), - }, + Config: *awsConfig, }) + if err != nil { return } From 3c49eedeaeceab4265801aec0aa6d0960b063404 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Thu, 22 Dec 2022 12:21:17 -0600 Subject: [PATCH 3/3] s3/util: use AnonymousCredentials if none are found --- util/s3/s3Helper.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/util/s3/s3Helper.go b/util/s3/s3Helper.go index a04a768118..b88709ae6f 100644 --- a/util/s3/s3Helper.go +++ b/util/s3/s3Helper.go @@ -22,6 +22,7 @@ import ( "io" "os" "path/filepath" + "reflect" "regexp" "runtime" "strconv" @@ -119,6 +120,7 @@ func makeS3Session(bucket string) (helper Helper, err error) { Region: aws.String(getS3Region()), } + // s3DefaultReleaseBucket should be public, use AnonymousCredentials if bucket == s3DefaultReleaseBucket { awsConfig.Credentials = credentials.AnonymousCredentials } @@ -127,10 +129,19 @@ func makeS3Session(bucket string) (helper Helper, err error) { SharedConfigState: session.SharedConfigEnable, Config: *awsConfig, }) - if err != nil { return } + + // use AnonymousCredentials if none are found + if creds, err := sess.Config.Credentials.Get(); err != nil && !reflect.DeepEqual(creds, credentials.AnonymousCredentials) { + sess.Config.Credentials = credentials.AnonymousCredentials + } + + if reflect.DeepEqual(sess.Config.Credentials, credentials.AnonymousCredentials) { + fmt.Println("Using anonymous credentials") + } + helper = Helper{ session: sess, bucket: bucket,