diff --git a/api/v4/common_types.go b/api/v4/common_types.go index 968ecd8ed..14b973e8f 100644 --- a/api/v4/common_types.go +++ b/api/v4/common_types.go @@ -316,6 +316,10 @@ type VolumeSpec struct { // Region of the remote storage volume where apps reside. Used for aws, if provided. Not used for minio and azure. Region string `json:"region"` + + // Type of URL for S3 endpoints. Configure true for path sytle URL. False by default(Host style) + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + S3PathUrl bool `json:"s3PathUrl"` } // VolumeAndTypeSpec used to add any custom varaibles for volume implementation diff --git a/config/crd/bases/enterprise.splunk.com_clustermanagers.yaml b/config/crd/bases/enterprise.splunk.com_clustermanagers.yaml index 911c7d7b9..3b9120759 100644 --- a/config/crd/bases/enterprise.splunk.com_clustermanagers.yaml +++ b/config/crd/bases/enterprise.splunk.com_clustermanagers.yaml @@ -1036,6 +1036,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -2129,6 +2133,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -4099,6 +4107,11 @@ spec: apps reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure + true for path sytle URL. False by default(Host style) + https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -4367,6 +4380,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string diff --git a/config/crd/bases/enterprise.splunk.com_clustermasters.yaml b/config/crd/bases/enterprise.splunk.com_clustermasters.yaml index 05097a4fc..320c3551b 100644 --- a/config/crd/bases/enterprise.splunk.com_clustermasters.yaml +++ b/config/crd/bases/enterprise.splunk.com_clustermasters.yaml @@ -1032,6 +1032,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -2125,6 +2129,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -4095,6 +4103,11 @@ spec: apps reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure + true for path sytle URL. False by default(Host style) + https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -4360,6 +4373,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string diff --git a/config/crd/bases/enterprise.splunk.com_licensemanagers.yaml b/config/crd/bases/enterprise.splunk.com_licensemanagers.yaml index 6842fab02..241fcda42 100644 --- a/config/crd/bases/enterprise.splunk.com_licensemanagers.yaml +++ b/config/crd/bases/enterprise.splunk.com_licensemanagers.yaml @@ -1026,6 +1026,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -3973,6 +3977,11 @@ spec: apps reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure + true for path sytle URL. False by default(Host style) + https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string diff --git a/config/crd/bases/enterprise.splunk.com_licensemasters.yaml b/config/crd/bases/enterprise.splunk.com_licensemasters.yaml index 130bfc599..fbe3b592b 100644 --- a/config/crd/bases/enterprise.splunk.com_licensemasters.yaml +++ b/config/crd/bases/enterprise.splunk.com_licensemasters.yaml @@ -1021,6 +1021,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -3968,6 +3972,11 @@ spec: apps reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure + true for path sytle URL. False by default(Host style) + https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string diff --git a/config/crd/bases/enterprise.splunk.com_monitoringconsoles.yaml b/config/crd/bases/enterprise.splunk.com_monitoringconsoles.yaml index c2b7c5179..d19e55bc6 100644 --- a/config/crd/bases/enterprise.splunk.com_monitoringconsoles.yaml +++ b/config/crd/bases/enterprise.splunk.com_monitoringconsoles.yaml @@ -1028,6 +1028,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -3974,6 +3978,11 @@ spec: apps reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure + true for path sytle URL. False by default(Host style) + https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -5154,6 +5163,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -8100,6 +8113,11 @@ spec: apps reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure + true for path sytle URL. False by default(Host style) + https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string diff --git a/config/crd/bases/enterprise.splunk.com_searchheadclusters.yaml b/config/crd/bases/enterprise.splunk.com_searchheadclusters.yaml index c7ab2b769..3fa15c17c 100644 --- a/config/crd/bases/enterprise.splunk.com_searchheadclusters.yaml +++ b/config/crd/bases/enterprise.splunk.com_searchheadclusters.yaml @@ -1034,6 +1034,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -3997,6 +4001,11 @@ spec: apps reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure + true for path sytle URL. False by default(Host style) + https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -5247,6 +5256,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -8210,6 +8223,11 @@ spec: apps reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure + true for path sytle URL. False by default(Host style) + https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string diff --git a/config/crd/bases/enterprise.splunk.com_standalones.yaml b/config/crd/bases/enterprise.splunk.com_standalones.yaml index a0baf787b..bc0a6c18f 100644 --- a/config/crd/bases/enterprise.splunk.com_standalones.yaml +++ b/config/crd/bases/enterprise.splunk.com_standalones.yaml @@ -1029,6 +1029,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -2126,6 +2130,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -4097,6 +4105,11 @@ spec: apps reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure + true for path sytle URL. False by default(Host style) + https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -4359,6 +4372,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -5399,6 +5416,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -6496,6 +6517,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -8467,6 +8492,11 @@ spec: apps reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure + true for path sytle URL. False by default(Host style) + https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string @@ -8732,6 +8762,10 @@ spec: reside. Used for aws, if provided. Not used for minio and azure. type: string + s3PathUrl: + description: Type of URL for S3 endpoints. Configure true + for path sytle URL. False by default(Host style) https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + type: boolean secretRef: description: Secret object name type: string diff --git a/go.mod b/go.mod index 9c0c7485c..a4a19bcbe 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,19 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect + github.com/aws/aws-sdk-go-v2/config v1.27.11 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect + github.com/aws/smithy-go v1.20.2 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/buger/jsonparser v1.1.1 // indirect diff --git a/pkg/splunk/client/awss3client.go b/pkg/splunk/client/awss3client.go index 3d00746d3..956f612f6 100644 --- a/pkg/splunk/client/awss3client.go +++ b/pkg/splunk/client/awss3client.go @@ -32,7 +32,7 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "sigs.k8s.io/controller-runtime/pkg/log" + log "sigs.k8s.io/controller-runtime/pkg/log" ) // blank assignment to verify that AWSS3Client implements RemoteDataClient @@ -76,12 +76,12 @@ func GetRegion(ctx context.Context, endpoint string, region *string) error { } // InitAWSClientWrapper is a wrapper around InitClientSession -func InitAWSClientWrapper(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { - return InitAWSClientSession(ctx, region, accessKeyID, secretAccessKey) +func InitAWSClientWrapper(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { + return InitAWSClientSession(ctx, region, accessKeyID, secretAccessKey, pathStyleUrl) } // InitAWSClientSession initializes and returns a client session object -func InitAWSClientSession(ctx context.Context, regionWithEndpoint, accessKeyID, secretAccessKey string) SplunkAWSS3Client { +func InitAWSClientSession(ctx context.Context, regionWithEndpoint, accessKeyID, secretAccessKey string, pathStyleUrl bool) SplunkAWSS3Client { reqLogger := log.FromContext(ctx) scopedLog := reqLogger.WithName("InitAWSClientSession") @@ -112,11 +112,14 @@ func InitAWSClientSession(ctx context.Context, regionWithEndpoint, accessKeyID, endpoint = regEndSl[1] config := &aws.Config{ - Region: aws.String(region), - MaxRetries: aws.Int(3), - HTTPClient: &httpClient, + Region: aws.String(region), + MaxRetries: aws.Int(3), + HTTPClient: &httpClient, + S3ForcePathStyle: aws.Bool(pathStyleUrl), } + scopedLog.Info("Setting up AWS SDK client", "regionWithEndpoint", regionWithEndpoint, "pathStyleUrl", pathStyleUrl) + if accessKeyID != "" && secretAccessKey != "" { config.WithCredentials(credentials.NewStaticCredentials( accessKeyID, // id @@ -146,7 +149,7 @@ func InitAWSClientSession(ctx context.Context, regionWithEndpoint, accessKeyID, } // NewAWSS3Client returns an AWS S3 client -func NewAWSS3Client(ctx context.Context, bucketName string, accessKeyID string, secretAccessKey string, prefix string, startAfter string, region string, endpoint string, fn GetInitFunc) (RemoteDataClient, error) { +func NewAWSS3Client(ctx context.Context, bucketName string, accessKeyID string, secretAccessKey string, prefix string, startAfter string, region string, endpoint string, pathStyleUrl bool, fn GetInitFunc) (RemoteDataClient, error) { var s3SplunkClient SplunkAWSS3Client var err error @@ -160,8 +163,7 @@ func NewAWSS3Client(ctx context.Context, bucketName string, accessKeyID string, } endpointWithRegion := fmt.Sprintf("%s%s%s", region, awsRegionEndPointDelimiter, endpoint) - - cl := fn(ctx, endpointWithRegion, accessKeyID, secretAccessKey) + cl := fn(ctx, endpointWithRegion, accessKeyID, secretAccessKey, pathStyleUrl) if cl == nil { err = fmt.Errorf("failed to create an AWS S3 client") return nil, err @@ -171,6 +173,7 @@ func NewAWSS3Client(ctx context.Context, bucketName string, accessKeyID string, if !ok { return nil, fmt.Errorf("unable to get s3 client") } + downloader := s3manager.NewDownloaderWithClient(cl.(*s3.S3)) return &AWSS3Client{ diff --git a/pkg/splunk/client/awss3client_test.go b/pkg/splunk/client/awss3client_test.go index a6af8efa2..901c1ed78 100644 --- a/pkg/splunk/client/awss3client_test.go +++ b/pkg/splunk/client/awss3client_test.go @@ -30,24 +30,24 @@ import ( func TestInitAWSClientWrapper(t *testing.T) { ctx := context.TODO() - awsS3ClientSession := InitAWSClientWrapper(ctx, "us-west-2|https://s3.amazon.com", "abcd", "1234") + awsS3ClientSession := InitAWSClientWrapper(ctx, "us-west-2|https://s3.amazon.com", "abcd", "1234", false) if awsS3ClientSession == nil { t.Errorf("We should have got a valid AWS S3 client session object") } - awsS3ClientSession = InitAWSClientWrapper(ctx, "us-west-2|https://s3.amazon.com", "", "") + awsS3ClientSession = InitAWSClientWrapper(ctx, "us-west-2|https://s3.amazon.com", "", "", false) if awsS3ClientSession == nil { t.Errorf("Case: Invalid secret/access keys, still returns a session") } - awsS3ClientSession = InitAWSClientWrapper(ctx, "us-west-2", "", "") + awsS3ClientSession = InitAWSClientWrapper(ctx, "us-west-2", "", "", false) if awsS3ClientSession != nil { t.Errorf("Endpoint not resolved, should receive a nil session") } // Invalid session test os.Setenv("AWS_STS_REGIONAL_ENDPOINTS", "abcde") - awsS3ClientSession = InitAWSClientWrapper(ctx, "us-west-2|https://s3.amazon.com", "abcd", "1234") + awsS3ClientSession = InitAWSClientWrapper(ctx, "us-west-2|https://s3.amazon.com", "abcd", "1234", false) os.Unsetenv("AWS_STS_REGIONAL_ENDPOINTS") } @@ -72,37 +72,37 @@ func TestGetTLSVersion(t *testing.T) { func TestNewAWSS3Client(t *testing.T) { ctx := context.TODO() fn := InitAWSClientWrapper - awsS3Client, err := NewAWSS3Client(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "us-west-2", "https://s3.us-west-2.amazonaws.com", fn) + awsS3Client, err := NewAWSS3Client(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "us-west-2", "https://s3.us-west-2.amazonaws.com", false, fn) if awsS3Client == nil || err != nil { t.Errorf("NewAWSS3Client should have returned a valid AWS S3 client.") } // just test the backward compatibility where we do not pass a region explicitly - awsS3Client, err = NewAWSS3Client(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "", "https://s3.us-west-2.amazonaws.com", fn) + awsS3Client, err = NewAWSS3Client(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "", "https://s3.us-west-2.amazonaws.com", false, fn) if awsS3Client == nil || err != nil { t.Errorf("NewAWSS3Client should have returned a valid AWS S3 client.") } // test the invalid scenario where we cannot extract region from endpoint - awsS3Client, err = NewAWSS3Client(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "", "https://s3.us-west-2.dummyprovider.com", fn) + awsS3Client, err = NewAWSS3Client(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "", "https://s3.us-west-2.dummyprovider.com", false, fn) if awsS3Client != nil || err == nil { t.Errorf("NewAWSS3Client should have returned a valid AWS S3 client.") } // Test for invalid scenario, where we return nil client - fn = func(context.Context, string, string, string) interface{} { + fn = func(context.Context, string, string, string, bool) interface{} { return nil } - _, err = NewAWSS3Client(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "us-west-2", "https://s3.us-west-2.amazonaws.com", fn) + _, err = NewAWSS3Client(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "us-west-2", "https://s3.us-west-2.amazonaws.com", false, fn) if err == nil { t.Errorf("NewAWSS3Client should have returned error.") } // Test for invalid scenario, where we return invalid client - fn = func(context.Context, string, string, string) interface{} { + fn = func(context.Context, string, string, string, bool) interface{} { return "abcd" } - _, err = NewAWSS3Client(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "us-west-2", "https://s3.us-west-2.amazonaws.com", fn) + _, err = NewAWSS3Client(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "us-west-2", "https://s3.us-west-2.amazonaws.com", false, fn) if err == nil { t.Errorf("NewAWSS3Client should have returned error.") } @@ -216,7 +216,7 @@ func TestAWSGetAppsListShouldNotFail(t *testing.T) { getClientWrapper := RemoteDataClientsMap[vol.Provider] getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAWSS3Client) - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockAWSS3Client{} cl.Objects = mockAwsObjects[index].Objects return cl @@ -225,7 +225,7 @@ func TestAWSGetAppsListShouldNotFail(t *testing.T) { getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - awsClient.Client = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(spltest.MockAWSS3Client) + awsClient.Client = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(spltest.MockAWSS3Client) RemoteDataListResponse, err := awsClient.GetAppsList(ctx) if err != nil { @@ -316,7 +316,7 @@ func TestAWSGetAppsListShouldFail(t *testing.T) { getClientWrapper := RemoteDataClientsMap[vol.Provider] getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAWSS3Client) - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockAWSS3Client{} // return empty objects list here to test the negative scenario return cl @@ -325,7 +325,7 @@ func TestAWSGetAppsListShouldFail(t *testing.T) { getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - awsClient.Client = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(spltest.MockAWSS3Client) + awsClient.Client = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(spltest.MockAWSS3Client) remoteDataClientResponse, err := awsClient.GetAppsList(ctx) if err != nil { @@ -337,7 +337,7 @@ func TestAWSGetAppsListShouldFail(t *testing.T) { // Update the GetRemoteDataClient with our mock call which initializes mock AWS client getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAWSS3Client) - initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockAWSS3ClientError{} // return empty objects list here to test the negative scenario return cl @@ -345,7 +345,7 @@ func TestAWSGetAppsListShouldFail(t *testing.T) { getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) getRemoteDataClientFn = getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - awsClient.Client = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(spltest.MockAWSS3ClientError) + awsClient.Client = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(spltest.MockAWSS3ClientError) remoteDataClientResponse, err = awsClient.GetAppsList(ctx) if err == nil { @@ -440,7 +440,7 @@ func TestAWSDownloadAppShouldNotFail(t *testing.T) { getClientWrapper := RemoteDataClientsMap[vol.Provider] getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAWSS3Client) - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockAWSS3Client{} return cl } @@ -449,7 +449,7 @@ func TestAWSDownloadAppShouldNotFail(t *testing.T) { getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - awsClient.Client = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(spltest.MockAWSS3Client) + awsClient.Client = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(spltest.MockAWSS3Client) downloadRequest := RemoteDataDownloadRequest{ LocalFile: LocalFiles[index], @@ -534,7 +534,7 @@ func TestAWSDownloadAppShouldFail(t *testing.T) { getClientWrapper := RemoteDataClientsMap[vol.Provider] getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAWSS3Client) - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockAWSS3Client{} return cl } @@ -543,7 +543,7 @@ func TestAWSDownloadAppShouldFail(t *testing.T) { getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - awsClient.Client = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(spltest.MockAWSS3Client) + awsClient.Client = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(spltest.MockAWSS3Client) downloadRequest := RemoteDataDownloadRequest{ LocalFile: LocalFile[0], diff --git a/pkg/splunk/client/azureblobclient.go b/pkg/splunk/client/azureblobclient.go index f308e607c..db124f217 100644 --- a/pkg/splunk/client/azureblobclient.go +++ b/pkg/splunk/client/azureblobclient.go @@ -198,9 +198,9 @@ func buildCanonicalizedResource(u *url.URL, accountName string) (string, error) } // NewAzureBlobClient returns an AzureBlob client -func NewAzureBlobClient(ctx context.Context, bucketName string, storageAccountName string, secretAccessKey string, prefix string, startAfter string, region string, endpoint string, fn GetInitFunc) (RemoteDataClient, error) { +func NewAzureBlobClient(ctx context.Context, bucketName string, storageAccountName string, secretAccessKey string, prefix string, startAfter string, region string, endpoint string, pathStyleUrl bool, fn GetInitFunc) (RemoteDataClient, error) { // Get http client - azureHTTPClient := fn(ctx, endpoint, storageAccountName, secretAccessKey) + azureHTTPClient := fn(ctx, endpoint, storageAccountName, secretAccessKey, false) return &AzureBlobClient{ BucketName: bucketName, @@ -214,7 +214,7 @@ func NewAzureBlobClient(ctx context.Context, bucketName string, storageAccountNa } // InitAzureBlobClientWrapper is a wrapper around InitAzureBlobClientSession -func InitAzureBlobClientWrapper(ctx context.Context, appAzureBlobEndPoint string, storageAccountName string, secretAccessKey string) interface{} { +func InitAzureBlobClientWrapper(ctx context.Context, appAzureBlobEndPoint string, storageAccountName string, secretAccessKey string, pathStyleUrl bool) interface{} { return InitAzureBlobClientSession(ctx) } diff --git a/pkg/splunk/client/azureblobclient_test.go b/pkg/splunk/client/azureblobclient_test.go index cd73f4160..186438aa2 100644 --- a/pkg/splunk/client/azureblobclient_test.go +++ b/pkg/splunk/client/azureblobclient_test.go @@ -42,7 +42,7 @@ func (errReader) Read(p []byte) (n int, err error) { func TestInitAzureBlobClientWrapper(t *testing.T) { ctx := context.TODO() - azureBlobClientSession := InitAzureBlobClientWrapper(ctx, "https://mystorageaccount.blob.core.windows.net", "abcd", "1234") + azureBlobClientSession := InitAzureBlobClientWrapper(ctx, "https://mystorageaccount.blob.core.windows.net", "abcd", "1234", false) if azureBlobClientSession == nil { t.Errorf("We should not have got a nil Azure Blob Client") } @@ -52,7 +52,7 @@ func TestNewAzureBlobClient(t *testing.T) { ctx := context.TODO() fn := InitAzureBlobClientWrapper - azureBlobClient, err := NewAzureBlobClient(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "us-west-2", "https://mystorageaccount.blob.core.windows.net", fn) + azureBlobClient, err := NewAzureBlobClient(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "us-west-2", "https://mystorageaccount.blob.core.windows.net", false, fn) if azureBlobClient == nil || err != nil { t.Errorf("NewAzureBlobClient should have returned a valid Azure Blob client.") } @@ -185,14 +185,14 @@ func TestAzureBlobGetAppsListShouldNotFail(t *testing.T) { getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAzureBlobClient) // Update the GetRemoteDataClientInit function pointer - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return &mclient } getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) // Init azure blob client getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(*spltest.MockHTTPClient) + azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(*spltest.MockHTTPClient) azureBlobClient.BucketName = vol.Path azureBlobClient.Prefix = appSource.Location azureBlobClient.Endpoint = vol.Endpoint @@ -338,14 +338,14 @@ func TestAzureBlobGetAppsListShouldFail(t *testing.T) { getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAzureBlobClient) // Update the GetRemoteDataClientInit function pointer - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return &mclient } getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) // Init azure blob client getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(*spltest.MockHTTPClient) + azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(*spltest.MockHTTPClient) azureBlobClient.BucketName = vol.Path azureBlobClient.Prefix = appSource.Location @@ -445,14 +445,14 @@ func TestAzureBlobDownloadAppShouldNotFail(t *testing.T) { getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAzureBlobClient) // Update the GetRemoteDataClientInit function pointer - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return &mclient } getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) // Init azure blob client getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(*spltest.MockHTTPClient) + azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(*spltest.MockHTTPClient) azureBlobClient.BucketName = vol.Path azureBlobClient.Prefix = appSource.Location azureBlobClient.Endpoint = vol.Endpoint @@ -556,14 +556,14 @@ func TestAzureBlobDownloadAppShouldFail(t *testing.T) { getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAzureBlobClient) // Update the GetRemoteDataClientInit function pointer - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return &mclient } getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) // Init azure blob client getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(*spltest.MockHTTPClient) + azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(*spltest.MockHTTPClient) azureBlobClient.BucketName = vol.Path azureBlobClient.Prefix = appSource.Location azureBlobClient.Endpoint = vol.Endpoint @@ -652,14 +652,14 @@ func TestAzureBlobGetAppsListShouldFailBadSecret(t *testing.T) { getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAzureBlobClient) // Update the GetRemoteDataClientInit function pointer - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return &mclient } getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) // Init azure blob client getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(*spltest.MockHTTPClient) + azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(*spltest.MockHTTPClient) azureBlobClient.BucketName = vol.Path azureBlobClient.Prefix = appSource.Location azureBlobClient.Endpoint = vol.Endpoint @@ -736,14 +736,14 @@ func TestAzureBlobGetAppsListShouldFailBadXmlResponse(t *testing.T) { getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAzureBlobClient) // Update the GetRemoteDataClientInit function pointer - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return &mclient } getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) // Init azure blob client getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(*spltest.MockHTTPClient) + azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(*spltest.MockHTTPClient) azureBlobClient.BucketName = vol.Path azureBlobClient.Prefix = appSource.Location azureBlobClient.Endpoint = vol.Endpoint @@ -822,14 +822,14 @@ func TestAzureBlobGetAppsListShouldFailNoIdentity(t *testing.T) { getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAzureBlobClient) // Update the GetRemoteDataClientInit function pointer - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return &mclient } getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) // Init azure blob client getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(*spltest.MockHTTPClient) + azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(*spltest.MockHTTPClient) azureBlobClient.BucketName = vol.Path azureBlobClient.Prefix = appSource.Location azureBlobClient.Endpoint = vol.Endpoint @@ -917,14 +917,14 @@ func TestAzureBlobGetAppsListShouldFailInvalidIdentity(t *testing.T) { getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAzureBlobClient) // Update the GetRemoteDataClientInit function pointer - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return &mclient } getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) // Init azure blob client getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(*spltest.MockHTTPClient) + azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(*spltest.MockHTTPClient) azureBlobClient.BucketName = vol.Path azureBlobClient.Prefix = appSource.Location azureBlobClient.Endpoint = vol.Endpoint @@ -996,14 +996,14 @@ func TestAzureBlobDownloadFailBadSecret(t *testing.T) { getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAzureBlobClient) // Update the GetRemoteDataClientInit function pointer - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return &mclient } getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) // Init azure blob client getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(*spltest.MockHTTPClient) + azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(*spltest.MockHTTPClient) azureBlobClient.BucketName = vol.Path azureBlobClient.Prefix = appSource.Location azureBlobClient.Endpoint = vol.Endpoint @@ -1080,14 +1080,14 @@ func TestAzureBlobDownloadAppShouldFailNoIdentity(t *testing.T) { getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAzureBlobClient) // Update the GetRemoteDataClientInit function pointer - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return &mclient } getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) // Init azure blob client getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(*spltest.MockHTTPClient) + azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(*spltest.MockHTTPClient) azureBlobClient.BucketName = vol.Path azureBlobClient.Prefix = appSource.Location azureBlobClient.Endpoint = vol.Endpoint @@ -1172,14 +1172,14 @@ func TestAzureBlobDownloadAppShouldFailInvalidIdentity(t *testing.T) { getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockAzureBlobClient) // Update the GetRemoteDataClientInit function pointer - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return &mclient } getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) // Init azure blob client getRemoteDataClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234").(*spltest.MockHTTPClient) + azureBlobClient.HTTPClient = getRemoteDataClientFn(ctx, "us-west-2", "abcd", "1234", false).(*spltest.MockHTTPClient) azureBlobClient.BucketName = vol.Path azureBlobClient.Prefix = appSource.Location azureBlobClient.Endpoint = vol.Endpoint diff --git a/pkg/splunk/client/minioclient.go b/pkg/splunk/client/minioclient.go index 02b8da7e9..1b2594261 100644 --- a/pkg/splunk/client/minioclient.go +++ b/pkg/splunk/client/minioclient.go @@ -50,12 +50,12 @@ type MinioClient struct { } // NewMinioClient returns an Minio client -func NewMinioClient(ctx context.Context, bucketName string, accessKeyID string, secretAccessKey string, prefix string, startAfter string, region string, endpoint string, fn GetInitFunc) (RemoteDataClient, error) { +func NewMinioClient(ctx context.Context, bucketName string, accessKeyID string, secretAccessKey string, prefix string, startAfter string, region string, endpoint string, pathStyleUrl bool, fn GetInitFunc) (RemoteDataClient, error) { var s3SplunkClient SplunkMinioClient var err error - cl := fn(ctx, endpoint, accessKeyID, secretAccessKey) + cl := fn(ctx, endpoint, accessKeyID, secretAccessKey, false) if cl == nil { err = fmt.Errorf("failed to create an Minio S3 client") return nil, err @@ -81,7 +81,7 @@ func RegisterMinioClient() { } // InitMinioClientWrapper is a wrapper around InitMinioClientSession -func InitMinioClientWrapper(ctx context.Context, appS3Endpoint string, accessKeyID string, secretAccessKey string) interface{} { +func InitMinioClientWrapper(ctx context.Context, appS3Endpoint string, accessKeyID string, secretAccessKey string, pathStyleUrl bool) interface{} { return InitMinioClientSession(ctx, appS3Endpoint, accessKeyID, secretAccessKey) } diff --git a/pkg/splunk/client/minioclient_test.go b/pkg/splunk/client/minioclient_test.go index e422904e0..831f6265d 100644 --- a/pkg/splunk/client/minioclient_test.go +++ b/pkg/splunk/client/minioclient_test.go @@ -27,19 +27,19 @@ import ( func TestInitMinioClientWrapper(t *testing.T) { ctx := context.TODO() - minioS3ClientSession := InitMinioClientWrapper(ctx, "https://s3.us-east-1.amazonaws.com", "abcd", "1234") + minioS3ClientSession := InitMinioClientWrapper(ctx, "https://s3.us-east-1.amazonaws.com", "abcd", "1234", false) if minioS3ClientSession == nil { t.Errorf("We should have got a valid Minio S3 client object") } // Test case without access and secret keys - minioS3ClientSession = InitMinioClientWrapper(ctx, "https://s3.us-east-1.amazonaws.com", "", "") + minioS3ClientSession = InitMinioClientWrapper(ctx, "https://s3.us-east-1.amazonaws.com", "", "", false) if minioS3ClientSession == nil { t.Errorf("We should have got a valid Minio S3 client object") } // Test erroneous minio session - minioS3ClientSession = InitMinioClientWrapper(ctx, "https://s3.us-east-1.amazonaws.com:-9000", "", "") + minioS3ClientSession = InitMinioClientWrapper(ctx, "https://s3.us-east-1.amazonaws.com:-9000", "", "", false) if minioS3ClientSession != nil { t.Errorf("Should have gotten a nil session due to error in URL") } @@ -50,21 +50,21 @@ func TestNewMinioClient(t *testing.T) { fn := InitMinioClientWrapper // Test1. Test for endpoint with https - minioS3Client, err := NewMinioClient(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "us-west-2", "https://s3.us-west-2.amazonaws.com", fn) + minioS3Client, err := NewMinioClient(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "us-west-2", "https://s3.us-west-2.amazonaws.com", false, fn) if minioS3Client == nil || err != nil { t.Errorf("NewMinioClient should have returned a valid Minio S3 client.") } // Test2. Test for endpoint with http - minioS3Client, err = NewMinioClient(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "us-west-2", "http://s3.us-west-2.amazonaws.com", fn) + minioS3Client, err = NewMinioClient(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "us-west-2", "http://s3.us-west-2.amazonaws.com", false, fn) if minioS3Client == nil || err != nil { t.Errorf("NewMinioClient should have returned a valid Minio S3 client.") } // Test3. Test for invalid endpoint - minioS3Client, err = NewMinioClient(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "us-west-2", "random-endpoint.com", fn) + minioS3Client, err = NewMinioClient(ctx, "sample_bucket", "abcd", "xyz", "admin/", "admin", "us-west-2", "random-endpoint.com", false, fn) if minioS3Client != nil || err == nil { t.Errorf("NewMinioClient should have returned a error.") @@ -178,7 +178,7 @@ func TestMinioGetAppsListShouldNotFail(t *testing.T) { getClientWrapper := RemoteDataClientsMap[vol.Provider] getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockMinioS3Client) - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockMinioS3Client{} cl.Objects = mockMinioObjects[index].Objects return cl @@ -187,7 +187,7 @@ func TestMinioGetAppsListShouldNotFail(t *testing.T) { getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) getS3ClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - minioClient.Client = getS3ClientFn(ctx, "us-west-2", "abcd", "1234").(spltest.MockMinioS3Client) + minioClient.Client = getS3ClientFn(ctx, "us-west-2", "abcd", "1234", false).(spltest.MockMinioS3Client) RemoteDataListResponse, err := minioClient.GetAppsList(ctx) if err != nil { @@ -278,7 +278,7 @@ func TestMinioGetAppsListShouldFail(t *testing.T) { getClientWrapper := RemoteDataClientsMap[vol.Provider] getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockMinioS3Client) - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockMinioS3Client{} // return empty objects list here to test the negative scenario return cl @@ -287,7 +287,7 @@ func TestMinioGetAppsListShouldFail(t *testing.T) { getClientWrapper.SetRemoteDataClientInitFuncPtr(ctx, vol.Provider, initFn) getS3ClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - minioClient.Client = getS3ClientFn(ctx, "us-west-2", "abcd", "1234").(spltest.MockMinioS3Client) + minioClient.Client = getS3ClientFn(ctx, "us-west-2", "abcd", "1234", false).(spltest.MockMinioS3Client) _, err = minioClient.GetAppsList(ctx) if err == nil { @@ -380,7 +380,7 @@ func TestMinioDownloadAppShouldNotFail(t *testing.T) { getClientWrapper := RemoteDataClientsMap[vol.Provider] getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockMinioS3Client) - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockMinioS3Client{} return cl } @@ -389,7 +389,7 @@ func TestMinioDownloadAppShouldNotFail(t *testing.T) { getS3ClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - minioClient.Client = getS3ClientFn(ctx, "us-west-2", "abcd", "1234").(spltest.MockMinioS3Client) + minioClient.Client = getS3ClientFn(ctx, "us-west-2", "abcd", "1234", false).(spltest.MockMinioS3Client) downloadRequest := RemoteDataDownloadRequest{ LocalFile: LocalFiles[index], @@ -473,7 +473,7 @@ func TestMinioDownloadAppShouldFail(t *testing.T) { getClientWrapper := RemoteDataClientsMap[vol.Provider] getClientWrapper.SetRemoteDataClientFuncPtr(ctx, vol.Provider, NewMockMinioS3Client) - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockMinioS3Client{} return cl } @@ -482,7 +482,7 @@ func TestMinioDownloadAppShouldFail(t *testing.T) { getS3ClientFn := getClientWrapper.GetRemoteDataClientInitFuncPtr(ctx) - minioClient.Client = getS3ClientFn(ctx, "us-west-2", "abcd", "1234").(spltest.MockMinioS3Client) + minioClient.Client = getS3ClientFn(ctx, "us-west-2", "abcd", "1234", false).(spltest.MockMinioS3Client) downloadRequest := RemoteDataDownloadRequest{ LocalFile: LocalFile[0], diff --git a/pkg/splunk/client/remotedataclient.go b/pkg/splunk/client/remotedataclient.go index 7e3cbecd7..18c6601f1 100644 --- a/pkg/splunk/client/remotedataclient.go +++ b/pkg/splunk/client/remotedataclient.go @@ -100,11 +100,12 @@ func (c *GetRemoteDataClientWrapper) GetRemoteDataClientInitFuncPtr(ctx context. } // GetInitFunc gets the init function pointer which returns the new RemoteDataClient session client object -type GetInitFunc func(context.Context, string, string, string) interface{} +type GetInitFunc func(context.Context, string, string, string, bool) interface{} // GetRemoteDataClient gets the required RemoteDataClient based on the storageType and provider type GetRemoteDataClient func(context.Context, string /* bucket */, string, /* Access key ID */ - string /* Secret access key */, string /* Prefix */, string /* StartAfter */, string /* Region */, string /* Endpoint */, GetInitFunc) (RemoteDataClient, error) + string /* Secret access key */, string /* Prefix */, string /* StartAfter */, string, /* Region */ + string /* Endpoint */, bool /* pathStyleUrl */, GetInitFunc) (RemoteDataClient, error) // SplunkRemoteDataClient is a simple object used to connect to RemoteDataClient type SplunkRemoteDataClient struct { diff --git a/pkg/splunk/client/util.go b/pkg/splunk/client/util.go index d53cd1f91..09aff6f72 100644 --- a/pkg/splunk/client/util.go +++ b/pkg/splunk/client/util.go @@ -29,12 +29,12 @@ import ( // Ideally this function should live in test package but due to // dependency of some variables in client package and to avoid // cyclic dependency this has to live here. -func NewMockAWSS3Client(ctx context.Context, bucketName string, accessKeyID string, secretAccessKey string, prefix string, startAfter string, region string, endpoint string, fn GetInitFunc) (RemoteDataClient, error) { +func NewMockAWSS3Client(ctx context.Context, bucketName string, accessKeyID string, secretAccessKey string, prefix string, startAfter string, region string, endpoint string, s3PathUrl bool, fn GetInitFunc) (RemoteDataClient, error) { var s3SplunkClient SplunkAWSS3Client var err error region = fmt.Sprintf("%s%s%s", region, awsRegionEndPointDelimiter, endpoint) - cl := fn(ctx, region, accessKeyID, secretAccessKey) + cl := fn(ctx, region, accessKeyID, secretAccessKey, false) if cl == nil { err = fmt.Errorf("failed to create an AWS S3 client") return nil, err @@ -56,11 +56,11 @@ func NewMockAWSS3Client(ctx context.Context, bucketName string, accessKeyID stri } // NewMockMinioS3Client is mock client for testing minio client -func NewMockMinioS3Client(ctx context.Context, bucketName string, accessKeyID string, secretAccessKey string, prefix string, startAfter string, region string, endpoint string, fn GetInitFunc) (RemoteDataClient, error) { +func NewMockMinioS3Client(ctx context.Context, bucketName string, accessKeyID string, secretAccessKey string, prefix string, startAfter string, region string, endpoint string, s3PathUrl bool, fn GetInitFunc) (RemoteDataClient, error) { var s3SplunkClient SplunkMinioClient var err error - cl := fn(ctx, endpoint, accessKeyID, secretAccessKey) + cl := fn(ctx, endpoint, accessKeyID, secretAccessKey, false) if cl == nil { err = fmt.Errorf("failed to create an AWS S3 client") return nil, err @@ -80,10 +80,10 @@ func NewMockMinioS3Client(ctx context.Context, bucketName string, accessKeyID st } // NewMockAzureBlobClient will create a mock azureblob client -func NewMockAzureBlobClient(ctx context.Context, bucketName string, storageAccountName string, secretAccessKey string, prefix string, startAfter string, region string, endpoint string, fn GetInitFunc) (RemoteDataClient, error) { +func NewMockAzureBlobClient(ctx context.Context, bucketName string, storageAccountName string, secretAccessKey string, prefix string, startAfter string, region string, endpoint string, s3PathUrl bool, fn GetInitFunc) (RemoteDataClient, error) { var err error - cl := fn(ctx, endpoint, storageAccountName, secretAccessKey) + cl := fn(ctx, endpoint, storageAccountName, secretAccessKey, false) if cl == nil { err = fmt.Errorf("failed to create an Azure blob client") return nil, err diff --git a/pkg/splunk/client/util_test.go b/pkg/splunk/client/util_test.go index 1c3ba5d47..8ef65fb2b 100644 --- a/pkg/splunk/client/util_test.go +++ b/pkg/splunk/client/util_test.go @@ -17,10 +17,11 @@ package client import ( "context" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" "reflect" "testing" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + spltest "github.com/splunk/splunk-operator/pkg/splunk/test" ) @@ -66,22 +67,22 @@ func TestCheckIfVolumeExists(t *testing.T) { func TestNewMockAWSS3Client(t *testing.T) { ctx := context.TODO() // Test 1. Test the valid case - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockAWSS3Client{} return cl } - _, err := NewMockAWSS3Client(ctx, "sample_bucket", "abcd", "1234", "admin/", "admin", "us-west-2", "htts://s3.us-west-2.amazonaws.com", initFn) + _, err := NewMockAWSS3Client(ctx, "sample_bucket", "abcd", "1234", "admin/", "admin", "us-west-2", "htts://s3.us-west-2.amazonaws.com", false, initFn) if err != nil { t.Errorf("NewMockAWSS3Client should have returned a Mock AWS client.") } // Test 2. Test the invalid case by returning nil client - initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return nil } - _, err = NewMockAWSS3Client(ctx, "sample_bucket", "abcd", "1234", "admin/", "admin", "us-west-2", "htts://s3.us-west-2.amazonaws.com", initFn) + _, err = NewMockAWSS3Client(ctx, "sample_bucket", "abcd", "1234", "admin/", "admin", "us-west-2", "htts://s3.us-west-2.amazonaws.com", false, initFn) if err == nil { t.Errorf("NewMockAWSS3Client should have returned an error since we passed nil client in init function.") @@ -91,22 +92,22 @@ func TestNewMockAWSS3Client(t *testing.T) { func TestNewMockAWMinioClient(t *testing.T) { ctx := context.TODO() // Test 1. Test the valid case - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockMinioS3Client{} return cl } - _, err := NewMockMinioS3Client(ctx, "sample_bucket", "abcd", "1234", "admin/", "admin", "us-west-2", "htts://s3.us-west-2.amazonaws.com", initFn) + _, err := NewMockMinioS3Client(ctx, "sample_bucket", "abcd", "1234", "admin/", "admin", "us-west-2", "htts://s3.us-west-2.amazonaws.com", false, initFn) if err != nil { t.Errorf("NewMockAWSS3Client should have returned a Mock AWS client.") } // Test 2. Test the invalid case by returning nil client - initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return nil } - _, err = NewMockMinioS3Client(ctx, "sample_bucket", "abcd", "1234", "admin/", "admin", "us-west-2", "htts://s3.us-west-2.amazonaws.com", initFn) + _, err = NewMockMinioS3Client(ctx, "sample_bucket", "abcd", "1234", "admin/", "admin", "us-west-2", "htts://s3.us-west-2.amazonaws.com", false, initFn) if err == nil { t.Errorf("NewMockMinioS3Client should have returned an error since we passed nil client in init function.") @@ -116,22 +117,22 @@ func TestNewMockAWMinioClient(t *testing.T) { func TestNewMockAzureBlobClient(t *testing.T) { ctx := context.TODO() // Test 1. Test the valid case - initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := &spltest.MockHTTPClient{} return cl } - _, err := NewMockAzureBlobClient(ctx, "sample_bucket", "abcd", "1234", "admin/", "admin", "us-west-2", "htts://s3.us-west-2.amazonaws.com", initFn) + _, err := NewMockAzureBlobClient(ctx, "sample_bucket", "abcd", "1234", "admin/", "admin", "us-west-2", "htts://s3.us-west-2.amazonaws.com", false, initFn) if err != nil { t.Errorf("NewMockAzureBlobClient should have returned a Mock AWS client.") } // Test 2. Test the invalid case by returning nil client - initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return nil } - _, err = NewMockAzureBlobClient(ctx, "sample_bucket", "abcd", "1234", "admin/", "admin", "us-west-2", "htts://s3.us-west-2.amazonaws.com", initFn) + _, err = NewMockAzureBlobClient(ctx, "sample_bucket", "abcd", "1234", "admin/", "admin", "us-west-2", "htts://s3.us-west-2.amazonaws.com", false, initFn) if err == nil { t.Errorf("NewMockAzureBlobClient should have returned an error since we passed nil client in init function.") diff --git a/pkg/splunk/enterprise/clustermanager_test.go b/pkg/splunk/enterprise/clustermanager_test.go index da3265482..e7909ed54 100644 --- a/pkg/splunk/enterprise/clustermanager_test.go +++ b/pkg/splunk/enterprise/clustermanager_test.go @@ -1113,7 +1113,7 @@ func TestClusterManagerGetAppsListForAWSS3ClientShouldNotFail(t *testing.T) { appFrameworkRef: &cm.Spec.AppFrameworkConfig, vol: &vol, location: appSource.Location, - initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockAWSS3Client{} cl.Objects = mockAwsObjects[index].Objects return cl @@ -1238,7 +1238,7 @@ func TestClusterManagerGetAppsListForAWSS3ClientShouldFail(t *testing.T) { appFrameworkRef: &cm.Spec.AppFrameworkConfig, vol: &vol, location: appSource.Location, - initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { // Purposefully return nil here so that we test the error scenario return nil }, @@ -1296,7 +1296,7 @@ func TestClusterManagerGetAppsListForAWSS3ClientShouldFail(t *testing.T) { t.Errorf("GetAppsList should have returned error as we could not get the S3 client") } - remoteDataClientMgr.initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + remoteDataClientMgr.initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { // To test the error scenario, do no set the Objects member yet cl := spltest.MockAWSS3Client{} return cl diff --git a/pkg/splunk/enterprise/clustermaster_test.go b/pkg/splunk/enterprise/clustermaster_test.go index 70ee16357..54dc224d2 100644 --- a/pkg/splunk/enterprise/clustermaster_test.go +++ b/pkg/splunk/enterprise/clustermaster_test.go @@ -813,7 +813,7 @@ func TestClusterMasterGetAppsListForAWSS3ClientShouldNotFail(t *testing.T) { appFrameworkRef: &cm.Spec.AppFrameworkConfig, vol: &vol, location: appSource.Location, - initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockAWSS3Client{} cl.Objects = mockAwsObjects[index].Objects return cl @@ -938,7 +938,7 @@ func TestClusterMasterGetAppsListForAWSS3ClientShouldFail(t *testing.T) { appFrameworkRef: &cm.Spec.AppFrameworkConfig, vol: &vol, location: appSource.Location, - initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { // Purposefully return nil here so that we test the error scenario return nil }, @@ -996,7 +996,7 @@ func TestClusterMasterGetAppsListForAWSS3ClientShouldFail(t *testing.T) { t.Errorf("GetAppsList should have returned error as we could not get the S3 client") } - remoteDataClientMgr.initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + remoteDataClientMgr.initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { // To test the error scenario, do no set the Objects member yet cl := spltest.MockAWSS3Client{} return cl diff --git a/pkg/splunk/enterprise/licensemanager_test.go b/pkg/splunk/enterprise/licensemanager_test.go index 71b32113c..789adcc0f 100644 --- a/pkg/splunk/enterprise/licensemanager_test.go +++ b/pkg/splunk/enterprise/licensemanager_test.go @@ -396,7 +396,7 @@ func TestLicensemanagerGetAppsListForAWSS3ClientShouldNotFail(t *testing.T) { cr: &cr, appFrameworkRef: &cr.Spec.AppFrameworkConfig, vol: &vol, location: appSource.Location, - initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockAWSS3Client{} cl.Objects = mockAwsObjects[index].Objects return cl @@ -518,7 +518,7 @@ func TestLicenseManagerGetAppsListForAWSS3ClientShouldFail(t *testing.T) { appFrameworkRef: &lm.Spec.AppFrameworkConfig, vol: &vol, location: appSource.Location, - initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { // Purposefully return nil here so that we test the error scenario return nil }, @@ -576,7 +576,7 @@ func TestLicenseManagerGetAppsListForAWSS3ClientShouldFail(t *testing.T) { t.Errorf("GetAppsList should have returned error as we could not get the S3 client") } - s3ClientMgr.initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + s3ClientMgr.initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { // To test the error scenario, do no set the Objects member yet cl := spltest.MockAWSS3Client{} return cl diff --git a/pkg/splunk/enterprise/licensemaster_test.go b/pkg/splunk/enterprise/licensemaster_test.go index 9e74afd5b..6dbf7b241 100644 --- a/pkg/splunk/enterprise/licensemaster_test.go +++ b/pkg/splunk/enterprise/licensemaster_test.go @@ -407,7 +407,7 @@ func TestLicensemasterGetAppsListForAWSS3ClientShouldNotFail(t *testing.T) { cr: &cr, appFrameworkRef: &cr.Spec.AppFrameworkConfig, vol: &vol, location: appSource.Location, - initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockAWSS3Client{} cl.Objects = mockAwsObjects[index].Objects return cl @@ -529,7 +529,7 @@ func TestLicenseMasterGetAppsListForAWSS3ClientShouldFail(t *testing.T) { appFrameworkRef: &lm.Spec.AppFrameworkConfig, vol: &vol, location: appSource.Location, - initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { // Purposefully return nil here so that we test the error scenario return nil }, @@ -587,7 +587,7 @@ func TestLicenseMasterGetAppsListForAWSS3ClientShouldFail(t *testing.T) { t.Errorf("GetAppsList should have returned error as we could not get the S3 client") } - remoteDataClientMgr.initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + remoteDataClientMgr.initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { // To test the error scenario, do no set the Objects member yet cl := spltest.MockAWSS3Client{} return cl diff --git a/pkg/splunk/enterprise/monitoringconsole_test.go b/pkg/splunk/enterprise/monitoringconsole_test.go index 07459c2a7..73a31df4a 100644 --- a/pkg/splunk/enterprise/monitoringconsole_test.go +++ b/pkg/splunk/enterprise/monitoringconsole_test.go @@ -608,7 +608,7 @@ func TestMonitoringConsoleGetAppsListForAWSS3ClientShouldNotFail(t *testing.T) { cr: &cr, appFrameworkRef: &cr.Spec.AppFrameworkConfig, vol: &vol, location: appSource.Location, - initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockAWSS3Client{} cl.Objects = mockAwsObjects[index].Objects return cl @@ -730,7 +730,7 @@ func TestMonitoringConsoleGetAppsListForAWSS3ClientShouldFail(t *testing.T) { appFrameworkRef: &cr.Spec.AppFrameworkConfig, vol: &vol, location: appSource.Location, - initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { // Purposefully return nil here so that we test the error scenario return nil }, @@ -788,7 +788,7 @@ func TestMonitoringConsoleGetAppsListForAWSS3ClientShouldFail(t *testing.T) { t.Errorf("GetAppsList should have returned error as we could not get the S3 client") } - remoteDataClientMgr.initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + remoteDataClientMgr.initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { // To test the error scenario, do no set the Objects member yet cl := spltest.MockAWSS3Client{} return cl diff --git a/pkg/splunk/enterprise/searchheadcluster_test.go b/pkg/splunk/enterprise/searchheadcluster_test.go index 8bc795308..e62d5c301 100644 --- a/pkg/splunk/enterprise/searchheadcluster_test.go +++ b/pkg/splunk/enterprise/searchheadcluster_test.go @@ -958,7 +958,7 @@ func TestSHCGetAppsListForAWSS3ClientShouldNotFail(t *testing.T) { cr: &cr, appFrameworkRef: &cr.Spec.AppFrameworkConfig, vol: &vol, location: appSource.Location, - initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockAWSS3Client{} cl.Objects = mockAwsObjects[index].Objects return cl @@ -1078,7 +1078,7 @@ func TestSHCGetAppsListForAWSS3ClientShouldFail(t *testing.T) { appFrameworkRef: &cr.Spec.AppFrameworkConfig, vol: &vol, location: appSource.Location, - initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { // Purposefully return nil here so that we test the error scenario return nil }, @@ -1136,7 +1136,7 @@ func TestSHCGetAppsListForAWSS3ClientShouldFail(t *testing.T) { t.Errorf("GetAppsList should have returned error as we could not get the S3 client") } - remoteDataClientMgr.initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + remoteDataClientMgr.initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { // To test the error scenario, do no set the Objects member yet cl := spltest.MockAWSS3Client{} return cl diff --git a/pkg/splunk/enterprise/standalone_test.go b/pkg/splunk/enterprise/standalone_test.go index a317d7677..abb9b5d76 100644 --- a/pkg/splunk/enterprise/standalone_test.go +++ b/pkg/splunk/enterprise/standalone_test.go @@ -726,7 +726,7 @@ func TestStandaloneGetAppsListForAWSS3ClientShouldNotFail(t *testing.T) { cr: &cr, appFrameworkRef: &cr.Spec.AppFrameworkConfig, vol: &vol, location: appSource.Location, - initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { cl := spltest.MockAWSS3Client{} cl.Objects = mockAwsObjects[index].Objects return cl @@ -847,7 +847,7 @@ func TestStandaloneGetAppsListForAWSS3ClientShouldFail(t *testing.T) { appFrameworkRef: &cr.Spec.AppFrameworkConfig, vol: &vol, location: appSource.Location, - initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + initFn: func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { // Purposefully return nil here so that we test the error scenario return nil }, @@ -905,7 +905,7 @@ func TestStandaloneGetAppsListForAWSS3ClientShouldFail(t *testing.T) { t.Errorf("GetAppsList should have returned error as we could not get the S3 client") } - remoteDataClientMgr.initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + remoteDataClientMgr.initFn = func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { // To test the error scenario, do no set the Objects member yet cl := spltest.MockAWSS3Client{} return cl diff --git a/pkg/splunk/enterprise/util.go b/pkg/splunk/enterprise/util.go index 4a70f0d6b..5f7e29ed1 100644 --- a/pkg/splunk/enterprise/util.go +++ b/pkg/splunk/enterprise/util.go @@ -193,8 +193,13 @@ func GetRemoteStorageClient(ctx context.Context, client splcommon.ControllerClie scopedLog.Info("Creating the client", "volume", vol.Name, "bucket", bucket, "bucket path", prefix) var err error + var s3PathUrl bool - remoteDataClient.Client, err = getClient(ctx, bucket, accessKeyID, secretAccessKey, prefix, prefix /* startAfter*/, vol.Region, vol.Endpoint, fn) + if vol.S3PathUrl { + s3PathUrl = true + } + + remoteDataClient.Client, err = getClient(ctx, bucket, accessKeyID, secretAccessKey, prefix, prefix /* startAfter*/, vol.Region, vol.Endpoint, s3PathUrl, fn) if err != nil { scopedLog.Error(err, "Failed to get the S3 client") diff --git a/pkg/splunk/enterprise/util_test.go b/pkg/splunk/enterprise/util_test.go index 1d7ed4d10..929ffcbfd 100644 --- a/pkg/splunk/enterprise/util_test.go +++ b/pkg/splunk/enterprise/util_test.go @@ -105,7 +105,7 @@ func TestGetRemoteStorageClient(t *testing.T) { }, } - fn := func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + fn := func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return nil } @@ -116,7 +116,7 @@ func TestGetRemoteStorageClient(t *testing.T) { // Cover no secret key, empty GetInitFunc case GetRemoteStorageClient(ctx, c, &cm, &cm.Spec.AppFrameworkConfig, &cm.Spec.AppFrameworkConfig.VolList[0], "location", fn) - fn = func(ctx context.Context, region, accessKeyID, secretAccessKey string) interface{} { + fn = func(ctx context.Context, region, accessKeyID, secretAccessKey string, pathStyleUrl bool) interface{} { return spltest.MockAWSS3Client{} } GetRemoteStorageClient(ctx, c, &cm, &cm.Spec.AppFrameworkConfig, &cm.Spec.AppFrameworkConfig.VolList[0], "location", fn)