From a3d5d1984df8524bec949891fd003cf58ab80ca9 Mon Sep 17 00:00:00 2001 From: AnnaHariprasad5123 Date: Tue, 4 Jun 2024 08:43:34 +0530 Subject: [PATCH 1/7] feat: add support for Google Cloud Storage in S3 plugin --- .../com/external/plugins/AmazonS3Plugin.java | 113 ++++++++++++++---- .../plugins/constants/S3PluginConstants.java | 4 + .../plugins/exceptions/S3ErrorMessages.java | 10 +- .../com/external/utils/DatasourceUtils.java | 19 +-- .../src/main/resources/form.json | 17 +++ .../external/plugins/AmazonS3PluginTest.java | 29 ++++- 6 files changed, 154 insertions(+), 38 deletions(-) diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java index 7f71cfbfda20..7b2e28f2c624 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java @@ -103,8 +103,10 @@ import static com.external.plugins.constants.S3PluginConstants.AWS_S3_SERVICE_PROVIDER; import static com.external.plugins.constants.S3PluginConstants.BASE64_DELIMITER; import static com.external.plugins.constants.S3PluginConstants.CUSTOM_ENDPOINT_INDEX; +import static com.external.plugins.constants.S3PluginConstants.DEFAULT_BUCKET_PROPERTY_INDEX; import static com.external.plugins.constants.S3PluginConstants.DEFAULT_FILE_NAME; import static com.external.plugins.constants.S3PluginConstants.DEFAULT_URL_EXPIRY_IN_MINUTES; +import static com.external.plugins.constants.S3PluginConstants.GOOGLE_CLOUD_SERVICE_PROVIDER; import static com.external.plugins.constants.S3PluginConstants.NO; import static com.external.plugins.constants.S3PluginConstants.S3_DRIVER; import static com.external.plugins.constants.S3PluginConstants.S3_SERVICE_PROVIDER_PROPERTY_INDEX; @@ -948,36 +950,97 @@ public Set validateDatasource(DatasourceConfiguration datasourceConfigur invalids.add(S3ErrorMessages.DS_MANDATORY_PARAMETER_ENDPOINT_URL_MISSING_ERROR_MSG); } + if (datasourceConfiguration != null) { + String serviceProvider = (String) + properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue(); + if (GOOGLE_CLOUD_SERVICE_PROVIDER.equals(serviceProvider)) { + String defaultBucket = (String) + properties.get(DEFAULT_BUCKET_PROPERTY_INDEX).getValue(); + if (StringUtils.isNullOrEmpty(defaultBucket)) { + invalids.add(S3ErrorMessages.DS_MANDATORY_PARAMETER_DEFAULT_BUCKET_MISSING_ERROR_MSG); + } + } + } + return invalids; } @Override - public Mono testDatasource(AmazonS3 connection) { - return Mono.fromCallable(() -> { - /* - * - Please note that as of 28 Jan 2021, the way AmazonS3 client works, creating a connection - * object with wrong credentials does not throw any exception. - * - Hence, adding a listBuckets() method call to test the connection. - */ - connection.listBuckets(); - return new DatasourceTestResult(); - }) - .onErrorResume(error -> { - if (error instanceof AmazonS3Exception - && ACCESS_DENIED_ERROR_CODE.equals(((AmazonS3Exception) error).getErrorCode())) { - /** - * Sometimes a valid account credential may not have permission to run listBuckets action - * . In this case `AccessDenied` error is returned. - * That fact that the credentials caused `AccessDenied` error instead of invalid access key - * id or signature mismatch error means that the credentials are valid, we are able to - * establish a connection as well, but the account does not have permission to run - * listBuckets. - */ - return Mono.just(new DatasourceTestResult()); - } + public Mono testDatasource(DatasourceConfiguration datasourceConfiguration) { + if (datasourceConfiguration == null) { + return Mono.just(new DatasourceTestResult( + S3ErrorMessages.DS_AT_LEAST_ONE_MANDATORY_PARAMETER_MISSING_ERROR_MSG)); + } + + List properties = datasourceConfiguration.getProperties(); + String s3Provider = + (String) properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue(); + + // Handle Google Cloud Storage in a separate method if necessary + if (GOOGLE_CLOUD_SERVICE_PROVIDER.equals(s3Provider)) { + return testGoogleCloudStorage(datasourceConfiguration); + } - return Mono.just(new DatasourceTestResult(amazonS3ErrorUtils.getReadableError(error))); - }); + // For Amazon S3 or other providers, perform a standard test by listing buckets in Amazon S3 + return datasourceCreate(datasourceConfiguration) + .flatMap(connection -> Mono.fromCallable(() -> { + /* + * - Please note that as of 28 Jan 2021, the way AmazonS3 client works, creating a connection + * object with wrong credentials does not throw any exception. + * - Hence, adding a listBuckets() method call to test the connection. + */ + connection.listBuckets(); + return new DatasourceTestResult(); + }) + .onErrorResume(error -> { + if (error instanceof AmazonS3Exception + && ACCESS_DENIED_ERROR_CODE.equals( + ((AmazonS3Exception) error).getErrorCode())) { + /** + * Sometimes a valid account credential may not have permission to run listBuckets action + * . In this case `AccessDenied` error is returned. + * That fact that the credentials caused `AccessDenied` error instead of invalid access key + * id or signature mismatch error means that the credentials are valid, we are able to + * establish a connection as well, but the account does not have permission to run + * listBuckets. + */ + return Mono.just(new DatasourceTestResult()); + } + + return Mono.just(new DatasourceTestResult(amazonS3ErrorUtils.getReadableError(error))); + }) + .doFinally(signalType -> connection.shutdown())) + .onErrorResume(error -> Mono.just(new DatasourceTestResult(error.getMessage()))) + .subscribeOn(scheduler); + } + + private Mono testGoogleCloudStorage(DatasourceConfiguration datasourceConfiguration) { + List properties = datasourceConfiguration.getProperties(); + String defaultBucket = + (String) properties.get(DEFAULT_BUCKET_PROPERTY_INDEX).getValue(); + if (StringUtils.isNullOrEmpty(defaultBucket)) { + return Mono.just(new DatasourceTestResult( + S3ErrorMessages.DS_MANDATORY_PARAMETER_DEFAULT_BUCKET_MISSING_ERROR_MSG)); + } + + return datasourceCreate(datasourceConfiguration) + .flatMap(connection -> Mono.fromCallable(() -> { + connection.listObjects(defaultBucket); + return new DatasourceTestResult(); + }) + .onErrorResume(error -> { + if (error instanceof AmazonS3Exception + && ((AmazonS3Exception) error).getStatusCode() == 404) { + return Mono.just( + new DatasourceTestResult(S3ErrorMessages.NON_EXITED_BUCKET_ERROR_MSG)); + } else { + return Mono.just( + new DatasourceTestResult(amazonS3ErrorUtils.getReadableError(error))); + } + }) + .doFinally(signalType -> connection.shutdown())) + .onErrorResume(error -> Mono.just(new DatasourceTestResult(error.getMessage()))) + .subscribeOn(scheduler); } /** diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/S3PluginConstants.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/S3PluginConstants.java index 74132ff566a1..ad4e6c3ff221 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/S3PluginConstants.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/S3PluginConstants.java @@ -4,6 +4,8 @@ public class S3PluginConstants { public static final String S3_DRIVER = "com.amazonaws.services.s3.AmazonS3"; public static final int S3_SERVICE_PROVIDER_PROPERTY_INDEX = 1; public static final int CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX = 2; + + public static final int DEFAULT_BUCKET_PROPERTY_INDEX = 3; public static final int CUSTOM_ENDPOINT_INDEX = 0; public static final String DEFAULT_URL_EXPIRY_IN_MINUTES = "5"; // max 7 days is possible public static final String YES = "YES"; @@ -12,4 +14,6 @@ public class S3PluginConstants { public static final String AWS_S3_SERVICE_PROVIDER = "amazon-s3"; public static String DEFAULT_FILE_NAME = "MyFile.txt"; public static final String ACCESS_DENIED_ERROR_CODE = "AccessDenied"; + public static final String GOOGLE_CLOUD_SERVICE_PROVIDER = "google-cloud-storage"; + public static final String AUTO = "auto"; } diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/exceptions/S3ErrorMessages.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/exceptions/S3ErrorMessages.java index aefb7e796bbf..ed517fccc0cb 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/exceptions/S3ErrorMessages.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/exceptions/S3ErrorMessages.java @@ -18,6 +18,9 @@ public class S3ErrorMessages extends BasePluginErrorMessages { "Appsmith has encountered an unexpected error when getting bucket name. Please reach out to " + "Appsmith customer support to resolve this."; + public static final String NON_EXITED_BUCKET_ERROR_MSG = + "Appsmith has encountered an unexpected error when getting bucket name. The specified bucket does not exist in Google Cloud Storage. "; + public static final String EMPTY_PREFIX_ERROR_MSG = "Appsmith has encountered an unexpected error when getting path prefix. Please reach out to " + "Appsmith customer support to resolve this."; @@ -126,4 +129,9 @@ public class S3ErrorMessages extends BasePluginErrorMessages { "Required parameter 'Endpoint URL' is empty. Did you forget to edit the 'Endpoint" + " URL' field in the datasource creation form ? You need to fill it with " + "the endpoint URL of your S3 instance."; -} + + public static final String DS_MANDATORY_PARAMETER_DEFAULT_BUCKET_MISSING_ERROR_MSG = + "Required parameter 'Default Bucket' is empty. Did you forget to edit the 'Default Bucket' " + + "field in the datasource creation form? You need to fill it with the default bucket name " + + "for your Google Cloud Storage instance."; +} \ No newline at end of file diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java index c53798f52708..5f1ec4b39a4b 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java @@ -20,9 +20,7 @@ import static com.amazonaws.regions.Regions.DEFAULT_REGION; import static com.appsmith.external.helpers.PluginUtils.getValueSafelyFromPropertyList; -import static com.external.plugins.constants.S3PluginConstants.CUSTOM_ENDPOINT_INDEX; -import static com.external.plugins.constants.S3PluginConstants.CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX; -import static com.external.plugins.constants.S3PluginConstants.S3_SERVICE_PROVIDER_PROPERTY_INDEX; +import static com.external.plugins.constants.S3PluginConstants.*; import static com.external.utils.DatasourceUtils.S3ServiceProvider.AMAZON; public class DatasourceUtils { @@ -67,6 +65,7 @@ public enum S3ServiceProvider { DIGITAL_OCEAN_SPACES("digital-ocean-spaces"), DREAM_OBJECTS("dream-objects"), MINIO("minio"), + GOOGLE_CLOUD_STORAGE("google-cloud-storage"), OTHER("other"); private String name; @@ -136,7 +135,6 @@ public static AmazonS3ClientBuilder getS3ClientBuilder(DatasourceConfiguration d S3ServiceProvider s3ServiceProvider = S3ServiceProvider.fromString( (String) properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue()); - /** * AmazonS3 provides an attribute `forceGlobalBucketAccessEnabled` that automatically routes the request to a * region such that request should succeed. @@ -156,6 +154,13 @@ public static AmazonS3ClientBuilder getS3ClientBuilder(DatasourceConfiguration d */ if (s3ServiceProvider.equals(AMAZON)) { s3ClientBuilder = s3ClientBuilder.withRegion(DEFAULT_REGION).enableForceGlobalBucketAccess(); + } else if (s3ServiceProvider.equals(GOOGLE_CLOUD_SERVICE_PROVIDER)) { + String endpoint = datasourceConfiguration + .getEndpoints() + .get(CUSTOM_ENDPOINT_INDEX) + .getHost(); + s3ClientBuilder = s3ClientBuilder.withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(endpoint, AUTO)); } else { String endpoint = datasourceConfiguration .getEndpoints() @@ -233,8 +238,8 @@ private static String getUserProvidedRegion(List properties) { /** * This method checks if the S3 endpoint URL has correct format and extracts region information from it. * - * @param endpoint : endpoint URL - * @param regex : expected endpoint URL pattern + * @param endpoint : endpoint URL + * @param regex : expected endpoint URL pattern * @param regionGroupIndex : pattern group index for region string * @return S3 object storage region. * @throws AppsmithPluginException when then endpoint URL does not match the expected regex pattern. @@ -260,4 +265,4 @@ private static String getRegionFromEndpointPattern(String endpoint, String regex AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, S3ErrorMessages.INCORRECT_S3_ENDPOINT_URL_ERROR_MSG); } -} +} \ No newline at end of file diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/form.json b/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/form.json index 56e8c272dc80..aca975ff4e43 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/form.json +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/form.json @@ -43,6 +43,10 @@ "label": "MinIO", "value": "minio" }, + { + "label": "Google Cloud Storage", + "value": "google-cloud-storage" + }, { "label": "Other", "value": "other" @@ -106,6 +110,19 @@ } ] } + }, + { + "label": "Default Bucket", + "configProperty": "datasourceConfiguration.properties[3].value", + "controlType": "INPUT_TEXT", + "initialValue": "", + "placeholderText": "Enter default bucket name", + "isRequired": true, + "hidden": { + "path": "datasourceConfiguration.properties[1].value", + "comparison": "NOT_EQUALS", + "value": "google-cloud-storage" + } } ] } diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java b/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java index fab27b5cd544..fd9788b244d1 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java @@ -70,10 +70,7 @@ import static com.external.plugins.constants.FieldName.READ_DATATYPE; import static com.external.plugins.constants.FieldName.READ_EXPIRY; import static com.external.plugins.constants.FieldName.SMART_SUBSTITUTION; -import static com.external.plugins.constants.S3PluginConstants.DEFAULT_FILE_NAME; -import static com.external.plugins.constants.S3PluginConstants.DEFAULT_URL_EXPIRY_IN_MINUTES; -import static com.external.plugins.constants.S3PluginConstants.NO; -import static com.external.plugins.constants.S3PluginConstants.YES; +import static com.external.plugins.constants.S3PluginConstants.*; import static com.external.utils.DatasourceUtils.getS3ClientBuilder; import static com.external.utils.TemplateUtils.CREATE_FILE_TEMPLATE_NAME; import static com.external.utils.TemplateUtils.CREATE_MULTIPLE_FILES_TEMPLATE_NAME; @@ -202,6 +199,28 @@ public void testValidateDatasourceWithMissingRegionWithOtherS3ServiceProvider() .verifyComplete(); } + @Test + public void testValidateDatasourceWithMissingRegionWithNonAmazonProvider() { + DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); + + datasourceConfiguration.getProperties().get(1).setValue(GOOGLE_CLOUD_SERVICE_PROVIDER); + + Property defaultBucketProperty = new Property("default bucket", ""); + datasourceConfiguration.getProperties().add(defaultBucketProperty); + + AmazonS3Plugin.S3PluginExecutor pluginExecutor = new AmazonS3Plugin.S3PluginExecutor(); + Mono pluginExecutorMono = Mono.just(pluginExecutor); + + StepVerifier.create(pluginExecutorMono) + .assertNext(executor -> { + Set res = executor.validateDatasource(datasourceConfiguration); + // There should be no errors related to the region for GCS, but it should validate the default + // bucket + assertTrue(res.contains(S3ErrorMessages.DS_MANDATORY_PARAMETER_DEFAULT_BUCKET_MISSING_ERROR_MSG)); + }) + .verifyComplete(); + } + @Test public void testValidateDatasourceWithMissingRegionWithListedProvider() { DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); @@ -1535,4 +1554,4 @@ public void verify_sanitizeGenerateCRUDPageTemplateInfo_addsInfoToReplaceTemplat .block(); assertEquals(userSelectedBucketName, mappedColumnsAndTableName.get("templateBucket")); } -} +} \ No newline at end of file From 4844aeda9c6aba42c08d586b93b01426b8a63a33 Mon Sep 17 00:00:00 2001 From: AnnaHariprasad5123 Date: Thu, 6 Jun 2024 13:38:41 +0530 Subject: [PATCH 2/7] Fixed: Wildcard imports --- .../java/com/external/plugins/exceptions/S3ErrorMessages.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/exceptions/S3ErrorMessages.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/exceptions/S3ErrorMessages.java index ed517fccc0cb..945a47bf55e8 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/exceptions/S3ErrorMessages.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/exceptions/S3ErrorMessages.java @@ -134,4 +134,4 @@ public class S3ErrorMessages extends BasePluginErrorMessages { "Required parameter 'Default Bucket' is empty. Did you forget to edit the 'Default Bucket' " + "field in the datasource creation form? You need to fill it with the default bucket name " + "for your Google Cloud Storage instance."; -} \ No newline at end of file +} From c27fecad1336af91a5399faa4cedc566fd18d920 Mon Sep 17 00:00:00 2001 From: AnnaHariprasad5123 Date: Thu, 6 Jun 2024 17:41:40 +0530 Subject: [PATCH 3/7] Formatted files --- .../src/main/java/com/external/utils/DatasourceUtils.java | 2 +- .../src/test/java/com/external/plugins/AmazonS3PluginTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java index 5f1ec4b39a4b..7eb62d1d0b91 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java @@ -265,4 +265,4 @@ private static String getRegionFromEndpointPattern(String endpoint, String regex AppsmithPluginError.PLUGIN_DATASOURCE_ARGUMENT_ERROR, S3ErrorMessages.INCORRECT_S3_ENDPOINT_URL_ERROR_MSG); } -} \ No newline at end of file +} diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java b/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java index fd9788b244d1..fc33d5ece6fd 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java @@ -1554,4 +1554,4 @@ public void verify_sanitizeGenerateCRUDPageTemplateInfo_addsInfoToReplaceTemplat .block(); assertEquals(userSelectedBucketName, mappedColumnsAndTableName.get("templateBucket")); } -} \ No newline at end of file +} From 64dbbd0cae7521f4c41b3b94ea49bc57c110e73a Mon Sep 17 00:00:00 2001 From: AnnaHariprasad5123 Date: Thu, 6 Jun 2024 17:53:10 +0530 Subject: [PATCH 4/7] Fix: wildcard imports in datasourceutils --- .../src/main/java/com/external/utils/DatasourceUtils.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java index 7eb62d1d0b91..010287ab4644 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java @@ -20,7 +20,11 @@ import static com.amazonaws.regions.Regions.DEFAULT_REGION; import static com.appsmith.external.helpers.PluginUtils.getValueSafelyFromPropertyList; -import static com.external.plugins.constants.S3PluginConstants.*; +import static com.external.plugins.constants.S3PluginConstants.AUTO; +import static com.external.plugins.constants.S3PluginConstants.CUSTOM_ENDPOINT_INDEX; +import static com.external.plugins.constants.S3PluginConstants.CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX; +import static com.external.plugins.constants.S3PluginConstants.GOOGLE_CLOUD_SERVICE_PROVIDER; +import static com.external.plugins.constants.S3PluginConstants.S3_SERVICE_PROVIDER_PROPERTY_INDEX; import static com.external.utils.DatasourceUtils.S3ServiceProvider.AMAZON; public class DatasourceUtils { @@ -94,7 +98,7 @@ public static S3ServiceProvider fromString(String name) throws AppsmithPluginExc * @param datasourceConfiguration * @return AmazonS3ClientBuilder object * @throws AppsmithPluginException when (1) there is an error with parsing credentials (2) required - * datasourceConfiguration properties are missing (3) endpoint URL is found incorrect. + * datasourceConfiguration properties are missing (3) endpoint URL is found incorrect. */ public static AmazonS3ClientBuilder getS3ClientBuilder(DatasourceConfiguration datasourceConfiguration) throws AppsmithPluginException { From 5e2bfcd1260de274a32fedc90f607a2e14a42b46 Mon Sep 17 00:00:00 2001 From: AnnaHariprasad5123 Date: Fri, 7 Jun 2024 06:39:26 +0530 Subject: [PATCH 5/7] Fix: done changes --- .../com/external/utils/DatasourceUtils.java | 12 +++---- .../external/plugins/AmazonS3PluginTest.java | 32 ++++++++++++++++++- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java index 010287ab4644..715e6a5df148 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java @@ -23,7 +23,6 @@ import static com.external.plugins.constants.S3PluginConstants.AUTO; import static com.external.plugins.constants.S3PluginConstants.CUSTOM_ENDPOINT_INDEX; import static com.external.plugins.constants.S3PluginConstants.CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX; -import static com.external.plugins.constants.S3PluginConstants.GOOGLE_CLOUD_SERVICE_PROVIDER; import static com.external.plugins.constants.S3PluginConstants.S3_SERVICE_PROVIDER_PROPERTY_INDEX; import static com.external.utils.DatasourceUtils.S3ServiceProvider.AMAZON; @@ -158,13 +157,6 @@ public static AmazonS3ClientBuilder getS3ClientBuilder(DatasourceConfiguration d */ if (s3ServiceProvider.equals(AMAZON)) { s3ClientBuilder = s3ClientBuilder.withRegion(DEFAULT_REGION).enableForceGlobalBucketAccess(); - } else if (s3ServiceProvider.equals(GOOGLE_CLOUD_SERVICE_PROVIDER)) { - String endpoint = datasourceConfiguration - .getEndpoints() - .get(CUSTOM_ENDPOINT_INDEX) - .getHost(); - s3ClientBuilder = s3ClientBuilder.withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(endpoint, AUTO)); } else { String endpoint = datasourceConfiguration .getEndpoints() @@ -178,6 +170,10 @@ public static AmazonS3ClientBuilder getS3ClientBuilder(DatasourceConfiguration d completeness. */ break; + case GOOGLE_CLOUD_STORAGE: + s3ClientBuilder = s3ClientBuilder.withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(endpoint, AUTO)); + break; case UPCLOUD: region = getRegionFromEndpointPattern( endpoint, UPCLOUD_URL_ENDPOINT_PATTERN, UPCLOUD_REGION_GROUP_INDEX); diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java b/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java index fc33d5ece6fd..9b3872097c1a 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java @@ -70,7 +70,11 @@ import static com.external.plugins.constants.FieldName.READ_DATATYPE; import static com.external.plugins.constants.FieldName.READ_EXPIRY; import static com.external.plugins.constants.FieldName.SMART_SUBSTITUTION; -import static com.external.plugins.constants.S3PluginConstants.*; +import static com.external.plugins.constants.S3PluginConstants.DEFAULT_FILE_NAME; +import static com.external.plugins.constants.S3PluginConstants.DEFAULT_URL_EXPIRY_IN_MINUTES; +import static com.external.plugins.constants.S3PluginConstants.GOOGLE_CLOUD_SERVICE_PROVIDER; +import static com.external.plugins.constants.S3PluginConstants.NO; +import static com.external.plugins.constants.S3PluginConstants.YES; import static com.external.utils.DatasourceUtils.getS3ClientBuilder; import static com.external.utils.TemplateUtils.CREATE_FILE_TEMPLATE_NAME; import static com.external.utils.TemplateUtils.CREATE_MULTIPLE_FILES_TEMPLATE_NAME; @@ -221,6 +225,32 @@ public void testValidateDatasourceWithMissingRegionWithNonAmazonProvider() { .verifyComplete(); } + @Test + public void testValidateDatasourceWithNonAmazonProvider() { + DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); + + datasourceConfiguration.getProperties().get(1).setValue(GOOGLE_CLOUD_SERVICE_PROVIDER); + + Property defaultBucketProperty = new Property("default bucket", "default-bucket"); + datasourceConfiguration.getProperties().add(defaultBucketProperty); + + AmazonS3Plugin.S3PluginExecutor pluginExecutor = new AmazonS3Plugin.S3PluginExecutor(); + Mono pluginExecutorMono = Mono.just(pluginExecutor); + + StepVerifier.create(pluginExecutorMono) + .assertNext(executor -> { + Set res = executor.validateDatasource(datasourceConfiguration); + assertFalse(res.contains(S3ErrorMessages.DS_AT_LEAST_ONE_MANDATORY_PARAMETER_MISSING_ERROR_MSG)); + assertFalse(res.contains(S3ErrorMessages.DS_MANDATORY_PARAMETER_DEFAULT_BUCKET_MISSING_ERROR_MSG)); + assertFalse(res.contains(S3ErrorMessages.DS_MANDATORY_PARAMETER_SECRET_KEY_MISSING_ERROR_MSG)); + assertFalse(res.contains(S3ErrorMessages.DS_MANDATORY_PARAMETER_ACCESS_KEY_MISSING_ERROR_MSG)); + assertFalse(res.contains(S3ErrorMessages.DS_MANDATORY_PARAMETER_ENDPOINT_URL_MISSING_ERROR_MSG)); + assertFalse(res.contains(S3ErrorMessages.NON_EXITED_BUCKET_ERROR_MSG)); + assertTrue(res.isEmpty()); + }) + .verifyComplete(); + } + @Test public void testValidateDatasourceWithMissingRegionWithListedProvider() { DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); From e73a14bec455b7a1401fd8ece21a6bedca6a90f5 Mon Sep 17 00:00:00 2001 From: AnnaHariprasad5123 Date: Fri, 7 Jun 2024 14:19:55 +0530 Subject: [PATCH 6/7] fix: refactor code --- .../src/main/java/com/external/utils/DatasourceUtils.java | 3 +-- .../test/java/com/external/plugins/AmazonS3PluginTest.java | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java index 715e6a5df148..96b5a3b5f500 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java @@ -171,8 +171,7 @@ public static AmazonS3ClientBuilder getS3ClientBuilder(DatasourceConfiguration d break; case GOOGLE_CLOUD_STORAGE: - s3ClientBuilder = s3ClientBuilder.withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(endpoint, AUTO)); + region = AUTO; break; case UPCLOUD: region = getRegionFromEndpointPattern( diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java b/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java index 9b3872097c1a..cbeb5ec9e51f 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java @@ -204,7 +204,7 @@ public void testValidateDatasourceWithMissingRegionWithOtherS3ServiceProvider() } @Test - public void testValidateDatasourceWithMissingRegionWithNonAmazonProvider() { + public void testValidateDatasourceWithMissingRegionAndDefaultBucketWithNonAmazonProvider() { DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); datasourceConfiguration.getProperties().get(1).setValue(GOOGLE_CLOUD_SERVICE_PROVIDER); @@ -226,7 +226,7 @@ public void testValidateDatasourceWithMissingRegionWithNonAmazonProvider() { } @Test - public void testValidateDatasourceWithNonAmazonProvider() { + public void testValidateDatasourceWithMissingRegionWithNonAmazonProvider() { DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); datasourceConfiguration.getProperties().get(1).setValue(GOOGLE_CLOUD_SERVICE_PROVIDER); From baae8b6dc01bfe2a12006cb75c93cbb3917d1248 Mon Sep 17 00:00:00 2001 From: AnnaHariprasad5123 Date: Tue, 11 Jun 2024 15:12:01 +0530 Subject: [PATCH 7/7] Modify deleteMultipleObjects for GCS --- .../com/external/plugins/AmazonS3Plugin.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java index 7b2e28f2c624..b4166191fe8c 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java @@ -771,7 +771,12 @@ private Mono executeCommon( requestParams.add( new RequestParamDTO(ACTION_CONFIGURATION_PATH, path, null, null, null)); - deleteMultipleObjects(connection, bucketName, path); + List properties = datasourceConfiguration.getProperties(); + String s3Provider = (String) properties + .get(S3_SERVICE_PROVIDER_PROPERTY_INDEX) + .getValue(); + + deleteMultipleObjects(s3Provider, connection, bucketName, path); actionResult = Map.of("status", "All files deleted successfully"); break; /** @@ -830,7 +835,7 @@ private Mono executeCommon( .subscribeOn(scheduler); } - private void deleteMultipleObjects(AmazonS3 connection, String bucketName, String path) + private void deleteMultipleObjects(String s3Provider, AmazonS3 connection, String bucketName, String path) throws AppsmithPluginException { List listOfFiles; try { @@ -844,7 +849,13 @@ private void deleteMultipleObjects(AmazonS3 connection, String bucketName, Strin DeleteObjectsRequest deleteObjectsRequest = getDeleteObjectsRequest(bucketName, listOfFiles); try { - connection.deleteObjects(deleteObjectsRequest); + if (GOOGLE_CLOUD_SERVICE_PROVIDER.equals(s3Provider)) { + for (String filePath : listOfFiles) { + connection.deleteObject(bucketName, filePath); + } + } else { + connection.deleteObjects(deleteObjectsRequest); + } } catch (SdkClientException e) { throw new AppsmithPluginException( S3PluginError.AMAZON_S3_QUERY_EXECUTION_FAILED,