diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java index 7f71cfbfda20..562a3b834433 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/AmazonS3Plugin.java @@ -99,12 +99,14 @@ import static com.external.plugins.constants.FieldName.PATH; import static com.external.plugins.constants.FieldName.READ_DATATYPE; import static com.external.plugins.constants.FieldName.SMART_SUBSTITUTION; -import static com.external.plugins.constants.S3PluginConstants.ACCESS_DENIED_ERROR_CODE; +import static com.external.plugins.constants.S3PluginConstants.*; import static com.external.plugins.constants.S3PluginConstants.AWS_S3_SERVICE_PROVIDER; import static com.external.plugins.constants.S3PluginConstants.BASE64_DELIMITER; import static com.external.plugins.constants.S3PluginConstants.CUSTOM_ENDPOINT_INDEX; +import static com.external.plugins.constants.S3PluginConstants.DEFAULT_BUCKET_PROPERTY_INDEX; import static com.external.plugins.constants.S3PluginConstants.DEFAULT_FILE_NAME; import static com.external.plugins.constants.S3PluginConstants.DEFAULT_URL_EXPIRY_IN_MINUTES; +import static com.external.plugins.constants.S3PluginConstants.GOOGLE_CLOUD_SERVICE_PROVIDER; import static com.external.plugins.constants.S3PluginConstants.NO; import static com.external.plugins.constants.S3PluginConstants.S3_DRIVER; import static com.external.plugins.constants.S3PluginConstants.S3_SERVICE_PROVIDER_PROPERTY_INDEX; @@ -948,36 +950,97 @@ public Set validateDatasource(DatasourceConfiguration datasourceConfigur invalids.add(S3ErrorMessages.DS_MANDATORY_PARAMETER_ENDPOINT_URL_MISSING_ERROR_MSG); } + if (datasourceConfiguration != null) { + String serviceProvider = (String) + properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue(); + if (GOOGLE_CLOUD_SERVICE_PROVIDER.equals(serviceProvider)) { + String defaultBucket = (String) + properties.get(DEFAULT_BUCKET_PROPERTY_INDEX).getValue(); + if (StringUtils.isNullOrEmpty(defaultBucket)) { + invalids.add(S3ErrorMessages.DS_MANDATORY_PARAMETER_DEFAULT_BUCKET_MISSING_ERROR_MSG); + } + } + } + return invalids; } @Override - public Mono testDatasource(AmazonS3 connection) { - return Mono.fromCallable(() -> { - /* - * - Please note that as of 28 Jan 2021, the way AmazonS3 client works, creating a connection - * object with wrong credentials does not throw any exception. - * - Hence, adding a listBuckets() method call to test the connection. - */ - connection.listBuckets(); - return new DatasourceTestResult(); - }) - .onErrorResume(error -> { - if (error instanceof AmazonS3Exception - && ACCESS_DENIED_ERROR_CODE.equals(((AmazonS3Exception) error).getErrorCode())) { - /** - * Sometimes a valid account credential may not have permission to run listBuckets action - * . In this case `AccessDenied` error is returned. - * That fact that the credentials caused `AccessDenied` error instead of invalid access key - * id or signature mismatch error means that the credentials are valid, we are able to - * establish a connection as well, but the account does not have permission to run - * listBuckets. - */ - return Mono.just(new DatasourceTestResult()); - } + public Mono testDatasource(DatasourceConfiguration datasourceConfiguration) { + if (datasourceConfiguration == null) { + return Mono.just(new DatasourceTestResult( + S3ErrorMessages.DS_AT_LEAST_ONE_MANDATORY_PARAMETER_MISSING_ERROR_MSG)); + } + + List properties = datasourceConfiguration.getProperties(); + String s3Provider = + (String) properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue(); + + // Handle Google Cloud Storage in a separate method if necessary + if (GOOGLE_CLOUD_SERVICE_PROVIDER.equals(s3Provider)) { + return testGoogleCloudStorage(datasourceConfiguration); + } - return Mono.just(new DatasourceTestResult(amazonS3ErrorUtils.getReadableError(error))); - }); + // For Amazon S3 or other providers, perform a standard test by listing buckets in Amazon S3 + return datasourceCreate(datasourceConfiguration) + .flatMap(connection -> Mono.fromCallable(() -> { + /* + * - Please note that as of 28 Jan 2021, the way AmazonS3 client works, creating a connection + * object with wrong credentials does not throw any exception. + * - Hence, adding a listBuckets() method call to test the connection. + */ + connection.listBuckets(); + return new DatasourceTestResult(); + }) + .onErrorResume(error -> { + if (error instanceof AmazonS3Exception + && ACCESS_DENIED_ERROR_CODE.equals( + ((AmazonS3Exception) error).getErrorCode())) { + /** + * Sometimes a valid account credential may not have permission to run listBuckets action + * . In this case `AccessDenied` error is returned. + * That fact that the credentials caused `AccessDenied` error instead of invalid access key + * id or signature mismatch error means that the credentials are valid, we are able to + * establish a connection as well, but the account does not have permission to run + * listBuckets. + */ + return Mono.just(new DatasourceTestResult()); + } + + return Mono.just(new DatasourceTestResult(amazonS3ErrorUtils.getReadableError(error))); + }) + .doFinally(signalType -> connection.shutdown())) + .onErrorResume(error -> Mono.just(new DatasourceTestResult(error.getMessage()))) + .subscribeOn(scheduler); + } + + private Mono testGoogleCloudStorage(DatasourceConfiguration datasourceConfiguration) { + List properties = datasourceConfiguration.getProperties(); + String defaultBucket = + (String) properties.get(DEFAULT_BUCKET_PROPERTY_INDEX).getValue(); + if (StringUtils.isNullOrEmpty(defaultBucket)) { + return Mono.just(new DatasourceTestResult( + S3ErrorMessages.DS_MANDATORY_PARAMETER_DEFAULT_BUCKET_MISSING_ERROR_MSG)); + } + + return datasourceCreate(datasourceConfiguration) + .flatMap(connection -> Mono.fromCallable(() -> { + connection.listObjects(defaultBucket); + return new DatasourceTestResult(); + }) + .onErrorResume(error -> { + if (error instanceof AmazonS3Exception + && ((AmazonS3Exception) error).getStatusCode() == 404) { + return Mono.just( + new DatasourceTestResult(S3ErrorMessages.NON_EXITED_BUCKET_ERROR_MSG)); + } else { + return Mono.just( + new DatasourceTestResult(amazonS3ErrorUtils.getReadableError(error))); + } + }) + .doFinally(signalType -> connection.shutdown())) + .onErrorResume(error -> Mono.just(new DatasourceTestResult(error.getMessage()))) + .subscribeOn(scheduler); } /** @@ -1098,4 +1161,4 @@ public Mono sanitizeGenerateCRUDPageTemplateInfo( return Mono.empty(); } } -} +} \ No newline at end of file diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/S3PluginConstants.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/S3PluginConstants.java index 74132ff566a1..ad4e6c3ff221 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/S3PluginConstants.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/constants/S3PluginConstants.java @@ -4,6 +4,8 @@ public class S3PluginConstants { public static final String S3_DRIVER = "com.amazonaws.services.s3.AmazonS3"; public static final int S3_SERVICE_PROVIDER_PROPERTY_INDEX = 1; public static final int CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX = 2; + + public static final int DEFAULT_BUCKET_PROPERTY_INDEX = 3; public static final int CUSTOM_ENDPOINT_INDEX = 0; public static final String DEFAULT_URL_EXPIRY_IN_MINUTES = "5"; // max 7 days is possible public static final String YES = "YES"; @@ -12,4 +14,6 @@ public class S3PluginConstants { public static final String AWS_S3_SERVICE_PROVIDER = "amazon-s3"; public static String DEFAULT_FILE_NAME = "MyFile.txt"; public static final String ACCESS_DENIED_ERROR_CODE = "AccessDenied"; + public static final String GOOGLE_CLOUD_SERVICE_PROVIDER = "google-cloud-storage"; + public static final String AUTO = "auto"; } diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/exceptions/S3ErrorMessages.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/exceptions/S3ErrorMessages.java index aefb7e796bbf..945a47bf55e8 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/exceptions/S3ErrorMessages.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/plugins/exceptions/S3ErrorMessages.java @@ -18,6 +18,9 @@ public class S3ErrorMessages extends BasePluginErrorMessages { "Appsmith has encountered an unexpected error when getting bucket name. Please reach out to " + "Appsmith customer support to resolve this."; + public static final String NON_EXITED_BUCKET_ERROR_MSG = + "Appsmith has encountered an unexpected error when getting bucket name. The specified bucket does not exist in Google Cloud Storage. "; + public static final String EMPTY_PREFIX_ERROR_MSG = "Appsmith has encountered an unexpected error when getting path prefix. Please reach out to " + "Appsmith customer support to resolve this."; @@ -126,4 +129,9 @@ public class S3ErrorMessages extends BasePluginErrorMessages { "Required parameter 'Endpoint URL' is empty. Did you forget to edit the 'Endpoint" + " URL' field in the datasource creation form ? You need to fill it with " + "the endpoint URL of your S3 instance."; + + public static final String DS_MANDATORY_PARAMETER_DEFAULT_BUCKET_MISSING_ERROR_MSG = + "Required parameter 'Default Bucket' is empty. Did you forget to edit the 'Default Bucket' " + + "field in the datasource creation form? You need to fill it with the default bucket name " + + "for your Google Cloud Storage instance."; } diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java index c53798f52708..96b5a3b5f500 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/java/com/external/utils/DatasourceUtils.java @@ -20,6 +20,7 @@ import static com.amazonaws.regions.Regions.DEFAULT_REGION; import static com.appsmith.external.helpers.PluginUtils.getValueSafelyFromPropertyList; +import static com.external.plugins.constants.S3PluginConstants.AUTO; import static com.external.plugins.constants.S3PluginConstants.CUSTOM_ENDPOINT_INDEX; import static com.external.plugins.constants.S3PluginConstants.CUSTOM_ENDPOINT_REGION_PROPERTY_INDEX; import static com.external.plugins.constants.S3PluginConstants.S3_SERVICE_PROVIDER_PROPERTY_INDEX; @@ -67,6 +68,7 @@ public enum S3ServiceProvider { DIGITAL_OCEAN_SPACES("digital-ocean-spaces"), DREAM_OBJECTS("dream-objects"), MINIO("minio"), + GOOGLE_CLOUD_STORAGE("google-cloud-storage"), OTHER("other"); private String name; @@ -95,7 +97,7 @@ public static S3ServiceProvider fromString(String name) throws AppsmithPluginExc * @param datasourceConfiguration * @return AmazonS3ClientBuilder object * @throws AppsmithPluginException when (1) there is an error with parsing credentials (2) required - * datasourceConfiguration properties are missing (3) endpoint URL is found incorrect. + * datasourceConfiguration properties are missing (3) endpoint URL is found incorrect. */ public static AmazonS3ClientBuilder getS3ClientBuilder(DatasourceConfiguration datasourceConfiguration) throws AppsmithPluginException { @@ -136,7 +138,6 @@ public static AmazonS3ClientBuilder getS3ClientBuilder(DatasourceConfiguration d S3ServiceProvider s3ServiceProvider = S3ServiceProvider.fromString( (String) properties.get(S3_SERVICE_PROVIDER_PROPERTY_INDEX).getValue()); - /** * AmazonS3 provides an attribute `forceGlobalBucketAccessEnabled` that automatically routes the request to a * region such that request should succeed. @@ -168,6 +169,9 @@ public static AmazonS3ClientBuilder getS3ClientBuilder(DatasourceConfiguration d /* This case can never be reached because of the if condition above. Just adding for sake of completeness. */ + break; + case GOOGLE_CLOUD_STORAGE: + region = AUTO; break; case UPCLOUD: region = getRegionFromEndpointPattern( @@ -233,8 +237,8 @@ private static String getUserProvidedRegion(List properties) { /** * This method checks if the S3 endpoint URL has correct format and extracts region information from it. * - * @param endpoint : endpoint URL - * @param regex : expected endpoint URL pattern + * @param endpoint : endpoint URL + * @param regex : expected endpoint URL pattern * @param regionGroupIndex : pattern group index for region string * @return S3 object storage region. * @throws AppsmithPluginException when then endpoint URL does not match the expected regex pattern. diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/form.json b/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/form.json index 56e8c272dc80..aca975ff4e43 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/form.json +++ b/app/server/appsmith-plugins/amazons3Plugin/src/main/resources/form.json @@ -43,6 +43,10 @@ "label": "MinIO", "value": "minio" }, + { + "label": "Google Cloud Storage", + "value": "google-cloud-storage" + }, { "label": "Other", "value": "other" @@ -106,6 +110,19 @@ } ] } + }, + { + "label": "Default Bucket", + "configProperty": "datasourceConfiguration.properties[3].value", + "controlType": "INPUT_TEXT", + "initialValue": "", + "placeholderText": "Enter default bucket name", + "isRequired": true, + "hidden": { + "path": "datasourceConfiguration.properties[1].value", + "comparison": "NOT_EQUALS", + "value": "google-cloud-storage" + } } ] } diff --git a/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java b/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java index fab27b5cd544..cbeb5ec9e51f 100644 --- a/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java +++ b/app/server/appsmith-plugins/amazons3Plugin/src/test/java/com/external/plugins/AmazonS3PluginTest.java @@ -72,6 +72,7 @@ import static com.external.plugins.constants.FieldName.SMART_SUBSTITUTION; import static com.external.plugins.constants.S3PluginConstants.DEFAULT_FILE_NAME; import static com.external.plugins.constants.S3PluginConstants.DEFAULT_URL_EXPIRY_IN_MINUTES; +import static com.external.plugins.constants.S3PluginConstants.GOOGLE_CLOUD_SERVICE_PROVIDER; import static com.external.plugins.constants.S3PluginConstants.NO; import static com.external.plugins.constants.S3PluginConstants.YES; import static com.external.utils.DatasourceUtils.getS3ClientBuilder; @@ -202,6 +203,54 @@ public void testValidateDatasourceWithMissingRegionWithOtherS3ServiceProvider() .verifyComplete(); } + @Test + public void testValidateDatasourceWithMissingRegionAndDefaultBucketWithNonAmazonProvider() { + DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); + + datasourceConfiguration.getProperties().get(1).setValue(GOOGLE_CLOUD_SERVICE_PROVIDER); + + Property defaultBucketProperty = new Property("default bucket", ""); + datasourceConfiguration.getProperties().add(defaultBucketProperty); + + AmazonS3Plugin.S3PluginExecutor pluginExecutor = new AmazonS3Plugin.S3PluginExecutor(); + Mono pluginExecutorMono = Mono.just(pluginExecutor); + + StepVerifier.create(pluginExecutorMono) + .assertNext(executor -> { + Set res = executor.validateDatasource(datasourceConfiguration); + // There should be no errors related to the region for GCS, but it should validate the default + // bucket + assertTrue(res.contains(S3ErrorMessages.DS_MANDATORY_PARAMETER_DEFAULT_BUCKET_MISSING_ERROR_MSG)); + }) + .verifyComplete(); + } + + @Test + public void testValidateDatasourceWithMissingRegionWithNonAmazonProvider() { + DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration(); + + datasourceConfiguration.getProperties().get(1).setValue(GOOGLE_CLOUD_SERVICE_PROVIDER); + + Property defaultBucketProperty = new Property("default bucket", "default-bucket"); + datasourceConfiguration.getProperties().add(defaultBucketProperty); + + AmazonS3Plugin.S3PluginExecutor pluginExecutor = new AmazonS3Plugin.S3PluginExecutor(); + Mono pluginExecutorMono = Mono.just(pluginExecutor); + + StepVerifier.create(pluginExecutorMono) + .assertNext(executor -> { + Set res = executor.validateDatasource(datasourceConfiguration); + assertFalse(res.contains(S3ErrorMessages.DS_AT_LEAST_ONE_MANDATORY_PARAMETER_MISSING_ERROR_MSG)); + assertFalse(res.contains(S3ErrorMessages.DS_MANDATORY_PARAMETER_DEFAULT_BUCKET_MISSING_ERROR_MSG)); + assertFalse(res.contains(S3ErrorMessages.DS_MANDATORY_PARAMETER_SECRET_KEY_MISSING_ERROR_MSG)); + assertFalse(res.contains(S3ErrorMessages.DS_MANDATORY_PARAMETER_ACCESS_KEY_MISSING_ERROR_MSG)); + assertFalse(res.contains(S3ErrorMessages.DS_MANDATORY_PARAMETER_ENDPOINT_URL_MISSING_ERROR_MSG)); + assertFalse(res.contains(S3ErrorMessages.NON_EXITED_BUCKET_ERROR_MSG)); + assertTrue(res.isEmpty()); + }) + .verifyComplete(); + } + @Test public void testValidateDatasourceWithMissingRegionWithListedProvider() { DatasourceConfiguration datasourceConfiguration = createDatasourceConfiguration();