diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ce/FieldNameCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ce/FieldNameCE.java index 4f96dee7d21d..f11aed539049 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ce/FieldNameCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/ce/FieldNameCE.java @@ -202,4 +202,6 @@ public class FieldNameCE { public static final String ARTIFACT_CONTEXT = "artifactContext"; public static final String ARTIFACT_ID = "artifactId"; public static final String BODY = "body"; + + public static final String CREATE = "save"; } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCE.java index 7e34656f20e9..b2f2953e4dce 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCE.java @@ -10,6 +10,8 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import java.util.List; +import java.util.Map; import java.util.Set; public interface DatasourceServiceCE { @@ -36,7 +38,7 @@ public interface DatasourceServiceCE { Mono> extractKeysFromDatasource(Datasource datasource); - Mono save(Datasource datasource); + Mono save(Datasource datasource, boolean isDryOps); /** * Retrieves all datasources based on input params, currently only workspaceId. @@ -58,10 +60,15 @@ public interface DatasourceServiceCE { */ Flux getAllByWorkspaceIdWithStorages(String workspaceId, AclPermission permission); + Flux saveAll(List datasourceList); + Mono create(Datasource datasource); Mono createWithoutPermissions(Datasource datasource); + Mono createWithoutPermissions( + Datasource datasource, Map> datasourceStorageDryRunQueries); + Mono updateDatasourceStorage( DatasourceStorageDTO datasourceStorageDTO, String activeEnvironmentId, Boolean IsUserRefreshedUpdate); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCEImpl.java index dbd201095448..c8dc4f898196 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCEImpl.java @@ -19,6 +19,7 @@ import com.appsmith.server.domains.Plugin; import com.appsmith.server.domains.User; import com.appsmith.server.domains.Workspace; +import com.appsmith.server.dtos.DBOpsType; import com.appsmith.server.exceptions.AppsmithError; import com.appsmith.server.exceptions.AppsmithException; import com.appsmith.server.helpers.PluginExecutorHelper; @@ -140,16 +141,26 @@ public DatasourceServiceCEImpl( @Override public Mono create(Datasource datasource) { - return createEx(datasource, workspacePermission.getDatasourceCreatePermission()); + return createEx(datasource, workspacePermission.getDatasourceCreatePermission(), false, null); } // TODO: Check usage + @Override + public Mono createWithoutPermissions( + Datasource datasource, Map> datasourceStorageDryRunQueries) { + return createEx(datasource, null, true, datasourceStorageDryRunQueries); + } + @Override public Mono createWithoutPermissions(Datasource datasource) { - return createEx(datasource, null); + return createEx(datasource, null, false, null); } - private Mono createEx(@NotNull Datasource datasource, AclPermission permission) { + private Mono createEx( + @NotNull Datasource datasource, + AclPermission permission, + boolean isDryOps, + Map> datasourceStorageDryRunQueries) { // Validate incoming request String workspaceId = datasource.getWorkspaceId(); if (!hasText(workspaceId)) { @@ -193,7 +204,7 @@ private Mono createEx(@NotNull Datasource datasource, AclPermission Mono userMono = sessionUserService.getCurrentUser(); return generateAndSetDatasourcePolicies(userMono, datasource1, permission); }) - .flatMap(this::validateAndSaveDatasourceToRepository) + .flatMap(datasourceInDb -> validateAndSaveDatasourceToRepository(datasourceInDb, isDryOps)) .flatMap(savedDatasource -> analyticsService.sendCreateEvent(savedDatasource, getAnalyticsProperties(savedDatasource))); } else { @@ -217,7 +228,16 @@ private Mono createEx(@NotNull Datasource datasource, AclPermission return Mono.just(datasourceStorage); } - return datasourceStorageService.create(datasourceStorage); + return datasourceStorageService + .create(datasourceStorage, isDryOps) + .map(datasourceStorage1 -> { + if (datasourceStorageDryRunQueries != null && isDryOps) { + datasourceStorageDryRunQueries + .computeIfAbsent(DBOpsType.SAVE.name(), k -> new ArrayList<>()) + .add(datasourceStorage1); + } + return datasourceStorage1; + }); }) .map(datasourceStorageService::createDatasourceStorageDTOFromDatasourceStorage) .collectMap(DatasourceStorageDTO::getEnvironmentId) @@ -303,7 +323,7 @@ public Mono updateDatasource( copyNestedNonNullProperties(datasource, datasourceInDb); return datasourceInDb; }) - .flatMap(this::validateAndSaveDatasourceToRepository) + .flatMap(datasourceInDb -> validateAndSaveDatasourceToRepository(datasourceInDb, false)) .map(savedDatasource -> { // not required by client side in order to avoid updating it to a null storage, // one alternative is that we find and send datasourceStorages along, but that is an expensive call @@ -355,7 +375,7 @@ public Mono updateDatasourceStorage( datasourceStorage.prepareTransientFields(dbDatasource); return datasourceStorageService - .updateDatasourceStorage(datasourceStorage, activeEnvironmentId, Boolean.TRUE) + .updateDatasourceStorage(datasourceStorage, activeEnvironmentId, Boolean.TRUE, false) .map(datasourceStorageService::createDatasourceStorageDTOFromDatasourceStorage) .map(datasourceStorageDTO1 -> { dbDatasource.getDatasourceStorages().put(trueEnvironmentId, datasourceStorageDTO1); @@ -365,20 +385,31 @@ public Mono updateDatasourceStorage( } @Override - public Mono save(Datasource datasource) { + public Mono save(Datasource datasource, boolean isDryOps) { if (datasource.getGitSyncId() == null) { datasource.setGitSyncId( datasource.getWorkspaceId() + "_" + Instant.now().toString()); } + if (isDryOps) { + datasource.updateForBulkWriteOperation(); + return Mono.just(datasource); + } return repository.save(datasource); } - private Mono validateAndSaveDatasourceToRepository(Datasource datasource) { + private Mono validateAndSaveDatasourceToRepository(Datasource datasource, boolean isDryOps) { return Mono.just(datasource) .flatMap(this::validateDatasource) .flatMap(unsavedDatasource -> { - return repository.save(unsavedDatasource).map(savedDatasource -> { + Mono datasourceMono; + if (isDryOps) { + unsavedDatasource.updateForBulkWriteOperation(); + datasourceMono = Mono.just(unsavedDatasource); + } else { + datasourceMono = repository.save(unsavedDatasource); + } + return datasourceMono.map(savedDatasource -> { // datasource.pluginName is a transient field. It was set by validateDatasource method // object from db will have pluginName=null so set it manually from the unsaved datasource obj savedDatasource.setPluginName(unsavedDatasource.getPluginName()); @@ -852,6 +883,15 @@ public Mono archiveById(String id) { }); } + @Override + public Flux saveAll(List datasourceList) { + datasourceList.stream() + .filter(datasource -> datasource.getGitSyncId() == null) + .forEach(datasource -> datasource.setGitSyncId( + datasource.getWorkspaceId() + "_" + Instant.now().toString())); + return repository.saveAll(datasourceList); + } + private Mono findPluginExecutor(String pluginId) { final Mono pluginMono = pluginService.findById(pluginId).cache(); return pluginExecutorHelper diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/importable/DatasourceImportableServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/importable/DatasourceImportableServiceCEImpl.java index c4115be2bee0..69e6d7a7b25e 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/importable/DatasourceImportableServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasources/importable/DatasourceImportableServiceCEImpl.java @@ -15,6 +15,7 @@ import com.appsmith.server.domains.Artifact; import com.appsmith.server.domains.Workspace; import com.appsmith.server.dtos.ArtifactExchangeJson; +import com.appsmith.server.dtos.DBOpsType; import com.appsmith.server.dtos.ImportingMetaDTO; import com.appsmith.server.dtos.MappedImportableResourcesDTO; import com.appsmith.server.exceptions.AppsmithError; @@ -211,9 +212,16 @@ private Mono> importDatasources( // Don't update the datasource configuration for already available datasources existingDatasource.setDatasourceConfiguration(null); return datasourceService - .save(existingDatasource) - .map(createdDatasource -> - Tuples.of(createdDatasource.getName(), createdDatasource.getId())); + .save(existingDatasource, true) + .map(createdDatasource -> { + // Add dry run queries for the datasource + addDryOpsForEntity( + DBOpsType.SAVE, + mappedImportableResourcesDTO.getDatasourceDryRunQueries(), + createdDatasource); + return Tuples.of( + createdDatasource.getName(), createdDatasource.getId()); + }); } else { // This is explicitly copied over from the map we created before datasourceStorage.setPluginId(pluginMap.get(datasourceStorage.getPluginId())); @@ -235,9 +243,16 @@ private Mono> importDatasources( datasourceStorage, workspace, environmentId, - importingMetaDTO.getPermissionProvider()) - .map(createdDatasource -> - Tuples.of(importedDatasourceName, createdDatasource.getId())); + importingMetaDTO.getPermissionProvider(), + mappedImportableResourcesDTO) + .map(createdDatasource -> { + // Add dry run queries for the datasource + addDryOpsForEntity( + DBOpsType.SAVE, + mappedImportableResourcesDTO.getDatasourceDryRunQueries(), + createdDatasource); + return Tuples.of(importedDatasourceName, createdDatasource.getId()); + }); } }); }) @@ -266,7 +281,8 @@ private Mono createUniqueDatasourceIfNotPresent( DatasourceStorage datasourceStorage, Workspace workspace, String environmentId, - ImportArtifactPermissionProvider permissionProvider) { + ImportArtifactPermissionProvider permissionProvider, + MappedImportableResourcesDTO mappedImportableResourcesDTO) { /* 1. If same datasource is present return 2. If unable to find the datasource create a new datasource with unique name and return @@ -311,11 +327,15 @@ private Mono createUniqueDatasourceIfNotPresent( getUniqueSuffixForDuplicateNameEntity(duplicateNameDatasource, workspace.getId())) .map(dsName -> { datasourceStorage.setName(datasourceStorage.getName() + dsName); + return datasourceService.createDatasourceFromDatasourceStorage(datasourceStorage); }) .switchIfEmpty(Mono.just( datasourceService.createDatasourceFromDatasourceStorage(datasourceStorage))) - .flatMap(datasourceService::createWithoutPermissions); + // DRY RUN queries are not saved, so we need to create them separately at the import service + // solution + .flatMap(datasource -> datasourceService.createWithoutPermissions( + datasource, mappedImportableResourcesDTO.getDatasourceStorageDryRunQueries())); })) .onErrorResume(throwable -> { log.error("failed to import datasource", throwable); @@ -386,4 +406,9 @@ private Mono getUniqueSuffixForDuplicateNameEntity(BaseDomain sourceEnti public Flux getEntitiesPresentInWorkspace(String workspaceId) { return datasourceService.getAllByWorkspaceIdWithStorages(workspaceId, null); } + + private void addDryOpsForEntity( + DBOpsType queryType, Map> dryRunOpsMap, Datasource createdDatasource) { + dryRunOpsMap.computeIfAbsent(queryType.name(), k -> new ArrayList<>()).add(createdDatasource); + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceCE.java index 0f9966566561..67252e8e67a5 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceCE.java @@ -15,6 +15,8 @@ public interface DatasourceStorageServiceCE { Mono create(DatasourceStorage datasourceStorage); + Mono create(DatasourceStorage datasourceStorage, boolean isDryOps); + Mono save(DatasourceStorage datasourceStorage); Mono archive(DatasourceStorage datasourceStorage); @@ -32,6 +34,12 @@ public interface DatasourceStorageServiceCE { Mono updateDatasourceStorage( DatasourceStorage datasourceStorage, String activeEnvironmentId, Boolean IsUserRefreshedUpdate); + Mono updateDatasourceStorage( + DatasourceStorage datasourceStorage, + String activeEnvironmentId, + Boolean IsUserRefreshedUpdate, + boolean isDryOps); + Mono validateDatasourceStorage(DatasourceStorage datasourceStorage); Mono validateDatasourceConfiguration(DatasourceStorage datasourceStorage); diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceCEImpl.java index 751a7fcf2814..556880330cc5 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceCEImpl.java @@ -58,7 +58,16 @@ public DatasourceStorageServiceCEImpl( @Override public Mono create(DatasourceStorage datasourceStorage) { return this.checkDuplicateDatasourceStorage(datasourceStorage) - .then(this.validateAndSaveDatasourceStorageToRepository(datasourceStorage)) + .then(validateAndSaveDatasourceStorageToRepository(datasourceStorage, false)) + .flatMap(this::populateHintMessages) // For REST API datasource create flow. + .flatMap(savedDatasourceStorage -> analyticsService.sendCreateEvent( + savedDatasourceStorage, getAnalyticsProperties(savedDatasourceStorage))); + } + + @Override + public Mono create(DatasourceStorage datasourceStorage, boolean isDryOps) { + return this.checkDuplicateDatasourceStorage(datasourceStorage) + .then(validateAndSaveDatasourceStorageToRepository(datasourceStorage, isDryOps)) .flatMap(this::populateHintMessages) // For REST API datasource create flow. .flatMap(savedDatasourceStorage -> analyticsService.sendCreateEvent( savedDatasourceStorage, getAnalyticsProperties(savedDatasourceStorage))); @@ -127,6 +136,15 @@ public Mono findStrictlyByDatasourceIdAndEnvironmentId( @Override public Mono updateDatasourceStorage( DatasourceStorage datasourceStorage, String activeEnvironmentId, Boolean isUserRefreshedUpdate) { + return updateDatasourceStorage(datasourceStorage, activeEnvironmentId, isUserRefreshedUpdate, false); + } + + @Override + public Mono updateDatasourceStorage( + DatasourceStorage datasourceStorage, + String activeEnvironmentId, + Boolean isUserRefreshedUpdate, + boolean isDryOps) { String datasourceId = datasourceStorage.getDatasourceId(); String environmentId = datasourceStorage.getEnvironmentId(); @@ -143,7 +161,8 @@ public Mono updateDatasourceStorage( } return dbStorage; }) - .flatMap(this::validateAndSaveDatasourceStorageToRepository) + .flatMap(datasourceStorage1 -> + validateAndSaveDatasourceStorageToRepository(datasourceStorage1, isDryOps)) .flatMap(savedDatasourceStorage -> { Map analyticsProperties = getAnalyticsProperties(savedDatasourceStorage); Boolean isUserInvokedUpdate = TRUE.equals(isUserRefreshedUpdate) ? TRUE : FALSE; @@ -211,14 +230,20 @@ public Mono validateDatasourceConfiguration(DatasourceStorage }); } - private Mono validateAndSaveDatasourceStorageToRepository(DatasourceStorage datasourceStorage) { + private Mono validateAndSaveDatasourceStorageToRepository( + DatasourceStorage datasourceStorage, boolean isDryOps) { return Mono.just(datasourceStorage) .map(this::sanitizeDatasourceStorage) .flatMap(datasourceStorage1 -> validateDatasourceStorage(datasourceStorage1)) .flatMap(this::executePreSaveActions) - .flatMap(unsavedDatasourceStorage -> - repository.save(unsavedDatasourceStorage).thenReturn(unsavedDatasourceStorage)); + .flatMap(unsavedDatasourceStorage -> { + if (isDryOps) { + unsavedDatasourceStorage.updateForBulkWriteOperation(); + return Mono.just(unsavedDatasourceStorage); + } + return repository.save(unsavedDatasourceStorage).thenReturn(unsavedDatasourceStorage); + }); } @Override diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/DBOpsType.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/DBOpsType.java new file mode 100644 index 000000000000..5278a695e227 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/DBOpsType.java @@ -0,0 +1,9 @@ +package com.appsmith.server.dtos; + +public enum DBOpsType { + SAVE, + + UPDATE, + + DELETE +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedImportableResourcesCE_DTO.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedImportableResourcesCE_DTO.java index e9247ccfa658..58363cdf2ef5 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedImportableResourcesCE_DTO.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedImportableResourcesCE_DTO.java @@ -1,5 +1,7 @@ package com.appsmith.server.dtos.ce; +import com.appsmith.external.models.Datasource; +import com.appsmith.external.models.DatasourceStorage; import com.appsmith.server.domains.Context; import com.appsmith.server.dtos.CustomJSLibContextDTO; import com.appsmith.server.dtos.ImportActionCollectionResultDTO; @@ -43,4 +45,9 @@ public class MappedImportableResourcesCE_DTO { // This is being used to carry the resources from ArtifactExchangeJson Map resourceStoreFromArtifactExchangeJson = new HashMap<>(); + + // Dry ops queries + Map> datasourceDryRunQueries = new HashMap<>(); + + Map> datasourceStorageDryRunQueries = new HashMap<>(); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/ImportServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/ImportServiceCEImpl.java index 6c62de863c2c..ca662a3b2d83 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/ImportServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/ImportServiceCEImpl.java @@ -24,6 +24,7 @@ import com.appsmith.server.imports.importable.ImportableService; import com.appsmith.server.imports.internal.artifactbased.ArtifactBasedImportService; import com.appsmith.server.migrations.JsonSchemaMigration; +import com.appsmith.server.repositories.DryOperationRepository; import com.appsmith.server.repositories.PermissionGroupRepository; import com.appsmith.server.services.AnalyticsService; import com.appsmith.server.services.SessionUserService; @@ -66,6 +67,7 @@ public class ImportServiceCEImpl implements ImportServiceCE { private final GsonBuilder gsonBuilder; private final ArtifactExchangeJsonAdapter artifactExchangeJsonAdapter; private final JsonSchemaMigration jsonSchemaMigration; + private final DryOperationRepository dryOperationRepository; /** * This method provides the importService specific to the artifact based on the ArtifactType. @@ -502,6 +504,10 @@ private Mono importArtifactInWorkspace( return Mono.error( new AppsmithException(AppsmithError.GENERIC_JSON_IMPORT_ERROR, workspaceId, errorMessage)); }) + // execute dry run for datasource + .flatMap(importableArtifact -> dryOperationRepository + .executeAllDbOps(mappedImportableResourcesDTO) + .thenReturn(importableArtifact)) .as(transactionalOperator::transactional); final Mono resultMono = importMono diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/ImportServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/ImportServiceImpl.java index b325a019733f..f2cc3c6ef9fa 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/ImportServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/ImportServiceImpl.java @@ -9,6 +9,7 @@ import com.appsmith.server.imports.importable.ImportableService; import com.appsmith.server.imports.internal.artifactbased.ArtifactBasedImportService; import com.appsmith.server.migrations.JsonSchemaMigration; +import com.appsmith.server.repositories.DryOperationRepository; import com.appsmith.server.repositories.PermissionGroupRepository; import com.appsmith.server.services.AnalyticsService; import com.appsmith.server.services.SessionUserService; @@ -30,7 +31,8 @@ public ImportServiceImpl( ImportableService datasourceImportableService, GsonBuilder gsonBuilder, ArtifactExchangeJsonAdapter artifactExchangeJsonAdapter, - JsonSchemaMigration jsonSchemaMigration) { + JsonSchemaMigration jsonSchemaMigration, + DryOperationRepository dryOperationRepository) { super( applicationImportService, sessionUserService, @@ -42,6 +44,7 @@ public ImportServiceImpl( datasourceImportableService, gsonBuilder, artifactExchangeJsonAdapter, - jsonSchemaMigration); + jsonSchemaMigration, + dryOperationRepository); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/partial/PartialImportServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/partial/PartialImportServiceCEImpl.java index 2f18023cde8d..71a056833062 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/partial/PartialImportServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/partial/PartialImportServiceCEImpl.java @@ -35,6 +35,7 @@ import com.appsmith.server.newactions.base.NewActionService; import com.appsmith.server.newpages.base.NewPageService; import com.appsmith.server.refactors.applications.RefactoringService; +import com.appsmith.server.repositories.DryOperationRepository; import com.appsmith.server.repositories.PermissionGroupRepository; import com.appsmith.server.services.AnalyticsService; import com.appsmith.server.services.ApplicationPageService; @@ -99,6 +100,7 @@ public class PartialImportServiceCEImpl implements PartialImportServiceCE { private final DatasourceService datasourceService; private final CustomJSLibService customJSLibService; private final UpdateLayoutService updateLayoutService; + private final DryOperationRepository dryOperationRepository; @Override public Mono importResourceInPage( @@ -247,6 +249,10 @@ private Mono importResourceInPage( .thenReturn(application); }); }) + // execute dry run ops + .flatMap(importableArtifact -> dryOperationRepository + .executeAllDbOps(mappedImportableResourcesDTO) + .thenReturn(importableArtifact)) .flatMap(application -> { Map fieldNameValueMap = Map.of( FieldName.UNPUBLISHED_JS_LIBS_IDENTIFIER_IN_APPLICATION_CLASS, diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/partial/PartialImportServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/partial/PartialImportServiceImpl.java index 48fe93526e8c..0a45692b6be7 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/partial/PartialImportServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/imports/internal/partial/PartialImportServiceImpl.java @@ -16,6 +16,7 @@ import com.appsmith.server.newactions.base.NewActionService; import com.appsmith.server.newpages.base.NewPageService; import com.appsmith.server.refactors.applications.RefactoringService; +import com.appsmith.server.repositories.DryOperationRepository; import com.appsmith.server.repositories.PermissionGroupRepository; import com.appsmith.server.services.AnalyticsService; import com.appsmith.server.services.ApplicationPageService; @@ -66,7 +67,8 @@ public PartialImportServiceImpl( ActionCollectionService actionCollectionService, DatasourceService datasourceService, CustomJSLibService customJSLibService, - UpdateLayoutService updateLayoutService) { + UpdateLayoutService updateLayoutService, + DryOperationRepository dryOperationRepository) { super( importService, workspaceService, @@ -95,6 +97,7 @@ public PartialImportServiceImpl( actionCollectionService, datasourceService, customJSLibService, - updateLayoutService); + updateLayoutService, + dryOperationRepository); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceCEImpl.java index e529e95957b5..bd276c9110c6 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/base/NewActionServiceCEImpl.java @@ -1529,7 +1529,7 @@ private Mono updateDatasourcePolicyForPublicAction(NewAction action, Datasource updatedDatasource = policySolution.addPoliciesToExistingObject(datasourcePolicyMap, datasource); - return datasourceService.save(updatedDatasource); + return datasourceService.save(updatedDatasource, false); }); }); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/importable/NewActionImportableServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/importable/NewActionImportableServiceCEImpl.java index aba07e6176f2..25980f2d3715 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/importable/NewActionImportableServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/newactions/importable/NewActionImportableServiceCEImpl.java @@ -1,6 +1,7 @@ package com.appsmith.server.newactions.importable; import com.appsmith.external.models.ActionDTO; +import com.appsmith.external.models.Datasource; import com.appsmith.external.models.Policy; import com.appsmith.server.actioncollections.base.ActionCollectionService; import com.appsmith.server.constants.FieldName; @@ -36,6 +37,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties; @@ -317,6 +319,32 @@ private Mono createImportNewActionsMono( branchedNewAction, newAction); + // Check if the action has datasource present + Datasource datasource = + newAction.getUnpublishedAction().getDatasource(); + if (datasource == null || datasource.getPluginId() == null) { + // Since the datasource are not yet saved to db, if we don't update the action with + // correct datasource, + // the action ave will fail due to validation + final String datasourceId = Objects.requireNonNull(newAction + .getUnpublishedAction() + .getDatasource()) + .getId() + != null + ? newAction + .getUnpublishedAction() + .getDatasource() + .getId() + : ""; + datasource = mappedImportableResourcesDTO.getDatasourceDryRunQueries().values().stream() + .flatMap(List::stream) + .filter(ds -> ds.getId().equals(datasourceId)) + .findFirst() + .orElse(datasource); + datasource.setIsAutoGenerated(false); + newAction.getUnpublishedAction().setDatasource(datasource); + } + // Check if the action has gitSyncId and if it's already in DB if (existingArtifactContainsAction(actionsInCurrentArtifact, newAction)) { diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/DryOperationRepository.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/DryOperationRepository.java new file mode 100644 index 000000000000..d957efc073fc --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/DryOperationRepository.java @@ -0,0 +1,70 @@ +package com.appsmith.server.repositories; + +import com.appsmith.external.models.Datasource; +import com.appsmith.external.models.DatasourceStorage; +import com.appsmith.server.dtos.MappedImportableResourcesDTO; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +@RequiredArgsConstructor +public class DryOperationRepository { + + private final DatasourceRepository datasourceRepository; + + private final DatasourceStorageRepository datasourceStorageRepository; + + private Map, AppsmithRepository> repoByEntityClass; + + @PostConstruct + public void init() { + final Map, AppsmithRepository> map = new HashMap<>(); + map.put(Datasource.class, datasourceRepository); + map.put(DatasourceStorage.class, datasourceStorageRepository); + repoByEntityClass = Collections.unmodifiableMap(map); + } + + public AppsmithRepository getRepositoryForEntity(Class entityClass) { + return (AppsmithRepository) repoByEntityClass.get(entityClass); + } + + public Flux saveDatasourceToDb(List datasources) { + return datasourceRepository.saveAll(datasources); + } + + public Flux saveDatasourceStorageToDb(List datasourceStorage) { + return datasourceStorageRepository.saveAll(datasourceStorage); + } + + public Mono executeAllDbOps(MappedImportableResourcesDTO mappedImportableResourcesDTO) { + + Flux> datasourceFLux = Flux.fromIterable(mappedImportableResourcesDTO + .getDatasourceStorageDryRunQueries() + .keySet()) + .flatMap(key -> { + List datasourceList = mappedImportableResourcesDTO + .getDatasourceDryRunQueries() + .get(key); + return saveDatasourceToDb(datasourceList).collectList(); + }); + + Flux> datasourceStorageFLux = Flux.fromIterable(mappedImportableResourcesDTO + .getDatasourceStorageDryRunQueries() + .keySet()) + .flatMap(key -> { + List datasourceStorageList = mappedImportableResourcesDTO + .getDatasourceStorageDryRunQueries() + .get(key); + return saveDatasourceStorageToDb(datasourceStorageList).collectList(); + }); + return Flux.merge(datasourceFLux, datasourceStorageFLux).then(); + } +} diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceContextServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceContextServiceCEImpl.java index b2f3a982362d..3693d1348c47 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceContextServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/DatasourceContextServiceCEImpl.java @@ -195,7 +195,7 @@ public Mono updateDatasourceAndSetAuthentication(Object connection, Data .setAuthentication(updatableConnection.getAuthenticationDTO( datasourceStorage.getDatasourceConfiguration().getAuthentication())); datasourceStorageMono = datasourceStorageService.updateDatasourceStorage( - datasourceStorage, datasourceStorage.getEnvironmentId(), Boolean.FALSE); + datasourceStorage, datasourceStorage.getEnvironmentId(), Boolean.FALSE, false); } return datasourceStorageMono.thenReturn(connection); } diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/exports/internal/ExportServiceTests.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/exports/internal/ExportServiceTests.java index cab998ec48d6..af8244bf6f9e 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/exports/internal/ExportServiceTests.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/exports/internal/ExportServiceTests.java @@ -2073,7 +2073,7 @@ public void exportApplicationByWhen_WhenGitConnectedAndDatasourceRenamed_Queries Application application = objects.getT2(); datasource.setName("DS_FOR_RENAME_TEST_RENAMED"); return datasourceService - .save(datasource) + .save(datasource, false) .then(exportService.exportByArtifactId( application.getId(), SerialiseArtifactObjective.VERSION_CONTROL, diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/imports/internal/ImportServiceTests.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/imports/internal/ImportServiceTests.java index 0f03290799b7..c881b2a88482 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/imports/internal/ImportServiceTests.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/imports/internal/ImportServiceTests.java @@ -5180,7 +5180,7 @@ public void exportApplicationByWhen_WhenGitConnectedAndDatasourceRenamed_Queries Application application = objects.getT2(); datasource.setName("DS_FOR_RENAME_TEST_RENAMED"); return datasourceService - .save(datasource) + .save(datasource, false) .then(exportService.exportByArtifactId(application.getId(), VERSION_CONTROL, APPLICATION)) .map(artifactExchangeJson -> (ApplicationJson) artifactExchangeJson); }); diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java index c23d8aa2fbf3..32d45f46864b 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/services/DatasourceServiceTest.java @@ -1284,7 +1284,7 @@ public void checkEncryptionOfAuthenticationDTOAfterRemoval() { datasourceConfiguration2.setUrl("http://test.com"); datasource1.setDatasourceConfiguration(datasourceConfiguration2); datasource1.setName("New Name for update to test that encryption is now gone"); - return datasourceService.save(datasource1); + return datasourceService.save(datasource1, false); }); StepVerifier.create(datasourceMono) @@ -1994,7 +1994,7 @@ public void verifyTestDatasourceWithSavedDatasourceButNoDatasourceStorageSucceed datasourceStorageService.createDatasourceStorageFromDatasourceStorageDTO(datasourceStorageDTO); Mockito.doReturn(Mono.just(datasourceStorage)) .when(datasourceStorageService) - .create(Mockito.any()); + .create(Mockito.any(), Mockito.anyBoolean()); Datasource dbDatasource = datasourceService.create(datasource).block(); assertThat(dbDatasource.getId()).isNotNull();