Skip to content

chore: move metadata calculation to datasource storage #39657

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nsarupr merged 1 commit into from
Mar 11, 2025
Merged

chore: move metadata calculation to datasource storage #39657

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 21 additions & 47 deletions ...th-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceCEImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,8 @@
import com.appsmith.server.repositories.DatasourceRepository;
import com.appsmith.server.repositories.NewActionRepository;
import com.appsmith.server.services.AnalyticsService;
import com.appsmith.server.services.ConfigService;
import com.appsmith.server.services.DatasourceContextService;
import com.appsmith.server.services.FeatureFlagService;
import com.appsmith.server.services.OrganizationService;
import com.appsmith.server.services.SequenceService;
import com.appsmith.server.services.SessionUserService;
import com.appsmith.server.services.WorkspaceService;
Expand Down Expand Up @@ -66,8 +64,6 @@
import static com.appsmith.external.constants.spans.DatasourceSpan.FETCH_ALL_DATASOURCES_WITH_STORAGES;
import static com.appsmith.external.constants.spans.DatasourceSpan.FETCH_ALL_PLUGINS_IN_WORKSPACE;
import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties;
import static com.appsmith.server.constants.ce.FieldNameCE.INSTANCE_ID;
import static com.appsmith.server.constants.ce.FieldNameCE.TENANT_ID;
import static com.appsmith.server.dtos.DBOpsType.SAVE;
import static com.appsmith.server.helpers.CollectionUtils.isNullOrEmpty;
import static com.appsmith.server.helpers.DatasourceAnalyticsUtils.getAnalyticsProperties;
Expand Down Expand Up @@ -97,9 +93,6 @@ public class DatasourceServiceCEImpl implements DatasourceServiceCE {
private final RateLimitService rateLimitService;
private final FeatureFlagService featureFlagService;
private final ObservationRegistry observationRegistry;
private final OrganizationService organizationService;
private final ConfigService configService;

// Defines blocking duration for test as well as connection created for query execution
// This will block the creation of datasource connection for 5 minutes, in case of more than 3 failed connection
// attempts
Expand All @@ -125,9 +118,7 @@ public DatasourceServiceCEImpl(
EnvironmentPermission environmentPermission,
RateLimitService rateLimitService,
FeatureFlagService featureFlagService,
ObservationRegistry observationRegistry,
OrganizationService organizationService,
ConfigService configService) {
ObservationRegistry observationRegistry) {

this.workspaceService = workspaceService;
this.sessionUserService = sessionUserService;
Expand All @@ -146,8 +137,6 @@ public DatasourceServiceCEImpl(
this.rateLimitService = rateLimitService;
this.featureFlagService = featureFlagService;
this.observationRegistry = observationRegistry;
this.organizationService = organizationService;
this.configService = configService;
}

@Override
Expand Down Expand Up @@ -235,28 +224,27 @@ private Mono<Datasource> createEx(
}

return datasourceMono.flatMap(savedDatasource -> this.organiseDatasourceStorages(savedDatasource)
.flatMap(datasourceStorageX -> setAdditionalMetadataInDatasourceStorage(datasourceStorageX)
.flatMap(datasourceStorage -> {
// Make sure that we are creating entries only if the id is not already populated
if (hasText(datasourceStorage.getId())) {
return Mono.just(datasourceStorage);
}
.flatMap(datasourceStorage -> {
// Make sure that we are creating entries only if the id is not already populated
if (hasText(datasourceStorage.getId())) {
return Mono.just(datasourceStorage);
}

return datasourceStorageService
.create(datasourceStorage, isDryOps)
.map(datasourceStorage1 -> {
if (datasourceStorageDryRunQueries != null && isDryOps) {
List<DatasourceStorage> datasourceStorages =
datasourceStorageDryRunQueries.get(SAVE);
if (datasourceStorages == null) {
datasourceStorages = new ArrayList<>();
}
datasourceStorages.add(datasourceStorage1);
datasourceStorageDryRunQueries.put(SAVE, datasourceStorages);
}
return datasourceStorage1;
});
}))
return datasourceStorageService
.create(datasourceStorage, isDryOps)
.map(datasourceStorage1 -> {
if (datasourceStorageDryRunQueries != null && isDryOps) {
List<DatasourceStorage> datasourceStorages =
datasourceStorageDryRunQueries.get(SAVE);
if (datasourceStorages == null) {
datasourceStorages = new ArrayList<>();
}
datasourceStorages.add(datasourceStorage1);
datasourceStorageDryRunQueries.put(SAVE, datasourceStorages);
}
return datasourceStorage1;
});
})
.map(datasourceStorageService::createDatasourceStorageDTOFromDatasourceStorage)
.collectMap(DatasourceStorageDTO::getEnvironmentId)
.map(savedStorages -> {
Expand All @@ -265,20 +253,6 @@ private Mono<Datasource> createEx(
}));
}

private Mono<DatasourceStorage> setAdditionalMetadataInDatasourceStorage(DatasourceStorage datasourceStorage) {
Mono<String> organizationIdMono = organizationService.getCurrentUserOrganizationId();
Mono<String> instanceIdMono = configService.getInstanceId();

Map<String, Object> metadata = new HashMap<>();

return organizationIdMono.zipWith(instanceIdMono).map(tuple -> {
metadata.put(TENANT_ID, tuple.getT1());
metadata.put(INSTANCE_ID, tuple.getT2());
datasourceStorage.setMetadata(metadata);
return datasourceStorage;
});
}

// this requires an EE override multiple environments
protected Flux<DatasourceStorage> organiseDatasourceStorages(@NotNull Datasource savedDatasource) {
Map<String, DatasourceStorageDTO> storages = savedDatasource.getDatasourceStorages();
Expand Down
10 changes: 2 additions & 8 deletions ...mith-server/src/main/java/com/appsmith/server/datasources/base/DatasourceServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,8 @@
import com.appsmith.server.repositories.DatasourceRepository;
import com.appsmith.server.repositories.NewActionRepository;
import com.appsmith.server.services.AnalyticsService;
import com.appsmith.server.services.ConfigService;
import com.appsmith.server.services.DatasourceContextService;
import com.appsmith.server.services.FeatureFlagService;
import com.appsmith.server.services.OrganizationService;
import com.appsmith.server.services.SequenceService;
import com.appsmith.server.services.SessionUserService;
import com.appsmith.server.services.WorkspaceService;
Expand Down Expand Up @@ -43,9 +41,7 @@ public DatasourceServiceImpl(
EnvironmentPermission environmentPermission,
RateLimitService rateLimitService,
FeatureFlagService featureFlagService,
ObservationRegistry observationRegistry,
OrganizationService organizationService,
ConfigService configService) {
ObservationRegistry observationRegistry) {

super(
repository,
Expand All @@ -64,8 +60,6 @@ public DatasourceServiceImpl(
environmentPermission,
rateLimitService,
featureFlagService,
observationRegistry,
organizationService,
configService);
observationRegistry);
}
}
15 changes: 13 additions & 2 deletions ...com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceCECompatibleImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import com.appsmith.server.plugins.base.PluginService;
import com.appsmith.server.repositories.DatasourceStorageRepository;
import com.appsmith.server.services.AnalyticsService;
import com.appsmith.server.services.ConfigService;
import com.appsmith.server.services.OrganizationService;
import com.appsmith.server.solutions.DatasourcePermission;
import org.springframework.stereotype.Service;

Expand All @@ -16,7 +18,16 @@ public DatasourceStorageServiceCECompatibleImpl(
DatasourcePermission datasourcePermission,
PluginService pluginService,
PluginExecutorHelper pluginExecutorHelper,
AnalyticsService analyticsService) {
super(repository, datasourcePermission, pluginService, pluginExecutorHelper, analyticsService);
AnalyticsService analyticsService,
ConfigService configService,
OrganizationService organizationService) {
super(
repository,
datasourcePermission,
pluginService,
pluginExecutorHelper,
analyticsService,
configService,
organizationService);
}
}
33 changes: 30 additions & 3 deletions ...main/java/com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceCEImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
import com.appsmith.server.plugins.base.PluginService;
import com.appsmith.server.repositories.DatasourceStorageRepository;
import com.appsmith.server.services.AnalyticsService;
import com.appsmith.server.services.ConfigService;
import com.appsmith.server.services.OrganizationService;
import com.appsmith.server.solutions.DatasourcePermission;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
Expand All @@ -30,6 +32,8 @@
import java.util.Set;

import static com.appsmith.external.helpers.AppsmithBeanUtils.copyNestedNonNullProperties;
import static com.appsmith.server.constants.FieldName.INSTANCE_ID;
import static com.appsmith.server.constants.FieldName.ORGANIZATION_ID;
import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;

Expand All @@ -41,18 +45,24 @@ public class DatasourceStorageServiceCEImpl implements DatasourceStorageServiceC
private final PluginService pluginService;
private final PluginExecutorHelper pluginExecutorHelper;
private final AnalyticsService analyticsService;
private final ConfigService configService;
private final OrganizationService organizationService;

public DatasourceStorageServiceCEImpl(
DatasourceStorageRepository repository,
DatasourcePermission datasourcePermission,
PluginService pluginService,
PluginExecutorHelper pluginExecutorHelper,
AnalyticsService analyticsService) {
AnalyticsService analyticsService,
ConfigService configService,
OrganizationService organizationService) {
this.repository = repository;
this.datasourcePermission = datasourcePermission;
this.pluginService = pluginService;
this.pluginExecutorHelper = pluginExecutorHelper;
this.analyticsService = analyticsService;
this.configService = configService;
this.organizationService = organizationService;
}

@Override
Expand Down Expand Up @@ -254,8 +264,10 @@ private Mono<DatasourceStorage> validateAndSaveDatasourceStorageToRepository(
}
return repository
.save(unsavedDatasourceStorage)
.then(this.executePostSaveActions(unsavedDatasourceStorage))
.thenReturn(unsavedDatasourceStorage);
.flatMap(savedDatasourceStorage -> setAdditionalMetadataInDatasourceStorage(
savedDatasourceStorage)
.flatMap(this::executePostSaveActions)
.thenReturn(savedDatasourceStorage));
});
}

Expand All @@ -265,6 +277,21 @@ public Mono<DatasourceStorage> checkEnvironment(DatasourceStorage datasourceStor
return Mono.just(datasourceStorage);
}

private Mono<DatasourceStorage> setAdditionalMetadataInDatasourceStorage(DatasourceStorage datasourceStorage) {
Mono<String> organizationIdMono = organizationService.getCurrentUserOrganizationId();
Mono<String> instanceIdMono = configService.getInstanceId();

Map<String, Object> metadata = new HashMap<>();

return organizationIdMono.zipWith(instanceIdMono).map(tuple -> {
// Change this to ORGANIZATION_ID once we have the organizationId field in the datasource storage
metadata.put(ORGANIZATION_ID, tuple.getT1());
metadata.put(INSTANCE_ID, tuple.getT2());
datasourceStorage.setMetadata(metadata);
return datasourceStorage;
});
}

private DatasourceStorage sanitizeDatasourceStorage(DatasourceStorage datasourceStorage) {
if (datasourceStorage.getDatasourceConfiguration() != null
&& !CollectionUtils.isEmpty(
Expand Down
15 changes: 13 additions & 2 deletions ...c/main/java/com/appsmith/server/datasourcestorages/base/DatasourceStorageServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import com.appsmith.server.plugins.base.PluginService;
import com.appsmith.server.repositories.DatasourceStorageRepository;
import com.appsmith.server.services.AnalyticsService;
import com.appsmith.server.services.ConfigService;
import com.appsmith.server.services.OrganizationService;
import com.appsmith.server.solutions.DatasourcePermission;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
Expand All @@ -17,7 +19,16 @@ public DatasourceStorageServiceImpl(
DatasourcePermission datasourcePermission,
PluginService pluginService,
PluginExecutorHelper pluginExecutorHelper,
AnalyticsService analyticsService) {
super(repository, datasourcePermission, pluginService, pluginExecutorHelper, analyticsService);
AnalyticsService analyticsService,
ConfigService configService,
OrganizationService organizationService) {
super(
repository,
datasourcePermission,
pluginService,
pluginExecutorHelper,
analyticsService,
configService,
organizationService);
}
}