diff --git a/app/client/craco.build.config.js b/app/client/craco.build.config.js index 574a6a5788ad..62a4c42261a0 100644 --- a/app/client/craco.build.config.js +++ b/app/client/craco.build.config.js @@ -14,9 +14,8 @@ const plugins = []; plugins.push( new WorkboxPlugin.InjectManifest({ swSrc: "./src/serviceWorker.ts", - mode: "development", + mode: "production", swDest: "./pageService.js", - maximumFileSizeToCacheInBytes: 11 * 1024 * 1024, exclude: [ // Don’t cache source maps and PWA manifests. // (These are the default values of the `exclude` option: https://developer.chrome.com/docs/workbox/reference/workbox-build/#type-WebpackPartial, @@ -32,9 +31,8 @@ plugins.push( // one by one (as the service worker does it) keeps the network busy for a long time // and delays the service worker installation /\/*\.svg$/, + /\.(js|css|html|png|jpg|jpeg|gif)$/, // Exclude JS, CSS, HTML, and image files ], - // Don’t cache-bust JS and CSS chunks - dontCacheBustURLsMatching: /\.[0-9a-zA-Z]{8}\.chunk\.(js|css)$/, }), ); diff --git a/app/client/cypress/support/Objects/FeatureFlags.ts b/app/client/cypress/support/Objects/FeatureFlags.ts index 78fc490c3f97..d9fcbc2e6fb5 100644 --- a/app/client/cypress/support/Objects/FeatureFlags.ts +++ b/app/client/cypress/support/Objects/FeatureFlags.ts @@ -32,6 +32,7 @@ export const getConsolidatedDataApi = ( reload = true, ) => { cy.intercept("GET", "/api/v1/consolidated-api/*?*", (req) => { + delete req.headers["if-none-match"]; req.reply((res: any) => { if ( res.statusCode === 200 || @@ -86,6 +87,7 @@ export const featureFlagInterceptForLicenseFlags = () => { cy.intercept("GET", "/api/v1/consolidated-api/*?*", (req) => { req.reply((res: any) => { + delete req.headers["if-none-match"]; if (res.statusCode === 200) { const originalResponse = res?.body; const updatedResponse = produce(originalResponse, (draft: any) => { diff --git a/app/client/src/serviceWorker.ts b/app/client/src/serviceWorker.ts index b2893a76c6ec..314ed67ffeb3 100644 --- a/app/client/src/serviceWorker.ts +++ b/app/client/src/serviceWorker.ts @@ -1,11 +1,6 @@ -import { precacheAndRoute } from "workbox-precaching"; -import { clientsClaim, setCacheNameDetails, skipWaiting } from "workbox-core"; +import { clientsClaim, skipWaiting } from "workbox-core"; import { registerRoute, Route } from "workbox-routing"; -import { - CacheFirst, - NetworkOnly, - StaleWhileRevalidate, -} from "workbox-strategies"; +import { NetworkOnly } from "workbox-strategies"; import { cachedApiUrlRegex, getApplicationParamsFromUrl, @@ -14,32 +9,11 @@ import { } from "ee/utils/serviceWorkerUtils"; import type { RouteHandlerCallback } from "workbox-core/types"; -setCacheNameDetails({ - prefix: "appsmith", - suffix: "", - precache: "precache-v1", - runtime: "runtime", - googleAnalytics: "appsmith-ga", -}); - -const regexMap = { - appViewPage: new RegExp(/api\/v1\/pages\/\w+\/view$/), - static3PAssets: new RegExp( - /(tiny.cloud|googleapis|gstatic|cloudfront).*.(js|css|woff2)/, - ), - shims: new RegExp(/shims\/.*.js/), - profile: new RegExp(/v1\/(users\/profile|workspaces)/), -}; - -/* eslint-disable no-restricted-globals */ -// Note: if you need to filter out some files from precaching, +// eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any +const wbManifest = (self as any).__WB_MANIFEST; -// do that in craco.build.config.js → workbox webpack plugin options -// TODO: Fix this the next time the file is edited -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const toPrecache = (self as any).__WB_MANIFEST; - -precacheAndRoute(toPrecache); +// Delete the old pre-fetch cache. All static files are now cached by cache control headers. +caches.delete("appsmith-precache-v1"); self.__WB_DISABLE_DEV_LOGS = true; skipWaiting(); @@ -75,23 +49,6 @@ const htmlRouteHandlerCallback: RouteHandlerCallback = async ({ return networkHandler.handle({ event, request }); }; -// This route's caching seems too aggressive. -// TODO(abhinav): Figure out if this is really necessary. -// Maybe add the assets locally? -registerRoute(({ url }) => { - return ( - regexMap.shims.test(url.pathname) || regexMap.static3PAssets.test(url.href) - ); -}, new CacheFirst()); - -registerRoute(({ url }) => { - return regexMap.profile.test(url.pathname); -}, new NetworkOnly()); - -registerRoute(({ url }) => { - return regexMap.appViewPage.test(url.pathname); -}, new StaleWhileRevalidate()); - registerRoute( new Route(({ request, sameOrigin }) => { return sameOrigin && request.destination === "document"; diff --git a/app/client/start-https.sh b/app/client/start-https.sh index b2e6599292f1..c3c706dc21cd 100755 --- a/app/client/start-https.sh +++ b/app/client/start-https.sh @@ -294,6 +294,14 @@ $(if [[ $use_https == 1 ]]; then echo " location /api { proxy_pass $backend; + + gzip off; # Etag stripped from upstream if gzip is off. + # Ref1: https://forum.nginx.org/read.php?2,242807,242810#msg-242810 + # Ref2: https://www.ruby-forum.com/t/reverse-proxy-deleting-etag-header-from-response/246209/2 + # Delete the Cache-Control header set in the server block above. + add_header Cache-Control '' always; + # Proxy pass the Cache-Control header from the upstream. + proxy_pass_header Cache-Control; } location /oauth2 { diff --git a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/spans/ce/ConsolidatedApiSpanNamesCE.java b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/spans/ce/ConsolidatedApiSpanNamesCE.java index e757e73bab51..9e2ac07b3ee8 100644 --- a/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/spans/ce/ConsolidatedApiSpanNamesCE.java +++ b/app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/spans/ce/ConsolidatedApiSpanNamesCE.java @@ -31,4 +31,5 @@ public class ConsolidatedApiSpanNamesCE { public static final String DATASOURCES_SPAN = "datasources"; public static final String FORM_CONFIG_SPAN = "form_config"; public static final String MOCK_DATASOURCES_SPAN = "mock_datasources"; + public static final String ETAG_SPAN = CONSOLIDATED_API_PREFIX + VIEW + "compute_etag"; } diff --git a/app/server/appsmith-server/pom.xml b/app/server/appsmith-server/pom.xml index 6aaa35893697..7eb7287df41e 100644 --- a/app/server/appsmith-server/pom.xml +++ b/app/server/appsmith-server/pom.xml @@ -412,6 +412,11 @@ 2.14.2.Final test + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + 2.17.0 + diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ConsolidatedAPIController.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ConsolidatedAPIController.java index 1f4770915131..4dc87ba917f2 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ConsolidatedAPIController.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ConsolidatedAPIController.java @@ -11,9 +11,12 @@ import com.fasterxml.jackson.annotation.JsonView; import io.micrometer.observation.ObservationRegistry; import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; @@ -24,6 +27,7 @@ import static com.appsmith.external.constants.spans.ConsolidatedApiSpanNames.CONSOLIDATED_API_ROOT_EDIT; import static com.appsmith.external.constants.spans.ConsolidatedApiSpanNames.CONSOLIDATED_API_ROOT_VIEW; +import static org.apache.commons.lang3.StringUtils.isBlank; @Slf4j @RestController @@ -78,13 +82,13 @@ public Mono> getAllDataForFirstPageLoadF @JsonView(Views.Public.class) @GetMapping("/view") - public Mono> getAllDataForFirstPageLoadForViewMode( + public Mono>> getAllDataForFirstPageLoadForViewMode( @RequestParam(required = false) String applicationId, @RequestParam(required = false) String defaultPageId, @RequestParam(required = false, defaultValue = "branch") RefType refType, @RequestParam(required = false) String refName, - @RequestParam(required = false) String branchName) { - + @RequestParam(required = false) String branchName, + @RequestHeader(required = false, name = "if-none-match") String ifNoneMatch) { if (!StringUtils.hasLength(refName)) { refName = branchName; } @@ -100,8 +104,37 @@ public Mono> getAllDataForFirstPageLoadF return consolidatedAPIService .getConsolidatedInfoForPageLoad( defaultPageId, applicationId, refType, refName, ApplicationMode.PUBLISHED) - .map(consolidatedAPIResponseDTO -> - new ResponseDTO<>(HttpStatus.OK.value(), consolidatedAPIResponseDTO, null)) + .map(consolidatedAPIResponseDTO -> { + long startTime = System.currentTimeMillis(); + + String responseHash = consolidatedAPIService.computeConsolidatedAPIResponseEtag( + consolidatedAPIResponseDTO, defaultPageId, applicationId); + long endTime = System.currentTimeMillis(); + long duration = endTime - startTime; + log.debug("Time taken to compute ETag: {} ms", duration); + + // if defaultPageId and applicationId are both null, then don't compute ETag + if (isBlank(responseHash)) { + ResponseDTO responseDTO = + new ResponseDTO<>(HttpStatus.OK.value(), consolidatedAPIResponseDTO, null); + return new ResponseEntity<>(responseDTO, HttpStatus.OK); + } + + HttpHeaders headers = new HttpHeaders(); + headers.add("ETag", responseHash); + headers.add("Cache-Control", "private, must-revalidate"); + + if (ifNoneMatch != null && ifNoneMatch.equals(responseHash)) { + ResponseDTO responseDTO = + new ResponseDTO<>(HttpStatus.NOT_MODIFIED.value(), null, null); + return new ResponseEntity<>(responseDTO, headers, HttpStatus.NOT_MODIFIED); + } + + ResponseDTO responseDTO = + new ResponseDTO<>(HttpStatus.OK.value(), consolidatedAPIResponseDTO, null); + + return new ResponseEntity<>(responseDTO, headers, HttpStatus.OK); + }) .tag("pageId", Objects.toString(defaultPageId)) .tag("applicationId", Objects.toString(applicationId)) .tag("refType", Objects.toString(refType)) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ConsolidatedAPIServiceImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ConsolidatedAPIServiceImpl.java index 9b0c48a44d0d..649619990d7e 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ConsolidatedAPIServiceImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ConsolidatedAPIServiceImpl.java @@ -1,5 +1,6 @@ package com.appsmith.server.services; +import com.appsmith.external.helpers.ObservationHelper; import com.appsmith.server.actioncollections.base.ActionCollectionService; import com.appsmith.server.applications.base.ApplicationService; import com.appsmith.server.datasources.base.DatasourceService; @@ -35,7 +36,8 @@ public ConsolidatedAPIServiceImpl( DatasourceService datasourceService, MockDataService mockDataService, ObservationRegistry observationRegistry, - CacheableRepositoryHelper cacheableRepositoryHelper) { + CacheableRepositoryHelper cacheableRepositoryHelper, + ObservationHelper observationHelper) { super( sessionUserService, userService, @@ -53,6 +55,7 @@ public ConsolidatedAPIServiceImpl( datasourceService, mockDataService, observationRegistry, - cacheableRepositoryHelper); + cacheableRepositoryHelper, + observationHelper); } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ConsolidatedAPIServiceCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ConsolidatedAPIServiceCE.java index 697664f8f01d..14f1d6c79648 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ConsolidatedAPIServiceCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ConsolidatedAPIServiceCE.java @@ -9,4 +9,7 @@ public interface ConsolidatedAPIServiceCE { Mono getConsolidatedInfoForPageLoad( String defaultPageId, String applicationId, RefType refType, String refName, ApplicationMode mode); + + String computeConsolidatedAPIResponseEtag( + ConsolidatedAPIResponseDTO consolidatedAPIResponseDTO, String defaultPageId, String applicationId); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ConsolidatedAPIServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ConsolidatedAPIServiceCEImpl.java index 7872f46995fe..c9263d484a2a 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ConsolidatedAPIServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ConsolidatedAPIServiceCEImpl.java @@ -2,6 +2,7 @@ import com.appsmith.external.exceptions.ErrorDTO; import com.appsmith.external.git.constants.ce.RefType; +import com.appsmith.external.helpers.ObservationHelper; import com.appsmith.external.models.CreatorContextType; import com.appsmith.external.models.Datasource; import com.appsmith.server.actioncollections.base.ActionCollectionService; @@ -32,9 +33,13 @@ import com.appsmith.server.services.UserDataService; import com.appsmith.server.services.UserService; import com.appsmith.server.themes.base.ThemeService; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import io.micrometer.observation.ObservationRegistry; +import io.micrometer.tracing.Span; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.springframework.data.util.Pair; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; @@ -46,7 +51,11 @@ import reactor.core.publisher.Mono; import reactor.util.function.Tuple2; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -64,6 +73,7 @@ import static com.appsmith.external.constants.spans.ConsolidatedApiSpanNames.CURRENT_THEME_SPAN; import static com.appsmith.external.constants.spans.ConsolidatedApiSpanNames.CUSTOM_JS_LIB_SPAN; import static com.appsmith.external.constants.spans.ConsolidatedApiSpanNames.DATASOURCES_SPAN; +import static com.appsmith.external.constants.spans.ConsolidatedApiSpanNames.ETAG_SPAN; import static com.appsmith.external.constants.spans.ConsolidatedApiSpanNames.FEATURE_FLAG_SPAN; import static com.appsmith.external.constants.spans.ConsolidatedApiSpanNames.FORM_CONFIG_SPAN; import static com.appsmith.external.constants.spans.ConsolidatedApiSpanNames.MOCK_DATASOURCES_SPAN; @@ -106,6 +116,7 @@ public class ConsolidatedAPIServiceCEImpl implements ConsolidatedAPIServiceCE { private final MockDataService mockDataService; private final ObservationRegistry observationRegistry; private final CacheableRepositoryHelper cacheableRepositoryHelper; + private final ObservationHelper observationHelper; protected ResponseDTO getSuccessResponse(T data) { return new ResponseDTO<>(HttpStatus.OK.value(), data, null); @@ -633,4 +644,60 @@ protected Mono> getApplicationAndPageTupleMono( private boolean isPossibleToCreateQueryWithoutDatasource(Plugin plugin) { return PLUGINS_THAT_ALLOW_QUERY_CREATION_WITHOUT_DATASOURCE.contains(plugin.getPackageName()); } + + @NotNull public String computeConsolidatedAPIResponseEtag( + ConsolidatedAPIResponseDTO consolidatedAPIResponseDTO, String defaultPageId, String applicationId) { + if (isBlank(defaultPageId) && isBlank(applicationId)) { + return ""; + } + + Span computeEtagSpan = observationHelper.createSpan(ETAG_SPAN).start(); + + try { + String lastDeployedAt = consolidatedAPIResponseDTO.getPages() != null + ? consolidatedAPIResponseDTO + .getPages() + .getData() + .getApplication() + .getLastDeployedAt() + .toString() + : null; + + if (lastDeployedAt == null) { + return ""; + } + + Object currentTheme = consolidatedAPIResponseDTO.getCurrentTheme() != null + ? consolidatedAPIResponseDTO.getCurrentTheme() + : ""; + Object themes = consolidatedAPIResponseDTO.getThemes() != null + ? consolidatedAPIResponseDTO.getThemes() + : Collections.emptyList(); + + Map consolidateAPISignature = Map.of( + "userProfile", consolidatedAPIResponseDTO.getUserProfile(), + "featureFlags", consolidatedAPIResponseDTO.getFeatureFlags(), + "tenantConfig", consolidatedAPIResponseDTO.getTenantConfig(), + "productAlert", consolidatedAPIResponseDTO.getProductAlert(), + "currentTheme", currentTheme, + "themes", themes, + "lastDeployedAt", lastDeployedAt); + + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new JavaTimeModule()); + + String consolidateAPISignatureJSON = objectMapper.writeValueAsString(consolidateAPISignature); + + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hashBytes = digest.digest(consolidateAPISignatureJSON.getBytes(StandardCharsets.UTF_8)); + String etag = Base64.getEncoder().encodeToString(hashBytes); + + return etag; + } catch (Exception e) { + log.error("Error while computing etag for ConsolidatedAPIResponseDTO", e); + return ""; + } finally { + observationHelper.endSpan(computeEtagSpan, true); + } + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce_compatible/ConsolidatedAPIServiceCECompatibleImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce_compatible/ConsolidatedAPIServiceCECompatibleImpl.java index c7cc66cc4360..c6f94b2cf303 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce_compatible/ConsolidatedAPIServiceCECompatibleImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce_compatible/ConsolidatedAPIServiceCECompatibleImpl.java @@ -1,5 +1,6 @@ package com.appsmith.server.services.ce_compatible; +import com.appsmith.external.helpers.ObservationHelper; import com.appsmith.server.actioncollections.base.ActionCollectionService; import com.appsmith.server.applications.base.ApplicationService; import com.appsmith.server.datasources.base.DatasourceService; @@ -38,7 +39,8 @@ public ConsolidatedAPIServiceCECompatibleImpl( DatasourceService datasourceService, MockDataService mockDataService, ObservationRegistry observationRegistry, - CacheableRepositoryHelper cacheableRepositoryHelper) { + CacheableRepositoryHelper cacheableRepositoryHelper, + ObservationHelper observationHelper) { super( sessionUserService, userService, @@ -56,6 +58,7 @@ public ConsolidatedAPIServiceCECompatibleImpl( datasourceService, mockDataService, observationRegistry, - cacheableRepositoryHelper); + cacheableRepositoryHelper, + observationHelper); } } diff --git a/deploy/docker/fs/opt/appsmith/caddy-reconfigure.mjs b/deploy/docker/fs/opt/appsmith/caddy-reconfigure.mjs index c2d05cff933a..edd4ebd710fb 100644 --- a/deploy/docker/fs/opt/appsmith/caddy-reconfigure.mjs +++ b/deploy/docker/fs/opt/appsmith/caddy-reconfigure.mjs @@ -130,6 +130,15 @@ parts.push(` import file_server } + handle /api/v1/consolidated-api/view { + reverse_proxy { + to 127.0.0.1:8080 + header_up -Forwarded + header_up X-Appsmith-Request-Id {http.request.uuid} + header_down +Etag + } + } + @backend path /api/* /oauth2/* /login/* handle @backend { import reverse_proxy 8080