Incremental static regeneration

packages/libs/lambda-at-edge/package.json

@@ -62,6 +62,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "1.0.0-rc.3",
+    "@aws-sdk/client-sqs": "1.0.0-rc.3",
     "@hapi/accept": "5.0.1",
     "@vercel/nft": "^0.9.3",
     "cookie": "^0.4.1",

packages/libs/lambda-at-edge/rollup.config.js

@@ -47,5 +47,7 @@ export default [
   { filename: "api-handler", minify: false },
   { filename: "api-handler", minify: true },
   { filename: "image-handler", minify: false },
-  { filename: "image-handler", minify: true }
+  { filename: "image-handler", minify: true },
+  { filename: "regeneration-handler", minify: false },
+  { filename: "regeneration-handler", minify: true }
 ].map(generateConfig);

packages/libs/lambda-at-edge/src/build.ts

@@ -31,6 +31,7 @@ import { Job } from "@vercel/nft/out/node-file-trace";
 export const DEFAULT_LAMBDA_CODE_DIR = "default-lambda";
 export const API_LAMBDA_CODE_DIR = "api-lambda";
 export const IMAGE_LAMBDA_CODE_DIR = "image-lambda";
+export const REGENERATION_LAMBDA_CODE_DIR = "regeneration-lambda";
 export const ASSETS_DIR = "assets";
 
 type BuildOptions = {
@@ -230,7 +231,11 @@ class Builder {
    * @param shouldMinify
    */
   async processAndCopyHandler(
-    handlerType: "api-handler" | "default-handler" | "image-handler",
+    handlerType:
+      | "api-handler"
+      | "default-handler"
+      | "image-handler"
+      | "regeneration-handler",
     destination: string,
     shouldMinify: boolean
   ) {
@@ -245,9 +250,9 @@ class Builder {
     await fse.copy(source, destination);
   }
 
-  async buildDefaultLambda(
+  async copyTraces(
     buildManifest: OriginRequestDefaultHandlerManifest
-  ): Promise<void[]> {
+  ): Promise<void> {
     let copyTraces: Promise<void>[] = [];
 
     if (this.buildOptions.useServerlessTraceTarget) {
@@ -284,7 +289,13 @@ class Builder {
       );
     }
 
-    let prerenderManifest = require(join(
+    await Promise.all(copyTraces);
+  }
+
+  async buildDefaultLambda(
+    buildManifest: OriginRequestDefaultHandlerManifest
+  ): Promise<void[]> {
+    const prerenderManifest = require(join(
       this.dotNextDir,
       "prerender-manifest.json"
     ));
@@ -294,7 +305,7 @@ class Builder {
     );
 
     return Promise.all([
-      ...copyTraces,
+      this.copyTraces(buildManifest),
       this.processAndCopyHandler(
         "default-handler",
         join(this.outputDir, DEFAULT_LAMBDA_CODE_DIR),
@@ -414,6 +425,36 @@ class Builder {
     ]);
   }
 
+  async buildRegenerationHandler(
+    buildManifest: OriginRequestDefaultHandlerManifest
+  ): Promise<void> {
+    await Promise.all([
+      this.copyTraces(buildManifest),
+      this.processAndCopyHandler(
+        "regeneration-handler",
+        join(this.outputDir, REGENERATION_LAMBDA_CODE_DIR, "index.js"),
+        !!this.buildOptions.minifyHandlers
+      ),
+      fse.copy(
+        join(this.serverlessDir, "pages"),
+        join(this.outputDir, REGENERATION_LAMBDA_CODE_DIR, "pages"),
+        {
+          filter: (file: string) => {
+            const isNotPrerenderedHTMLPage = path.extname(file) !== ".html";
+            const isNotStaticPropsJSONFile = path.extname(file) !== ".json";
+            const isNotApiPage = pathToPosix(file).indexOf("pages/api") === -1;
+
+            return (
+              isNotPrerenderedHTMLPage &&
+              isNotStaticPropsJSONFile &&
+              isNotApiPage
+            );
+          }
+        }
+      )
+    ]);
+  }
+
   /**
    * Build image optimization lambda (supported by Next.js 10)
    * @param buildManifest
@@ -943,9 +984,9 @@ class Builder {
       path.join(dotNextDirectory, "prerender-manifest.json")
     );
 
-    let prerenderManifestJSONPropFileAssets: Promise<void>[] = [];
-    let prerenderManifestHTMLPageAssets: Promise<void>[] = [];
-    let fallbackHTMLPageAssets: Promise<void>[] = [];
+    const prerenderManifestJSONPropFileAssets: Promise<void>[] = [];
+    const prerenderManifestHTMLPageAssets: Promise<void>[] = [];
+    const fallbackHTMLPageAssets: Promise<void>[] = [];
 
     // Copy locale-specific prerendered files if defined, otherwise use empty locale
     // which would copy to root only
@@ -1121,6 +1162,7 @@ class Builder {
     await fse.emptyDir(join(this.outputDir, DEFAULT_LAMBDA_CODE_DIR));
     await fse.emptyDir(join(this.outputDir, API_LAMBDA_CODE_DIR));
     await fse.emptyDir(join(this.outputDir, IMAGE_LAMBDA_CODE_DIR));
+    await fse.emptyDir(join(this.outputDir, REGENERATION_LAMBDA_CODE_DIR));
     await fse.emptyDir(join(this.outputDir, ASSETS_DIR));
 
     const { restoreUserConfig } = await createServerlessConfig(
@@ -1162,6 +1204,7 @@ class Builder {
     } = await this.prepareBuildManifests(routesManifest, prerenderManifest);
 
     await this.buildDefaultLambda(defaultBuildManifest);
+    await this.buildRegenerationHandler(defaultBuildManifest);
 
     const hasAPIPages =
       Object.keys(apiBuildManifest.apis.nonDynamic).length > 0 ||

packages/libs/lambda-at-edge/src/default-handler.ts

@@ -44,6 +44,11 @@ import {
   getLocalePrefixFromUri
 } from "./routing/locale-utils";
 import { removeBlacklistedHeaders } from "./headers/removeBlacklistedHeaders";
+import { getStaticRegenerationResponse } from "./lib/getStaticRegenerationResponse";
+import { s3BucketNameFromEventRequest } from "./s3/s3BucketNameFromEventRequest";
+import { triggerStaticRegeneration } from "./lib/triggerStaticRegeneration";
+import { s3StorePage } from "./s3/s3StorePage";
+import { cleanRequestUriForRouter } from "./lib/cleanRequestUriForRouter";
 
 const basePath = RoutesManifestJson.basePath;
 
@@ -325,8 +330,8 @@ const handleOriginRequest = async ({
   const decodedUri = decodeURI(uri);
   const { pages, publicFiles } = manifest;
 
-  let isPublicFile = publicFiles[decodedUri];
-  let isDataReq = isDataRequest(uri);
+  const isPublicFile = publicFiles[decodedUri];
+  const isDataReq = isDataRequest(uri);
 
   // Handle redirects
   // TODO: refactor redirect logic to another file since this is getting quite large
@@ -590,7 +595,6 @@ const handleOriginRequest = async ({
 const handleOriginResponse = async ({
   event,
   manifest,
-  prerenderManifest,
   routesManifest
 }: {
   event: OriginResponseEvent;
@@ -602,12 +606,46 @@ const handleOriginResponse = async ({
   const request = event.Records[0].cf.request;
   const { uri } = request;
   const { status } = response;
+  const bucketName = s3BucketNameFromEventRequest(request);
+
   if (status !== "403") {
     // Set 404 status code for 404.html page. We do not need normalised URI as it will always be "/404.html"
     if (uri.endsWith("/404.html")) {
       response.status = "404";
       response.statusDescription = "Not Found";
+      return response;
     }
+
+    const staticRegenerationResponse = getStaticRegenerationResponse({
+      requestedOriginUri: uri,
+      expiresHeader: response.headers.expires?.[0]?.value || "",
+      lastModifiedHeader: response.headers["last-modified"]?.[0]?.value || "",
+      manifest
+    });
+
+    if (staticRegenerationResponse) {
+      response.headers["cache-control"] = [
+        {
+          key: "Cache-Control",
+          value: staticRegenerationResponse.cacheControl
+        }
+      ];
+
+      // We don't want the `expires` header to be sent to the client we manage
+      // the cache at the edge using the s-maxage directive in the cache-control
+      // header
+      delete response.headers.expires;
+
+      if (staticRegenerationResponse.secondsRemainingUntilRevalidation === 0) {
+        await triggerStaticRegeneration({
+          basePath,
+          manifest,
+          request,
+          response
+        });
+      }
+    }
+
     return response;
   }
 
@@ -616,9 +654,6 @@ const handleOriginResponse = async ({
     return response;
   }
 
-  const { domainName, region } = request.origin!.s3!;
-  const bucketName = domainName.replace(`.s3.${region}.amazonaws.com`, "");
-
   // Lazily import only S3Client to reduce init times until actually needed
   const { S3Client } = await import("@aws-sdk/client-s3/S3Client");
 
@@ -643,12 +678,7 @@ const handleOriginResponse = async ({
     // eslint-disable-next-line
     const page = require(`./${pagePath}`);
     // Reconstruct original uri for next/router
-    if (uri.endsWith(".html")) {
-      request.uri = uri.slice(0, uri.length - 5);
-      if (manifest.trailingSlash) {
-        request.uri += "/";
-      }
-    }
+    request.uri = cleanRequestUriForRouter(request.uri, manifest.trailingSlash);
     const { req, res, responsePromise } = lambdaAtEdgeCompat(
       event.Records[0].cf,
       {
@@ -661,44 +691,42 @@ const handleOriginResponse = async ({
       res,
       "passthrough"
     );
+    let cacheControl = "public, max-age=0, s-maxage=2678400, must-revalidate";
     if (isSSG) {
-      const baseKey = uri
-        .replace(/^\//, "")
-        .replace(/\.(json|html)$/, "")
-        .replace(/^_next\/data\/[^\/]*\//, "");
-      const jsonKey = `_next/data/${manifest.buildId}/${baseKey}.json`;
-      const htmlKey = `static-pages/${manifest.buildId}/${baseKey}.html`;
-      const s3JsonParams = {
-        Bucket: bucketName,
-        Key: `${s3BasePath}${jsonKey}`,
-        Body: JSON.stringify(renderOpts.pageData),
-        ContentType: "application/json",
-        CacheControl: "public, max-age=0, s-maxage=2678400, must-revalidate"
-      };
-      const s3HtmlParams = {
-        Bucket: bucketName,
-        Key: `${s3BasePath}${htmlKey}`,
-        Body: html,
-        ContentType: "text/html",
-        CacheControl: "public, max-age=0, s-maxage=2678400, must-revalidate"
-      };
-      const { PutObjectCommand } = await import(
-        "@aws-sdk/client-s3/commands/PutObjectCommand"
-      );
-      await Promise.all([
-        s3.send(new PutObjectCommand(s3JsonParams)),
-        s3.send(new PutObjectCommand(s3HtmlParams))
-      ]);
+      const { expires } = await s3StorePage({
+        html,
+        uri,
+        basePath,
+        bucketName: bucketName || "",
+        buildId: manifest.buildId,
+        pageData: renderOpts.pageData,
+        region: request.origin?.s3?.region || "",
+        revalidate: renderOpts.revalidate
+      });
+
+      const isrResponse = expires
+        ? getStaticRegenerationResponse({
+            expiresHeader: expires.toJSON(),
+            manifest,
+            requestedOriginUri: uri,
+            lastModifiedHeader: undefined
+          })
+        : null;
+
+      cacheControl = (isrResponse && isrResponse.cacheControl) || cacheControl;
     }
     const outHeaders: OutgoingHttpHeaders = {};
     Object.entries(response.headers).map(([name, headers]) => {
       outHeaders[name] = headers.map(({ value }) => value);
     });
     res.writeHead(200, outHeaders);
-    res.setHeader(
-      "Cache-Control",
-      "public, max-age=0, s-maxage=2678400, must-revalidate"
-    );
+
+    if (cacheControl) {
+      res.setHeader("Cache-Control", cacheControl);
+    } else {
+      res.removeHeader("Cache-Control");
+    }
+
     if (isDataRequest(uri)) {
       res.setHeader("Content-Type", "application/json");
       res.end(JSON.stringify(renderOpts.pageData));

packages/libs/lambda-at-edge/src/image-handler.ts

@@ -20,6 +20,7 @@ import {
 } from "./routing/redirector";
 import { getUnauthenticatedResponse } from "./auth/authenticator";
 import { removeBlacklistedHeaders } from "./headers/removeBlacklistedHeaders";
+import { s3BucketNameFromEventRequest } from "./s3/s3BucketNameFromEventRequest";
 
 const basePath = RoutesManifestJson.basePath;
 
@@ -88,11 +89,11 @@ export const handler = async (
       true
     );
 
-    const { domainName, region } = request.origin!.s3!;
-    const bucketName = domainName.replace(`.s3.${region}.amazonaws.com`, "");
+    const { region } = request.origin!.s3!;
+    const bucketName = s3BucketNameFromEventRequest(request);
 
     await imageOptimizer(
-      { basePath: basePath, bucketName: bucketName, region: region },
+      { basePath: basePath, bucketName: bucketName || "", region: region },
       imagesManifest,
       req,
       res,

packages/libs/lambda-at-edge/src/lib/cleanRequestUriForRouter.ts

@@ -0,0 +1,16 @@
+/**
+ * Removes html and adds the trailing slash if needed. This is used when
+ * regenerating an SSG page.
+ */
+export const cleanRequestUriForRouter = (
+  uri: string,
+  trailingSlash?: boolean
+): string => {
+  if (uri.endsWith(".html")) {
+    uri = uri.slice(0, uri.length - 5);
+    if (trailingSlash) {
+      uri += "/";
+    }
+  }
+  return uri;
+};