feat(gatsby): Exclude the internal/fields objects for greater consistency creating contentDigest (#33671)

* feat(gatsby-core-utils): Exclude the internal object for greater consistency

I was puzzled why a site was having more nodes than expected by marked as changed on each data update. I use jest-diff to show the diff between the nodes and as it turns out, it was all trivial differences in the `internal` object. Since the `internal` object is managed by the framework and source/transformer plugins typically don't ensure that the `internal` object hasn't changed between updates — we should just exclude it when generating the digest.

Making this change resulted in ~20 less nodes being changed which reduced the number of invalidated queries from ~40k to ~6k, dramatically speeding up the incremental builds.

I tested whether the destructuring would slow things down any by running the ~700k nodes on this site through both the old and new version of the site. They were essentially the same — ~16.5s vs. ~16.9s with the new algorithm being faster, presumably as we don't now have to hash the internal object.

I expect this will result in fewer changed nodes in a lot of plugins.

* Update create-content-digest.ts

* Only hash as node if node & update snapshots

* Only remove autogenerated fields from internal object

* Move node-specific createContentDigest code to api-runner-node

* revert snapshot changes

* update snapshots

* Update packages/gatsby/src/utils/api-runner-node.js

Co-authored-by: Michal Piechowiak <[email protected]>

Co-authored-by: Ward Peeters <[email protected]>
Co-authored-by: Michal Piechowiak <[email protected]>

Author: 3 people

integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/source-changed/scenario.js

@@ -57,8 +57,8 @@ const nodesTest = ({
       -   \\"foo\\": \\"run-1\\",
           \\"id\\": \\"parent_parentChangeForTransformer\\",
           \\"internal\\": Object {
-      -     \\"contentDigest\\": \\"9d6d458358c77dbe8f4247752ebe41f0\\",
-      +     \\"contentDigest\\": \\"3021b9f76357d1cffb3c40fabc9e08fb\\",
+      -     \\"contentDigest\\": \\"dc3d69faa290879f166b8cf2a459e123\\",
+      +     \\"contentDigest\\": \\"5b2b719234b7823a8a7743409714b7a7\\",
             \\"owner\\": \\"source-and-transformers-child-nodes/source-changed/gatsby-source\\",
             \\"type\\": \\"Parent_ParentChangeForTransformer\\",
           },
@@ -75,8 +75,8 @@ const nodesTest = ({
       +   \\"foo\\": undefined,
           \\"id\\": \\"parent_parentChangeForTransformer >>> Child\\",
           \\"internal\\": Object {
-      -     \\"contentDigest\\": \\"603e50c1fe96279688538ab046d1d70a\\",
-      +     \\"contentDigest\\": \\"f784f9722081b56fee8ca34708299a37\\",
+      -     \\"contentDigest\\": \\"7ec9e8b4f83ebca01990608f961b1a5d\\",
+      +     \\"contentDigest\\": \\"4cecf7ebaf0c19b22eef8835c725c2e0\\",
             \\"owner\\": \\"source-and-transformers-child-nodes/source-changed/gatsby-transformer\\",
             \\"type\\": \\"ChildOfParent_ParentChangeForTransformer\\",
           },

integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-added/scenario.js

@@ -55,7 +55,7 @@ const nodesTest = ({
           \\"foo\\": \\"run-1\\",
           \\"id\\": \\"parent_childAdditionForTransformer\\",
           \\"internal\\": Object {
-            \\"contentDigest\\": \\"f85e860f002547e9da9e893e3e44e162\\",
+            \\"contentDigest\\": \\"f99c0539a6bb6061b072fe782a8e441f\\",
             \\"owner\\": \\"source-and-transformers-child-nodes/transformer-added/gatsby-source\\",
             \\"type\\": \\"Parent_ChildAdditionForTransformer\\",
           },

integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-changed/scenario.js

@@ -78,8 +78,8 @@ const nodesTest = ({
       +   \\"foo\\": \\"baz\\",
           \\"id\\": \\"parent_childChangeForTransformer >>> Child\\",
           \\"internal\\": Object {
-      -     \\"contentDigest\\": \\"25ad44d8db6f7cee17e248d3b4c94538\\",
-      +     \\"contentDigest\\": \\"3c57fc662e03b2b443b47575f7e82ec0\\",
+      -     \\"contentDigest\\": \\"04b03439f5e1d249d999e6763e23f306\\",
+      +     \\"contentDigest\\": \\"66875f4e1170801ce795d9d863a12b2a\\",
             \\"owner\\": \\"source-and-transformers-child-nodes/transformer-changed/gatsby-transformer\\",
             \\"type\\": \\"ChildOfParent_ChildChangeForTransformer\\",
           },

integration-tests/cache-resilience/plugins/source-and-transformers-child-nodes/transformer-removed/scenario.js

@@ -85,7 +85,7 @@ const nodesTest = ({
           \\"foo\\": \\"run-1\\",
           \\"id\\": \\"parent_childDeletionForTransformer\\",
           \\"internal\\": Object {
-            \\"contentDigest\\": \\"872081fdfb66891ee6ccdcd13716a5ce\\",
+            \\"contentDigest\\": \\"6140f27e9454bda710ae6da2cda698f0\\",
             \\"owner\\": \\"source-and-transformers-child-nodes/transformer-removed/gatsby-source\\",
             \\"type\\": \\"Parent_ChildDeletionForTransformer\\",
           },

integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/source-changed/scenario.js

@@ -49,8 +49,8 @@ const nodesTest = ({
       -   \\"foo\\": \\"run-1\\",
           \\"id\\": \\"parent_parentChangeForFields\\",
           \\"internal\\": Object {
-      -     \\"contentDigest\\": \\"e88540d53597617cf99d612601037013\\",
-      +     \\"contentDigest\\": \\"3b78e62e87d3f1d8e92d274aa8dbe548\\",
+      -     \\"contentDigest\\": \\"bde8140cd815ea2bb9e7ae2b45330d08\\",
+      +     \\"contentDigest\\": \\"76dc03ca64a18a99d1581b9e7dd224ef\\",
             \\"fieldOwners\\": Object {
               \\"bar\\": \\"source-and-transformers-node-fields/source-changed/gatsby-transformer\\",
               \\"foo\\": \\"source-and-transformers-node-fields/source-changed/gatsby-transformer\\",

integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/transformer-added/scenario.js

@@ -50,7 +50,7 @@ const nodesTest = ({
           \\"foo\\": \\"run-1\\",
           \\"id\\": \\"parent_childAdditionForFields\\",
           \\"internal\\": Object {
-            \\"contentDigest\\": \\"bdf44fdce30b104b4f290d66c2dc3ca1\\",
+            \\"contentDigest\\": \\"b6d6df9ccdfc5388cde03e3140520cf5\\",
       +     \\"fieldOwners\\": Object {
       +       \\"foo1\\": \\"source-and-transformers-node-fields/transformer-added/gatsby-transformer\\",
       +     },

integration-tests/cache-resilience/plugins/source-and-transformers-node-fields/transformer-changed/scenario.js

@@ -67,7 +67,7 @@ const nodesTest = ({
           \\"foo\\": \\"run-1\\",
           \\"id\\": \\"parent_childChangeForFields\\",
           \\"internal\\": Object {
-            \\"contentDigest\\": \\"fb9e9b9c26522bceaa1f51c537b2aff2\\",
+            \\"contentDigest\\": \\"c6c5c686a949351323ba4a04b585840a\\",
             \\"fieldOwners\\": Object {
       -       \\"foo1\\": \\"source-and-transformers-node-fields/transformer-changed/gatsby-transformer\\",
       +       \\"foo2\\": \\"source-and-transformers-node-fields/transformer-changed/gatsby-transformer\\",

packages/gatsby/src/utils/api-runner-node.js

@@ -13,7 +13,9 @@ const { codeFrameColumns } = require(`@babel/code-frame`)
 const fs = require(`fs-extra`)
 const { getCache } = require(`./get-cache`)
 import { createNodeId } from "./create-node-id"
-const { createContentDigest } = require(`gatsby-core-utils`)
+const {
+  createContentDigest: _createContentDigest,
+} = require(`gatsby-core-utils`)
 import {
   buildObjectType,
   buildUnionType,
@@ -32,6 +34,29 @@ const { trackBuildError, decorateEvent } = require(`gatsby-telemetry`)
 import errorParser from "./api-runner-error-parser"
 import { wrapNode, wrapNodes } from "./detect-node-mutations"
 
+// Override createContentDigest to remove autogenerated data from nodes to
+// ensure consistent digests.
+function createContentDigest(node) {
+  if (!node?.internal?.type) {
+    // this doesn't look like a node, so let's just pass it as-is
+    return _createContentDigest(node)
+  }
+  return _createContentDigest({
+    ...node,
+    internal: {
+      ...node.internal,
+      // Remove auto-generated fields that'd prevent
+      // creating a consistent contentDigest.
+      contentDigest: undefined,
+      owner: undefined,
+      fieldOwners: undefined,
+      ignoreType: undefined,
+      counter: undefined,
+    },
+    fields: undefined,
+  })
+}
+
 if (!process.env.BLUEBIRD_DEBUG && !process.env.BLUEBIRD_LONG_STACK_TRACES) {
   // Unless specified - disable longStackTraces
   // as this have severe perf penalty ( http://bluebirdjs.com/docs/api/promise.longstacktraces.html )