[v2] Hulksmash build slowdowns on larger sites (#6226)

* Hulksmash slowdowns on larger sites

* Switch more existsSync to use caching version

* Convert node reducer to use Map

* Fix most lint errors

* Split writing page data json files into 999 folders to speed writes

* Use forEach instead of reduce when prepping page data

For building a 25k page site, it reduced the time spent writing out page
data from 32 seconds to 0.32 seconds 😱

* Profiled code and fixed hot functions

* WIP commit to dramatically speed up graphql queries

Not really loving all the nested caching I'm adding... but it's
necessary. Will revisit this Monday to see if there's a cleaner way
to cache things.

* Handle picking directory to write StaticQuery results

* Speed up resolving queries when the query is querying a node by id

* Show pages rendered / second while building HTML

* Add sites for benchmarking

* Persist activity status at end and show queries/second for graphql

* Restore creating SitePage nodes as no longer slow

* Correct storing/using/deleting nodes

* Disable profiling

* Remove extra dependencies

* Format + fix linting

* remove console.log and unused profile

* Remove another profile

* Fix tests (hopefully)

* Check if nodes exists, sometimes a test fails otherwise

* debugging...

* Work around...

* Don't need to run bootstrap on tests as jest compiles code

* Lodash uses Object.entries which fails on node 6... :-"

* Ok, do need bootstrap to link packages

* Try try again

* Ugh, problem was Object.entries isn't in Node 6

benchmarks/create-pages/.gitignore

@@ -0,0 +1,62 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Typescript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+
+.cache/
+public
+yarn-error.log

benchmarks/create-pages/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 gatsbyjs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

benchmarks/create-pages/README.md

@@ -0,0 +1,5 @@
+# createPages benchmark
+
+Stress tests creating lots of tiny pages.
+
+Defaults to building a site with 5k pages. Set the `NUM_PAGES` environment variable to change that e.g. `NUM_PAGES=25000 gatsby build`

benchmarks/create-pages/gatsby-node.js

@@ -0,0 +1,18 @@
+let NUM_PAGES = 5000
+
+if (process.env.NUM_PAGES) {
+  NUM_PAGES = process.env.NUM_PAGES
+}
+
+exports.createPages = ({ actions: { createPage } }) => {
+  var step
+  for (step = 0; step < NUM_PAGES; step++) {
+    createPage({
+      path: `/path/${step}/`,
+      component: require.resolve(`./src/templates/blank.js`),
+      context: {
+        id: step,
+      },
+    })
+  }
+}

benchmarks/create-pages/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "gatsby-starter-hello-world",
+  "description": "Gatsby hello world starter",
+  "license": "MIT",
+  "scripts": {
+    "develop": "gatsby develop",
+    "build": "gatsby build",
+    "serve": "gatsby serve"
+  },
+  "dependencies": {
+    "gatsby": "next",
+    "react": "^16.3.2",
+    "react-dom": "^16.3.2"
+  }
+}

benchmarks/create-pages/src/pages/index.js

@@ -0,0 +1,3 @@
+import React from "react"
+
+export default () => <div>Hello world!</div>

benchmarks/create-pages/src/templates/blank.js

@@ -0,0 +1,3 @@
+import React from "react"
+
+export default () => <div>Yo!</div>

benchmarks/markdown/.gitignore

@@ -0,0 +1,62 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Typescript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+
+.cache/
+public
+yarn-error.log

benchmarks/markdown/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 gatsbyjs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

benchmarks/markdown/README.md

@@ -0,0 +1,5 @@
+# Markdown benchmark
+
+Stress tests creating lots of pages rendered from Markdown.
+
+Defaults to building a site with 5k markdown pages. Set the `NUM_PAGES` environment variable to change that e.g. `NUM_PAGES=25000 gatsby build`

benchmarks/markdown/gatsby-config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  plugins: [`gatsby-transformer-remark`],
+};

benchmarks/markdown/gatsby-node.js

@@ -0,0 +1,36 @@
+let NUM_PAGES = 5000
+
+if (process.env.NUM_PAGES) {
+  NUM_PAGES = process.env.NUM_PAGES
+}
+
+exports.sourceNodes = ({ actions: { createNode } }) => {
+  // Create markdown nodes
+  let step
+  for (step = 0; step < NUM_PAGES; step++) {
+    createNode({
+      id: step.toString(),
+      parent: null,
+      children: [],
+      internal: {
+        type: `FakeMarkdown`,
+        mediaType: `text/markdown`,
+        content: `---\ntitle: "hi"\n---\nWhat up folks? This is _lit_\n\n##Page #${step}`,
+        contentDigest: step.toString(),
+      },
+    })
+  }
+}
+
+exports.createPages = ({ actions: { createPage } }) => {
+  let step
+  for (step = 0; step < NUM_PAGES; step++) {
+    createPage({
+      path: `/path/${step}/`,
+      component: require.resolve(`./src/templates/blank.js`),
+      context: {
+        id: step,
+      },
+    })
+  }
+}

benchmarks/markdown/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "gatsby-starter-hello-world",
+  "description": "Gatsby hello world starter",
+  "license": "MIT",
+  "scripts": {
+    "develop": "gatsby develop",
+    "build": "gatsby build",
+    "serve": "gatsby serve"
+  },
+  "dependencies": {
+    "gatsby": "next",
+    "gatsby-transformer-remark": "^2.1.1-beta.2",
+    "react": "^16.3.2",
+    "react-dom": "^16.3.2"
+  }
+}

benchmarks/markdown/src/pages/index.js

@@ -0,0 +1,3 @@
+import React from "react"
+
+export default () => <div>Hello world!</div>

benchmarks/markdown/src/templates/blank.js

@@ -0,0 +1,25 @@
+import React from "react"
+import { graphql } from "gatsby"
+
+export default ({ data }) => {
+  const markdown = data.fakeMarkdown.childMarkdownRemark
+  return (
+    <div>
+      <h1>{markdown.frontmatter.title}</h1>
+      <div dangerouslySetInnerHTML={{ __html: markdown.html }} />
+    </div>
+  )
+}
+
+export const query = graphql`
+  query testing($id: String!) {
+    fakeMarkdown(id: { eq: $id }) {
+      childMarkdownRemark {
+        frontmatter {
+          title
+        }
+        html
+      }
+    }
+  }
+`

packages/gatsby-cli/package.json

@@ -18,6 +18,7 @@
     "core-js": "^2.5.0",
     "envinfo": "^5.8.1",
     "execa": "^0.8.0",
+    "fs-exists-cached": "^1.0.0",
     "fs-extra": "^4.0.1",
     "hosted-git-info": "^2.6.0",
     "lodash": "^4.17.4",

packages/gatsby-cli/src/create-cli.js

@@ -2,8 +2,8 @@ const path = require(`path`)
 const resolveCwd = require(`resolve-cwd`)
 const yargs = require(`yargs`)
 const report = require(`./reporter`)
-const fs = require(`fs`)
 const envinfo = require(`envinfo`)
+const existsSync = require(`fs-exists-cached`).sync
 
 const DEFAULT_BROWSERS = [`>0.25%`, `not dead`]
 
@@ -19,7 +19,7 @@ function buildLocalCommands(cli, isLocalSite) {
   const directory = path.resolve(`.`)
 
   let siteInfo = { directory, browserslist: DEFAULT_BROWSERS }
-  const useYarn = fs.existsSync(path.join(directory, `yarn.lock`))
+  const useYarn = existsSync(path.join(directory, `yarn.lock`))
   if (isLocalSite) {
     const json = require(path.join(directory, `package.json`))
     siteInfo.sitePackageJson = json

packages/gatsby-cli/src/init-starter.js

@@ -6,6 +6,7 @@ const fs = require(`fs-extra`)
 const sysPath = require(`path`)
 const report = require(`./reporter`)
 const url = require(`url`)
+const existsSync = require(`fs-exists-cached`).sync
 
 const spawn = (cmd: string) => {
   const [file, ...args] = cmd.split(/\s+/)
@@ -49,7 +50,7 @@ const copy = async (starterPath: string, rootPath: string) => {
   // 493 = parseInt('755', 8)
   await fs.mkdirp(rootPath, { mode: 493 })
 
-  if (!fs.existsSync(starterPath)) {
+  if (!existsSync(starterPath)) {
     throw new Error(`starter ${starterPath} doesn't exist`)
   }
 
@@ -117,7 +118,7 @@ module.exports = async (starter: string, options: InitOptions = {}) => {
     return
   }
 
-  if (fs.existsSync(sysPath.join(rootPath, `package.json`))) {
+  if (existsSync(sysPath.join(rootPath, `package.json`))) {
     report.panic(`Directory ${rootPath} is already an npm project`)
     return
   }

packages/gatsby-cli/src/reporter/index.js

@@ -75,6 +75,7 @@ module.exports = Object.assign(reporter, {
   activityTimer(name) {
     const spinner = reporter.activity()
     const start = process.hrtime()
+    let status
 
     const elapsedTime = () => {
       var elapsed = process.hrtime(start)
@@ -85,8 +86,15 @@ module.exports = Object.assign(reporter, {
       start: () => {
         spinner.tick(name)
       },
+      setStatus: s => {
+        status = s
+        spinner.tick(`${name} — ${status}`)
+      },
       end: () => {
-        reporter.success(`${name} — ${elapsedTime()}`)
+        const str = status
+          ? `${name} — ${elapsedTime()} — ${status}`
+          : `${name} — ${elapsedTime()}`
+        reporter.success(str)
         spinner.end()
       },
     }

packages/gatsby-plugin-netlify/src/gatsby-node.js

@@ -34,7 +34,7 @@ exports.onPostBuild = async ({ store, pathPrefix }, userPluginOptions) => {
   let rewrites = []
   if (pluginOptions.generateMatchPathRewrites) {
     const { pages } = store.getState()
-    rewrites = pages
+    rewrites = Array.from(pages.values())
       .filter(page => page.matchPath && page.matchPath !== page.path)
       .map(page => {
         return {