diff --git a/.gitignore b/.gitignore index 37b0891d..e4f1f89c 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,7 @@ storybook-static # mdx enhanced .mdx-data .mongo + +# next-sitemap +/public/sitemap.xml +/public/robots.txt \ No newline at end of file diff --git a/package.json b/package.json index 66119dbf..8c61918e 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "debug": "NODE_OPTIONS='--inspect' next", "build": "next build", "start": "next start", + "postbuild": "next-sitemap --config vulcan-next-sitemap.js", "test": "npm run test:unit && npm run test:e2e", "clean": "rm -Rf ./dist # clean various build folders", "postinstall": "", @@ -76,6 +77,7 @@ "next-connect": "^0.9.1", "next-i18next": "^5.1.0", "next-mdx-enhanced": "^4.0.0", + "next-sitemap": "^1.4.17", "passport": "^0.4.1", "passport-local": "1.0.0", "polished": "^3.6.5", diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 00000000..4100ba25 --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,14 @@ +# * +User-agent: * +Disallow: /vns/* +Disallow: /login +Disallow: /profile +Disallow: /signup +Disallow: /admin +Disallow: /auth + +# Host +Host: https://vulcan.next + +# Sitemaps +Sitemap: https://vulcan.next/sitemap.xml diff --git a/public/sitemap.xml b/public/sitemap.xml new file mode 100644 index 00000000..6b12b53c --- /dev/null +++ b/public/sitemap.xml @@ -0,0 +1,11 @@ + + +https://vulcan.nextdaily0.72021-02-04T08:22:19.495Z +https://vulcan.next/docsdaily0.72021-02-04T08:22:19.495Z +https://vulcan.next/docs/contributingdaily0.72021-02-04T08:22:19.495Z +https://vulcan.next/docs/featuresdaily0.72021-02-04T08:22:19.495Z +https://vulcan.next/docs/from-vulcan-v1daily0.72021-02-04T08:22:19.495Z +https://vulcan.next/docs/learningsdaily0.72021-02-04T08:22:19.495Z +https://vulcan.next/docs/recipesdaily0.72021-02-04T08:22:19.495Z +https://vulcan.next/docs/release-processdaily0.72021-02-04T08:22:19.495Z + \ No newline at end of file diff --git a/src/pages/docs/features.md b/src/pages/docs/features.md index 6af6b5c5..2df6b60c 100644 --- a/src/pages/docs/features.md +++ b/src/pages/docs/features.md @@ -131,6 +131,10 @@ In `next.config.js`, you'll find a demonstration of how to **safely** inject inf For example, we use it to inject current version into the `html` tag for better deployment tracking. +### Sitemap.xml and Robots.txt with next-sitemap + +We use [next-sitemap](https://github.com/iamvishnusankar/next-sitemap#readme) to create both the `robots.txt` and `sitemap.xml` in the `postbuild` script. Change `https://vulcan.next` to your root url in `/vulcan-next-sitemap.js`. Here's more [configuration options](https://github.com/iamvishnusankar/next-sitemap#configuration-options). + ### Performance debugging [See official doc](https://nextjs.org/docs/advanced-features/measuring-performance). diff --git a/vulcan-next-sitemap.js b/vulcan-next-sitemap.js new file mode 100644 index 00000000..3530a5ec --- /dev/null +++ b/vulcan-next-sitemap.js @@ -0,0 +1,15 @@ +module.exports = { + siteUrl: 'https://vulcan.next', + generateRobotsTxt: true, + //outDir: './out/', for static builds + exclude: ['/vns/*', '/login', '/profile', '/signup', '/admin', '/auth'], + robotsTxtOptions: { + policies: [ + { + userAgent: '*', + disallow: ['/vns/*', '/login', '/profile', '/signup', '/admin', '/auth'], + //allow: ['/','/docs/*'], + }, + ] + } +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 1e90cb88..0b9157d0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2234,6 +2234,11 @@ exec-sh "^0.3.2" minimist "^1.2.0" +"@corex/deepmerge@^2.5.3": + version "2.5.3" + resolved "https://registry.yarnpkg.com/@corex/deepmerge/-/deepmerge-2.5.3.tgz#ecdb8e4c0617ada2a5af0c0400f56562d1f0ea31" + integrity sha512-9q+jWQ6zqBc7iU2ItxYcoA+G/QUxPt0eo7d3PVNy30HOCpIEX/oZPmtoC3+XSJNyHdV39MGrq+EIpVh81jcapw== + "@cypress/browserify-preprocessor@3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@cypress/browserify-preprocessor/-/browserify-preprocessor-3.0.0.tgz#2d1fa6a96ed7130a1b172c540448a5955cbc1264" @@ -8885,6 +8890,11 @@ escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5: resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= +escape-string-regexp@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" + integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== + escodegen@^1.12.0, escodegen@^1.14.1: version "1.14.3" resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-1.14.3.tgz#4e7b81fba61581dc97582ed78cab7f0e8d63f503" @@ -12654,6 +12664,13 @@ markdown-to-jsx@^6.11.4: prop-types "^15.6.2" unquote "^1.1.0" +matcher@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/matcher/-/matcher-3.0.0.tgz#bd9060f4c5b70aa8041ccc6f80368760994f30ca" + integrity sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng== + dependencies: + escape-string-regexp "^4.0.0" + material-colors@^1.2.1: version "1.2.6" resolved "https://registry.yarnpkg.com/material-colors/-/material-colors-1.2.6.tgz#6d1958871126992ceecc72f4bcc4d8f010865f46" @@ -13288,6 +13305,15 @@ next-mdx-enhanced@^4.0.0: prebuild-webpack-plugin "1.1.1" stringify-object "^3.3.0" +next-sitemap@^1.4.17: + version "1.4.17" + resolved "https://registry.yarnpkg.com/next-sitemap/-/next-sitemap-1.4.17.tgz#24bb24105f59383ff0145b89a771849e3babbb5b" + integrity sha512-+y1RxdH/O4O2TifaNDQPmIU4h9gFxqJrul2CN+Qx/ziBEyIrWumSgb6XfMquEpwfdq/ZtjD1TfxdRujyyHDybQ== + dependencies: + "@corex/deepmerge" "^2.5.3" + matcher "^3.0.0" + minimist "^1.2.5" + next-tick@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c"