fix: generate better slugs for non latin langs (close #45)

docs/config/README.md

@@ -113,19 +113,26 @@ Provide config options to the used theme. The options will vary depending on the
 
 ## Markdown
 
+### markdown.slugify
+
+- Type: `Function`
+- Default: [source](https://github.com/vuejs/vuepress/blob/master/lib/markdown/slugify.js)
+
+Function for transforming header texts into slugs. This affects the ids/links generated for header anchors, table of contents and sidebar links.
+
 ### markdown.anchor
 
 - Type: `Object`
 - Default: `{ permalink: true, permalinkBefore: true, permalinkSymbol: '#' }`
 
-Options for [markdown-it-anchor](https://github.com/valeriangalliat/markdown-it-anchor).
+Options for [markdown-it-anchor](https://github.com/valeriangalliat/markdown-it-anchor). (Note: prefer `markdown.slugify` if you want to customize header ids.)
 
 ### markdown.toc
 
 - Type: `Object`
 - Default: `{ includeLevel: [2, 3] }`
 
-Options for [markdown-it-table-of-contents](https://github.com/Oktavilla/markdown-it-table-of-contents).
+Options for [markdown-it-table-of-contents](https://github.com/Oktavilla/markdown-it-table-of-contents). (Note: prefer `markdown.slugify` if you want to customize header ids.)
 
 ### markdown.config
 

lib/markdown/index.js

@@ -7,8 +7,12 @@ const containers = require('./containers')
 const emoji = require('markdown-it-emoji')
 const anchor = require('markdown-it-anchor')
 const toc = require('markdown-it-table-of-contents')
+const _slugify = require('./slugify')
+
+module.exports = ({ markdown = {}} = {}) => {
+  // allow user config slugify
+  const slugify = markdown.slugify || _slugify
 
-module.exports = ({ markdown = {}}) => {
   const md = require('markdown-it')({
     html: true,
     typographer: true,
@@ -24,11 +28,13 @@ module.exports = ({ markdown = {}}) => {
     // 3rd party plugins
     .use(emoji)
     .use(anchor, Object.assign({
+      slugify,
       permalink: true,
       permalinkBefore: true,
       permalinkSymbol: '#'
     }, markdown.anchor))
     .use(toc, Object.assign({
+      slugify,
       includeLevel: [2, 3]
     }, markdown.toc))
 
@@ -48,5 +54,8 @@ module.exports = ({ markdown = {}}) => {
     }
   }
 
+  // expose slugify
+  md.slugify = slugify
+
   return md
 }

lib/markdown/slugify.js

@@ -0,0 +1,20 @@
+// string.js slugify drops non ascii chars so we have to
+// use a custom implementation here
+const removeDiacritics = require('diacritics').remove
+// eslint-disable-next-line no-control-regex
+const rControl = /[\u0000-\u001f]/g
+const rSpecial = /[\s~`!@#$%^&*()\-_+=[\]{}|\\;:"'<>,.?/]+/g
+
+module.exports = function slugify (str) {
+  return removeDiacritics(str)
+    // Remove control characters
+    .replace(rControl, '')
+    // Replace special characters
+    .replace(rSpecial, '-')
+    // Remove continous separators
+    .replace(/\-{2,}/g, '-')
+    // Remove prefixing and trailing separtors
+    .replace(/^\-+|\-+$/g, '')
+    // lowercase
+    .toLowerCase()
+}

lib/util.js → lib/util/index.js

@@ -45,6 +45,7 @@ exports.parseFrontmatter = content => {
 
 const LRU = require('lru-cache')
 const cache = LRU({ max: 1000 })
+const md = require('../markdown')()
 
 exports.extractHeaders = (content, include = []) => {
   const key = content + include.join(',')
@@ -53,8 +54,6 @@ exports.extractHeaders = (content, include = []) => {
     return hit
   }
 
-  const md = require('./markdown')({})
-  const S = require('string')
   const tokens = md.parse(content, {})
 
   const res = []
@@ -64,7 +63,7 @@ exports.extractHeaders = (content, include = []) => {
       res.push({
         level: parseInt(t.tag.slice(1), 10),
         title,
-        slug: S(title).slugify().s
+        slug: md.slugify(title)
       })
     }
   })

package.json

@@ -50,6 +50,7 @@
     "connect-history-api-fallback": "^1.5.0",
     "copy-webpack-plugin": "^4.5.1",
     "css-loader": "^0.28.11",
+    "diacritics": "^1.3.0",
     "es6-promise": "^4.2.4",
     "escape-html": "^1.0.3",
     "file-loader": "^1.1.11",

yarn.lock

@@ -1381,6 +1381,10 @@ detect-libc@^1.0.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b"
 
+diacritics@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/diacritics/-/diacritics-1.3.0.tgz#3efa87323ebb863e6696cebb0082d48ff3d6f7a1"
+
 diff@^3.2.0:
   version "3.5.0"
   resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12"