From 2ef8181f1fea8b02314dddcb8061c7ac0808fd26 Mon Sep 17 00:00:00 2001 From: Dougal Matthews Date: Sun, 29 Mar 2015 10:04:59 +0100 Subject: [PATCH] Create a new search implementation with Lunr.js This rebases the work done in #222 and adapts it for to use Lunr rather than Tipue. --- .../assets/search/mkdocs/js/lunr-0.5.7.min.js | 7 + .../assets/search/mkdocs/js/mustache.min.js | 1 + mkdocs/assets/search/mkdocs/js/require.js | 36 ++ .../js/search-results-template.mustache | 4 + mkdocs/assets/search/mkdocs/js/search.js | 68 +++ mkdocs/assets/search/mkdocs/js/text.js | 390 +++++++++++++++++ mkdocs/build.py | 38 +- mkdocs/compat.py | 34 +- mkdocs/config.py | 13 +- mkdocs/nav.py | 3 + mkdocs/search.py | 203 +++++++++ mkdocs/serve.py | 13 +- mkdocs/test.py | 158 ++++++- mkdocs/themes/amelia/base.html | 2 - mkdocs/themes/amelia/nav.html | 5 - mkdocs/themes/bootstrap/base.html | 1 - mkdocs/themes/bootstrap/nav.html | 5 - mkdocs/themes/cerulean/base.html | 2 - mkdocs/themes/cerulean/nav.html | 5 - mkdocs/themes/cosmo/base.html | 2 - mkdocs/themes/cosmo/nav.html | 5 - mkdocs/themes/cyborg/base.html | 2 - mkdocs/themes/cyborg/nav.html | 5 - mkdocs/themes/flatly/base.html | 2 - mkdocs/themes/flatly/nav.html | 5 - mkdocs/themes/journal/base.html | 2 - mkdocs/themes/journal/nav.html | 5 - mkdocs/themes/mkdocs/base.html | 36 +- mkdocs/themes/mkdocs/css/base.css | 2 +- mkdocs/themes/mkdocs/js/base.js | 102 +++-- mkdocs/themes/mkdocs/nav.html | 51 ++- mkdocs/themes/mkdocs/search.html | 17 - mkdocs/themes/readable/base.html | 2 - mkdocs/themes/readable/nav.html | 5 - mkdocs/themes/readthedocs/base.html | 28 +- mkdocs/themes/readthedocs/breadcrumbs.html | 16 +- mkdocs/themes/readthedocs/js/theme.js | 4 + mkdocs/themes/readthedocs/search.html | 56 +-- mkdocs/themes/readthedocs/searchbox.html | 4 +- mkdocs/themes/simplex/base.html | 2 - mkdocs/themes/simplex/nav.html | 5 - mkdocs/themes/slate/base.html | 2 - mkdocs/themes/slate/nav.html | 5 - mkdocs/themes/spacelab/base.html | 2 - mkdocs/themes/spacelab/nav.html | 5 - mkdocs/themes/united/base.html | 2 - mkdocs/themes/united/nav.html | 5 - mkdocs/themes/yeti/base.html | 2 - mkdocs/themes/yeti/nav.html | 5 - package.json | 7 + text.js | 391 ++++++++++++++++++ tox.ini | 2 + 52 files changed, 1477 insertions(+), 297 deletions(-) create mode 100644 mkdocs/assets/search/mkdocs/js/lunr-0.5.7.min.js create mode 100644 mkdocs/assets/search/mkdocs/js/mustache.min.js create mode 100644 mkdocs/assets/search/mkdocs/js/require.js create mode 100644 mkdocs/assets/search/mkdocs/js/search-results-template.mustache create mode 100644 mkdocs/assets/search/mkdocs/js/search.js create mode 100644 mkdocs/assets/search/mkdocs/js/text.js create mode 100644 mkdocs/search.py delete mode 100644 mkdocs/themes/mkdocs/search.html create mode 100644 package.json create mode 100644 text.js diff --git a/mkdocs/assets/search/mkdocs/js/lunr-0.5.7.min.js b/mkdocs/assets/search/mkdocs/js/lunr-0.5.7.min.js new file mode 100644 index 0000000000..b72449aab6 --- /dev/null +++ b/mkdocs/assets/search/mkdocs/js/lunr-0.5.7.min.js @@ -0,0 +1,7 @@ +/** + * lunr - http://lunrjs.com - A bit like Solr, but much smaller and not as bright - 0.5.7 + * Copyright (C) 2014 Oliver Nightingale + * MIT Licensed + * @license + */ +!function(){var t=function(e){var n=new t.Index;return n.pipeline.add(t.trimmer,t.stopWordFilter,t.stemmer),e&&e.call(n,n),n};t.version="0.5.7",t.utils={},t.utils.warn=function(t){return function(e){t.console&&console.warn&&console.warn(e)}}(this),t.EventEmitter=function(){this.events={}},t.EventEmitter.prototype.addListener=function(){var t=Array.prototype.slice.call(arguments),e=t.pop(),n=t;if("function"!=typeof e)throw new TypeError("last argument must be a function");n.forEach(function(t){this.hasHandler(t)||(this.events[t]=[]),this.events[t].push(e)},this)},t.EventEmitter.prototype.removeListener=function(t,e){if(this.hasHandler(t)){var n=this.events[t].indexOf(e);this.events[t].splice(n,1),this.events[t].length||delete this.events[t]}},t.EventEmitter.prototype.emit=function(t){if(this.hasHandler(t)){var e=Array.prototype.slice.call(arguments,1);this.events[t].forEach(function(t){t.apply(void 0,e)})}},t.EventEmitter.prototype.hasHandler=function(t){return t in this.events},t.tokenizer=function(t){if(!arguments.length||null==t||void 0==t)return[];if(Array.isArray(t))return t.map(function(t){return t.toLowerCase()});for(var e=t.toString().replace(/^\s+/,""),n=e.length-1;n>=0;n--)if(/\S/.test(e.charAt(n))){e=e.substring(0,n+1);break}return e.split(/(?:\s+|\-)/).filter(function(t){return!!t}).map(function(t){return t.toLowerCase()})},t.Pipeline=function(){this._stack=[]},t.Pipeline.registeredFunctions={},t.Pipeline.registerFunction=function(e,n){n in this.registeredFunctions&&t.utils.warn("Overwriting existing registered function: "+n),e.label=n,t.Pipeline.registeredFunctions[e.label]=e},t.Pipeline.warnIfFunctionNotRegistered=function(e){var n=e.label&&e.label in this.registeredFunctions;n||t.utils.warn("Function is not registered with pipeline. This may cause problems when serialising the index.\n",e)},t.Pipeline.load=function(e){var n=new t.Pipeline;return e.forEach(function(e){var i=t.Pipeline.registeredFunctions[e];if(!i)throw new Error("Cannot load un-registered function: "+e);n.add(i)}),n},t.Pipeline.prototype.add=function(){var e=Array.prototype.slice.call(arguments);e.forEach(function(e){t.Pipeline.warnIfFunctionNotRegistered(e),this._stack.push(e)},this)},t.Pipeline.prototype.after=function(e,n){t.Pipeline.warnIfFunctionNotRegistered(n);var i=this._stack.indexOf(e)+1;this._stack.splice(i,0,n)},t.Pipeline.prototype.before=function(e,n){t.Pipeline.warnIfFunctionNotRegistered(n);var i=this._stack.indexOf(e);this._stack.splice(i,0,n)},t.Pipeline.prototype.remove=function(t){var e=this._stack.indexOf(t);this._stack.splice(e,1)},t.Pipeline.prototype.run=function(t){for(var e=[],n=t.length,i=this._stack.length,o=0;n>o;o++){for(var r=t[o],s=0;i>s&&(r=this._stack[s](r,o,t),void 0!==r);s++);void 0!==r&&e.push(r)}return e},t.Pipeline.prototype.reset=function(){this._stack=[]},t.Pipeline.prototype.toJSON=function(){return this._stack.map(function(e){return t.Pipeline.warnIfFunctionNotRegistered(e),e.label})},t.Vector=function(){this._magnitude=null,this.list=void 0,this.length=0},t.Vector.Node=function(t,e,n){this.idx=t,this.val=e,this.next=n},t.Vector.prototype.insert=function(e,n){var i=this.list;if(!i)return this.list=new t.Vector.Node(e,n,i),this.length++;for(var o=i,r=i.next;void 0!=r;){if(en.idx?n=n.next:(i+=e.val*n.val,e=e.next,n=n.next);return i},t.Vector.prototype.similarity=function(t){return this.dot(t)/(this.magnitude()*t.magnitude())},t.SortedSet=function(){this.length=0,this.elements=[]},t.SortedSet.load=function(t){var e=new this;return e.elements=t,e.length=t.length,e},t.SortedSet.prototype.add=function(){Array.prototype.slice.call(arguments).forEach(function(t){~this.indexOf(t)||this.elements.splice(this.locationFor(t),0,t)},this),this.length=this.elements.length},t.SortedSet.prototype.toArray=function(){return this.elements.slice()},t.SortedSet.prototype.map=function(t,e){return this.elements.map(t,e)},t.SortedSet.prototype.forEach=function(t,e){return this.elements.forEach(t,e)},t.SortedSet.prototype.indexOf=function(t,e,n){var e=e||0,n=n||this.elements.length,i=n-e,o=e+Math.floor(i/2),r=this.elements[o];return 1>=i?r===t?o:-1:t>r?this.indexOf(t,o,n):r>t?this.indexOf(t,e,o):r===t?o:void 0},t.SortedSet.prototype.locationFor=function(t,e,n){var e=e||0,n=n||this.elements.length,i=n-e,o=e+Math.floor(i/2),r=this.elements[o];if(1>=i){if(r>t)return o;if(t>r)return o+1}return t>r?this.locationFor(t,o,n):r>t?this.locationFor(t,e,o):void 0},t.SortedSet.prototype.intersect=function(e){for(var n=new t.SortedSet,i=0,o=0,r=this.length,s=e.length,a=this.elements,h=e.elements;;){if(i>r-1||o>s-1)break;a[i]!==h[o]?a[i]h[o]&&o++:(n.add(a[i]),i++,o++)}return n},t.SortedSet.prototype.clone=function(){var e=new t.SortedSet;return e.elements=this.toArray(),e.length=e.elements.length,e},t.SortedSet.prototype.union=function(t){var e,n,i;return this.length>=t.length?(e=this,n=t):(e=t,n=this),i=e.clone(),i.add.apply(i,n.toArray()),i},t.SortedSet.prototype.toJSON=function(){return this.toArray()},t.Index=function(){this._fields=[],this._ref="id",this.pipeline=new t.Pipeline,this.documentStore=new t.Store,this.tokenStore=new t.TokenStore,this.corpusTokens=new t.SortedSet,this.eventEmitter=new t.EventEmitter,this._idfCache={},this.on("add","remove","update",function(){this._idfCache={}}.bind(this))},t.Index.prototype.on=function(){var t=Array.prototype.slice.call(arguments);return this.eventEmitter.addListener.apply(this.eventEmitter,t)},t.Index.prototype.off=function(t,e){return this.eventEmitter.removeListener(t,e)},t.Index.load=function(e){e.version!==t.version&&t.utils.warn("version mismatch: current "+t.version+" importing "+e.version);var n=new this;return n._fields=e.fields,n._ref=e.ref,n.documentStore=t.Store.load(e.documentStore),n.tokenStore=t.TokenStore.load(e.tokenStore),n.corpusTokens=t.SortedSet.load(e.corpusTokens),n.pipeline=t.Pipeline.load(e.pipeline),n},t.Index.prototype.field=function(t,e){var e=e||{},n={name:t,boost:e.boost||1};return this._fields.push(n),this},t.Index.prototype.ref=function(t){return this._ref=t,this},t.Index.prototype.add=function(e,n){var i={},o=new t.SortedSet,r=e[this._ref],n=void 0===n?!0:n;this._fields.forEach(function(n){var r=this.pipeline.run(t.tokenizer(e[n.name]));i[n.name]=r,t.SortedSet.prototype.add.apply(o,r)},this),this.documentStore.set(r,o),t.SortedSet.prototype.add.apply(this.corpusTokens,o.toArray());for(var s=0;s0&&(i=1+Math.log(this.tokenStore.length/n)),this._idfCache[e]=i},t.Index.prototype.search=function(e){var n=this.pipeline.run(t.tokenizer(e)),i=new t.Vector,o=[],r=this._fields.reduce(function(t,e){return t+e.boost},0),s=n.some(function(t){return this.tokenStore.has(t)},this);if(!s)return[];n.forEach(function(e,n,s){var a=1/s.length*this._fields.length*r,h=this,u=this.tokenStore.expand(e).reduce(function(n,o){var r=h.corpusTokens.indexOf(o),s=h.idf(o),u=1,l=new t.SortedSet;if(o!==e){var c=Math.max(3,o.length-e.length);u=1/Math.log(c)}return r>-1&&i.insert(r,a*s*u),Object.keys(h.tokenStore.get(o)).forEach(function(t){l.add(t)}),n.union(l)},new t.SortedSet);o.push(u)},this);var a=o.reduce(function(t,e){return t.intersect(e)});return a.map(function(t){return{ref:t,score:i.similarity(this.documentVector(t))}},this).sort(function(t,e){return e.score-t.score})},t.Index.prototype.documentVector=function(e){for(var n=this.documentStore.get(e),i=n.length,o=new t.Vector,r=0;i>r;r++){var s=n.elements[r],a=this.tokenStore.get(s)[e].tf,h=this.idf(s);o.insert(this.corpusTokens.indexOf(s),a*h)}return o},t.Index.prototype.toJSON=function(){return{version:t.version,fields:this._fields,ref:this._ref,documentStore:this.documentStore.toJSON(),tokenStore:this.tokenStore.toJSON(),corpusTokens:this.corpusTokens.toJSON(),pipeline:this.pipeline.toJSON()}},t.Index.prototype.use=function(t){var e=Array.prototype.slice.call(arguments,1);e.unshift(this),t.apply(this,e)},t.Store=function(){this.store={},this.length=0},t.Store.load=function(e){var n=new this;return n.length=e.length,n.store=Object.keys(e.store).reduce(function(n,i){return n[i]=t.SortedSet.load(e.store[i]),n},{}),n},t.Store.prototype.set=function(t,e){this.has(t)||this.length++,this.store[t]=e},t.Store.prototype.get=function(t){return this.store[t]},t.Store.prototype.has=function(t){return t in this.store},t.Store.prototype.remove=function(t){this.has(t)&&(delete this.store[t],this.length--)},t.Store.prototype.toJSON=function(){return{store:this.store,length:this.length}},t.stemmer=function(){var t={ational:"ate",tional:"tion",enci:"ence",anci:"ance",izer:"ize",bli:"ble",alli:"al",entli:"ent",eli:"e",ousli:"ous",ization:"ize",ation:"ate",ator:"ate",alism:"al",iveness:"ive",fulness:"ful",ousness:"ous",aliti:"al",iviti:"ive",biliti:"ble",logi:"log"},e={icate:"ic",ative:"",alize:"al",iciti:"ic",ical:"ic",ful:"",ness:""},n="[^aeiou]",i="[aeiouy]",o=n+"[^aeiouy]*",r=i+"[aeiou]*",s="^("+o+")?"+r+o,a="^("+o+")?"+r+o+"("+r+")?$",h="^("+o+")?"+r+o+r+o,u="^("+o+")?"+i,l=new RegExp(s),c=new RegExp(h),p=new RegExp(a),f=new RegExp(u),d=/^(.+?)(ss|i)es$/,v=/^(.+?)([^s])s$/,m=/^(.+?)eed$/,g=/^(.+?)(ed|ing)$/,y=/.$/,S=/(at|bl|iz)$/,w=new RegExp("([^aeiouylsz])\\1$"),x=new RegExp("^"+o+i+"[^aeiouwxy]$"),k=/^(.+?[^aeiou])y$/,b=/^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/,E=/^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/,_=/^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/,O=/^(.+?)(s|t)(ion)$/,F=/^(.+?)e$/,P=/ll$/,T=new RegExp("^"+o+i+"[^aeiouwxy]$"),$=function(n){var i,o,r,s,a,h,u;if(n.length<3)return n;if(r=n.substr(0,1),"y"==r&&(n=r.toUpperCase()+n.substr(1)),s=d,a=v,s.test(n)?n=n.replace(s,"$1$2"):a.test(n)&&(n=n.replace(a,"$1$2")),s=m,a=g,s.test(n)){var $=s.exec(n);s=l,s.test($[1])&&(s=y,n=n.replace(s,""))}else if(a.test(n)){var $=a.exec(n);i=$[1],a=f,a.test(i)&&(n=i,a=S,h=w,u=x,a.test(n)?n+="e":h.test(n)?(s=y,n=n.replace(s,"")):u.test(n)&&(n+="e"))}if(s=k,s.test(n)){var $=s.exec(n);i=$[1],n=i+"i"}if(s=b,s.test(n)){var $=s.exec(n);i=$[1],o=$[2],s=l,s.test(i)&&(n=i+t[o])}if(s=E,s.test(n)){var $=s.exec(n);i=$[1],o=$[2],s=l,s.test(i)&&(n=i+e[o])}if(s=_,a=O,s.test(n)){var $=s.exec(n);i=$[1],s=c,s.test(i)&&(n=i)}else if(a.test(n)){var $=a.exec(n);i=$[1]+$[2],a=c,a.test(i)&&(n=i)}if(s=F,s.test(n)){var $=s.exec(n);i=$[1],s=c,a=p,h=T,(s.test(i)||a.test(i)&&!h.test(i))&&(n=i)}return s=P,a=c,s.test(n)&&a.test(n)&&(s=y,n=n.replace(s,"")),"y"==r&&(n=r.toLowerCase()+n.substr(1)),n};return $}(),t.Pipeline.registerFunction(t.stemmer,"stemmer"),t.stopWordFilter=function(e){return-1===t.stopWordFilter.stopWords.indexOf(e)?e:void 0},t.stopWordFilter.stopWords=new t.SortedSet,t.stopWordFilter.stopWords.length=119,t.stopWordFilter.stopWords.elements=["","a","able","about","across","after","all","almost","also","am","among","an","and","any","are","as","at","be","because","been","but","by","can","cannot","could","dear","did","do","does","either","else","ever","every","for","from","get","got","had","has","have","he","her","hers","him","his","how","however","i","if","in","into","is","it","its","just","least","let","like","likely","may","me","might","most","must","my","neither","no","nor","not","of","off","often","on","only","or","other","our","own","rather","said","say","says","she","should","since","so","some","than","that","the","their","them","then","there","these","they","this","tis","to","too","twas","us","wants","was","we","were","what","when","where","which","while","who","whom","why","will","with","would","yet","you","your"],t.Pipeline.registerFunction(t.stopWordFilter,"stopWordFilter"),t.trimmer=function(t){return t.replace(/^\W+/,"").replace(/\W+$/,"")},t.Pipeline.registerFunction(t.trimmer,"trimmer"),t.TokenStore=function(){this.root={docs:{}},this.length=0},t.TokenStore.load=function(t){var e=new this;return e.root=t.root,e.length=t.length,e},t.TokenStore.prototype.add=function(t,e,n){var n=n||this.root,i=t[0],o=t.slice(1);return i in n||(n[i]={docs:{}}),0===o.length?(n[i].docs[e.ref]=e,void(this.length+=1)):this.add(o,e,n[i])},t.TokenStore.prototype.has=function(t){if(!t)return!1;for(var e=this.root,n=0;n":">",'"':""","'":"'","/":"/"};function escapeHtml(string){return String(string).replace(/[&<>"'\/]/g,function(s){return entityMap[s]})}var whiteRe=/\s*/;var spaceRe=/\s+/;var equalsRe=/\s*=/;var curlyRe=/\s*\}/;var tagRe=/#|\^|\/|>|\{|&|=|!/;function parseTemplate(template,tags){if(!template)return[];var sections=[];var tokens=[];var spaces=[];var hasTag=false;var nonSpace=false;function stripSpace(){if(hasTag&&!nonSpace){while(spaces.length)delete tokens[spaces.pop()]}else{spaces=[]}hasTag=false;nonSpace=false}var openingTagRe,closingTagRe,closingCurlyRe;function compileTags(tags){if(typeof tags==="string")tags=tags.split(spaceRe,2);if(!isArray(tags)||tags.length!==2)throw new Error("Invalid tags: "+tags);openingTagRe=new RegExp(escapeRegExp(tags[0])+"\\s*");closingTagRe=new RegExp("\\s*"+escapeRegExp(tags[1]));closingCurlyRe=new RegExp("\\s*"+escapeRegExp("}"+tags[1]))}compileTags(tags||mustache.tags);var scanner=new Scanner(template);var start,type,value,chr,token,openSection;while(!scanner.eos()){start=scanner.pos;value=scanner.scanUntil(openingTagRe);if(value){for(var i=0,valueLength=value.length;i0?sections[sections.length-1][4]:nestedTokens;break;default:collector.push(token)}}return nestedTokens}function Scanner(string){this.string=string;this.tail=string;this.pos=0}Scanner.prototype.eos=function(){return this.tail===""};Scanner.prototype.scan=function(re){var match=this.tail.match(re);if(!match||match.index!==0)return"";var string=match[0];this.tail=this.tail.substring(string.length);this.pos+=string.length;return string};Scanner.prototype.scanUntil=function(re){var index=this.tail.search(re),match;switch(index){case-1:match=this.tail;this.tail="";break;case 0:match="";break;default:match=this.tail.substring(0,index);this.tail=this.tail.substring(index)}this.pos+=match.length;return match};function Context(view,parentContext){this.view=view;this.cache={".":this.view};this.parent=parentContext}Context.prototype.push=function(view){return new Context(view,this)};Context.prototype.lookup=function(name){var cache=this.cache;var value;if(name in cache){value=cache[name]}else{var context=this,names,index,lookupHit=false;while(context){if(name.indexOf(".")>0){value=context.view;names=name.split(".");index=0;while(value!=null&&index")value=this._renderPartial(token,context,partials,originalTemplate);else if(symbol==="&")value=this._unescapedValue(token,context);else if(symbol==="name")value=this._escapedValue(token,context);else if(symbol==="text")value=this._rawValue(token);if(value!==undefined)buffer+=value}return buffer};Writer.prototype._renderSection=function(token,context,partials,originalTemplate){var self=this;var buffer="";var value=context.lookup(token[1]);function subRender(template){return self.render(template,context,partials)}if(!value)return;if(isArray(value)){for(var j=0,valueLength=value.length;jthis.depCount&&!this.defined){if(G(l)){if(this.events.error&&this.map.isDefine||g.onError!==ca)try{f=i.execCb(c,l,b,f)}catch(d){a=d}else f=i.execCb(c,l,b,f);this.map.isDefine&&void 0===f&&((b=this.module)?f=b.exports:this.usingExports&& +(f=this.exports));if(a)return a.requireMap=this.map,a.requireModules=this.map.isDefine?[this.map.id]:null,a.requireType=this.map.isDefine?"define":"require",w(this.error=a)}else f=l;this.exports=f;if(this.map.isDefine&&!this.ignore&&(r[c]=f,g.onResourceLoad))g.onResourceLoad(i,this.map,this.depMaps);y(c);this.defined=!0}this.defining=!1;this.defined&&!this.defineEmitted&&(this.defineEmitted=!0,this.emit("defined",this.exports),this.defineEmitComplete=!0)}}else this.fetch()}},callPlugin:function(){var a= +this.map,b=a.id,d=p(a.prefix);this.depMaps.push(d);q(d,"defined",u(this,function(f){var l,d;d=m(aa,this.map.id);var e=this.map.name,P=this.map.parentMap?this.map.parentMap.name:null,n=i.makeRequire(a.parentMap,{enableBuildCallback:!0});if(this.map.unnormalized){if(f.normalize&&(e=f.normalize(e,function(a){return c(a,P,!0)})||""),f=p(a.prefix+"!"+e,this.map.parentMap),q(f,"defined",u(this,function(a){this.init([],function(){return a},null,{enabled:!0,ignore:!0})})),d=m(h,f.id)){this.depMaps.push(f); +if(this.events.error)d.on("error",u(this,function(a){this.emit("error",a)}));d.enable()}}else d?(this.map.url=i.nameToUrl(d),this.load()):(l=u(this,function(a){this.init([],function(){return a},null,{enabled:!0})}),l.error=u(this,function(a){this.inited=!0;this.error=a;a.requireModules=[b];B(h,function(a){0===a.map.id.indexOf(b+"_unnormalized")&&y(a.map.id)});w(a)}),l.fromText=u(this,function(f,c){var d=a.name,e=p(d),P=M;c&&(f=c);P&&(M=!1);s(e);t(j.config,b)&&(j.config[d]=j.config[b]);try{g.exec(f)}catch(h){return w(C("fromtexteval", +"fromText eval for "+b+" failed: "+h,h,[b]))}P&&(M=!0);this.depMaps.push(e);i.completeLoad(d);n([d],l)}),f.load(a.name,n,l,j))}));i.enable(d,this);this.pluginMaps[d.id]=d},enable:function(){V[this.map.id]=this;this.enabling=this.enabled=!0;v(this.depMaps,u(this,function(a,b){var c,f;if("string"===typeof a){a=p(a,this.map.isDefine?this.map:this.map.parentMap,!1,!this.skipMap);this.depMaps[b]=a;if(c=m(L,a.id)){this.depExports[b]=c(this);return}this.depCount+=1;q(a,"defined",u(this,function(a){this.defineDep(b, +a);this.check()}));this.errback?q(a,"error",u(this,this.errback)):this.events.error&&q(a,"error",u(this,function(a){this.emit("error",a)}))}c=a.id;f=h[c];!t(L,c)&&(f&&!f.enabled)&&i.enable(a,this)}));B(this.pluginMaps,u(this,function(a){var b=m(h,a.id);b&&!b.enabled&&i.enable(a,this)}));this.enabling=!1;this.check()},on:function(a,b){var c=this.events[a];c||(c=this.events[a]=[]);c.push(b)},emit:function(a,b){v(this.events[a],function(a){a(b)});"error"===a&&delete this.events[a]}};i={config:j,contextName:b, +registry:h,defined:r,urlFetched:S,defQueue:A,Module:Z,makeModuleMap:p,nextTick:g.nextTick,onError:w,configure:function(a){a.baseUrl&&"/"!==a.baseUrl.charAt(a.baseUrl.length-1)&&(a.baseUrl+="/");var b=j.shim,c={paths:!0,bundles:!0,config:!0,map:!0};B(a,function(a,b){c[b]?(j[b]||(j[b]={}),U(j[b],a,!0,!0)):j[b]=a});a.bundles&&B(a.bundles,function(a,b){v(a,function(a){a!==b&&(aa[a]=b)})});a.shim&&(B(a.shim,function(a,c){H(a)&&(a={deps:a});if((a.exports||a.init)&&!a.exportsFn)a.exportsFn=i.makeShimExports(a); +b[c]=a}),j.shim=b);a.packages&&v(a.packages,function(a){var b,a="string"===typeof a?{name:a}:a;b=a.name;a.location&&(j.paths[b]=a.location);j.pkgs[b]=a.name+"/"+(a.main||"main").replace(ia,"").replace(Q,"")});B(h,function(a,b){!a.inited&&!a.map.unnormalized&&(a.map=p(b))});if(a.deps||a.callback)i.require(a.deps||[],a.callback)},makeShimExports:function(a){return function(){var b;a.init&&(b=a.init.apply(ba,arguments));return b||a.exports&&da(a.exports)}},makeRequire:function(a,e){function j(c,d,m){var n, +q;e.enableBuildCallback&&(d&&G(d))&&(d.__requireJsBuild=!0);if("string"===typeof c){if(G(d))return w(C("requireargs","Invalid require call"),m);if(a&&t(L,c))return L[c](h[a.id]);if(g.get)return g.get(i,c,a,j);n=p(c,a,!1,!0);n=n.id;return!t(r,n)?w(C("notloaded",'Module name "'+n+'" has not been loaded yet for context: '+b+(a?"":". Use require([])"))):r[n]}J();i.nextTick(function(){J();q=s(p(null,a));q.skipMap=e.skipMap;q.init(c,d,m,{enabled:!0});D()});return j}e=e||{};U(j,{isBrowser:z,toUrl:function(b){var d, +e=b.lastIndexOf("."),k=b.split("/")[0];if(-1!==e&&(!("."===k||".."===k)||1e.attachEvent.toString().indexOf("[native code"))&& +!Y?(M=!0,e.attachEvent("onreadystatechange",b.onScriptLoad)):(e.addEventListener("load",b.onScriptLoad,!1),e.addEventListener("error",b.onScriptError,!1)),e.src=d,J=e,D?y.insertBefore(e,D):y.appendChild(e),J=null,e;if(ea)try{importScripts(d),b.completeLoad(c)}catch(m){b.onError(C("importscripts","importScripts failed for "+c+" at "+d,m,[c]))}};z&&!q.skipDataMain&&T(document.getElementsByTagName("script"),function(b){y||(y=b.parentNode);if(I=b.getAttribute("data-main"))return s=I,q.baseUrl||(E=s.split("/"), +s=E.pop(),O=E.length?E.join("/")+"/":"./",q.baseUrl=O),s=s.replace(Q,""),g.jsExtRegExp.test(s)&&(s=I),q.deps=q.deps?q.deps.concat(s):[s],!0});define=function(b,c,d){var e,g;"string"!==typeof b&&(d=c,c=b,b=null);H(c)||(d=c,c=null);!c&&G(d)&&(c=[],d.length&&(d.toString().replace(ka,"").replace(la,function(b,d){c.push(d)}),c=(1===d.length?["require"]:["require","exports","module"]).concat(c)));if(M){if(!(e=J))N&&"interactive"===N.readyState||T(document.getElementsByTagName("script"),function(b){if("interactive"=== +b.readyState)return N=b}),e=N;e&&(b||(b=e.getAttribute("data-requiremodule")),g=F[e.getAttribute("data-requirecontext")])}(g?g.defQueue:R).push([b,c,d])};define.amd={jQuery:!0};g.exec=function(b){return eval(b)};g(q)}})(this); diff --git a/mkdocs/assets/search/mkdocs/js/search-results-template.mustache b/mkdocs/assets/search/mkdocs/js/search-results-template.mustache new file mode 100644 index 0000000000..a8b3862f20 --- /dev/null +++ b/mkdocs/assets/search/mkdocs/js/search-results-template.mustache @@ -0,0 +1,4 @@ + diff --git a/mkdocs/assets/search/mkdocs/js/search.js b/mkdocs/assets/search/mkdocs/js/search.js new file mode 100644 index 0000000000..e32528e822 --- /dev/null +++ b/mkdocs/assets/search/mkdocs/js/search.js @@ -0,0 +1,68 @@ +require([ + base_url + '/mkdocs/js/mustache.min.js', + base_url + '/mkdocs/js/lunr-0.5.7.min.js', + 'text!search-results-template.mustache', + 'text!../search_index.json', +], function (Mustache, lunr, results_template, data) { + + function getSearchTerm() + { + var sPageURL = window.location.search.substring(1); + var sURLVariables = sPageURL.split('&'); + for (var i = 0; i < sURLVariables.length; i++) + { + var sParameterName = sURLVariables[i].split('='); + if (sParameterName[0] == 'q') + { + return sParameterName[1]; + } + } + } + + var index = lunr(function () { + this.field('title', {boost: 10}); + this.field('text'); + this.ref('location'); + }); + + data = JSON.parse(data); + var documents = {}; + + $.each(data.docs, function(i, doc){ + index.add(doc); + documents[doc.location] = doc; + }); + + var search = function(){ + + var query = $('#mkdocs-search-query').val(); + var search_results = $('#mkdocs-search-results'); + search_results.empty(); + + if(query === ''){ + return; + } + + var results = index.search(query); + + if (results.length > 0){ + $.each(results, function(i, result){ + doc = documents[result.ref]; + doc.base_url = base_url; + doc.summary = doc.text.substring(0, 200); + search_results.append(Mustache.to_html(results_template, doc)); + }); + } else { + search_results.append("

No results found

"); + } + }; + + var term = getSearchTerm(); + if (term){ + $('#mkdocs-search-query').val(term); + search(); + } + + $('#mkdocs-search-query').keyup(search); + +}); diff --git a/mkdocs/assets/search/mkdocs/js/text.js b/mkdocs/assets/search/mkdocs/js/text.js new file mode 100644 index 0000000000..17921b6e5e --- /dev/null +++ b/mkdocs/assets/search/mkdocs/js/text.js @@ -0,0 +1,390 @@ +/** + * @license RequireJS text 2.0.12 Copyright (c) 2010-2014, The Dojo Foundation All Rights Reserved. + * Available via the MIT or new BSD license. + * see: http://github.com/requirejs/text for details + */ +/*jslint regexp: true */ +/*global require, XMLHttpRequest, ActiveXObject, + define, window, process, Packages, + java, location, Components, FileUtils */ + +define(['module'], function (module) { + 'use strict'; + + var text, fs, Cc, Ci, xpcIsWindows, + progIds = ['Msxml2.XMLHTTP', 'Microsoft.XMLHTTP', 'Msxml2.XMLHTTP.4.0'], + xmlRegExp = /^\s*<\?xml(\s)+version=[\'\"](\d)*.(\d)*[\'\"](\s)*\?>/im, + bodyRegExp = /]*>\s*([\s\S]+)\s*<\/body>/im, + hasLocation = typeof location !== 'undefined' && location.href, + defaultProtocol = hasLocation && location.protocol && location.protocol.replace(/\:/, ''), + defaultHostName = hasLocation && location.hostname, + defaultPort = hasLocation && (location.port || undefined), + buildMap = {}, + masterConfig = (module.config && module.config()) || {}; + + text = { + version: '2.0.12', + + strip: function (content) { + //Strips declarations so that external SVG and XML + //documents can be added to a document without worry. Also, if the string + //is an HTML document, only the part inside the body tag is returned. + if (content) { + content = content.replace(xmlRegExp, ""); + var matches = content.match(bodyRegExp); + if (matches) { + content = matches[1]; + } + } else { + content = ""; + } + return content; + }, + + jsEscape: function (content) { + return content.replace(/(['\\])/g, '\\$1') + .replace(/[\f]/g, "\\f") + .replace(/[\b]/g, "\\b") + .replace(/[\n]/g, "\\n") + .replace(/[\t]/g, "\\t") + .replace(/[\r]/g, "\\r") + .replace(/[\u2028]/g, "\\u2028") + .replace(/[\u2029]/g, "\\u2029"); + }, + + createXhr: masterConfig.createXhr || function () { + //Would love to dump the ActiveX crap in here. Need IE 6 to die first. + var xhr, i, progId; + if (typeof XMLHttpRequest !== "undefined") { + return new XMLHttpRequest(); + } else if (typeof ActiveXObject !== "undefined") { + for (i = 0; i < 3; i += 1) { + progId = progIds[i]; + try { + xhr = new ActiveXObject(progId); + } catch (e) {} + + if (xhr) { + progIds = [progId]; // so faster next time + break; + } + } + } + + return xhr; + }, + + /** + * Parses a resource name into its component parts. Resource names + * look like: module/name.ext!strip, where the !strip part is + * optional. + * @param {String} name the resource name + * @returns {Object} with properties "moduleName", "ext" and "strip" + * where strip is a boolean. + */ + parseName: function (name) { + var modName, ext, temp, + strip = false, + index = name.indexOf("."), + isRelative = name.indexOf('./') === 0 || + name.indexOf('../') === 0; + + if (index !== -1 && (!isRelative || index > 1)) { + modName = name.substring(0, index); + ext = name.substring(index + 1, name.length); + } else { + modName = name; + } + + temp = ext || modName; + index = temp.indexOf("!"); + if (index !== -1) { + //Pull off the strip arg. + strip = temp.substring(index + 1) === "strip"; + temp = temp.substring(0, index); + if (ext) { + ext = temp; + } else { + modName = temp; + } + } + + return { + moduleName: modName, + ext: ext, + strip: strip + }; + }, + + xdRegExp: /^((\w+)\:)?\/\/([^\/\\]+)/, + + /** + * Is an URL on another domain. Only works for browser use, returns + * false in non-browser environments. Only used to know if an + * optimized .js version of a text resource should be loaded + * instead. + * @param {String} url + * @returns Boolean + */ + useXhr: function (url, protocol, hostname, port) { + var uProtocol, uHostName, uPort, + match = text.xdRegExp.exec(url); + if (!match) { + return true; + } + uProtocol = match[2]; + uHostName = match[3]; + + uHostName = uHostName.split(':'); + uPort = uHostName[1]; + uHostName = uHostName[0]; + + return (!uProtocol || uProtocol === protocol) && + (!uHostName || uHostName.toLowerCase() === hostname.toLowerCase()) && + ((!uPort && !uHostName) || uPort === port); + }, + + finishLoad: function (name, strip, content, onLoad) { + content = strip ? text.strip(content) : content; + if (masterConfig.isBuild) { + buildMap[name] = content; + } + onLoad(content); + }, + + load: function (name, req, onLoad, config) { + //Name has format: some.module.filext!strip + //The strip part is optional. + //if strip is present, then that means only get the string contents + //inside a body tag in an HTML string. For XML/SVG content it means + //removing the declarations so the content can be inserted + //into the current doc without problems. + + // Do not bother with the work if a build and text will + // not be inlined. + if (config && config.isBuild && !config.inlineText) { + onLoad(); + return; + } + + masterConfig.isBuild = config && config.isBuild; + + var parsed = text.parseName(name), + nonStripName = parsed.moduleName + + (parsed.ext ? '.' + parsed.ext : ''), + url = req.toUrl(nonStripName), + useXhr = (masterConfig.useXhr) || + text.useXhr; + + // Do not load if it is an empty: url + if (url.indexOf('empty:') === 0) { + onLoad(); + return; + } + + //Load the text. Use XHR if possible and in a browser. + if (!hasLocation || useXhr(url, defaultProtocol, defaultHostName, defaultPort)) { + text.get(url, function (content) { + text.finishLoad(name, parsed.strip, content, onLoad); + }, function (err) { + if (onLoad.error) { + onLoad.error(err); + } + }); + } else { + //Need to fetch the resource across domains. Assume + //the resource has been optimized into a JS module. Fetch + //by the module name + extension, but do not include the + //!strip part to avoid file system issues. + req([nonStripName], function (content) { + text.finishLoad(parsed.moduleName + '.' + parsed.ext, + parsed.strip, content, onLoad); + }); + } + }, + + write: function (pluginName, moduleName, write, config) { + if (buildMap.hasOwnProperty(moduleName)) { + var content = text.jsEscape(buildMap[moduleName]); + write.asModule(pluginName + "!" + moduleName, + "define(function () { return '" + + content + + "';});\n"); + } + }, + + writeFile: function (pluginName, moduleName, req, write, config) { + var parsed = text.parseName(moduleName), + extPart = parsed.ext ? '.' + parsed.ext : '', + nonStripName = parsed.moduleName + extPart, + //Use a '.js' file name so that it indicates it is a + //script that can be loaded across domains. + fileName = req.toUrl(parsed.moduleName + extPart) + '.js'; + + //Leverage own load() method to load plugin value, but only + //write out values that do not have the strip argument, + //to avoid any potential issues with ! in file names. + text.load(nonStripName, req, function (value) { + //Use own write() method to construct full module value. + //But need to create shell that translates writeFile's + //write() to the right interface. + var textWrite = function (contents) { + return write(fileName, contents); + }; + textWrite.asModule = function (moduleName, contents) { + return write.asModule(moduleName, fileName, contents); + }; + + text.write(pluginName, nonStripName, textWrite, config); + }, config); + } + }; + + if (masterConfig.env === 'node' || (!masterConfig.env && + typeof process !== "undefined" && + process.versions && + !!process.versions.node && + !process.versions['node-webkit'])) { + //Using special require.nodeRequire, something added by r.js. + fs = require.nodeRequire('fs'); + + text.get = function (url, callback, errback) { + try { + var file = fs.readFileSync(url, 'utf8'); + //Remove BOM (Byte Mark Order) from utf8 files if it is there. + if (file.indexOf('\uFEFF') === 0) { + file = file.substring(1); + } + callback(file); + } catch (e) { + if (errback) { + errback(e); + } + } + }; + } else if (masterConfig.env === 'xhr' || (!masterConfig.env && + text.createXhr())) { + text.get = function (url, callback, errback, headers) { + var xhr = text.createXhr(), header; + xhr.open('GET', url, true); + + //Allow plugins direct access to xhr headers + if (headers) { + for (header in headers) { + if (headers.hasOwnProperty(header)) { + xhr.setRequestHeader(header.toLowerCase(), headers[header]); + } + } + } + + //Allow overrides specified in config + if (masterConfig.onXhr) { + masterConfig.onXhr(xhr, url); + } + + xhr.onreadystatechange = function (evt) { + var status, err; + //Do not explicitly handle errors, those should be + //visible via console output in the browser. + if (xhr.readyState === 4) { + status = xhr.status || 0; + if (status > 399 && status < 600) { + //An http 4xx or 5xx error. Signal an error. + err = new Error(url + ' HTTP status: ' + status); + err.xhr = xhr; + if (errback) { + errback(err); + } + } else { + callback(xhr.responseText); + } + + if (masterConfig.onXhrComplete) { + masterConfig.onXhrComplete(xhr, url); + } + } + }; + xhr.send(null); + }; + } else if (masterConfig.env === 'rhino' || (!masterConfig.env && + typeof Packages !== 'undefined' && typeof java !== 'undefined')) { + //Why Java, why is this so awkward? + text.get = function (url, callback) { + var stringBuffer, line, + encoding = "utf-8", + file = new java.io.File(url), + lineSeparator = java.lang.System.getProperty("line.separator"), + input = new java.io.BufferedReader(new java.io.InputStreamReader(new java.io.FileInputStream(file), encoding)), + content = ''; + try { + stringBuffer = new java.lang.StringBuffer(); + line = input.readLine(); + + // Byte Order Mark (BOM) - The Unicode Standard, version 3.0, page 324 + // http://www.unicode.org/faq/utf_bom.html + + // Note that when we use utf-8, the BOM should appear as "EF BB BF", but it doesn't due to this bug in the JDK: + // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4508058 + if (line && line.length() && line.charAt(0) === 0xfeff) { + // Eat the BOM, since we've already found the encoding on this file, + // and we plan to concatenating this buffer with others; the BOM should + // only appear at the top of a file. + line = line.substring(1); + } + + if (line !== null) { + stringBuffer.append(line); + } + + while ((line = input.readLine()) !== null) { + stringBuffer.append(lineSeparator); + stringBuffer.append(line); + } + //Make sure we return a JavaScript string and not a Java string. + content = String(stringBuffer.toString()); //String + } finally { + input.close(); + } + callback(content); + }; + } else if (masterConfig.env === 'xpconnect' || (!masterConfig.env && + typeof Components !== 'undefined' && Components.classes && + Components.interfaces)) { + //Avert your gaze! + Cc = Components.classes; + Ci = Components.interfaces; + Components.utils['import']('resource://gre/modules/FileUtils.jsm'); + xpcIsWindows = ('@mozilla.org/windows-registry-key;1' in Cc); + + text.get = function (url, callback) { + var inStream, convertStream, fileObj, + readData = {}; + + if (xpcIsWindows) { + url = url.replace(/\//g, '\\'); + } + + fileObj = new FileUtils.File(url); + + //XPCOM, you so crazy + try { + inStream = Cc['@mozilla.org/network/file-input-stream;1'] + .createInstance(Ci.nsIFileInputStream); + inStream.init(fileObj, 1, 0, false); + + convertStream = Cc['@mozilla.org/intl/converter-input-stream;1'] + .createInstance(Ci.nsIConverterInputStream); + convertStream.init(inStream, "utf-8", inStream.available(), + Ci.nsIConverterInputStream.DEFAULT_REPLACEMENT_CHARACTER); + + convertStream.readString(inStream.available(), readData); + convertStream.close(); + inStream.close(); + callback(readData.value); + } catch (e) { + throw new Error((fileObj && fileObj.path || '') + ': ' + e); + } + }; + } + return text; +}); diff --git a/mkdocs/build.py b/mkdocs/build.py index 2253c6b021..fbfd85bbab 100644 --- a/mkdocs/build.py +++ b/mkdocs/build.py @@ -5,7 +5,7 @@ from jinja2.exceptions import TemplateNotFound import mkdocs -from mkdocs import nav, toc, utils +from mkdocs import nav, search, toc, utils from mkdocs.compat import urljoin, PY2 from mkdocs.relative_path_ext import RelativePathExtension import jinja2 @@ -85,7 +85,6 @@ def get_global_context(nav, config): 'include_nav': config['include_nav'], 'include_next_prev': config['include_next_prev'], - 'include_search': config['include_search'], 'copyright': config['copyright'], 'google_analytics': config['google_analytics'], @@ -132,22 +131,29 @@ def get_page_context(page, content, nav, toc, meta, config): 'current_page': page, 'previous_page': page.previous_page, - 'next_page': page.next_page, + 'next_page': page.next_page } -def build_404(config, env, site_navigation): +def build_template(template_name, env, config, site_navigation=None, extra_context=None): try: - template = env.get_template('404.html') + template = env.get_template(template_name) except TemplateNotFound: - return + return False - global_context = get_global_context(site_navigation, config) + if site_navigation is not None: + context = get_global_context(site_navigation, config) + else: + context = {} + + if extra_context is not None: + context.update(extra_context) - output_content = template.render(global_context) - output_path = os.path.join(config['site_dir'], '404.html') + output_content = template.render(context) + output_path = os.path.join(config['site_dir'], template_name) utils.write_file(output_content.encode('utf-8'), output_path) + return True def build_pages(config, dump_json=False): @@ -157,8 +163,14 @@ def build_pages(config, dump_json=False): site_navigation = nav.SiteNavigation(config['pages'], config['use_directory_urls']) loader = jinja2.FileSystemLoader(config['theme_dir']) env = jinja2.Environment(loader=loader) + search_index = search.SearchIndex() - build_404(config, env, site_navigation) + build_template('404.html', env, config, site_navigation) + + if not build_template('search.html', env, config, site_navigation): + log.debug("Search is enabled but the theme doesn't contain a " + "search.html file. Assuming the theme implements search " + "within a modal.") for page in site_navigation.walk_pages(): # Read the input file @@ -207,6 +219,12 @@ def build_pages(config, dump_json=False): else: utils.write_file(output_content.encode('utf-8'), output_path) + search_index.add_entry_from_context(page, html_content, table_of_contents) + + search_index = search_index.generate_search_index() + json_output_path = os.path.join(config['site_dir'], 'mkdocs', 'search_index.json') + utils.write_file(search_index.encode('utf-8'), json_output_path) + def build(config, live_server=False, dump_json=False, clean_site_dir=False): """ diff --git a/mkdocs/compat.py b/mkdocs/compat.py index 49bd396a8d..7e2bdfd890 100644 --- a/mkdocs/compat.py +++ b/mkdocs/compat.py @@ -1,40 +1,30 @@ # coding: utf-8 +# flake8: noqa """Python 2/3 compatibility module.""" + import sys -PY2 = int(sys.version[0]) == 2 +PY2 = sys.version_info < (3, ) if PY2: + from HTMLParser import HTMLParser from urlparse import urljoin, urlparse, urlunparse - import urllib - urlunquote = urllib.unquote - import SimpleHTTPServer as httpserver - httpserver = httpserver - import SocketServer - socketserver = SocketServer + import SocketServer as socketserver + import urllib - import itertools - zip = itertools.izip + urlunquote = urllib.unquote - text_type = unicode - binary_type = str - string_types = (str, unicode) + from itertools import izip as zip unicode = unicode - basestring = basestring -else: # PY3 - from urllib.parse import urljoin, urlparse, urlunparse, unquote - urlunquote = unquote +else: # PY3 + from html.parser import HTMLParser + from urllib.parse import (urljoin, urlparse, urlunparse, + unquote as urlunquote) import http.server as httpserver - httpserver = httpserver import socketserver - socketserver = socketserver zip = zip - text_type = str - binary_type = bytes - string_types = (str,) unicode = str - basestring = (str, bytes) diff --git a/mkdocs/config.py b/mkdocs/config.py index aeaad94e86..f8fbecdbe2 100644 --- a/mkdocs/config.py +++ b/mkdocs/config.py @@ -58,10 +58,6 @@ # PyMarkdown extension names. 'markdown_extensions': (), - # Determine if the site should generate a json search index and include - # search elements in the theme. - TODO - 'include_search': False, - # Determine if the site should include a 404.html page. # TODO: Implment this. Make this None, have it True if a 404.html # template exists in the theme or docs dir. @@ -69,7 +65,7 @@ # enabling strict mode causes MkDocs to stop the build when a problem is # encountered rather than display an error. - 'strict': False, + 'strict': False } @@ -129,7 +125,12 @@ def validate_config(user_config): config['extra_javascript'] = extra_javascript package_dir = os.path.dirname(__file__) - theme_dir = [os.path.join(package_dir, 'themes', config['theme'])] + theme_dir = [os.path.join(package_dir, 'themes', config['theme']), ] + + # Add the search assets to the theme_dir, this means that + # they will then we copied into the output directory but can + # be overwritten by themes if needed. + theme_dir.append(os.path.join(package_dir, 'assets', 'search')) if config['theme_dir'] is not None: # If the user has given us a custom theme but not a diff --git a/mkdocs/nav.py b/mkdocs/nav.py index c8257e126b..8f56979348 100644 --- a/mkdocs/nav.py +++ b/mkdocs/nav.py @@ -91,6 +91,7 @@ def make_relative(self, url): Given a URL path return it as a relative URL, given the context of the current page. """ + suffix = '/' if (url.endswith('/') and len(url) > 1) else '' # Workaround for bug on `posixpath.relpath()` in Python 2.6 if self.base_path == '/': @@ -98,6 +99,8 @@ def make_relative(self, url): # Workaround for static assets return '.' return url.lstrip('/') + relative_path = posixpath.relpath(url, start=self.base_path) + suffix + # Under Python 2.6, relative_path adds an extra '/' at the end. relative_path = posixpath.relpath(url, start=self.base_path).rstrip('/') + suffix diff --git a/mkdocs/search.py b/mkdocs/search.py new file mode 100644 index 0000000000..ca6a35b574 --- /dev/null +++ b/mkdocs/search.py @@ -0,0 +1,203 @@ +from __future__ import unicode_literals + +from mkdocs.compat import HTMLParser, unicode +import json + + +class SearchIndex(object): + """ + Search index is a collection of pages and sections (H1 and H2 + tags and their following content are sections). + """ + + def __init__(self): + self._entries = [] + + def _find_toc_by_id(self, toc, id_): + """ + Given a table of contents and HTML ID, iterate through + and return the matched item in the TOC. + """ + for toc_item in toc: + if toc_item.url[1:] == id_: + return toc_item + for toc_sub_item in toc_item.children: + if toc_sub_item.url[1:] == id_: + return toc_sub_item + + def _add_entry(self, title, text, loc): + """ + A simple wrapper to add an entry and ensure the contents + is UTF8 encoded. + """ + self._entries.append({ + 'title': title, + 'text': unicode(text.strip().encode('utf-8'), encoding='utf-8'), + 'location': loc + }) + + def add_entry_from_context(self, page, content, toc): + """ + Create a set of entries in the index for a page. One for + the page itself and then one for each of it's H1 and H2 + tags. + """ + + # Create the content parser and feed in the HTML for the + # full page. This handles all the parsing and prepares + # us to iterate through it. + parser = ContentParser() + parser.feed(content) + + # Get the absolute URL for the page, this is then + # prepended to the urls of the sections + abs_url = page.abs_url + + # Create an entry for the full page. + self._add_entry( + title=page.title, + text=self.strip_tags(content).rstrip('\n'), + loc=abs_url + ) + + for section in parser.data: + self.create_entry_for_section(section, toc, abs_url) + + def create_entry_for_section(self, section, toc, abs_url): + """ + Given a section on the page, the table of contents and + the absolute url for the page create an entry in the + index + """ + + toc_item = self._find_toc_by_id(toc, section.id) + + if toc_item is not None: + self._add_entry( + title=toc_item.title, + text=u" ".join(section.text), + loc=abs_url + toc_item.url + ) + + def generate_search_index(self): + """python to json conversion""" + page_dicts = { + 'docs': self._entries, + } + return json.dumps(page_dicts, sort_keys=True, indent=4) + + def strip_tags(self, html): + """strip html tags from data""" + s = HTMLStripper() + s.feed(html) + return s.get_data() + + +class HTMLStripper(HTMLParser): + """ + A simple HTML parser that stores all of the data within tags + but ignores the tags themselves and thus strips them from the + content. + """ + + def __init__(self, *args, **kwargs): + # HTMLParser is a old-style class in Python 2, so + # super() wont work here. + HTMLParser.__init__(self, *args, **kwargs) + + self.data = [] + + def handle_data(self, d): + """ + Called for the text contents of each tag. + """ + self.data.append(d) + + def get_data(self): + return '\n'.join(self.data) + + +class ContentSection(): + """ + Used by the ContentParser class to capture the information we + need when it is parsing the HMTL. + """ + + def __init__(self, text=None, id_=None, title=None): + self.text = text or [] + self.id = id_ + self.title = title + + def __eq__(self, other): + return all([ + self.text == other.text, + self.id == other.id, + self.title == other.title + ]) + + def __repr__(self): + return "".format(self.id, self.title) + + +class ContentParser(HTMLParser): + """ + Given a block of HTML, group the content under the preceding + H1 or H2 tags which can then be used for creating an index + for that section. + """ + + def __init__(self, *args, **kwargs): + + # HTMLParser is a old-style class in Python 2, so + # super() wont work here. + HTMLParser.__init__(self, *args, **kwargs) + + self.data = [] + self.section = None + self.is_header_tag = False + + def handle_starttag(self, tag, attrs): + """Called at the start of every HTML tag.""" + + # We only care about the opening tag for H1 and H2. + if tag not in ("h1", "h2"): + return + + # We are dealing with a new header, create a new section + # for it and assign the ID if it has one. + self.is_header_tag = True + self.section = ContentSection() + self.data.append(self.section) + + for attr in attrs: + if attr[0] == "id": + self.section.id = attr[1] + + def handle_endtag(self, tag): + """Called at the end of every HTML tag.""" + + # We only care about the opening tag for H1 and H2. + if tag not in ("h1", "h2"): + return + + self.is_header_tag = False + + def handle_data(self, data): + """ + Called for the text contents of each tag. + """ + + if self.section is None: + # This means we have some content at the start of the + # HTML before we reach a H1 or H2. We don't actually + # care about that content as it will be added to the + # overall page entry in the search. So just skip it. + return + + # If this is a header, then the data is the title. + # Otherwise it is content of something under that header + # section. + if self.is_header_tag: + self.section.title = data + else: + self.section.text.append(data.rstrip('\n')) diff --git a/mkdocs/serve.py b/mkdocs/serve.py index 6220aa2522..6d5f19e514 100644 --- a/mkdocs/serve.py +++ b/mkdocs/serve.py @@ -4,7 +4,7 @@ from watchdog import events from watchdog.observers.polling import PollingObserver from mkdocs.build import build -from mkdocs.compat import httpserver, socketserver, urlunquote +from mkdocs.compat import httpserver, socketserver, urlunquote, urlparse, urlunparse from mkdocs.config import load_config import os import posixpath @@ -48,6 +48,17 @@ class FixedDirectoryHandler(httpserver.SimpleHTTPRequestHandler): """ base_dir = os.getcwd() + def do_GET(self): + """ + The SimpleHTTPRequestHandler isn't designed to work with + query strings. Everything we do with the query string is + handled on the client-side, so throw it away here. + """ + scheme, netloc, path, query, query, fragment = urlparse(self.path) + if query is not '': + self.path = urlunparse((scheme, netloc, path, '', '', fragment)) + return httpserver.SimpleHTTPRequestHandler.do_GET(self) + def translate_path(self, path): # abandon query parameters path = path.split('?', 1)[0] diff --git a/mkdocs/test.py b/mkdocs/test.py index 7d81f6b51a..114c3f0d84 100755 --- a/mkdocs/test.py +++ b/mkdocs/test.py @@ -2,7 +2,7 @@ # coding: utf-8 -from mkdocs import build, main, nav, toc, utils, config +from mkdocs import build, config, main, nav, search, toc, utils from mkdocs.compat import PY2, zip from mkdocs.exceptions import ConfigurationError, MarkdownNotFound import logging @@ -23,6 +23,10 @@ def ensure_utf(string): return string.encode('utf-8') if PY2 else string +def strip_whitespace(string): + return string.replace("\n", "").replace(" ", "") + + class MainTests(unittest.TestCase): def test_arg_to_option(self): """ @@ -212,13 +216,14 @@ def test_create_media_urls(self): self.assertEqual(urls[0], expected_result) -class TableOfContentsTests(unittest.TestCase): - def markdown_to_toc(self, markdown_source): - md = markdown.Markdown(extensions=['toc']) - md.convert(markdown_source) - toc_output = md.toc - return toc.TableOfContents(toc_output) +def _markdown_to_toc(markdown_source): + md = markdown.Markdown(extensions=['toc']) + md.convert(markdown_source) + toc_output = md.toc + return toc.TableOfContents(toc_output) + +class TableOfContentsTests(unittest.TestCase): def test_indented_toc(self): md = dedent(""" # Heading 1 @@ -230,7 +235,7 @@ def test_indented_toc(self): Heading 2 - #heading-2 Heading 3 - #heading-3 """) - toc = self.markdown_to_toc(md) + toc = _markdown_to_toc(md) self.assertEqual(str(toc).strip(), expected) def test_flat_toc(self): @@ -244,7 +249,7 @@ def test_flat_toc(self): Heading 2 - #heading-2 Heading 3 - #heading-3 """) - toc = self.markdown_to_toc(md) + toc = _markdown_to_toc(md) self.assertEqual(str(toc).strip(), expected) def test_flat_h2_toc(self): @@ -258,7 +263,7 @@ def test_flat_h2_toc(self): Heading 2 - #heading-2 Heading 3 - #heading-3 """) - toc = self.markdown_to_toc(md) + toc = _markdown_to_toc(md) self.assertEqual(str(toc).strip(), expected) def test_mixed_toc(self): @@ -276,7 +281,7 @@ def test_mixed_toc(self): Heading 4 - #heading-4 Heading 5 - #heading-5 """) - toc = self.markdown_to_toc(md) + toc = _markdown_to_toc(md) self.assertEqual(str(toc).strip(), expected) @@ -820,6 +825,137 @@ def test_strict_mode_invalid(self): MarkdownNotFound, build.convert_markdown, invalid, site_nav, strict=True) + +class SearchTests(unittest.TestCase): + + def test_html_stripper(self): + + stripper = search.HTMLStripper() + + stripper.feed("

Testing

Content

") + + self.assertEquals(stripper.data, ["Testing", "Content"]) + + def test_content_parser(self): + + parser = search.ContentParser() + + parser.feed('

Title

TEST') + + self.assertEquals(parser.data, [search.ContentSection( + text=["TEST"], + id_="title", + title="Title" + )]) + + def test_content_parser_no_id(self): + + parser = search.ContentParser() + + parser.feed("

Title

TEST") + + self.assertEquals(parser.data, [search.ContentSection( + text=["TEST"], + id_=None, + title="Title" + )]) + + def test_content_parser_content_before_header(self): + + parser = search.ContentParser() + + parser.feed("Content Before H1

Title

TEST") + + self.assertEquals(parser.data, [search.ContentSection( + text=["TEST"], + id_=None, + title="Title" + )]) + + def test_content_parser_no_sections(self): + + parser = search.ContentParser() + + parser.feed("No H1 or H2TitleTEST") + + self.assertEquals(parser.data, []) + + def test_find_toc_by_id(self): + """ + Test finding the relevant TOC item by the tag ID. + """ + + index = search.SearchIndex() + + md = dedent(""" + # Heading 1 + ## Heading 2 + ### Heading 3 + """) + toc = _markdown_to_toc(md) + + toc_item = index._find_toc_by_id(toc, "heading-1") + self.assertEqual(toc_item.url, "#heading-1") + self.assertEqual(toc_item.title, "Heading 1") + + toc_item2 = index._find_toc_by_id(toc, "heading-2") + self.assertEqual(toc_item2.url, "#heading-2") + self.assertEqual(toc_item2.title, "Heading 2") + + toc_item3 = index._find_toc_by_id(toc, "heading-3") + self.assertEqual(toc_item3, None) + + def test_create_search_index(self): + + html_content = """ +

Heading 1

+

Content 1

+

Heading 2

+

Content 2

+

Heading 3

+

Content 3

+ """ + + pages = [ + ('index.md', 'Home'), + ('about.md', 'About') + ] + site_navigation = nav.SiteNavigation(pages) + + md = dedent(""" + # Heading 1 + ## Heading 2 + ### Heading 3 + """) + toc = _markdown_to_toc(md) + + full_content = ''.join("""Heading{0}Content{0}""".format(i) for i in range(1, 4)) + + for page in site_navigation: + + index = search.SearchIndex() + index.add_entry_from_context(page, html_content, toc) + + self.assertEqual(len(index._entries), 3) + + loc = page.abs_url + + self.assertEqual(index._entries[0]['title'], page.title) + self.assertEqual(strip_whitespace(index._entries[0]['text']), full_content) + self.assertEqual(index._entries[0]['tags'], "") + self.assertEqual(index._entries[0]['loc'], loc) + + self.assertEqual(index._entries[1]['title'], "Heading 1") + self.assertEqual(index._entries[1]['text'], "Content 1") + self.assertEqual(index._entries[1]['tags'], "") + self.assertEqual(index._entries[1]['loc'], "{0}#heading-1".format(loc)) + + self.assertEqual(index._entries[2]['title'], "Heading 2") + self.assertEqual(strip_whitespace(index._entries[2]['text']), "Content2Heading3Content3") + self.assertEqual(index._entries[2]['tags'], "") + self.assertEqual(index._entries[2]['loc'], "{0}#heading-2".format(loc)) + + # class IntegrationTests(unittest.TestCase): # def test_mkdocs_site(self): # """ diff --git a/mkdocs/themes/amelia/base.html b/mkdocs/themes/amelia/base.html index 595b40bad1..a502bb4ae9 100644 --- a/mkdocs/themes/amelia/base.html +++ b/mkdocs/themes/amelia/base.html @@ -48,8 +48,6 @@
{% include "content.html" %}
- {% if include_search %}{% include "search.html" %}{% endif %} - diff --git a/mkdocs/themes/amelia/nav.html b/mkdocs/themes/amelia/nav.html index 1cb451f896..0a62e33a10 100644 --- a/mkdocs/themes/amelia/nav.html +++ b/mkdocs/themes/amelia/nav.html @@ -41,11 +41,6 @@