Improve SQLite and map performance

dashing.go

@@ -425,6 +425,16 @@ func initDB(name string, fresh bool) (*sql.DB, error) {
 
 // texasRanger is... wait for it... a WALKER!
 func texasRanger(base string, base_depth int, name string, dashing Dashing, db *sql.DB) error {
+	tx, err := db.Begin()
+	if err != nil {
+		fmt.Printf("Error on db begin")
+		return nil
+	}
+	query, err := tx.Prepare(`INSERT OR IGNORE INTO searchIndex(name, type, path) VALUES (?,?,?)`)
+	if err != nil {
+		return err
+	}
+	defer query.Close()
 	filepath.Walk(base, func(path string, info os.FileInfo, err error) error {
 		fmt.Printf("Reading %s\n", path)
 		if strings.HasPrefix(path, name+".docset") {
@@ -460,6 +470,12 @@ func texasRanger(base string, base_depth int, name string, dashing Dashing, db *
 		}
 		return nil
 	})
+	err = tx.Commit()
+	if err != nil {
+		fmt.Printf("Error on commit")
+		return nil
+	}
+
 	return nil
 }
 
@@ -484,8 +500,9 @@ func ignore(src string) bool {
 
 func encodeHTMLentities(orig string) string {
 	escaped := new(bytes.Buffer)
+	escaped.Grow(len(orig))
 	for _, c := range orig {
-		if point_to_entity[c] == "" {
+		if c < 160 || point_to_entity[c] == "" {
 			escaped.WriteRune(c)
 		} else {
 			escaped.WriteString(point_to_entity[c])
@@ -699,88 +716,23 @@ func copyFile(src, dest string) error {
 }
 
 var point_to_entity = map[rune]string{
-	8704: "&forall;",
-	8194: "&ensp;",
-	8195: "&emsp;",
-	8709: "&empty;",
-	8711: "&nabla;",
-	8712: "&isin;",
-	8201: "&thinsp;",
-	8715: "&ni;",
-	8204: "&zwnj;",
-	8205: "&zwj;",
-	8206: "&lrm;",
-	8719: "&prod;",
-	8721: "&sum;",
-	8722: "&minus;",
-	8211: "&ndash;",
-	8212: "&mdash;",
-	8727: "&lowast;",
-	8216: "&lsquo;",
-	8217: "&rsquo;",
-	8730: "&radic;",
-	175:  "&macr;",
-	8220: "&ldquo;",
-	8221: "&rdquo;",
-	8222: "&bdquo;",
-	8224: "&dagger;",
-	8225: "&Dagger;",
-	8226: "&bull;",
-	8230: "&hellip;",
-	8743: "&and;",
-	8744: "&or;",
-	8745: "&cap;",
-	8746: "&cup;",
-	8747: "&int;",
-	8240: "&permil;",
-	8242: "&prime;",
-	8243: "&Prime;",
-	8756: "&there4;",
-	8713: "&notin;",
-	8249: "&lsaquo;",
-	8250: "&rsaquo;",
-	8764: "&sim;",
-	// 62:   "&gt;",	// this is already encoded for us
-	8629: "&crarr;",
-	9824: "&spades;",
-	8260: "&frasl;",
-	8773: "&cong;",
-	8776: "&asymp;",
-	8207: "&rlm;",
-	9829: "&hearts;",
-	8800: "&ne;",
-	8801: "&equiv;",
-	9827: "&clubs;",
-	8804: "&le;",
-	8805: "&ge;",
-	9830: "&diams;",
-	// 38:   "&amp;",	// this is already encoded for us
-	8834: "&sub;",
-	8835: "&sup;",
-	8836: "&nsub;",
-	8838: "&sube;",
-	8839: "&supe;",
-	8853: "&oplus;",
-	8855: "&otimes;",
-	8734: "&infin;",
-	8218: "&sbquo;",
-	8901: "&sdot;",
+	// 34:   "&quot;",	// this is already encoded
 	160:  "&nbsp;",
 	161:  "&iexcl;",
 	162:  "&cent;",
 	163:  "&pound;",
 	164:  "&curren;",
-	8869: "&perp;",
 	166:  "&brvbar;",
+	165:  "&yen;",
 	167:  "&sect;",
 	168:  "&uml;",
 	169:  "&copy;",
 	170:  "&ordf;",
 	171:  "&laquo;",
-	8364: "&euro;",
+	172:  "&not;",
 	173:  "&shy;",
 	174:  "&reg;",
-	8733: "&prop;",
+	175:  "&macr;",
 	176:  "&deg;",
 	177:  "&plusmn;",
 	178:  "&sup2;",
@@ -810,7 +762,7 @@ var point_to_entity = map[rune]string{
 	202:  "&Ecirc;",
 	203:  "&Euml;",
 	204:  "&Igrave;",
-	// 34:   "&quot;",	// this is already encoded
+	205:  "&Iacute;",
 	206:  "&Icirc;",
 	207:  "&Iuml;",
 	208:  "&ETH;",
@@ -838,11 +790,11 @@ var point_to_entity = map[rune]string{
 	230:  "&aelig;",
 	231:  "&ccedil;",
 	232:  "&egrave;",
-	205:  "&Iacute;",
+	233:  "&eacute;",
 	234:  "&ecirc;",
 	235:  "&euml;",
 	236:  "&igrave;",
-	8658: "&rArr;",
+	237:  "&iacute;",
 	238:  "&icirc;",
 	239:  "&iuml;",
 	240:  "&eth;",
@@ -861,41 +813,18 @@ var point_to_entity = map[rune]string{
 	253:  "&yacute;",
 	254:  "&thorn;",
 	255:  "&yuml;",
-	172:  "&not;",
-	8968: "&lceil;",
-	8969: "&rceil;",
-	8970: "&lfloor;",
-	8971: "&rfloor;",
-	8465: "&image;",
-	8472: "&weierp;",
-	8476: "&real;",
-	8482: "&trade;",
-	732:  "&tilde;",
-	9002: "&rang;",
-	8736: "&ang;",
-	402:  "&fnof;",
-	8706: "&part;",
-	8501: "&alefsym;",
-	710:  "&circ;",
+	376:  "&Yuml;",
 	338:  "&OElig;",
 	339:  "&oelig;",
 	352:  "&Scaron;",
 	353:  "&scaron;",
-	8593: "&uarr;",
-	// 60:   "&lt;",	// this is already encoded for us
-	8594: "&rarr;",
-	8707: "&exist;",
-	8595: "&darr;",
-	8254: "&oline;",
-	233:  "&eacute;",
-	376:  "&Yuml;",
-	916:  "&Delta;",
-	237:  "&iacute;",
-	8592: "&larr;",
+	402:  "&fnof;",
+	710:  "&circ;",
+	732:  "&tilde;",
 	913:  "&Alpha;",
 	914:  "&Beta;",
 	915:  "&Gamma;",
-	8596: "&harr;",
+	916:  "&Delta;",
 	917:  "&Epsilon;",
 	918:  "&Zeta;",
 	919:  "&Eta;",
@@ -941,14 +870,102 @@ var point_to_entity = map[rune]string{
 	967:  "&chi;",
 	968:  "&psi;",
 	969:  "&omega;",
-	9674: "&loz;",
-	8656: "&lArr;",
 	977:  "&thetasym;",
 	978:  "&upsih;",
-	8659: "&dArr;",
-	8660: "&hArr;",
 	982:  "&piv;",
-	165:  "&yen;",
+	8194: "&ensp;",
+	8195: "&emsp;",
+	8220: "&ldquo;",
+	8221: "&rdquo;",
+	8222: "&bdquo;",
+	8224: "&dagger;",
+	8225: "&Dagger;",
+	8226: "&bull;",
+	8230: "&hellip;",
+	8704: "&forall;",
+	8709: "&empty;",
+	8711: "&nabla;",
+	8712: "&isin;",
+	8201: "&thinsp;",
+	8715: "&ni;",
+	8204: "&zwnj;",
+	8205: "&zwj;",
+	8206: "&lrm;",
+	8719: "&prod;",
+	8721: "&sum;",
+	8722: "&minus;",
+	8211: "&ndash;",
+	8212: "&mdash;",
+	8727: "&lowast;",
+	8216: "&lsquo;",
+	8217: "&rsquo;",
+	8730: "&radic;",
+	8743: "&and;",
+	8744: "&or;",
+	8745: "&cap;",
+	8746: "&cup;",
+	8747: "&int;",
+	8240: "&permil;",
+	8242: "&prime;",
+	8243: "&Prime;",
+	8756: "&there4;",
+	8713: "&notin;",
+	8249: "&lsaquo;",
+	8250: "&rsaquo;",
+	8764: "&sim;",
+	// 62:   "&gt;",	// this is already encoded for us
+	8629: "&crarr;",
+	9824: "&spades;",
+	8260: "&frasl;",
+	8773: "&cong;",
+	8776: "&asymp;",
+	8207: "&rlm;",
+	9829: "&hearts;",
+	8800: "&ne;",
+	8801: "&equiv;",
+	9827: "&clubs;",
+	8804: "&le;",
+	8805: "&ge;",
+	9830: "&diams;",
+	// 38:   "&amp;",	// this is already encoded for us
+	8834: "&sub;",
+	8835: "&sup;",
+	8836: "&nsub;",
+	8838: "&sube;",
+	8839: "&supe;",
+	8853: "&oplus;",
+	8855: "&otimes;",
+	8734: "&infin;",
+	8218: "&sbquo;",
+	8901: "&sdot;",
+	8869: "&perp;",
+	8364: "&euro;",
+	8733: "&prop;",
+	8658: "&rArr;",
+	8968: "&lceil;",
+	8969: "&rceil;",
+	8970: "&lfloor;",
+	8971: "&rfloor;",
+	8465: "&image;",
+	8472: "&weierp;",
+	8476: "&real;",
+	8482: "&trade;",
+	9002: "&rang;",
+	8736: "&ang;",
+	8706: "&part;",
+	8501: "&alefsym;",
+	8593: "&uarr;",
+	// 60:   "&lt;",	// this is already encoded for us
+	8594: "&rarr;",
+	8707: "&exist;",
+	8595: "&darr;",
+	8254: "&oline;",
+	8592: "&larr;",
+	8656: "&lArr;",
 	8657: "&uArr;",
+	8659: "&dArr;",
+	8660: "&hArr;",
 	9001: "&lang;",
+	9674: "&loz;",
+	8596: "&harr;",
 }