Replace problematic chars in tsvector (#103)

bitmagnet-io · Jan 13, 2024 · d999b4c · d999b4c
1 parent 01b1448
commit d999b4c
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/internal/database/fts/tokenizer.go b/internal/database/fts/tokenizer.go
@@ -64,7 +64,11 @@ func (l *tokenizerLexer) readPhrase() []string {
 				if tb, ok := table.Tables[section]; ok {
 					if len(tb) > int(position) {
 						subst := tb[position]
-						appendStr(strings.TrimSpace(subst))
+						// replace some problematic characters
+						subst = strings.ReplaceAll(subst, "'", "_sq_")
+						subst = strings.ReplaceAll(subst, "\\", "_bs_")
+						subst = strings.TrimSpace(subst)
+						appendStr(subst)
 						if isNonBreakingLang || len(subst) == 0 || subst[len(subst)-1] == ' ' {
 							breakWord()
 						}

diff --git a/internal/database/fts/tsvector.go b/internal/database/fts/tsvector.go
@@ -199,7 +199,7 @@ var nonWordChar = regexp.MustCompile(`\W`)
 
 func quoteLexeme(str string, force bool) string {
 	if force || nonWordChar.MatchString(str) {
-		str = "'" + strings.Replace(str, "'", "''", -1) + "'"
+		str = "'" + strings.ReplaceAll(str, "'", "''") + "'"
 	}
 	return str
 }