Skip to content

Commit 36ead72

Browse files
committed
Use utf8.RuneCountInString() rather than len() :(
Fixes #10. Thanks @curio77.
1 parent 3f941dd commit 36ead72

File tree

1 file changed

+2
-1
lines changed

1 file changed

+2
-1
lines changed

Diff for: regexp.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"regexp"
66
"strings"
77
"sync"
8+
"unicode/utf8"
89

910
"github.com/dlclark/regexp2"
1011
)
@@ -221,7 +222,7 @@ func (r *RegexLexer) Tokenise(options *TokeniseOptions, text string, out func(*T
221222
state.Rule = ruleIndex
222223

223224
state.Groups = groups
224-
state.Pos += len(groups[0])
225+
state.Pos += utf8.RuneCountInString(groups[0])
225226
if rule.Mutator != nil {
226227
if err := rule.Mutator.Mutate(state); err != nil {
227228
return err

0 commit comments

Comments
 (0)