diff --git a/internal/buffer/buffer.go b/internal/buffer/buffer.go index c623fd586..997b6407f 100644 --- a/internal/buffer/buffer.go +++ b/internal/buffer/buffer.go @@ -148,11 +148,7 @@ func (b *SharedBuffer) MarkModified(start, end int) { end = util.Clamp(end, 0, len(b.lines)-1) if b.Settings["syntax"].(bool) && b.SyntaxDef != nil { - l := -1 - for i := start; i <= end; i++ { - l = util.Max(b.Highlighter.ReHighlightStates(b, i), l) - } - b.Highlighter.HighlightMatches(b, start, l) + b.Highlighter.Highlight(b, start, end) } for i := start; i <= end; i++ { @@ -961,8 +957,7 @@ func (b *Buffer) UpdateRules() { b.Highlighter = highlight.NewHighlighter(b.SyntaxDef) if b.Settings["syntax"].(bool) { go func() { - b.Highlighter.HighlightStates(b) - b.Highlighter.HighlightMatches(b, 0, b.End().Y) + b.Highlighter.Highlight(b, 0, b.End().Y) screen.Redraw() }() } diff --git a/internal/util/util.go b/internal/util/util.go index 83dc4458a..e5af4a834 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -133,6 +133,16 @@ func SliceStartStr(str string, index int) string { return str[:totalSize] } +// SliceStartEnd combines SliceStart and SliceEnd into one +func SliceStartEnd(slc []byte, start int, end int) []byte { + return SliceEnd(SliceStart(slc, end), start) +} + +// SliceStartEndStr is the same as SliceStartEnd but for strings +func SliceStartEndStr(str string, start int, end int) string { + return SliceEndStr(SliceStartStr(str, end), start) +} + // SliceVisualEnd will take a byte slice and slice off the start // up to a given visual index. If the index is in the middle of a // rune the number of visual columns into the rune will be returned diff --git a/pkg/highlight/highlighter.go b/pkg/highlight/highlighter.go index ebcd2aaa7..724bd828c 100644 --- a/pkg/highlight/highlighter.go +++ b/pkg/highlight/highlighter.go @@ -1,55 +1,12 @@ package highlight import ( + // "log" "regexp" "strings" -) - -func sliceStart(slc []byte, index int) []byte { - len := len(slc) - i := 0 - totalSize := 0 - for totalSize < len { - if i >= index { - return slc[totalSize:] - } - - _, _, size := DecodeCharacter(slc[totalSize:]) - totalSize += size - i++ - } - return slc[totalSize:] -} - -func sliceEnd(slc []byte, index int) []byte { - len := len(slc) - i := 0 - totalSize := 0 - for totalSize < len { - if i >= index { - return slc[:totalSize] - } - - _, _, size := DecodeCharacter(slc[totalSize:]) - totalSize += size - i++ - } - - return slc[:totalSize] -} - -// RunePos returns the rune index of a given byte index -// This could cause problems if the byte index is between code points -func runePos(p int, str []byte) int { - if p < 0 { - return 0 - } - if p >= len(str) { - return CharacterCount(str) - } - return CharacterCount(str[:p]) -} + "github.com/zyedidia/micro/v2/internal/util" +) func combineLineMatch(src, dst LineMatch) LineMatch { for k, v := range src { @@ -78,10 +35,24 @@ type LineStates interface { Unlock() } +// highlightStorage is used to store the found ranges +type highlightStorage struct { + start int + end int + group Group + region *region + children []*highlightStorage + pattern bool +} + // A Highlighter contains the information needed to highlight a string type Highlighter struct { lastRegion *region + lastStart int + lastEnd int Def *Def + storage []highlightStorage + removed []highlightStorage } // NewHighlighter returns a new highlighter from the given syntax definition @@ -99,7 +70,7 @@ func findIndex(regex *regexp.Regexp, skip *regexp.Regexp, str []byte) []int { var strbytes []byte if skip != nil { strbytes = skip.ReplaceAllFunc(str, func(match []byte) []byte { - res := make([]byte, CharacterCount(match)) + res := make([]byte, util.CharacterCount(match)) return res }) } else { @@ -111,171 +82,351 @@ func findIndex(regex *regexp.Regexp, skip *regexp.Regexp, str []byte) []int { return nil } // return []int{match.Index, match.Index + match.Length} - return []int{runePos(match[0], str), runePos(match[1], str)} + return []int{util.RunePos(str, match[0]), util.RunePos(str, match[1])} } -func findAllIndex(regex *regexp.Regexp, str []byte) [][]int { - matches := regex.FindAllIndex(str, -1) +func findAllIndex(regex *regexp.Regexp, skip *regexp.Regexp, str []byte) [][]int { + var strbytes []byte + if skip != nil { + strbytes = skip.ReplaceAllFunc(str, func(match []byte) []byte { + res := make([]byte, util.CharacterCount(match)) + return res + }) + } else { + strbytes = str + } + + matches := regex.FindAllIndex(strbytes, -1) for i, m := range matches { - matches[i][0] = runePos(m[0], str) - matches[i][1] = runePos(m[1], str) + matches[i][0] = util.RunePos(str, m[0]) + matches[i][1] = util.RunePos(str, m[1]) } return matches } -func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchEnd bool, lineNum int, line []byte, curRegion *region, statesOnly bool) LineMatch { - lineLen := CharacterCount(line) - if start == 0 { - if !statesOnly { - if _, ok := highlights[0]; !ok { - highlights[0] = curRegion.group +func (h *Highlighter) removeRange(start int, end int, removeStart int) { + var children []highlightStorage + removeEnd := removeStart + for i := removeStart; i < len(h.storage); i++ { + e := h.storage[i] + if start < e.start && e.start < end { + // log.Println("remove: start:", e.start, "end:", e.end, "group:", e.group) + removeEnd++ + h.removed = append(h.removed, e) + for childIdx, _ := range h.storage[i].children { + // log.Println("attached child: start:", h.storage[i].children[childIdx].start, "end:", h.storage[i].children[childIdx].end, "group:", h.storage[i].children[childIdx].group) + children = append(children, *(h.storage[i].children[childIdx])) } } } - - var firstRegion *region - firstLoc := []int{lineLen, 0} - searchNesting := true - endLoc := findIndex(curRegion.end, curRegion.skip, line) - if endLoc != nil { - if start == endLoc[0] { - searchNesting = false - } else { - firstLoc = endLoc - } + if removeStart < removeEnd { + h.storage = append(h.storage[:removeStart], h.storage[removeEnd:]...) } - if searchNesting { - for _, r := range curRegion.rules.regions { - loc := findIndex(r.start, r.skip, line) - if loc != nil { - if loc[0] < firstLoc[0] { - firstLoc = loc - firstRegion = r - } + + // remove possible children too +childLoop: + for childIdx, _ := range children { + for storageIdx, _ := range h.storage { + if children[childIdx].start == h.storage[storageIdx].start && children[childIdx].end == h.storage[storageIdx].end && children[childIdx].group == h.storage[storageIdx].group && children[childIdx].region == h.storage[storageIdx].region { + // log.Println("remove child: start:", h.storage[storageIdx].start, "end:", h.storage[storageIdx].end, "group:", h.storage[storageIdx].group) + h.storage = append(h.storage[:storageIdx], h.storage[storageIdx+1:]...) + continue childLoop } } } - if firstRegion != nil && firstLoc[0] != lineLen { - if !statesOnly { - highlights[start+firstLoc[0]] = firstRegion.limitGroup - } - h.highlightEmptyRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, sliceStart(line, firstLoc[1]), statesOnly) - h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, sliceStart(line, firstLoc[1]), firstRegion, statesOnly) - return highlights +} + +func (h *Highlighter) storeRange(start int, end int, group Group, r *region, isPattern bool) { + // log.Println("storeRange: start:", start, "end:", end, "group:", group) + var parent *region + if isPattern { + parent = r + } else if r != nil { + parent = r.parent } - if !statesOnly { - fullHighlights := make([]Group, lineLen) - for i := 0; i < len(fullHighlights); i++ { - fullHighlights[i] = curRegion.group + updated := false + for k, e := range h.storage { + if r == e.region && group == e.group && start == e.end { + // same region, update ... + h.storage[k].end = end + // log.Println("exchanged to: start:", h.storage[k].start, "end:", h.storage[k].end, "group:", h.storage[k].group) + updated = true + start = h.storage[k].start } + } - if searchNesting { - for _, p := range curRegion.rules.patterns { - if curRegion.group == curRegion.limitGroup || p.group == curRegion.limitGroup { - matches := findAllIndex(p.regex, line) - for _, m := range matches { - if ((endLoc == nil) || (m[0] < endLoc[0])) { - for i := m[0]; i < m[1]; i++ { - fullHighlights[i] = p.group - } - } + for k, e := range h.storage { + if e.region != nil && r != nil { + if e.region.parent == parent { + if r != e.region { + // sibling regions, search for overlaps ... + if start < e.start && end > e.start { + // overlap from left + } else if start == e.start && end == e.end { + // same match + continue + } else if start <= e.start && end >= e.end { + // larger match + } else if start >= e.start && end <= e.end { + // smaller match + return + } else if start > e.start && start < e.end && end > e.end { + // overlap from right + return + } else { + continue + } + + if !updated { + // log.Println("exchanged from: start:", e.start, "end:", e.end, "group:", e.group) + h.storage[k] = highlightStorage{start, end, group, r, nil, isPattern} + + // check and remove follow-ups matching the same + h.removeRange(start, end, k+1) + } else { + h.removeRange(start, end, k) } + return + } + } else { + if parent != e.region && start >= e.start && end <= e.end { + return } - } - } - for i, h := range fullHighlights { - if i == 0 || h != fullHighlights[i-1] { - highlights[start+i] = h } } } - loc := endLoc - if loc != nil { - if !statesOnly { - highlights[start+loc[0]] = curRegion.limitGroup - } - if curRegion.parent == nil { - if !statesOnly { - highlights[start+loc[1]] = 0 + if !updated { + h.storage = append(h.storage, highlightStorage{start, end, group, r, nil, isPattern}) + } + + // add possible child entry + if parent != nil { + storageLoop: + for k, e := range h.storage { + if e.region == parent && e.start < start && end < e.end { + for _, child := range h.storage[k].children { + if child == &(h.storage[len(h.storage)-1]) { + continue storageLoop + } + } + + // log.Println("add child: start:", h.storage[k].start, "end:", h.storage[k].end, "group:", h.storage[k].group) + h.storage[k].children = append(h.storage[k].children, &(h.storage[len(h.storage)-1])) } - h.highlightEmptyRegion(highlights, start+loc[1], canMatchEnd, lineNum, sliceStart(line, loc[1]), statesOnly) - return highlights } - if !statesOnly { - highlights[start+loc[1]] = curRegion.parent.group - } - h.highlightRegion(highlights, start+loc[1], canMatchEnd, lineNum, sliceStart(line, loc[1]), curRegion.parent, statesOnly) - return highlights + } +} + +func (h *Highlighter) highlightPatterns(start int, lineNum int, line []byte, curRegion *region) { + lineLen := util.CharacterCount(line) + // log.Println("highlightPatterns: lineNum:", lineNum, "start:", start, "line:", string(line)) + if lineLen == 0 { + return } - if canMatchEnd { - h.lastRegion = curRegion + var patterns []*pattern + if curRegion == nil { + patterns = h.Def.rules.patterns + } else { + patterns = curRegion.rules.patterns } - return highlights + for _, p := range patterns { + matches := findAllIndex(p.regex, nil, line) + for _, m := range matches { + h.storeRange(start+m[0], start+m[1], p.group, curRegion, true) + } + } } -func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canMatchEnd bool, lineNum int, line []byte, statesOnly bool) LineMatch { - lineLen := CharacterCount(line) +func (h *Highlighter) highlightRegions(start int, lineNum int, line []byte, curRegion *region, regions []*region, nestedRegion bool) { + lineLen := util.CharacterCount(line) + // log.Println("highlightRegions: lineNum:", lineNum, "start:", start, "line:", string(line)) if lineLen == 0 { - if canMatchEnd { - h.lastRegion = nil - } - return highlights + return + } + + if nestedRegion { + h.highlightPatterns(start, lineNum, line, curRegion) + } else { + h.highlightPatterns(start, lineNum, line, nil) } - var firstRegion *region - firstLoc := []int{lineLen, 0} - for _, r := range h.Def.rules.regions { - loc := findIndex(r.start, r.skip, line) - if loc != nil { - if loc[0] < firstLoc[0] { - firstLoc = loc - firstRegion = r +regionLoop: + for _, r := range regions { + // log.Println("r.start:", r.start.String(), "r.end", r.end.String()) + if !nestedRegion && curRegion != nil && curRegion != r { + continue + } + startMatches := findAllIndex(r.start, r.skip, line) + endMatches := findAllIndex(r.end, r.skip, line) + samePattern := false + startLoop: + for startIdx := 0; startIdx < len(startMatches); startIdx++ { + // log.Println("startIdx:", startIdx, "of", len(startMatches)) + startMatch := startMatches[startIdx] + for endIdx := 0; endIdx < len(endMatches); endIdx++ { + // log.Println("startIdx:", startIdx, "of", len(startMatches), "/ endIdx:", endIdx, "of", len(endMatches), "/ h.lastStart:", h.lastStart, "/ h.lastEnd:", h.lastEnd) + endMatch := endMatches[endIdx] + if startMatch[0] == endMatch[0] { + // start and end are the same (pattern) + // log.Println("start == end") + samePattern = true + if len(startMatches) == len(endMatches) { + // special case in the moment both are the same + if curRegion == r { + if len(startMatches) > 1 { + // end < start + continue startLoop + } else if len(startMatches) > 0 { + // ... end + startIdx = len(startMatches) + continue startLoop + } + } else { + // start ... or start < end + } + } + } else if startMatch[1] <= endMatch[0] { + if !nestedRegion && h.lastStart < start+startMatch[0] && start+startMatch[0] < h.lastEnd { + continue + } + // start and end at the current line + // log.Println("start < end") + update := false + if h.lastStart == -1 || h.lastStart < start+endMatch[1] { + h.lastStart = start + startMatch[0] + h.lastEnd = start + endMatch[1] + update = true + } + h.storeRange(start+startMatch[0], start+startMatch[1], r.limitGroup, r, false) + h.storeRange(start+startMatch[1], start+endMatch[0], r.group, r, false) + h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r, false) + h.highlightRegions(start+startMatch[1], lineNum, util.SliceStartEnd(line, startMatch[1], endMatch[0]), r, r.rules.regions, true) + if samePattern { + startIdx += 1 + } + if update { + if curRegion != nil { + h.lastRegion = r.parent + } else { + h.lastRegion = nil + } + curRegion = h.lastRegion + } + continue startLoop + } else if endMatch[1] <= startMatch[0] { + if start+endMatch[0] < h.lastEnd || curRegion == nil { + continue + } + // start and end at the current line, but switched + // log.Println("end < start") + h.lastStart = start + h.lastEnd = start + endMatch[1] + h.storeRange(start, start+endMatch[0], r.group, r, false) + h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r, false) + h.highlightRegions(start, lineNum, util.SliceStart(line, endMatch[0]), r, r.rules.regions, true) + h.highlightPatterns(start+endMatch[1], lineNum, util.SliceStartEnd(line, endMatch[1], startMatch[0]), nil) + if curRegion != nil { + h.lastRegion = r.parent + } else { + h.lastRegion = nil + } + curRegion = h.lastRegion + } + } + if nestedRegion || start+startMatch[0] < h.lastStart || h.lastEnd < start+startMatch[0] { + // start at the current, but end at the next line + // log.Println("start ...") + if h.lastStart == -1 || start+startMatch[0] < h.lastStart || h.lastEnd < start+startMatch[0] { + h.lastStart = start + startMatch[0] + h.lastEnd = start + lineLen - 1 + h.lastRegion = r + } + h.storeRange(start+startMatch[0], start+startMatch[1], r.limitGroup, r, false) + h.storeRange(start+startMatch[1], start+lineLen, r.group, r, false) + h.highlightRegions(start+startMatch[1], lineNum, util.SliceEnd(line, startMatch[1]), r, r.rules.regions, true) + continue regionLoop } } - } - if firstRegion != nil && firstLoc[0] != lineLen { - if !statesOnly { - highlights[start+firstLoc[0]] = firstRegion.limitGroup + if curRegion == r { + if (len(startMatches) == 0 && len(endMatches) > 0) || (samePattern && (len(startMatches) == len(endMatches))) { + for _, endMatch := range endMatches { + // end at the current, but start at the previous line + // log.Println("... end") + h.lastStart = start + h.lastEnd = start + endMatch[1] + h.storeRange(start, start+endMatch[0], r.group, r, false) + h.storeRange(start+endMatch[0], start+endMatch[1], r.limitGroup, r, false) + h.highlightRegions(start, lineNum, util.SliceStart(line, endMatch[0]), r, r.rules.regions, true) + if curRegion != nil { + h.lastRegion = r.parent + } else { + h.lastRegion = nil + } + curRegion = h.lastRegion + h.highlightRegions(start+endMatch[1], lineNum, util.SliceEnd(line, endMatch[1]), curRegion, h.Def.rules.regions, false) + break + } + } else if len(startMatches) == 0 && len(endMatches) == 0 { + // no start and end found in this region + h.storeRange(start, start+lineLen, curRegion.group, r, false) + } } - h.highlightEmptyRegion(highlights, start, false, lineNum, sliceEnd(line, firstLoc[0]), statesOnly) - h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, sliceStart(line, firstLoc[1]), firstRegion, statesOnly) - return highlights } - if statesOnly { - if canMatchEnd { - h.lastRegion = nil + if curRegion != nil && !nestedRegion { + // current region still open + // log.Println("...") + if curRegion.rules != nil { + h.highlightRegions(start, lineNum, line, curRegion, curRegion.rules.regions, true) } + if curRegion == h.lastRegion && curRegion.parent != nil { + var regions []*region + regions = append(regions, curRegion) + h.highlightRegions(start, lineNum, line, curRegion, regions, true) + } + } +} - return highlights +func (h *Highlighter) highlight(highlights LineMatch, start int, lineNum int, line []byte, curRegion *region) (LineMatch, *region) { + lineLen := util.CharacterCount(line) + // log.Println("highlight: lineNum:", lineNum, "start:", start, "line:", string(line)) + if lineLen == 0 { + return highlights, curRegion } - fullHighlights := make([]Group, len(line)) - for _, p := range h.Def.rules.patterns { - matches := findAllIndex(p.regex, line) - for _, m := range matches { - for i := m[0]; i < m[1]; i++ { - fullHighlights[i] = p.group - } + h.lastRegion = curRegion + h.lastStart = -1 + h.lastEnd = -1 + h.storage = h.storage[:0] + h.removed = h.removed[:0] + + h.highlightRegions(start, lineNum, line, curRegion, h.Def.rules.regions, false) + + // check if entries have been removed by invalid region + for _, e := range h.removed { + h.storeRange(e.start, e.end, e.group, e.region, e.pattern) + } + + fullHighlights := make([]Group, lineLen) + + for _, e := range h.storage { + for i := e.start; i < e.end; i++ { + fullHighlights[i] = e.group + // log.Println("fullHighlights[", i, "]:", e.group) } } + for i, h := range fullHighlights { if i == 0 || h != fullHighlights[i-1] { - // if _, ok := highlights[start+i]; !ok { - highlights[start+i] = h - // } + highlights[i] = h } } - if canMatchEnd { - h.lastRegion = nil - } - - return highlights + return highlights, h.lastRegion } // HighlightString syntax highlights a string @@ -289,82 +440,27 @@ func (h *Highlighter) HighlightString(input string) []LineMatch { for i := 0; i < len(lines); i++ { line := []byte(lines[i]) highlights := make(LineMatch) - - if i == 0 || h.lastRegion == nil { - lineMatches = append(lineMatches, h.highlightEmptyRegion(highlights, 0, true, i, line, false)) - } else { - lineMatches = append(lineMatches, h.highlightRegion(highlights, 0, true, i, line, h.lastRegion, false)) - } + match, _ := h.highlight(highlights, 0, i, line, nil) + lineMatches = append(lineMatches, match) } return lineMatches } -// HighlightStates correctly sets all states for the buffer -func (h *Highlighter) HighlightStates(input LineStates) { - for i := 0; ; i++ { - input.Lock() - if i >= input.LinesNum() { - input.Unlock() - break - } - - line := input.LineBytes(i) - // highlights := make(LineMatch) - - if i == 0 || h.lastRegion == nil { - h.highlightEmptyRegion(nil, 0, true, i, line, true) - } else { - h.highlightRegion(nil, 0, true, i, line, h.lastRegion, true) - } - - curState := h.lastRegion - - input.SetState(i, curState) - input.Unlock() - } -} - -// HighlightMatches sets the matches for each line from startline to endline -// It sets all other matches in the buffer to nil to conserve memory -// This assumes that all the states are set correctly -func (h *Highlighter) HighlightMatches(input LineStates, startline, endline int) { - for i := startline; i <= endline; i++ { - input.Lock() - if i >= input.LinesNum() { - input.Unlock() - break - } - - line := input.LineBytes(i) - highlights := make(LineMatch) - - var match LineMatch - if i == 0 || input.State(i-1) == nil { - match = h.highlightEmptyRegion(highlights, 0, true, i, line, false) - } else { - match = h.highlightRegion(highlights, 0, true, i, line, input.State(i-1), false) - } - - input.SetMatch(i, match) - input.Unlock() - } -} - -// ReHighlightStates will scan down from `startline` and set the appropriate end of line state -// for each line until it comes across a line whose state does not change -// returns the number of the final line -func (h *Highlighter) ReHighlightStates(input LineStates, startline int) int { - // lines := input.LineData() - - h.lastRegion = nil +// Highlight sets the state and matches for each line from startline to endline, +// and also for some amount of lines after endline, until it detects a line +// whose state does not change, which means that the lines after it do not change +// their highlighting and therefore do not need to be updated. +func (h *Highlighter) Highlight(input LineStates, startline, endline int) { + var curState *region if startline > 0 { input.Lock() if startline-1 < input.LinesNum() { - h.lastRegion = input.State(startline - 1) + curState = input.State(startline - 1) } input.Unlock() } + for i := startline; ; i++ { input.Lock() if i >= input.LinesNum() { @@ -373,26 +469,24 @@ func (h *Highlighter) ReHighlightStates(input LineStates, startline int) int { } line := input.LineBytes(i) - // highlights := make(LineMatch) + highlights := make(LineMatch) - // var match LineMatch - if i == 0 || h.lastRegion == nil { - h.highlightEmptyRegion(nil, 0, true, i, line, true) - } else { - h.highlightRegion(nil, 0, true, i, line, h.lastRegion, true) + match, newState := h.highlight(highlights, 0, i, line, curState) + curState = newState + + var lastState *region + if i >= endline { + lastState = input.State(i) } - curState := h.lastRegion - lastState := input.State(i) input.SetState(i, curState) + input.SetMatch(i, match) input.Unlock() - if curState == lastState { - return i + if i >= endline && curState == lastState { + break } } - - return input.LinesNum() - 1 } // ReHighlightLine will rehighlight the state and match for a single line @@ -403,19 +497,14 @@ func (h *Highlighter) ReHighlightLine(input LineStates, lineN int) { line := input.LineBytes(lineN) highlights := make(LineMatch) - h.lastRegion = nil + var curState *region if lineN > 0 { - h.lastRegion = input.State(lineN - 1) + curState = input.State(lineN - 1) } - var match LineMatch - if lineN == 0 || h.lastRegion == nil { - match = h.highlightEmptyRegion(highlights, 0, true, lineN, line, false) - } else { - match = h.highlightRegion(highlights, 0, true, lineN, line, h.lastRegion, false) - } - curState := h.lastRegion + match, newState := h.highlight(highlights, 0, lineN, line, curState) + curState = newState - input.SetMatch(lineN, match) input.SetState(lineN, curState) + input.SetMatch(lineN, match) } diff --git a/pkg/highlight/unicode.go b/pkg/highlight/unicode.go deleted file mode 100644 index a18118a6c..000000000 --- a/pkg/highlight/unicode.go +++ /dev/null @@ -1,85 +0,0 @@ -package highlight - -import ( - "unicode" - "unicode/utf8" -) - -var minMark = rune(unicode.Mark.R16[0].Lo) - -func isMark(r rune) bool { - // Fast path - if r < minMark { - return false - } - return unicode.In(r, unicode.Mark) -} - -// DecodeCharacter returns the next character from an array of bytes -// A character is a rune along with any accompanying combining runes -func DecodeCharacter(b []byte) (rune, []rune, int) { - r, size := utf8.DecodeRune(b) - b = b[size:] - c, s := utf8.DecodeRune(b) - - var combc []rune - for isMark(c) { - combc = append(combc, c) - size += s - - b = b[s:] - c, s = utf8.DecodeRune(b) - } - - return r, combc, size -} - -// DecodeCharacterInString returns the next character from a string -// A character is a rune along with any accompanying combining runes -func DecodeCharacterInString(str string) (rune, []rune, int) { - r, size := utf8.DecodeRuneInString(str) - str = str[size:] - c, s := utf8.DecodeRuneInString(str) - - var combc []rune - for isMark(c) { - combc = append(combc, c) - size += s - - str = str[s:] - c, s = utf8.DecodeRuneInString(str) - } - - return r, combc, size -} - -// CharacterCount returns the number of characters in a byte array -// Similar to utf8.RuneCount but for unicode characters -func CharacterCount(b []byte) int { - s := 0 - - for len(b) > 0 { - r, size := utf8.DecodeRune(b) - if !isMark(r) { - s++ - } - - b = b[size:] - } - - return s -} - -// CharacterCount returns the number of characters in a string -// Similar to utf8.RuneCountInString but for unicode characters -func CharacterCountInString(str string) int { - s := 0 - - for _, r := range str { - if !isMark(r) { - s++ - } - } - - return s -}