Refactored and strengthened completion system (#52)

* Load default options

* fix stupid

* Fix clearline after input buffer

* Change UNIX newlines with returned newlines in display (more reliable)

* Always use original terminal file descriptor for low-level cursor
requests, and potentially most other virtual terminal sequences.

* Don't code too late in the night

* Fix term length

* Fix

* Add options related to completions

* Add option to keep escape sequences in inserted comp values

* Add cursor style options for any keymap (set cursor-<keymap> block)

* Fix some excess completion hint bugs

* Try to enhance support for arbitrary colors in descriptions

* Fix comp messages not used

* Fix/clean/enhance many details and logic in completions

* Fixes and enhancements to alias overflowing.
Note done yet:
- Bug when no more aliases to show: does not recompute the compound
  padding. Related, selected highlighting not applied correctly.

* New completion generation code, entirely refactored

* Fix padding computations

* Fix alias detection

* Fix alias selection

* Fix all display completion stuff.

* Reintegrate fuzzy search

* Fix fuzzy search highlighting

* Multiple fixes to complex completion (quotes/escapes/prefixes)

* Fix autosuggest clearing

* Cleanup

* Lots of cleanup and fixes, still things to do.

* Cleanup

* Other fixes to display, better support for color

* Final fixes to display

* Fix sorting

* Cleanup

completions.go

@@ -22,6 +22,7 @@ type Completions struct {
 	noSort   map[string]bool
 	listSep  map[string]string
 	pad      map[string]bool
+	escapes  map[string]bool
 
 	// Initially this will be set to the part of the current word
 	// from the beginning of the word up to the position of the cursor.
@@ -48,7 +49,7 @@ func CompleteValues(values ...string) Completions {
 // CompleteStyledValues is like CompleteValues but also accepts a style.
 func CompleteStyledValues(values ...string) Completions {
 	if length := len(values); length%2 != 0 {
-		return Message("invalid amount of arguments [CompleteStyledValues]: %v", length)
+		return CompleteMessage("invalid amount of arguments [CompleteStyledValues]: %v", length)
 	}
 
 	vals := make([]Completion, 0, len(values)/2)
@@ -62,7 +63,7 @@ func CompleteStyledValues(values ...string) Completions {
 // CompleteValuesDescribed completes arbitrary key (values) with an additional description (value, description pairs).
 func CompleteValuesDescribed(values ...string) Completions {
 	if length := len(values); length%2 != 0 {
-		return Message("invalid amount of arguments [CompleteValuesDescribed]: %v", length)
+		return CompleteMessage("invalid amount of arguments [CompleteValuesDescribed]: %v", length)
 	}
 
 	vals := make([]Completion, 0, len(values)/2)
@@ -76,7 +77,7 @@ func CompleteValuesDescribed(values ...string) Completions {
 // CompleteStyledValuesDescribed is like CompleteValues but also accepts a style.
 func CompleteStyledValuesDescribed(values ...string) Completions {
 	if length := len(values); length%3 != 0 {
-		return Message("invalid amount of arguments [CompleteStyledValuesDescribed]: %v", length)
+		return CompleteMessage("invalid amount of arguments [CompleteStyledValuesDescribed]: %v", length)
 	}
 
 	vals := make([]Completion, 0, len(values)/3)
@@ -87,13 +88,27 @@ func CompleteStyledValuesDescribed(values ...string) Completions {
 	return Completions{values: vals}
 }
 
+// CompleteMessage ads a help message to display along with
+// or in places where no completions can be generated.
+func CompleteMessage(msg string, args ...any) Completions {
+	comps := Completions{}
+
+	if len(args) > 0 {
+		msg = fmt.Sprintf(msg, args...)
+	}
+
+	comps.messages.Add(msg)
+
+	return comps
+}
+
 // CompleteRaw directly accepts a list of prepared Completion values.
 func CompleteRaw(values []Completion) Completions {
 	return Completions{values: completion.RawValues(values)}
 }
 
 // Message displays a help messages in places where no completions can be generated.
-func Message(msg string, args ...interface{}) Completions {
+func Message(msg string, args ...any) Completions {
 	comps := Completions{}
 
 	if len(args) > 0 {
@@ -108,7 +123,7 @@ func Message(msg string, args ...interface{}) Completions {
 // Suppress suppresses specific error messages using regular expressions.
 func (c Completions) Suppress(expr ...string) Completions {
 	if err := c.messages.Suppress(expr...); err != nil {
-		return Message(err.Error())
+		return CompleteMessage(err.Error())
 	}
 
 	return c
@@ -153,7 +168,7 @@ func (c Completions) Suffix(suffix string) Completions {
 }
 
 // Usage sets the usage.
-func (c Completions) Usage(usage string, args ...interface{}) Completions {
+func (c Completions) Usage(usage string, args ...any) Completions {
 	return c.UsageF(func() string {
 		return fmt.Sprintf(usage, args...)
 	})
@@ -255,7 +270,7 @@ func (c Completions) ListSeparator(seps ...string) Completions {
 	}
 
 	if length := len(seps); len(seps) > 1 && length%2 != 0 {
-		return Message("invalid amount of arguments (ListSeparator): %v", length)
+		return CompleteMessage("invalid amount of arguments (ListSeparator): %v", length)
 	}
 
 	if len(seps) == 1 {
@@ -322,6 +337,35 @@ func (c Completions) JustifyDescriptions(tags ...string) Completions {
 	return c
 }
 
+// PreserveEscapes forces the completion engine to keep all escaped characters in
+// the inserted completion (c.Value of the Completion type). By default, those are
+// stripped out and only kept in the completion.Display. If no arguments are given,
+// escape sequence preservation will apply to all tags.
+//
+// This has very few use cases: one of them might be when you want to read a string
+// from the readline shell that might include color sequences to be preserved.
+// In such cases, this function gives a double advantage: the resulting completion
+// is still "color-displayed" in the input line, and returned to the readline with
+// them. A classic example is where you want to read a prompt string configuration.
+//
+// Note that this option might have various undefined behaviors when it comes to
+// completion prefix matching, insertion, removal and related things.
+func (c Completions) PreserveEscapes(tags ...string) Completions {
+	if c.escapes == nil {
+		c.escapes = make(map[string]bool)
+	}
+
+	if len(tags) == 0 {
+		c.escapes["*"] = true
+	}
+
+	for _, tag := range tags {
+		c.escapes[tag] = true
+	}
+
+	return c
+}
+
 // Merge merges Completions (existing values are overwritten)
 //
 //	a := CompleteValues("A", "B").Invoke(c)
@@ -400,6 +444,10 @@ func (c *Completions) convert() completion.Values {
 	comps.NoSort = c.noSort
 	comps.ListSep = c.listSep
 	comps.Pad = c.pad
+	comps.Escapes = c.escapes
+
+	comps.PREFIX = c.PREFIX
+	comps.SUFFIX = c.SUFFIX
 
 	return comps
 }

go.mod

@@ -1,6 +1,6 @@
 module github.com/reeflective/readline
 
-go 1.20
+go 1.21
 
 require (
 	github.com/google/go-cmp v0.5.8

internal/color/color.go

@@ -3,6 +3,8 @@ package color
 import (
 	"os"
 	"regexp"
+	"strconv"
+	"strings"
 )
 
 // Base text effects.
@@ -82,6 +84,44 @@ func Fmt(color string) string {
 	return SGRStart + color + SGREnd
 }
 
+// Trim accepts a string including arbitrary escaped sequences at arbitrary
+// index positions, and returns the first 'n' printable characters in this
+// string, including all escape codes found between and immediately around
+// those characters (including surrounding 1st and 80th ones).
+func Trim(input string, maxPrintableLength int) string {
+	if len(input) < maxPrintableLength {
+		return input
+	}
+
+	// Find all escape sequences in the input
+	escapeIndices := re.FindAllStringIndex(input, -1)
+
+	// Iterate over escape sequences to find the
+	// last escape index within maxPrintableLength
+	for _, indices := range escapeIndices {
+		if indices[0] <= maxPrintableLength {
+			maxPrintableLength += indices[1] - indices[0]
+		} else {
+			break
+		}
+	}
+
+	// Determine the end index for limiting printable content
+	return input[:maxPrintableLength]
+}
+
+// UnquoteRC removes the `\e` escape used in readline .inputrc
+// configuration values and replaces it with the printable escape.
+func UnquoteRC(color string) string {
+	color = strings.ReplaceAll(color, `\e`, "\x1b")
+
+	if unquoted, err := strconv.Unquote(color); err == nil {
+		return unquoted
+	}
+
+	return color
+}
+
 // HasEffects returns true if colors and effects are supported
 // on the current terminal.
 func HasEffects() bool {
@@ -159,14 +199,3 @@ var re = regexp.MustCompile(ansi)
 func Strip(str string) string {
 	return re.ReplaceAllString(str, "")
 }
-
-// wrong: reapplies fg/bg escapes regardless of the string passed.
-// Users should be in charge of applying any effect as they wish.
-// func SGR(color string, fg bool) string {
-// 	if fg {
-// 		return SGRStart + FgColorStart + color + SGREnd
-// 		// return SGRStart + color + SGREnd
-// 	}
-//
-// 	return SGRStart + BgColorStart + color + SGREnd
-// }

internal/completion/completion.go

@@ -17,6 +17,9 @@ type Candidate struct {
 	// inserted immediately after the completion. This is used for slash-autoremoval in path
 	// completions, comma-separated completions, etc.
 	noSpace SuffixMatcher
+
+	displayLen int // Real length of the displayed candidate, that is not counting escaped sequences.
+	descLen    int
 }
 
 // Values is used internally to hold all completion candidates and their associated data.
@@ -29,6 +32,7 @@ type Values struct {
 	NoSort   map[string]bool
 	ListSep  map[string]string
 	Pad      map[string]bool
+	Escapes  map[string]bool
 
 	// Initially this will be set to the part of the current word
 	// from the beginning of the word up to the position of the cursor.

internal/completion/display.go

@@ -3,6 +3,7 @@ package completion
 import (
 	"bufio"
 	"fmt"
+	"regexp"
 	"strings"
 
 	"github.com/reeflective/readline/internal/color"
@@ -30,7 +31,7 @@ func Display(eng *Engine, maxRows int) {
 	completions := term.ClearLineAfter
 
 	for _, group := range eng.groups {
-		completions += group.writeComps(eng)
+		completions += eng.renderCompletions(group)
 	}
 
 	// Crop the completions so that it fits within our terminal
@@ -47,6 +48,129 @@ func Coordinates(e *Engine) int {
 	return e.usedY
 }
 
+// renderCompletions renders all completions in a given list (with aliases or not).
+// The descriptions list argument is optional.
+func (e *Engine) renderCompletions(grp *group) string {
+	var builder strings.Builder
+
+	if len(grp.rows) == 0 {
+		return ""
+	}
+
+	if grp.tag != "" {
+		tag := fmt.Sprintf("%s%s%s %s", color.Bold, color.FgYellow, grp.tag, color.Reset)
+		builder.WriteString(tag + term.ClearLineAfter + term.NewlineReturn)
+	}
+
+	for rowIndex, row := range grp.rows {
+		for columnIndex := range grp.columnsWidth {
+			var value Candidate
+
+			// If there are aliases, we might have no completions at the current
+			// coordinates, so just print the corresponding padding and return.
+			if len(row) > columnIndex {
+				value = row[columnIndex]
+			}
+
+			// Apply all highlightings to the displayed value:
+			// selection, prefixes, styles and other things,
+			padding := grp.getPad(value, columnIndex, false)
+			isSelected := rowIndex == grp.posY && columnIndex == grp.posX && grp.isCurrent
+			display := e.highlightDisplay(grp, value, padding, columnIndex, isSelected)
+
+			builder.WriteString(display)
+
+			// Add description if no aliases, or if done with them.
+			onLast := columnIndex == len(grp.columnsWidth)-1
+			if grp.aliased && onLast && value.Description == "" {
+				value = row[0]
+			}
+
+			if !grp.aliased || onLast {
+				grp.maxDescAllowed = grp.setMaximumSizes(columnIndex)
+
+				descPad := grp.getPad(value, columnIndex, true)
+				desc := e.highlightDesc(grp, value, descPad, rowIndex, columnIndex, isSelected)
+				builder.WriteString(desc)
+			}
+		}
+
+		// We're done for this line.
+		builder.WriteString(term.ClearLineAfter + term.NewlineReturn)
+	}
+
+	return builder.String()
+}
+
+func (e *Engine) highlightDisplay(grp *group, val Candidate, pad, col int, selected bool) (candidate string) {
+	// An empty display value means padding.
+	if val.Display == "" {
+		return padSpace(pad)
+	}
+
+	reset := color.Fmt(val.Style)
+	candidate, padded := grp.trimDisplay(val, pad, col)
+
+	if e.IsearchRegex != nil && e.isearchBuf.Len() > 0 && !selected {
+		match := e.IsearchRegex.FindString(candidate)
+		match = color.Fmt(color.Bg+"244") + match + color.Reset + reset
+		candidate = e.IsearchRegex.ReplaceAllLiteralString(candidate, match)
+	}
+
+	if selected {
+		// If the comp is currently selected, overwrite any highlighting already applied.
+		userStyle := color.UnquoteRC(e.config.GetString("completion-selection-style"))
+		selectionHighlightStyle := color.Fmt(color.Bg+"255") + userStyle
+		candidate = selectionHighlightStyle + candidate
+
+		if grp.aliased {
+			candidate += color.Reset
+		}
+	} else {
+		// Highlight the prefix if any and configured for it.
+		if e.config.GetBool("colored-completion-prefix") && e.prefix != "" {
+			if prefixMatch, err := regexp.Compile(fmt.Sprintf("^%s", e.prefix)); err == nil {
+				prefixColored := color.Bold + color.FgBlue + e.prefix + color.BoldReset + color.FgDefault + reset
+				candidate = prefixMatch.ReplaceAllString(candidate, prefixColored)
+			}
+		}
+
+		candidate = reset + candidate + color.Reset
+	}
+
+	return candidate + padded
+}
+
+func (e *Engine) highlightDesc(grp *group, val Candidate, pad, row, col int, selected bool) (desc string) {
+	if val.Description == "" {
+		return color.Reset
+	}
+
+	desc, padded := grp.trimDesc(val, pad)
+
+	// If the next row has the same completions, replace the description with our hint.
+	if len(grp.rows) > row+1 && grp.rows[row+1][0].Description == val.Description {
+		desc = "|"
+	} else if e.IsearchRegex != nil && e.isearchBuf.Len() > 0 && !selected {
+		match := e.IsearchRegex.FindString(desc)
+		match = color.Fmt(color.Bg+"244") + match + color.Reset + color.Dim
+		desc = e.IsearchRegex.ReplaceAllLiteralString(desc, match)
+	}
+
+	// If the comp is currently selected, overwrite any highlighting already applied.
+	// Replace all background reset escape sequences in it, to ensure correct display.
+	if row == grp.posY && col == grp.posX && grp.isCurrent && !grp.aliased {
+		userDescStyle := color.UnquoteRC(e.config.GetString("completion-selection-style"))
+		selectionHighlightStyle := color.Fmt(color.Bg+"255") + userDescStyle
+		desc = strings.ReplaceAll(desc, color.BgDefault, userDescStyle)
+		desc = selectionHighlightStyle + desc
+	}
+
+	compDescStyle := color.UnquoteRC(e.config.GetString("completion-description-style"))
+
+	return compDescStyle + desc + color.Reset + padded
+}
+
 // cropCompletions - When the user cycles through a completion list longer
 // than the console MaxTabCompleterRows value, we crop the completions string
 // so that "global" cycling (across all groups) is printed correctly.

internal/completion/engine.go

@@ -116,7 +116,7 @@ func (e *Engine) SkipDisplay() {
 func (e *Engine) Select(row, column int) {
 	grp := e.currentGroup()
 
-	if grp == nil || len(grp.values) == 0 {
+	if grp == nil || len(grp.rows) == 0 {
 		return
 	}