Merge pull request #81 from h3x0c4t/unicode

Unicode support

Author: maxlandon

internal/core/keys.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/reeflective/readline/inputrc"
 	"github.com/reeflective/readline/internal/strutil"
+	"github.com/rivo/uniseg"
 )
 
 const (
@@ -22,7 +23,7 @@ var Stdin io.ReadCloser = os.Stdin
 
 var rxRcvCursorPos = regexp.MustCompile(`\x1b\[([0-9]+);([0-9]+)R`)
 
-// Keys is used read, manage and use keys input by the shell user.
+// Keys is used to read, manage and use keys input by the shell user.
 type Keys struct {
 	buf       []byte      // Keys read and waiting to be used.
 	matched   []rune      // Keys that have been successfully matched against a bind.
@@ -97,6 +98,20 @@ func WaitAvailableKeys(keys *Keys, cfg *inputrc.Config) {
 	}
 }
 
+// PeekKey returns the first key in the stack, without removing it.
+func PeekKey(keys *Keys) (key byte, empty bool) {
+	switch {
+	case len(keys.buf) > 0:
+		key = keys.buf[0]
+	case len(keys.macroKeys) > 0:
+		key = byte(keys.macroKeys[0])
+	default:
+		return byte(0), true
+	}
+
+	return key, false
+}
+
 // PopKey is used to pop a key off the key stack without
 // yet marking this key as having matched a bind command.
 func PopKey(keys *Keys) (key byte, empty bool) {
@@ -114,18 +129,40 @@ func PopKey(keys *Keys) (key byte, empty bool) {
 	return key, false
 }
 
-// PeekKey returns the first key in the stack, without removing it.
-func PeekKey(keys *Keys) (key byte, empty bool) {
+// PeekChar returns the first character in the stack, without
+// removing it (in order to provide Unicode support).
+func PeekChar(keys *Keys) (char []byte, empty bool) {
 	switch {
 	case len(keys.buf) > 0:
-		key = keys.buf[0]
+		// Use the uniseg library to correctly determine where each characters stop.
+		char, _, _, _ = uniseg.FirstGraphemeCluster(keys.buf, -1)
+		return char, false
+
 	case len(keys.macroKeys) > 0:
-		key = byte(keys.macroKeys[0])
+		// Macros already store keys as runes, just pick one.
+		// Maybe long-term we should find a remedy to this: storing
+		// macro keys as runes is inconsistent with the rest of our code.
+		return []byte(string(keys.macroKeys[0])), false
+
 	default:
-		return byte(0), true
+		return nil, true
 	}
+}
 
-	return key, false
+// PopChar is used to pop a character off the key stack without
+// yet marking this key as having matched a bind command.
+func PopChar(keys *Keys) (char []byte, empty bool) {
+	switch {
+	case len(keys.buf) > 0:
+		char, keys.buf, _, _ = uniseg.FirstGraphemeCluster(keys.buf, -1)
+	case len(keys.macroKeys) > 0:
+		char = []byte(string(keys.macroKeys[0]))
+		keys.macroKeys = keys.macroKeys[1:]
+	default:
+		return nil, true
+	}
+
+	return char, false
 }
 
 // MatchedKeys is used to indicate how many keys have been evaluated against the shell

internal/keymap/dispatch.go

@@ -96,7 +96,35 @@ func MatchMain(eng *Engine) (bind inputrc.Bind, command func(), prefix bool) {
 	return bind, command, prefix
 }
 
+func (m *Engine) makeMatch(active, prefixed inputrc.Bind) (prefix bool) {
+	m.active = active
+	m.prefixed = prefixed
+	return m.prefixed.Action != ""
+}
+
 func (m *Engine) dispatchKeys(binds map[string]inputrc.Bind) (bind inputrc.Bind, prefix bool, read, matched []byte) {
+	// Support for Unicode: if the character is multi-byte (UTF-8), consume all its bytes
+	// and treat it as a single self-insert action. Note that we just peek the characters
+	// here, so if it's actually not a UTF-8 character, we just keep going and re-peek later.
+	char, empty := core.PeekChar(m.keys)
+	if empty {
+		return m.active, prefix, read, matched
+	}
+
+	if len(char) > 1 {
+		read = append(read, char...)
+		match := inputrc.Bind{
+			Action: "self-insert",
+			Macro:  false,
+		}
+
+		matched = append(matched, char...)
+		prefix = m.makeMatch(match, inputrc.Bind{})
+		core.PopChar(m.keys)
+
+		return m.active, prefix, read, matched
+	}
+
 	for {
 		// Read a single byte from the input buffer.
 		// This mimics the way Bash reads input when the inputrc option `byte-oriented` is set.
@@ -114,9 +142,7 @@ func (m *Engine) dispatchKeys(binds map[string]inputrc.Bind) (bind inputrc.Bind,
 		// If the current keys have no matches but the previous
 		// matching process found a prefix, use it with the keys.
 		if match.Action == "" && len(prefixed) == 0 {
-			prefix = false
-			m.active = m.prefixed
-			m.prefixed = inputrc.Bind{}
+			prefix = m.makeMatch(m.prefixed, inputrc.Bind{})
 
 			// FIX related to Github issue #73, where someone
 			// complains not being able to input Unicode characters
@@ -148,9 +174,7 @@ func (m *Engine) dispatchKeys(binds map[string]inputrc.Bind) (bind inputrc.Bind,
 		}
 
 		// Or an exact match, so drop any prefixed one.
-		prefix = false
-		m.active = match
-		m.prefixed = inputrc.Bind{}
+		prefix = m.makeMatch(match, inputrc.Bind{})
 
 		break
 	}

internal/strutil/len.go

@@ -3,9 +3,10 @@ package strutil
 import (
 	"strings"
 
+	"github.com/rivo/uniseg"
+
 	"github.com/reeflective/readline/internal/color"
 	"github.com/reeflective/readline/internal/term"
-	"github.com/rivo/uniseg"
 )
 
 // FormatTabs replaces all '\t' occurrences in a string with 6 spaces each.