Add grapheme clustering support for cursor movement

**Problem** Test sequence (copy and paste into any REPL/edit area)::

     👨‍👩‍👧 👩‍❤‍👨 👩‍💻👋🏿 ❤️⭐ 🇯🇵🇩🇪 café niño Åộ 中文!.

Moving the cursor over and around emojis get strange. insertions become
chaotic. Cursor position becomes indeterminate (even negative!), input
result becomes more corrupted with user confusion as draws become
corrupted. This is briefly described in #274 by @jonathanslenders:

> Notice that it still requires multiple cursor movements (left/right
arrow) to move across these characters.

**Solution**: Close #274 "Handle decomposed unicode characters" (2018)
through careful integration of new functions,
[wcwidth.iter_graphemes](https://wcwidth.readthedocs.io/en/latest/intro.html#iter-graphemes)
and
[wcwidth.grapheme_boundary_before](https://wcwidth.readthedocs.io/en/latest/api.html#wcwidth.grapheme_boundary_before).

getting there, working on a PTY test suite

I don't feel comfortable changing so much code for a large library
without also including more detailed tests -- i keep fixing all errors
with TDD/automatic tests, but when using it interactively, the cursor
position is out of control

Author: jquast

.github/workflows/test.yaml

@@ -33,6 +33,8 @@ jobs:
       - name: Unit test
         run: |
           uvx --with . --with pytest coverage run -m pytest tests/
+          uvx coverage combine
+          uvx coverage report
       - name: Type Checking
         if: ${{ matrix.python-version != '3.8' }}
         run: |

pyproject.toml

@@ -22,7 +22,7 @@ classifiers = [
 ]
 requires-python = ">=3.8"
 dependencies = [
-    "wcwidth>=0.1.4",
+    "wcwidth>=0.5.1",
 ]
 
 [project.urls]
@@ -118,6 +118,10 @@ warn_return_any = true
 warn_unused_configs = true
 warn_unused_ignores = true
 
+[tool.coverage.run]
+source = ["src/prompt_toolkit"]
+parallel = true
+
 [build-system]
 requires = ["setuptools>=68"]
 build-backend = "setuptools.build_meta"

src/prompt_toolkit/buffer.py

@@ -18,6 +18,8 @@
 from functools import wraps
 from typing import Any, Callable, Coroutine, Iterable, TypeVar, cast
 
+import wcwidth
+
 from .application.current import get_app
 from .application.run_in_terminal import run_in_terminal
 from .auto_suggest import AutoSuggest, Suggestion
@@ -764,20 +766,24 @@ def auto_down(
 
     def delete_before_cursor(self, count: int = 1) -> str:
         """
-        Delete specified number of characters before cursor and return the
-        deleted text.
+        Delete specified number of grapheme clusters before cursor and return
+        the deleted text.
         """
         assert count >= 0
         deleted = ""
 
         if self.cursor_position > 0:
-            deleted = self.text[self.cursor_position - count : self.cursor_position]
-
-            new_text = (
-                self.text[: self.cursor_position - count]
-                + self.text[self.cursor_position :]
-            )
-            new_cursor_position = self.cursor_position - len(deleted)
+            # Find position after deleting `count` grapheme clusters.
+            # Loop is required since grapheme clusters have variable length.
+            pos = self.cursor_position
+            for _ in range(count):
+                if pos <= 0:
+                    break
+                pos = wcwidth.grapheme_boundary_before(self.text, pos)
+
+            deleted = self.text[pos:self.cursor_position]
+            new_text = self.text[:pos] + self.text[self.cursor_position :]
+            new_cursor_position = pos
 
             # Set new Document atomically.
             self.document = Document(new_text, new_cursor_position)
@@ -786,14 +792,19 @@ def delete_before_cursor(self, count: int = 1) -> str:
 
     def delete(self, count: int = 1) -> str:
         """
-        Delete specified number of characters and Return the deleted text.
+        Delete specified number of grapheme clusters and return the deleted text.
         """
         if self.cursor_position < len(self.text):
-            deleted = self.document.text_after_cursor[:count]
-            self.text = (
-                self.text[: self.cursor_position]
-                + self.text[self.cursor_position + len(deleted) :]
-            )
+            # Find position after `count` grapheme clusters.
+            text_after = self.text[self.cursor_position :]
+            pos = 0
+            for i, grapheme in enumerate(wcwidth.iter_graphemes(text_after)):
+                if i >= count:
+                    break
+                pos += len(grapheme)
+
+            deleted = text_after[:pos]
+            self.text = self.text[: self.cursor_position] + text_after[pos:]
             return deleted
         else:
             return ""

src/prompt_toolkit/document.py

@@ -10,6 +10,8 @@
 import weakref
 from typing import Callable, Dict, Iterable, List, NoReturn, Pattern, cast
 
+import wcwidth
+
 from .clipboard import ClipboardData
 from .filters import vi_mode
 from .selection import PasteMode, SelectionState, SelectionType
@@ -158,13 +160,49 @@ def selection(self) -> SelectionState | None:
 
     @property
     def current_char(self) -> str:
-        """Return character under cursor or an empty string."""
-        return self._get_char_relative_to_cursor(0) or ""
+        """
+        Return grapheme cluster at cursor position, or empty string at end.
+
+        Note: Returns a grapheme cluster which may contain multiple code points.
+        If cursor is inside a grapheme cluster (e.g., on a combining character),
+        returns the complete grapheme containing the cursor.
+        """
+        if self.cursor_position >= len(self.text):
+            return ""
+        grapheme_start = wcwidth.grapheme_boundary_before(
+            self.text, self.cursor_position + 1
+        )
+        for g in wcwidth.iter_graphemes(self.text[grapheme_start:]):
+            return g
+        return ""
 
     @property
     def char_before_cursor(self) -> str:
-        """Return character before the cursor or an empty string."""
-        return self._get_char_relative_to_cursor(-1) or ""
+        """
+        Return grapheme cluster before the cursor, or empty string at start.
+
+        Note: Returns a grapheme cluster which may contain multiple code points.
+        If cursor is inside a grapheme cluster (e.g., on a combining character),
+        returns the grapheme before the one containing the cursor.
+        """
+        if self.cursor_position == 0:
+            return ""
+
+        text = self.text
+        cursor = self.cursor_position
+
+        # Find reference point: cursor position or start of containing grapheme.
+        if cursor >= len(text):
+            reference = len(text)
+        else:
+            grapheme_start = wcwidth.grapheme_boundary_before(text, cursor + 1)
+            reference = grapheme_start if grapheme_start < cursor else cursor
+
+        if reference == 0:
+            return ""
+
+        prev_start = wcwidth.grapheme_boundary_before(text, reference)
+        return text[prev_start:reference]
 
     @property
     def text_before_cursor(self) -> str:
@@ -251,15 +289,6 @@ def leading_whitespace_in_current_line(self) -> str:
         length = len(current_line) - len(current_line.lstrip())
         return current_line[:length]
 
-    def _get_char_relative_to_cursor(self, offset: int = 0) -> str:
-        """
-        Return character relative to cursor position, or empty string
-        """
-        try:
-            return self.text[self.cursor_position + offset]
-        except IndexError:
-            return ""
-
     @property
     def on_first_line(self) -> bool:
         """
@@ -692,21 +721,44 @@ def find_previous_matching_line(
 
     def get_cursor_left_position(self, count: int = 1) -> int:
         """
-        Relative position for cursor left.
+        Relative position for cursor left (grapheme cluster aware).
         """
         if count < 0:
             return self.get_cursor_right_position(-count)
 
-        return -min(self.cursor_position_col, count)
+        line_before = self.current_line_before_cursor
+        if not line_before:
+            return 0
+
+        pos = len(line_before)
+        for _ in range(count):
+            if pos <= 0:
+                break
+            new_pos = wcwidth.grapheme_boundary_before(line_before, pos)
+            if new_pos == pos:
+                break
+            pos = new_pos
+
+        return pos - len(line_before)
 
     def get_cursor_right_position(self, count: int = 1) -> int:
         """
-        Relative position for cursor_right.
+        Relative position for cursor right (grapheme cluster aware).
         """
         if count < 0:
             return self.get_cursor_left_position(-count)
 
-        return min(count, len(self.current_line_after_cursor))
+        line_after = self.current_line_after_cursor
+        if not line_after:
+            return 0
+
+        pos = 0
+        for i, grapheme in enumerate(wcwidth.iter_graphemes(line_after)):
+            if i >= count:
+                break
+            pos += len(grapheme)
+
+        return pos
 
     def get_cursor_up_position(
         self, count: int = 1, preferred_column: int | None = None

src/prompt_toolkit/formatted_text/utils.py

@@ -9,7 +9,7 @@
 
 from typing import Iterable, cast
 
-from prompt_toolkit.utils import get_cwidth
+import wcwidth
 
 from .base import (
     AnyFormattedText,
@@ -48,17 +48,15 @@ def fragment_list_len(fragments: StyleAndTextTuples) -> int:
 def fragment_list_width(fragments: StyleAndTextTuples) -> int:
     """
     Return the character width of this text fragment list.
-    (Take double width characters into account.)
+    (Take double width characters and grapheme clusters into account.)
 
     :param fragments: List of ``(style_str, text)`` or
         ``(style_str, text, mouse_handler)`` tuples.
     """
-    ZeroWidthEscape = "[ZeroWidthEscape]"
     return sum(
-        get_cwidth(c)
+        wcwidth.width(item[1], control_codes="ignore")
         for item in fragments
-        for c in item[1]
-        if ZeroWidthEscape not in item[0]
+        if "[ZeroWidthEscape]" not in item[0]
     )
 
 

src/prompt_toolkit/layout/containers.py

@@ -10,6 +10,8 @@
 from functools import partial
 from typing import TYPE_CHECKING, Callable, Sequence, Union, cast
 
+import wcwidth
+
 from prompt_toolkit.application.current import get_app
 from prompt_toolkit.cache import SimpleCache
 from prompt_toolkit.data_structures import Point
@@ -2014,7 +2016,7 @@ def copy_line(
                     new_screen.zero_width_escapes[y + ypos][x + xpos] += text
                     continue
 
-                for c in text:
+                for c in wcwidth.iter_graphemes(text):
                     char = _CHAR_CACHE[c, style]
                     char_width = char.width
 
@@ -2052,26 +2054,7 @@ def copy_line(
                             for i in range(1, char_width):
                                 new_buffer_row[x + xpos + i] = empty_char
 
-                        # If this is a zero width characters, then it's
-                        # probably part of a decomposed unicode character.
-                        # See: https://en.wikipedia.org/wiki/Unicode_equivalence
-                        # Merge it in the previous cell.
-                        elif char_width == 0:
-                            # Handle all character widths. If the previous
-                            # character is a multiwidth character, then
-                            # merge it two positions back.
-                            for pw in [2, 1]:  # Previous character width.
-                                if (
-                                    x - pw >= 0
-                                    and new_buffer_row[x + xpos - pw].width == pw
-                                ):
-                                    prev_char = new_buffer_row[x + xpos - pw]
-                                    char2 = _CHAR_CACHE[
-                                        prev_char.char + c, prev_char.style
-                                    ]
-                                    new_buffer_row[x + xpos - pw] = char2
-
-                        # Keep track of write position for each character.
+                        # Keep track of write position for each grapheme.
                         current_rowcol_to_yx[lineno, col + skipped] = (
                             y + ypos,
                             x + xpos,

src/prompt_toolkit/layout/controls.py

@@ -8,6 +8,8 @@
 from abc import ABCMeta, abstractmethod
 from typing import TYPE_CHECKING, Callable, Hashable, Iterable, NamedTuple
 
+import wcwidth
+
 from prompt_toolkit.application.current import get_app
 from prompt_toolkit.buffer import Buffer
 from prompt_toolkit.cache import SimpleCache
@@ -674,29 +676,45 @@ def transform(
         ) -> _ProcessedLine:
             "Transform the fragments for a given line number."
 
-            # Get cursor position at this line.
-            def source_to_display(i: int) -> int:
-                """X position from the buffer to the x position in the
-                processed fragment list. By default, we start from the 'identity'
-                operation."""
-                return i
+            # Build code point to grapheme index mapping for cursor positioning.
+            line_text = fragment_list_to_text(fragments)
+            codepoint_to_grapheme: dict[int, int] = {}
+            grapheme_idx = 0
+            codepoint_idx = 0
+            for grapheme in wcwidth.iter_graphemes(line_text):
+                for _ in grapheme:
+                    codepoint_to_grapheme[codepoint_idx] = grapheme_idx
+                    codepoint_idx += 1
+                grapheme_idx += 1
+
+            def grapheme_source_to_display(i: int) -> int:
+                """Map code point index to grapheme index."""
+                if i >= codepoint_idx:
+                    return grapheme_idx + (i - codepoint_idx)
+                return codepoint_to_grapheme.get(i, grapheme_idx)
 
             transformation = merged_processor.apply_transformation(
                 TransformationInput(
                     self,
                     document,
                     lineno,
-                    source_to_display,
+                    grapheme_source_to_display,
                     fragments,
                     width,
                     height,
                     get_line,
                 )
             )
 
+            # Compose grapheme mapping with processor transformations.
+            proc_s2d = transformation.source_to_display
+
+            def final_source_to_display(i: int) -> int:
+                return proc_s2d(grapheme_source_to_display(i))
+
             return _ProcessedLine(
                 transformation.fragments,
-                transformation.source_to_display,
+                final_source_to_display,
                 transformation.display_to_source,
             )
 

src/prompt_toolkit/layout/utils.py

@@ -2,6 +2,8 @@
 
 from typing import TYPE_CHECKING, Iterable, List, TypeVar, cast, overload
 
+import wcwidth
+
 from prompt_toolkit.formatted_text.base import OneStyleAndTextTuple
 
 if TYPE_CHECKING:
@@ -60,7 +62,7 @@ def __setitem__(
 def explode_text_fragments(fragments: Iterable[_T]) -> _ExplodedList[_T]:
     """
     Turn a list of (style_str, text) tuples into another list where each string is
-    exactly one character.
+    exactly one grapheme cluster.
 
     It should be fine to call this function several times. Calling this on a
     list that is already exploded, is a null operation.
@@ -74,7 +76,7 @@ def explode_text_fragments(fragments: Iterable[_T]) -> _ExplodedList[_T]:
     result: list[_T] = []
 
     for style, string, *rest in fragments:
-        for c in string:
-            result.append((style, c, *rest))  # type: ignore
+        for grapheme in wcwidth.iter_graphemes(string):
+            result.append((style, grapheme, *rest))  # type: ignore
 
     return _ExplodedList(result)