shumengya/sproutclaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(tui): leverage Intl.Segmenter for proper Unicode word boundaries (#5022)

Browse Source
This commit is contained in:
xu0o0
2026-05-27 06:28:47 +08:00
committed by GitHub
parent 8fb1e877cd
commit 4402100830
4 changed files with 141 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
packages/tui/src/components/editor.ts
View File

@@ -4,10 +4,11 @@ import { decodePrintableKey, matchesKey } from "../keys.ts";
import { KillRing } from "../kill-ring.ts";
import { type Component, CURSOR_MARKER, type Focusable, type TUI } from "../tui.ts";
import { UndoStack } from "../undo-stack.ts";
import { getSegmenter, isPunctuationChar, isWhitespaceChar, truncateToWidth, visibleWidth } from "../utils.ts";
import { getGraphemeSegmenter, getWordSegmenter, isWhitespaceChar, truncateToWidth, visibleWidth } from "../utils.ts";
import { SelectList, type SelectListLayoutOptions, type SelectListTheme } from "./select-list.ts";
const baseSegmenter = getSegmenter();
const graphemeSegmenter = getGraphemeSegmenter();
const wordSegmenter = getWordSegmenter();
/** Regex matching paste markers like `[paste #1 +123 lines]` or `[paste #2 1234 chars]`. */
const PASTE_MARKER_REGEX = /\[paste #(\d+)( (\+\d+ lines|\d+ chars))?\]/g;
@@ -27,7 +28,11 @@ function isPasteMarker(segment: string): boolean {
*
* Only markers whose numeric ID exists in `validIds` are merged.
*/
function segmentWithMarkers(text: string, validIds: Set<number>): Iterable<Intl.SegmentData> {
function segmentWithMarkers(
text: string,
baseSegmenter: Intl.Segmenter,
validIds: Set<number>,
): Iterable<Intl.SegmentData> {
// Fast path: no paste markers in the text or no valid IDs.
if (validIds.size === 0 || !text.includes("[paste #")) {
return baseSegmenter.segment(text);
@@ -109,7 +114,7 @@ export function wordWrapLine(line: string, maxWidth: number, preSegmented?: Intl
}
const chunks: TextChunk[] = [];
const segments = preSegmented ?? [...baseSegmenter.segment(line)];
const segments = preSegmented ?? [...graphemeSegmenter.segment(line)];
let currentWidth = 0;
let chunkStart = 0;
@@ -301,8 +306,8 @@ export class Editor implements Component, Focusable {
}
/** Segment text with paste-marker awareness, only merging markers with valid IDs. */
private segment(text: string): Iterable<Intl.SegmentData> {
return segmentWithMarkers(text, this.validPasteIds());
private segment(text: string, mode: "word" | "grapheme"): Iterable<Intl.SegmentData> {
return segmentWithMarkers(text, mode === "word" ? wordSegmenter : graphemeSegmenter, this.validPasteIds());
}
getPaddingX(): number {
@@ -482,7 +487,7 @@ export class Editor implements Component, Focusable {
if (after.length > 0) {
// Cursor is on a character (grapheme) - replace it with highlighted version
// Get the first grapheme from 'after'
const afterGraphemes = [...this.segment(after)];
const afterGraphemes = [...this.segment(after, "grapheme")];
const firstGrapheme = afterGraphemes[0]?.segment || "";
const restAfter = after.slice(firstGrapheme.length);
const cursor = `\x1b[7m${firstGrapheme}\x1b[0m`;
@@ -855,7 +860,7 @@ export class Editor implements Component, Focusable {
}
} else {
// Line needs wrapping - use word-aware wrapping
const chunks = wordWrapLine(line, contentWidth, [...this.segment(line)]);
const chunks = wordWrapLine(line, contentWidth, [...this.segment(line, "grapheme")]);
for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
const chunk = chunks[chunkIndex];
@@ -1213,7 +1218,7 @@ export class Editor implements Component, Focusable {
const beforeCursor = line.slice(0, this.state.cursorCol);
// Find the last grapheme in the text before cursor
const graphemes = [...this.segment(beforeCursor)];
const graphemes = [...this.segment(beforeCursor, "grapheme")];
const lastGrapheme = graphemes[graphemes.length - 1];
const graphemeLength = lastGrapheme ? lastGrapheme.segment.length : 1;
@@ -1314,7 +1319,7 @@ export class Editor implements Component, Focusable {
// Snap cursor to atomic segment boundary (e.g. paste markers)
// so the cursor never lands in the middle of a multi-grapheme unit.
// Single-grapheme segments don't need snapping.
const segments = [...this.segment(logicalLine)];
const segments = [...this.segment(logicalLine, "grapheme")];
for (const seg of segments) {
if (seg.index > this.state.cursorCol) break;
if (seg.segment.length <= 1) continue;
@@ -1585,7 +1590,7 @@ export class Editor implements Component, Focusable {
const afterCursor = currentLine.slice(this.state.cursorCol);
// Find the first grapheme at cursor
const graphemes = [...this.segment(afterCursor)];
const graphemes = [...this.segment(afterCursor, "grapheme")];
const firstGrapheme = graphemes[0];
const graphemeLength = firstGrapheme ? firstGrapheme.segment.length : 1;
@@ -1642,7 +1647,7 @@ export class Editor implements Component, Focusable {
visualLines.push({ logicalLine: i, startCol: 0, length: line.length });
} else {
// Line needs wrapping - use word-aware wrapping
const chunks = wordWrapLine(line, width, [...this.segment(line)]);
const chunks = wordWrapLine(line, width, [...this.segment(line, "grapheme")]);
for (const chunk of chunks) {
visualLines.push({
logicalLine: i,
@@ -1707,7 +1712,7 @@ export class Editor implements Component, Focusable {
// Moving right - move by one grapheme (handles emojis, combining characters, etc.)
if (this.state.cursorCol < currentLine.length) {
const afterCursor = currentLine.slice(this.state.cursorCol);
const graphemes = [...this.segment(afterCursor)];
const graphemes = [...this.segment(afterCursor, "grapheme")];
const firstGrapheme = graphemes[0];
this.setCursorCol(this.state.cursorCol + (firstGrapheme ? firstGrapheme.segment.length : 1));
} else if (this.state.cursorLine < this.state.lines.length - 1) {
@@ -1725,7 +1730,7 @@ export class Editor implements Component, Focusable {
// Moving left - move by one grapheme (handles emojis, combining characters, etc.)
if (this.state.cursorCol > 0) {
const beforeCursor = currentLine.slice(0, this.state.cursorCol);
const graphemes = [...this.segment(beforeCursor)];
const graphemes = [...this.segment(beforeCursor, "grapheme")];
const lastGrapheme = graphemes[graphemes.length - 1];
this.setCursorCol(this.state.cursorCol - (lastGrapheme ? lastGrapheme.segment.length : 1));
} else if (this.state.cursorLine > 0) {
@@ -1769,41 +1774,32 @@ export class Editor implements Component, Focusable {
}
const textBeforeCursor = currentLine.slice(0, this.state.cursorCol);
const graphemes = [...this.segment(textBeforeCursor)];
const segments = [...this.segment(textBeforeCursor, "word")];
let newCol = this.state.cursorCol;
// Skip trailing whitespace
while (
graphemes.length > 0 &&
!isPasteMarker(graphemes[graphemes.length - 1]?.segment || "") &&
isWhitespaceChar(graphemes[graphemes.length - 1]?.segment || "")
segments.length > 0 &&
!isPasteMarker(segments[segments.length - 1]?.segment || "") &&
isWhitespaceChar(segments[segments.length - 1]?.segment || "")
) {
newCol -= graphemes.pop()?.segment.length || 0;
newCol -= segments.pop()?.segment.length || 0;
}
if (graphemes.length > 0) {
const lastGrapheme = graphemes[graphemes.length - 1]?.segment || "";
if (isPasteMarker(lastGrapheme)) {
// Paste marker is a single atomic word
newCol -= graphemes.pop()?.segment.length || 0;
} else if (isPunctuationChar(lastGrapheme)) {
// Skip punctuation run
while (
graphemes.length > 0 &&
isPunctuationChar(graphemes[graphemes.length - 1]?.segment || "") &&
!isPasteMarker(graphemes[graphemes.length - 1]?.segment || "")
) {
newCol -= graphemes.pop()?.segment.length || 0;
}
if (segments.length > 0) {
const last = segments[segments.length - 1]!;
if (isPasteMarker(last.segment) || last.isWordLike) {
// Skip one word-like segment (or paste marker)
newCol -= segments.pop()?.segment.length || 0;
} else {
// Skip word run
// Skip non-word non-whitespace run (punctuation)
while (
graphemes.length > 0 &&
!isWhitespaceChar(graphemes[graphemes.length - 1]?.segment || "") &&
!isPunctuationChar(graphemes[graphemes.length - 1]?.segment || "") &&
!isPasteMarker(graphemes[graphemes.length - 1]?.segment || "")
segments.length > 0 &&
!isPasteMarker(segments[segments.length - 1]?.segment || "") &&
!segments[segments.length - 1]?.isWordLike &&
!isWhitespaceChar(segments[segments.length - 1]?.segment || "")
) {
newCol -= graphemes.pop()?.segment.length || 0;
newCol -= segments.pop()?.segment.length || 0;
}
}
}
@@ -1996,7 +1992,7 @@ export class Editor implements Component, Focusable {
}
const textAfterCursor = currentLine.slice(this.state.cursorCol);
const segments = this.segment(textAfterCursor);
const segments = this.segment(textAfterCursor, "word");
const iterator = segments[Symbol.iterator]();
let next = iterator.next();
let newCol = this.state.cursorCol;
@@ -2008,23 +2004,16 @@ export class Editor implements Component, Focusable {
}
if (!next.done) {
const firstGrapheme = next.value.segment;
if (isPasteMarker(firstGrapheme)) {
// Paste marker is a single atomic word
newCol += firstGrapheme.length;
} else if (isPunctuationChar(firstGrapheme)) {
// Skip punctuation run
while (!next.done && isPunctuationChar(next.value.segment) && !isPasteMarker(next.value.segment)) {
newCol += next.value.segment.length;
next = iterator.next();
}
if (isPasteMarker(next.value.segment) || next.value.isWordLike) {
// Skip one word-like segment (or paste marker)
newCol += next.value.segment.length;
} else {
// Skip word run
// Skip non-word non-whitespace run (punctuation)
while (
!next.done &&
!isWhitespaceChar(next.value.segment) &&
!isPunctuationChar(next.value.segment) &&
!isPasteMarker(next.value.segment)
!isPasteMarker(next.value.segment) &&
!next.value.isWordLike &&
!isWhitespaceChar(next.value.segment)
) {
newCol += next.value.segment.length;
next = iterator.next();