shumengya/sproutclaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(tui): leverage Intl.Segmenter for proper Unicode word boundaries (#5022)

Browse Source
This commit is contained in:
xu0o0
2026-05-27 06:28:47 +08:00
committed by GitHub
parent 8fb1e877cd
commit 4402100830
4 changed files with 141 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
packages/tui/src/components/editor.ts
View File

@@ -4,10 +4,11 @@ import { decodePrintableKey, matchesKey } from "../keys.ts";
import { KillRing } from "../kill-ring.ts";
import { type Component, CURSOR_MARKER, type Focusable, type TUI } from "../tui.ts";
import { UndoStack } from "../undo-stack.ts";
import { getSegmenter, isPunctuationChar, isWhitespaceChar, truncateToWidth, visibleWidth } from "../utils.ts";
import { getGraphemeSegmenter, getWordSegmenter, isWhitespaceChar, truncateToWidth, visibleWidth } from "../utils.ts";
import { SelectList, type SelectListLayoutOptions, type SelectListTheme } from "./select-list.ts";
const baseSegmenter = getSegmenter();
const graphemeSegmenter = getGraphemeSegmenter();
const wordSegmenter = getWordSegmenter();
/** Regex matching paste markers like `[paste #1 +123 lines]` or `[paste #2 1234 chars]`. */
const PASTE_MARKER_REGEX = /\[paste #(\d+)( (\+\d+ lines|\d+ chars))?\]/g;
@@ -27,7 +28,11 @@ function isPasteMarker(segment: string): boolean {
*
* Only markers whose numeric ID exists in `validIds` are merged.
*/
function segmentWithMarkers(text: string, validIds: Set<number>): Iterable<Intl.SegmentData> {
function segmentWithMarkers(
text: string,
baseSegmenter: Intl.Segmenter,
validIds: Set<number>,
): Iterable<Intl.SegmentData> {
// Fast path: no paste markers in the text or no valid IDs.
if (validIds.size === 0 || !text.includes("[paste #")) {
return baseSegmenter.segment(text);
@@ -109,7 +114,7 @@ export function wordWrapLine(line: string, maxWidth: number, preSegmented?: Intl
}
const chunks: TextChunk[] = [];
const segments = preSegmented ?? [...baseSegmenter.segment(line)];
const segments = preSegmented ?? [...graphemeSegmenter.segment(line)];
let currentWidth = 0;
let chunkStart = 0;
@@ -301,8 +306,8 @@ export class Editor implements Component, Focusable {
}
/** Segment text with paste-marker awareness, only merging markers with valid IDs. */
private segment(text: string): Iterable<Intl.SegmentData> {
return segmentWithMarkers(text, this.validPasteIds());
private segment(text: string, mode: "word" | "grapheme"): Iterable<Intl.SegmentData> {
return segmentWithMarkers(text, mode === "word" ? wordSegmenter : graphemeSegmenter, this.validPasteIds());
}
getPaddingX(): number {
@@ -482,7 +487,7 @@ export class Editor implements Component, Focusable {
if (after.length > 0) {
// Cursor is on a character (grapheme) - replace it with highlighted version
// Get the first grapheme from 'after'
const afterGraphemes = [...this.segment(after)];
const afterGraphemes = [...this.segment(after, "grapheme")];
const firstGrapheme = afterGraphemes[0]?.segment || "";
const restAfter = after.slice(firstGrapheme.length);
const cursor = `\x1b[7m${firstGrapheme}\x1b[0m`;
@@ -855,7 +860,7 @@ export class Editor implements Component, Focusable {
}
} else {
// Line needs wrapping - use word-aware wrapping
const chunks = wordWrapLine(line, contentWidth, [...this.segment(line)]);
const chunks = wordWrapLine(line, contentWidth, [...this.segment(line, "grapheme")]);
for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
const chunk = chunks[chunkIndex];
@@ -1213,7 +1218,7 @@ export class Editor implements Component, Focusable {
const beforeCursor = line.slice(0, this.state.cursorCol);
// Find the last grapheme in the text before cursor
const graphemes = [...this.segment(beforeCursor)];
const graphemes = [...this.segment(beforeCursor, "grapheme")];
const lastGrapheme = graphemes[graphemes.length - 1];
const graphemeLength = lastGrapheme ? lastGrapheme.segment.length : 1;
@@ -1314,7 +1319,7 @@ export class Editor implements Component, Focusable {
// Snap cursor to atomic segment boundary (e.g. paste markers)
// so the cursor never lands in the middle of a multi-grapheme unit.
// Single-grapheme segments don't need snapping.
const segments = [...this.segment(logicalLine)];
const segments = [...this.segment(logicalLine, "grapheme")];
for (const seg of segments) {
if (seg.index > this.state.cursorCol) break;
if (seg.segment.length <= 1) continue;
@@ -1585,7 +1590,7 @@ export class Editor implements Component, Focusable {
const afterCursor = currentLine.slice(this.state.cursorCol);
// Find the first grapheme at cursor
const graphemes = [...this.segment(afterCursor)];
const graphemes = [...this.segment(afterCursor, "grapheme")];
const firstGrapheme = graphemes[0];
const graphemeLength = firstGrapheme ? firstGrapheme.segment.length : 1;
@@ -1642,7 +1647,7 @@ export class Editor implements Component, Focusable {
visualLines.push({ logicalLine: i, startCol: 0, length: line.length });
} else {
// Line needs wrapping - use word-aware wrapping
const chunks = wordWrapLine(line, width, [...this.segment(line)]);
const chunks = wordWrapLine(line, width, [...this.segment(line, "grapheme")]);
for (const chunk of chunks) {
visualLines.push({
logicalLine: i,
@@ -1707,7 +1712,7 @@ export class Editor implements Component, Focusable {
// Moving right - move by one grapheme (handles emojis, combining characters, etc.)
if (this.state.cursorCol < currentLine.length) {
const afterCursor = currentLine.slice(this.state.cursorCol);
const graphemes = [...this.segment(afterCursor)];
const graphemes = [...this.segment(afterCursor, "grapheme")];
const firstGrapheme = graphemes[0];
this.setCursorCol(this.state.cursorCol + (firstGrapheme ? firstGrapheme.segment.length : 1));
} else if (this.state.cursorLine < this.state.lines.length - 1) {
@@ -1725,7 +1730,7 @@ export class Editor implements Component, Focusable {
// Moving left - move by one grapheme (handles emojis, combining characters, etc.)
if (this.state.cursorCol > 0) {
const beforeCursor = currentLine.slice(0, this.state.cursorCol);
const graphemes = [...this.segment(beforeCursor)];
const graphemes = [...this.segment(beforeCursor, "grapheme")];
const lastGrapheme = graphemes[graphemes.length - 1];
this.setCursorCol(this.state.cursorCol - (lastGrapheme ? lastGrapheme.segment.length : 1));
} else if (this.state.cursorLine > 0) {
@@ -1769,41 +1774,32 @@ export class Editor implements Component, Focusable {
}
const textBeforeCursor = currentLine.slice(0, this.state.cursorCol);
const graphemes = [...this.segment(textBeforeCursor)];
const segments = [...this.segment(textBeforeCursor, "word")];
let newCol = this.state.cursorCol;
// Skip trailing whitespace
while (
graphemes.length > 0 &&
!isPasteMarker(graphemes[graphemes.length - 1]?.segment || "") &&
isWhitespaceChar(graphemes[graphemes.length - 1]?.segment || "")
segments.length > 0 &&
!isPasteMarker(segments[segments.length - 1]?.segment || "") &&
isWhitespaceChar(segments[segments.length - 1]?.segment || "")
) {
newCol -= graphemes.pop()?.segment.length || 0;
newCol -= segments.pop()?.segment.length || 0;
}
if (graphemes.length > 0) {
const lastGrapheme = graphemes[graphemes.length - 1]?.segment || "";
if (isPasteMarker(lastGrapheme)) {
// Paste marker is a single atomic word
newCol -= graphemes.pop()?.segment.length || 0;
} else if (isPunctuationChar(lastGrapheme)) {
// Skip punctuation run
while (
graphemes.length > 0 &&
isPunctuationChar(graphemes[graphemes.length - 1]?.segment || "") &&
!isPasteMarker(graphemes[graphemes.length - 1]?.segment || "")
) {
newCol -= graphemes.pop()?.segment.length || 0;
}
if (segments.length > 0) {
const last = segments[segments.length - 1]!;
if (isPasteMarker(last.segment) || last.isWordLike) {
// Skip one word-like segment (or paste marker)
newCol -= segments.pop()?.segment.length || 0;
} else {
// Skip word run
// Skip non-word non-whitespace run (punctuation)
while (
graphemes.length > 0 &&
!isWhitespaceChar(graphemes[graphemes.length - 1]?.segment || "") &&
!isPunctuationChar(graphemes[graphemes.length - 1]?.segment || "") &&
!isPasteMarker(graphemes[graphemes.length - 1]?.segment || "")
segments.length > 0 &&
!isPasteMarker(segments[segments.length - 1]?.segment || "") &&
!segments[segments.length - 1]?.isWordLike &&
!isWhitespaceChar(segments[segments.length - 1]?.segment || "")
) {
newCol -= graphemes.pop()?.segment.length || 0;
newCol -= segments.pop()?.segment.length || 0;
}
}
}
@@ -1996,7 +1992,7 @@ export class Editor implements Component, Focusable {
}
const textAfterCursor = currentLine.slice(this.state.cursorCol);
const segments = this.segment(textAfterCursor);
const segments = this.segment(textAfterCursor, "word");
const iterator = segments[Symbol.iterator]();
let next = iterator.next();
let newCol = this.state.cursorCol;
@@ -2008,23 +2004,16 @@ export class Editor implements Component, Focusable {
}
if (!next.done) {
const firstGrapheme = next.value.segment;
if (isPasteMarker(firstGrapheme)) {
// Paste marker is a single atomic word
newCol += firstGrapheme.length;
} else if (isPunctuationChar(firstGrapheme)) {
// Skip punctuation run
while (!next.done && isPunctuationChar(next.value.segment) && !isPasteMarker(next.value.segment)) {
newCol += next.value.segment.length;
next = iterator.next();
}
if (isPasteMarker(next.value.segment) || next.value.isWordLike) {
// Skip one word-like segment (or paste marker)
newCol += next.value.segment.length;
} else {
// Skip word run
// Skip non-word non-whitespace run (punctuation)
while (
!next.done &&
!isWhitespaceChar(next.value.segment) &&
!isPunctuationChar(next.value.segment) &&
!isPasteMarker(next.value.segment)
!isPasteMarker(next.value.segment) &&
!next.value.isWordLike &&
!isWhitespaceChar(next.value.segment)
) {
newCol += next.value.segment.length;
next = iterator.next();

4
packages/tui/src/components/input.ts
View File

@@ -3,9 +3,9 @@ import { decodeKittyPrintable } from "../keys.ts";
import { KillRing } from "../kill-ring.ts";
import { type Component, CURSOR_MARKER, type Focusable } from "../tui.ts";
import { UndoStack } from "../undo-stack.ts";
import { getSegmenter, isPunctuationChar, isWhitespaceChar, sliceByColumn, visibleWidth } from "../utils.ts";
import { getGraphemeSegmenter, isPunctuationChar, isWhitespaceChar, sliceByColumn, visibleWidth } from "../utils.ts";
const segmenter = getSegmenter();
const segmenter = getGraphemeSegmenter();
interface InputState {
value: string;

32
packages/tui/src/utils.ts
View File

@@ -1,13 +1,21 @@
import { eastAsianWidth } from "get-east-asian-width";
// Grapheme segmenter (shared instance)
const segmenter = new Intl.Segmenter(undefined, { granularity: "grapheme" });
// segmenters (shared instance)
const graphemeSegmenter = new Intl.Segmenter(undefined, { granularity: "grapheme" });
const wordSegmenter = new Intl.Segmenter(undefined, { granularity: "word" });
/**
* Get the shared grapheme segmenter instance.
*/
export function getSegmenter(): Intl.Segmenter {
return segmenter;
export function getGraphemeSegmenter(): Intl.Segmenter {
return graphemeSegmenter;
}
/**
* Get the shared word segmenter instance.
*/
export function getWordSegmenter(): Intl.Segmenter {
return wordSegmenter;
}
/**
@@ -62,7 +70,7 @@ function truncateFragmentToWidth(text: string, maxWidth: number): { text: string
if (!hasAnsi && !hasTabs) {
let result = "";
let width = 0;
for (const { segment } of segmenter.segment(text)) {
for (const { segment } of graphemeSegmenter.segment(text)) {
const w = graphemeWidth(segment);
if (width + w > maxWidth) {
break;
@@ -109,7 +117,7 @@ function truncateFragmentToWidth(text: string, maxWidth: number): { text: string
end++;
}
for (const { segment } of segmenter.segment(text.slice(i, end))) {
for (const { segment } of graphemeSegmenter.segment(text.slice(i, end))) {
const w = graphemeWidth(segment);
if (width + w > maxWidth) {
return { text: result, width };
@@ -239,7 +247,7 @@ export function visibleWidth(str: string): number {
// Calculate width
let width = 0;
for (const { segment } of segmenter.segment(clean)) {
for (const { segment } of graphemeSegmenter.segment(clean)) {
width += graphemeWidth(segment);
}
@@ -790,7 +798,7 @@ function breakLongWord(word: string, width: number, tracker: AnsiCodeTracker): s
}
// Segment this non-ANSI portion into graphemes
const textPortion = word.slice(i, end);
for (const seg of segmenter.segment(textPortion)) {
for (const seg of graphemeSegmenter.segment(textPortion)) {
segments.push({ type: "grapheme", value: seg.segment });
}
i = end;
@@ -912,7 +920,7 @@ export function truncateToWidth(
const hasTabs = text.includes("\t");
if (!hasAnsi && !hasTabs) {
for (const { segment } of segmenter.segment(text)) {
for (const { segment } of graphemeSegmenter.segment(text)) {
const width = graphemeWidth(segment);
if (keepContiguousPrefix && keptWidth + width <= targetWidth) {
result += segment;
@@ -967,7 +975,7 @@ export function truncateToWidth(
end++;
}
for (const { segment } of segmenter.segment(text.slice(i, end))) {
for (const { segment } of graphemeSegmenter.segment(text.slice(i, end))) {
const width = graphemeWidth(segment);
if (keepContiguousPrefix && keptWidth + width <= targetWidth) {
if (pendingAnsi) {
@@ -1037,7 +1045,7 @@ export function sliceWithWidth(
let textEnd = i;
while (textEnd < line.length && !extractAnsiCode(line, textEnd)) textEnd++;
for (const { segment } of segmenter.segment(line.slice(i, textEnd))) {
for (const { segment } of graphemeSegmenter.segment(line.slice(i, textEnd))) {
const w = graphemeWidth(segment);
const inRange = currentCol >= startCol && currentCol < endCol;
const fits = !strict || currentCol + w <= endCol;
@@ -1105,7 +1113,7 @@ export function extractSegments(
let textEnd = i;
while (textEnd < line.length && !extractAnsiCode(line, textEnd)) textEnd++;
for (const { segment } of segmenter.segment(line.slice(i, textEnd))) {
for (const { segment } of graphemeSegmenter.segment(line.slice(i, textEnd))) {
const w = graphemeWidth(segment);
if (currentCol < beforeEnd) {

76
packages/tui/test/editor.test.ts
View File

@@ -591,6 +591,82 @@ describe("Editor component", () => {
editor.handleInput("\x1b[1;5C"); // Ctrl+Right
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 6 }); // after 'foo'
});
it("stops at fullwidth Chinese punctuation (issue #4972)", () => {
const editor = new Editor(createTestTUI(), defaultEditorTheme);
// 你好，世界 = 你好(0-2) ，(2-3) 世界(3-5)
editor.setText("你好，世界");
// Cursor at end (col 5)
// Move left over 世界
editor.handleInput("\x1b[1;5D"); // Ctrl+Left
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 3 }); // after ，
// Move left over ，
editor.handleInput("\x1b[1;5D"); // Ctrl+Left
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 2 }); // after 你好
// Move left over 你好
editor.handleInput("\x1b[1;5D"); // Ctrl+Left
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 0 }); // start
// Move right over 你好
editor.handleInput("\x1b[1;5C"); // Ctrl+Right
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 2 }); // after 你好
// Move right over ，
editor.handleInput("\x1b[1;5C"); // Ctrl+Right
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 3 }); // after ，
// Move right over 世界
editor.handleInput("\x1b[1;5C"); // Ctrl+Right
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 5 }); // end
});
it("handles mixed CJK and ASCII word movement", () => {
const editor = new Editor(createTestTUI(), defaultEditorTheme);
// "hello你好，world世界" = hello(0-5) 你好(5-7) ，(7-8) world(8-13) 世界(13-15)
editor.setText("hello你好，world世界");
// Cursor at end (col 15)
// Move left over 世界
editor.handleInput("\x1b[1;5D"); // Ctrl+Left
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 13 }); // after 'world'
// Move left over world
editor.handleInput("\x1b[1;5D"); // Ctrl+Left
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 8 }); // after ，
// Move left over ，
editor.handleInput("\x1b[1;5D"); // Ctrl+Left
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 7 }); // after 你好
// Move left over 你好
editor.handleInput("\x1b[1;5D"); // Ctrl+Left
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 5 }); // after 'hello'
// Move left over hello
editor.handleInput("\x1b[1;5D"); // Ctrl+Left
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 0 }); // start
// Forward from start
editor.handleInput("\x1b[1;5C"); // Ctrl+Right
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 5 }); // after 'hello'
editor.handleInput("\x1b[1;5C"); // Ctrl+Right
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 7 }); // after 你好
editor.handleInput("\x1b[1;5C"); // Ctrl+Right
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 8 }); // after ，
editor.handleInput("\x1b[1;5C"); // Ctrl+Right
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 13 }); // after 'world'
editor.handleInput("\x1b[1;5C"); // Ctrl+Right
assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 15 }); // end
});
});
describe("Grapheme-aware text wrapping", () => {