fix(tui): preserve ASCII punctuation word boundaries with Intl.Segmenter (#5067)

packages/tui/src/components/editor.ts

@@ -4,7 +4,14 @@ import { decodePrintableKey, matchesKey } from "../keys.ts";
 import { KillRing } from "../kill-ring.ts";
 import { type Component, CURSOR_MARKER, type Focusable, type TUI } from "../tui.ts";
 import { UndoStack } from "../undo-stack.ts";
-import { getGraphemeSegmenter, getWordSegmenter, isWhitespaceChar, truncateToWidth, visibleWidth } from "../utils.ts";
+import {
+	getGraphemeSegmenter,
+	getWordSegmenter,
+	isWhitespaceChar,
+	PUNCTUATION_REGEX,
+	truncateToWidth,
+	visibleWidth,
+} from "../utils.ts";
 import { SelectList, type SelectListLayoutOptions, type SelectListTheme } from "./select-list.ts";
 
 const graphemeSegmenter = getGraphemeSegmenter();
@@ -1788,9 +1795,19 @@ export class Editor implements Component, Focusable {
 
 		if (segments.length > 0) {
 			const last = segments[segments.length - 1]!;
-			if (isPasteMarker(last.segment) || last.isWordLike) {
-				// Skip one word-like segment (or paste marker)
+			if (isPasteMarker(last.segment)) {
+				// Skip one paste marker.
 				newCol -= segments.pop()?.segment.length || 0;
+			} else if (last.isWordLike) {
+				// Skip inside one word-like segment, preserving existing ASCII punctuation boundaries.
+				const segment = last.segment;
+				const matches = [...segment.matchAll(new RegExp(PUNCTUATION_REGEX, "g"))];
+				if (matches.length <= 0) {
+					newCol -= segment.length;
+				} else {
+					const lastMatch = matches[matches.length - 1]!;
+					newCol -= segment.length - (lastMatch.index + lastMatch[0].length);
+				}
 			} else {
 				// Skip non-word non-whitespace run (punctuation)
 				while (
@@ -2004,9 +2021,12 @@ export class Editor implements Component, Focusable {
 		}
 
 		if (!next.done) {
-			if (isPasteMarker(next.value.segment) || next.value.isWordLike) {
-				// Skip one word-like segment (or paste marker)
+			if (isPasteMarker(next.value.segment)) {
+				// Skip one paste marker.
 				newCol += next.value.segment.length;
+			} else if (next.value.isWordLike) {
+				// Skip inside one word-like segment, preserving existing ASCII punctuation boundaries.
+				newCol += PUNCTUATION_REGEX.exec(next.value.segment)?.index ?? next.value.segment.length;
 			} else {
 				// Skip non-word non-whitespace run (punctuation)
 				while (

packages/tui/src/utils.ts

@@ -757,7 +757,7 @@ function wrapSingleLine(line: string, width: number): string[] {
 	return wrapped.length > 0 ? wrapped.map((line) => line.trimEnd()) : [""];
 }
 
-const PUNCTUATION_REGEX = /[(){}[\]<>.,;:'"!?+\-=*/\\|&%^$#@~`]/;
+export const PUNCTUATION_REGEX = /[(){}[\]<>.,;:'"!?+\-=*/\\|&%^$#@~`]/;
 
 /**
  * Check if a character is whitespace.

packages/tui/test/editor.test.ts

@@ -532,6 +532,15 @@ describe("Editor component", () => {
 			editor.handleInput("\x17");
 			assert.strictEqual(editor.getText(), "foo bar");
 
+			// ASCII punctuation inside Intl word-like segments preserves old boundaries
+			editor.setText("foo.bar");
+			editor.handleInput("\x17");
+			assert.strictEqual(editor.getText(), "foo.");
+
+			editor.setText("foo:bar");
+			editor.handleInput("\x17");
+			assert.strictEqual(editor.getText(), "foo:");
+
 			// Delete across multiple lines
 			editor.setText("line one\nline two");
 			editor.handleInput("\x17");
@@ -590,6 +599,23 @@ describe("Editor component", () => {
 			editor.handleInput("\x01"); // Ctrl+A to go to start
 			editor.handleInput("\x1b[1;5C"); // Ctrl+Right
 			assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 6 }); // after 'foo'
+
+			// ASCII punctuation inside Intl word-like segments preserves old boundaries
+			editor.setText("foo.bar baz");
+			editor.handleInput("\x1b[1;5D"); // Ctrl+Left over baz
+			assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 8 });
+			editor.handleInput("\x1b[1;5D"); // Ctrl+Left over bar
+			assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 4 });
+			editor.handleInput("\x1b[1;5D"); // Ctrl+Left over .
+			assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 3 });
+
+			editor.handleInput("\x01"); // Ctrl+A
+			editor.handleInput("\x1b[1;5C"); // Ctrl+Right over foo
+			assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 3 });
+			editor.handleInput("\x1b[1;5C"); // Ctrl+Right over .
+			assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 4 });
+			editor.handleInput("\x1b[1;5C"); // Ctrl+Right over bar
+			assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 7 });
 		});
 
 		it("stops at fullwidth Chinese punctuation (issue #4972)", () => {

packages/tui/test/input.test.ts

@@ -100,6 +100,20 @@ describe("Input component", () => {
 			assert.strictEqual(input.getValue(), "bazfoo bar ");
 		});
 
+		it("Ctrl+W preserves ASCII punctuation boundaries", () => {
+			const input = new Input();
+
+			input.setValue("foo.bar");
+			input.handleInput("\x05"); // Ctrl+E
+			input.handleInput("\x17"); // Ctrl+W - deletes "bar"
+			assert.strictEqual(input.getValue(), "foo.");
+
+			input.setValue("foo:bar");
+			input.handleInput("\x05"); // Ctrl+E
+			input.handleInput("\x17"); // Ctrl+W - deletes "bar"
+			assert.strictEqual(input.getValue(), "foo:");
+		});
+
 		it("Ctrl+U saves deleted text to kill ring", () => {
 			const input = new Input();