Merge pull request #4261 from badlogic/kitty-image-ids

fix(tui): keep kitty image redraws inside TUI

packages/tui/src/components/image.ts

@@ -1,4 +1,5 @@
 import {
+	allocateImageId,
 	getCapabilities,
 	getImageDimensions,
 	type ImageDimensions,
@@ -66,9 +67,13 @@ export class Image implements Component {
 		let lines: string[];
 
 		if (caps.images) {
+			if (caps.images === "kitty" && this.imageId === undefined) {
+				this.imageId = allocateImageId();
+			}
 			const result = renderImage(this.base64Data, this.dimensions, {
 				maxWidthCells: maxWidth,
 				imageId: this.imageId,
+				moveCursor: false,
 			});
 
 			if (result) {
@@ -77,16 +82,19 @@ export class Image implements Component {
 					this.imageId = result.imageId;
 				}
 
-				// Return `rows` lines so TUI accounts for image height
-				// First (rows-1) lines are empty (TUI clears them)
-				// Last line: move cursor back up, then output image sequence
+				// Return `rows` lines so TUI accounts for image height.
+				// First (rows-1) lines are empty and cleared before the image is drawn.
+				// Last line: move cursor back up, draw the image, then move back down
+				// for Kitty (this component disables Kitty's terminal-side cursor movement)
+				// so TUI cursor accounting stays inside the scroll area.
 				lines = [];
 				for (let i = 0; i < result.rows - 1; i++) {
 					lines.push("");
 				}
-				// Move cursor up to first row, then output image
-				const moveUp = result.rows > 1 ? `\x1b[${result.rows - 1}A` : "";
-				lines.push(moveUp + result.sequence);
+				const rowOffset = result.rows - 1;
+				const moveUp = rowOffset > 0 ? `\x1b[${rowOffset}A` : "";
+				const moveDown = caps.images === "kitty" && rowOffset > 0 ? `\x1b[${rowOffset}B` : "";
+				lines.push(moveUp + result.sequence + moveDown);
 			} else {
 				const fallback = imageFallback(this.mimeType, this.dimensions, this.options.filename);
 				lines = [this.theme.fallbackColor(fallback)];

packages/tui/src/terminal-image.ts

@@ -22,6 +22,8 @@ export interface ImageRenderOptions {
 	preserveAspectRatio?: boolean;
 	/** Kitty image ID. If provided, reuses/replaces existing image with this ID. */
 	imageId?: number;
+	/** Whether Kitty should apply its default cursor movement after placement. */
+	moveCursor?: boolean;
 }
 
 let cachedCapabilities: TerminalCapabilities | null = null;
@@ -46,10 +48,7 @@ export function detectCapabilities(): TerminalCapabilities {
 	// sequences differently). Force hyperlinks off whenever we detect them, even
 	// when the outer terminal would otherwise support OSC 8. Image protocols are
 	// also unreliable under tmux/screen, so leave `images: null` for safety.
-	// cmux currently also gets this conservative fallback due to image corruption:
-	// https://github.com/badlogic/pi-mono/issues/4208
-	const inTmuxOrScreen =
-		!!process.env.TMUX || !!process.env.CMUX_WORKSPACE_ID || term.startsWith("tmux") || term.startsWith("screen");
+	const inTmuxOrScreen = !!process.env.TMUX || term.startsWith("tmux") || term.startsWith("screen");
 	if (inTmuxOrScreen) {
 		const trueColor = colorTerm === "truecolor" || colorTerm === "24bit";
 		return { images: null, trueColor, hyperlinks: false };
@@ -131,12 +130,15 @@ export function encodeKitty(
 		columns?: number;
 		rows?: number;
 		imageId?: number;
+		/** Whether Kitty should apply its default cursor movement after placement. Default: true. */
+		moveCursor?: boolean;
 	} = {},
 ): string {
 	const CHUNK_SIZE = 4096;
 
 	const params: string[] = ["a=T", "f=100", "q=2"];
 
+	if (options.moveCursor === false) params.push("C=1");
 	if (options.columns) params.push(`c=${options.columns}`);
 	if (options.rows) params.push(`r=${options.rows}`);
 	if (options.imageId) params.push(`i=${options.imageId}`);
@@ -173,7 +175,7 @@ export function encodeKitty(
  * Uses uppercase 'I' to also free the image data.
  */
 export function deleteKittyImage(imageId: number): string {
-	return `\x1b_Ga=d,d=I,i=${imageId}\x1b\\`;
+	return `\x1b_Ga=d,d=I,i=${imageId},q=2\x1b\\`;
 }
 
 /**
@@ -181,7 +183,7 @@ export function deleteKittyImage(imageId: number): string {
  * Uses uppercase 'A' to also free the image data.
  */
 export function deleteAllKittyImages(): string {
-	return `\x1b_Ga=d,d=A\x1b\\`;
+	return "\x1b_Ga=d,d=A,q=2\x1b\\";
 }
 
 export function encodeITerm2(
@@ -377,8 +379,12 @@ export function renderImage(
 	const rows = calculateImageRows(imageDimensions, maxWidth, getCellDimensions());
 
 	if (caps.images === "kitty") {
-		// Only use imageId if explicitly provided - static images don't need IDs
-		const sequence = encodeKitty(base64Data, { columns: maxWidth, rows, imageId: options.imageId });
+		const sequence = encodeKitty(base64Data, {
+			columns: maxWidth,
+			rows,
+			imageId: options.imageId,
+			moveCursor: options.moveCursor,
+		});
 		return { sequence, rows, imageId: options.imageId };
 	}
 

packages/tui/src/tui.ts

@@ -8,9 +8,26 @@ import * as path from "node:path";
 import { performance } from "node:perf_hooks";
 import { isKeyRelease, matchesKey } from "./keys.js";
 import type { Terminal } from "./terminal.js";
-import { getCapabilities, isImageLine, setCellDimensions } from "./terminal-image.js";
+import { deleteKittyImage, getCapabilities, isImageLine, setCellDimensions } from "./terminal-image.js";
 import { extractSegments, normalizeTerminalOutput, sliceByColumn, sliceWithWidth, visibleWidth } from "./utils.js";
 
+const KITTY_SEQUENCE_PATTERN = /\x1b_G([^;\x1b]*);/g;
+
+function extractKittyImageIds(line: string): number[] {
+	const ids: number[] = [];
+	for (const match of line.matchAll(KITTY_SEQUENCE_PATTERN)) {
+		for (const param of match[1].split(",")) {
+			const [key, value] = param.split("=", 2);
+			if (key !== "i" || value === undefined) continue;
+			const id = Number(value);
+			if (Number.isInteger(id) && id > 0 && id <= 0xffffffff && !ids.includes(id)) {
+				ids.push(id);
+			}
+		}
+	}
+	return ids;
+}
+
 /**
  * Component interface - all components must implement this
  */
@@ -217,6 +234,7 @@ export class Container implements Component {
 export class TUI extends Container {
 	public terminal: Terminal;
 	private previousLines: string[] = [];
+	private previousKittyImageIds = new Set<number>();
 	private previousWidth = 0;
 	private previousHeight = 0;
 	private focusedComponent: Component | null = null;
@@ -806,6 +824,48 @@ export class TUI extends Container {
 		return lines;
 	}
 
+	private collectKittyImageIds(lines: string[]): Set<number> {
+		const ids = new Set<number>();
+		for (const line of lines) {
+			for (const id of extractKittyImageIds(line)) {
+				ids.add(id);
+			}
+		}
+		return ids;
+	}
+
+	private deleteKittyImages(ids: Iterable<number>): string {
+		let buffer = "";
+		for (const id of ids) {
+			buffer += deleteKittyImage(id);
+		}
+		return buffer;
+	}
+
+	private expandLastChangedForKittyImages(firstChanged: number, lastChanged: number): number {
+		let expandedLastChanged = lastChanged;
+		for (let i = firstChanged; i < this.previousLines.length; i++) {
+			if (extractKittyImageIds(this.previousLines[i]).length > 0) {
+				expandedLastChanged = Math.max(expandedLastChanged, i);
+			}
+		}
+		return expandedLastChanged;
+	}
+
+	private deleteChangedKittyImages(firstChanged: number, lastChanged: number): string {
+		if (firstChanged < 0 || lastChanged < firstChanged) return "";
+
+		const ids = new Set<number>();
+		const maxLine = Math.min(lastChanged, this.previousLines.length - 1);
+		for (let i = firstChanged; i <= maxLine; i++) {
+			for (const id of extractKittyImageIds(this.previousLines[i] ?? "")) {
+				ids.add(id);
+			}
+		}
+
+		return this.deleteKittyImages(ids);
+	}
+
 	/** Splice overlay content into a base line at a specific column. Single-pass optimized. */
 	private compositeLineAt(
 		baseLine: string,
@@ -918,7 +978,10 @@ export class TUI extends Container {
 		const fullRender = (clear: boolean): void => {
 			this.fullRedrawCount += 1;
 			let buffer = "\x1b[?2026h"; // Begin synchronized output
-			if (clear) buffer += "\x1b[2J\x1b[H\x1b[3J"; // Clear screen, home, then clear scrollback
+			if (clear) {
+				buffer += this.deleteKittyImages(this.previousKittyImageIds);
+				buffer += "\x1b[2J\x1b[H\x1b[3J"; // Clear screen, home, then clear scrollback
+			}
 			for (let i = 0; i < newLines.length; i++) {
 				if (i > 0) buffer += "\r\n";
 				buffer += newLines[i];
@@ -937,6 +1000,7 @@ export class TUI extends Container {
 			this.previousViewportTop = Math.max(0, bufferLength - height);
 			this.positionHardwareCursor(cursorPos, newLines.length);
 			this.previousLines = newLines;
+			this.previousKittyImageIds = this.collectKittyImageIds(newLines);
 			this.previousWidth = width;
 			this.previousHeight = height;
 		};
@@ -1003,6 +1067,9 @@ export class TUI extends Container {
 			}
 			lastChanged = newLines.length - 1;
 		}
+		if (firstChanged !== -1) {
+			lastChanged = this.expandLastChangedForKittyImages(firstChanged, lastChanged);
+		}
 		const appendStart = appendedLines && firstChanged === this.previousLines.length && firstChanged > 0;
 
 		// No changes - but still need to update hardware cursor position if it moved
@@ -1017,6 +1084,7 @@ export class TUI extends Container {
 		if (firstChanged >= newLines.length) {
 			if (this.previousLines.length > newLines.length) {
 				let buffer = "\x1b[?2026h";
+				buffer += this.deleteChangedKittyImages(firstChanged, lastChanged);
 				// Move to end of new content (clamp to 0 for empty content)
 				const targetRow = Math.max(0, newLines.length - 1);
 				if (targetRow < prevViewportTop) {
@@ -1052,6 +1120,7 @@ export class TUI extends Container {
 			}
 			this.positionHardwareCursor(cursorPos, newLines.length);
 			this.previousLines = newLines;
+			this.previousKittyImageIds = this.collectKittyImageIds(newLines);
 			this.previousWidth = width;
 			this.previousHeight = height;
 			this.previousViewportTop = prevViewportTop;
@@ -1069,6 +1138,7 @@ export class TUI extends Container {
 		// Render from first changed line to end
 		// Build buffer with all updates wrapped in synchronized output
 		let buffer = "\x1b[?2026h"; // Begin synchronized output
+		buffer += this.deleteChangedKittyImages(firstChanged, lastChanged);
 		const prevViewportBottom = prevViewportTop + height - 1;
 		const moveTargetRow = appendStart ? firstChanged - 1 : firstChanged;
 		if (moveTargetRow > prevViewportBottom) {
@@ -1199,6 +1269,7 @@ export class TUI extends Container {
 		this.positionHardwareCursor(cursorPos, newLines.length);
 
 		this.previousLines = newLines;
+		this.previousKittyImageIds = this.collectKittyImageIds(newLines);
 		this.previousWidth = width;
 		this.previousHeight = height;
 	}

packages/tui/test/terminal-image.test.ts

@@ -4,7 +4,19 @@
 
 import assert from "node:assert";
 import { describe, it } from "node:test";
-import { detectCapabilities, hyperlink, isImageLine } from "../src/terminal-image.js";
+import { Image } from "../src/components/image.js";
+import {
+	deleteAllKittyImages,
+	deleteKittyImage,
+	detectCapabilities,
+	encodeKitty,
+	hyperlink,
+	isImageLine,
+	renderImage,
+	resetCapabilitiesCache,
+	setCapabilities,
+	setCellDimensions,
+} from "../src/terminal-image.js";
 
 const ENV_KEYS = [
 	"TERM",
@@ -15,6 +27,7 @@ const ENV_KEYS = [
 	"GHOSTTY_RESOURCES_DIR",
 	"WEZTERM_PANE",
 	"ITERM_SESSION_ID",
+	"CMUX_WORKSPACE_ID",
 ] as const;
 
 function withEnv(overrides: Record<string, string | undefined>, fn: () => void): void {
@@ -223,6 +236,14 @@ describe("detectCapabilities", () => {
 		});
 	});
 
+	it("does not disable Ghostty images solely because cmux is present", () => {
+		withEnv({ TERM_PROGRAM: "ghostty", CMUX_WORKSPACE_ID: "workspace" }, () => {
+			const caps = detectCapabilities();
+			assert.strictEqual(caps.images, "kitty");
+			assert.strictEqual(caps.hyperlinks, true);
+		});
+	});
+
 	it("enables hyperlinks for Kitty", () => {
 		withEnv({ KITTY_WINDOW_ID: "1" }, () => {
 			const caps = detectCapabilities();
@@ -252,6 +273,71 @@ describe("detectCapabilities", () => {
 	});
 });
 
+describe("Kitty image cursor movement", () => {
+	it("can request no terminal-side cursor movement", () => {
+		const sequence = encodeKitty("AAAA", { columns: 2, rows: 2, moveCursor: false });
+		assert.ok(sequence.startsWith("\x1b_Ga=T,f=100,q=2,C=1,c=2,r=2;"));
+	});
+
+	it("suppresses Kitty replies for delete commands", () => {
+		assert.strictEqual(deleteKittyImage(42), "\x1b_Ga=d,d=I,i=42,q=2\x1b\\");
+		assert.strictEqual(deleteAllKittyImages(), "\x1b_Ga=d,d=A,q=2\x1b\\");
+	});
+
+	it("preserves renderImage's default terminal-side cursor movement", () => {
+		setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });
+		setCellDimensions({ widthPx: 10, heightPx: 10 });
+		try {
+			const result = renderImage("AAAA", { widthPx: 20, heightPx: 20 }, { maxWidthCells: 2 });
+			assert.ok(result);
+			assert.ok(!result.sequence.includes(",C=1,"));
+			assert.strictEqual(result.rows, 2);
+		} finally {
+			resetCapabilitiesCache();
+			setCellDimensions({ widthPx: 9, heightPx: 18 });
+		}
+	});
+
+	it("can opt renderImage into no terminal-side cursor movement", () => {
+		setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });
+		setCellDimensions({ widthPx: 10, heightPx: 10 });
+		try {
+			const result = renderImage("AAAA", { widthPx: 20, heightPx: 20 }, { maxWidthCells: 2, moveCursor: false });
+			assert.ok(result);
+			assert.ok(result.sequence.includes(",C=1,"));
+			assert.strictEqual(result.rows, 2);
+		} finally {
+			resetCapabilitiesCache();
+			setCellDimensions({ widthPx: 9, heightPx: 18 });
+		}
+	});
+
+	it("restores the cursor to the reserved image row after Kitty rendering", () => {
+		setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });
+		setCellDimensions({ widthPx: 10, heightPx: 10 });
+		try {
+			const image = new Image(
+				"AAAA",
+				"image/png",
+				{ fallbackColor: (value) => value },
+				{ maxWidthCells: 2 },
+				{ widthPx: 20, heightPx: 20 },
+			);
+			const lines = image.render(4);
+			const imageId = image.getImageId();
+			assert.strictEqual(typeof imageId, "number");
+			assert.deepStrictEqual(lines.slice(0, -1), [""]);
+			assert.ok(lines[1].startsWith("\x1b[1A\x1b_G"));
+			assert.ok(lines[1].includes(",C=1,"));
+			assert.ok(lines[1].includes(`,i=${imageId}`));
+			assert.ok(lines[1].endsWith("\x1b[1B"));
+		} finally {
+			resetCapabilitiesCache();
+			setCellDimensions({ widthPx: 9, heightPx: 18 });
+		}
+	});
+});
+
 describe("hyperlink", () => {
 	it("wraps text in OSC 8 open and close sequences", () => {
 		const result = hyperlink("click me", "https://example.com");

packages/tui/test/tui-render.test.ts

@@ -1,6 +1,7 @@
 import assert from "node:assert";
 import { describe, it } from "node:test";
 import type { Terminal as XtermTerminalType } from "@xterm/headless";
+import { deleteKittyImage, encodeKitty } from "../src/terminal-image.js";
 import { type Component, TUI } from "../src/tui.js";
 import { VirtualTerminal } from "./virtual-terminal.js";
 
@@ -63,6 +64,87 @@ function getCellItalic(terminal: VirtualTerminal, row: number, col: number): num
 	return cell.isItalic();
 }
 
+describe("TUI Kitty image cleanup", () => {
+	it("deletes changed image ids before drawing moved placements", async () => {
+		const terminal = new LoggingVirtualTerminal(40, 10);
+		const tui = new TUI(terminal);
+		const component = new TestComponent();
+		tui.addChild(component);
+
+		const oldImage = encodeKitty("AAAA", { columns: 2, rows: 2, imageId: 42, moveCursor: false });
+		component.lines = ["top", oldImage];
+		tui.start();
+		await terminal.waitForRender();
+		terminal.clearWrites();
+
+		const newImage = encodeKitty("BBBB", { columns: 2, rows: 1, imageId: 42, moveCursor: false });
+		component.lines = [newImage, ""];
+		tui.requestRender();
+		await terminal.waitForRender();
+
+		const writes = terminal.getWrites();
+		const deleteIndex = writes.indexOf(deleteKittyImage(42));
+		const drawIndex = writes.indexOf(newImage);
+		assert.ok(deleteIndex >= 0, "changed old image should be deleted");
+		assert.ok(drawIndex >= 0, "new image should be drawn");
+		assert.ok(deleteIndex < drawIndex, "old image must be deleted before the new placement is drawn");
+
+		tui.stop();
+	});
+
+	it("redraws image lines when an earlier reserved image row changes", async () => {
+		const terminal = new LoggingVirtualTerminal(40, 10);
+		const tui = new TUI(terminal);
+		const component = new TestComponent();
+		tui.addChild(component);
+
+		const image = encodeKitty("AAAA", { columns: 2, rows: 2, imageId: 88, moveCursor: false });
+		component.lines = ["", image];
+		tui.start();
+		await terminal.waitForRender();
+		terminal.clearWrites();
+
+		component.lines = ["covered", image];
+		tui.requestRender();
+		await terminal.waitForRender();
+
+		const writes = terminal.getWrites();
+		const deleteIndex = writes.indexOf(deleteKittyImage(88));
+		const drawIndex = writes.indexOf(image);
+		assert.ok(deleteIndex >= 0, "image should be deleted when a reserved row changes");
+		assert.ok(drawIndex >= 0, "unchanged image line should be redrawn after deleting the placement");
+		assert.ok(deleteIndex < drawIndex, "old placement must be deleted before the image line is redrawn");
+		assert.ok(!writes.includes("\x1b[2J"), "reserved row changes should not force a full redraw");
+
+		tui.stop();
+	});
+
+	it("deletes previously rendered image ids during full redraws", async () => {
+		const terminal = new LoggingVirtualTerminal(40, 10);
+		const tui = new TUI(terminal);
+		const component = new TestComponent();
+		tui.addChild(component);
+
+		component.lines = [encodeKitty("AAAA", { columns: 2, rows: 2, imageId: 77, moveCursor: false })];
+		tui.start();
+		await terminal.waitForRender();
+		terminal.clearWrites();
+
+		component.lines = ["plain text"];
+		tui.requestRender(true);
+		await terminal.waitForRender();
+
+		const writes = terminal.getWrites();
+		const deleteIndex = writes.indexOf(deleteKittyImage(77));
+		const clearIndex = writes.indexOf("\x1b[2J");
+		assert.ok(deleteIndex >= 0, "previous image should be deleted during full redraw");
+		assert.ok(clearIndex >= 0, "full redraw should clear the screen");
+		assert.ok(deleteIndex < clearIndex, "old image should be deleted before the screen is cleared");
+
+		tui.stop();
+	});
+});
+
 describe("TUI resize handling", () => {
 	it("triggers full re-render when terminal height changes", async () => {
 		await withEnv({ TERMUX_VERSION: undefined }, async () => {