From 6b271842e27aa00d70003877ae58a197b664681e Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Thu, 7 May 2026 00:12:19 +0200 Subject: [PATCH 1/5] fix(ai): handle mixed chat completion deltas Fixes #4228 --- .../ai/src/providers/openai-completions.ts | 256 +++++++++--------- .../openai-completions-tool-choice.test.ts | 232 ++++++++++++++++ 2 files changed, 367 insertions(+), 121 deletions(-) diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index b4e19e5f..4f001514 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -160,45 +160,104 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenA partialArgs?: string; streamIndex?: number; } + type StreamingBlock = TextContent | ThinkingContent | StreamingToolCallBlock; + type StreamingToolCallDelta = NonNullable[number]; - let currentBlock: TextContent | ThinkingContent | StreamingToolCallBlock | null = null; - const blocks = output.content; - const getContentIndex = (block: typeof currentBlock) => (block ? blocks.indexOf(block) : -1); - const currentContentIndex = () => getContentIndex(currentBlock); - const finishCurrentBlock = (block?: typeof currentBlock) => { - if (block) { - const contentIndex = getContentIndex(block); - if (contentIndex === -1) { - return; - } - if (block.type === "text") { - stream.push({ - type: "text_end", - contentIndex, - content: block.text, - partial: output, - }); - } else if (block.type === "thinking") { - stream.push({ - type: "thinking_end", - contentIndex, - content: block.thinking, - partial: output, - }); - } else if (block.type === "toolCall") { - block.arguments = parseStreamingJson(block.partialArgs); - // Finalize in-place and strip the scratch buffers so replay only - // carries parsed arguments. - delete block.partialArgs; - delete block.streamIndex; - stream.push({ - type: "toolcall_end", - contentIndex, - toolCall: block, - partial: output, - }); - } + let textBlock: TextContent | null = null; + let thinkingBlock: ThinkingContent | null = null; + const toolCallBlocksByIndex = new Map(); + const toolCallBlocksById = new Map(); + const blocks = output.content as StreamingBlock[]; + const getContentIndex = (block: StreamingBlock) => blocks.indexOf(block); + const finishBlock = (block: StreamingBlock) => { + const contentIndex = getContentIndex(block); + if (contentIndex === -1) { + return; } + if (block.type === "text") { + stream.push({ + type: "text_end", + contentIndex, + content: block.text, + partial: output, + }); + } else if (block.type === "thinking") { + stream.push({ + type: "thinking_end", + contentIndex, + content: block.thinking, + partial: output, + }); + } else if (block.type === "toolCall") { + block.arguments = parseStreamingJson(block.partialArgs); + // Finalize in-place and strip the scratch buffers so replay only + // carries parsed arguments. + delete block.partialArgs; + delete block.streamIndex; + stream.push({ + type: "toolcall_end", + contentIndex, + toolCall: block, + partial: output, + }); + } + }; + const ensureTextBlock = () => { + if (!textBlock) { + textBlock = { type: "text", text: "" }; + blocks.push(textBlock); + stream.push({ type: "text_start", contentIndex: getContentIndex(textBlock), partial: output }); + } + return textBlock; + }; + const ensureThinkingBlock = (thinkingSignature: string) => { + if (!thinkingBlock) { + thinkingBlock = { + type: "thinking", + thinking: "", + thinkingSignature, + }; + blocks.push(thinkingBlock); + stream.push({ type: "thinking_start", contentIndex: getContentIndex(thinkingBlock), partial: output }); + } + return thinkingBlock; + }; + const ensureToolCallBlock = (toolCall: StreamingToolCallDelta) => { + const streamIndex = typeof toolCall.index === "number" ? toolCall.index : undefined; + let block = streamIndex !== undefined ? toolCallBlocksByIndex.get(streamIndex) : undefined; + if (!block && toolCall.id) { + block = toolCallBlocksById.get(toolCall.id); + } + if (!block) { + block = { + type: "toolCall", + id: toolCall.id || "", + name: toolCall.function?.name || "", + arguments: {}, + partialArgs: "", + streamIndex, + }; + if (streamIndex !== undefined) { + toolCallBlocksByIndex.set(streamIndex, block); + } + if (toolCall.id) { + toolCallBlocksById.set(toolCall.id, block); + } + blocks.push(block); + stream.push({ + type: "toolcall_start", + contentIndex: getContentIndex(block), + partial: output, + }); + } + if (streamIndex !== undefined && block.streamIndex === undefined) { + block.streamIndex = streamIndex; + toolCallBlocksByIndex.set(streamIndex, block); + } + if (toolCall.id) { + toolCallBlocksById.set(toolCall.id, block); + } + return block; }; for await (const chunk of openaiStream) { @@ -237,22 +296,14 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenA choice.delta.content !== undefined && choice.delta.content.length > 0 ) { - if (!currentBlock || currentBlock.type !== "text") { - finishCurrentBlock(currentBlock); - currentBlock = { type: "text", text: "" }; - output.content.push(currentBlock); - stream.push({ type: "text_start", contentIndex: currentContentIndex(), partial: output }); - } - - if (currentBlock.type === "text") { - currentBlock.text += choice.delta.content; - stream.push({ - type: "text_delta", - contentIndex: currentContentIndex(), - delta: choice.delta.content, - partial: output, - }); - } + const block = ensureTextBlock(); + block.text += choice.delta.content; + stream.push({ + type: "text_delta", + contentIndex: getContentIndex(block), + delta: choice.delta.content, + partial: output, + }); } // Some endpoints return reasoning in reasoning_content (llama.cpp), @@ -260,38 +311,24 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenA // Use the first non-empty reasoning field to avoid duplication // (e.g., chutes.ai returns both reasoning_content and reasoning with same content) const reasoningFields = ["reasoning_content", "reasoning", "reasoning_text"]; + const deltaFields = choice.delta as Record; let foundReasoningField: string | null = null; for (const field of reasoningFields) { - if ( - (choice.delta as any)[field] !== null && - (choice.delta as any)[field] !== undefined && - (choice.delta as any)[field].length > 0 - ) { - if (!foundReasoningField) { - foundReasoningField = field; - break; - } + const value = deltaFields[field]; + if (typeof value === "string" && value.length > 0) { + foundReasoningField = field; + break; } } if (foundReasoningField) { - if (!currentBlock || currentBlock.type !== "thinking") { - finishCurrentBlock(currentBlock); - currentBlock = { - type: "thinking", - thinking: "", - thinkingSignature: foundReasoningField, - }; - output.content.push(currentBlock); - stream.push({ type: "thinking_start", contentIndex: currentContentIndex(), partial: output }); - } - - if (currentBlock.type === "thinking") { - const delta = (choice.delta as any)[foundReasoningField]; - currentBlock.thinking += delta; + const delta = deltaFields[foundReasoningField]; + if (typeof delta === "string" && delta.length > 0) { + const block = ensureThinkingBlock(foundReasoningField); + block.thinking += delta; stream.push({ type: "thinking_delta", - contentIndex: currentContentIndex(), + contentIndex: getContentIndex(block), delta, partial: output, }); @@ -300,52 +337,27 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenA if (choice?.delta?.tool_calls) { for (const toolCall of choice.delta.tool_calls) { - const streamIndex = typeof toolCall.index === "number" ? toolCall.index : undefined; - const sameToolCall = - currentBlock?.type === "toolCall" && - ((streamIndex !== undefined && currentBlock.streamIndex === streamIndex) || - (streamIndex === undefined && toolCall.id && currentBlock.id === toolCall.id)); - - if (!sameToolCall) { - finishCurrentBlock(currentBlock); - currentBlock = { - type: "toolCall", - id: toolCall.id || "", - name: toolCall.function?.name || "", - arguments: {}, - partialArgs: "", - streamIndex, - }; - output.content.push(currentBlock); - stream.push({ - type: "toolcall_start", - contentIndex: getContentIndex(currentBlock), - partial: output, - }); + const block = ensureToolCallBlock(toolCall); + if (!block.id && toolCall.id) { + block.id = toolCall.id; + toolCallBlocksById.set(toolCall.id, block); + } + if (!block.name && toolCall.function?.name) { + block.name = toolCall.function.name; } - const currentToolCallBlock = currentBlock?.type === "toolCall" ? currentBlock : null; - if (currentToolCallBlock) { - if (!currentToolCallBlock.id && toolCall.id) currentToolCallBlock.id = toolCall.id; - if (!currentToolCallBlock.name && toolCall.function?.name) { - currentToolCallBlock.name = toolCall.function.name; - } - if (currentToolCallBlock.streamIndex === undefined && streamIndex !== undefined) { - currentToolCallBlock.streamIndex = streamIndex; - } - let delta = ""; - if (toolCall.function?.arguments) { - delta = toolCall.function.arguments; - currentToolCallBlock.partialArgs += toolCall.function.arguments; - currentToolCallBlock.arguments = parseStreamingJson(currentToolCallBlock.partialArgs); - } - stream.push({ - type: "toolcall_delta", - contentIndex: getContentIndex(currentToolCallBlock), - delta, - partial: output, - }); + let delta = ""; + if (toolCall.function?.arguments) { + delta = toolCall.function.arguments; + block.partialArgs = (block.partialArgs ?? "") + toolCall.function.arguments; + block.arguments = parseStreamingJson(block.partialArgs); } + stream.push({ + type: "toolcall_delta", + contentIndex: getContentIndex(block), + delta, + partial: output, + }); } } @@ -365,7 +377,9 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenA } } - finishCurrentBlock(currentBlock); + for (const block of blocks) { + finishBlock(block); + } if (options?.signal?.aborted) { throw new Error("Request was aborted"); } diff --git a/packages/ai/test/openai-completions-tool-choice.test.ts b/packages/ai/test/openai-completions-tool-choice.test.ts index 0fa00341..c9c8ec9f 100644 --- a/packages/ai/test/openai-completions-tool-choice.test.ts +++ b/packages/ai/test/openai-completions-tool-choice.test.ts @@ -552,6 +552,238 @@ describe("openai-completions tool_choice", () => { expect(toolCall).not.toHaveProperty("partialArgs"); }); + it("accumulates mixed content, reasoning, and parallel tool call deltas independently", async () => { + mockState.chunks = [ + { + id: "chatcmpl-mixed-deltas", + choices: [ + { + delta: { + content: "answer 1", + reasoning_content: "think 1", + tool_calls: [ + { + index: 0, + id: "tc_read_initial", + type: "function", + function: { name: "read", arguments: '{"path":"README' }, + }, + { + index: 1, + id: "tc_grep_initial", + type: "function", + function: { name: "grep", arguments: '{"pattern":"TODO' }, + }, + { + id: "tc_list_no_index", + type: "function", + function: { name: "list", arguments: '{"path":"packages' }, + }, + { + id: "tc_write_no_index", + type: "function", + function: { name: "write", arguments: '{"path":"out' }, + }, + ], + }, + finish_reason: null, + }, + ], + }, + { + id: "chatcmpl-mixed-deltas", + choices: [ + { + delta: { + content: " answer 2", + tool_calls: [ + { + index: 1, + id: "tc_grep_changed", + type: "function", + function: { arguments: '","path":"src' }, + }, + { + id: "tc_write_no_index", + type: "function", + function: { arguments: '.txt","content":"ok"}' }, + }, + { + id: "tc_list_no_index", + type: "function", + function: { arguments: '/ai"}' }, + }, + ], + }, + finish_reason: null, + }, + ], + }, + { + id: "chatcmpl-mixed-deltas", + choices: [ + { + delta: { + content: "\n", + reasoning_content: " think 2", + tool_calls: [ + { + index: 0, + id: "tc_read_changed", + type: "function", + function: { arguments: '.md"}' }, + }, + { + index: 1, + type: "function", + function: { arguments: '"}' }, + }, + ], + }, + finish_reason: "tool_calls", + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 8, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 2 }, + }, + }, + ]; + + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + const tools: Tool[] = [ + { + name: "read", + description: "Read a file", + parameters: Type.Object({ path: Type.String() }), + }, + { + name: "grep", + description: "Search a file", + parameters: Type.Object({ pattern: Type.String(), path: Type.String() }), + }, + { + name: "list", + description: "List a directory", + parameters: Type.Object({ path: Type.String() }), + }, + { + name: "write", + description: "Write a file", + parameters: Type.Object({ path: Type.String(), content: Type.String() }), + }, + ]; + const s = streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Think, answer, and use tools.", + timestamp: Date.now(), + }, + ], + tools, + }, + { apiKey: "test" }, + ); + + const eventTypes: string[] = []; + const toolEventsByContentIndex = new Map(); + for await (const event of s) { + eventTypes.push(event.type); + if (event.type === "toolcall_start" || event.type === "toolcall_delta" || event.type === "toolcall_end") { + const events = toolEventsByContentIndex.get(event.contentIndex) ?? []; + events.push(event.type); + toolEventsByContentIndex.set(event.contentIndex, events); + } + } + + const response = await s.result(); + expect(response.stopReason).toBe("toolUse"); + expect(eventTypes.filter((type) => type === "text_start")).toHaveLength(1); + expect(eventTypes.filter((type) => type === "text_delta")).toHaveLength(3); + expect(eventTypes.filter((type) => type === "text_end")).toHaveLength(1); + expect(eventTypes.filter((type) => type === "thinking_start")).toHaveLength(1); + expect(eventTypes.filter((type) => type === "thinking_delta")).toHaveLength(2); + expect(eventTypes.filter((type) => type === "thinking_end")).toHaveLength(1); + expect(eventTypes.filter((type) => type === "toolcall_start")).toHaveLength(4); + expect(eventTypes.filter((type) => type === "toolcall_delta")).toHaveLength(9); + expect(eventTypes.filter((type) => type === "toolcall_end")).toHaveLength(4); + expect(toolEventsByContentIndex.get(2)).toEqual([ + "toolcall_start", + "toolcall_delta", + "toolcall_delta", + "toolcall_end", + ]); + expect(toolEventsByContentIndex.get(3)).toEqual([ + "toolcall_start", + "toolcall_delta", + "toolcall_delta", + "toolcall_delta", + "toolcall_end", + ]); + expect(toolEventsByContentIndex.get(4)).toEqual([ + "toolcall_start", + "toolcall_delta", + "toolcall_delta", + "toolcall_end", + ]); + expect(toolEventsByContentIndex.get(5)).toEqual([ + "toolcall_start", + "toolcall_delta", + "toolcall_delta", + "toolcall_end", + ]); + + expect(response.content).toHaveLength(6); + expect(response.content[0]).toEqual({ type: "text", text: "answer 1 answer 2\n" }); + expect(response.content[1]).toEqual({ + type: "thinking", + thinking: "think 1 think 2", + thinkingSignature: "reasoning_content", + }); + const readCall = response.content[2]; + const grepCall = response.content[3]; + const listCall = response.content[4]; + const writeCall = response.content[5]; + expect(readCall.type).toBe("toolCall"); + expect(grepCall.type).toBe("toolCall"); + expect(listCall.type).toBe("toolCall"); + expect(writeCall.type).toBe("toolCall"); + if ( + readCall.type !== "toolCall" || + grepCall.type !== "toolCall" || + listCall.type !== "toolCall" || + writeCall.type !== "toolCall" + ) { + throw new Error("Expected toolCall content"); + } + expect(readCall.id).toBe("tc_read_initial"); + expect(readCall.name).toBe("read"); + expect(readCall.arguments).toEqual({ path: "README.md" }); + expect(readCall).not.toHaveProperty("streamIndex"); + expect(readCall).not.toHaveProperty("partialArgs"); + expect(grepCall.id).toBe("tc_grep_initial"); + expect(grepCall.name).toBe("grep"); + expect(grepCall.arguments).toEqual({ pattern: "TODO", path: "src" }); + expect(grepCall).not.toHaveProperty("streamIndex"); + expect(grepCall).not.toHaveProperty("partialArgs"); + expect(listCall.id).toBe("tc_list_no_index"); + expect(listCall.name).toBe("list"); + expect(listCall.arguments).toEqual({ path: "packages/ai" }); + expect(listCall).not.toHaveProperty("streamIndex"); + expect(listCall).not.toHaveProperty("partialArgs"); + expect(writeCall.id).toBe("tc_write_no_index"); + expect(writeCall.name).toBe("write"); + expect(writeCall.arguments).toEqual({ path: "out.txt", content: "ok" }); + expect(writeCall).not.toHaveProperty("streamIndex"); + expect(writeCall).not.toHaveProperty("partialArgs"); + }); + it("does not double-count reasoning tokens in completion usage", async () => { mockState.chunks = [ { From b8712457d2c46987b5f8ad33de5465d2cb3a47e3 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Thu, 7 May 2026 10:34:37 +0200 Subject: [PATCH 2/5] fix(tui): keep kitty image redraws inside TUI Fixes #4208. --- packages/tui/src/components/image.ts | 26 ++++--- packages/tui/src/terminal-image.ts | 22 +++--- packages/tui/src/tui.ts | 75 +++++++++++++++++++- packages/tui/test/terminal-image.test.ts | 87 +++++++++++++++++++++++- packages/tui/test/tui-render.test.ts | 82 ++++++++++++++++++++++ 5 files changed, 273 insertions(+), 19 deletions(-) diff --git a/packages/tui/src/components/image.ts b/packages/tui/src/components/image.ts index ca76cddd..876996f3 100644 --- a/packages/tui/src/components/image.ts +++ b/packages/tui/src/components/image.ts @@ -1,4 +1,5 @@ import { + allocateImageId, getCapabilities, getImageDimensions, type ImageDimensions, @@ -26,6 +27,7 @@ export class Image implements Component { private theme: ImageTheme; private options: ImageOptions; private imageId?: number; + private readonly explicitImageId?: number; private cachedLines?: string[]; private cachedWidth?: number; @@ -42,12 +44,13 @@ export class Image implements Component { this.theme = theme; this.options = options; this.dimensions = dimensions || getImageDimensions(base64Data, mimeType) || { widthPx: 800, heightPx: 600 }; + this.explicitImageId = options.imageId; this.imageId = options.imageId; } - /** Get the Kitty image ID used by this image (if any). */ + /** Get the explicit Kitty image ID configured for this image (if any). */ getImageId(): number | undefined { - return this.imageId; + return this.explicitImageId; } invalidate(): void { @@ -66,9 +69,13 @@ export class Image implements Component { let lines: string[]; if (caps.images) { + if (caps.images === "kitty" && this.imageId === undefined) { + this.imageId = allocateImageId(); + } const result = renderImage(this.base64Data, this.dimensions, { maxWidthCells: maxWidth, imageId: this.imageId, + moveCursor: false, }); if (result) { @@ -77,16 +84,19 @@ export class Image implements Component { this.imageId = result.imageId; } - // Return `rows` lines so TUI accounts for image height - // First (rows-1) lines are empty (TUI clears them) - // Last line: move cursor back up, then output image sequence + // Return `rows` lines so TUI accounts for image height. + // First (rows-1) lines are empty and cleared before the image is drawn. + // Last line: move cursor back up, draw the image, then move back down + // for Kitty (this component disables Kitty's terminal-side cursor movement) + // so TUI cursor accounting stays inside the scroll area. lines = []; for (let i = 0; i < result.rows - 1; i++) { lines.push(""); } - // Move cursor up to first row, then output image - const moveUp = result.rows > 1 ? `\x1b[${result.rows - 1}A` : ""; - lines.push(moveUp + result.sequence); + const rowOffset = result.rows - 1; + const moveUp = rowOffset > 0 ? `\x1b[${rowOffset}A` : ""; + const moveDown = caps.images === "kitty" && rowOffset > 0 ? `\x1b[${rowOffset}B` : ""; + lines.push(moveUp + result.sequence + moveDown); } else { const fallback = imageFallback(this.mimeType, this.dimensions, this.options.filename); lines = [this.theme.fallbackColor(fallback)]; diff --git a/packages/tui/src/terminal-image.ts b/packages/tui/src/terminal-image.ts index a396b497..6a84716c 100644 --- a/packages/tui/src/terminal-image.ts +++ b/packages/tui/src/terminal-image.ts @@ -22,6 +22,8 @@ export interface ImageRenderOptions { preserveAspectRatio?: boolean; /** Kitty image ID. If provided, reuses/replaces existing image with this ID. */ imageId?: number; + /** Whether Kitty should apply its default cursor movement after placement. */ + moveCursor?: boolean; } let cachedCapabilities: TerminalCapabilities | null = null; @@ -46,10 +48,7 @@ export function detectCapabilities(): TerminalCapabilities { // sequences differently). Force hyperlinks off whenever we detect them, even // when the outer terminal would otherwise support OSC 8. Image protocols are // also unreliable under tmux/screen, so leave `images: null` for safety. - // cmux currently also gets this conservative fallback due to image corruption: - // https://github.com/badlogic/pi-mono/issues/4208 - const inTmuxOrScreen = - !!process.env.TMUX || !!process.env.CMUX_WORKSPACE_ID || term.startsWith("tmux") || term.startsWith("screen"); + const inTmuxOrScreen = !!process.env.TMUX || term.startsWith("tmux") || term.startsWith("screen"); if (inTmuxOrScreen) { const trueColor = colorTerm === "truecolor" || colorTerm === "24bit"; return { images: null, trueColor, hyperlinks: false }; @@ -131,12 +130,15 @@ export function encodeKitty( columns?: number; rows?: number; imageId?: number; + /** Whether Kitty should apply its default cursor movement after placement. Default: true. */ + moveCursor?: boolean; } = {}, ): string { const CHUNK_SIZE = 4096; const params: string[] = ["a=T", "f=100", "q=2"]; + if (options.moveCursor === false) params.push("C=1"); if (options.columns) params.push(`c=${options.columns}`); if (options.rows) params.push(`r=${options.rows}`); if (options.imageId) params.push(`i=${options.imageId}`); @@ -173,7 +175,7 @@ export function encodeKitty( * Uses uppercase 'I' to also free the image data. */ export function deleteKittyImage(imageId: number): string { - return `\x1b_Ga=d,d=I,i=${imageId}\x1b\\`; + return `\x1b_Ga=d,d=I,i=${imageId},q=2\x1b\\`; } /** @@ -181,7 +183,7 @@ export function deleteKittyImage(imageId: number): string { * Uses uppercase 'A' to also free the image data. */ export function deleteAllKittyImages(): string { - return `\x1b_Ga=d,d=A\x1b\\`; + return "\x1b_Ga=d,d=A,q=2\x1b\\"; } export function encodeITerm2( @@ -377,8 +379,12 @@ export function renderImage( const rows = calculateImageRows(imageDimensions, maxWidth, getCellDimensions()); if (caps.images === "kitty") { - // Only use imageId if explicitly provided - static images don't need IDs - const sequence = encodeKitty(base64Data, { columns: maxWidth, rows, imageId: options.imageId }); + const sequence = encodeKitty(base64Data, { + columns: maxWidth, + rows, + imageId: options.imageId, + moveCursor: options.moveCursor, + }); return { sequence, rows, imageId: options.imageId }; } diff --git a/packages/tui/src/tui.ts b/packages/tui/src/tui.ts index fda30282..e4b22d5b 100644 --- a/packages/tui/src/tui.ts +++ b/packages/tui/src/tui.ts @@ -8,9 +8,26 @@ import * as path from "node:path"; import { performance } from "node:perf_hooks"; import { isKeyRelease, matchesKey } from "./keys.js"; import type { Terminal } from "./terminal.js"; -import { getCapabilities, isImageLine, setCellDimensions } from "./terminal-image.js"; +import { deleteKittyImage, getCapabilities, isImageLine, setCellDimensions } from "./terminal-image.js"; import { extractSegments, normalizeTerminalOutput, sliceByColumn, sliceWithWidth, visibleWidth } from "./utils.js"; +const KITTY_SEQUENCE_PATTERN = /\x1b_G([^;\x1b]*);/g; + +function extractKittyImageIds(line: string): number[] { + const ids: number[] = []; + for (const match of line.matchAll(KITTY_SEQUENCE_PATTERN)) { + for (const param of match[1].split(",")) { + const [key, value] = param.split("=", 2); + if (key !== "i" || value === undefined) continue; + const id = Number(value); + if (Number.isInteger(id) && id > 0 && id <= 0xffffffff && !ids.includes(id)) { + ids.push(id); + } + } + } + return ids; +} + /** * Component interface - all components must implement this */ @@ -217,6 +234,7 @@ export class Container implements Component { export class TUI extends Container { public terminal: Terminal; private previousLines: string[] = []; + private previousKittyImageIds = new Set(); private previousWidth = 0; private previousHeight = 0; private focusedComponent: Component | null = null; @@ -806,6 +824,48 @@ export class TUI extends Container { return lines; } + private collectKittyImageIds(lines: string[]): Set { + const ids = new Set(); + for (const line of lines) { + for (const id of extractKittyImageIds(line)) { + ids.add(id); + } + } + return ids; + } + + private deleteKittyImages(ids: Iterable): string { + let buffer = ""; + for (const id of ids) { + buffer += deleteKittyImage(id); + } + return buffer; + } + + private expandLastChangedForKittyImages(firstChanged: number, lastChanged: number): number { + let expandedLastChanged = lastChanged; + for (let i = firstChanged; i < this.previousLines.length; i++) { + if (extractKittyImageIds(this.previousLines[i]).length > 0) { + expandedLastChanged = Math.max(expandedLastChanged, i); + } + } + return expandedLastChanged; + } + + private deleteChangedKittyImages(firstChanged: number, lastChanged: number): string { + if (firstChanged < 0 || lastChanged < firstChanged) return ""; + + const ids = new Set(); + const maxLine = Math.min(lastChanged, this.previousLines.length - 1); + for (let i = firstChanged; i <= maxLine; i++) { + for (const id of extractKittyImageIds(this.previousLines[i] ?? "")) { + ids.add(id); + } + } + + return this.deleteKittyImages(ids); + } + /** Splice overlay content into a base line at a specific column. Single-pass optimized. */ private compositeLineAt( baseLine: string, @@ -918,7 +978,10 @@ export class TUI extends Container { const fullRender = (clear: boolean): void => { this.fullRedrawCount += 1; let buffer = "\x1b[?2026h"; // Begin synchronized output - if (clear) buffer += "\x1b[2J\x1b[H\x1b[3J"; // Clear screen, home, then clear scrollback + if (clear) { + buffer += this.deleteKittyImages(this.previousKittyImageIds); + buffer += "\x1b[2J\x1b[H\x1b[3J"; // Clear screen, home, then clear scrollback + } for (let i = 0; i < newLines.length; i++) { if (i > 0) buffer += "\r\n"; buffer += newLines[i]; @@ -937,6 +1000,7 @@ export class TUI extends Container { this.previousViewportTop = Math.max(0, bufferLength - height); this.positionHardwareCursor(cursorPos, newLines.length); this.previousLines = newLines; + this.previousKittyImageIds = this.collectKittyImageIds(newLines); this.previousWidth = width; this.previousHeight = height; }; @@ -1003,6 +1067,9 @@ export class TUI extends Container { } lastChanged = newLines.length - 1; } + if (firstChanged !== -1) { + lastChanged = this.expandLastChangedForKittyImages(firstChanged, lastChanged); + } const appendStart = appendedLines && firstChanged === this.previousLines.length && firstChanged > 0; // No changes - but still need to update hardware cursor position if it moved @@ -1017,6 +1084,7 @@ export class TUI extends Container { if (firstChanged >= newLines.length) { if (this.previousLines.length > newLines.length) { let buffer = "\x1b[?2026h"; + buffer += this.deleteChangedKittyImages(firstChanged, lastChanged); // Move to end of new content (clamp to 0 for empty content) const targetRow = Math.max(0, newLines.length - 1); if (targetRow < prevViewportTop) { @@ -1052,6 +1120,7 @@ export class TUI extends Container { } this.positionHardwareCursor(cursorPos, newLines.length); this.previousLines = newLines; + this.previousKittyImageIds = this.collectKittyImageIds(newLines); this.previousWidth = width; this.previousHeight = height; this.previousViewportTop = prevViewportTop; @@ -1069,6 +1138,7 @@ export class TUI extends Container { // Render from first changed line to end // Build buffer with all updates wrapped in synchronized output let buffer = "\x1b[?2026h"; // Begin synchronized output + buffer += this.deleteChangedKittyImages(firstChanged, lastChanged); const prevViewportBottom = prevViewportTop + height - 1; const moveTargetRow = appendStart ? firstChanged - 1 : firstChanged; if (moveTargetRow > prevViewportBottom) { @@ -1199,6 +1269,7 @@ export class TUI extends Container { this.positionHardwareCursor(cursorPos, newLines.length); this.previousLines = newLines; + this.previousKittyImageIds = this.collectKittyImageIds(newLines); this.previousWidth = width; this.previousHeight = height; } diff --git a/packages/tui/test/terminal-image.test.ts b/packages/tui/test/terminal-image.test.ts index dd67e6bf..bdb2220c 100644 --- a/packages/tui/test/terminal-image.test.ts +++ b/packages/tui/test/terminal-image.test.ts @@ -4,7 +4,19 @@ import assert from "node:assert"; import { describe, it } from "node:test"; -import { detectCapabilities, hyperlink, isImageLine } from "../src/terminal-image.js"; +import { Image } from "../src/components/image.js"; +import { + deleteAllKittyImages, + deleteKittyImage, + detectCapabilities, + encodeKitty, + hyperlink, + isImageLine, + renderImage, + resetCapabilitiesCache, + setCapabilities, + setCellDimensions, +} from "../src/terminal-image.js"; const ENV_KEYS = [ "TERM", @@ -15,6 +27,7 @@ const ENV_KEYS = [ "GHOSTTY_RESOURCES_DIR", "WEZTERM_PANE", "ITERM_SESSION_ID", + "CMUX_WORKSPACE_ID", ] as const; function withEnv(overrides: Record, fn: () => void): void { @@ -223,6 +236,14 @@ describe("detectCapabilities", () => { }); }); + it("does not disable Ghostty images solely because cmux is present", () => { + withEnv({ TERM_PROGRAM: "ghostty", CMUX_WORKSPACE_ID: "workspace" }, () => { + const caps = detectCapabilities(); + assert.strictEqual(caps.images, "kitty"); + assert.strictEqual(caps.hyperlinks, true); + }); + }); + it("enables hyperlinks for Kitty", () => { withEnv({ KITTY_WINDOW_ID: "1" }, () => { const caps = detectCapabilities(); @@ -252,6 +273,70 @@ describe("detectCapabilities", () => { }); }); +describe("Kitty image cursor movement", () => { + it("can request no terminal-side cursor movement", () => { + const sequence = encodeKitty("AAAA", { columns: 2, rows: 2, moveCursor: false }); + assert.ok(sequence.startsWith("\x1b_Ga=T,f=100,q=2,C=1,c=2,r=2;")); + }); + + it("suppresses Kitty replies for delete commands", () => { + assert.strictEqual(deleteKittyImage(42), "\x1b_Ga=d,d=I,i=42,q=2\x1b\\"); + assert.strictEqual(deleteAllKittyImages(), "\x1b_Ga=d,d=A,q=2\x1b\\"); + }); + + it("preserves renderImage's default terminal-side cursor movement", () => { + setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true }); + setCellDimensions({ widthPx: 10, heightPx: 10 }); + try { + const result = renderImage("AAAA", { widthPx: 20, heightPx: 20 }, { maxWidthCells: 2 }); + assert.ok(result); + assert.ok(!result.sequence.includes(",C=1,")); + assert.strictEqual(result.rows, 2); + } finally { + resetCapabilitiesCache(); + setCellDimensions({ widthPx: 9, heightPx: 18 }); + } + }); + + it("can opt renderImage into no terminal-side cursor movement", () => { + setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true }); + setCellDimensions({ widthPx: 10, heightPx: 10 }); + try { + const result = renderImage("AAAA", { widthPx: 20, heightPx: 20 }, { maxWidthCells: 2, moveCursor: false }); + assert.ok(result); + assert.ok(result.sequence.includes(",C=1,")); + assert.strictEqual(result.rows, 2); + } finally { + resetCapabilitiesCache(); + setCellDimensions({ widthPx: 9, heightPx: 18 }); + } + }); + + it("restores the cursor to the reserved image row after Kitty rendering", () => { + setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true }); + setCellDimensions({ widthPx: 10, heightPx: 10 }); + try { + const image = new Image( + "AAAA", + "image/png", + { fallbackColor: (value) => value }, + { maxWidthCells: 2 }, + { widthPx: 20, heightPx: 20 }, + ); + const lines = image.render(4); + assert.strictEqual(image.getImageId(), undefined); + assert.deepStrictEqual(lines.slice(0, -1), [""]); + assert.ok(lines[1].startsWith("\x1b[1A\x1b_G")); + assert.ok(lines[1].includes(",C=1,")); + assert.ok(lines[1].includes(",i=")); + assert.ok(lines[1].endsWith("\x1b[1B")); + } finally { + resetCapabilitiesCache(); + setCellDimensions({ widthPx: 9, heightPx: 18 }); + } + }); +}); + describe("hyperlink", () => { it("wraps text in OSC 8 open and close sequences", () => { const result = hyperlink("click me", "https://example.com"); diff --git a/packages/tui/test/tui-render.test.ts b/packages/tui/test/tui-render.test.ts index 4357a367..91cb9695 100644 --- a/packages/tui/test/tui-render.test.ts +++ b/packages/tui/test/tui-render.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert"; import { describe, it } from "node:test"; import type { Terminal as XtermTerminalType } from "@xterm/headless"; +import { deleteKittyImage, encodeKitty } from "../src/terminal-image.js"; import { type Component, TUI } from "../src/tui.js"; import { VirtualTerminal } from "./virtual-terminal.js"; @@ -63,6 +64,87 @@ function getCellItalic(terminal: VirtualTerminal, row: number, col: number): num return cell.isItalic(); } +describe("TUI Kitty image cleanup", () => { + it("deletes changed image ids before drawing moved placements", async () => { + const terminal = new LoggingVirtualTerminal(40, 10); + const tui = new TUI(terminal); + const component = new TestComponent(); + tui.addChild(component); + + const oldImage = encodeKitty("AAAA", { columns: 2, rows: 2, imageId: 42, moveCursor: false }); + component.lines = ["top", oldImage]; + tui.start(); + await terminal.waitForRender(); + terminal.clearWrites(); + + const newImage = encodeKitty("BBBB", { columns: 2, rows: 1, imageId: 42, moveCursor: false }); + component.lines = [newImage, ""]; + tui.requestRender(); + await terminal.waitForRender(); + + const writes = terminal.getWrites(); + const deleteIndex = writes.indexOf(deleteKittyImage(42)); + const drawIndex = writes.indexOf(newImage); + assert.ok(deleteIndex >= 0, "changed old image should be deleted"); + assert.ok(drawIndex >= 0, "new image should be drawn"); + assert.ok(deleteIndex < drawIndex, "old image must be deleted before the new placement is drawn"); + + tui.stop(); + }); + + it("redraws image lines when an earlier reserved image row changes", async () => { + const terminal = new LoggingVirtualTerminal(40, 10); + const tui = new TUI(terminal); + const component = new TestComponent(); + tui.addChild(component); + + const image = encodeKitty("AAAA", { columns: 2, rows: 2, imageId: 88, moveCursor: false }); + component.lines = ["", image]; + tui.start(); + await terminal.waitForRender(); + terminal.clearWrites(); + + component.lines = ["covered", image]; + tui.requestRender(); + await terminal.waitForRender(); + + const writes = terminal.getWrites(); + const deleteIndex = writes.indexOf(deleteKittyImage(88)); + const drawIndex = writes.indexOf(image); + assert.ok(deleteIndex >= 0, "image should be deleted when a reserved row changes"); + assert.ok(drawIndex >= 0, "unchanged image line should be redrawn after deleting the placement"); + assert.ok(deleteIndex < drawIndex, "old placement must be deleted before the image line is redrawn"); + assert.ok(!writes.includes("\x1b[2J"), "reserved row changes should not force a full redraw"); + + tui.stop(); + }); + + it("deletes previously rendered image ids during full redraws", async () => { + const terminal = new LoggingVirtualTerminal(40, 10); + const tui = new TUI(terminal); + const component = new TestComponent(); + tui.addChild(component); + + component.lines = [encodeKitty("AAAA", { columns: 2, rows: 2, imageId: 77, moveCursor: false })]; + tui.start(); + await terminal.waitForRender(); + terminal.clearWrites(); + + component.lines = ["plain text"]; + tui.requestRender(true); + await terminal.waitForRender(); + + const writes = terminal.getWrites(); + const deleteIndex = writes.indexOf(deleteKittyImage(77)); + const clearIndex = writes.indexOf("\x1b[2J"); + assert.ok(deleteIndex >= 0, "previous image should be deleted during full redraw"); + assert.ok(clearIndex >= 0, "full redraw should clear the screen"); + assert.ok(deleteIndex < clearIndex, "old image should be deleted before the screen is cleared"); + + tui.stop(); + }); +}); + describe("TUI resize handling", () => { it("triggers full re-render when terminal height changes", async () => { await withEnv({ TERMUX_VERSION: undefined }, async () => { From 83a917073c986673839efc66ca6b473e32e91152 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Thu, 7 May 2026 10:40:15 +0200 Subject: [PATCH 3/5] fix(tui): remove explicit image ids --- packages/tui/src/components/image.ts | 6 ++---- packages/tui/test/terminal-image.test.ts | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/tui/src/components/image.ts b/packages/tui/src/components/image.ts index 876996f3..74992b34 100644 --- a/packages/tui/src/components/image.ts +++ b/packages/tui/src/components/image.ts @@ -27,7 +27,6 @@ export class Image implements Component { private theme: ImageTheme; private options: ImageOptions; private imageId?: number; - private readonly explicitImageId?: number; private cachedLines?: string[]; private cachedWidth?: number; @@ -44,13 +43,12 @@ export class Image implements Component { this.theme = theme; this.options = options; this.dimensions = dimensions || getImageDimensions(base64Data, mimeType) || { widthPx: 800, heightPx: 600 }; - this.explicitImageId = options.imageId; this.imageId = options.imageId; } - /** Get the explicit Kitty image ID configured for this image (if any). */ + /** Get the Kitty image ID used by this image (if any). */ getImageId(): number | undefined { - return this.explicitImageId; + return this.imageId; } invalidate(): void { diff --git a/packages/tui/test/terminal-image.test.ts b/packages/tui/test/terminal-image.test.ts index bdb2220c..58e018d2 100644 --- a/packages/tui/test/terminal-image.test.ts +++ b/packages/tui/test/terminal-image.test.ts @@ -324,11 +324,12 @@ describe("Kitty image cursor movement", () => { { widthPx: 20, heightPx: 20 }, ); const lines = image.render(4); - assert.strictEqual(image.getImageId(), undefined); + const imageId = image.getImageId(); + assert.strictEqual(typeof imageId, "number"); assert.deepStrictEqual(lines.slice(0, -1), [""]); assert.ok(lines[1].startsWith("\x1b[1A\x1b_G")); assert.ok(lines[1].includes(",C=1,")); - assert.ok(lines[1].includes(",i=")); + assert.ok(lines[1].includes(`,i=${imageId}`)); assert.ok(lines[1].endsWith("\x1b[1B")); } finally { resetCapabilitiesCache(); From 9eb126e7b2fe3a539b653e9c846638ce15f32ee1 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 7 May 2026 11:42:35 +0200 Subject: [PATCH 4/5] docs(ai): document interleaved stream events --- packages/ai/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/ai/README.md b/packages/ai/README.md index 51b3ef53..df8136a4 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -383,6 +383,8 @@ All streaming events emitted during assistant message generation: | `done` | Stream complete | `reason`: Stop reason ("stop", "length", "toolUse"), `message`: Final assistant message | | `error` | Error occurred | `reason`: Error type ("error" or "aborted"), `error`: AssistantMessage with partial content | +Streaming events for different content blocks are not guaranteed to be contiguous. Providers may emit deltas for text, thinking, and tool calls in the same upstream chunk, and pi may surface corresponding events interleaved, for example `text_start`, `text_delta`, `toolcall_start`, `text_delta`, `toolcall_delta`. Consumers must use `contentIndex` to associate each delta/end event with its block and must not assume that a block's `*_start`/`*_delta`/`*_end` sequence is uninterrupted by events for other blocks. + ## Image Input Models with vision capabilities can process images. You can check if a model supports images via the `input` property. If you pass images to a non-vision model, they are silently ignored. From 801db80b65210b25462bce1700675e073fe3dbe5 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 7 May 2026 12:21:29 +0200 Subject: [PATCH 5/5] fix(tui): bound kitty image id parsing --- packages/tui/src/tui.ts | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/tui/src/tui.ts b/packages/tui/src/tui.ts index e4b22d5b..824ac82f 100644 --- a/packages/tui/src/tui.ts +++ b/packages/tui/src/tui.ts @@ -11,21 +11,26 @@ import type { Terminal } from "./terminal.js"; import { deleteKittyImage, getCapabilities, isImageLine, setCellDimensions } from "./terminal-image.js"; import { extractSegments, normalizeTerminalOutput, sliceByColumn, sliceWithWidth, visibleWidth } from "./utils.js"; -const KITTY_SEQUENCE_PATTERN = /\x1b_G([^;\x1b]*);/g; +const KITTY_SEQUENCE_PREFIX = "\x1b_G"; function extractKittyImageIds(line: string): number[] { - const ids: number[] = []; - for (const match of line.matchAll(KITTY_SEQUENCE_PATTERN)) { - for (const param of match[1].split(",")) { - const [key, value] = param.split("=", 2); - if (key !== "i" || value === undefined) continue; - const id = Number(value); - if (Number.isInteger(id) && id > 0 && id <= 0xffffffff && !ids.includes(id)) { - ids.push(id); - } + const sequenceStart = line.indexOf(KITTY_SEQUENCE_PREFIX); + if (sequenceStart === -1) return []; + + const paramsStart = sequenceStart + KITTY_SEQUENCE_PREFIX.length; + const paramsEnd = line.indexOf(";", paramsStart); + if (paramsEnd === -1) return []; + + const params = line.slice(paramsStart, paramsEnd); + for (const param of params.split(",")) { + const [key, value] = param.split("=", 2); + if (key !== "i" || value === undefined) continue; + const id = Number(value); + if (Number.isInteger(id) && id > 0 && id <= 0xffffffff) { + return [id]; } } - return ids; + return []; } /**