shumengya/sproutclaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ai,coding-agent): preserve non-vision image placeholders closes #3429

Browse Source
This commit is contained in:
Mario Zechner
2026-04-20 16:52:27 +02:00
parent 74139c3f66
commit 2f4f283cc2
11 changed files with 184 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
packages/coding-agent/src/core/tools/read.ts
View File

@@ -1,5 +1,5 @@
import type { AgentTool } from "@mariozechner/pi-agent-core";
import type { ImageContent, TextContent } from "@mariozechner/pi-ai";
import type { Api, ImageContent, Model, TextContent } from "@mariozechner/pi-ai";
import { Text } from "@mariozechner/pi-tui";
import { type Static, Type } from "@sinclair/typebox";
import { constants } from "fs";
@@ -78,6 +78,13 @@ function trimTrailingEmptyLines(lines: string[]): string[] {
return lines.slice(0, end);
}
function getNonVisionImageNote(model: Model<Api> | undefined): string | undefined {
if (!model || model.input.includes("image")) {
return undefined;
}
return "[Current model does not support images. The image will be omitted from this request.]";
}
function formatReadResult(
args: { path?: string; file_path?: string; offset?: number; limit?: number } | undefined,
result: { content: (TextContent | ImageContent)[]; details?: ReadToolDetails },
@@ -129,7 +136,7 @@ export function createReadToolDefinition(
{ path, offset, limit }: { path: string; offset?: number; limit?: number },
signal?: AbortSignal,
_onUpdate?,
_ctx?,
ctx?,
) {
const absolutePath = resolveReadPath(path, cwd);
return new Promise<{ content: (TextContent | ImageContent)[]; details: ReadToolDetails | undefined }>(
@@ -153,6 +160,7 @@ export function createReadToolDefinition(
const mimeType = ops.detectImageMimeType ? await ops.detectImageMimeType(absolutePath) : undefined;
let content: (TextContent | ImageContent)[];
let details: ReadToolDetails | undefined;
const nonVisionImageNote = getNonVisionImageNote(ctx?.model);
if (mimeType) {
// Read image as binary.
const buffer = await ops.readFile(absolutePath);
@@ -161,24 +169,24 @@ export function createReadToolDefinition(
// Resize image if needed before sending it back to the model.
const resized = await resizeImage({ type: "image", data: base64, mimeType });
if (!resized) {
content = [
{
type: "text",
text: `Read image file [${mimeType}]\n[Image omitted: could not be resized below the inline image size limit.]`,
},
];
let textNote = `Read image file [${mimeType}]\n[Image omitted: could not be resized below the inline image size limit.]`;
if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`;
content = [{ type: "text", text: textNote }];
} else {
const dimensionNote = formatDimensionNote(resized);
let textNote = `Read image file [${resized.mimeType}]`;
if (dimensionNote) textNote += `\n${dimensionNote}`;
if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`;
content = [
{ type: "text", text: textNote },
{ type: "image", data: resized.data, mimeType: resized.mimeType },
];
}
} else {
let textNote = `Read image file [${mimeType}]`;
if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`;
content = [
{ type: "text", text: `Read image file [${mimeType}]` },
{ type: "text", text: textNote },
{ type: "image", data: base64, mimeType },
];
}