fix(coding-agent): enforce safe auto-resized image limits closes #2055

packages/coding-agent/CHANGELOG.md

@@ -13,6 +13,7 @@
 
 ### Fixed
 
+- Fixed auto-resized image handling to enforce the inline image size limit on the final base64 payload, return text-only fallbacks when resizing cannot produce a safe image, and avoid falling back to the original image in `read` and `@file` auto-resize paths ([#2055](https://github.com/badlogic/pi-mono/issues/2055))
 - Fixed `pi update` for git packages to skip destructive reset, clean, and reinstall steps when the fetched target already matches the local checkout ([#2503](https://github.com/badlogic/pi-mono/issues/2503))
 - Fixed print and JSON mode to take over stdout during non-interactive startup, keeping package-manager and other incidental chatter off protocol/output stdout ([#2482](https://github.com/badlogic/pi-mono/issues/2482))
 

packages/coding-agent/src/cli/file-processor.ts

@@ -57,6 +57,10 @@ export async function processFileArguments(fileArgs: string[], options?: Process
 
 			if (autoResizeImages) {
 				const resized = await resizeImage({ type: "image", data: base64Content, mimeType });
+				if (!resized) {
+					text += `<file name="${absolutePath}">[Image omitted: could not be resized below the inline image size limit.]</file>\n`;
+					continue;
+				}
 				dimensionNote = formatDimensionNote(resized);
 				attachment = {
 					type: "image",

packages/coding-agent/src/core/tools/read.ts

@@ -160,13 +160,22 @@ export function createReadToolDefinition(
 								if (autoResizeImages) {
 									// Resize image if needed before sending it back to the model.
 									const resized = await resizeImage({ type: "image", data: base64, mimeType });
-									const dimensionNote = formatDimensionNote(resized);
+									if (!resized) {
-									let textNote = `Read image file [${resized.mimeType}]`;
+										content = [
-									if (dimensionNote) textNote += `\n${dimensionNote}`;
+											{
-									content = [
+												type: "text",
-										{ type: "text", text: textNote },
+												text: `Read image file [${mimeType}]\n[Image omitted: could not be resized below the inline image size limit.]`,
-										{ type: "image", data: resized.data, mimeType: resized.mimeType },
+											},
-									];
+										];
+									} else {
+										const dimensionNote = formatDimensionNote(resized);
+										let textNote = `Read image file [${resized.mimeType}]`;
+										if (dimensionNote) textNote += `\n${dimensionNote}`;
+										content = [
+											{ type: "text", text: textNote },
+											{ type: "image", data: resized.data, mimeType: resized.mimeType },
+										];
+									}
 								} else {
 									content = [
 										{ type: "text", text: `Read image file [${mimeType}]` },

packages/coding-agent/src/utils/image-resize.ts

@@ -5,7 +5,7 @@ import { loadPhoton } from "./photon.js";
 export interface ImageResizeOptions {
 	maxWidth?: number; // Default: 2000
 	maxHeight?: number; // Default: 2000
-	maxBytes?: number; // Default: 4.5MB (below Anthropic's 5MB limit)
+	maxBytes?: number; // Default: 4.5MB of base64 payload (below Anthropic's 5MB limit)
 	jpegQuality?: number; // Default: 80
 }
 
@@ -19,7 +19,7 @@ export interface ResizedImage {
 	wasResized: boolean;
 }
 
-// 4.5MB - provides headroom below Anthropic's 5MB limit
+// 4.5MB of base64 payload. Provides headroom below Anthropic's 5MB limit.
 const DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024;
 
 const DEFAULT_OPTIONS: Required<ImageResizeOptions> = {
@@ -29,43 +29,42 @@ const DEFAULT_OPTIONS: Required<ImageResizeOptions> = {
 	jpegQuality: 80,
 };
 
-/** Helper to pick the smaller of two buffers */
+interface EncodedCandidate {
-function pickSmaller(
+	data: string;
-	a: { buffer: Uint8Array; mimeType: string },
+	encodedSize: number;
-	b: { buffer: Uint8Array; mimeType: string },
+	mimeType: string;
-): { buffer: Uint8Array; mimeType: string } {
+}
-	return a.buffer.length <= b.buffer.length ? a : b;
+
+function encodeCandidate(buffer: Uint8Array, mimeType: string): EncodedCandidate {
+	const data = Buffer.from(buffer).toString("base64");
+	return {
+		data,
+		encodedSize: Buffer.byteLength(data, "utf-8"),
+		mimeType,
+	};
 }
 
 /**
- * Resize an image to fit within the specified max dimensions and file size.
+ * Resize an image to fit within the specified max dimensions and encoded file size.
- * Returns the original image if it already fits within the limits.
+ * Returns null if the image cannot be resized below maxBytes.
  *
  * Uses Photon (Rust/WASM) for image processing. If Photon is not available,
- * returns the original image unchanged.
+ * returns null.
  *
  * Strategy for staying under maxBytes:
  * 1. First resize to maxWidth/maxHeight
  * 2. Try both PNG and JPEG formats, pick the smaller one
  * 3. If still too large, try JPEG with decreasing quality
- * 4. If still too large, progressively reduce dimensions
+ * 4. If still too large, progressively reduce dimensions until 1x1
  */
-export async function resizeImage(img: ImageContent, options?: ImageResizeOptions): Promise<ResizedImage> {
+export async function resizeImage(img: ImageContent, options?: ImageResizeOptions): Promise<ResizedImage | null> {
 	const opts = { ...DEFAULT_OPTIONS, ...options };
 	const inputBuffer = Buffer.from(img.data, "base64");
+	const inputBase64Size = Buffer.byteLength(img.data, "utf-8");
 
 	const photon = await loadPhoton();
 	if (!photon) {
-		// Photon not available, return original image
+		return null;
-		return {
-			data: img.data,
-			mimeType: img.mimeType,
-			originalWidth: 0,
-			originalHeight: 0,
-			width: 0,
-			height: 0,
-			wasResized: false,
-		};
 	}
 
 	let image: ReturnType<typeof photon.PhotonImage.new_from_byteslice> | undefined;
@@ -79,9 +78,8 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
 		const originalHeight = image.get_height();
 		const format = img.mimeType?.split("/")[1] ?? "png";
 
-		// Check if already within all limits (dimensions AND size)
+		// Check if already within all limits (dimensions AND encoded size)
-		const originalSize = inputBuffer.length;
+		if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && inputBase64Size < opts.maxBytes) {
-		if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && originalSize <= opts.maxBytes) {
 			return {
 				data: img.data,
 				mimeType: img.mimeType ?? `image/${format}`,
@@ -106,114 +104,57 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
 			targetHeight = opts.maxHeight;
 		}
 
-		// Helper to resize and encode in both formats, returning the smaller one
+		function tryEncodings(width: number, height: number, jpegQualities: number[]): EncodedCandidate[] {
-		function tryBothFormats(
-			width: number,
-			height: number,
-			jpegQuality: number,
-		): { buffer: Uint8Array; mimeType: string } {
 			const resized = photon!.resize(image!, width, height, photon!.SamplingFilter.Lanczos3);
 
 			try {
-				const pngBuffer = resized.get_bytes();
+				const candidates: EncodedCandidate[] = [encodeCandidate(resized.get_bytes(), "image/png")];
-				const jpegBuffer = resized.get_bytes_jpeg(jpegQuality);
+				for (const quality of jpegQualities) {
-
+					candidates.push(encodeCandidate(resized.get_bytes_jpeg(quality), "image/jpeg"));
-				return pickSmaller(
+				}
-					{ buffer: pngBuffer, mimeType: "image/png" },
+				return candidates;
-					{ buffer: jpegBuffer, mimeType: "image/jpeg" },
-				);
 			} finally {
 				resized.free();
 			}
 		}
 
-		// Try to produce an image under maxBytes
+		const qualitySteps = Array.from(new Set([opts.jpegQuality, 85, 70, 55, 40]));
-		const qualitySteps = [85, 70, 55, 40];
+		let currentWidth = targetWidth;
-		const scaleSteps = [1.0, 0.75, 0.5, 0.35, 0.25];
+		let currentHeight = targetHeight;
 
-		let best: { buffer: Uint8Array; mimeType: string };
+		while (true) {
-		let finalWidth = targetWidth;
+			const candidates = tryEncodings(currentWidth, currentHeight, qualitySteps);
-		let finalHeight = targetHeight;
+			for (const candidate of candidates) {
-
+				if (candidate.encodedSize < opts.maxBytes) {
-		// First attempt: resize to target dimensions, try both formats
-		best = tryBothFormats(targetWidth, targetHeight, opts.jpegQuality);
-
-		if (best.buffer.length <= opts.maxBytes) {
-			return {
-				data: Buffer.from(best.buffer).toString("base64"),
-				mimeType: best.mimeType,
-				originalWidth,
-				originalHeight,
-				width: finalWidth,
-				height: finalHeight,
-				wasResized: true,
-			};
-		}
-
-		// Still too large - try JPEG with decreasing quality
-		for (const quality of qualitySteps) {
-			best = tryBothFormats(targetWidth, targetHeight, quality);
-
-			if (best.buffer.length <= opts.maxBytes) {
-				return {
-					data: Buffer.from(best.buffer).toString("base64"),
-					mimeType: best.mimeType,
-					originalWidth,
-					originalHeight,
-					width: finalWidth,
-					height: finalHeight,
-					wasResized: true,
-				};
-			}
-		}
-
-		// Still too large - reduce dimensions progressively
-		for (const scale of scaleSteps) {
-			finalWidth = Math.round(targetWidth * scale);
-			finalHeight = Math.round(targetHeight * scale);
-
-			if (finalWidth < 100 || finalHeight < 100) {
-				break;
-			}
-
-			for (const quality of qualitySteps) {
-				best = tryBothFormats(finalWidth, finalHeight, quality);
-
-				if (best.buffer.length <= opts.maxBytes) {
 					return {
-						data: Buffer.from(best.buffer).toString("base64"),
+						data: candidate.data,
-						mimeType: best.mimeType,
+						mimeType: candidate.mimeType,
 						originalWidth,
 						originalHeight,
-						width: finalWidth,
+						width: currentWidth,
-						height: finalHeight,
+						height: currentHeight,
 						wasResized: true,
 					};
 				}
 			}
+
+			if (currentWidth === 1 && currentHeight === 1) {
+				break;
+			}
+
+			const nextWidth = currentWidth === 1 ? 1 : Math.max(1, Math.floor(currentWidth * 0.75));
+			const nextHeight = currentHeight === 1 ? 1 : Math.max(1, Math.floor(currentHeight * 0.75));
+			if (nextWidth === currentWidth && nextHeight === currentHeight) {
+				break;
+			}
+
+			currentWidth = nextWidth;
+			currentHeight = nextHeight;
 		}
 
-		// Last resort: return smallest version we produced
+		return null;
-		return {
-			data: Buffer.from(best.buffer).toString("base64"),
-			mimeType: best.mimeType,
-			originalWidth,
-			originalHeight,
-			width: finalWidth,
-			height: finalHeight,
-			wasResized: true,
-		};
 	} catch {
-		// Failed to load image
+		return null;
-		return {
-			data: img.data,
-			mimeType: img.mimeType,
-			originalWidth: 0,
-			originalHeight: 0,
-			width: 0,
-			height: 0,
-			wasResized: false,
-		};
 	} finally {
 		if (image) {
 			image.free();

packages/coding-agent/test/image-processing.test.ts

@@ -52,12 +52,13 @@ describe("resizeImage", () => {
 			{ maxWidth: 100, maxHeight: 100, maxBytes: 1024 * 1024 },
 		);
 
-		expect(result.wasResized).toBe(false);
+		expect(result).not.toBeNull();
-		expect(result.data).toBe(TINY_PNG);
+		expect(result!.wasResized).toBe(false);
-		expect(result.originalWidth).toBe(2);
+		expect(result!.data).toBe(TINY_PNG);
-		expect(result.originalHeight).toBe(2);
+		expect(result!.originalWidth).toBe(2);
-		expect(result.width).toBe(2);
+		expect(result!.originalHeight).toBe(2);
-		expect(result.height).toBe(2);
+		expect(result!.width).toBe(2);
+		expect(result!.height).toBe(2);
 	});
 
 	it("should resize image exceeding dimension limits", async () => {
@@ -66,26 +67,38 @@ describe("resizeImage", () => {
 			{ maxWidth: 50, maxHeight: 50, maxBytes: 1024 * 1024 },
 		);
 
-		expect(result.wasResized).toBe(true);
+		expect(result).not.toBeNull();
-		expect(result.originalWidth).toBe(100);
+		expect(result!.wasResized).toBe(true);
-		expect(result.originalHeight).toBe(100);
+		expect(result!.originalWidth).toBe(100);
-		expect(result.width).toBeLessThanOrEqual(50);
+		expect(result!.originalHeight).toBe(100);
-		expect(result.height).toBeLessThanOrEqual(50);
+		expect(result!.width).toBeLessThanOrEqual(50);
+		expect(result!.height).toBeLessThanOrEqual(50);
 	});
 
 	it("should resize image exceeding byte limit", async () => {
 		const originalBuffer = Buffer.from(LARGE_PNG_200x200, "base64");
 		const originalSize = originalBuffer.length;
 
-		// Set maxBytes to less than the original image size
+		// Set maxBytes to less than the original encoded image size
 		const result = await resizeImage(
 			{ type: "image", data: LARGE_PNG_200x200, mimeType: "image/png" },
-			{ maxWidth: 2000, maxHeight: 2000, maxBytes: Math.floor(originalSize / 2) },
+			{ maxWidth: 2000, maxHeight: 2000, maxBytes: Math.floor(LARGE_PNG_200x200.length * 0.9) },
 		);
 
 		// Should have tried to reduce size
-		const resultBuffer = Buffer.from(result.data, "base64");
+		expect(result).not.toBeNull();
+		const resultBuffer = Buffer.from(result!.data, "base64");
 		expect(resultBuffer.length).toBeLessThan(originalSize);
+		expect(result!.data.length).toBeLessThan(LARGE_PNG_200x200.length);
+	});
+
+	it("should return null when image cannot be resized below maxBytes", async () => {
+		const result = await resizeImage(
+			{ type: "image", data: LARGE_PNG_200x200, mimeType: "image/png" },
+			{ maxWidth: 2000, maxHeight: 2000, maxBytes: 1 },
+		);
+
+		expect(result).toBeNull();
 	});
 
 	it("should handle JPEG input", async () => {
@@ -94,9 +107,10 @@ describe("resizeImage", () => {
 			{ maxWidth: 100, maxHeight: 100, maxBytes: 1024 * 1024 },
 		);
 
-		expect(result.wasResized).toBe(false);
+		expect(result).not.toBeNull();
-		expect(result.originalWidth).toBe(2);
+		expect(result!.wasResized).toBe(false);
-		expect(result.originalHeight).toBe(2);
+		expect(result!.originalWidth).toBe(2);
+		expect(result!.originalHeight).toBe(2);
 	});
 });
 

packages/coding-agent/test/image-resize-callers.test.ts

@@ -0,0 +1,53 @@
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("../src/utils/image-resize.js", () => ({
+	resizeImage: vi.fn(),
+	formatDimensionNote: vi.fn(() => undefined),
+}));
+
+import { processFileArguments } from "../src/cli/file-processor.js";
+import { createReadTool } from "../src/core/tools/read.js";
+import { resizeImage } from "../src/utils/image-resize.js";
+
+const TINY_PNG_BASE64 =
+	"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==";
+
+describe("image resize callers", () => {
+	let testDir: string;
+
+	beforeEach(() => {
+		testDir = join(tmpdir(), `image-resize-callers-${Date.now()}`);
+		mkdirSync(testDir, { recursive: true });
+		vi.mocked(resizeImage).mockReset();
+		vi.mocked(resizeImage).mockResolvedValue(null);
+	});
+
+	afterEach(() => {
+		rmSync(testDir, { recursive: true, force: true });
+	});
+
+	it("read tool returns text-only output when auto-resize cannot produce a safe image", async () => {
+		const imagePath = join(testDir, "test.png");
+		writeFileSync(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
+
+		const tool = createReadTool(testDir);
+		const result = await tool.execute("test-read-image", { path: imagePath });
+
+		expect(result.content).toHaveLength(1);
+		expect(result.content[0].type).toBe("text");
+		expect((result.content[0] as { type: "text"; text: string }).text).toContain("Image omitted");
+	});
+
+	it("file processor omits image attachments when auto-resize cannot produce a safe image", async () => {
+		const imagePath = join(testDir, "test.png");
+		writeFileSync(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
+
+		const result = await processFileArguments([imagePath]);
+
+		expect(result.images).toHaveLength(0);
+		expect(result.text).toContain("Image omitted");
+	});
+});