fix(ai): keep image tool results inline for gemini 3+ and antigravity closes #2052

packages/ai/src/providers/google-shared.ts

@@ -70,6 +70,20 @@ export function requiresToolCallId(modelId: string): boolean {
 	return modelId.startsWith("claude-") || modelId.startsWith("gpt-oss-");
 }
 
+function getGeminiMajorVersion(modelId: string): number | undefined {
+	const match = modelId.toLowerCase().match(/^gemini(?:-live)?-(\d+)/);
+	if (!match) return undefined;
+	return Number.parseInt(match[1], 10);
+}
+
+function supportsMultimodalFunctionResponse(modelId: string): boolean {
+	const geminiMajorVersion = getGeminiMajorVersion(modelId);
+	if (geminiMajorVersion !== undefined) {
+		return geminiMajorVersion >= 3;
+	}
+	return true;
+}
+
 /**
  * Convert internal messages to Gemini Content[] format.
  */
@@ -175,10 +189,10 @@ export function convertMessages<T extends GoogleApiType>(model: Model<T>, contex
 			const hasText = textResult.length > 0;
 			const hasImages = imageContent.length > 0;
 
-			// Gemini 3 supports multimodal function responses with images nested inside functionResponse.parts
+			// Gemini 3+ models support multimodal function responses with images nested inside
-			// See: https://ai.google.dev/gemini-api/docs/function-calling#multimodal
+			// functionResponse.parts. Claude and other non-Gemini models behind Cloud Code Assist /
-			// Older models don't support this, so we put images in a separate user message.
+			// Antigravity also accept this shape. Gemini < 3 still needs a separate user image turn.
-			const supportsMultimodalFunctionResponse = model.id.includes("gemini-3");
+			const modelSupportsMultimodalFunctionResponse = supportsMultimodalFunctionResponse(model.id);
 
 			// Use "output" key for success, "error" key for errors as per SDK documentation
 			const responseValue = hasText ? sanitizeSurrogates(textResult) : hasImages ? "(see attached image)" : "";
@@ -195,8 +209,7 @@ export function convertMessages<T extends GoogleApiType>(model: Model<T>, contex
 				functionResponse: {
 					name: msg.toolName,
 					response: msg.isError ? { error: responseValue } : { output: responseValue },
-					// Nest images inside functionResponse.parts for Gemini 3
+					...(hasImages && modelSupportsMultimodalFunctionResponse && { parts: imageParts }),
-					...(hasImages && supportsMultimodalFunctionResponse && { parts: imageParts }),
 					...(includeId ? { id: msg.toolCallId } : {}),
 				},
 			};
@@ -213,8 +226,8 @@ export function convertMessages<T extends GoogleApiType>(model: Model<T>, contex
 				});
 			}
 
-			// For older models, add images in a separate user message
+			// For Gemini < 3, add images in a separate user message
-			if (hasImages && !supportsMultimodalFunctionResponse) {
+			if (hasImages && !modelSupportsMultimodalFunctionResponse) {
 				contents.push({
 					role: "user",
 					parts: [{ text: "Tool result image:" }, ...imageParts],

packages/ai/test/google-shared-image-tool-result-routing.test.ts

@@ -0,0 +1,124 @@
+import { describe, expect, it } from "vitest";
+import { convertMessages } from "../src/providers/google-shared.js";
+import type { Context, Model } from "../src/types.js";
+
+function makeModel<TApi extends "google-generative-ai" | "google-gemini-cli">(
+	api: TApi,
+	provider: Model<TApi>["provider"],
+	id: string,
+): Model<TApi> {
+	return {
+		id,
+		name: id,
+		api,
+		provider,
+		baseUrl: "https://example.com",
+		reasoning: true,
+		input: ["text", "image"],
+		cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+		contextWindow: 128000,
+		maxTokens: 8192,
+	};
+}
+
+function makeContext(model: { api: string; provider: string; id: string }): Context {
+	const now = Date.now();
+	return {
+		messages: [
+			{ role: "user", content: "read the files", timestamp: now },
+			{
+				role: "assistant",
+				content: [
+					{ type: "toolCall", id: "call_a", name: "read", arguments: { path: "a.txt" } },
+					{ type: "toolCall", id: "call_img", name: "read", arguments: { path: "image.png" } },
+					{ type: "toolCall", id: "call_b", name: "read", arguments: { path: "b.txt" } },
+				],
+				api: model.api,
+				provider: model.provider,
+				model: model.id,
+				usage: {
+					input: 0,
+					output: 0,
+					cacheRead: 0,
+					cacheWrite: 0,
+					totalTokens: 0,
+					cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+				},
+				stopReason: "toolUse",
+				timestamp: now,
+			},
+			{
+				role: "toolResult",
+				toolCallId: "call_a",
+				toolName: "read",
+				content: [{ type: "text", text: "alpha text" }],
+				isError: false,
+				timestamp: now,
+			},
+			{
+				role: "toolResult",
+				toolCallId: "call_img",
+				toolName: "read",
+				content: [{ type: "image", data: "abc", mimeType: "image/png" }],
+				isError: false,
+				timestamp: now,
+			},
+			{
+				role: "toolResult",
+				toolCallId: "call_b",
+				toolName: "read",
+				content: [{ type: "text", text: "beta text" }],
+				isError: false,
+				timestamp: now,
+			},
+		],
+	};
+}
+
+describe("google-shared image tool result routing", () => {
+	it("keeps separate synthetic image turn for Gemini 2.x Google API models", () => {
+		const model = makeModel("google-generative-ai", "google", "gemini-2.5-flash");
+		const contents = convertMessages(model, makeContext(model));
+
+		expect(contents).toHaveLength(5);
+		expect(contents[2].parts?.every((part) => part.functionResponse)).toBe(true);
+		expect(contents[3].parts?.[0]?.text).toBe("Tool result image:");
+		expect(contents[3].parts?.[1]?.inlineData).toBeTruthy();
+		expect(contents[4].parts?.[0]?.functionResponse).toBeTruthy();
+	});
+
+	it("nests image tool results for Gemini 3 Google API models", () => {
+		const model = makeModel("google-generative-ai", "google", "gemini-3-pro-preview");
+		const contents = convertMessages(model, makeContext(model));
+
+		expect(contents).toHaveLength(3);
+		const toolResultTurn = contents[2];
+		expect(toolResultTurn.parts).toHaveLength(3);
+		const imageResponse = toolResultTurn.parts?.[1]?.functionResponse;
+		expect(imageResponse).toBeTruthy();
+		expect(imageResponse?.parts).toHaveLength(1);
+		expect(imageResponse?.parts?.[0]?.inlineData).toBeTruthy();
+	});
+
+	it("nests image tool results for non-Gemini models on Antigravity / Cloud Code Assist", () => {
+		const model = makeModel("google-gemini-cli", "google-antigravity", "claude-sonnet-4-6");
+		const contents = convertMessages(model, makeContext(model));
+
+		expect(contents).toHaveLength(3);
+		const toolResultTurn = contents[2];
+		expect(toolResultTurn.parts).toHaveLength(3);
+		const imageResponse = toolResultTurn.parts?.[1]?.functionResponse;
+		expect(imageResponse).toBeTruthy();
+		expect(imageResponse?.parts).toHaveLength(1);
+		expect(imageResponse?.parts?.[0]?.inlineData).toBeTruthy();
+	});
+
+	it("keeps separate synthetic image turn for Gemini 2.x Cloud Code Assist models", () => {
+		const model = makeModel("google-gemini-cli", "google-gemini-cli", "gemini-2.5-flash");
+		const contents = convertMessages(model, makeContext(model));
+
+		expect(contents).toHaveLength(5);
+		expect(contents[3].parts?.[0]?.text).toBe("Tool result image:");
+		expect(contents[3].parts?.[1]?.inlineData).toBeTruthy();
+	});
+});