From 453b22397d1eca31c999d11856639c0c310efe13 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 18 Mar 2026 02:17:44 +0100 Subject: [PATCH] fix(ai): keep image tool results inline for gemini 3+ and antigravity closes #2052 --- packages/ai/src/providers/google-shared.ts | 29 ++-- ...e-shared-image-tool-result-routing.test.ts | 124 ++++++++++++++++++ 2 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 packages/ai/test/google-shared-image-tool-result-routing.test.ts diff --git a/packages/ai/src/providers/google-shared.ts b/packages/ai/src/providers/google-shared.ts index e942314f..57b59238 100644 --- a/packages/ai/src/providers/google-shared.ts +++ b/packages/ai/src/providers/google-shared.ts @@ -70,6 +70,20 @@ export function requiresToolCallId(modelId: string): boolean { return modelId.startsWith("claude-") || modelId.startsWith("gpt-oss-"); } +function getGeminiMajorVersion(modelId: string): number | undefined { + const match = modelId.toLowerCase().match(/^gemini(?:-live)?-(\d+)/); + if (!match) return undefined; + return Number.parseInt(match[1], 10); +} + +function supportsMultimodalFunctionResponse(modelId: string): boolean { + const geminiMajorVersion = getGeminiMajorVersion(modelId); + if (geminiMajorVersion !== undefined) { + return geminiMajorVersion >= 3; + } + return true; +} + /** * Convert internal messages to Gemini Content[] format. */ @@ -175,10 +189,10 @@ export function convertMessages(model: Model, contex const hasText = textResult.length > 0; const hasImages = imageContent.length > 0; - // Gemini 3 supports multimodal function responses with images nested inside functionResponse.parts - // See: https://ai.google.dev/gemini-api/docs/function-calling#multimodal - // Older models don't support this, so we put images in a separate user message. - const supportsMultimodalFunctionResponse = model.id.includes("gemini-3"); + // Gemini 3+ models support multimodal function responses with images nested inside + // functionResponse.parts. Claude and other non-Gemini models behind Cloud Code Assist / + // Antigravity also accept this shape. Gemini < 3 still needs a separate user image turn. + const modelSupportsMultimodalFunctionResponse = supportsMultimodalFunctionResponse(model.id); // Use "output" key for success, "error" key for errors as per SDK documentation const responseValue = hasText ? sanitizeSurrogates(textResult) : hasImages ? "(see attached image)" : ""; @@ -195,8 +209,7 @@ export function convertMessages(model: Model, contex functionResponse: { name: msg.toolName, response: msg.isError ? { error: responseValue } : { output: responseValue }, - // Nest images inside functionResponse.parts for Gemini 3 - ...(hasImages && supportsMultimodalFunctionResponse && { parts: imageParts }), + ...(hasImages && modelSupportsMultimodalFunctionResponse && { parts: imageParts }), ...(includeId ? { id: msg.toolCallId } : {}), }, }; @@ -213,8 +226,8 @@ export function convertMessages(model: Model, contex }); } - // For older models, add images in a separate user message - if (hasImages && !supportsMultimodalFunctionResponse) { + // For Gemini < 3, add images in a separate user message + if (hasImages && !modelSupportsMultimodalFunctionResponse) { contents.push({ role: "user", parts: [{ text: "Tool result image:" }, ...imageParts], diff --git a/packages/ai/test/google-shared-image-tool-result-routing.test.ts b/packages/ai/test/google-shared-image-tool-result-routing.test.ts new file mode 100644 index 00000000..c3a991d5 --- /dev/null +++ b/packages/ai/test/google-shared-image-tool-result-routing.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from "vitest"; +import { convertMessages } from "../src/providers/google-shared.js"; +import type { Context, Model } from "../src/types.js"; + +function makeModel( + api: TApi, + provider: Model["provider"], + id: string, +): Model { + return { + id, + name: id, + api, + provider, + baseUrl: "https://example.com", + reasoning: true, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 8192, + }; +} + +function makeContext(model: { api: string; provider: string; id: string }): Context { + const now = Date.now(); + return { + messages: [ + { role: "user", content: "read the files", timestamp: now }, + { + role: "assistant", + content: [ + { type: "toolCall", id: "call_a", name: "read", arguments: { path: "a.txt" } }, + { type: "toolCall", id: "call_img", name: "read", arguments: { path: "image.png" } }, + { type: "toolCall", id: "call_b", name: "read", arguments: { path: "b.txt" } }, + ], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp: now, + }, + { + role: "toolResult", + toolCallId: "call_a", + toolName: "read", + content: [{ type: "text", text: "alpha text" }], + isError: false, + timestamp: now, + }, + { + role: "toolResult", + toolCallId: "call_img", + toolName: "read", + content: [{ type: "image", data: "abc", mimeType: "image/png" }], + isError: false, + timestamp: now, + }, + { + role: "toolResult", + toolCallId: "call_b", + toolName: "read", + content: [{ type: "text", text: "beta text" }], + isError: false, + timestamp: now, + }, + ], + }; +} + +describe("google-shared image tool result routing", () => { + it("keeps separate synthetic image turn for Gemini 2.x Google API models", () => { + const model = makeModel("google-generative-ai", "google", "gemini-2.5-flash"); + const contents = convertMessages(model, makeContext(model)); + + expect(contents).toHaveLength(5); + expect(contents[2].parts?.every((part) => part.functionResponse)).toBe(true); + expect(contents[3].parts?.[0]?.text).toBe("Tool result image:"); + expect(contents[3].parts?.[1]?.inlineData).toBeTruthy(); + expect(contents[4].parts?.[0]?.functionResponse).toBeTruthy(); + }); + + it("nests image tool results for Gemini 3 Google API models", () => { + const model = makeModel("google-generative-ai", "google", "gemini-3-pro-preview"); + const contents = convertMessages(model, makeContext(model)); + + expect(contents).toHaveLength(3); + const toolResultTurn = contents[2]; + expect(toolResultTurn.parts).toHaveLength(3); + const imageResponse = toolResultTurn.parts?.[1]?.functionResponse; + expect(imageResponse).toBeTruthy(); + expect(imageResponse?.parts).toHaveLength(1); + expect(imageResponse?.parts?.[0]?.inlineData).toBeTruthy(); + }); + + it("nests image tool results for non-Gemini models on Antigravity / Cloud Code Assist", () => { + const model = makeModel("google-gemini-cli", "google-antigravity", "claude-sonnet-4-6"); + const contents = convertMessages(model, makeContext(model)); + + expect(contents).toHaveLength(3); + const toolResultTurn = contents[2]; + expect(toolResultTurn.parts).toHaveLength(3); + const imageResponse = toolResultTurn.parts?.[1]?.functionResponse; + expect(imageResponse).toBeTruthy(); + expect(imageResponse?.parts).toHaveLength(1); + expect(imageResponse?.parts?.[0]?.inlineData).toBeTruthy(); + }); + + it("keeps separate synthetic image turn for Gemini 2.x Cloud Code Assist models", () => { + const model = makeModel("google-gemini-cli", "google-gemini-cli", "gemini-2.5-flash"); + const contents = convertMessages(model, makeContext(model)); + + expect(contents).toHaveLength(5); + expect(contents[3].parts?.[0]?.text).toBe("Tool result image:"); + expect(contents[3].parts?.[1]?.inlineData).toBeTruthy(); + }); +});