shumengya/sproutclaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ai): keep image tool results inline for gemini 3+ and antigravity closes #2052

Browse Source
This commit is contained in:
Mario Zechner
2026-03-18 02:17:44 +01:00
parent 2becbbdff3
commit 453b22397d
2 changed files with 145 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
packages/ai/src/providers/google-shared.ts
View File

@@ -70,6 +70,20 @@ export function requiresToolCallId(modelId: string): boolean {
return modelId.startsWith("claude-") || modelId.startsWith("gpt-oss-");
}
function getGeminiMajorVersion(modelId: string): number | undefined {
const match = modelId.toLowerCase().match(/^gemini(?:-live)?-(\d+)/);
if (!match) return undefined;
return Number.parseInt(match[1], 10);
}
function supportsMultimodalFunctionResponse(modelId: string): boolean {
const geminiMajorVersion = getGeminiMajorVersion(modelId);
if (geminiMajorVersion !== undefined) {
return geminiMajorVersion >= 3;
}
return true;
}
/**
* Convert internal messages to Gemini Content[] format.
*/
@@ -175,10 +189,10 @@ export function convertMessages<T extends GoogleApiType>(model: Model<T>, contex
const hasText = textResult.length > 0;
const hasImages = imageContent.length > 0;
// Gemini 3 supports multimodal function responses with images nested inside functionResponse.parts
// See: https://ai.google.dev/gemini-api/docs/function-calling#multimodal
// Older models don't support this, so we put images in a separate user message.
const supportsMultimodalFunctionResponse = model.id.includes("gemini-3");
// Gemini 3+ models support multimodal function responses with images nested inside
// functionResponse.parts. Claude and other non-Gemini models behind Cloud Code Assist /
// Antigravity also accept this shape. Gemini < 3 still needs a separate user image turn.
const modelSupportsMultimodalFunctionResponse = supportsMultimodalFunctionResponse(model.id);
// Use "output" key for success, "error" key for errors as per SDK documentation
const responseValue = hasText ? sanitizeSurrogates(textResult) : hasImages ? "(see attached image)" : "";
@@ -195,8 +209,7 @@ export function convertMessages<T extends GoogleApiType>(model: Model<T>, contex
functionResponse: {
name: msg.toolName,
response: msg.isError ? { error: responseValue } : { output: responseValue },
// Nest images inside functionResponse.parts for Gemini 3
...(hasImages && supportsMultimodalFunctionResponse && { parts: imageParts }),
...(hasImages && modelSupportsMultimodalFunctionResponse && { parts: imageParts }),
...(includeId ? { id: msg.toolCallId } : {}),
},
};
@@ -213,8 +226,8 @@ export function convertMessages<T extends GoogleApiType>(model: Model<T>, contex
});
}
// For older models, add images in a separate user message
if (hasImages && !supportsMultimodalFunctionResponse) {
// For Gemini < 3, add images in a separate user message
if (hasImages && !modelSupportsMultimodalFunctionResponse) {
contents.push({
role: "user",
parts: [{ text: "Tool result image:" }, ...imageParts],