Add image support in tool results across all providers
Tool results now use content blocks and can include both text and images. All providers (Anthropic, Google, OpenAI Completions, OpenAI Responses) correctly pass images from tool results to LLMs. - Update ToolResultMessage type to use content blocks - Add placeholder text for image-only tool results in Google/Anthropic - OpenAI providers send tool result + follow-up user message with images - Fix Anthropic JSON parsing for empty tool arguments - Add comprehensive tests for image-only and text+image tool results - Update README with tool result content blocks API
This commit is contained in:
@@ -62,7 +62,10 @@ async function testEmojiInToolResults<TApi extends Api>(llm: Model<TApi>, option
|
||||
role: "toolResult",
|
||||
toolCallId: "test_1",
|
||||
toolName: "test_tool",
|
||||
output: `Test with emoji 🙈 and other characters:
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: `Test with emoji 🙈 and other characters:
|
||||
- Monkey emoji: 🙈
|
||||
- Thumbs up: 👍
|
||||
- Heart: ❤️
|
||||
@@ -73,6 +76,8 @@ async function testEmojiInToolResults<TApi extends Api>(llm: Model<TApi>, option
|
||||
- Chinese: 你好
|
||||
- Mathematical symbols: ∑∫∂√
|
||||
- Special quotes: "curly" 'quotes'`,
|
||||
},
|
||||
],
|
||||
isError: false,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
@@ -141,7 +146,10 @@ async function testRealWorldLinkedInData<TApi extends Api>(llm: Model<TApi>, opt
|
||||
role: "toolResult",
|
||||
toolCallId: "linkedin_1",
|
||||
toolName: "linkedin_skill",
|
||||
output: `Post: Hab einen "Generative KI für Nicht-Techniker" Workshop gebaut.
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: `Post: Hab einen "Generative KI für Nicht-Techniker" Workshop gebaut.
|
||||
Unanswered Comments: 2
|
||||
|
||||
=> {
|
||||
@@ -156,6 +164,8 @@ Unanswered Comments: 2
|
||||
}
|
||||
]
|
||||
}`,
|
||||
},
|
||||
],
|
||||
isError: false,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
@@ -226,7 +236,7 @@ async function testUnpairedHighSurrogate<TApi extends Api>(llm: Model<TApi>, opt
|
||||
role: "toolResult",
|
||||
toolCallId: "test_2",
|
||||
toolName: "test_tool",
|
||||
output: `Text with unpaired surrogate: ${unpairedSurrogate} <- should be sanitized`,
|
||||
content: [{ type: "text", text: `Text with unpaired surrogate: ${unpairedSurrogate} <- should be sanitized` }],
|
||||
isError: false,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user