shumengya/sproutclaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ai,coding-agent): preserve non-vision image placeholders closes #3429

Browse Source
This commit is contained in:
Mario Zechner
2026-04-20 16:52:27 +02:00
parent 74139c3f66
commit 2f4f283cc2
11 changed files with 184 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/ai/CHANGELOG.md
View File

@@ -4,8 +4,10 @@
### Fixed ### Fixed
- Fixed non-vision model requests to replace user and tool-result image blocks with explicit text placeholders instead of silently dropping them during provider payload conversion ([#3429](https://github.com/badlogic/pi-mono/issues/3429))
- Fixed OpenRouter Meta tests by switching `meta-llama/llama-4-maverick` to `meta-llama/llama-4-scout` to avoid type-check failures from model-catalog drift. - Fixed OpenRouter Meta tests by switching `meta-llama/llama-4-maverick` to `meta-llama/llama-4-scout` to avoid type-check failures from model-catalog drift.
- Fixed direct OpenAI Chat Completions requests to map `sessionId` and `cacheRetention` to OpenAI prompt caching fields, sending `prompt_cache_key` when caching is enabled and `prompt_cache_retention: "24h"` for direct `api.openai.com` requests with long retention ([#3426](https://github.com/badlogic/pi-mono/issues/3426)) - Fixed direct OpenAI Chat Completions requests to map `sessionId` and `cacheRetention` to OpenAI prompt caching fields, sending `prompt_cache_key` when caching is enabled and `prompt_cache_retention: "24h"` for direct `api.openai.com` requests with long retention ([#3426](https://github.com/badlogic/pi-mono/issues/3426))
- Fixed OpenAI-compatible Chat Completions requests to optionally send aligned `session_id`, `x-client-request-id`, and `x-session-affinity` session-affinity headers from `sessionId` via `compat.sendSessionAffinityHeaders`, enabling cache-affinity routing for backends such as Fireworks ([#3430](https://github.com/badlogic/pi-mono/issues/3430))
## [0.67.68] - 2026-04-17 ## [0.67.68] - 2026-04-17

3
packages/ai/src/providers/anthropic.ts
View File

@@ -783,8 +783,7 @@ function convertMessages(
}; };
} }
}); });
let filteredBlocks = !model?.input.includes("image") ? blocks.filter((b) => b.type !== "image") : blocks; const filteredBlocks = blocks.filter((b) => {
filteredBlocks = filteredBlocks.filter((b) => {
if (b.type === "text") { if (b.type === "text") {
return b.text.trim().length > 0; return b.text.trim().length > 0;
} }

5
packages/ai/src/providers/google-shared.ts
View File

@@ -116,11 +116,10 @@ export function convertMessages<T extends GoogleApiType>(model: Model<T>, contex
}; };
} }
}); });
const filteredParts = !model.input.includes("image") ? parts.filter((p) => p.text !== undefined) : parts; if (parts.length === 0) continue;
if (filteredParts.length === 0) continue;
contents.push({ contents.push({
role: "user", role: "user",
parts: filteredParts, parts,
}); });
} }
} else if (msg.role === "assistant") { } else if (msg.role === "assistant") {

34
packages/ai/src/providers/openai-completions.ts
View File

@@ -97,8 +97,11 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenA
try { try {
const apiKey = options?.apiKey || getEnvApiKey(model.provider) || ""; const apiKey = options?.apiKey || getEnvApiKey(model.provider) || "";
const client = createClient(model, context, apiKey, options?.headers); const compat = getCompat(model);
let params = buildParams(model, context, options); const cacheRetention = resolveCacheRetention(options?.cacheRetention);
const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId;
const client = createClient(model, context, apiKey, options?.headers, cacheSessionId, compat);
let params = buildParams(model, context, options, compat, cacheRetention);
const nextParams = await options?.onPayload?.(params, model); const nextParams = await options?.onPayload?.(params, model);
if (nextParams !== undefined) { if (nextParams !== undefined) {
params = nextParams as OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming; params = nextParams as OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming;
@@ -350,6 +353,8 @@ function createClient(
context: Context, context: Context,
apiKey?: string, apiKey?: string,
optionsHeaders?: Record<string, string>, optionsHeaders?: Record<string, string>,
sessionId?: string,
compat: Required<OpenAICompletionsCompat> = getCompat(model),
) { ) {
if (!apiKey) { if (!apiKey) {
if (!process.env.OPENAI_API_KEY) { if (!process.env.OPENAI_API_KEY) {
@@ -370,6 +375,12 @@ function createClient(
Object.assign(headers, copilotHeaders); Object.assign(headers, copilotHeaders);
} }
if (sessionId && compat.sendSessionAffinityHeaders) {
headers.session_id = sessionId;
headers["x-client-request-id"] = sessionId;
headers["x-session-affinity"] = sessionId;
}
// Merge options headers last so they can override defaults // Merge options headers last so they can override defaults
if (optionsHeaders) { if (optionsHeaders) {
Object.assign(headers, optionsHeaders); Object.assign(headers, optionsHeaders);
@@ -383,12 +394,16 @@ function createClient(
}); });
} }
function buildParams(model: Model<"openai-completions">, context: Context, options?: OpenAICompletionsOptions) { function buildParams(
const compat = getCompat(model); model: Model<"openai-completions">,
context: Context,
options?: OpenAICompletionsOptions,
compat: Required<OpenAICompletionsCompat> = getCompat(model),
cacheRetention: CacheRetention = resolveCacheRetention(options?.cacheRetention),
) {
const messages = convertMessages(model, context, compat); const messages = convertMessages(model, context, compat);
maybeAddOpenRouterAnthropicCacheControl(model, messages); maybeAddOpenRouterAnthropicCacheControl(model, messages);
const cacheRetention = resolveCacheRetention(options?.cacheRetention);
const params: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { const params: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = {
model: model.id, model: model.id,
messages, messages,
@@ -580,13 +595,10 @@ export function convertMessages(
} satisfies ChatCompletionContentPartImage; } satisfies ChatCompletionContentPartImage;
} }
}); });
const filteredContent = !model.input.includes("image") if (content.length === 0) continue;
? content.filter((c) => c.type !== "image_url")
: content;
if (filteredContent.length === 0) continue;
params.push({ params.push({
role: "user", role: "user",
content: filteredContent, content,
}); });
} }
} else if (msg.role === "assistant") { } else if (msg.role === "assistant") {
@@ -878,6 +890,7 @@ function detectCompat(model: Model<"openai-completions">): Required<OpenAIComple
vercelGatewayRouting: {}, vercelGatewayRouting: {},
zaiToolStream: false, zaiToolStream: false,
supportsStrictMode: true, supportsStrictMode: true,
sendSessionAffinityHeaders: false,
}; };
} }
@@ -905,5 +918,6 @@ function getCompat(model: Model<"openai-completions">): Required<OpenAICompletio
vercelGatewayRouting: model.compat.vercelGatewayRouting ?? detected.vercelGatewayRouting, vercelGatewayRouting: model.compat.vercelGatewayRouting ?? detected.vercelGatewayRouting,
zaiToolStream: model.compat.zaiToolStream ?? detected.zaiToolStream, zaiToolStream: model.compat.zaiToolStream ?? detected.zaiToolStream,
supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode, supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode,
sendSessionAffinityHeaders: model.compat.sendSessionAffinityHeaders ?? detected.sendSessionAffinityHeaders,
}; };
} }

7
packages/ai/src/providers/openai-responses-shared.ts
View File

@@ -153,13 +153,10 @@ export function convertResponsesMessages<TApi extends Api>(
image_url: `data:${item.mimeType};base64,${item.data}`, image_url: `data:${item.mimeType};base64,${item.data}`,
} satisfies ResponseInputImage; } satisfies ResponseInputImage;
}); });
const filteredContent = !model.input.includes("image") if (content.length === 0) continue;
? content.filter((c) => c.type !== "input_image")
: content;
if (filteredContent.length === 0) continue;
messages.push({ messages.push({
role: "user", role: "user",
content: filteredContent, content,
}); });
} }
} else if (msg.role === "assistant") { } else if (msg.role === "assistant") {

63
packages/ai/src/providers/transform-messages.ts
View File

@@ -1,4 +1,60 @@
import type { Api, AssistantMessage, Message, Model, ToolCall, ToolResultMessage } from "../types.js"; import type {
Api,
AssistantMessage,
ImageContent,
Message,
Model,
TextContent,
ToolCall,
ToolResultMessage,
} from "../types.js";
const NON_VISION_USER_IMAGE_PLACEHOLDER = "(image omitted: model does not support images)";
const NON_VISION_TOOL_IMAGE_PLACEHOLDER = "(tool image omitted: model does not support images)";
function replaceImagesWithPlaceholder(content: (TextContent | ImageContent)[], placeholder: string): TextContent[] {
const result: TextContent[] = [];
let previousWasPlaceholder = false;
for (const block of content) {
if (block.type === "image") {
if (!previousWasPlaceholder) {
result.push({ type: "text", text: placeholder });
}
previousWasPlaceholder = true;
continue;
}
result.push(block);
previousWasPlaceholder = block.text === placeholder;
}
return result;
}
function downgradeUnsupportedImages<TApi extends Api>(messages: Message[], model: Model<TApi>): Message[] {
if (model.input.includes("image")) {
return messages;
}
return messages.map((msg) => {
if (msg.role === "user" && Array.isArray(msg.content)) {
return {
...msg,
content: replaceImagesWithPlaceholder(msg.content, NON_VISION_USER_IMAGE_PLACEHOLDER),
};
}
if (msg.role === "toolResult") {
return {
...msg,
content: replaceImagesWithPlaceholder(msg.content, NON_VISION_TOOL_IMAGE_PLACEHOLDER),
};
}
return msg;
});
}
/** /**
* Normalize tool call ID for cross-provider compatibility. * Normalize tool call ID for cross-provider compatibility.
@@ -12,9 +68,10 @@ export function transformMessages<TApi extends Api>(
): Message[] { ): Message[] {
// Build a map of original tool call IDs to normalized IDs // Build a map of original tool call IDs to normalized IDs
const toolCallIdMap = new Map<string, string>(); const toolCallIdMap = new Map<string, string>();
const imageAwareMessages = downgradeUnsupportedImages(messages, model);
// First pass: transform messages (thinking blocks, tool call ID normalization) // First pass: transform messages (unsupported image downgrade, thinking blocks, tool call ID normalization)
const transformed = messages.map((msg) => { const transformed = imageAwareMessages.map((msg) => {
// User messages pass through unchanged // User messages pass through unchanged
if (msg.role === "user") { if (msg.role === "user") {
return msg; return msg;

1
packages/coding-agent/CHANGELOG.md
View File

@@ -9,6 +9,7 @@
### Fixed ### Fixed
- Fixed built-in tool wrapping to use the same extension-runner context path as extension tools, so built-in tools receive execution context and `read` can warn when the current model does not support images ([#3429](https://github.com/badlogic/pi-mono/issues/3429))
- Fixed threaded `/resume` session relationships and current-session detection to canonicalize symlinked session paths during selector comparisons, so shared session directories no longer break parent-child matching or active-session delete protection ([#3364](https://github.com/badlogic/pi-mono/issues/3364)) - Fixed threaded `/resume` session relationships and current-session detection to canonicalize symlinked session paths during selector comparisons, so shared session directories no longer break parent-child matching or active-session delete protection ([#3364](https://github.com/badlogic/pi-mono/issues/3364))
- Fixed `/session`, Sessions docs, and CLI help to consistently document that session reuse supports both file paths and session IDs, and that `/session` shows the current session ID ([#3390](https://github.com/badlogic/pi-mono/issues/3390)) - Fixed `/session`, Sessions docs, and CLI help to consistently document that session reuse supports both file paths and session IDs, and that `/session` shows the current session ID ([#3390](https://github.com/badlogic/pi-mono/issues/3390))
- Fixed Windows pnpm global install detection to recognize `\\.pnpm\\` store paths, so update notices now suggest `pnpm install -g @mariozechner/pi-coding-agent` instead of falling back to npm ([#3378](https://github.com/badlogic/pi-mono/issues/3378)) - Fixed Windows pnpm global install detection to recognize `\\.pnpm\\` store paths, so update notices now suggest `pnpm install -g @mariozechner/pi-coding-agent` instead of falling back to npm ([#3378](https://github.com/badlogic/pi-mono/issues/3378))

4
packages/coding-agent/src/core/agent-session-runtime.ts
View File

@@ -98,7 +98,7 @@ export class AgentSessionRuntime {
targetSessionFile?: string, targetSessionFile?: string,
): Promise<{ cancelled: boolean }> { ): Promise<{ cancelled: boolean }> {
const runner = this.session.extensionRunner; const runner = this.session.extensionRunner;
if (!runner?.hasHandlers("session_before_switch")) { if (!runner.hasHandlers("session_before_switch")) {
return { cancelled: false }; return { cancelled: false };
} }
@@ -115,7 +115,7 @@ export class AgentSessionRuntime {
options: { position: "before" | "at" }, options: { position: "before" | "at" },
): Promise<{ cancelled: boolean }> { ): Promise<{ cancelled: boolean }> {
const runner = this.session.extensionRunner; const runner = this.session.extensionRunner;
if (!runner?.hasHandlers("session_before_fork")) { if (!runner.hasHandlers("session_before_fork")) {
return { cancelled: false }; return { cancelled: false };
} }

155
packages/coding-agent/src/core/agent-session.ts
View File

@@ -79,7 +79,7 @@ import { createSyntheticSourceInfo, type SourceInfo } from "./source-info.js";
import { buildSystemPrompt } from "./system-prompt.js"; import { buildSystemPrompt } from "./system-prompt.js";
import { type BashOperations, createLocalBashOperations } from "./tools/bash.js"; import { type BashOperations, createLocalBashOperations } from "./tools/bash.js";
import { createAllToolDefinitions } from "./tools/index.js"; import { createAllToolDefinitions } from "./tools/index.js";
import { createToolDefinitionFromAgentTool, wrapToolDefinition } from "./tools/tool-definition-wrapper.js"; import { createToolDefinitionFromAgentTool } from "./tools/tool-definition-wrapper.js";
// ============================================================================ // ============================================================================
// Skill Block Parsing // Skill Block Parsing
@@ -269,7 +269,7 @@ export class AgentSession {
private _pendingBashMessages: BashExecutionMessage[] = []; private _pendingBashMessages: BashExecutionMessage[] = [];
// Extension system // Extension system
private _extensionRunner: ExtensionRunner | undefined = undefined; private _extensionRunner!: ExtensionRunner;
private _turnIndex = 0; private _turnIndex = 0;
private _resourceLoader: ResourceLoader; private _resourceLoader: ResourceLoader;
@@ -365,7 +365,7 @@ export class AgentSession {
private _installAgentToolHooks(): void { private _installAgentToolHooks(): void {
this.agent.beforeToolCall = async ({ toolCall, args }) => { this.agent.beforeToolCall = async ({ toolCall, args }) => {
const runner = this._extensionRunner; const runner = this._extensionRunner;
if (!runner?.hasHandlers("tool_call")) { if (!runner.hasHandlers("tool_call")) {
return undefined; return undefined;
} }
@@ -388,7 +388,7 @@ export class AgentSession {
this.agent.afterToolCall = async ({ toolCall, args, result, isError }) => { this.agent.afterToolCall = async ({ toolCall, args, result, isError }) => {
const runner = this._extensionRunner; const runner = this._extensionRunner;
if (!runner?.hasHandlers("tool_result")) { if (!runner.hasHandlers("tool_result")) {
return undefined; return undefined;
} }
@@ -604,8 +604,6 @@ export class AgentSession {
/** Emit extension events based on agent events */ /** Emit extension events based on agent events */
private async _emitExtensionEvent(event: AgentEvent): Promise<void> { private async _emitExtensionEvent(event: AgentEvent): Promise<void> {
if (!this._extensionRunner) return;
if (event.type === "agent_start") { if (event.type === "agent_start") {
this._turnIndex = 0; this._turnIndex = 0;
await this._extensionRunner.emit({ type: "agent_start" }); await this._extensionRunner.emit({ type: "agent_start" });
@@ -949,7 +947,7 @@ export class AgentSession {
// Emit input event for extension interception (before skill/template expansion) // Emit input event for extension interception (before skill/template expansion)
let currentText = text; let currentText = text;
let currentImages = options?.images; let currentImages = options?.images;
if (this._extensionRunner?.hasHandlers("input")) { if (this._extensionRunner.hasHandlers("input")) {
const inputResult = await this._extensionRunner.emitInput( const inputResult = await this._extensionRunner.emitInput(
currentText, currentText,
currentImages, currentImages,
@@ -1042,33 +1040,31 @@ export class AgentSession {
this._pendingNextTurnMessages = []; this._pendingNextTurnMessages = [];
// Emit before_agent_start extension event // Emit before_agent_start extension event
if (this._extensionRunner) { const result = await this._extensionRunner.emitBeforeAgentStart(
const result = await this._extensionRunner.emitBeforeAgentStart( expandedText,
expandedText, currentImages,
currentImages, this._baseSystemPrompt,
this._baseSystemPrompt, );
); // Add all custom messages from extensions
// Add all custom messages from extensions if (result?.messages) {
if (result?.messages) { for (const msg of result.messages) {
for (const msg of result.messages) { messages.push({
messages.push({ role: "custom",
role: "custom", customType: msg.customType,
customType: msg.customType, content: msg.content,
content: msg.content, display: msg.display,
display: msg.display, details: msg.details,
details: msg.details, timestamp: Date.now(),
timestamp: Date.now(), });
});
}
}
// Apply extension-modified system prompt, or reset to base
if (result?.systemPrompt) {
this.agent.state.systemPrompt = result.systemPrompt;
} else {
// Ensure we're using the base prompt (in case previous turn had modifications)
this.agent.state.systemPrompt = this._baseSystemPrompt;
} }
} }
// Apply extension-modified system prompt, or reset to base
if (result?.systemPrompt) {
this.agent.state.systemPrompt = result.systemPrompt;
} else {
// Ensure we're using the base prompt (in case previous turn had modifications)
this.agent.state.systemPrompt = this._baseSystemPrompt;
}
} catch (error) { } catch (error) {
preflightResult?.(false); preflightResult?.(false);
throw error; throw error;
@@ -1087,8 +1083,6 @@ export class AgentSession {
* Try to execute an extension command. Returns true if command was found and executed. * Try to execute an extension command. Returns true if command was found and executed.
*/ */
private async _tryExecuteExtensionCommand(text: string): Promise<boolean> { private async _tryExecuteExtensionCommand(text: string): Promise<boolean> {
if (!this._extensionRunner) return false;
// Parse command name and args // Parse command name and args
const spaceIndex = text.indexOf(" "); const spaceIndex = text.indexOf(" ");
const commandName = spaceIndex === -1 ? text.slice(1) : text.slice(1, spaceIndex); const commandName = spaceIndex === -1 ? text.slice(1) : text.slice(1, spaceIndex);
@@ -1136,7 +1130,7 @@ export class AgentSession {
return args ? `${skillBlock}\n\n${args}` : skillBlock; return args ? `${skillBlock}\n\n${args}` : skillBlock;
} catch (err) { } catch (err) {
// Emit error like extension commands do // Emit error like extension commands do
this._extensionRunner?.emitError({ this._extensionRunner.emitError({
extensionPath: skill.filePath, extensionPath: skill.filePath,
event: "skill_expansion", event: "skill_expansion",
error: err instanceof Error ? err.message : String(err), error: err instanceof Error ? err.message : String(err),
@@ -1224,8 +1218,6 @@ export class AgentSession {
* Throw an error if the text is an extension command. * Throw an error if the text is an extension command.
*/ */
private _throwIfExtensionCommand(text: string): void { private _throwIfExtensionCommand(text: string): void {
if (!this._extensionRunner) return;
const spaceIndex = text.indexOf(" "); const spaceIndex = text.indexOf(" ");
const commandName = spaceIndex === -1 ? text.slice(1) : text.slice(1, spaceIndex); const commandName = spaceIndex === -1 ? text.slice(1) : text.slice(1, spaceIndex);
const command = this._extensionRunner.getCommand(commandName); const command = this._extensionRunner.getCommand(commandName);
@@ -1376,7 +1368,6 @@ export class AgentSession {
previousModel: Model<any> | undefined, previousModel: Model<any> | undefined,
source: "set" | "cycle" | "restore", source: "set" | "cycle" | "restore",
): Promise<void> { ): Promise<void> {
if (!this._extensionRunner) return;
if (modelsAreEqual(previousModel, nextModel)) return; if (modelsAreEqual(previousModel, nextModel)) return;
await this._extensionRunner.emit({ await this._extensionRunner.emit({
type: "model_select", type: "model_select",
@@ -1628,7 +1619,7 @@ export class AgentSession {
let extensionCompaction: CompactionResult | undefined; let extensionCompaction: CompactionResult | undefined;
let fromExtension = false; let fromExtension = false;
if (this._extensionRunner?.hasHandlers("session_before_compact")) { if (this._extensionRunner.hasHandlers("session_before_compact")) {
const result = (await this._extensionRunner.emit({ const result = (await this._extensionRunner.emit({
type: "session_before_compact", type: "session_before_compact",
preparation, preparation,
@@ -1886,7 +1877,7 @@ export class AgentSession {
let extensionCompaction: CompactionResult | undefined; let extensionCompaction: CompactionResult | undefined;
let fromExtension = false; let fromExtension = false;
if (this._extensionRunner?.hasHandlers("session_before_compact")) { if (this._extensionRunner.hasHandlers("session_before_compact")) {
const extensionResult = (await this._extensionRunner.emit({ const extensionResult = (await this._extensionRunner.emit({
type: "session_before_compact", type: "session_before_compact",
preparation, preparation,
@@ -2038,15 +2029,13 @@ export class AgentSession {
this._extensionErrorListener = bindings.onError; this._extensionErrorListener = bindings.onError;
} }
if (this._extensionRunner) { this._applyExtensionBindings(this._extensionRunner);
this._applyExtensionBindings(this._extensionRunner); await this._extensionRunner.emit(this._sessionStartEvent);
await this._extensionRunner.emit(this._sessionStartEvent); await this.extendResourcesFromExtensions(this._sessionStartEvent.reason === "reload" ? "reload" : "startup");
await this.extendResourcesFromExtensions(this._sessionStartEvent.reason === "reload" ? "reload" : "startup");
}
} }
private async extendResourcesFromExtensions(reason: "startup" | "reload"): Promise<void> { private async extendResourcesFromExtensions(reason: "startup" | "reload"): Promise<void> {
if (!this._extensionRunner?.hasHandlers("resources_discover")) { if (!this._extensionRunner.hasHandlers("resources_discover")) {
return; return;
} }
@@ -2233,7 +2222,7 @@ export class AgentSession {
const previousRegistryNames = new Set(this._toolRegistry.keys()); const previousRegistryNames = new Set(this._toolRegistry.keys());
const previousActiveToolNames = this.getActiveToolNames(); const previousActiveToolNames = this.getActiveToolNames();
const registeredTools = this._extensionRunner?.getAllRegisteredTools() ?? []; const registeredTools = this._extensionRunner.getAllRegisteredTools();
const allCustomTools = [ const allCustomTools = [
...registeredTools, ...registeredTools,
...this._customTools.map((definition) => ({ ...this._customTools.map((definition) => ({
@@ -2273,16 +2262,17 @@ export class AgentSession {
}) })
.filter((entry): entry is readonly [string, string[]] => entry !== undefined), .filter((entry): entry is readonly [string, string[]] => entry !== undefined),
); );
const wrappedExtensionTools = this._extensionRunner const runner = this._extensionRunner;
? wrapRegisteredTools(allCustomTools, this._extensionRunner) const wrappedExtensionTools = wrapRegisteredTools(allCustomTools, runner);
: []; const wrappedBuiltInTools = wrapRegisteredTools(
Array.from(this._baseToolDefinitions.values()).map((definition) => ({
const toolRegistry = new Map( definition,
Array.from(this._baseToolDefinitions.values()).map((definition) => [ sourceInfo: createSyntheticSourceInfo(`<builtin:${definition.name}>`, { source: "builtin" }),
definition.name, })),
wrapToolDefinition(definition), runner,
]),
); );
const toolRegistry = new Map(wrappedBuiltInTools.map((tool) => [tool.name, tool]));
for (const tool of wrappedExtensionTools as AgentTool[]) { for (const tool of wrappedExtensionTools as AgentTool[]) {
toolRegistry.set(tool.name, tool); toolRegistry.set(tool.name, tool);
} }
@@ -2337,25 +2327,18 @@ export class AgentSession {
} }
} }
const hasExtensions = extensionsResult.extensions.length > 0; this._extensionRunner = new ExtensionRunner(
const hasCustomTools = this._customTools.length > 0; extensionsResult.extensions,
this._extensionRunner = extensionsResult.runtime,
hasExtensions || hasCustomTools this._cwd,
? new ExtensionRunner( this.sessionManager,
extensionsResult.extensions, this._modelRegistry,
extensionsResult.runtime, );
this._cwd,
this.sessionManager,
this._modelRegistry,
)
: undefined;
if (this._extensionRunnerRef) { if (this._extensionRunnerRef) {
this._extensionRunnerRef.current = this._extensionRunner; this._extensionRunnerRef.current = this._extensionRunner;
} }
if (this._extensionRunner) { this._bindExtensionCore(this._extensionRunner);
this._bindExtensionCore(this._extensionRunner); this._applyExtensionBindings(this._extensionRunner);
this._applyExtensionBindings(this._extensionRunner);
}
const defaultActiveToolNames = this._baseToolsOverride const defaultActiveToolNames = this._baseToolsOverride
? Object.keys(this._baseToolsOverride) ? Object.keys(this._baseToolsOverride)
@@ -2368,8 +2351,8 @@ export class AgentSession {
} }
async reload(): Promise<void> { async reload(): Promise<void> {
const previousFlagValues = this._extensionRunner?.getFlagValues(); const previousFlagValues = this._extensionRunner.getFlagValues();
await this._extensionRunner?.emit({ type: "session_shutdown" }); await this._extensionRunner.emit({ type: "session_shutdown" });
await this.settingsManager.reload(); await this.settingsManager.reload();
resetApiProviders(); resetApiProviders();
await this._resourceLoader.reload(); await this._resourceLoader.reload();
@@ -2384,7 +2367,7 @@ export class AgentSession {
this._extensionCommandContextActions || this._extensionCommandContextActions ||
this._extensionShutdownHandler || this._extensionShutdownHandler ||
this._extensionErrorListener; this._extensionErrorListener;
if (this._extensionRunner && hasBindings) { if (hasBindings) {
await this._extensionRunner.emit({ type: "session_start", reason: "reload" }); await this._extensionRunner.emit({ type: "session_start", reason: "reload" });
await this.extendResourcesFromExtensions("reload"); await this.extendResourcesFromExtensions("reload");
} }
@@ -2713,7 +2696,7 @@ export class AgentSession {
let fromExtension = false; let fromExtension = false;
// Emit session_before_tree event // Emit session_before_tree event
if (this._extensionRunner?.hasHandlers("session_before_tree")) { if (this._extensionRunner.hasHandlers("session_before_tree")) {
const result = (await this._extensionRunner.emit({ const result = (await this._extensionRunner.emit({
type: "session_before_tree", type: "session_before_tree",
preparation, preparation,
@@ -2827,15 +2810,13 @@ export class AgentSession {
this.agent.state.messages = sessionContext.messages; this.agent.state.messages = sessionContext.messages;
// Emit session_tree event // Emit session_tree event
if (this._extensionRunner) { await this._extensionRunner.emit({
await this._extensionRunner.emit({ type: "session_tree",
type: "session_tree", newLeafId: this.sessionManager.getLeafId(),
newLeafId: this.sessionManager.getLeafId(), oldLeafId,
oldLeafId, summaryEntry,
summaryEntry, fromExtension: summaryText ? fromExtension : undefined,
fromExtension: summaryText ? fromExtension : undefined, });
});
}
// Emit to custom tools // Emit to custom tools
@@ -3067,13 +3048,13 @@ export class AgentSession {
* Check if extensions have handlers for a specific event type. * Check if extensions have handlers for a specific event type.
*/ */
hasExtensionHandlers(eventType: string): boolean { hasExtensionHandlers(eventType: string): boolean {
return this._extensionRunner?.hasHandlers(eventType) ?? false; return this._extensionRunner.hasHandlers(eventType);
} }
/** /**
* Get the extension runner (for setting UI context and error handlers). * Get the extension runner (for setting UI context and error handlers).
*/ */
get extensionRunner(): ExtensionRunner | undefined { get extensionRunner(): ExtensionRunner {
return this._extensionRunner; return this._extensionRunner;
} }
} }

7
packages/coding-agent/src/core/extensions/wrapper.ts
View File

@@ -6,7 +6,7 @@
*/ */
import type { AgentTool } from "@mariozechner/pi-agent-core"; import type { AgentTool } from "@mariozechner/pi-agent-core";
import { wrapToolDefinition } from "../tools/tool-definition-wrapper.js"; import { wrapToolDefinition, wrapToolDefinitions } from "../tools/tool-definition-wrapper.js";
import type { ExtensionRunner } from "./runner.js"; import type { ExtensionRunner } from "./runner.js";
import type { RegisteredTool } from "./types.js"; import type { RegisteredTool } from "./types.js";
@@ -23,5 +23,8 @@ export function wrapRegisteredTool(registeredTool: RegisteredTool, runner: Exten
* Uses the runner's createContext() for consistent context across tools and event handlers. * Uses the runner's createContext() for consistent context across tools and event handlers.
*/ */
export function wrapRegisteredTools(registeredTools: RegisteredTool[], runner: ExtensionRunner): AgentTool[] { export function wrapRegisteredTools(registeredTools: RegisteredTool[], runner: ExtensionRunner): AgentTool[] {
return registeredTools.map((rt) => wrapRegisteredTool(rt, runner)); return wrapToolDefinitions(
registeredTools.map((registeredTool) => registeredTool.definition),
() => runner.createContext(),
);
} }

26
packages/coding-agent/src/core/tools/read.ts
View File

@@ -1,5 +1,5 @@
import type { AgentTool } from "@mariozechner/pi-agent-core"; import type { AgentTool } from "@mariozechner/pi-agent-core";
import type { ImageContent, TextContent } from "@mariozechner/pi-ai"; import type { Api, ImageContent, Model, TextContent } from "@mariozechner/pi-ai";
import { Text } from "@mariozechner/pi-tui"; import { Text } from "@mariozechner/pi-tui";
import { type Static, Type } from "@sinclair/typebox"; import { type Static, Type } from "@sinclair/typebox";
import { constants } from "fs"; import { constants } from "fs";
@@ -78,6 +78,13 @@ function trimTrailingEmptyLines(lines: string[]): string[] {
return lines.slice(0, end); return lines.slice(0, end);
} }
function getNonVisionImageNote(model: Model<Api> | undefined): string | undefined {
if (!model || model.input.includes("image")) {
return undefined;
}
return "[Current model does not support images. The image will be omitted from this request.]";
}
function formatReadResult( function formatReadResult(
args: { path?: string; file_path?: string; offset?: number; limit?: number } | undefined, args: { path?: string; file_path?: string; offset?: number; limit?: number } | undefined,
result: { content: (TextContent | ImageContent)[]; details?: ReadToolDetails }, result: { content: (TextContent | ImageContent)[]; details?: ReadToolDetails },
@@ -129,7 +136,7 @@ export function createReadToolDefinition(
{ path, offset, limit }: { path: string; offset?: number; limit?: number }, { path, offset, limit }: { path: string; offset?: number; limit?: number },
signal?: AbortSignal, signal?: AbortSignal,
_onUpdate?, _onUpdate?,
_ctx?, ctx?,
) { ) {
const absolutePath = resolveReadPath(path, cwd); const absolutePath = resolveReadPath(path, cwd);
return new Promise<{ content: (TextContent | ImageContent)[]; details: ReadToolDetails | undefined }>( return new Promise<{ content: (TextContent | ImageContent)[]; details: ReadToolDetails | undefined }>(
@@ -153,6 +160,7 @@ export function createReadToolDefinition(
const mimeType = ops.detectImageMimeType ? await ops.detectImageMimeType(absolutePath) : undefined; const mimeType = ops.detectImageMimeType ? await ops.detectImageMimeType(absolutePath) : undefined;
let content: (TextContent | ImageContent)[]; let content: (TextContent | ImageContent)[];
let details: ReadToolDetails | undefined; let details: ReadToolDetails | undefined;
const nonVisionImageNote = getNonVisionImageNote(ctx?.model);
if (mimeType) { if (mimeType) {
// Read image as binary. // Read image as binary.
const buffer = await ops.readFile(absolutePath); const buffer = await ops.readFile(absolutePath);
@@ -161,24 +169,24 @@ export function createReadToolDefinition(
// Resize image if needed before sending it back to the model. // Resize image if needed before sending it back to the model.
const resized = await resizeImage({ type: "image", data: base64, mimeType }); const resized = await resizeImage({ type: "image", data: base64, mimeType });
if (!resized) { if (!resized) {
content = [ let textNote = `Read image file [${mimeType}]\n[Image omitted: could not be resized below the inline image size limit.]`;
{ if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`;
type: "text", content = [{ type: "text", text: textNote }];
text: `Read image file [${mimeType}]\n[Image omitted: could not be resized below the inline image size limit.]`,
},
];
} else { } else {
const dimensionNote = formatDimensionNote(resized); const dimensionNote = formatDimensionNote(resized);
let textNote = `Read image file [${resized.mimeType}]`; let textNote = `Read image file [${resized.mimeType}]`;
if (dimensionNote) textNote += `\n${dimensionNote}`; if (dimensionNote) textNote += `\n${dimensionNote}`;
if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`;
content = [ content = [
{ type: "text", text: textNote }, { type: "text", text: textNote },
{ type: "image", data: resized.data, mimeType: resized.mimeType }, { type: "image", data: resized.data, mimeType: resized.mimeType },
]; ];
} }
} else { } else {
let textNote = `Read image file [${mimeType}]`;
if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`;
content = [ content = [
{ type: "text", text: `Read image file [${mimeType}]` }, { type: "text", text: textNote },
{ type: "image", data: base64, mimeType }, { type: "image", data: base64, mimeType },
]; ];
} }