From c3c10737d86f76ec61fc46ccc957eda883010d6d Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Tue, 28 Apr 2026 13:43:28 +0200 Subject: [PATCH 01/19] feat: image content --- packages/agent/test/e2e.test.ts | 7 +- packages/ai/CHANGELOG.md | 2 + packages/ai/scripts/generate-models.ts | 26 +++-- packages/ai/src/providers/faux.ts | 12 +- packages/ai/src/providers/google.ts | 38 +++++++ packages/ai/src/providers/mistral.ts | 3 + .../ai/src/providers/openai-completions.ts | 96 +++++++++++++++- packages/ai/src/types.ts | 6 +- packages/ai/test/google-image-output.test.ts | 103 ++++++++++++++++++ .../openai-completions-image-output.test.ts | 98 +++++++++++++++++ ...penai-completions-thinking-as-text.test.ts | 1 + ...nai-completions-tool-result-images.test.ts | 1 + .../coding-agent/src/core/session-manager.ts | 6 +- .../2860-replaced-session-context.test.ts | 13 +-- 14 files changed, 385 insertions(+), 27 deletions(-) create mode 100644 packages/ai/test/google-image-output.test.ts create mode 100644 packages/ai/test/openai-completions-image-output.test.ts diff --git a/packages/agent/test/e2e.test.ts b/packages/agent/test/e2e.test.ts index f032555e..052d327b 100644 --- a/packages/agent/test/e2e.test.ts +++ b/packages/agent/test/e2e.test.ts @@ -7,6 +7,7 @@ import { fauxToolCall, type Model, registerFauxProvider, + type TextContent, type ToolResultMessage, type UserMessage, } from "@mariozechner/pi-ai"; @@ -23,10 +24,8 @@ function createFauxRegistration(options: Parameters } function getTextContent(message: AssistantMessage | ToolResultMessage): string { - return message.content - .filter((block) => block.type === "text") - .map((block) => block.text) - .join("\n"); + const textBlocks = message.content.filter((block): block is TextContent => block.type === "text") as TextContent[]; + return textBlocks.map((block) => block.text).join("\n"); } afterEach(() => { diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 8050ec83..9f0e60f0 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -15,6 +15,8 @@ ### Added - Added Azure Cognitive Services endpoint support for Azure OpenAI Responses base URLs ([#3799](https://github.com/badlogic/pi-mono/pull/3799) by [@marcbloech](https://github.com/marcbloech)). +- Added OpenRouter image-generation support by registering Flux and Gemini image-preview models, accepting OpenRouter image-output models from discovery, and parsing generated image payloads from OpenAI-compatible completions responses. +- Added Google streaming to surface inline image output parts as assistant image content blocks and `image_start`/`image_end` events. ### Changed diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 2c848ae4..242bdeaf 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -88,20 +88,29 @@ function getBedrockBaseUrl(modelId: string): string { async function fetchOpenRouterModels(): Promise[]> { try { console.log("Fetching models from OpenRouter API..."); - const response = await fetch("https://openrouter.ai/api/v1/models"); - const data = await response.json(); + const [generalResponse, imageResponse] = await Promise.all([ + fetch("https://openrouter.ai/api/v1/models"), + fetch("https://openrouter.ai/api/v1/models?output_modalities=image"), + ]); + const generalData = await generalResponse.json(); + const imageData = await imageResponse.json(); + + const combinedModels = new Map(); + for (const model of [...generalData.data, ...imageData.data]) { + combinedModels.set(model.id, model); + } const models: Model[] = []; - for (const model of data.data) { - // Only include models that support tools - if (!model.supported_parameters?.includes("tools")) continue; - // Parse provider from model ID let provider: KnownProvider = "openrouter"; let modelKey = model.id; modelKey = model.id; // Keep full ID for OpenRouter + for (const model of combinedModels.values()) { + const supportsTools = model.supported_parameters?.includes("tools"); + const supportsImageOutput = model.architecture?.output_modalities?.includes("image"); + if (!supportsTools && !supportsImageOutput) continue; // Parse input modalities const input: ("text" | "image")[] = ["text"]; @@ -123,6 +132,7 @@ async function fetchOpenRouterModels(): Promise[]> { provider, reasoning: model.supported_parameters?.includes("reasoning") || false, input, + compat: supportsImageOutput ? { openRouterImageGeneration: true } : undefined, cost: { input: inputCost, output: outputCost, @@ -130,12 +140,12 @@ async function fetchOpenRouterModels(): Promise[]> { cacheWrite: cacheWriteCost, }, contextWindow: model.context_length || 4096, - maxTokens: model.top_provider?.max_completion_tokens || 4096, + maxTokens: model.top_provider?.max_completion_tokens || model.context_length || 4096, }; models.push(normalizedModel); } - console.log(`Fetched ${models.length} tool-capable models from OpenRouter`); + console.log(`Fetched ${models.length} tool-capable or image-generation models from OpenRouter`); return models; } catch (error) { console.error("Failed to fetch OpenRouter models:", error); diff --git a/packages/ai/src/providers/faux.ts b/packages/ai/src/providers/faux.ts index 7bba1b72..8c0277b0 100644 --- a/packages/ai/src/providers/faux.ts +++ b/packages/ai/src/providers/faux.ts @@ -147,7 +147,7 @@ function contentToText(content: string | Array): str .join("\n"); } -function assistantContentToText(content: Array): string { +function assistantContentToText(content: Array): string { return content .map((block) => { if (block.type === "text") { @@ -156,6 +156,9 @@ function assistantContentToText(content: Array } } + if (part.inlineData) { + if (currentBlock) { + if (currentBlock.type === "text") { + stream.push({ + type: "text_end", + contentIndex: blockIndex(), + content: currentBlock.text, + partial: output, + }); + } else { + stream.push({ + type: "thinking_end", + contentIndex: blockIndex(), + content: currentBlock.thinking, + partial: output, + }); + } + currentBlock = null; + } + + if (part.inlineData.data && part.inlineData.mimeType) { + const imageBlock: ImageContent = { + type: "image", + data: part.inlineData.data, + mimeType: part.inlineData.mimeType, + }; + output.content.push(imageBlock); + stream.push({ type: "image_start", contentIndex: blockIndex(), partial: output }); + stream.push({ + type: "image_end", + contentIndex: blockIndex(), + image: imageBlock, + partial: output, + }); + } + } + if (part.functionCall) { if (currentBlock) { if (currentBlock.type === "text") { diff --git a/packages/ai/src/providers/mistral.ts b/packages/ai/src/providers/mistral.ts index 68affc98..501b1f53 100644 --- a/packages/ai/src/providers/mistral.ts +++ b/packages/ai/src/providers/mistral.ts @@ -524,6 +524,9 @@ function toChatMessages(messages: Message[], supportsImages: boolean): ChatCompl } continue; } + if (block.type !== "toolCall") { + continue; + } toolCalls.push({ id: block.id, type: "function", diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index f0e167e9..ffe4e28e 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -1,5 +1,6 @@ import OpenAI from "openai"; import type { + ChatCompletion, ChatCompletionAssistantMessageParam, ChatCompletionChunk, ChatCompletionContentPart, @@ -98,6 +99,26 @@ type ChatCompletionToolWithCacheControl = OpenAI.Chat.Completions.ChatCompletion cache_control?: OpenAICompatCacheControl; }; +interface OpenRouterGeneratedImage { + image_url?: string | { url?: string }; +} + +type OpenRouterImageGenerationMessage = ChatCompletion["choices"][number]["message"] & { + images?: OpenRouterGeneratedImage[]; +}; + +type OpenRouterImageGenerationChoice = ChatCompletion["choices"][number] & { + message: OpenRouterImageGenerationMessage; +}; + +type OpenRouterImageGenerationResponse = ChatCompletion & { + choices: OpenRouterImageGenerationChoice[]; +}; + +function isOpenRouterImageGenerationModel(model: Model<"openai-completions">): boolean { + return model.provider === "openrouter" && model.compat?.openRouterImageGeneration === true; +} + function resolveCacheRetention(cacheRetention?: CacheRetention): CacheRetention { if (cacheRetention) { return cacheRetention; @@ -150,6 +171,75 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenA ...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}), ...(options?.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}), }; + if (isOpenRouterImageGenerationModel(model)) { + const { stream_options: _streamOptions, ...nonStreamingBaseParams } = params; + const nonStreamingParams = { + ...nonStreamingBaseParams, + stream: false, + } as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming; + const { data: response, response: rawResponse } = await client.chat.completions + .create(nonStreamingParams, requestOptions) + .withResponse(); + await options?.onResponse?.( + { status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, + model, + ); + stream.push({ type: "start", partial: output }); + + const imageResponse = response as OpenRouterImageGenerationResponse; + output.responseId ||= imageResponse.id; + output.usage = parseChunkUsage(imageResponse.usage ?? {}, model); + + const choice = imageResponse.choices[0]; + if (choice) { + const text = choice.message.content; + if (text) { + const textBlock: TextContent = { type: "text", text }; + output.content.push(textBlock); + const contentIndex = output.content.length - 1; + stream.push({ type: "text_start", contentIndex, partial: output }); + stream.push({ type: "text_delta", contentIndex, delta: text, partial: output }); + stream.push({ type: "text_end", contentIndex, content: text, partial: output }); + } + + for (const image of choice.message.images ?? []) { + const imageUrl = typeof image.image_url === "string" ? image.image_url : image.image_url?.url; + if (!imageUrl?.startsWith("data:")) { + continue; + } + const matches = imageUrl.match(/^data:([^;]+);base64,(.+)$/); + if (!matches) { + continue; + } + const imageBlock: ImageContent = { + type: "image", + mimeType: matches[1], + data: matches[2], + }; + output.content.push(imageBlock); + const contentIndex = output.content.length - 1; + stream.push({ type: "image_start", contentIndex, partial: output }); + stream.push({ type: "image_end", contentIndex, image: imageBlock, partial: output }); + } + + if (choice.finish_reason) { + const finishReasonResult = mapStopReason(choice.finish_reason); + output.stopReason = finishReasonResult.stopReason; + if (finishReasonResult.errorMessage) { + output.errorMessage = finishReasonResult.errorMessage; + } + } + } + + if (output.stopReason === "stop" || output.stopReason === "length" || output.stopReason === "toolUse") { + stream.push({ type: "done", reason: output.stopReason, message: output }); + } else { + stream.push({ type: "error", reason: output.stopReason, error: output }); + } + stream.end(); + return; + } + const { data: openaiStream, response } = await client.chat.completions .create(params, requestOptions) .withResponse(); @@ -505,12 +595,12 @@ function buildParams( params.temperature = options.temperature; } - if (context.tools && context.tools.length > 0) { + if (!isOpenRouterImageGenerationModel(model) && context.tools && context.tools.length > 0) { params.tools = convertTools(context.tools, compat); if (compat.zaiToolStream) { (params as any).tool_stream = true; } - } else if (hasToolHistory(context.messages)) { + } else if (!isOpenRouterImageGenerationModel(model) && hasToolHistory(context.messages)) { // Anthropic (via LiteLLM/proxy) requires tools param when conversation has tool_calls/tool_results params.tools = []; } @@ -1080,6 +1170,7 @@ function detectCompat(model: Model<"openai-completions">): ResolvedOpenAIComplet ? "openrouter" : "openai", openRouterRouting: {}, + openRouterImageGeneration: false, vercelGatewayRouting: {}, zaiToolStream: false, supportsStrictMode: true, @@ -1113,6 +1204,7 @@ function getCompat(model: Model<"openai-completions">): ResolvedOpenAICompletion detected.requiresReasoningContentOnAssistantMessages, thinkingFormat: model.compat.thinkingFormat ?? detected.thinkingFormat, openRouterRouting: model.compat.openRouterRouting ?? {}, + openRouterImageGeneration: model.compat.openRouterImageGeneration ?? detected.openRouterImageGeneration, vercelGatewayRouting: model.compat.vercelGatewayRouting ?? detected.vercelGatewayRouting, zaiToolStream: model.compat.zaiToolStream ?? detected.zaiToolStream, supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode, diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 8ffd785f..9a4c7a9f 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -212,7 +212,7 @@ export interface UserMessage { export interface AssistantMessage { role: "assistant"; - content: (TextContent | ThinkingContent | ToolCall)[]; + content: (TextContent | ThinkingContent | ImageContent | ToolCall)[]; api: Api; provider: Provider; model: string; @@ -265,6 +265,8 @@ export type AssistantMessageEvent = | { type: "thinking_start"; contentIndex: number; partial: AssistantMessage } | { type: "thinking_delta"; contentIndex: number; delta: string; partial: AssistantMessage } | { type: "thinking_end"; contentIndex: number; content: string; partial: AssistantMessage } + | { type: "image_start"; contentIndex: number; partial: AssistantMessage } + | { type: "image_end"; contentIndex: number; image: ImageContent; partial: AssistantMessage } | { type: "toolcall_start"; contentIndex: number; partial: AssistantMessage } | { type: "toolcall_delta"; contentIndex: number; delta: string; partial: AssistantMessage } | { type: "toolcall_end"; contentIndex: number; toolCall: ToolCall; partial: AssistantMessage } @@ -300,6 +302,8 @@ export interface OpenAICompletionsCompat { thinkingFormat?: "openai" | "openrouter" | "deepseek" | "zai" | "qwen" | "qwen-chat-template"; /** OpenRouter-specific routing preferences. Only used when baseUrl points to OpenRouter. */ openRouterRouting?: OpenRouterRouting; + /** Whether the model uses OpenRouter's image-generation response shape with assistant `message.images`. */ + openRouterImageGeneration?: boolean; /** Vercel AI Gateway routing preferences. Only used when baseUrl points to Vercel AI Gateway. */ vercelGatewayRouting?: VercelGatewayRouting; /** Whether z.ai supports top-level `tool_stream: true` for streaming tool call deltas. Default: false. */ diff --git a/packages/ai/test/google-image-output.test.ts b/packages/ai/test/google-image-output.test.ts new file mode 100644 index 00000000..13563093 --- /dev/null +++ b/packages/ai/test/google-image-output.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@google/genai", () => { + class GoogleGenAI { + models = { + generateContentStream: async function* () { + yield { + responseId: "google-image-1", + candidates: [ + { + content: { + parts: [ + { text: "Here is your image." }, + { inlineData: { mimeType: "image/png", data: "ZmFrZS1wbmc=" } }, + ], + }, + finishReason: "STOP", + }, + ], + usageMetadata: { + promptTokenCount: 5, + candidatesTokenCount: 7, + totalTokenCount: 12, + }, + }; + }, + }; + } + + return { + GoogleGenAI, + FinishReason: { + STOP: "STOP", + MAX_TOKENS: "MAX_TOKENS", + BLOCKLIST: "BLOCKLIST", + PROHIBITED_CONTENT: "PROHIBITED_CONTENT", + SPII: "SPII", + SAFETY: "SAFETY", + IMAGE_SAFETY: "IMAGE_SAFETY", + IMAGE_PROHIBITED_CONTENT: "IMAGE_PROHIBITED_CONTENT", + IMAGE_RECITATION: "IMAGE_RECITATION", + IMAGE_OTHER: "IMAGE_OTHER", + RECITATION: "RECITATION", + FINISH_REASON_UNSPECIFIED: "FINISH_REASON_UNSPECIFIED", + OTHER: "OTHER", + LANGUAGE: "LANGUAGE", + MALFORMED_FUNCTION_CALL: "MALFORMED_FUNCTION_CALL", + UNEXPECTED_TOOL_CALL: "UNEXPECTED_TOOL_CALL", + NO_IMAGE: "NO_IMAGE", + }, + FunctionCallingConfigMode: { + AUTO: "AUTO", + ANY: "ANY", + NONE: "NONE", + }, + ThinkingLevel: { + THINKING_LEVEL_UNSPECIFIED: "THINKING_LEVEL_UNSPECIFIED", + MINIMAL: "MINIMAL", + LOW: "LOW", + MEDIUM: "MEDIUM", + HIGH: "HIGH", + }, + }; +}); + +import { streamGoogle } from "../src/providers/google.js"; +import type { Context, Model } from "../src/types.js"; + +describe("google image output", () => { + it("emits assistant image blocks for inlineData parts", async () => { + const model: Model<"google-generative-ai"> = { + id: "gemini-2.5-flash", + name: "Gemini 2.5 Flash", + api: "google-generative-ai", + provider: "google", + baseUrl: "https://generativelanguage.googleapis.com", + reasoning: false, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1000000, + maxTokens: 8192, + }; + const context: Context = { + messages: [{ role: "user", content: "Generate an image", timestamp: Date.now() }], + }; + + const result = streamGoogle(model, context, { apiKey: "test" }); + const eventTypes: string[] = []; + for await (const event of result) { + eventTypes.push(event.type); + } + + const message = await result.result(); + expect(eventTypes).toEqual(["start", "text_start", "text_delta", "text_end", "image_start", "image_end", "done"]); + expect(message.responseId).toBe("google-image-1"); + expect(message.content[0]).toMatchObject({ type: "text", text: "Here is your image." }); + expect(message.content[1]).toMatchObject({ + type: "image", + mimeType: "image/png", + data: "ZmFrZS1wbmc=", + }); + }); +}); diff --git a/packages/ai/test/openai-completions-image-output.test.ts b/packages/ai/test/openai-completions-image-output.test.ts new file mode 100644 index 00000000..cb9b6b32 --- /dev/null +++ b/packages/ai/test/openai-completions-image-output.test.ts @@ -0,0 +1,98 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { streamSimple } from "../src/stream.js"; +import type { Context, Model } from "../src/types.js"; + +const mockState = vi.hoisted(() => ({ + lastParams: undefined as unknown, +})); + +vi.mock("openai", () => { + class FakeOpenAI { + chat = { + completions: { + create: (params: unknown) => { + mockState.lastParams = params; + const response = { + id: "chatcmpl-image-1", + usage: { + prompt_tokens: 12, + completion_tokens: 34, + prompt_tokens_details: { cached_tokens: 0 }, + }, + choices: [ + { + finish_reason: "stop", + message: { + role: "assistant", + content: "Here is your image.", + images: [ + { + image_url: "data:image/png;base64,ZmFrZS1wbmc=", + }, + ], + }, + }, + ], + }; + const promise = Promise.resolve(response) as Promise & { + withResponse: () => Promise<{ + data: typeof response; + response: { status: number; headers: Headers }; + }>; + }; + promise.withResponse = async () => ({ + data: response, + response: { status: 200, headers: new Headers() }, + }); + return promise; + }, + }, + }; + } + + return { default: FakeOpenAI }; +}); + +describe("openai-completions image output", () => { + beforeEach(() => { + mockState.lastParams = undefined; + }); + + it("switches OpenRouter image generation models to non-streaming and emits image events", async () => { + const model: Model<"openai-completions"> = { + id: "black-forest-labs/flux.2-pro", + name: "Black Forest Labs: FLUX.2 Pro", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: false, + input: ["text", "image"], + cost: { input: 0.015, output: 0.03, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 100000, + maxTokens: 100000, + }; + const context: Context = { + messages: [{ role: "user", content: "Generate a cat", timestamp: Date.now() }], + }; + + const result = streamSimple(model, context, { apiKey: "test" }); + const eventTypes: string[] = []; + for await (const event of result) { + eventTypes.push(event.type); + } + + const message = await result.result(); + expect(eventTypes).toEqual(["start", "text_start", "text_delta", "text_end", "image_start", "image_end", "done"]); + expect(message.responseId).toBe("chatcmpl-image-1"); + expect(message.content[0]).toMatchObject({ type: "text", text: "Here is your image." }); + expect(message.content[1]).toMatchObject({ + type: "image", + mimeType: "image/png", + data: "ZmFrZS1wbmc=", + }); + + const params = mockState.lastParams as { stream?: boolean; stream_options?: unknown }; + expect(params.stream).toBe(false); + expect("stream_options" in (params as object)).toBe(false); + }); +}); diff --git a/packages/ai/test/openai-completions-thinking-as-text.test.ts b/packages/ai/test/openai-completions-thinking-as-text.test.ts index 24425f3a..012e434b 100644 --- a/packages/ai/test/openai-completions-thinking-as-text.test.ts +++ b/packages/ai/test/openai-completions-thinking-as-text.test.ts @@ -34,6 +34,7 @@ const compat = { requiresReasoningContentOnAssistantMessages: false, thinkingFormat: "openai", openRouterRouting: {}, + openRouterImageGeneration: false, vercelGatewayRouting: {}, zaiToolStream: false, supportsStrictMode: true, diff --git a/packages/ai/test/openai-completions-tool-result-images.test.ts b/packages/ai/test/openai-completions-tool-result-images.test.ts index ac1eb28a..8154c9ac 100644 --- a/packages/ai/test/openai-completions-tool-result-images.test.ts +++ b/packages/ai/test/openai-completions-tool-result-images.test.ts @@ -32,6 +32,7 @@ const compat: Required = { requiresReasoningContentOnAssistantMessages: false, thinkingFormat: "openai", openRouterRouting: {}, + openRouterImageGeneration: false, vercelGatewayRouting: {}, zaiToolStream: false, supportsStrictMode: true, diff --git a/packages/coding-agent/src/core/session-manager.ts b/packages/coding-agent/src/core/session-manager.ts index f366274b..26299b2b 100644 --- a/packages/coding-agent/src/core/session-manager.ts +++ b/packages/coding-agent/src/core/session-manager.ts @@ -502,10 +502,8 @@ function extractTextContent(message: Message): string { if (typeof content === "string") { return content; } - return content - .filter((block): block is TextContent => block.type === "text") - .map((block) => block.text) - .join(" "); + const textBlocks = content.filter((block): block is TextContent => block.type === "text") as TextContent[]; + return textBlocks.map((block) => block.text).join(" "); } function getLastActivityTime(entries: FileEntry[]): number | undefined { diff --git a/packages/coding-agent/test/suite/regressions/2860-replaced-session-context.test.ts b/packages/coding-agent/test/suite/regressions/2860-replaced-session-context.test.ts index 626365c6..678d6400 100644 --- a/packages/coding-agent/test/suite/regressions/2860-replaced-session-context.test.ts +++ b/packages/coding-agent/test/suite/regressions/2860-replaced-session-context.test.ts @@ -1,7 +1,7 @@ import { existsSync, mkdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { fauxAssistantMessage, registerFauxProvider } from "@mariozechner/pi-ai"; +import { fauxAssistantMessage, registerFauxProvider, type TextContent } from "@mariozechner/pi-ai"; import { afterEach, describe, expect, it } from "vitest"; import type { AgentSession } from "../../../src/core/agent-session.js"; import { @@ -18,12 +18,11 @@ function getText(message: AgentSession["messages"][number]): string { if (!("content" in message)) { return ""; } - return typeof message.content === "string" - ? message.content - : message.content - .filter((part): part is { type: "text"; text: string } => part.type === "text") - .map((part) => part.text) - .join(""); + if (typeof message.content === "string") { + return message.content; + } + const textParts = message.content.filter((part): part is TextContent => part.type === "text") as TextContent[]; + return textParts.map((part) => part.text).join(""); } describe("regression #2860: replaced session callbacks", () => { From 364ac0f3ab2e115bf5d10ac6918cce3376452a9f Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Tue, 28 Apr 2026 14:40:13 +0200 Subject: [PATCH 02/19] fix: test (openRouterImageGeneration) --- packages/ai/test/openai-completions-image-output.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ai/test/openai-completions-image-output.test.ts b/packages/ai/test/openai-completions-image-output.test.ts index cb9b6b32..2cf1b7fb 100644 --- a/packages/ai/test/openai-completions-image-output.test.ts +++ b/packages/ai/test/openai-completions-image-output.test.ts @@ -65,6 +65,7 @@ describe("openai-completions image output", () => { api: "openai-completions", provider: "openrouter", baseUrl: "https://openrouter.ai/api/v1", + compat: { openRouterImageGeneration: true }, reasoning: false, input: ["text", "image"], cost: { input: 0.015, output: 0.03, cacheRead: 0, cacheWrite: 0 }, From e9414b050046fc46cf90bcff31c9aeb830428a61 Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Wed, 29 Apr 2026 09:49:09 +0200 Subject: [PATCH 03/19] fix --- packages/ai/scripts/generate-models.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 242bdeaf..4b1a7d75 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -102,12 +102,12 @@ async function fetchOpenRouterModels(): Promise[]> { const models: Model[] = []; - // Parse provider from model ID - let provider: KnownProvider = "openrouter"; - let modelKey = model.id; - - modelKey = model.id; // Keep full ID for OpenRouter for (const model of combinedModels.values()) { + // OpenRouter model IDs already include the upstream provider prefix. + // Keep the full ID (e.g. "google/gemini-2.5-flash-image"). + const provider: KnownProvider = "openrouter"; + const modelKey = model.id; + const supportsTools = model.supported_parameters?.includes("tools"); const supportsImageOutput = model.architecture?.output_modalities?.includes("image"); if (!supportsTools && !supportsImageOutput) continue; From 59a89e0c1cb315ee41d33da72f3a407b844191ea Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Wed, 29 Apr 2026 09:51:57 +0200 Subject: [PATCH 04/19] fix: modalities from openrouter --- packages/ai/src/providers/openai-completions.ts | 4 ++++ packages/ai/test/openai-completions-image-output.test.ts | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index ffe4e28e..f2d36e7c 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -575,6 +575,10 @@ function buildParams( prompt_cache_retention: cacheRetention === "long" && compat.supportsLongCacheRetention ? "24h" : undefined, }; + if (isOpenRouterImageGenerationModel(model)) { + Reflect.set(params, "modalities", ["image"]); + } + if (compat.supportsUsageInStreaming !== false) { (params as any).stream_options = { include_usage: true }; } diff --git a/packages/ai/test/openai-completions-image-output.test.ts b/packages/ai/test/openai-completions-image-output.test.ts index 2cf1b7fb..745305c4 100644 --- a/packages/ai/test/openai-completions-image-output.test.ts +++ b/packages/ai/test/openai-completions-image-output.test.ts @@ -92,8 +92,9 @@ describe("openai-completions image output", () => { data: "ZmFrZS1wbmc=", }); - const params = mockState.lastParams as { stream?: boolean; stream_options?: unknown }; + const params = mockState.lastParams as { stream?: boolean; stream_options?: unknown; modalities?: string[] }; expect(params.stream).toBe(false); expect("stream_options" in (params as object)).toBe(false); + expect(params.modalities).toEqual(["image"]); }); }); From cbf3c333ef3e7e5827030cb744f6d75a92726f28 Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Mon, 4 May 2026 15:43:45 +0200 Subject: [PATCH 05/19] revert --- packages/agent/test/e2e.test.ts | 7 +- packages/ai/CHANGELOG.md | 2 - packages/ai/scripts/generate-models.ts | 34 ++---- packages/ai/src/providers/faux.ts | 12 +- packages/ai/src/providers/google.ts | 38 ------- packages/ai/src/providers/mistral.ts | 3 - .../ai/src/providers/openai-completions.ts | 100 +---------------- packages/ai/src/types.ts | 6 +- packages/ai/test/google-image-output.test.ts | 103 ------------------ .../openai-completions-image-output.test.ts | 100 ----------------- ...penai-completions-thinking-as-text.test.ts | 1 - ...nai-completions-tool-result-images.test.ts | 1 - .../coding-agent/src/core/session-manager.ts | 6 +- .../2860-replaced-session-context.test.ts | 13 ++- 14 files changed, 31 insertions(+), 395 deletions(-) delete mode 100644 packages/ai/test/google-image-output.test.ts delete mode 100644 packages/ai/test/openai-completions-image-output.test.ts diff --git a/packages/agent/test/e2e.test.ts b/packages/agent/test/e2e.test.ts index 052d327b..f032555e 100644 --- a/packages/agent/test/e2e.test.ts +++ b/packages/agent/test/e2e.test.ts @@ -7,7 +7,6 @@ import { fauxToolCall, type Model, registerFauxProvider, - type TextContent, type ToolResultMessage, type UserMessage, } from "@mariozechner/pi-ai"; @@ -24,8 +23,10 @@ function createFauxRegistration(options: Parameters } function getTextContent(message: AssistantMessage | ToolResultMessage): string { - const textBlocks = message.content.filter((block): block is TextContent => block.type === "text") as TextContent[]; - return textBlocks.map((block) => block.text).join("\n"); + return message.content + .filter((block) => block.type === "text") + .map((block) => block.text) + .join("\n"); } afterEach(() => { diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 9f0e60f0..8050ec83 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -15,8 +15,6 @@ ### Added - Added Azure Cognitive Services endpoint support for Azure OpenAI Responses base URLs ([#3799](https://github.com/badlogic/pi-mono/pull/3799) by [@marcbloech](https://github.com/marcbloech)). -- Added OpenRouter image-generation support by registering Flux and Gemini image-preview models, accepting OpenRouter image-output models from discovery, and parsing generated image payloads from OpenAI-compatible completions responses. -- Added Google streaming to surface inline image output parts as assistant image content blocks and `image_start`/`image_end` events. ### Changed diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 4b1a7d75..2c848ae4 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -88,29 +88,20 @@ function getBedrockBaseUrl(modelId: string): string { async function fetchOpenRouterModels(): Promise[]> { try { console.log("Fetching models from OpenRouter API..."); - const [generalResponse, imageResponse] = await Promise.all([ - fetch("https://openrouter.ai/api/v1/models"), - fetch("https://openrouter.ai/api/v1/models?output_modalities=image"), - ]); - const generalData = await generalResponse.json(); - const imageData = await imageResponse.json(); - - const combinedModels = new Map(); - for (const model of [...generalData.data, ...imageData.data]) { - combinedModels.set(model.id, model); - } + const response = await fetch("https://openrouter.ai/api/v1/models"); + const data = await response.json(); const models: Model[] = []; - for (const model of combinedModels.values()) { - // OpenRouter model IDs already include the upstream provider prefix. - // Keep the full ID (e.g. "google/gemini-2.5-flash-image"). - const provider: KnownProvider = "openrouter"; - const modelKey = model.id; + for (const model of data.data) { + // Only include models that support tools + if (!model.supported_parameters?.includes("tools")) continue; - const supportsTools = model.supported_parameters?.includes("tools"); - const supportsImageOutput = model.architecture?.output_modalities?.includes("image"); - if (!supportsTools && !supportsImageOutput) continue; + // Parse provider from model ID + let provider: KnownProvider = "openrouter"; + let modelKey = model.id; + + modelKey = model.id; // Keep full ID for OpenRouter // Parse input modalities const input: ("text" | "image")[] = ["text"]; @@ -132,7 +123,6 @@ async function fetchOpenRouterModels(): Promise[]> { provider, reasoning: model.supported_parameters?.includes("reasoning") || false, input, - compat: supportsImageOutput ? { openRouterImageGeneration: true } : undefined, cost: { input: inputCost, output: outputCost, @@ -140,12 +130,12 @@ async function fetchOpenRouterModels(): Promise[]> { cacheWrite: cacheWriteCost, }, contextWindow: model.context_length || 4096, - maxTokens: model.top_provider?.max_completion_tokens || model.context_length || 4096, + maxTokens: model.top_provider?.max_completion_tokens || 4096, }; models.push(normalizedModel); } - console.log(`Fetched ${models.length} tool-capable or image-generation models from OpenRouter`); + console.log(`Fetched ${models.length} tool-capable models from OpenRouter`); return models; } catch (error) { console.error("Failed to fetch OpenRouter models:", error); diff --git a/packages/ai/src/providers/faux.ts b/packages/ai/src/providers/faux.ts index 8c0277b0..7bba1b72 100644 --- a/packages/ai/src/providers/faux.ts +++ b/packages/ai/src/providers/faux.ts @@ -147,7 +147,7 @@ function contentToText(content: string | Array): str .join("\n"); } -function assistantContentToText(content: Array): string { +function assistantContentToText(content: Array): string { return content .map((block) => { if (block.type === "text") { @@ -156,9 +156,6 @@ function assistantContentToText(content: Array } } - if (part.inlineData) { - if (currentBlock) { - if (currentBlock.type === "text") { - stream.push({ - type: "text_end", - contentIndex: blockIndex(), - content: currentBlock.text, - partial: output, - }); - } else { - stream.push({ - type: "thinking_end", - contentIndex: blockIndex(), - content: currentBlock.thinking, - partial: output, - }); - } - currentBlock = null; - } - - if (part.inlineData.data && part.inlineData.mimeType) { - const imageBlock: ImageContent = { - type: "image", - data: part.inlineData.data, - mimeType: part.inlineData.mimeType, - }; - output.content.push(imageBlock); - stream.push({ type: "image_start", contentIndex: blockIndex(), partial: output }); - stream.push({ - type: "image_end", - contentIndex: blockIndex(), - image: imageBlock, - partial: output, - }); - } - } - if (part.functionCall) { if (currentBlock) { if (currentBlock.type === "text") { diff --git a/packages/ai/src/providers/mistral.ts b/packages/ai/src/providers/mistral.ts index 501b1f53..68affc98 100644 --- a/packages/ai/src/providers/mistral.ts +++ b/packages/ai/src/providers/mistral.ts @@ -524,9 +524,6 @@ function toChatMessages(messages: Message[], supportsImages: boolean): ChatCompl } continue; } - if (block.type !== "toolCall") { - continue; - } toolCalls.push({ id: block.id, type: "function", diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index f2d36e7c..f0e167e9 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -1,6 +1,5 @@ import OpenAI from "openai"; import type { - ChatCompletion, ChatCompletionAssistantMessageParam, ChatCompletionChunk, ChatCompletionContentPart, @@ -99,26 +98,6 @@ type ChatCompletionToolWithCacheControl = OpenAI.Chat.Completions.ChatCompletion cache_control?: OpenAICompatCacheControl; }; -interface OpenRouterGeneratedImage { - image_url?: string | { url?: string }; -} - -type OpenRouterImageGenerationMessage = ChatCompletion["choices"][number]["message"] & { - images?: OpenRouterGeneratedImage[]; -}; - -type OpenRouterImageGenerationChoice = ChatCompletion["choices"][number] & { - message: OpenRouterImageGenerationMessage; -}; - -type OpenRouterImageGenerationResponse = ChatCompletion & { - choices: OpenRouterImageGenerationChoice[]; -}; - -function isOpenRouterImageGenerationModel(model: Model<"openai-completions">): boolean { - return model.provider === "openrouter" && model.compat?.openRouterImageGeneration === true; -} - function resolveCacheRetention(cacheRetention?: CacheRetention): CacheRetention { if (cacheRetention) { return cacheRetention; @@ -171,75 +150,6 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions", OpenA ...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}), ...(options?.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}), }; - if (isOpenRouterImageGenerationModel(model)) { - const { stream_options: _streamOptions, ...nonStreamingBaseParams } = params; - const nonStreamingParams = { - ...nonStreamingBaseParams, - stream: false, - } as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming; - const { data: response, response: rawResponse } = await client.chat.completions - .create(nonStreamingParams, requestOptions) - .withResponse(); - await options?.onResponse?.( - { status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, - model, - ); - stream.push({ type: "start", partial: output }); - - const imageResponse = response as OpenRouterImageGenerationResponse; - output.responseId ||= imageResponse.id; - output.usage = parseChunkUsage(imageResponse.usage ?? {}, model); - - const choice = imageResponse.choices[0]; - if (choice) { - const text = choice.message.content; - if (text) { - const textBlock: TextContent = { type: "text", text }; - output.content.push(textBlock); - const contentIndex = output.content.length - 1; - stream.push({ type: "text_start", contentIndex, partial: output }); - stream.push({ type: "text_delta", contentIndex, delta: text, partial: output }); - stream.push({ type: "text_end", contentIndex, content: text, partial: output }); - } - - for (const image of choice.message.images ?? []) { - const imageUrl = typeof image.image_url === "string" ? image.image_url : image.image_url?.url; - if (!imageUrl?.startsWith("data:")) { - continue; - } - const matches = imageUrl.match(/^data:([^;]+);base64,(.+)$/); - if (!matches) { - continue; - } - const imageBlock: ImageContent = { - type: "image", - mimeType: matches[1], - data: matches[2], - }; - output.content.push(imageBlock); - const contentIndex = output.content.length - 1; - stream.push({ type: "image_start", contentIndex, partial: output }); - stream.push({ type: "image_end", contentIndex, image: imageBlock, partial: output }); - } - - if (choice.finish_reason) { - const finishReasonResult = mapStopReason(choice.finish_reason); - output.stopReason = finishReasonResult.stopReason; - if (finishReasonResult.errorMessage) { - output.errorMessage = finishReasonResult.errorMessage; - } - } - } - - if (output.stopReason === "stop" || output.stopReason === "length" || output.stopReason === "toolUse") { - stream.push({ type: "done", reason: output.stopReason, message: output }); - } else { - stream.push({ type: "error", reason: output.stopReason, error: output }); - } - stream.end(); - return; - } - const { data: openaiStream, response } = await client.chat.completions .create(params, requestOptions) .withResponse(); @@ -575,10 +485,6 @@ function buildParams( prompt_cache_retention: cacheRetention === "long" && compat.supportsLongCacheRetention ? "24h" : undefined, }; - if (isOpenRouterImageGenerationModel(model)) { - Reflect.set(params, "modalities", ["image"]); - } - if (compat.supportsUsageInStreaming !== false) { (params as any).stream_options = { include_usage: true }; } @@ -599,12 +505,12 @@ function buildParams( params.temperature = options.temperature; } - if (!isOpenRouterImageGenerationModel(model) && context.tools && context.tools.length > 0) { + if (context.tools && context.tools.length > 0) { params.tools = convertTools(context.tools, compat); if (compat.zaiToolStream) { (params as any).tool_stream = true; } - } else if (!isOpenRouterImageGenerationModel(model) && hasToolHistory(context.messages)) { + } else if (hasToolHistory(context.messages)) { // Anthropic (via LiteLLM/proxy) requires tools param when conversation has tool_calls/tool_results params.tools = []; } @@ -1174,7 +1080,6 @@ function detectCompat(model: Model<"openai-completions">): ResolvedOpenAIComplet ? "openrouter" : "openai", openRouterRouting: {}, - openRouterImageGeneration: false, vercelGatewayRouting: {}, zaiToolStream: false, supportsStrictMode: true, @@ -1208,7 +1113,6 @@ function getCompat(model: Model<"openai-completions">): ResolvedOpenAICompletion detected.requiresReasoningContentOnAssistantMessages, thinkingFormat: model.compat.thinkingFormat ?? detected.thinkingFormat, openRouterRouting: model.compat.openRouterRouting ?? {}, - openRouterImageGeneration: model.compat.openRouterImageGeneration ?? detected.openRouterImageGeneration, vercelGatewayRouting: model.compat.vercelGatewayRouting ?? detected.vercelGatewayRouting, zaiToolStream: model.compat.zaiToolStream ?? detected.zaiToolStream, supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode, diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 9a4c7a9f..8ffd785f 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -212,7 +212,7 @@ export interface UserMessage { export interface AssistantMessage { role: "assistant"; - content: (TextContent | ThinkingContent | ImageContent | ToolCall)[]; + content: (TextContent | ThinkingContent | ToolCall)[]; api: Api; provider: Provider; model: string; @@ -265,8 +265,6 @@ export type AssistantMessageEvent = | { type: "thinking_start"; contentIndex: number; partial: AssistantMessage } | { type: "thinking_delta"; contentIndex: number; delta: string; partial: AssistantMessage } | { type: "thinking_end"; contentIndex: number; content: string; partial: AssistantMessage } - | { type: "image_start"; contentIndex: number; partial: AssistantMessage } - | { type: "image_end"; contentIndex: number; image: ImageContent; partial: AssistantMessage } | { type: "toolcall_start"; contentIndex: number; partial: AssistantMessage } | { type: "toolcall_delta"; contentIndex: number; delta: string; partial: AssistantMessage } | { type: "toolcall_end"; contentIndex: number; toolCall: ToolCall; partial: AssistantMessage } @@ -302,8 +300,6 @@ export interface OpenAICompletionsCompat { thinkingFormat?: "openai" | "openrouter" | "deepseek" | "zai" | "qwen" | "qwen-chat-template"; /** OpenRouter-specific routing preferences. Only used when baseUrl points to OpenRouter. */ openRouterRouting?: OpenRouterRouting; - /** Whether the model uses OpenRouter's image-generation response shape with assistant `message.images`. */ - openRouterImageGeneration?: boolean; /** Vercel AI Gateway routing preferences. Only used when baseUrl points to Vercel AI Gateway. */ vercelGatewayRouting?: VercelGatewayRouting; /** Whether z.ai supports top-level `tool_stream: true` for streaming tool call deltas. Default: false. */ diff --git a/packages/ai/test/google-image-output.test.ts b/packages/ai/test/google-image-output.test.ts deleted file mode 100644 index 13563093..00000000 --- a/packages/ai/test/google-image-output.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; - -vi.mock("@google/genai", () => { - class GoogleGenAI { - models = { - generateContentStream: async function* () { - yield { - responseId: "google-image-1", - candidates: [ - { - content: { - parts: [ - { text: "Here is your image." }, - { inlineData: { mimeType: "image/png", data: "ZmFrZS1wbmc=" } }, - ], - }, - finishReason: "STOP", - }, - ], - usageMetadata: { - promptTokenCount: 5, - candidatesTokenCount: 7, - totalTokenCount: 12, - }, - }; - }, - }; - } - - return { - GoogleGenAI, - FinishReason: { - STOP: "STOP", - MAX_TOKENS: "MAX_TOKENS", - BLOCKLIST: "BLOCKLIST", - PROHIBITED_CONTENT: "PROHIBITED_CONTENT", - SPII: "SPII", - SAFETY: "SAFETY", - IMAGE_SAFETY: "IMAGE_SAFETY", - IMAGE_PROHIBITED_CONTENT: "IMAGE_PROHIBITED_CONTENT", - IMAGE_RECITATION: "IMAGE_RECITATION", - IMAGE_OTHER: "IMAGE_OTHER", - RECITATION: "RECITATION", - FINISH_REASON_UNSPECIFIED: "FINISH_REASON_UNSPECIFIED", - OTHER: "OTHER", - LANGUAGE: "LANGUAGE", - MALFORMED_FUNCTION_CALL: "MALFORMED_FUNCTION_CALL", - UNEXPECTED_TOOL_CALL: "UNEXPECTED_TOOL_CALL", - NO_IMAGE: "NO_IMAGE", - }, - FunctionCallingConfigMode: { - AUTO: "AUTO", - ANY: "ANY", - NONE: "NONE", - }, - ThinkingLevel: { - THINKING_LEVEL_UNSPECIFIED: "THINKING_LEVEL_UNSPECIFIED", - MINIMAL: "MINIMAL", - LOW: "LOW", - MEDIUM: "MEDIUM", - HIGH: "HIGH", - }, - }; -}); - -import { streamGoogle } from "../src/providers/google.js"; -import type { Context, Model } from "../src/types.js"; - -describe("google image output", () => { - it("emits assistant image blocks for inlineData parts", async () => { - const model: Model<"google-generative-ai"> = { - id: "gemini-2.5-flash", - name: "Gemini 2.5 Flash", - api: "google-generative-ai", - provider: "google", - baseUrl: "https://generativelanguage.googleapis.com", - reasoning: false, - input: ["text", "image"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 1000000, - maxTokens: 8192, - }; - const context: Context = { - messages: [{ role: "user", content: "Generate an image", timestamp: Date.now() }], - }; - - const result = streamGoogle(model, context, { apiKey: "test" }); - const eventTypes: string[] = []; - for await (const event of result) { - eventTypes.push(event.type); - } - - const message = await result.result(); - expect(eventTypes).toEqual(["start", "text_start", "text_delta", "text_end", "image_start", "image_end", "done"]); - expect(message.responseId).toBe("google-image-1"); - expect(message.content[0]).toMatchObject({ type: "text", text: "Here is your image." }); - expect(message.content[1]).toMatchObject({ - type: "image", - mimeType: "image/png", - data: "ZmFrZS1wbmc=", - }); - }); -}); diff --git a/packages/ai/test/openai-completions-image-output.test.ts b/packages/ai/test/openai-completions-image-output.test.ts deleted file mode 100644 index 745305c4..00000000 --- a/packages/ai/test/openai-completions-image-output.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; -import { streamSimple } from "../src/stream.js"; -import type { Context, Model } from "../src/types.js"; - -const mockState = vi.hoisted(() => ({ - lastParams: undefined as unknown, -})); - -vi.mock("openai", () => { - class FakeOpenAI { - chat = { - completions: { - create: (params: unknown) => { - mockState.lastParams = params; - const response = { - id: "chatcmpl-image-1", - usage: { - prompt_tokens: 12, - completion_tokens: 34, - prompt_tokens_details: { cached_tokens: 0 }, - }, - choices: [ - { - finish_reason: "stop", - message: { - role: "assistant", - content: "Here is your image.", - images: [ - { - image_url: "data:image/png;base64,ZmFrZS1wbmc=", - }, - ], - }, - }, - ], - }; - const promise = Promise.resolve(response) as Promise & { - withResponse: () => Promise<{ - data: typeof response; - response: { status: number; headers: Headers }; - }>; - }; - promise.withResponse = async () => ({ - data: response, - response: { status: 200, headers: new Headers() }, - }); - return promise; - }, - }, - }; - } - - return { default: FakeOpenAI }; -}); - -describe("openai-completions image output", () => { - beforeEach(() => { - mockState.lastParams = undefined; - }); - - it("switches OpenRouter image generation models to non-streaming and emits image events", async () => { - const model: Model<"openai-completions"> = { - id: "black-forest-labs/flux.2-pro", - name: "Black Forest Labs: FLUX.2 Pro", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - compat: { openRouterImageGeneration: true }, - reasoning: false, - input: ["text", "image"], - cost: { input: 0.015, output: 0.03, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 100000, - maxTokens: 100000, - }; - const context: Context = { - messages: [{ role: "user", content: "Generate a cat", timestamp: Date.now() }], - }; - - const result = streamSimple(model, context, { apiKey: "test" }); - const eventTypes: string[] = []; - for await (const event of result) { - eventTypes.push(event.type); - } - - const message = await result.result(); - expect(eventTypes).toEqual(["start", "text_start", "text_delta", "text_end", "image_start", "image_end", "done"]); - expect(message.responseId).toBe("chatcmpl-image-1"); - expect(message.content[0]).toMatchObject({ type: "text", text: "Here is your image." }); - expect(message.content[1]).toMatchObject({ - type: "image", - mimeType: "image/png", - data: "ZmFrZS1wbmc=", - }); - - const params = mockState.lastParams as { stream?: boolean; stream_options?: unknown; modalities?: string[] }; - expect(params.stream).toBe(false); - expect("stream_options" in (params as object)).toBe(false); - expect(params.modalities).toEqual(["image"]); - }); -}); diff --git a/packages/ai/test/openai-completions-thinking-as-text.test.ts b/packages/ai/test/openai-completions-thinking-as-text.test.ts index 012e434b..24425f3a 100644 --- a/packages/ai/test/openai-completions-thinking-as-text.test.ts +++ b/packages/ai/test/openai-completions-thinking-as-text.test.ts @@ -34,7 +34,6 @@ const compat = { requiresReasoningContentOnAssistantMessages: false, thinkingFormat: "openai", openRouterRouting: {}, - openRouterImageGeneration: false, vercelGatewayRouting: {}, zaiToolStream: false, supportsStrictMode: true, diff --git a/packages/ai/test/openai-completions-tool-result-images.test.ts b/packages/ai/test/openai-completions-tool-result-images.test.ts index 8154c9ac..ac1eb28a 100644 --- a/packages/ai/test/openai-completions-tool-result-images.test.ts +++ b/packages/ai/test/openai-completions-tool-result-images.test.ts @@ -32,7 +32,6 @@ const compat: Required = { requiresReasoningContentOnAssistantMessages: false, thinkingFormat: "openai", openRouterRouting: {}, - openRouterImageGeneration: false, vercelGatewayRouting: {}, zaiToolStream: false, supportsStrictMode: true, diff --git a/packages/coding-agent/src/core/session-manager.ts b/packages/coding-agent/src/core/session-manager.ts index 26299b2b..f366274b 100644 --- a/packages/coding-agent/src/core/session-manager.ts +++ b/packages/coding-agent/src/core/session-manager.ts @@ -502,8 +502,10 @@ function extractTextContent(message: Message): string { if (typeof content === "string") { return content; } - const textBlocks = content.filter((block): block is TextContent => block.type === "text") as TextContent[]; - return textBlocks.map((block) => block.text).join(" "); + return content + .filter((block): block is TextContent => block.type === "text") + .map((block) => block.text) + .join(" "); } function getLastActivityTime(entries: FileEntry[]): number | undefined { diff --git a/packages/coding-agent/test/suite/regressions/2860-replaced-session-context.test.ts b/packages/coding-agent/test/suite/regressions/2860-replaced-session-context.test.ts index 678d6400..626365c6 100644 --- a/packages/coding-agent/test/suite/regressions/2860-replaced-session-context.test.ts +++ b/packages/coding-agent/test/suite/regressions/2860-replaced-session-context.test.ts @@ -1,7 +1,7 @@ import { existsSync, mkdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { fauxAssistantMessage, registerFauxProvider, type TextContent } from "@mariozechner/pi-ai"; +import { fauxAssistantMessage, registerFauxProvider } from "@mariozechner/pi-ai"; import { afterEach, describe, expect, it } from "vitest"; import type { AgentSession } from "../../../src/core/agent-session.js"; import { @@ -18,11 +18,12 @@ function getText(message: AgentSession["messages"][number]): string { if (!("content" in message)) { return ""; } - if (typeof message.content === "string") { - return message.content; - } - const textParts = message.content.filter((part): part is TextContent => part.type === "text") as TextContent[]; - return textParts.map((part) => part.text).join(""); + return typeof message.content === "string" + ? message.content + : message.content + .filter((part): part is { type: "text"; text: string } => part.type === "text") + .map((part) => part.text) + .join(""); } describe("regression #2860: replaced session callbacks", () => { From e3d066daa0e81ad74185e631dbd8a1c289e34903 Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Mon, 4 May 2026 16:39:38 +0200 Subject: [PATCH 06/19] feat: images api types --- packages/ai/src/types.ts | 98 ++++++++++++++++++++++++++- packages/ai/src/utils/event-stream.ts | 23 ++++++- 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 8ffd785f..babb0f89 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -1,6 +1,9 @@ -import type { AssistantMessageEventStream } from "./utils/event-stream.js"; +import type { AssistantImagesEventStream, AssistantMessageEventStream } from "./utils/event-stream.js"; -export type { AssistantMessageEventStream } from "./utils/event-stream.js"; +export type { + AssistantImagesEventStream, + AssistantMessageEventStream, +} from "./utils/event-stream.js"; export type KnownApi = | "openai-completions" @@ -16,6 +19,10 @@ export type KnownApi = export type Api = KnownApi | (string & {}); +export type KnownImagesApi = "openrouter-images"; + +export type ImagesApi = KnownImagesApi | (string & {}); + export type KnownProvider = | "amazon-bedrock" | "anthropic" @@ -45,6 +52,10 @@ export type KnownProvider = | "cloudflare-workers-ai"; export type Provider = KnownProvider | string; +export type KnownImagesProvider = "openrouter"; + +export type ImagesProvider = KnownImagesProvider | string; + export type ThinkingLevel = "minimal" | "low" | "medium" | "high" | "xhigh"; /** Token budgets for each thinking level (token-based providers only) */ @@ -130,6 +141,48 @@ export interface StreamOptions { export type ProviderStreamOptions = StreamOptions & Record; +export interface ImagesOptions { + signal?: AbortSignal; + apiKey?: string; + /** + * Optional callback for inspecting or replacing provider payloads before sending. + * Return undefined to keep the payload unchanged. + */ + onPayload?: (payload: unknown, model: ImagesModel) => unknown | undefined | Promise; + /** + * Optional callback invoked after an HTTP response is received. + */ + onResponse?: (response: ProviderResponse, model: ImagesModel) => void | Promise; + /** + * Optional custom HTTP headers to include in API requests. + * Merged with provider defaults; can override default headers. + */ + headers?: Record; + /** + * HTTP request timeout in milliseconds for providers/SDKs that support it. + */ + timeoutMs?: number; + /** + * Maximum retry attempts for providers/SDKs that support client-side retries. + */ + maxRetries?: number; + /** + * Maximum delay in milliseconds to wait for a retry when the server requests a long wait. + * If the server's requested delay exceeds this value, the request fails immediately + * with an error containing the requested delay, allowing higher-level retry logic + * to handle it with user visibility. + * Default: 60000 (60 seconds). Set to 0 to disable the cap. + */ + maxRetryDelayMs?: number; + /** + * Optional metadata to include in API requests. + * Providers extract the fields they understand and ignore the rest. + */ + metadata?: Record; +} + +export type ProviderImagesOptions = ImagesOptions & Record; + // Unified options with reasoning passed to streamSimple() and completeSimple() export interface SimpleStreamOptions extends StreamOptions { reasoning?: ThinkingLevel; @@ -151,6 +204,12 @@ export type StreamFunction AssistantMessageEventStream; +export type ImagesFunction = ( + model: ImagesModel, + context: ImagesContext, + options?: TOptions, +) => AssistantImagesEventStream; + export interface TextSignatureV1 { v: 1; id: string; @@ -235,6 +294,27 @@ export interface ToolResultMessage { export type Message = UserMessage | AssistantMessage | ToolResultMessage; +export type ImagesInputContent = TextContent | ImageContent; +export type ImagesOutputContent = TextContent | ImageContent; + +export interface ImagesContext { + input: ImagesInputContent[]; +} + +export type ImagesStopReason = "stop" | "error" | "aborted"; + +export interface AssistantImages { + api: ImagesApi; + provider: ImagesProvider; + model: string; + output: ImagesOutputContent[]; + responseId?: string; + usage?: Usage; + stopReason: ImagesStopReason; + errorMessage?: string; + timestamp: number; // Unix timestamp in milliseconds +} + import type { TSchema } from "typebox"; export interface Tool { @@ -271,6 +351,13 @@ export type AssistantMessageEvent = | { type: "done"; reason: Extract; message: AssistantMessage } | { type: "error"; reason: Extract; error: AssistantMessage }; +export type AssistantImagesEvent = + | { type: "start"; partial: AssistantImages } + | { type: "image_start"; contentIndex: number; partial: AssistantImages } + | { type: "image_end"; contentIndex: number; image: ImageContent; partial: AssistantImages } + | { type: "done"; reason: Extract; images: AssistantImages } + | { type: "error"; reason: Extract; error: AssistantImages }; + /** * Compatibility settings for OpenAI-compatible completions APIs. * Use this to override URL-based auto-detection for custom providers. @@ -450,3 +537,10 @@ export interface Model { ? AnthropicMessagesCompat : never; } + +export interface ImagesModel + extends Omit, "api" | "provider" | "reasoning" | "contextWindow" | "maxTokens" | "compat"> { + api: TApi; + provider: ImagesProvider; + output: ("text" | "image")[]; +} diff --git a/packages/ai/src/utils/event-stream.ts b/packages/ai/src/utils/event-stream.ts index f4a7ceba..9fb1c114 100644 --- a/packages/ai/src/utils/event-stream.ts +++ b/packages/ai/src/utils/event-stream.ts @@ -1,4 +1,4 @@ -import type { AssistantMessage, AssistantMessageEvent } from "../types.js"; +import type { AssistantImages, AssistantImagesEvent, AssistantMessage, AssistantMessageEvent } from "../types.js"; // Generic event stream class for async iteration export class EventStream implements AsyncIterable { @@ -81,7 +81,28 @@ export class AssistantMessageEventStream extends EventStream { + constructor() { + super( + (event) => event.type === "done" || event.type === "error", + (event) => { + if (event.type === "done") { + return event.images; + } else if (event.type === "error") { + return event.error; + } + throw new Error("Unexpected event type for final result"); + }, + ); + } +} + /** Factory function for AssistantMessageEventStream (for use in extensions) */ export function createAssistantMessageEventStream(): AssistantMessageEventStream { return new AssistantMessageEventStream(); } + +/** Factory function for AssistantImagesEventStream (for use in extensions) */ +export function createAssistantImagesEventStream(): AssistantImagesEventStream { + return new AssistantImagesEventStream(); +} From 62d9132648edb9ccfe1ec49853825ef485af65dd Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Mon, 4 May 2026 16:43:51 +0200 Subject: [PATCH 07/19] feat: images registry --- packages/ai/src/images-api-registry.ts | 76 ++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 packages/ai/src/images-api-registry.ts diff --git a/packages/ai/src/images-api-registry.ts b/packages/ai/src/images-api-registry.ts new file mode 100644 index 00000000..6db07dff --- /dev/null +++ b/packages/ai/src/images-api-registry.ts @@ -0,0 +1,76 @@ +import type { + AssistantImagesEventStream, + ImagesApi, + ImagesContext, + ImagesFunction, + ImagesModel, + ImagesOptions, +} from "./types.js"; + +export type ImagesApiFunction = ( + model: ImagesModel, + context: ImagesContext, + options?: ImagesOptions, +) => AssistantImagesEventStream; + +export interface ImagesApiProvider { + api: TApi; + images: ImagesFunction; +} + +interface ImagesApiProviderInternal { + api: ImagesApi; + images: ImagesApiFunction; +} + +type RegisteredImagesApiProvider = { + provider: ImagesApiProviderInternal; + sourceId?: string; +}; + +const imagesApiProviderRegistry = new Map(); + +function wrapImages( + api: TApi, + images: ImagesFunction, +): ImagesApiFunction { + return (model, context, options) => { + if (model.api !== api) { + throw new Error(`Mismatched api: ${model.api} expected ${api}`); + } + return images(model as ImagesModel, context, options as TOptions); + }; +} + +export function registerImagesApiProvider( + provider: ImagesApiProvider, + sourceId?: string, +): void { + imagesApiProviderRegistry.set(provider.api, { + provider: { + api: provider.api, + images: wrapImages(provider.api, provider.images), + }, + sourceId, + }); +} + +export function getImagesApiProvider(api: ImagesApi): ImagesApiProviderInternal | undefined { + return imagesApiProviderRegistry.get(api)?.provider; +} + +export function getImagesApiProviders(): ImagesApiProviderInternal[] { + return Array.from(imagesApiProviderRegistry.values(), (entry) => entry.provider); +} + +export function unregisterImagesApiProviders(sourceId: string): void { + for (const [api, entry] of imagesApiProviderRegistry.entries()) { + if (entry.sourceId === sourceId) { + imagesApiProviderRegistry.delete(api); + } + } +} + +export function clearImagesApiProviders(): void { + imagesApiProviderRegistry.clear(); +} From e9b0af0a11c53437e4db0773e706921d63cce0fd Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Mon, 4 May 2026 16:46:20 +0200 Subject: [PATCH 08/19] feat: images stream --- packages/ai/src/images.ts | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 packages/ai/src/images.ts diff --git a/packages/ai/src/images.ts b/packages/ai/src/images.ts new file mode 100644 index 00000000..089c5f31 --- /dev/null +++ b/packages/ai/src/images.ts @@ -0,0 +1,36 @@ +import { getImagesApiProvider } from "./images-api-registry.js"; +import type { + AssistantImages, + AssistantImagesEventStream, + ImagesApi, + ImagesContext, + ImagesModel, + ImagesOptions, + ProviderImagesOptions, +} from "./types.js"; + +function resolveImagesApiProvider(api: ImagesApi) { + const provider = getImagesApiProvider(api); + if (!provider) { + throw new Error(`No API provider registered for api: ${api}`); + } + return provider; +} + +export function images( + model: ImagesModel, + context: ImagesContext, + options?: ProviderImagesOptions, +): AssistantImagesEventStream { + const provider = resolveImagesApiProvider(model.api); + return provider.images(model, context, options as ImagesOptions); +} + +export async function completeImages( + model: ImagesModel, + context: ImagesContext, + options?: ProviderImagesOptions, +): Promise { + const s = images(model, context, options); + return s.result(); +} From d9adc536f5d39f3fd259ff5f047ddd03a33d25e4 Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Mon, 4 May 2026 17:00:58 +0200 Subject: [PATCH 09/19] feat: openrouter images --- packages/ai/src/images.ts | 2 + packages/ai/src/index.ts | 3 + .../ai/src/providers/openrouter-images.ts | 202 ++++++++++++++++++ .../src/providers/register-images-builtins.ts | 80 +++++++ packages/ai/test/openrouter-images.test.ts | 111 ++++++++++ 5 files changed, 398 insertions(+) create mode 100644 packages/ai/src/providers/openrouter-images.ts create mode 100644 packages/ai/src/providers/register-images-builtins.ts create mode 100644 packages/ai/test/openrouter-images.test.ts diff --git a/packages/ai/src/images.ts b/packages/ai/src/images.ts index 089c5f31..8595f6b5 100644 --- a/packages/ai/src/images.ts +++ b/packages/ai/src/images.ts @@ -1,3 +1,5 @@ +import "./providers/register-images-builtins.js"; + import { getImagesApiProvider } from "./images-api-registry.js"; import type { AssistantImages, diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 7235ecc7..d3c808e9 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -3,6 +3,8 @@ export { Type } from "typebox"; export * from "./api-registry.js"; export * from "./env-api-keys.js"; +export * from "./images.js"; +export * from "./images-api-registry.js"; export * from "./models.js"; export type { BedrockOptions, BedrockThinkingDisplay } from "./providers/amazon-bedrock.js"; export type { AnthropicEffort, AnthropicOptions, AnthropicThinkingDisplay } from "./providers/anthropic.js"; @@ -16,6 +18,7 @@ export type { OpenAICodexResponsesOptions } from "./providers/openai-codex-respo export type { OpenAICompletionsOptions } from "./providers/openai-completions.js"; export type { OpenAIResponsesOptions } from "./providers/openai-responses.js"; export * from "./providers/register-builtins.js"; +export * from "./providers/register-images-builtins.js"; export * from "./stream.js"; export * from "./types.js"; export * from "./utils/event-stream.js"; diff --git a/packages/ai/src/providers/openrouter-images.ts b/packages/ai/src/providers/openrouter-images.ts new file mode 100644 index 00000000..4dafc196 --- /dev/null +++ b/packages/ai/src/providers/openrouter-images.ts @@ -0,0 +1,202 @@ +import OpenAI from "openai"; +import type { + ChatCompletion, + ChatCompletionContentPart, + ChatCompletionContentPartImage, + ChatCompletionContentPartText, + ChatCompletionCreateParamsNonStreaming, +} from "openai/resources/chat/completions.js"; +import { getEnvApiKey } from "../env-api-keys.js"; +import type { + AssistantImages, + ImageContent, + ImagesContext, + ImagesFunction, + ImagesModel, + ImagesOptions, + TextContent, +} from "../types.js"; +import { AssistantImagesEventStream } from "../utils/event-stream.js"; +import { headersToRecord } from "../utils/headers.js"; +import { sanitizeSurrogates } from "../utils/sanitize-unicode.js"; + +interface OpenRouterGeneratedImage { + image_url?: string | { url?: string }; +} + +type OpenRouterImageGenerationMessage = ChatCompletion["choices"][number]["message"] & { + images?: OpenRouterGeneratedImage[]; +}; + +type OpenRouterImageGenerationChoice = ChatCompletion["choices"][number] & { + message: OpenRouterImageGenerationMessage; +}; + +type OpenRouterImageGenerationResponse = ChatCompletion & { + choices: OpenRouterImageGenerationChoice[]; +}; + +export const imagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = ( + model: ImagesModel<"openrouter-images">, + context: ImagesContext, + options?: ImagesOptions, +) => { + const stream = new AssistantImagesEventStream(); + + (async () => { + const output: AssistantImages = { + api: model.api, + provider: model.provider, + model: model.id, + output: [], + stopReason: "stop", + timestamp: Date.now(), + }; + + try { + const apiKey = options?.apiKey || getEnvApiKey(model.provider) || ""; + const client = createClient(model, apiKey, options?.headers); + let params = buildParams(model, context); + const nextParams = await options?.onPayload?.(params, model); + if (nextParams !== undefined) { + params = nextParams as typeof params; + } + const requestOptions = { + ...(options?.signal ? { signal: options.signal } : {}), + ...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}), + ...(options?.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}), + }; + const { data: response, response: rawResponse } = await client.chat.completions + .create(params as unknown as ChatCompletionCreateParamsNonStreaming, requestOptions) + .withResponse(); + await options?.onResponse?.( + { status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, + model, + ); + + stream.push({ type: "start", partial: output }); + + const imageResponse = response as OpenRouterImageGenerationResponse; + output.responseId = imageResponse.id; + if (imageResponse.usage) { + output.usage = parseUsage(imageResponse.usage, model); + } + + const choice = imageResponse.choices[0]; + if (choice) { + const content = choice.message.content; + if (typeof content === "string" && content.length > 0) { + output.output.push({ type: "text", text: content } satisfies TextContent); + } + + for (const image of choice.message.images ?? []) { + const imageUrl = typeof image.image_url === "string" ? image.image_url : image.image_url?.url; + if (!imageUrl?.startsWith("data:")) continue; + const matches = imageUrl.match(/^data:([^;]+);base64,(.+)$/); + if (!matches) continue; + const imageBlock: ImageContent = { + type: "image", + mimeType: matches[1], + data: matches[2], + }; + output.output.push(imageBlock); + const contentIndex = output.output.length - 1; + stream.push({ type: "image_start", contentIndex, partial: output }); + stream.push({ type: "image_end", contentIndex, image: imageBlock, partial: output }); + } + } + + stream.push({ type: "done", reason: "stop", images: output }); + stream.end(); + } catch (error) { + output.stopReason = options?.signal?.aborted ? "aborted" : "error"; + output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error); + stream.push({ type: "error", reason: output.stopReason, error: output }); + stream.end(output); + } + })(); + + return stream; +}; + +function createClient( + model: ImagesModel<"openrouter-images">, + apiKey: string, + optionsHeaders?: Record, +): OpenAI { + return new OpenAI({ + apiKey, + baseURL: model.baseUrl, + dangerouslyAllowBrowser: true, + defaultHeaders: { + ...model.headers, + ...optionsHeaders, + }, + }); +} + +type OpenRouterImagesCreateParams = Omit & { + modalities: Array<"image" | "text">; +}; + +function buildParams(model: ImagesModel<"openrouter-images">, context: ImagesContext): OpenRouterImagesCreateParams { + const content: ChatCompletionContentPart[] = context.input.map((item): ChatCompletionContentPart => { + if (item.type === "text") { + return { + type: "text", + text: sanitizeSurrogates(item.text), + } satisfies ChatCompletionContentPartText; + } + return { + type: "image_url", + image_url: { + url: `data:${item.mimeType};base64,${item.data}`, + }, + } satisfies ChatCompletionContentPartImage; + }); + + return { + model: model.id, + messages: [ + { + role: "user" as const, + content, + }, + ], + stream: false, + modalities: model.output.includes("text") ? ["image", "text"] : ["image"], + }; +} + +function parseUsage( + rawUsage: { + prompt_tokens?: number; + completion_tokens?: number; + prompt_tokens_details?: { cached_tokens?: number; cache_write_tokens?: number }; + }, + model: ImagesModel<"openrouter-images">, +) { + const promptTokens = rawUsage.prompt_tokens || 0; + const reportedCachedTokens = rawUsage.prompt_tokens_details?.cached_tokens || 0; + const cacheWriteTokens = rawUsage.prompt_tokens_details?.cache_write_tokens || 0; + const cacheReadTokens = + cacheWriteTokens > 0 ? Math.max(0, reportedCachedTokens - cacheWriteTokens) : reportedCachedTokens; + const input = Math.max(0, promptTokens - cacheReadTokens - cacheWriteTokens); + const output = rawUsage.completion_tokens || 0; + const usage = { + input, + output, + cacheRead: cacheReadTokens, + cacheWrite: cacheWriteTokens, + totalTokens: input + output + cacheReadTokens + cacheWriteTokens, + cost: { + input: (model.cost.input / 1000000) * input, + output: (model.cost.output / 1000000) * output, + cacheRead: (model.cost.cacheRead / 1000000) * cacheReadTokens, + cacheWrite: (model.cost.cacheWrite / 1000000) * cacheWriteTokens, + total: 0, + }, + }; + usage.cost.total = usage.cost.input + usage.cost.output + usage.cost.cacheRead + usage.cost.cacheWrite; + return usage; +} diff --git a/packages/ai/src/providers/register-images-builtins.ts b/packages/ai/src/providers/register-images-builtins.ts new file mode 100644 index 00000000..9c5c04ef --- /dev/null +++ b/packages/ai/src/providers/register-images-builtins.ts @@ -0,0 +1,80 @@ +import { clearImagesApiProviders, registerImagesApiProvider } from "../images-api-registry.js"; +import type { + AssistantImages, + AssistantImagesEvent, + ImagesContext, + ImagesFunction, + ImagesModel, + ImagesOptions, +} from "../types.js"; +import { AssistantImagesEventStream } from "../utils/event-stream.js"; +import type { imagesOpenRouter as imagesOpenRouterFunction } from "./openrouter-images.js"; + +interface OpenRouterImagesProviderModule { + imagesOpenRouter: typeof imagesOpenRouterFunction; +} + +let openRouterImagesProviderModulePromise: Promise | undefined; + +function forwardImagesStream(target: AssistantImagesEventStream, source: AsyncIterable): void { + (async () => { + for await (const event of source) { + target.push(event); + } + target.end(); + })(); +} + +function createLazyLoadErrorImages(model: ImagesModel<"openrouter-images">, error: unknown): AssistantImages { + return { + api: model.api, + provider: model.provider, + model: model.id, + output: [], + stopReason: "error", + errorMessage: error instanceof Error ? error.message : String(error), + timestamp: Date.now(), + }; +} + +function loadOpenRouterImagesProviderModule(): Promise { + openRouterImagesProviderModulePromise ||= import("./openrouter-images.js").then( + (module) => module as OpenRouterImagesProviderModule, + ); + return openRouterImagesProviderModulePromise; +} + +export const imagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = ( + model: ImagesModel<"openrouter-images">, + context: ImagesContext, + options?: ImagesOptions, +) => { + const outer = new AssistantImagesEventStream(); + + loadOpenRouterImagesProviderModule() + .then((module) => { + const inner = module.imagesOpenRouter(model, context, options); + forwardImagesStream(outer, inner); + }) + .catch((error) => { + const images = createLazyLoadErrorImages(model, error); + outer.push({ type: "error", reason: "error", error: images }); + outer.end(images); + }); + + return outer; +}; + +export function registerBuiltInImagesApiProviders(): void { + registerImagesApiProvider({ + api: "openrouter-images", + images: imagesOpenRouter, + }); +} + +export function resetImagesApiProviders(): void { + clearImagesApiProviders(); + registerBuiltInImagesApiProviders(); +} + +registerBuiltInImagesApiProviders(); diff --git a/packages/ai/test/openrouter-images.test.ts b/packages/ai/test/openrouter-images.test.ts new file mode 100644 index 00000000..5926fbf4 --- /dev/null +++ b/packages/ai/test/openrouter-images.test.ts @@ -0,0 +1,111 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { completeImages, images } from "../src/images.js"; +import type { ImagesContext, ImagesModel } from "../src/types.js"; + +const mockState = vi.hoisted(() => ({ + lastParams: undefined as unknown, +})); + +vi.mock("openai", () => { + class FakeOpenAI { + chat = { + completions: { + create: (params: unknown) => { + mockState.lastParams = params; + const response = { + id: "img-1", + usage: { + prompt_tokens: 12, + completion_tokens: 34, + prompt_tokens_details: { cached_tokens: 0 }, + }, + choices: [ + { + message: { + content: "Here is your image.", + images: [{ image_url: "data:image/png;base64,ZmFrZS1wbmc=" }], + }, + }, + ], + }; + const promise = Promise.resolve(response) as Promise & { + withResponse: () => Promise<{ + data: typeof response; + response: { status: number; headers: Headers }; + }>; + }; + promise.withResponse = async () => ({ + data: response, + response: { status: 200, headers: new Headers() }, + }); + return promise; + }, + }, + }; + } + + return { default: FakeOpenAI }; +}); + +describe("openrouter images", () => { + beforeEach(() => { + mockState.lastParams = undefined; + }); + + it("emits image events and returns text plus images in final output", async () => { + const model: ImagesModel<"openrouter-images"> = { + id: "google/gemini-3.1-flash-image-preview", + name: "Gemini 3.1 Flash Image Preview", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["text", "image"], + cost: { input: 0.015, output: 0.03, cacheRead: 0, cacheWrite: 0 }, + headers: { "HTTP-Referer": "https://example.com" }, + }; + const context: ImagesContext = { + input: [{ type: "text", text: "Generate a cat" }], + }; + + const result = images(model, context, { apiKey: "test" }); + const eventTypes: string[] = []; + for await (const event of result) { + eventTypes.push(event.type); + } + + const output = await result.result(); + expect(eventTypes).toEqual(["start", "image_start", "image_end", "done"]); + expect(output.responseId).toBe("img-1"); + expect(output.output[0]).toMatchObject({ type: "text", text: "Here is your image." }); + expect(output.output[1]).toMatchObject({ type: "image", mimeType: "image/png", data: "ZmFrZS1wbmc=" }); + + const params = mockState.lastParams as { + stream?: boolean; + modalities?: string[]; + messages?: [{ content?: [{ type: string; text?: string }] }]; + }; + expect(params.stream).toBe(false); + expect(params.modalities).toEqual(["image", "text"]); + expect(params.messages?.[0]?.content?.[0]).toMatchObject({ type: "text", text: "Generate a cat" }); + }); + + it("completeImages resolves the final assistant images result", async () => { + const model: ImagesModel<"openrouter-images"> = { + id: "black-forest-labs/flux.2-pro", + name: "FLUX.2 Pro", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { input: 0.015, output: 0.03, cacheRead: 0, cacheWrite: 0 }, + }; + const context: ImagesContext = { + input: [{ type: "text", text: "Generate a cat" }], + }; + + const output = await completeImages(model, context, { apiKey: "test" }); + expect(output.output.some((item) => item.type === "image")).toBe(true); + }); +}); From 63c61aac6f36815651a54fc01cce33f5042ba667 Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Mon, 4 May 2026 17:07:02 +0200 Subject: [PATCH 10/19] feat: image models --- packages/ai/README.md | 5 +- packages/ai/package.json | 3 +- packages/ai/scripts/generate-image-models.ts | 131 +++++++++ packages/ai/src/image-models.generated.ts | 264 +++++++++++++++++++ packages/ai/src/image-models.ts | 42 +++ packages/ai/src/index.ts | 1 + 6 files changed, 443 insertions(+), 3 deletions(-) create mode 100644 packages/ai/scripts/generate-image-models.ts create mode 100644 packages/ai/src/image-models.generated.ts create mode 100644 packages/ai/src/image-models.ts diff --git a/packages/ai/README.md b/packages/ai/README.md index 18d3b945..8d09c8ef 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -1262,10 +1262,11 @@ Create a new provider file (for example `amazon-bedrock.ts`) that exports: - Add credential detection in `env-api-keys.ts` for the new provider - Ensure `streamSimple` handles auth lookup via `getEnvApiKey()` or provider-specific auth -#### 4. Model Generation (`scripts/generate-models.ts`) +#### 4. Model Generation (`scripts/generate-models.ts`, `scripts/generate-image-models.ts`) - Add logic to fetch and parse models from the provider's source (e.g., models.dev API) -- Map provider model data to the standardized `Model` interface +- Map chat/tool-capable provider model data to the standardized `Model` interface via `scripts/generate-models.ts` +- Map image-generation provider model data to the standardized `ImagesModel` interface via `scripts/generate-image-models.ts` - Handle provider-specific quirks (pricing format, capability flags, model ID transformations) #### 5. Tests (`test/`) diff --git a/packages/ai/package.json b/packages/ai/package.json index 51a70afa..e2cd9357 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -65,7 +65,8 @@ "scripts": { "clean": "shx rm -rf dist", "generate-models": "npx tsx scripts/generate-models.ts", - "build": "npm run generate-models && tsgo -p tsconfig.build.json", + "generate-image-models": "npx tsx scripts/generate-image-models.ts", + "build": "npm run generate-models && npm run generate-image-models && tsgo -p tsconfig.build.json", "dev": "tsgo -p tsconfig.build.json --watch --preserveWatchOutput", "dev:tsc": "tsgo -p tsconfig.build.json --watch --preserveWatchOutput", "test": "vitest --run", diff --git a/packages/ai/scripts/generate-image-models.ts b/packages/ai/scripts/generate-image-models.ts new file mode 100644 index 00000000..bd0c6727 --- /dev/null +++ b/packages/ai/scripts/generate-image-models.ts @@ -0,0 +1,131 @@ +#!/usr/bin/env tsx + +import { writeFileSync } from "fs"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import type { ImagesModel } from "../src/types.js"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const packageRoot = join(__dirname, ".."); +const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"; + +interface OpenRouterModelRecord { + id: string; + name: string; + context_length?: number; + architecture?: { + input_modalities?: string[]; + output_modalities?: string[]; + }; + pricing?: { + prompt?: string; + completion?: string; + input_cache_read?: string; + input_cache_write?: string; + }; +} + +async function fetchOpenRouterImageModels(): Promise[]> { + try { + console.log("Fetching image models from OpenRouter API..."); + const response = await fetch(`${OPENROUTER_BASE_URL}/models?output_modalities=image`); + const data = (await response.json()) as { data?: OpenRouterModelRecord[] }; + const models: ImagesModel<"openrouter-images">[] = []; + + for (const model of data.data ?? []) { + const input = Array.from( + new Set( + (model.architecture?.input_modalities ?? []) + .filter((modality): modality is "text" | "image" => modality === "text" || modality === "image"), + ), + ); + const output = Array.from( + new Set( + (model.architecture?.output_modalities ?? []).filter( + (modality): modality is "text" | "image" => modality === "text" || modality === "image", + ), + ), + ); + + if (!output.includes("image")) continue; + if (input.length === 0) input.push("text"); + + models.push({ + id: model.id, + name: model.name, + api: "openrouter-images", + provider: "openrouter", + baseUrl: OPENROUTER_BASE_URL, + input, + output, + cost: { + input: parseFloat(model.pricing?.prompt || "0") * 1_000_000, + output: parseFloat(model.pricing?.completion || "0") * 1_000_000, + cacheRead: parseFloat(model.pricing?.input_cache_read || "0") * 1_000_000, + cacheWrite: parseFloat(model.pricing?.input_cache_write || "0") * 1_000_000, + }, + }); + } + + console.log(`Fetched ${models.length} image models from OpenRouter`); + return models; + } catch (error) { + console.error("Failed to fetch OpenRouter image models:", error); + return []; + } +} + +function generateImageModelsFile(models: ImagesModel<"openrouter-images">[]): string { + const imageModelsByProvider = { + openrouter: Object.fromEntries( + models + .sort((a, b) => a.id.localeCompare(b.id)) + .map((model) => [ + model.id, + `{ + id: ${JSON.stringify(model.id)}, + name: ${JSON.stringify(model.name)}, + api: ${JSON.stringify(model.api)}, + provider: ${JSON.stringify(model.provider)}, + baseUrl: ${JSON.stringify(model.baseUrl)}, + input: ${JSON.stringify(model.input)}, + output: ${JSON.stringify(model.output)}, + cost: ${JSON.stringify(model.cost, null, 2).replace(/^/gm, "\t")} + } satisfies ImagesModel<${JSON.stringify(model.api)}>`, + ]), + ), + }; + + const providerEntries = Object.entries(imageModelsByProvider) + .map(([provider, providerModels]) => { + const modelEntries = Object.entries(providerModels) + .map(([id, serialized]) => `\t\t${JSON.stringify(id)}: ${serialized},`) + .join("\n"); + return `\t${JSON.stringify(provider)}: {\n${modelEntries}\n\t},`; + }) + .join("\n"); + + return `// This file is auto-generated by scripts/generate-image-models.ts +// Do not edit manually - run 'npm run generate-image-models' to update + +import type { ImagesApi, ImagesModel } from "./types.js"; + +export const IMAGE_MODELS = { +${providerEntries} +} as const satisfies Record>>; +`; +} + +async function main(): Promise { + const models = await fetchOpenRouterImageModels(); + const output = generateImageModelsFile(models); + const outputPath = join(packageRoot, "src", "image-models.generated.ts"); + writeFileSync(outputPath, output, "utf-8"); + console.log(`Generated ${outputPath}`); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/packages/ai/src/image-models.generated.ts b/packages/ai/src/image-models.generated.ts new file mode 100644 index 00000000..9bbee4c8 --- /dev/null +++ b/packages/ai/src/image-models.generated.ts @@ -0,0 +1,264 @@ +// This file is auto-generated by scripts/generate-image-models.ts +// Do not edit manually - run 'npm run generate-image-models' to update + +import type { ImagesApi, ImagesModel } from "./types.js"; + +export const IMAGE_MODELS = { + openrouter: { + "black-forest-labs/flux.2-flex": { + id: "black-forest-labs/flux.2-flex", + name: "Black Forest Labs: FLUX.2 Flex", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "black-forest-labs/flux.2-klein-4b": { + id: "black-forest-labs/flux.2-klein-4b", + name: "Black Forest Labs: FLUX.2 Klein 4B", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "black-forest-labs/flux.2-max": { + id: "black-forest-labs/flux.2-max", + name: "Black Forest Labs: FLUX.2 Max", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "black-forest-labs/flux.2-pro": { + id: "black-forest-labs/flux.2-pro", + name: "Black Forest Labs: FLUX.2 Pro", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "bytedance-seed/seedream-4.5": { + id: "bytedance-seed/seedream-4.5", + name: "ByteDance Seed: Seedream 4.5", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["image", "text"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "google/gemini-2.5-flash-image": { + id: "google/gemini-2.5-flash-image", + name: "Google: Nano Banana (Gemini 2.5 Flash Image)", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["image", "text"], + output: ["image", "text"], + cost: { + input: 0.3, + output: 2.5, + cacheRead: 0.03, + cacheWrite: 0.08333333333333334, + }, + } satisfies ImagesModel<"openrouter-images">, + "google/gemini-3-pro-image-preview": { + id: "google/gemini-3-pro-image-preview", + name: "Google: Nano Banana Pro (Gemini 3 Pro Image Preview)", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["image", "text"], + output: ["image", "text"], + cost: { + input: 2, + output: 12, + cacheRead: 0.19999999999999998, + cacheWrite: 0.375, + }, + } satisfies ImagesModel<"openrouter-images">, + "google/gemini-3.1-flash-image-preview": { + id: "google/gemini-3.1-flash-image-preview", + name: "Google: Nano Banana 2 (Gemini 3.1 Flash Image Preview)", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["image", "text"], + output: ["image", "text"], + cost: { + input: 0.5, + output: 3, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "openai/gpt-5-image": { + id: "openai/gpt-5-image", + name: "OpenAI: GPT-5 Image", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["image", "text"], + output: ["image", "text"], + cost: { + input: 10, + output: 10, + cacheRead: 1.25, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "openai/gpt-5-image-mini": { + id: "openai/gpt-5-image-mini", + name: "OpenAI: GPT-5 Image Mini", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["image", "text"], + output: ["image", "text"], + cost: { + input: 2.5, + output: 2, + cacheRead: 0.25, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "openai/gpt-5.4-image-2": { + id: "openai/gpt-5.4-image-2", + name: "OpenAI: GPT-5.4 Image 2", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["image", "text"], + output: ["image", "text"], + cost: { + input: 8, + output: 15, + cacheRead: 2, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "openrouter/auto": { + id: "openrouter/auto", + name: "Auto Router", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["text", "image"], + cost: { + input: -1000000, + output: -1000000, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "sourceful/riverflow-v2-fast": { + id: "sourceful/riverflow-v2-fast", + name: "Sourceful: Riverflow V2 Fast", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "sourceful/riverflow-v2-fast-preview": { + id: "sourceful/riverflow-v2-fast-preview", + name: "Sourceful: Riverflow V2 Fast Preview", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "sourceful/riverflow-v2-max-preview": { + id: "sourceful/riverflow-v2-max-preview", + name: "Sourceful: Riverflow V2 Max Preview", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "sourceful/riverflow-v2-pro": { + id: "sourceful/riverflow-v2-pro", + name: "Sourceful: Riverflow V2 Pro", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "sourceful/riverflow-v2-standard-preview": { + id: "sourceful/riverflow-v2-standard-preview", + name: "Sourceful: Riverflow V2 Standard Preview", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + }, +} as const satisfies Record>>; diff --git a/packages/ai/src/image-models.ts b/packages/ai/src/image-models.ts new file mode 100644 index 00000000..afa4f415 --- /dev/null +++ b/packages/ai/src/image-models.ts @@ -0,0 +1,42 @@ +import { IMAGE_MODELS } from "./image-models.generated.js"; +import type { ImagesApi, ImagesModel, KnownImagesProvider } from "./types.js"; + +const imageModelRegistry: Map>> = new Map(); + +for (const [provider, models] of Object.entries(IMAGE_MODELS)) { + const providerModels = new Map>(); + for (const [id, model] of Object.entries(models)) { + providerModels.set(id, model as ImagesModel); + } + imageModelRegistry.set(provider, providerModels); +} + +type ImageModelApi< + TProvider extends KnownImagesProvider, + TModelId extends keyof (typeof IMAGE_MODELS)[TProvider], +> = (typeof IMAGE_MODELS)[TProvider][TModelId] extends { api: infer TApi } + ? TApi extends ImagesApi + ? TApi + : never + : never; + +export function getImageModel< + TProvider extends KnownImagesProvider, + TModelId extends keyof (typeof IMAGE_MODELS)[TProvider], +>(provider: TProvider, modelId: TModelId): ImagesModel> { + const providerModels = imageModelRegistry.get(provider); + return providerModels?.get(modelId as string) as ImagesModel>; +} + +export function getImageProviders(): KnownImagesProvider[] { + return Array.from(imageModelRegistry.keys()) as KnownImagesProvider[]; +} + +export function getImageModels( + provider: TProvider, +): ImagesModel>[] { + const models = imageModelRegistry.get(provider); + return models + ? (Array.from(models.values()) as ImagesModel>[]) + : []; +} diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index d3c808e9..48670068 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -3,6 +3,7 @@ export { Type } from "typebox"; export * from "./api-registry.js"; export * from "./env-api-keys.js"; +export * from "./image-models.js"; export * from "./images.js"; export * from "./images-api-registry.js"; export * from "./models.js"; From b9c19135d98b1956946b1dfd2cb177194eb72a2d Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Mon, 4 May 2026 17:10:44 +0200 Subject: [PATCH 11/19] cleanup --- packages/ai/src/images-api-registry.ts | 16 ---------------- .../ai/src/providers/register-images-builtins.ts | 7 +------ 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/packages/ai/src/images-api-registry.ts b/packages/ai/src/images-api-registry.ts index 6db07dff..395a060d 100644 --- a/packages/ai/src/images-api-registry.ts +++ b/packages/ai/src/images-api-registry.ts @@ -58,19 +58,3 @@ export function registerImagesApiProvider entry.provider); -} - -export function unregisterImagesApiProviders(sourceId: string): void { - for (const [api, entry] of imagesApiProviderRegistry.entries()) { - if (entry.sourceId === sourceId) { - imagesApiProviderRegistry.delete(api); - } - } -} - -export function clearImagesApiProviders(): void { - imagesApiProviderRegistry.clear(); -} diff --git a/packages/ai/src/providers/register-images-builtins.ts b/packages/ai/src/providers/register-images-builtins.ts index 9c5c04ef..afc6be55 100644 --- a/packages/ai/src/providers/register-images-builtins.ts +++ b/packages/ai/src/providers/register-images-builtins.ts @@ -1,4 +1,4 @@ -import { clearImagesApiProviders, registerImagesApiProvider } from "../images-api-registry.js"; +import { registerImagesApiProvider } from "../images-api-registry.js"; import type { AssistantImages, AssistantImagesEvent, @@ -72,9 +72,4 @@ export function registerBuiltInImagesApiProviders(): void { }); } -export function resetImagesApiProviders(): void { - clearImagesApiProviders(); - registerBuiltInImagesApiProviders(); -} - registerBuiltInImagesApiProviders(); From ab14238f1d1dcb34c2d4278a98937db397223c93 Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Mon, 4 May 2026 17:26:50 +0200 Subject: [PATCH 12/19] dog --- packages/ai/test/openrouter-images.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ai/test/openrouter-images.test.ts b/packages/ai/test/openrouter-images.test.ts index 5926fbf4..e3098f15 100644 --- a/packages/ai/test/openrouter-images.test.ts +++ b/packages/ai/test/openrouter-images.test.ts @@ -65,7 +65,7 @@ describe("openrouter images", () => { headers: { "HTTP-Referer": "https://example.com" }, }; const context: ImagesContext = { - input: [{ type: "text", text: "Generate a cat" }], + input: [{ type: "text", text: "Generate a dog" }], }; const result = images(model, context, { apiKey: "test" }); @@ -87,7 +87,7 @@ describe("openrouter images", () => { }; expect(params.stream).toBe(false); expect(params.modalities).toEqual(["image", "text"]); - expect(params.messages?.[0]?.content?.[0]).toMatchObject({ type: "text", text: "Generate a cat" }); + expect(params.messages?.[0]?.content?.[0]).toMatchObject({ type: "text", text: "Generate a dog" }); }); it("completeImages resolves the final assistant images result", async () => { @@ -102,7 +102,7 @@ describe("openrouter images", () => { cost: { input: 0.015, output: 0.03, cacheRead: 0, cacheWrite: 0 }, }; const context: ImagesContext = { - input: [{ type: "text", text: "Generate a cat" }], + input: [{ type: "text", text: "Generate a dog" }], }; const output = await completeImages(model, context, { apiKey: "test" }); From 60e885f985eb1001ba1e5c13d1e57d7d986d72d2 Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Tue, 5 May 2026 09:30:15 +0200 Subject: [PATCH 13/19] refactor: image providers into subfolder --- packages/ai/src/images.ts | 2 +- packages/ai/src/index.ts | 2 +- .../{openrouter-images.ts => images/openrouter.ts} | 10 +++++----- .../register-builtins.ts} | 10 +++++----- 4 files changed, 12 insertions(+), 12 deletions(-) rename packages/ai/src/providers/{openrouter-images.ts => images/openrouter.ts} (95%) rename packages/ai/src/providers/{register-images-builtins.ts => images/register-builtins.ts} (87%) diff --git a/packages/ai/src/images.ts b/packages/ai/src/images.ts index 8595f6b5..ecf374a8 100644 --- a/packages/ai/src/images.ts +++ b/packages/ai/src/images.ts @@ -1,4 +1,4 @@ -import "./providers/register-images-builtins.js"; +import "./providers/images/register-builtins.js"; import { getImagesApiProvider } from "./images-api-registry.js"; import type { diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index d535aeb3..d3c8d59f 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -14,6 +14,7 @@ export * from "./providers/faux.js"; export type { GoogleOptions } from "./providers/google.js"; export type { GoogleThinkingLevel } from "./providers/google-shared.js"; export type { GoogleVertexOptions } from "./providers/google-vertex.js"; +export * from "./providers/images/register-builtins.js"; export type { MistralOptions } from "./providers/mistral.js"; export type { OpenAICodexResponsesOptions, @@ -22,7 +23,6 @@ export type { export type { OpenAICompletionsOptions } from "./providers/openai-completions.js"; export type { OpenAIResponsesOptions } from "./providers/openai-responses.js"; export * from "./providers/register-builtins.js"; -export * from "./providers/register-images-builtins.js"; export * from "./session-resources.js"; export * from "./stream.js"; export * from "./types.js"; diff --git a/packages/ai/src/providers/openrouter-images.ts b/packages/ai/src/providers/images/openrouter.ts similarity index 95% rename from packages/ai/src/providers/openrouter-images.ts rename to packages/ai/src/providers/images/openrouter.ts index 4dafc196..c1a5d966 100644 --- a/packages/ai/src/providers/openrouter-images.ts +++ b/packages/ai/src/providers/images/openrouter.ts @@ -6,7 +6,7 @@ import type { ChatCompletionContentPartText, ChatCompletionCreateParamsNonStreaming, } from "openai/resources/chat/completions.js"; -import { getEnvApiKey } from "../env-api-keys.js"; +import { getEnvApiKey } from "../../env-api-keys.js"; import type { AssistantImages, ImageContent, @@ -15,10 +15,10 @@ import type { ImagesModel, ImagesOptions, TextContent, -} from "../types.js"; -import { AssistantImagesEventStream } from "../utils/event-stream.js"; -import { headersToRecord } from "../utils/headers.js"; -import { sanitizeSurrogates } from "../utils/sanitize-unicode.js"; +} from "../../types.js"; +import { AssistantImagesEventStream } from "../../utils/event-stream.js"; +import { headersToRecord } from "../../utils/headers.js"; +import { sanitizeSurrogates } from "../../utils/sanitize-unicode.js"; interface OpenRouterGeneratedImage { image_url?: string | { url?: string }; diff --git a/packages/ai/src/providers/register-images-builtins.ts b/packages/ai/src/providers/images/register-builtins.ts similarity index 87% rename from packages/ai/src/providers/register-images-builtins.ts rename to packages/ai/src/providers/images/register-builtins.ts index afc6be55..7a874b2a 100644 --- a/packages/ai/src/providers/register-images-builtins.ts +++ b/packages/ai/src/providers/images/register-builtins.ts @@ -1,4 +1,4 @@ -import { registerImagesApiProvider } from "../images-api-registry.js"; +import { registerImagesApiProvider } from "../../images-api-registry.js"; import type { AssistantImages, AssistantImagesEvent, @@ -6,9 +6,9 @@ import type { ImagesFunction, ImagesModel, ImagesOptions, -} from "../types.js"; -import { AssistantImagesEventStream } from "../utils/event-stream.js"; -import type { imagesOpenRouter as imagesOpenRouterFunction } from "./openrouter-images.js"; +} from "../../types.js"; +import { AssistantImagesEventStream } from "../../utils/event-stream.js"; +import type { imagesOpenRouter as imagesOpenRouterFunction } from "./openrouter.js"; interface OpenRouterImagesProviderModule { imagesOpenRouter: typeof imagesOpenRouterFunction; @@ -38,7 +38,7 @@ function createLazyLoadErrorImages(model: ImagesModel<"openrouter-images">, erro } function loadOpenRouterImagesProviderModule(): Promise { - openRouterImagesProviderModulePromise ||= import("./openrouter-images.js").then( + openRouterImagesProviderModulePromise ||= import("./openrouter.js").then( (module) => module as OpenRouterImagesProviderModule, ); return openRouterImagesProviderModulePromise; From 0d96b9be7f822a3e16c8da15187b3c3f8a2cf28e Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Tue, 5 May 2026 09:30:44 +0200 Subject: [PATCH 14/19] rename: generateImages --- packages/ai/src/images.ts | 2 +- packages/ai/test/openrouter-images.test.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/ai/src/images.ts b/packages/ai/src/images.ts index ecf374a8..c1008cdc 100644 --- a/packages/ai/src/images.ts +++ b/packages/ai/src/images.ts @@ -28,7 +28,7 @@ export function images( return provider.images(model, context, options as ImagesOptions); } -export async function completeImages( +export async function generateImages( model: ImagesModel, context: ImagesContext, options?: ProviderImagesOptions, diff --git a/packages/ai/test/openrouter-images.test.ts b/packages/ai/test/openrouter-images.test.ts index e3098f15..b4a79127 100644 --- a/packages/ai/test/openrouter-images.test.ts +++ b/packages/ai/test/openrouter-images.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { completeImages, images } from "../src/images.js"; +import { generateImages, images } from "../src/images.js"; import type { ImagesContext, ImagesModel } from "../src/types.js"; const mockState = vi.hoisted(() => ({ @@ -90,7 +90,7 @@ describe("openrouter images", () => { expect(params.messages?.[0]?.content?.[0]).toMatchObject({ type: "text", text: "Generate a dog" }); }); - it("completeImages resolves the final assistant images result", async () => { + it("generateImages resolves the final assistant images result", async () => { const model: ImagesModel<"openrouter-images"> = { id: "black-forest-labs/flux.2-pro", name: "FLUX.2 Pro", @@ -105,7 +105,7 @@ describe("openrouter images", () => { input: [{ type: "text", text: "Generate a dog" }], }; - const output = await completeImages(model, context, { apiKey: "test" }); + const output = await generateImages(model, context, { apiKey: "test" }); expect(output.output.some((item) => item.type === "image")).toBe(true); }); }); From 3728e4b9e62a2fea8f353fa4cf834fd11b834241 Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Tue, 5 May 2026 09:47:33 +0200 Subject: [PATCH 15/19] e2e: add images test --- packages/ai/test/images.test.ts | 124 ++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 packages/ai/test/images.test.ts diff --git a/packages/ai/test/images.test.ts b/packages/ai/test/images.test.ts new file mode 100644 index 00000000..84aac37a --- /dev/null +++ b/packages/ai/test/images.test.ts @@ -0,0 +1,124 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { getImageModel } from "../src/image-models.js"; +import { generateImages, images } from "../src/images.js"; +import type { ImageContent, ImagesContext, ImagesModel, ProviderImagesOptions } from "../src/types.js"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +type ImagesOptionsWithExtras = ProviderImagesOptions & Record; + +async function basicImageGeneration(model: ImagesModel, options?: ImagesOptionsWithExtras) { + const context: ImagesContext = { + input: [{ type: "text", text: "Generate a simple red circle on a plain white background. No text." }], + }; + + const response = await generateImages(model, context, options); + + expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop"); + expect(response.errorMessage).toBeFalsy(); + expect(response.output.some((item) => item.type === "image")).toBe(true); + expect(response.timestamp).toBeGreaterThan(0); +} + +async function handleStreaming(model: ImagesModel, options?: ImagesOptionsWithExtras) { + const context: ImagesContext = { + input: [{ type: "text", text: "Generate a simple blue square on a plain white background. No text." }], + }; + + const s = images(model, context, options); + let started = false; + let imageStarted = false; + let imageCompleted = false; + + for await (const event of s) { + if (event.type === "start") { + started = true; + } else if (event.type === "image_start") { + imageStarted = true; + } else if (event.type === "image_end") { + imageCompleted = true; + expect(event.image.type).toBe("image"); + expect(event.image.data.length).toBeGreaterThan(0); + } + } + + const response = await s.result(); + expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop"); + expect(started).toBe(true); + expect(imageStarted).toBe(true); + expect(imageCompleted).toBe(true); + expect(response.output.some((item) => item.type === "image")).toBe(true); +} + +async function handleTextAndImageOutput( + model: ImagesModel, + options?: ImagesOptionsWithExtras, +) { + if (!model.output.includes("text")) { + console.log(`Skipping text+image output test - model ${model.id} doesn't support text output`); + return; + } + + const context: ImagesContext = { + input: [{ type: "text", text: "Generate a red circle and include a brief description of the image." }], + }; + + const response = await generateImages(model, context, options); + + expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop"); + expect(response.output.some((item) => item.type === "image")).toBe(true); + expect(response.output.some((item) => item.type === "text" && item.text.trim().length > 0)).toBe(true); +} + +async function handleImageInput(model: ImagesModel, options?: ImagesOptionsWithExtras) { + if (!model.input.includes("image")) { + console.log(`Skipping image input test - model ${model.id} doesn't support image input`); + return; + } + + const imagePath = join(__dirname, "data", "red-circle.png"); + const imageBuffer = readFileSync(imagePath); + const imageContent: ImageContent = { + type: "image", + data: imageBuffer.toString("base64"), + mimeType: "image/png", + }; + + const context: ImagesContext = { + input: [{ type: "text", text: "Create a variation of this image with a blue background." }, imageContent], + }; + + const response = await generateImages(model, context, options); + + expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop"); + expect(response.output.some((item) => item.type === "image")).toBe(true); +} + +describe("Images E2E Tests", () => { + describe.skipIf(!process.env.OPENROUTER_API_KEY)( + "OpenRouter Images Provider (google/gemini-2.5-flash-image)", + () => { + const model = getImageModel("openrouter", "google/gemini-2.5-flash-image"); + + it("should generate a basic image", { retry: 3 }, async () => { + await basicImageGeneration(model); + }); + + it("should handle image streaming", { retry: 3 }, async () => { + await handleStreaming(model); + }); + + it("should handle text plus image output", { retry: 3 }, async () => { + await handleTextAndImageOutput(model); + }); + + it("should handle image input", { retry: 3 }, async () => { + await handleImageInput(model); + }); + }, + ); +}); From 074747c6873edcb734695b5c25ad1fc555ec8d5e Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Tue, 5 May 2026 10:06:27 +0200 Subject: [PATCH 16/19] docs: readme image section --- packages/ai/README.md | 115 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/packages/ai/README.md b/packages/ai/README.md index 5e5526b6..042c1bb5 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -16,6 +16,10 @@ Unified LLM API with automatic model discovery, provider configuration, token an - [Validating Tool Arguments](#validating-tool-arguments) - [Complete Event Reference](#complete-event-reference) - [Image Input](#image-input) +- [Image Generation](#image-generation) + - [Basic Image Generation](#basic-image-generation) + - [Streaming Image Events](#streaming-image-events) + - [Notes and Limitations](#notes-and-limitations) - [Thinking/Reasoning](#thinkingreasoning) - [Unified Interface](#unified-interface-streamsimplecompletesimple) - [Provider-Specific Options](#provider-specific-options-streamcomplete) @@ -419,6 +423,117 @@ for (const block of response.content) { } ``` +## Image Generation + +Image generation uses a separate API surface from text/chat generation. Use `getImageModel()` / `getImageModels()` / `getImageProviders()` to discover image-generation models, `images()` to stream events, and `generateImages()` to get the final result. + +Do not use `stream()` or `complete()` for image generation. Request options, error handling, and event/result flow follow the same overall pattern as the streaming APIs, but use image-specific types and events. + +### Basic Image Generation + +```typescript +import { getImageModel, generateImages } from '@mariozechner/pi-ai'; + +const model = getImageModel('openrouter', 'google/gemini-2.5-flash-image'); + +const result = await generateImages(model, { + input: [{ type: 'text', text: 'Generate a red circle on a plain white background.' }] +}, { + apiKey: process.env.OPENROUTER_API_KEY +}); + +for (const block of result.output) { + if (block.type === 'text') { + console.log(block.text); + } else if (block.type === 'image') { + console.log(block.mimeType); + console.log(block.data.substring(0, 32)); + } +} +``` + +Some models also support image input: + +```typescript +import { readFileSync } from 'fs'; + +const imageBuffer = readFileSync('input.png'); +const result = await generateImages(model, { + input: [ + { type: 'text', text: 'Create a variation of this image with a blue background.' }, + { type: 'image', data: imageBuffer.toString('base64'), mimeType: 'image/png' } + ] +}, { + apiKey: process.env.OPENROUTER_API_KEY +}); +``` + +Check capabilities on the model metadata: + +```typescript +console.log(model.input); // ['text', 'image'] +console.log(model.output); // ['image'] or ['image', 'text'] +``` + +### Streaming Image Events + +`images()` returns an `AssistantImagesEventStream` with image-specific events: + +```typescript +import { getImageModel, images } from '@mariozechner/pi-ai'; + +const model = getImageModel('openrouter', 'google/gemini-2.5-flash-image'); +const s = images(model, { + input: [{ type: 'text', text: 'Generate a simple green triangle.' }] +}, { + apiKey: process.env.OPENROUTER_API_KEY +}); + +for await (const event of s) { + switch (event.type) { + case 'start': + console.log(`Starting image generation with ${event.partial.model}`); + break; + case 'image_start': + console.log(`Image block started at index ${event.contentIndex}`); + break; + case 'image_end': + console.log(`Image block complete: ${event.image.mimeType}`); + break; + case 'done': + console.log(`Finished: ${event.reason}`); + break; + case 'error': + console.error(`Error: ${event.error.errorMessage}`); + break; + } +} + +const finalResult = await s.result(); +``` + +| Event Type | Description | Key Properties | +|------------|-------------|----------------| +| `start` | Stream begins | `partial`: Initial `AssistantImages` structure | +| `image_start` | Image block starts | `contentIndex`: Position in output array | +| `image_end` | Image block complete | `image`: Final `ImageContent`, `contentIndex`: Position | +| `done` | Stream complete | `reason`: Stop reason (`"stop"`), `images`: Final `AssistantImages` | +| `error` | Error occurred | `reason`: Error type (`"error"` or `"aborted"`), `error`: Final `AssistantImages` | + +Text output, if the model returns any, is available on the final `AssistantImages.output` array. + +### Notes and Limitations + +- Use `getImageModel(...)`, not `getModel(...)`. +- Use `images()` / `generateImages()`, not `stream()` / `complete()`. +- Image-generation models do not participate in tool calling. +- Outputs are returned as base64-encoded `ImageContent` blocks in `AssistantImages.output`. +- Some models return only images, others return images plus text. Check `model.output`. +- Some models accept image input, others are text-to-image only. Check `model.input`. +- Like the streaming APIs, image generation supports options such as `apiKey`, `signal`, `headers`, `onPayload`, and `onResponse`, and results may include `stopReason`, `responseId`, and `usage`. +- If you want a model to analyze images in a conversation or call tools, use the regular `stream()` / `complete()` APIs with a model that supports image input. +- At the moment, image generation is available through only one provider, OpenRouter. + ## Thinking/Reasoning Many models support thinking/reasoning capabilities where they can show their internal thought process. You can check if a model supports reasoning via the `reasoning` property. If you pass reasoning options to a non-reasoning model, they are silently ignored. From 32b2cd07208115b842a6567c2c8ccb4b3c359f11 Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Thu, 7 May 2026 16:21:50 +0200 Subject: [PATCH 17/19] delete: images() --- packages/ai/README.md | 56 +------- packages/ai/src/images-api-registry.ts | 23 ++- packages/ai/src/images.ts | 23 +-- .../ai/src/providers/images/openrouter.ts | 132 ++++++++---------- .../src/providers/images/register-builtins.ts | 68 +-------- packages/ai/src/types.ts | 16 +-- packages/ai/src/utils/event-stream.ts | 23 +-- packages/ai/test/images.test.ts | 36 +---- packages/ai/test/openrouter-images.test.ts | 51 +++++-- 9 files changed, 119 insertions(+), 309 deletions(-) diff --git a/packages/ai/README.md b/packages/ai/README.md index 042c1bb5..6228dfb3 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -18,7 +18,6 @@ Unified LLM API with automatic model discovery, provider configuration, token an - [Image Input](#image-input) - [Image Generation](#image-generation) - [Basic Image Generation](#basic-image-generation) - - [Streaming Image Events](#streaming-image-events) - [Notes and Limitations](#notes-and-limitations) - [Thinking/Reasoning](#thinkingreasoning) - [Unified Interface](#unified-interface-streamsimplecompletesimple) @@ -425,9 +424,9 @@ for (const block of response.content) { ## Image Generation -Image generation uses a separate API surface from text/chat generation. Use `getImageModel()` / `getImageModels()` / `getImageProviders()` to discover image-generation models, `images()` to stream events, and `generateImages()` to get the final result. +Image generation uses a separate API surface from text/chat generation. Use `getImageModel()` / `getImageModels()` / `getImageProviders()` to discover image-generation models, and `generateImages()` to get the final result. -Do not use `stream()` or `complete()` for image generation. Request options, error handling, and event/result flow follow the same overall pattern as the streaming APIs, but use image-specific types and events. +Do not use `stream()` or `complete()` for image generation. Image generation is a one-shot API: `generateImages()` waits for the provider response and returns the final `AssistantImages` result. ### Basic Image Generation @@ -475,59 +474,12 @@ console.log(model.input); // ['text', 'image'] console.log(model.output); // ['image'] or ['image', 'text'] ``` -### Streaming Image Events - -`images()` returns an `AssistantImagesEventStream` with image-specific events: - -```typescript -import { getImageModel, images } from '@mariozechner/pi-ai'; - -const model = getImageModel('openrouter', 'google/gemini-2.5-flash-image'); -const s = images(model, { - input: [{ type: 'text', text: 'Generate a simple green triangle.' }] -}, { - apiKey: process.env.OPENROUTER_API_KEY -}); - -for await (const event of s) { - switch (event.type) { - case 'start': - console.log(`Starting image generation with ${event.partial.model}`); - break; - case 'image_start': - console.log(`Image block started at index ${event.contentIndex}`); - break; - case 'image_end': - console.log(`Image block complete: ${event.image.mimeType}`); - break; - case 'done': - console.log(`Finished: ${event.reason}`); - break; - case 'error': - console.error(`Error: ${event.error.errorMessage}`); - break; - } -} - -const finalResult = await s.result(); -``` - -| Event Type | Description | Key Properties | -|------------|-------------|----------------| -| `start` | Stream begins | `partial`: Initial `AssistantImages` structure | -| `image_start` | Image block starts | `contentIndex`: Position in output array | -| `image_end` | Image block complete | `image`: Final `ImageContent`, `contentIndex`: Position | -| `done` | Stream complete | `reason`: Stop reason (`"stop"`), `images`: Final `AssistantImages` | -| `error` | Error occurred | `reason`: Error type (`"error"` or `"aborted"`), `error`: Final `AssistantImages` | - -Text output, if the model returns any, is available on the final `AssistantImages.output` array. - ### Notes and Limitations - Use `getImageModel(...)`, not `getModel(...)`. -- Use `images()` / `generateImages()`, not `stream()` / `complete()`. +- Use `generateImages()`, not `stream()` / `complete()`. - Image-generation models do not participate in tool calling. -- Outputs are returned as base64-encoded `ImageContent` blocks in `AssistantImages.output`. +- Outputs are returned in `AssistantImages.output` and can include both base64-encoded `ImageContent` blocks and `TextContent` blocks. - Some models return only images, others return images plus text. Check `model.output`. - Some models accept image input, others are text-to-image only. Check `model.input`. - Like the streaming APIs, image generation supports options such as `apiKey`, `signal`, `headers`, `onPayload`, and `onResponse`, and results may include `stopReason`, `responseId`, and `usage`. diff --git a/packages/ai/src/images-api-registry.ts b/packages/ai/src/images-api-registry.ts index 395a060d..2a0c0fd9 100644 --- a/packages/ai/src/images-api-registry.ts +++ b/packages/ai/src/images-api-registry.ts @@ -1,26 +1,19 @@ -import type { - AssistantImagesEventStream, - ImagesApi, - ImagesContext, - ImagesFunction, - ImagesModel, - ImagesOptions, -} from "./types.js"; +import type { AssistantImages, ImagesApi, ImagesContext, ImagesFunction, ImagesModel, ImagesOptions } from "./types.js"; export type ImagesApiFunction = ( model: ImagesModel, context: ImagesContext, options?: ImagesOptions, -) => AssistantImagesEventStream; +) => Promise; export interface ImagesApiProvider { api: TApi; - images: ImagesFunction; + generateImages: ImagesFunction; } interface ImagesApiProviderInternal { api: ImagesApi; - images: ImagesApiFunction; + generateImages: ImagesApiFunction; } type RegisteredImagesApiProvider = { @@ -30,15 +23,15 @@ type RegisteredImagesApiProvider = { const imagesApiProviderRegistry = new Map(); -function wrapImages( +function wrapGenerateImages( api: TApi, - images: ImagesFunction, + generateImages: ImagesFunction, ): ImagesApiFunction { return (model, context, options) => { if (model.api !== api) { throw new Error(`Mismatched api: ${model.api} expected ${api}`); } - return images(model as ImagesModel, context, options as TOptions); + return generateImages(model as ImagesModel, context, options as TOptions); }; } @@ -49,7 +42,7 @@ export function registerImagesApiProvider( - model: ImagesModel, - context: ImagesContext, - options?: ProviderImagesOptions, -): AssistantImagesEventStream { - const provider = resolveImagesApiProvider(model.api); - return provider.images(model, context, options as ImagesOptions); -} - export async function generateImages( model: ImagesModel, context: ImagesContext, options?: ProviderImagesOptions, ): Promise { - const s = images(model, context, options); - return s.result(); + const provider = resolveImagesApiProvider(model.api); + return provider.generateImages(model, context, options); } diff --git a/packages/ai/src/providers/images/openrouter.ts b/packages/ai/src/providers/images/openrouter.ts index c1a5d966..8a7f91d3 100644 --- a/packages/ai/src/providers/images/openrouter.ts +++ b/packages/ai/src/providers/images/openrouter.ts @@ -16,7 +16,6 @@ import type { ImagesOptions, TextContent, } from "../../types.js"; -import { AssistantImagesEventStream } from "../../utils/event-stream.js"; import { headersToRecord } from "../../utils/headers.js"; import { sanitizeSurrogates } from "../../utils/sanitize-unicode.js"; @@ -36,87 +35,70 @@ type OpenRouterImageGenerationResponse = ChatCompletion & { choices: OpenRouterImageGenerationChoice[]; }; -export const imagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = ( +export const generateImagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = async ( model: ImagesModel<"openrouter-images">, context: ImagesContext, options?: ImagesOptions, ) => { - const stream = new AssistantImagesEventStream(); + const output: AssistantImages = { + api: model.api, + provider: model.provider, + model: model.id, + output: [], + stopReason: "stop", + timestamp: Date.now(), + }; - (async () => { - const output: AssistantImages = { - api: model.api, - provider: model.provider, - model: model.id, - output: [], - stopReason: "stop", - timestamp: Date.now(), - }; - - try { - const apiKey = options?.apiKey || getEnvApiKey(model.provider) || ""; - const client = createClient(model, apiKey, options?.headers); - let params = buildParams(model, context); - const nextParams = await options?.onPayload?.(params, model); - if (nextParams !== undefined) { - params = nextParams as typeof params; - } - const requestOptions = { - ...(options?.signal ? { signal: options.signal } : {}), - ...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}), - ...(options?.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}), - }; - const { data: response, response: rawResponse } = await client.chat.completions - .create(params as unknown as ChatCompletionCreateParamsNonStreaming, requestOptions) - .withResponse(); - await options?.onResponse?.( - { status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, - model, - ); - - stream.push({ type: "start", partial: output }); - - const imageResponse = response as OpenRouterImageGenerationResponse; - output.responseId = imageResponse.id; - if (imageResponse.usage) { - output.usage = parseUsage(imageResponse.usage, model); - } - - const choice = imageResponse.choices[0]; - if (choice) { - const content = choice.message.content; - if (typeof content === "string" && content.length > 0) { - output.output.push({ type: "text", text: content } satisfies TextContent); - } - - for (const image of choice.message.images ?? []) { - const imageUrl = typeof image.image_url === "string" ? image.image_url : image.image_url?.url; - if (!imageUrl?.startsWith("data:")) continue; - const matches = imageUrl.match(/^data:([^;]+);base64,(.+)$/); - if (!matches) continue; - const imageBlock: ImageContent = { - type: "image", - mimeType: matches[1], - data: matches[2], - }; - output.output.push(imageBlock); - const contentIndex = output.output.length - 1; - stream.push({ type: "image_start", contentIndex, partial: output }); - stream.push({ type: "image_end", contentIndex, image: imageBlock, partial: output }); - } - } - - stream.push({ type: "done", reason: "stop", images: output }); - stream.end(); - } catch (error) { - output.stopReason = options?.signal?.aborted ? "aborted" : "error"; - output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error); - stream.push({ type: "error", reason: output.stopReason, error: output }); - stream.end(output); + try { + const apiKey = options?.apiKey || getEnvApiKey(model.provider) || ""; + const client = createClient(model, apiKey, options?.headers); + let params = buildParams(model, context); + const nextParams = await options?.onPayload?.(params, model); + if (nextParams !== undefined) { + params = nextParams as typeof params; } - })(); + const requestOptions = { + ...(options?.signal ? { signal: options.signal } : {}), + ...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}), + ...(options?.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}), + }; + const { data: response, response: rawResponse } = await client.chat.completions + .create(params as unknown as ChatCompletionCreateParamsNonStreaming, requestOptions) + .withResponse(); + await options?.onResponse?.({ status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, model); - return stream; + const imageResponse = response as OpenRouterImageGenerationResponse; + output.responseId = imageResponse.id; + if (imageResponse.usage) { + output.usage = parseUsage(imageResponse.usage, model); + } + + const choice = imageResponse.choices[0]; + if (choice) { + const content = choice.message.content; + if (typeof content === "string" && content.length > 0) { + output.output.push({ type: "text", text: content } satisfies TextContent); + } + + for (const image of choice.message.images ?? []) { + const imageUrl = typeof image.image_url === "string" ? image.image_url : image.image_url?.url; + if (!imageUrl?.startsWith("data:")) continue; + const matches = imageUrl.match(/^data:([^;]+);base64,(.+)$/); + if (!matches) continue; + output.output.push({ + type: "image", + mimeType: matches[1], + data: matches[2], + } satisfies ImageContent); + } + } + + return output; + } catch (error) { + output.stopReason = options?.signal?.aborted ? "aborted" : "error"; + output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error); + return output; + } }; function createClient( diff --git a/packages/ai/src/providers/images/register-builtins.ts b/packages/ai/src/providers/images/register-builtins.ts index 7a874b2a..f7d1adc3 100644 --- a/packages/ai/src/providers/images/register-builtins.ts +++ b/packages/ai/src/providers/images/register-builtins.ts @@ -1,74 +1,10 @@ import { registerImagesApiProvider } from "../../images-api-registry.js"; -import type { - AssistantImages, - AssistantImagesEvent, - ImagesContext, - ImagesFunction, - ImagesModel, - ImagesOptions, -} from "../../types.js"; -import { AssistantImagesEventStream } from "../../utils/event-stream.js"; -import type { imagesOpenRouter as imagesOpenRouterFunction } from "./openrouter.js"; - -interface OpenRouterImagesProviderModule { - imagesOpenRouter: typeof imagesOpenRouterFunction; -} - -let openRouterImagesProviderModulePromise: Promise | undefined; - -function forwardImagesStream(target: AssistantImagesEventStream, source: AsyncIterable): void { - (async () => { - for await (const event of source) { - target.push(event); - } - target.end(); - })(); -} - -function createLazyLoadErrorImages(model: ImagesModel<"openrouter-images">, error: unknown): AssistantImages { - return { - api: model.api, - provider: model.provider, - model: model.id, - output: [], - stopReason: "error", - errorMessage: error instanceof Error ? error.message : String(error), - timestamp: Date.now(), - }; -} - -function loadOpenRouterImagesProviderModule(): Promise { - openRouterImagesProviderModulePromise ||= import("./openrouter.js").then( - (module) => module as OpenRouterImagesProviderModule, - ); - return openRouterImagesProviderModulePromise; -} - -export const imagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = ( - model: ImagesModel<"openrouter-images">, - context: ImagesContext, - options?: ImagesOptions, -) => { - const outer = new AssistantImagesEventStream(); - - loadOpenRouterImagesProviderModule() - .then((module) => { - const inner = module.imagesOpenRouter(model, context, options); - forwardImagesStream(outer, inner); - }) - .catch((error) => { - const images = createLazyLoadErrorImages(model, error); - outer.push({ type: "error", reason: "error", error: images }); - outer.end(images); - }); - - return outer; -}; +import { generateImagesOpenRouter } from "./openrouter.js"; export function registerBuiltInImagesApiProviders(): void { registerImagesApiProvider({ api: "openrouter-images", - images: imagesOpenRouter, + generateImages: generateImagesOpenRouter, }); } diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 29bc6bb8..db6ef893 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -1,10 +1,7 @@ import type { AssistantMessageDiagnostic } from "./utils/diagnostics.js"; -import type { AssistantImagesEventStream, AssistantMessageEventStream } from "./utils/event-stream.js"; +import type { AssistantMessageEventStream } from "./utils/event-stream.js"; -export type { - AssistantImagesEventStream, - AssistantMessageEventStream, -} from "./utils/event-stream.js"; +export type { AssistantMessageEventStream } from "./utils/event-stream.js"; export type KnownApi = | "openai-completions" @@ -215,7 +212,7 @@ export type ImagesFunction, context: ImagesContext, options?: TOptions, -) => AssistantImagesEventStream; +) => Promise; export interface TextSignatureV1 { v: 1; @@ -360,13 +357,6 @@ export type AssistantMessageEvent = | { type: "done"; reason: Extract; message: AssistantMessage } | { type: "error"; reason: Extract; error: AssistantMessage }; -export type AssistantImagesEvent = - | { type: "start"; partial: AssistantImages } - | { type: "image_start"; contentIndex: number; partial: AssistantImages } - | { type: "image_end"; contentIndex: number; image: ImageContent; partial: AssistantImages } - | { type: "done"; reason: Extract; images: AssistantImages } - | { type: "error"; reason: Extract; error: AssistantImages }; - /** * Compatibility settings for OpenAI-compatible completions APIs. * Use this to override URL-based auto-detection for custom providers. diff --git a/packages/ai/src/utils/event-stream.ts b/packages/ai/src/utils/event-stream.ts index 9fb1c114..f4a7ceba 100644 --- a/packages/ai/src/utils/event-stream.ts +++ b/packages/ai/src/utils/event-stream.ts @@ -1,4 +1,4 @@ -import type { AssistantImages, AssistantImagesEvent, AssistantMessage, AssistantMessageEvent } from "../types.js"; +import type { AssistantMessage, AssistantMessageEvent } from "../types.js"; // Generic event stream class for async iteration export class EventStream implements AsyncIterable { @@ -81,28 +81,7 @@ export class AssistantMessageEventStream extends EventStream { - constructor() { - super( - (event) => event.type === "done" || event.type === "error", - (event) => { - if (event.type === "done") { - return event.images; - } else if (event.type === "error") { - return event.error; - } - throw new Error("Unexpected event type for final result"); - }, - ); - } -} - /** Factory function for AssistantMessageEventStream (for use in extensions) */ export function createAssistantMessageEventStream(): AssistantMessageEventStream { return new AssistantMessageEventStream(); } - -/** Factory function for AssistantImagesEventStream (for use in extensions) */ -export function createAssistantImagesEventStream(): AssistantImagesEventStream { - return new AssistantImagesEventStream(); -} diff --git a/packages/ai/test/images.test.ts b/packages/ai/test/images.test.ts index 84aac37a..59912c70 100644 --- a/packages/ai/test/images.test.ts +++ b/packages/ai/test/images.test.ts @@ -3,7 +3,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; import { getImageModel } from "../src/image-models.js"; -import { generateImages, images } from "../src/images.js"; +import { generateImages } from "../src/images.js"; import type { ImageContent, ImagesContext, ImagesModel, ProviderImagesOptions } from "../src/types.js"; const __filename = fileURLToPath(import.meta.url); @@ -24,36 +24,6 @@ async function basicImageGeneration(model: ImagesModel(model: ImagesModel, options?: ImagesOptionsWithExtras) { - const context: ImagesContext = { - input: [{ type: "text", text: "Generate a simple blue square on a plain white background. No text." }], - }; - - const s = images(model, context, options); - let started = false; - let imageStarted = false; - let imageCompleted = false; - - for await (const event of s) { - if (event.type === "start") { - started = true; - } else if (event.type === "image_start") { - imageStarted = true; - } else if (event.type === "image_end") { - imageCompleted = true; - expect(event.image.type).toBe("image"); - expect(event.image.data.length).toBeGreaterThan(0); - } - } - - const response = await s.result(); - expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop"); - expect(started).toBe(true); - expect(imageStarted).toBe(true); - expect(imageCompleted).toBe(true); - expect(response.output.some((item) => item.type === "image")).toBe(true); -} - async function handleTextAndImageOutput( model: ImagesModel, options?: ImagesOptionsWithExtras, @@ -108,10 +78,6 @@ describe("Images E2E Tests", () => { await basicImageGeneration(model); }); - it("should handle image streaming", { retry: 3 }, async () => { - await handleStreaming(model); - }); - it("should handle text plus image output", { retry: 3 }, async () => { await handleTextAndImageOutput(model); }); diff --git a/packages/ai/test/openrouter-images.test.ts b/packages/ai/test/openrouter-images.test.ts index b4a79127..de2ddbba 100644 --- a/packages/ai/test/openrouter-images.test.ts +++ b/packages/ai/test/openrouter-images.test.ts @@ -1,17 +1,28 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { generateImages, images } from "../src/images.js"; +import { generateImages } from "../src/images.js"; import type { ImagesContext, ImagesModel } from "../src/types.js"; const mockState = vi.hoisted(() => ({ lastParams: undefined as unknown, + lastRequestOptions: undefined as unknown, })); vi.mock("openai", () => { class FakeOpenAI { chat = { completions: { - create: (params: unknown) => { + create: (params: unknown, requestOptions?: unknown) => { mockState.lastParams = params; + mockState.lastRequestOptions = requestOptions; + const signal = (requestOptions as { signal?: AbortSignal } | undefined)?.signal; + if (signal?.aborted) { + const error = new Error("Request aborted"); + return { + withResponse: async () => { + throw error; + }, + }; + } const response = { id: "img-1", usage: { @@ -50,9 +61,10 @@ vi.mock("openai", () => { describe("openrouter images", () => { beforeEach(() => { mockState.lastParams = undefined; + mockState.lastRequestOptions = undefined; }); - it("emits image events and returns text plus images in final output", async () => { + it("returns text plus images in final output", async () => { const model: ImagesModel<"openrouter-images"> = { id: "google/gemini-3.1-flash-image-preview", name: "Gemini 3.1 Flash Image Preview", @@ -68,14 +80,8 @@ describe("openrouter images", () => { input: [{ type: "text", text: "Generate a dog" }], }; - const result = images(model, context, { apiKey: "test" }); - const eventTypes: string[] = []; - for await (const event of result) { - eventTypes.push(event.type); - } - - const output = await result.result(); - expect(eventTypes).toEqual(["start", "image_start", "image_end", "done"]); + const output = await generateImages(model, context, { apiKey: "test" }); + expect(output.stopReason).toBe("stop"); expect(output.responseId).toBe("img-1"); expect(output.output[0]).toMatchObject({ type: "text", text: "Here is your image." }); expect(output.output[1]).toMatchObject({ type: "image", mimeType: "image/png", data: "ZmFrZS1wbmc=" }); @@ -90,6 +96,29 @@ describe("openrouter images", () => { expect(params.messages?.[0]?.content?.[0]).toMatchObject({ type: "text", text: "Generate a dog" }); }); + it("passes through abort signal and returns aborted result", async () => { + const model: ImagesModel<"openrouter-images"> = { + id: "black-forest-labs/flux.2-pro", + name: "FLUX.2 Pro", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { input: 0.015, output: 0.03, cacheRead: 0, cacheWrite: 0 }, + }; + const context: ImagesContext = { + input: [{ type: "text", text: "Generate a dog" }], + }; + const controller = new AbortController(); + controller.abort(); + + const output = await generateImages(model, context, { apiKey: "test", signal: controller.signal }); + expect(output.stopReason).toBe("aborted"); + expect(output.errorMessage).toBe("Request aborted"); + expect(mockState.lastRequestOptions).toMatchObject({ signal: controller.signal }); + }); + it("generateImages resolves the final assistant images result", async () => { const model: ImagesModel<"openrouter-images"> = { id: "black-forest-labs/flux.2-pro", From 0e4f845cf4e63a9d7af324bf25c48852cd7d5526 Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Fri, 8 May 2026 11:08:04 +0200 Subject: [PATCH 18/19] fix: lazy loading --- .../src/providers/images/register-builtins.ts | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/packages/ai/src/providers/images/register-builtins.ts b/packages/ai/src/providers/images/register-builtins.ts index f7d1adc3..04338f20 100644 --- a/packages/ai/src/providers/images/register-builtins.ts +++ b/packages/ai/src/providers/images/register-builtins.ts @@ -1,5 +1,44 @@ import { registerImagesApiProvider } from "../../images-api-registry.js"; -import { generateImagesOpenRouter } from "./openrouter.js"; +import type { AssistantImages, ImagesContext, ImagesFunction, ImagesModel, ImagesOptions } from "../../types.js"; +import type { generateImagesOpenRouter as generateImagesOpenRouterFunction } from "./openrouter.js"; + +interface OpenRouterImagesProviderModule { + generateImagesOpenRouter: typeof generateImagesOpenRouterFunction; +} + +let openRouterImagesProviderModulePromise: Promise | undefined; + +function createLazyLoadErrorImages(model: ImagesModel<"openrouter-images">, error: unknown): AssistantImages { + return { + api: model.api, + provider: model.provider, + model: model.id, + output: [], + stopReason: "error", + errorMessage: error instanceof Error ? error.message : String(error), + timestamp: Date.now(), + }; +} + +function loadOpenRouterImagesProviderModule(): Promise { + openRouterImagesProviderModulePromise ||= import("./openrouter.js").then( + (module) => module as OpenRouterImagesProviderModule, + ); + return openRouterImagesProviderModulePromise; +} + +export const generateImagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = async ( + model: ImagesModel<"openrouter-images">, + context: ImagesContext, + options?: ImagesOptions, +) => { + try { + const module = await loadOpenRouterImagesProviderModule(); + return await module.generateImagesOpenRouter(model, context, options); + } catch (error) { + return createLazyLoadErrorImages(model, error); + } +}; export function registerBuiltInImagesApiProviders(): void { registerImagesApiProvider({ From 8fa1194c2322a82ce1215724d3856a3bb5e5d314 Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Fri, 8 May 2026 11:23:15 +0200 Subject: [PATCH 19/19] fix: no fallback for apiKey --- packages/ai/src/providers/images/openrouter.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/ai/src/providers/images/openrouter.ts b/packages/ai/src/providers/images/openrouter.ts index 8a7f91d3..70c95a8c 100644 --- a/packages/ai/src/providers/images/openrouter.ts +++ b/packages/ai/src/providers/images/openrouter.ts @@ -50,7 +50,10 @@ export const generateImagesOpenRouter: ImagesFunction<"openrouter-images", Image }; try { - const apiKey = options?.apiKey || getEnvApiKey(model.provider) || ""; + const apiKey = options?.apiKey || getEnvApiKey(model.provider); + if (!apiKey) { + throw new Error(`No API key available for provider: ${model.provider}`); + } const client = createClient(model, apiKey, options?.headers); let params = buildParams(model, context); const nextParams = await options?.onPayload?.(params, model);