diff --git a/packages/ai/README.md b/packages/ai/README.md index 6bed1bbb..5d35f217 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -16,6 +16,9 @@ Unified LLM API with automatic model discovery, provider configuration, token an - [Validating Tool Arguments](#validating-tool-arguments) - [Complete Event Reference](#complete-event-reference) - [Image Input](#image-input) +- [Image Generation](#image-generation) + - [Basic Image Generation](#basic-image-generation) + - [Notes and Limitations](#notes-and-limitations) - [Thinking/Reasoning](#thinkingreasoning) - [Unified Interface](#unified-interface-streamsimplecompletesimple) - [Provider-Specific Options](#provider-specific-options-streamcomplete) @@ -421,6 +424,70 @@ for (const block of response.content) { } ``` +## Image Generation + +Image generation uses a separate API surface from text/chat generation. Use `getImageModel()` / `getImageModels()` / `getImageProviders()` to discover image-generation models, and `generateImages()` to get the final result. + +Do not use `stream()` or `complete()` for image generation. Image generation is a one-shot API: `generateImages()` waits for the provider response and returns the final `AssistantImages` result. + +### Basic Image Generation + +```typescript +import { getImageModel, generateImages } from '@mariozechner/pi-ai'; + +const model = getImageModel('openrouter', 'google/gemini-2.5-flash-image'); + +const result = await generateImages(model, { + input: [{ type: 'text', text: 'Generate a red circle on a plain white background.' }] +}, { + apiKey: process.env.OPENROUTER_API_KEY +}); + +for (const block of result.output) { + if (block.type === 'text') { + console.log(block.text); + } else if (block.type === 'image') { + console.log(block.mimeType); + console.log(block.data.substring(0, 32)); + } +} +``` + +Some models also support image input: + +```typescript +import { readFileSync } from 'fs'; + +const imageBuffer = readFileSync('input.png'); +const result = await generateImages(model, { + input: [ + { type: 'text', text: 'Create a variation of this image with a blue background.' }, + { type: 'image', data: imageBuffer.toString('base64'), mimeType: 'image/png' } + ] +}, { + apiKey: process.env.OPENROUTER_API_KEY +}); +``` + +Check capabilities on the model metadata: + +```typescript +console.log(model.input); // ['text', 'image'] +console.log(model.output); // ['image'] or ['image', 'text'] +``` + +### Notes and Limitations + +- Use `getImageModel(...)`, not `getModel(...)`. +- Use `generateImages()`, not `stream()` / `complete()`. +- Image-generation models do not participate in tool calling. +- Outputs are returned in `AssistantImages.output` and can include both base64-encoded `ImageContent` blocks and `TextContent` blocks. +- Some models return only images, others return images plus text. Check `model.output`. +- Some models accept image input, others are text-to-image only. Check `model.input`. +- Like the streaming APIs, image generation supports options such as `apiKey`, `signal`, `headers`, `onPayload`, and `onResponse`, and results may include `stopReason`, `responseId`, and `usage`. +- If you want a model to analyze images in a conversation or call tools, use the regular `stream()` / `complete()` APIs with a model that supports image input. +- At the moment, image generation is available through only one provider, OpenRouter. + ## Thinking/Reasoning Many models support thinking/reasoning capabilities where they can show their internal thought process. You can check if a model supports reasoning via the `reasoning` property. If you pass reasoning options to a non-reasoning model, they are silently ignored. @@ -1251,10 +1318,11 @@ Create a new provider file (for example `amazon-bedrock.ts`) that exports: - Add credential detection in `env-api-keys.ts` for the new provider - Ensure `streamSimple` handles auth lookup via `getEnvApiKey()` or provider-specific auth -#### 4. Model Generation (`scripts/generate-models.ts`) +#### 4. Model Generation (`scripts/generate-models.ts`, `scripts/generate-image-models.ts`) - Add logic to fetch and parse models from the provider's source (e.g., models.dev API) -- Map provider model data to the standardized `Model` interface +- Map chat/tool-capable provider model data to the standardized `Model` interface via `scripts/generate-models.ts` +- Map image-generation provider model data to the standardized `ImagesModel` interface via `scripts/generate-image-models.ts` - Handle provider-specific quirks (pricing format, capability flags, model ID transformations) #### 5. Tests (`test/`) diff --git a/packages/ai/package.json b/packages/ai/package.json index 79c8dc3c..f8d7b260 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -61,7 +61,8 @@ "scripts": { "clean": "shx rm -rf dist", "generate-models": "npx tsx scripts/generate-models.ts", - "build": "npm run generate-models && tsgo -p tsconfig.build.json", + "generate-image-models": "npx tsx scripts/generate-image-models.ts", + "build": "npm run generate-models && npm run generate-image-models && tsgo -p tsconfig.build.json", "dev": "tsgo -p tsconfig.build.json --watch --preserveWatchOutput", "dev:tsc": "tsgo -p tsconfig.build.json --watch --preserveWatchOutput", "test": "vitest --run", diff --git a/packages/ai/scripts/generate-image-models.ts b/packages/ai/scripts/generate-image-models.ts new file mode 100644 index 00000000..bd0c6727 --- /dev/null +++ b/packages/ai/scripts/generate-image-models.ts @@ -0,0 +1,131 @@ +#!/usr/bin/env tsx + +import { writeFileSync } from "fs"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import type { ImagesModel } from "../src/types.js"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const packageRoot = join(__dirname, ".."); +const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"; + +interface OpenRouterModelRecord { + id: string; + name: string; + context_length?: number; + architecture?: { + input_modalities?: string[]; + output_modalities?: string[]; + }; + pricing?: { + prompt?: string; + completion?: string; + input_cache_read?: string; + input_cache_write?: string; + }; +} + +async function fetchOpenRouterImageModels(): Promise[]> { + try { + console.log("Fetching image models from OpenRouter API..."); + const response = await fetch(`${OPENROUTER_BASE_URL}/models?output_modalities=image`); + const data = (await response.json()) as { data?: OpenRouterModelRecord[] }; + const models: ImagesModel<"openrouter-images">[] = []; + + for (const model of data.data ?? []) { + const input = Array.from( + new Set( + (model.architecture?.input_modalities ?? []) + .filter((modality): modality is "text" | "image" => modality === "text" || modality === "image"), + ), + ); + const output = Array.from( + new Set( + (model.architecture?.output_modalities ?? []).filter( + (modality): modality is "text" | "image" => modality === "text" || modality === "image", + ), + ), + ); + + if (!output.includes("image")) continue; + if (input.length === 0) input.push("text"); + + models.push({ + id: model.id, + name: model.name, + api: "openrouter-images", + provider: "openrouter", + baseUrl: OPENROUTER_BASE_URL, + input, + output, + cost: { + input: parseFloat(model.pricing?.prompt || "0") * 1_000_000, + output: parseFloat(model.pricing?.completion || "0") * 1_000_000, + cacheRead: parseFloat(model.pricing?.input_cache_read || "0") * 1_000_000, + cacheWrite: parseFloat(model.pricing?.input_cache_write || "0") * 1_000_000, + }, + }); + } + + console.log(`Fetched ${models.length} image models from OpenRouter`); + return models; + } catch (error) { + console.error("Failed to fetch OpenRouter image models:", error); + return []; + } +} + +function generateImageModelsFile(models: ImagesModel<"openrouter-images">[]): string { + const imageModelsByProvider = { + openrouter: Object.fromEntries( + models + .sort((a, b) => a.id.localeCompare(b.id)) + .map((model) => [ + model.id, + `{ + id: ${JSON.stringify(model.id)}, + name: ${JSON.stringify(model.name)}, + api: ${JSON.stringify(model.api)}, + provider: ${JSON.stringify(model.provider)}, + baseUrl: ${JSON.stringify(model.baseUrl)}, + input: ${JSON.stringify(model.input)}, + output: ${JSON.stringify(model.output)}, + cost: ${JSON.stringify(model.cost, null, 2).replace(/^/gm, "\t")} + } satisfies ImagesModel<${JSON.stringify(model.api)}>`, + ]), + ), + }; + + const providerEntries = Object.entries(imageModelsByProvider) + .map(([provider, providerModels]) => { + const modelEntries = Object.entries(providerModels) + .map(([id, serialized]) => `\t\t${JSON.stringify(id)}: ${serialized},`) + .join("\n"); + return `\t${JSON.stringify(provider)}: {\n${modelEntries}\n\t},`; + }) + .join("\n"); + + return `// This file is auto-generated by scripts/generate-image-models.ts +// Do not edit manually - run 'npm run generate-image-models' to update + +import type { ImagesApi, ImagesModel } from "./types.js"; + +export const IMAGE_MODELS = { +${providerEntries} +} as const satisfies Record>>; +`; +} + +async function main(): Promise { + const models = await fetchOpenRouterImageModels(); + const output = generateImageModelsFile(models); + const outputPath = join(packageRoot, "src", "image-models.generated.ts"); + writeFileSync(outputPath, output, "utf-8"); + console.log(`Generated ${outputPath}`); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/packages/ai/src/image-models.generated.ts b/packages/ai/src/image-models.generated.ts new file mode 100644 index 00000000..9bbee4c8 --- /dev/null +++ b/packages/ai/src/image-models.generated.ts @@ -0,0 +1,264 @@ +// This file is auto-generated by scripts/generate-image-models.ts +// Do not edit manually - run 'npm run generate-image-models' to update + +import type { ImagesApi, ImagesModel } from "./types.js"; + +export const IMAGE_MODELS = { + openrouter: { + "black-forest-labs/flux.2-flex": { + id: "black-forest-labs/flux.2-flex", + name: "Black Forest Labs: FLUX.2 Flex", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "black-forest-labs/flux.2-klein-4b": { + id: "black-forest-labs/flux.2-klein-4b", + name: "Black Forest Labs: FLUX.2 Klein 4B", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "black-forest-labs/flux.2-max": { + id: "black-forest-labs/flux.2-max", + name: "Black Forest Labs: FLUX.2 Max", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "black-forest-labs/flux.2-pro": { + id: "black-forest-labs/flux.2-pro", + name: "Black Forest Labs: FLUX.2 Pro", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "bytedance-seed/seedream-4.5": { + id: "bytedance-seed/seedream-4.5", + name: "ByteDance Seed: Seedream 4.5", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["image", "text"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "google/gemini-2.5-flash-image": { + id: "google/gemini-2.5-flash-image", + name: "Google: Nano Banana (Gemini 2.5 Flash Image)", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["image", "text"], + output: ["image", "text"], + cost: { + input: 0.3, + output: 2.5, + cacheRead: 0.03, + cacheWrite: 0.08333333333333334, + }, + } satisfies ImagesModel<"openrouter-images">, + "google/gemini-3-pro-image-preview": { + id: "google/gemini-3-pro-image-preview", + name: "Google: Nano Banana Pro (Gemini 3 Pro Image Preview)", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["image", "text"], + output: ["image", "text"], + cost: { + input: 2, + output: 12, + cacheRead: 0.19999999999999998, + cacheWrite: 0.375, + }, + } satisfies ImagesModel<"openrouter-images">, + "google/gemini-3.1-flash-image-preview": { + id: "google/gemini-3.1-flash-image-preview", + name: "Google: Nano Banana 2 (Gemini 3.1 Flash Image Preview)", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["image", "text"], + output: ["image", "text"], + cost: { + input: 0.5, + output: 3, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "openai/gpt-5-image": { + id: "openai/gpt-5-image", + name: "OpenAI: GPT-5 Image", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["image", "text"], + output: ["image", "text"], + cost: { + input: 10, + output: 10, + cacheRead: 1.25, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "openai/gpt-5-image-mini": { + id: "openai/gpt-5-image-mini", + name: "OpenAI: GPT-5 Image Mini", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["image", "text"], + output: ["image", "text"], + cost: { + input: 2.5, + output: 2, + cacheRead: 0.25, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "openai/gpt-5.4-image-2": { + id: "openai/gpt-5.4-image-2", + name: "OpenAI: GPT-5.4 Image 2", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["image", "text"], + output: ["image", "text"], + cost: { + input: 8, + output: 15, + cacheRead: 2, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "openrouter/auto": { + id: "openrouter/auto", + name: "Auto Router", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["text", "image"], + cost: { + input: -1000000, + output: -1000000, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "sourceful/riverflow-v2-fast": { + id: "sourceful/riverflow-v2-fast", + name: "Sourceful: Riverflow V2 Fast", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "sourceful/riverflow-v2-fast-preview": { + id: "sourceful/riverflow-v2-fast-preview", + name: "Sourceful: Riverflow V2 Fast Preview", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "sourceful/riverflow-v2-max-preview": { + id: "sourceful/riverflow-v2-max-preview", + name: "Sourceful: Riverflow V2 Max Preview", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "sourceful/riverflow-v2-pro": { + id: "sourceful/riverflow-v2-pro", + name: "Sourceful: Riverflow V2 Pro", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + "sourceful/riverflow-v2-standard-preview": { + id: "sourceful/riverflow-v2-standard-preview", + name: "Sourceful: Riverflow V2 Standard Preview", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + }, + } satisfies ImagesModel<"openrouter-images">, + }, +} as const satisfies Record>>; diff --git a/packages/ai/src/image-models.ts b/packages/ai/src/image-models.ts new file mode 100644 index 00000000..afa4f415 --- /dev/null +++ b/packages/ai/src/image-models.ts @@ -0,0 +1,42 @@ +import { IMAGE_MODELS } from "./image-models.generated.js"; +import type { ImagesApi, ImagesModel, KnownImagesProvider } from "./types.js"; + +const imageModelRegistry: Map>> = new Map(); + +for (const [provider, models] of Object.entries(IMAGE_MODELS)) { + const providerModels = new Map>(); + for (const [id, model] of Object.entries(models)) { + providerModels.set(id, model as ImagesModel); + } + imageModelRegistry.set(provider, providerModels); +} + +type ImageModelApi< + TProvider extends KnownImagesProvider, + TModelId extends keyof (typeof IMAGE_MODELS)[TProvider], +> = (typeof IMAGE_MODELS)[TProvider][TModelId] extends { api: infer TApi } + ? TApi extends ImagesApi + ? TApi + : never + : never; + +export function getImageModel< + TProvider extends KnownImagesProvider, + TModelId extends keyof (typeof IMAGE_MODELS)[TProvider], +>(provider: TProvider, modelId: TModelId): ImagesModel> { + const providerModels = imageModelRegistry.get(provider); + return providerModels?.get(modelId as string) as ImagesModel>; +} + +export function getImageProviders(): KnownImagesProvider[] { + return Array.from(imageModelRegistry.keys()) as KnownImagesProvider[]; +} + +export function getImageModels( + provider: TProvider, +): ImagesModel>[] { + const models = imageModelRegistry.get(provider); + return models + ? (Array.from(models.values()) as ImagesModel>[]) + : []; +} diff --git a/packages/ai/src/images-api-registry.ts b/packages/ai/src/images-api-registry.ts new file mode 100644 index 00000000..2a0c0fd9 --- /dev/null +++ b/packages/ai/src/images-api-registry.ts @@ -0,0 +1,53 @@ +import type { AssistantImages, ImagesApi, ImagesContext, ImagesFunction, ImagesModel, ImagesOptions } from "./types.js"; + +export type ImagesApiFunction = ( + model: ImagesModel, + context: ImagesContext, + options?: ImagesOptions, +) => Promise; + +export interface ImagesApiProvider { + api: TApi; + generateImages: ImagesFunction; +} + +interface ImagesApiProviderInternal { + api: ImagesApi; + generateImages: ImagesApiFunction; +} + +type RegisteredImagesApiProvider = { + provider: ImagesApiProviderInternal; + sourceId?: string; +}; + +const imagesApiProviderRegistry = new Map(); + +function wrapGenerateImages( + api: TApi, + generateImages: ImagesFunction, +): ImagesApiFunction { + return (model, context, options) => { + if (model.api !== api) { + throw new Error(`Mismatched api: ${model.api} expected ${api}`); + } + return generateImages(model as ImagesModel, context, options as TOptions); + }; +} + +export function registerImagesApiProvider( + provider: ImagesApiProvider, + sourceId?: string, +): void { + imagesApiProviderRegistry.set(provider.api, { + provider: { + api: provider.api, + generateImages: wrapGenerateImages(provider.api, provider.generateImages), + }, + sourceId, + }); +} + +export function getImagesApiProvider(api: ImagesApi): ImagesApiProviderInternal | undefined { + return imagesApiProviderRegistry.get(api)?.provider; +} diff --git a/packages/ai/src/images.ts b/packages/ai/src/images.ts new file mode 100644 index 00000000..88eb09f2 --- /dev/null +++ b/packages/ai/src/images.ts @@ -0,0 +1,21 @@ +import "./providers/images/register-builtins.js"; + +import { getImagesApiProvider } from "./images-api-registry.js"; +import type { AssistantImages, ImagesApi, ImagesContext, ImagesModel, ProviderImagesOptions } from "./types.js"; + +function resolveImagesApiProvider(api: ImagesApi) { + const provider = getImagesApiProvider(api); + if (!provider) { + throw new Error(`No API provider registered for api: ${api}`); + } + return provider; +} + +export async function generateImages( + model: ImagesModel, + context: ImagesContext, + options?: ProviderImagesOptions, +): Promise { + const provider = resolveImagesApiProvider(model.api); + return provider.generateImages(model, context, options); +} diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 2502008a..0a5ce4ed 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -3,6 +3,9 @@ export { Type } from "typebox"; export * from "./api-registry.js"; export * from "./env-api-keys.js"; +export * from "./image-models.js"; +export * from "./images.js"; +export * from "./images-api-registry.js"; export * from "./models.js"; export type { BedrockOptions, BedrockThinkingDisplay } from "./providers/amazon-bedrock.js"; export type { AnthropicEffort, AnthropicOptions, AnthropicThinkingDisplay } from "./providers/anthropic.js"; @@ -11,6 +14,7 @@ export * from "./providers/faux.js"; export type { GoogleOptions } from "./providers/google.js"; export type { GoogleThinkingLevel } from "./providers/google-shared.js"; export type { GoogleVertexOptions } from "./providers/google-vertex.js"; +export * from "./providers/images/register-builtins.js"; export type { MistralOptions } from "./providers/mistral.js"; export type { OpenAICodexResponsesOptions, diff --git a/packages/ai/src/providers/images/openrouter.ts b/packages/ai/src/providers/images/openrouter.ts new file mode 100644 index 00000000..70c95a8c --- /dev/null +++ b/packages/ai/src/providers/images/openrouter.ts @@ -0,0 +1,187 @@ +import OpenAI from "openai"; +import type { + ChatCompletion, + ChatCompletionContentPart, + ChatCompletionContentPartImage, + ChatCompletionContentPartText, + ChatCompletionCreateParamsNonStreaming, +} from "openai/resources/chat/completions.js"; +import { getEnvApiKey } from "../../env-api-keys.js"; +import type { + AssistantImages, + ImageContent, + ImagesContext, + ImagesFunction, + ImagesModel, + ImagesOptions, + TextContent, +} from "../../types.js"; +import { headersToRecord } from "../../utils/headers.js"; +import { sanitizeSurrogates } from "../../utils/sanitize-unicode.js"; + +interface OpenRouterGeneratedImage { + image_url?: string | { url?: string }; +} + +type OpenRouterImageGenerationMessage = ChatCompletion["choices"][number]["message"] & { + images?: OpenRouterGeneratedImage[]; +}; + +type OpenRouterImageGenerationChoice = ChatCompletion["choices"][number] & { + message: OpenRouterImageGenerationMessage; +}; + +type OpenRouterImageGenerationResponse = ChatCompletion & { + choices: OpenRouterImageGenerationChoice[]; +}; + +export const generateImagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = async ( + model: ImagesModel<"openrouter-images">, + context: ImagesContext, + options?: ImagesOptions, +) => { + const output: AssistantImages = { + api: model.api, + provider: model.provider, + model: model.id, + output: [], + stopReason: "stop", + timestamp: Date.now(), + }; + + try { + const apiKey = options?.apiKey || getEnvApiKey(model.provider); + if (!apiKey) { + throw new Error(`No API key available for provider: ${model.provider}`); + } + const client = createClient(model, apiKey, options?.headers); + let params = buildParams(model, context); + const nextParams = await options?.onPayload?.(params, model); + if (nextParams !== undefined) { + params = nextParams as typeof params; + } + const requestOptions = { + ...(options?.signal ? { signal: options.signal } : {}), + ...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}), + ...(options?.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}), + }; + const { data: response, response: rawResponse } = await client.chat.completions + .create(params as unknown as ChatCompletionCreateParamsNonStreaming, requestOptions) + .withResponse(); + await options?.onResponse?.({ status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, model); + + const imageResponse = response as OpenRouterImageGenerationResponse; + output.responseId = imageResponse.id; + if (imageResponse.usage) { + output.usage = parseUsage(imageResponse.usage, model); + } + + const choice = imageResponse.choices[0]; + if (choice) { + const content = choice.message.content; + if (typeof content === "string" && content.length > 0) { + output.output.push({ type: "text", text: content } satisfies TextContent); + } + + for (const image of choice.message.images ?? []) { + const imageUrl = typeof image.image_url === "string" ? image.image_url : image.image_url?.url; + if (!imageUrl?.startsWith("data:")) continue; + const matches = imageUrl.match(/^data:([^;]+);base64,(.+)$/); + if (!matches) continue; + output.output.push({ + type: "image", + mimeType: matches[1], + data: matches[2], + } satisfies ImageContent); + } + } + + return output; + } catch (error) { + output.stopReason = options?.signal?.aborted ? "aborted" : "error"; + output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error); + return output; + } +}; + +function createClient( + model: ImagesModel<"openrouter-images">, + apiKey: string, + optionsHeaders?: Record, +): OpenAI { + return new OpenAI({ + apiKey, + baseURL: model.baseUrl, + dangerouslyAllowBrowser: true, + defaultHeaders: { + ...model.headers, + ...optionsHeaders, + }, + }); +} + +type OpenRouterImagesCreateParams = Omit & { + modalities: Array<"image" | "text">; +}; + +function buildParams(model: ImagesModel<"openrouter-images">, context: ImagesContext): OpenRouterImagesCreateParams { + const content: ChatCompletionContentPart[] = context.input.map((item): ChatCompletionContentPart => { + if (item.type === "text") { + return { + type: "text", + text: sanitizeSurrogates(item.text), + } satisfies ChatCompletionContentPartText; + } + return { + type: "image_url", + image_url: { + url: `data:${item.mimeType};base64,${item.data}`, + }, + } satisfies ChatCompletionContentPartImage; + }); + + return { + model: model.id, + messages: [ + { + role: "user" as const, + content, + }, + ], + stream: false, + modalities: model.output.includes("text") ? ["image", "text"] : ["image"], + }; +} + +function parseUsage( + rawUsage: { + prompt_tokens?: number; + completion_tokens?: number; + prompt_tokens_details?: { cached_tokens?: number; cache_write_tokens?: number }; + }, + model: ImagesModel<"openrouter-images">, +) { + const promptTokens = rawUsage.prompt_tokens || 0; + const reportedCachedTokens = rawUsage.prompt_tokens_details?.cached_tokens || 0; + const cacheWriteTokens = rawUsage.prompt_tokens_details?.cache_write_tokens || 0; + const cacheReadTokens = + cacheWriteTokens > 0 ? Math.max(0, reportedCachedTokens - cacheWriteTokens) : reportedCachedTokens; + const input = Math.max(0, promptTokens - cacheReadTokens - cacheWriteTokens); + const output = rawUsage.completion_tokens || 0; + const usage = { + input, + output, + cacheRead: cacheReadTokens, + cacheWrite: cacheWriteTokens, + totalTokens: input + output + cacheReadTokens + cacheWriteTokens, + cost: { + input: (model.cost.input / 1000000) * input, + output: (model.cost.output / 1000000) * output, + cacheRead: (model.cost.cacheRead / 1000000) * cacheReadTokens, + cacheWrite: (model.cost.cacheWrite / 1000000) * cacheWriteTokens, + total: 0, + }, + }; + usage.cost.total = usage.cost.input + usage.cost.output + usage.cost.cacheRead + usage.cost.cacheWrite; + return usage; +} diff --git a/packages/ai/src/providers/images/register-builtins.ts b/packages/ai/src/providers/images/register-builtins.ts new file mode 100644 index 00000000..04338f20 --- /dev/null +++ b/packages/ai/src/providers/images/register-builtins.ts @@ -0,0 +1,50 @@ +import { registerImagesApiProvider } from "../../images-api-registry.js"; +import type { AssistantImages, ImagesContext, ImagesFunction, ImagesModel, ImagesOptions } from "../../types.js"; +import type { generateImagesOpenRouter as generateImagesOpenRouterFunction } from "./openrouter.js"; + +interface OpenRouterImagesProviderModule { + generateImagesOpenRouter: typeof generateImagesOpenRouterFunction; +} + +let openRouterImagesProviderModulePromise: Promise | undefined; + +function createLazyLoadErrorImages(model: ImagesModel<"openrouter-images">, error: unknown): AssistantImages { + return { + api: model.api, + provider: model.provider, + model: model.id, + output: [], + stopReason: "error", + errorMessage: error instanceof Error ? error.message : String(error), + timestamp: Date.now(), + }; +} + +function loadOpenRouterImagesProviderModule(): Promise { + openRouterImagesProviderModulePromise ||= import("./openrouter.js").then( + (module) => module as OpenRouterImagesProviderModule, + ); + return openRouterImagesProviderModulePromise; +} + +export const generateImagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = async ( + model: ImagesModel<"openrouter-images">, + context: ImagesContext, + options?: ImagesOptions, +) => { + try { + const module = await loadOpenRouterImagesProviderModule(); + return await module.generateImagesOpenRouter(model, context, options); + } catch (error) { + return createLazyLoadErrorImages(model, error); + } +}; + +export function registerBuiltInImagesApiProviders(): void { + registerImagesApiProvider({ + api: "openrouter-images", + generateImages: generateImagesOpenRouter, + }); +} + +registerBuiltInImagesApiProviders(); diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index e7947756..db6ef893 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -16,6 +16,10 @@ export type KnownApi = export type Api = KnownApi | (string & {}); +export type KnownImagesApi = "openrouter-images"; + +export type ImagesApi = KnownImagesApi | (string & {}); + export type KnownProvider = | "amazon-bedrock" | "anthropic" @@ -50,6 +54,10 @@ export type KnownProvider = | "xiaomi-token-plan-sgp"; export type Provider = KnownProvider | string; +export type KnownImagesProvider = "openrouter"; + +export type ImagesProvider = KnownImagesProvider | string; + export type ThinkingLevel = "minimal" | "low" | "medium" | "high" | "xhigh"; export type ModelThinkingLevel = "off" | ThinkingLevel; export type ThinkingLevelMap = Partial>; @@ -137,6 +145,48 @@ export interface StreamOptions { export type ProviderStreamOptions = StreamOptions & Record; +export interface ImagesOptions { + signal?: AbortSignal; + apiKey?: string; + /** + * Optional callback for inspecting or replacing provider payloads before sending. + * Return undefined to keep the payload unchanged. + */ + onPayload?: (payload: unknown, model: ImagesModel) => unknown | undefined | Promise; + /** + * Optional callback invoked after an HTTP response is received. + */ + onResponse?: (response: ProviderResponse, model: ImagesModel) => void | Promise; + /** + * Optional custom HTTP headers to include in API requests. + * Merged with provider defaults; can override default headers. + */ + headers?: Record; + /** + * HTTP request timeout in milliseconds for providers/SDKs that support it. + */ + timeoutMs?: number; + /** + * Maximum retry attempts for providers/SDKs that support client-side retries. + */ + maxRetries?: number; + /** + * Maximum delay in milliseconds to wait for a retry when the server requests a long wait. + * If the server's requested delay exceeds this value, the request fails immediately + * with an error containing the requested delay, allowing higher-level retry logic + * to handle it with user visibility. + * Default: 60000 (60 seconds). Set to 0 to disable the cap. + */ + maxRetryDelayMs?: number; + /** + * Optional metadata to include in API requests. + * Providers extract the fields they understand and ignore the rest. + */ + metadata?: Record; +} + +export type ProviderImagesOptions = ImagesOptions & Record; + // Unified options with reasoning passed to streamSimple() and completeSimple() export interface SimpleStreamOptions extends StreamOptions { reasoning?: ThinkingLevel; @@ -158,6 +208,12 @@ export type StreamFunction AssistantMessageEventStream; +export type ImagesFunction = ( + model: ImagesModel, + context: ImagesContext, + options?: TOptions, +) => Promise; + export interface TextSignatureV1 { v: 1; id: string; @@ -244,6 +300,27 @@ export interface ToolResultMessage { export type Message = UserMessage | AssistantMessage | ToolResultMessage; +export type ImagesInputContent = TextContent | ImageContent; +export type ImagesOutputContent = TextContent | ImageContent; + +export interface ImagesContext { + input: ImagesInputContent[]; +} + +export type ImagesStopReason = "stop" | "error" | "aborted"; + +export interface AssistantImages { + api: ImagesApi; + provider: ImagesProvider; + model: string; + output: ImagesOutputContent[]; + responseId?: string; + usage?: Usage; + stopReason: ImagesStopReason; + errorMessage?: string; + timestamp: number; // Unix timestamp in milliseconds +} + import type { TSchema } from "typebox"; export interface Tool { @@ -462,3 +539,10 @@ export interface Model { ? AnthropicMessagesCompat : never; } + +export interface ImagesModel + extends Omit, "api" | "provider" | "reasoning" | "contextWindow" | "maxTokens" | "compat"> { + api: TApi; + provider: ImagesProvider; + output: ("text" | "image")[]; +} diff --git a/packages/ai/test/images.test.ts b/packages/ai/test/images.test.ts new file mode 100644 index 00000000..59912c70 --- /dev/null +++ b/packages/ai/test/images.test.ts @@ -0,0 +1,90 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { getImageModel } from "../src/image-models.js"; +import { generateImages } from "../src/images.js"; +import type { ImageContent, ImagesContext, ImagesModel, ProviderImagesOptions } from "../src/types.js"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +type ImagesOptionsWithExtras = ProviderImagesOptions & Record; + +async function basicImageGeneration(model: ImagesModel, options?: ImagesOptionsWithExtras) { + const context: ImagesContext = { + input: [{ type: "text", text: "Generate a simple red circle on a plain white background. No text." }], + }; + + const response = await generateImages(model, context, options); + + expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop"); + expect(response.errorMessage).toBeFalsy(); + expect(response.output.some((item) => item.type === "image")).toBe(true); + expect(response.timestamp).toBeGreaterThan(0); +} + +async function handleTextAndImageOutput( + model: ImagesModel, + options?: ImagesOptionsWithExtras, +) { + if (!model.output.includes("text")) { + console.log(`Skipping text+image output test - model ${model.id} doesn't support text output`); + return; + } + + const context: ImagesContext = { + input: [{ type: "text", text: "Generate a red circle and include a brief description of the image." }], + }; + + const response = await generateImages(model, context, options); + + expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop"); + expect(response.output.some((item) => item.type === "image")).toBe(true); + expect(response.output.some((item) => item.type === "text" && item.text.trim().length > 0)).toBe(true); +} + +async function handleImageInput(model: ImagesModel, options?: ImagesOptionsWithExtras) { + if (!model.input.includes("image")) { + console.log(`Skipping image input test - model ${model.id} doesn't support image input`); + return; + } + + const imagePath = join(__dirname, "data", "red-circle.png"); + const imageBuffer = readFileSync(imagePath); + const imageContent: ImageContent = { + type: "image", + data: imageBuffer.toString("base64"), + mimeType: "image/png", + }; + + const context: ImagesContext = { + input: [{ type: "text", text: "Create a variation of this image with a blue background." }, imageContent], + }; + + const response = await generateImages(model, context, options); + + expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop"); + expect(response.output.some((item) => item.type === "image")).toBe(true); +} + +describe("Images E2E Tests", () => { + describe.skipIf(!process.env.OPENROUTER_API_KEY)( + "OpenRouter Images Provider (google/gemini-2.5-flash-image)", + () => { + const model = getImageModel("openrouter", "google/gemini-2.5-flash-image"); + + it("should generate a basic image", { retry: 3 }, async () => { + await basicImageGeneration(model); + }); + + it("should handle text plus image output", { retry: 3 }, async () => { + await handleTextAndImageOutput(model); + }); + + it("should handle image input", { retry: 3 }, async () => { + await handleImageInput(model); + }); + }, + ); +}); diff --git a/packages/ai/test/openrouter-images.test.ts b/packages/ai/test/openrouter-images.test.ts new file mode 100644 index 00000000..de2ddbba --- /dev/null +++ b/packages/ai/test/openrouter-images.test.ts @@ -0,0 +1,140 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { generateImages } from "../src/images.js"; +import type { ImagesContext, ImagesModel } from "../src/types.js"; + +const mockState = vi.hoisted(() => ({ + lastParams: undefined as unknown, + lastRequestOptions: undefined as unknown, +})); + +vi.mock("openai", () => { + class FakeOpenAI { + chat = { + completions: { + create: (params: unknown, requestOptions?: unknown) => { + mockState.lastParams = params; + mockState.lastRequestOptions = requestOptions; + const signal = (requestOptions as { signal?: AbortSignal } | undefined)?.signal; + if (signal?.aborted) { + const error = new Error("Request aborted"); + return { + withResponse: async () => { + throw error; + }, + }; + } + const response = { + id: "img-1", + usage: { + prompt_tokens: 12, + completion_tokens: 34, + prompt_tokens_details: { cached_tokens: 0 }, + }, + choices: [ + { + message: { + content: "Here is your image.", + images: [{ image_url: "data:image/png;base64,ZmFrZS1wbmc=" }], + }, + }, + ], + }; + const promise = Promise.resolve(response) as Promise & { + withResponse: () => Promise<{ + data: typeof response; + response: { status: number; headers: Headers }; + }>; + }; + promise.withResponse = async () => ({ + data: response, + response: { status: 200, headers: new Headers() }, + }); + return promise; + }, + }, + }; + } + + return { default: FakeOpenAI }; +}); + +describe("openrouter images", () => { + beforeEach(() => { + mockState.lastParams = undefined; + mockState.lastRequestOptions = undefined; + }); + + it("returns text plus images in final output", async () => { + const model: ImagesModel<"openrouter-images"> = { + id: "google/gemini-3.1-flash-image-preview", + name: "Gemini 3.1 Flash Image Preview", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["text", "image"], + cost: { input: 0.015, output: 0.03, cacheRead: 0, cacheWrite: 0 }, + headers: { "HTTP-Referer": "https://example.com" }, + }; + const context: ImagesContext = { + input: [{ type: "text", text: "Generate a dog" }], + }; + + const output = await generateImages(model, context, { apiKey: "test" }); + expect(output.stopReason).toBe("stop"); + expect(output.responseId).toBe("img-1"); + expect(output.output[0]).toMatchObject({ type: "text", text: "Here is your image." }); + expect(output.output[1]).toMatchObject({ type: "image", mimeType: "image/png", data: "ZmFrZS1wbmc=" }); + + const params = mockState.lastParams as { + stream?: boolean; + modalities?: string[]; + messages?: [{ content?: [{ type: string; text?: string }] }]; + }; + expect(params.stream).toBe(false); + expect(params.modalities).toEqual(["image", "text"]); + expect(params.messages?.[0]?.content?.[0]).toMatchObject({ type: "text", text: "Generate a dog" }); + }); + + it("passes through abort signal and returns aborted result", async () => { + const model: ImagesModel<"openrouter-images"> = { + id: "black-forest-labs/flux.2-pro", + name: "FLUX.2 Pro", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { input: 0.015, output: 0.03, cacheRead: 0, cacheWrite: 0 }, + }; + const context: ImagesContext = { + input: [{ type: "text", text: "Generate a dog" }], + }; + const controller = new AbortController(); + controller.abort(); + + const output = await generateImages(model, context, { apiKey: "test", signal: controller.signal }); + expect(output.stopReason).toBe("aborted"); + expect(output.errorMessage).toBe("Request aborted"); + expect(mockState.lastRequestOptions).toMatchObject({ signal: controller.signal }); + }); + + it("generateImages resolves the final assistant images result", async () => { + const model: ImagesModel<"openrouter-images"> = { + id: "black-forest-labs/flux.2-pro", + name: "FLUX.2 Pro", + api: "openrouter-images", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + input: ["text", "image"], + output: ["image"], + cost: { input: 0.015, output: 0.03, cacheRead: 0, cacheWrite: 0 }, + }; + const context: ImagesContext = { + input: [{ type: "text", text: "Generate a dog" }], + }; + + const output = await generateImages(model, context, { apiKey: "test" }); + expect(output.output.some((item) => item.type === "image")).toBe(true); + }); +});