Merge pull request #3887 from cristinaponcela/feat/image-outputs
feat: image content
This commit is contained in:
Mario Zechner
@@ -16,6 +16,9 @@ Unified LLM API with automatic model discovery, provider configuration, token an
|
|||||||
- [Validating Tool Arguments](#validating-tool-arguments)
|
- [Validating Tool Arguments](#validating-tool-arguments)
|
||||||
- [Complete Event Reference](#complete-event-reference)
|
- [Complete Event Reference](#complete-event-reference)
|
||||||
- [Image Input](#image-input)
|
- [Image Input](#image-input)
|
||||||
|
- [Image Generation](#image-generation)
|
||||||
|
- [Basic Image Generation](#basic-image-generation)
|
||||||
|
- [Notes and Limitations](#notes-and-limitations)
|
||||||
- [Thinking/Reasoning](#thinkingreasoning)
|
- [Thinking/Reasoning](#thinkingreasoning)
|
||||||
- [Unified Interface](#unified-interface-streamsimplecompletesimple)
|
- [Unified Interface](#unified-interface-streamsimplecompletesimple)
|
||||||
- [Provider-Specific Options](#provider-specific-options-streamcomplete)
|
- [Provider-Specific Options](#provider-specific-options-streamcomplete)
|
||||||
@@ -421,6 +424,70 @@ for (const block of response.content) {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Image Generation
|
||||||
|
|
||||||
|
Image generation uses a separate API surface from text/chat generation. Use `getImageModel()` / `getImageModels()` / `getImageProviders()` to discover image-generation models, and `generateImages()` to get the final result.
|
||||||
|
|
||||||
|
Do not use `stream()` or `complete()` for image generation. Image generation is a one-shot API: `generateImages()` waits for the provider response and returns the final `AssistantImages` result.
|
||||||
|
|
||||||
|
### Basic Image Generation
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import { getImageModel, generateImages } from '@mariozechner/pi-ai';
|
||||||
|
|
||||||
|
const model = getImageModel('openrouter', 'google/gemini-2.5-flash-image');
|
||||||
|
|
||||||
|
const result = await generateImages(model, {
|
||||||
|
input: [{ type: 'text', text: 'Generate a red circle on a plain white background.' }]
|
||||||
|
}, {
|
||||||
|
apiKey: process.env.OPENROUTER_API_KEY
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const block of result.output) {
|
||||||
|
if (block.type === 'text') {
|
||||||
|
console.log(block.text);
|
||||||
|
} else if (block.type === 'image') {
|
||||||
|
console.log(block.mimeType);
|
||||||
|
console.log(block.data.substring(0, 32));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Some models also support image input:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import { readFileSync } from 'fs';
|
||||||
|
|
||||||
|
const imageBuffer = readFileSync('input.png');
|
||||||
|
const result = await generateImages(model, {
|
||||||
|
input: [
|
||||||
|
{ type: 'text', text: 'Create a variation of this image with a blue background.' },
|
||||||
|
{ type: 'image', data: imageBuffer.toString('base64'), mimeType: 'image/png' }
|
||||||
|
]
|
||||||
|
}, {
|
||||||
|
apiKey: process.env.OPENROUTER_API_KEY
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
Check capabilities on the model metadata:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
console.log(model.input); // ['text', 'image']
|
||||||
|
console.log(model.output); // ['image'] or ['image', 'text']
|
||||||
|
```
|
||||||
|
|
||||||
|
### Notes and Limitations
|
||||||
|
|
||||||
|
- Use `getImageModel(...)`, not `getModel(...)`.
|
||||||
|
- Use `generateImages()`, not `stream()` / `complete()`.
|
||||||
|
- Image-generation models do not participate in tool calling.
|
||||||
|
- Outputs are returned in `AssistantImages.output` and can include both base64-encoded `ImageContent` blocks and `TextContent` blocks.
|
||||||
|
- Some models return only images, others return images plus text. Check `model.output`.
|
||||||
|
- Some models accept image input, others are text-to-image only. Check `model.input`.
|
||||||
|
- Like the streaming APIs, image generation supports options such as `apiKey`, `signal`, `headers`, `onPayload`, and `onResponse`, and results may include `stopReason`, `responseId`, and `usage`.
|
||||||
|
- If you want a model to analyze images in a conversation or call tools, use the regular `stream()` / `complete()` APIs with a model that supports image input.
|
||||||
|
- At the moment, image generation is available through only one provider, OpenRouter.
|
||||||
|
|
||||||
## Thinking/Reasoning
|
## Thinking/Reasoning
|
||||||
|
|
||||||
Many models support thinking/reasoning capabilities where they can show their internal thought process. You can check if a model supports reasoning via the `reasoning` property. If you pass reasoning options to a non-reasoning model, they are silently ignored.
|
Many models support thinking/reasoning capabilities where they can show their internal thought process. You can check if a model supports reasoning via the `reasoning` property. If you pass reasoning options to a non-reasoning model, they are silently ignored.
|
||||||
@@ -1251,10 +1318,11 @@ Create a new provider file (for example `amazon-bedrock.ts`) that exports:
|
|||||||
- Add credential detection in `env-api-keys.ts` for the new provider
|
- Add credential detection in `env-api-keys.ts` for the new provider
|
||||||
- Ensure `streamSimple` handles auth lookup via `getEnvApiKey()` or provider-specific auth
|
- Ensure `streamSimple` handles auth lookup via `getEnvApiKey()` or provider-specific auth
|
||||||
|
|
||||||
#### 4. Model Generation (`scripts/generate-models.ts`)
|
#### 4. Model Generation (`scripts/generate-models.ts`, `scripts/generate-image-models.ts`)
|
||||||
|
|
||||||
- Add logic to fetch and parse models from the provider's source (e.g., models.dev API)
|
- Add logic to fetch and parse models from the provider's source (e.g., models.dev API)
|
||||||
- Map provider model data to the standardized `Model` interface
|
- Map chat/tool-capable provider model data to the standardized `Model` interface via `scripts/generate-models.ts`
|
||||||
|
- Map image-generation provider model data to the standardized `ImagesModel` interface via `scripts/generate-image-models.ts`
|
||||||
- Handle provider-specific quirks (pricing format, capability flags, model ID transformations)
|
- Handle provider-specific quirks (pricing format, capability flags, model ID transformations)
|
||||||
|
|
||||||
#### 5. Tests (`test/`)
|
#### 5. Tests (`test/`)
|
||||||
|
|||||||
@@ -61,7 +61,8 @@
|
|||||||
"scripts": {
|
"scripts": {
|
||||||
"clean": "shx rm -rf dist",
|
"clean": "shx rm -rf dist",
|
||||||
"generate-models": "npx tsx scripts/generate-models.ts",
|
"generate-models": "npx tsx scripts/generate-models.ts",
|
||||||
"build": "npm run generate-models && tsgo -p tsconfig.build.json",
|
"generate-image-models": "npx tsx scripts/generate-image-models.ts",
|
||||||
|
"build": "npm run generate-models && npm run generate-image-models && tsgo -p tsconfig.build.json",
|
||||||
"dev": "tsgo -p tsconfig.build.json --watch --preserveWatchOutput",
|
"dev": "tsgo -p tsconfig.build.json --watch --preserveWatchOutput",
|
||||||
"dev:tsc": "tsgo -p tsconfig.build.json --watch --preserveWatchOutput",
|
"dev:tsc": "tsgo -p tsconfig.build.json --watch --preserveWatchOutput",
|
||||||
"test": "vitest --run",
|
"test": "vitest --run",
|
||||||
|
|||||||
131
packages/ai/scripts/generate-image-models.ts
Normal file
131
packages/ai/scripts/generate-image-models.ts
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
#!/usr/bin/env tsx
|
||||||
|
|
||||||
|
import { writeFileSync } from "fs";
|
||||||
|
import { dirname, join } from "path";
|
||||||
|
import { fileURLToPath } from "url";
|
||||||
|
import type { ImagesModel } from "../src/types.js";
|
||||||
|
|
||||||
|
const __filename = fileURLToPath(import.meta.url);
|
||||||
|
const __dirname = dirname(__filename);
|
||||||
|
const packageRoot = join(__dirname, "..");
|
||||||
|
const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1";
|
||||||
|
|
||||||
|
interface OpenRouterModelRecord {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
context_length?: number;
|
||||||
|
architecture?: {
|
||||||
|
input_modalities?: string[];
|
||||||
|
output_modalities?: string[];
|
||||||
|
};
|
||||||
|
pricing?: {
|
||||||
|
prompt?: string;
|
||||||
|
completion?: string;
|
||||||
|
input_cache_read?: string;
|
||||||
|
input_cache_write?: string;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchOpenRouterImageModels(): Promise<ImagesModel<"openrouter-images">[]> {
|
||||||
|
try {
|
||||||
|
console.log("Fetching image models from OpenRouter API...");
|
||||||
|
const response = await fetch(`${OPENROUTER_BASE_URL}/models?output_modalities=image`);
|
||||||
|
const data = (await response.json()) as { data?: OpenRouterModelRecord[] };
|
||||||
|
const models: ImagesModel<"openrouter-images">[] = [];
|
||||||
|
|
||||||
|
for (const model of data.data ?? []) {
|
||||||
|
const input = Array.from(
|
||||||
|
new Set(
|
||||||
|
(model.architecture?.input_modalities ?? [])
|
||||||
|
.filter((modality): modality is "text" | "image" => modality === "text" || modality === "image"),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
const output = Array.from(
|
||||||
|
new Set(
|
||||||
|
(model.architecture?.output_modalities ?? []).filter(
|
||||||
|
(modality): modality is "text" | "image" => modality === "text" || modality === "image",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!output.includes("image")) continue;
|
||||||
|
if (input.length === 0) input.push("text");
|
||||||
|
|
||||||
|
models.push({
|
||||||
|
id: model.id,
|
||||||
|
name: model.name,
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: OPENROUTER_BASE_URL,
|
||||||
|
input,
|
||||||
|
output,
|
||||||
|
cost: {
|
||||||
|
input: parseFloat(model.pricing?.prompt || "0") * 1_000_000,
|
||||||
|
output: parseFloat(model.pricing?.completion || "0") * 1_000_000,
|
||||||
|
cacheRead: parseFloat(model.pricing?.input_cache_read || "0") * 1_000_000,
|
||||||
|
cacheWrite: parseFloat(model.pricing?.input_cache_write || "0") * 1_000_000,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`Fetched ${models.length} image models from OpenRouter`);
|
||||||
|
return models;
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Failed to fetch OpenRouter image models:", error);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function generateImageModelsFile(models: ImagesModel<"openrouter-images">[]): string {
|
||||||
|
const imageModelsByProvider = {
|
||||||
|
openrouter: Object.fromEntries(
|
||||||
|
models
|
||||||
|
.sort((a, b) => a.id.localeCompare(b.id))
|
||||||
|
.map((model) => [
|
||||||
|
model.id,
|
||||||
|
`{
|
||||||
|
id: ${JSON.stringify(model.id)},
|
||||||
|
name: ${JSON.stringify(model.name)},
|
||||||
|
api: ${JSON.stringify(model.api)},
|
||||||
|
provider: ${JSON.stringify(model.provider)},
|
||||||
|
baseUrl: ${JSON.stringify(model.baseUrl)},
|
||||||
|
input: ${JSON.stringify(model.input)},
|
||||||
|
output: ${JSON.stringify(model.output)},
|
||||||
|
cost: ${JSON.stringify(model.cost, null, 2).replace(/^/gm, "\t")}
|
||||||
|
} satisfies ImagesModel<${JSON.stringify(model.api)}>`,
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
const providerEntries = Object.entries(imageModelsByProvider)
|
||||||
|
.map(([provider, providerModels]) => {
|
||||||
|
const modelEntries = Object.entries(providerModels)
|
||||||
|
.map(([id, serialized]) => `\t\t${JSON.stringify(id)}: ${serialized},`)
|
||||||
|
.join("\n");
|
||||||
|
return `\t${JSON.stringify(provider)}: {\n${modelEntries}\n\t},`;
|
||||||
|
})
|
||||||
|
.join("\n");
|
||||||
|
|
||||||
|
return `// This file is auto-generated by scripts/generate-image-models.ts
|
||||||
|
// Do not edit manually - run 'npm run generate-image-models' to update
|
||||||
|
|
||||||
|
import type { ImagesApi, ImagesModel } from "./types.js";
|
||||||
|
|
||||||
|
export const IMAGE_MODELS = {
|
||||||
|
${providerEntries}
|
||||||
|
} as const satisfies Record<string, Record<string, ImagesModel<ImagesApi>>>;
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main(): Promise<void> {
|
||||||
|
const models = await fetchOpenRouterImageModels();
|
||||||
|
const output = generateImageModelsFile(models);
|
||||||
|
const outputPath = join(packageRoot, "src", "image-models.generated.ts");
|
||||||
|
writeFileSync(outputPath, output, "utf-8");
|
||||||
|
console.log(`Generated ${outputPath}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((error) => {
|
||||||
|
console.error(error);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
264
packages/ai/src/image-models.generated.ts
Normal file
264
packages/ai/src/image-models.generated.ts
Normal file
@@ -0,0 +1,264 @@
|
|||||||
|
// This file is auto-generated by scripts/generate-image-models.ts
|
||||||
|
// Do not edit manually - run 'npm run generate-image-models' to update
|
||||||
|
|
||||||
|
import type { ImagesApi, ImagesModel } from "./types.js";
|
||||||
|
|
||||||
|
export const IMAGE_MODELS = {
|
||||||
|
openrouter: {
|
||||||
|
"black-forest-labs/flux.2-flex": {
|
||||||
|
id: "black-forest-labs/flux.2-flex",
|
||||||
|
name: "Black Forest Labs: FLUX.2 Flex",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["text", "image"],
|
||||||
|
output: ["image"],
|
||||||
|
cost: {
|
||||||
|
input: 0,
|
||||||
|
output: 0,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"black-forest-labs/flux.2-klein-4b": {
|
||||||
|
id: "black-forest-labs/flux.2-klein-4b",
|
||||||
|
name: "Black Forest Labs: FLUX.2 Klein 4B",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["text", "image"],
|
||||||
|
output: ["image"],
|
||||||
|
cost: {
|
||||||
|
input: 0,
|
||||||
|
output: 0,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"black-forest-labs/flux.2-max": {
|
||||||
|
id: "black-forest-labs/flux.2-max",
|
||||||
|
name: "Black Forest Labs: FLUX.2 Max",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["text", "image"],
|
||||||
|
output: ["image"],
|
||||||
|
cost: {
|
||||||
|
input: 0,
|
||||||
|
output: 0,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"black-forest-labs/flux.2-pro": {
|
||||||
|
id: "black-forest-labs/flux.2-pro",
|
||||||
|
name: "Black Forest Labs: FLUX.2 Pro",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["text", "image"],
|
||||||
|
output: ["image"],
|
||||||
|
cost: {
|
||||||
|
input: 0,
|
||||||
|
output: 0,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"bytedance-seed/seedream-4.5": {
|
||||||
|
id: "bytedance-seed/seedream-4.5",
|
||||||
|
name: "ByteDance Seed: Seedream 4.5",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["image", "text"],
|
||||||
|
output: ["image"],
|
||||||
|
cost: {
|
||||||
|
input: 0,
|
||||||
|
output: 0,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"google/gemini-2.5-flash-image": {
|
||||||
|
id: "google/gemini-2.5-flash-image",
|
||||||
|
name: "Google: Nano Banana (Gemini 2.5 Flash Image)",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["image", "text"],
|
||||||
|
output: ["image", "text"],
|
||||||
|
cost: {
|
||||||
|
input: 0.3,
|
||||||
|
output: 2.5,
|
||||||
|
cacheRead: 0.03,
|
||||||
|
cacheWrite: 0.08333333333333334,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"google/gemini-3-pro-image-preview": {
|
||||||
|
id: "google/gemini-3-pro-image-preview",
|
||||||
|
name: "Google: Nano Banana Pro (Gemini 3 Pro Image Preview)",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["image", "text"],
|
||||||
|
output: ["image", "text"],
|
||||||
|
cost: {
|
||||||
|
input: 2,
|
||||||
|
output: 12,
|
||||||
|
cacheRead: 0.19999999999999998,
|
||||||
|
cacheWrite: 0.375,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"google/gemini-3.1-flash-image-preview": {
|
||||||
|
id: "google/gemini-3.1-flash-image-preview",
|
||||||
|
name: "Google: Nano Banana 2 (Gemini 3.1 Flash Image Preview)",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["image", "text"],
|
||||||
|
output: ["image", "text"],
|
||||||
|
cost: {
|
||||||
|
input: 0.5,
|
||||||
|
output: 3,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"openai/gpt-5-image": {
|
||||||
|
id: "openai/gpt-5-image",
|
||||||
|
name: "OpenAI: GPT-5 Image",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["image", "text"],
|
||||||
|
output: ["image", "text"],
|
||||||
|
cost: {
|
||||||
|
input: 10,
|
||||||
|
output: 10,
|
||||||
|
cacheRead: 1.25,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"openai/gpt-5-image-mini": {
|
||||||
|
id: "openai/gpt-5-image-mini",
|
||||||
|
name: "OpenAI: GPT-5 Image Mini",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["image", "text"],
|
||||||
|
output: ["image", "text"],
|
||||||
|
cost: {
|
||||||
|
input: 2.5,
|
||||||
|
output: 2,
|
||||||
|
cacheRead: 0.25,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"openai/gpt-5.4-image-2": {
|
||||||
|
id: "openai/gpt-5.4-image-2",
|
||||||
|
name: "OpenAI: GPT-5.4 Image 2",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["image", "text"],
|
||||||
|
output: ["image", "text"],
|
||||||
|
cost: {
|
||||||
|
input: 8,
|
||||||
|
output: 15,
|
||||||
|
cacheRead: 2,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"openrouter/auto": {
|
||||||
|
id: "openrouter/auto",
|
||||||
|
name: "Auto Router",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["text", "image"],
|
||||||
|
output: ["text", "image"],
|
||||||
|
cost: {
|
||||||
|
input: -1000000,
|
||||||
|
output: -1000000,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"sourceful/riverflow-v2-fast": {
|
||||||
|
id: "sourceful/riverflow-v2-fast",
|
||||||
|
name: "Sourceful: Riverflow V2 Fast",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["text", "image"],
|
||||||
|
output: ["image"],
|
||||||
|
cost: {
|
||||||
|
input: 0,
|
||||||
|
output: 0,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"sourceful/riverflow-v2-fast-preview": {
|
||||||
|
id: "sourceful/riverflow-v2-fast-preview",
|
||||||
|
name: "Sourceful: Riverflow V2 Fast Preview",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["text", "image"],
|
||||||
|
output: ["image"],
|
||||||
|
cost: {
|
||||||
|
input: 0,
|
||||||
|
output: 0,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"sourceful/riverflow-v2-max-preview": {
|
||||||
|
id: "sourceful/riverflow-v2-max-preview",
|
||||||
|
name: "Sourceful: Riverflow V2 Max Preview",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["text", "image"],
|
||||||
|
output: ["image"],
|
||||||
|
cost: {
|
||||||
|
input: 0,
|
||||||
|
output: 0,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"sourceful/riverflow-v2-pro": {
|
||||||
|
id: "sourceful/riverflow-v2-pro",
|
||||||
|
name: "Sourceful: Riverflow V2 Pro",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["text", "image"],
|
||||||
|
output: ["image"],
|
||||||
|
cost: {
|
||||||
|
input: 0,
|
||||||
|
output: 0,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
"sourceful/riverflow-v2-standard-preview": {
|
||||||
|
id: "sourceful/riverflow-v2-standard-preview",
|
||||||
|
name: "Sourceful: Riverflow V2 Standard Preview",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["text", "image"],
|
||||||
|
output: ["image"],
|
||||||
|
cost: {
|
||||||
|
input: 0,
|
||||||
|
output: 0,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
},
|
||||||
|
} satisfies ImagesModel<"openrouter-images">,
|
||||||
|
},
|
||||||
|
} as const satisfies Record<string, Record<string, ImagesModel<ImagesApi>>>;
|
||||||
42
packages/ai/src/image-models.ts
Normal file
42
packages/ai/src/image-models.ts
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
import { IMAGE_MODELS } from "./image-models.generated.js";
|
||||||
|
import type { ImagesApi, ImagesModel, KnownImagesProvider } from "./types.js";
|
||||||
|
|
||||||
|
const imageModelRegistry: Map<string, Map<string, ImagesModel<ImagesApi>>> = new Map();
|
||||||
|
|
||||||
|
for (const [provider, models] of Object.entries(IMAGE_MODELS)) {
|
||||||
|
const providerModels = new Map<string, ImagesModel<ImagesApi>>();
|
||||||
|
for (const [id, model] of Object.entries(models)) {
|
||||||
|
providerModels.set(id, model as ImagesModel<ImagesApi>);
|
||||||
|
}
|
||||||
|
imageModelRegistry.set(provider, providerModels);
|
||||||
|
}
|
||||||
|
|
||||||
|
type ImageModelApi<
|
||||||
|
TProvider extends KnownImagesProvider,
|
||||||
|
TModelId extends keyof (typeof IMAGE_MODELS)[TProvider],
|
||||||
|
> = (typeof IMAGE_MODELS)[TProvider][TModelId] extends { api: infer TApi }
|
||||||
|
? TApi extends ImagesApi
|
||||||
|
? TApi
|
||||||
|
: never
|
||||||
|
: never;
|
||||||
|
|
||||||
|
export function getImageModel<
|
||||||
|
TProvider extends KnownImagesProvider,
|
||||||
|
TModelId extends keyof (typeof IMAGE_MODELS)[TProvider],
|
||||||
|
>(provider: TProvider, modelId: TModelId): ImagesModel<ImageModelApi<TProvider, TModelId>> {
|
||||||
|
const providerModels = imageModelRegistry.get(provider);
|
||||||
|
return providerModels?.get(modelId as string) as ImagesModel<ImageModelApi<TProvider, TModelId>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getImageProviders(): KnownImagesProvider[] {
|
||||||
|
return Array.from(imageModelRegistry.keys()) as KnownImagesProvider[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getImageModels<TProvider extends KnownImagesProvider>(
|
||||||
|
provider: TProvider,
|
||||||
|
): ImagesModel<ImageModelApi<TProvider, keyof (typeof IMAGE_MODELS)[TProvider]>>[] {
|
||||||
|
const models = imageModelRegistry.get(provider);
|
||||||
|
return models
|
||||||
|
? (Array.from(models.values()) as ImagesModel<ImageModelApi<TProvider, keyof (typeof IMAGE_MODELS)[TProvider]>>[])
|
||||||
|
: [];
|
||||||
|
}
|
||||||
53
packages/ai/src/images-api-registry.ts
Normal file
53
packages/ai/src/images-api-registry.ts
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
import type { AssistantImages, ImagesApi, ImagesContext, ImagesFunction, ImagesModel, ImagesOptions } from "./types.js";
|
||||||
|
|
||||||
|
export type ImagesApiFunction = (
|
||||||
|
model: ImagesModel<ImagesApi>,
|
||||||
|
context: ImagesContext,
|
||||||
|
options?: ImagesOptions,
|
||||||
|
) => Promise<AssistantImages>;
|
||||||
|
|
||||||
|
export interface ImagesApiProvider<TApi extends ImagesApi = ImagesApi, TOptions extends ImagesOptions = ImagesOptions> {
|
||||||
|
api: TApi;
|
||||||
|
generateImages: ImagesFunction<TApi, TOptions>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ImagesApiProviderInternal {
|
||||||
|
api: ImagesApi;
|
||||||
|
generateImages: ImagesApiFunction;
|
||||||
|
}
|
||||||
|
|
||||||
|
type RegisteredImagesApiProvider = {
|
||||||
|
provider: ImagesApiProviderInternal;
|
||||||
|
sourceId?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const imagesApiProviderRegistry = new Map<string, RegisteredImagesApiProvider>();
|
||||||
|
|
||||||
|
function wrapGenerateImages<TApi extends ImagesApi, TOptions extends ImagesOptions>(
|
||||||
|
api: TApi,
|
||||||
|
generateImages: ImagesFunction<TApi, TOptions>,
|
||||||
|
): ImagesApiFunction {
|
||||||
|
return (model, context, options) => {
|
||||||
|
if (model.api !== api) {
|
||||||
|
throw new Error(`Mismatched api: ${model.api} expected ${api}`);
|
||||||
|
}
|
||||||
|
return generateImages(model as ImagesModel<TApi>, context, options as TOptions);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function registerImagesApiProvider<TApi extends ImagesApi, TOptions extends ImagesOptions>(
|
||||||
|
provider: ImagesApiProvider<TApi, TOptions>,
|
||||||
|
sourceId?: string,
|
||||||
|
): void {
|
||||||
|
imagesApiProviderRegistry.set(provider.api, {
|
||||||
|
provider: {
|
||||||
|
api: provider.api,
|
||||||
|
generateImages: wrapGenerateImages(provider.api, provider.generateImages),
|
||||||
|
},
|
||||||
|
sourceId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getImagesApiProvider(api: ImagesApi): ImagesApiProviderInternal | undefined {
|
||||||
|
return imagesApiProviderRegistry.get(api)?.provider;
|
||||||
|
}
|
||||||
21
packages/ai/src/images.ts
Normal file
21
packages/ai/src/images.ts
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
import "./providers/images/register-builtins.js";
|
||||||
|
|
||||||
|
import { getImagesApiProvider } from "./images-api-registry.js";
|
||||||
|
import type { AssistantImages, ImagesApi, ImagesContext, ImagesModel, ProviderImagesOptions } from "./types.js";
|
||||||
|
|
||||||
|
function resolveImagesApiProvider(api: ImagesApi) {
|
||||||
|
const provider = getImagesApiProvider(api);
|
||||||
|
if (!provider) {
|
||||||
|
throw new Error(`No API provider registered for api: ${api}`);
|
||||||
|
}
|
||||||
|
return provider;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function generateImages<TApi extends ImagesApi>(
|
||||||
|
model: ImagesModel<TApi>,
|
||||||
|
context: ImagesContext,
|
||||||
|
options?: ProviderImagesOptions,
|
||||||
|
): Promise<AssistantImages> {
|
||||||
|
const provider = resolveImagesApiProvider(model.api);
|
||||||
|
return provider.generateImages(model, context, options);
|
||||||
|
}
|
||||||
@@ -3,6 +3,9 @@ export { Type } from "typebox";
|
|||||||
|
|
||||||
export * from "./api-registry.js";
|
export * from "./api-registry.js";
|
||||||
export * from "./env-api-keys.js";
|
export * from "./env-api-keys.js";
|
||||||
|
export * from "./image-models.js";
|
||||||
|
export * from "./images.js";
|
||||||
|
export * from "./images-api-registry.js";
|
||||||
export * from "./models.js";
|
export * from "./models.js";
|
||||||
export type { BedrockOptions, BedrockThinkingDisplay } from "./providers/amazon-bedrock.js";
|
export type { BedrockOptions, BedrockThinkingDisplay } from "./providers/amazon-bedrock.js";
|
||||||
export type { AnthropicEffort, AnthropicOptions, AnthropicThinkingDisplay } from "./providers/anthropic.js";
|
export type { AnthropicEffort, AnthropicOptions, AnthropicThinkingDisplay } from "./providers/anthropic.js";
|
||||||
@@ -11,6 +14,7 @@ export * from "./providers/faux.js";
|
|||||||
export type { GoogleOptions } from "./providers/google.js";
|
export type { GoogleOptions } from "./providers/google.js";
|
||||||
export type { GoogleThinkingLevel } from "./providers/google-shared.js";
|
export type { GoogleThinkingLevel } from "./providers/google-shared.js";
|
||||||
export type { GoogleVertexOptions } from "./providers/google-vertex.js";
|
export type { GoogleVertexOptions } from "./providers/google-vertex.js";
|
||||||
|
export * from "./providers/images/register-builtins.js";
|
||||||
export type { MistralOptions } from "./providers/mistral.js";
|
export type { MistralOptions } from "./providers/mistral.js";
|
||||||
export type {
|
export type {
|
||||||
OpenAICodexResponsesOptions,
|
OpenAICodexResponsesOptions,
|
||||||
|
|||||||
187
packages/ai/src/providers/images/openrouter.ts
Normal file
187
packages/ai/src/providers/images/openrouter.ts
Normal file
@@ -0,0 +1,187 @@
|
|||||||
|
import OpenAI from "openai";
|
||||||
|
import type {
|
||||||
|
ChatCompletion,
|
||||||
|
ChatCompletionContentPart,
|
||||||
|
ChatCompletionContentPartImage,
|
||||||
|
ChatCompletionContentPartText,
|
||||||
|
ChatCompletionCreateParamsNonStreaming,
|
||||||
|
} from "openai/resources/chat/completions.js";
|
||||||
|
import { getEnvApiKey } from "../../env-api-keys.js";
|
||||||
|
import type {
|
||||||
|
AssistantImages,
|
||||||
|
ImageContent,
|
||||||
|
ImagesContext,
|
||||||
|
ImagesFunction,
|
||||||
|
ImagesModel,
|
||||||
|
ImagesOptions,
|
||||||
|
TextContent,
|
||||||
|
} from "../../types.js";
|
||||||
|
import { headersToRecord } from "../../utils/headers.js";
|
||||||
|
import { sanitizeSurrogates } from "../../utils/sanitize-unicode.js";
|
||||||
|
|
||||||
|
interface OpenRouterGeneratedImage {
|
||||||
|
image_url?: string | { url?: string };
|
||||||
|
}
|
||||||
|
|
||||||
|
type OpenRouterImageGenerationMessage = ChatCompletion["choices"][number]["message"] & {
|
||||||
|
images?: OpenRouterGeneratedImage[];
|
||||||
|
};
|
||||||
|
|
||||||
|
type OpenRouterImageGenerationChoice = ChatCompletion["choices"][number] & {
|
||||||
|
message: OpenRouterImageGenerationMessage;
|
||||||
|
};
|
||||||
|
|
||||||
|
type OpenRouterImageGenerationResponse = ChatCompletion & {
|
||||||
|
choices: OpenRouterImageGenerationChoice[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export const generateImagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = async (
|
||||||
|
model: ImagesModel<"openrouter-images">,
|
||||||
|
context: ImagesContext,
|
||||||
|
options?: ImagesOptions,
|
||||||
|
) => {
|
||||||
|
const output: AssistantImages = {
|
||||||
|
api: model.api,
|
||||||
|
provider: model.provider,
|
||||||
|
model: model.id,
|
||||||
|
output: [],
|
||||||
|
stopReason: "stop",
|
||||||
|
timestamp: Date.now(),
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const apiKey = options?.apiKey || getEnvApiKey(model.provider);
|
||||||
|
if (!apiKey) {
|
||||||
|
throw new Error(`No API key available for provider: ${model.provider}`);
|
||||||
|
}
|
||||||
|
const client = createClient(model, apiKey, options?.headers);
|
||||||
|
let params = buildParams(model, context);
|
||||||
|
const nextParams = await options?.onPayload?.(params, model);
|
||||||
|
if (nextParams !== undefined) {
|
||||||
|
params = nextParams as typeof params;
|
||||||
|
}
|
||||||
|
const requestOptions = {
|
||||||
|
...(options?.signal ? { signal: options.signal } : {}),
|
||||||
|
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
|
||||||
|
...(options?.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}),
|
||||||
|
};
|
||||||
|
const { data: response, response: rawResponse } = await client.chat.completions
|
||||||
|
.create(params as unknown as ChatCompletionCreateParamsNonStreaming, requestOptions)
|
||||||
|
.withResponse();
|
||||||
|
await options?.onResponse?.({ status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, model);
|
||||||
|
|
||||||
|
const imageResponse = response as OpenRouterImageGenerationResponse;
|
||||||
|
output.responseId = imageResponse.id;
|
||||||
|
if (imageResponse.usage) {
|
||||||
|
output.usage = parseUsage(imageResponse.usage, model);
|
||||||
|
}
|
||||||
|
|
||||||
|
const choice = imageResponse.choices[0];
|
||||||
|
if (choice) {
|
||||||
|
const content = choice.message.content;
|
||||||
|
if (typeof content === "string" && content.length > 0) {
|
||||||
|
output.output.push({ type: "text", text: content } satisfies TextContent);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const image of choice.message.images ?? []) {
|
||||||
|
const imageUrl = typeof image.image_url === "string" ? image.image_url : image.image_url?.url;
|
||||||
|
if (!imageUrl?.startsWith("data:")) continue;
|
||||||
|
const matches = imageUrl.match(/^data:([^;]+);base64,(.+)$/);
|
||||||
|
if (!matches) continue;
|
||||||
|
output.output.push({
|
||||||
|
type: "image",
|
||||||
|
mimeType: matches[1],
|
||||||
|
data: matches[2],
|
||||||
|
} satisfies ImageContent);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return output;
|
||||||
|
} catch (error) {
|
||||||
|
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
||||||
|
output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
function createClient(
|
||||||
|
model: ImagesModel<"openrouter-images">,
|
||||||
|
apiKey: string,
|
||||||
|
optionsHeaders?: Record<string, string>,
|
||||||
|
): OpenAI {
|
||||||
|
return new OpenAI({
|
||||||
|
apiKey,
|
||||||
|
baseURL: model.baseUrl,
|
||||||
|
dangerouslyAllowBrowser: true,
|
||||||
|
defaultHeaders: {
|
||||||
|
...model.headers,
|
||||||
|
...optionsHeaders,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
type OpenRouterImagesCreateParams = Omit<ChatCompletionCreateParamsNonStreaming, "modalities"> & {
|
||||||
|
modalities: Array<"image" | "text">;
|
||||||
|
};
|
||||||
|
|
||||||
|
function buildParams(model: ImagesModel<"openrouter-images">, context: ImagesContext): OpenRouterImagesCreateParams {
|
||||||
|
const content: ChatCompletionContentPart[] = context.input.map((item): ChatCompletionContentPart => {
|
||||||
|
if (item.type === "text") {
|
||||||
|
return {
|
||||||
|
type: "text",
|
||||||
|
text: sanitizeSurrogates(item.text),
|
||||||
|
} satisfies ChatCompletionContentPartText;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
type: "image_url",
|
||||||
|
image_url: {
|
||||||
|
url: `data:${item.mimeType};base64,${item.data}`,
|
||||||
|
},
|
||||||
|
} satisfies ChatCompletionContentPartImage;
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
model: model.id,
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
role: "user" as const,
|
||||||
|
content,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
stream: false,
|
||||||
|
modalities: model.output.includes("text") ? ["image", "text"] : ["image"],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseUsage(
|
||||||
|
rawUsage: {
|
||||||
|
prompt_tokens?: number;
|
||||||
|
completion_tokens?: number;
|
||||||
|
prompt_tokens_details?: { cached_tokens?: number; cache_write_tokens?: number };
|
||||||
|
},
|
||||||
|
model: ImagesModel<"openrouter-images">,
|
||||||
|
) {
|
||||||
|
const promptTokens = rawUsage.prompt_tokens || 0;
|
||||||
|
const reportedCachedTokens = rawUsage.prompt_tokens_details?.cached_tokens || 0;
|
||||||
|
const cacheWriteTokens = rawUsage.prompt_tokens_details?.cache_write_tokens || 0;
|
||||||
|
const cacheReadTokens =
|
||||||
|
cacheWriteTokens > 0 ? Math.max(0, reportedCachedTokens - cacheWriteTokens) : reportedCachedTokens;
|
||||||
|
const input = Math.max(0, promptTokens - cacheReadTokens - cacheWriteTokens);
|
||||||
|
const output = rawUsage.completion_tokens || 0;
|
||||||
|
const usage = {
|
||||||
|
input,
|
||||||
|
output,
|
||||||
|
cacheRead: cacheReadTokens,
|
||||||
|
cacheWrite: cacheWriteTokens,
|
||||||
|
totalTokens: input + output + cacheReadTokens + cacheWriteTokens,
|
||||||
|
cost: {
|
||||||
|
input: (model.cost.input / 1000000) * input,
|
||||||
|
output: (model.cost.output / 1000000) * output,
|
||||||
|
cacheRead: (model.cost.cacheRead / 1000000) * cacheReadTokens,
|
||||||
|
cacheWrite: (model.cost.cacheWrite / 1000000) * cacheWriteTokens,
|
||||||
|
total: 0,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
usage.cost.total = usage.cost.input + usage.cost.output + usage.cost.cacheRead + usage.cost.cacheWrite;
|
||||||
|
return usage;
|
||||||
|
}
|
||||||
50
packages/ai/src/providers/images/register-builtins.ts
Normal file
50
packages/ai/src/providers/images/register-builtins.ts
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
import { registerImagesApiProvider } from "../../images-api-registry.js";
|
||||||
|
import type { AssistantImages, ImagesContext, ImagesFunction, ImagesModel, ImagesOptions } from "../../types.js";
|
||||||
|
import type { generateImagesOpenRouter as generateImagesOpenRouterFunction } from "./openrouter.js";
|
||||||
|
|
||||||
|
interface OpenRouterImagesProviderModule {
|
||||||
|
generateImagesOpenRouter: typeof generateImagesOpenRouterFunction;
|
||||||
|
}
|
||||||
|
|
||||||
|
let openRouterImagesProviderModulePromise: Promise<OpenRouterImagesProviderModule> | undefined;
|
||||||
|
|
||||||
|
function createLazyLoadErrorImages(model: ImagesModel<"openrouter-images">, error: unknown): AssistantImages {
|
||||||
|
return {
|
||||||
|
api: model.api,
|
||||||
|
provider: model.provider,
|
||||||
|
model: model.id,
|
||||||
|
output: [],
|
||||||
|
stopReason: "error",
|
||||||
|
errorMessage: error instanceof Error ? error.message : String(error),
|
||||||
|
timestamp: Date.now(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadOpenRouterImagesProviderModule(): Promise<OpenRouterImagesProviderModule> {
|
||||||
|
openRouterImagesProviderModulePromise ||= import("./openrouter.js").then(
|
||||||
|
(module) => module as OpenRouterImagesProviderModule,
|
||||||
|
);
|
||||||
|
return openRouterImagesProviderModulePromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const generateImagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = async (
|
||||||
|
model: ImagesModel<"openrouter-images">,
|
||||||
|
context: ImagesContext,
|
||||||
|
options?: ImagesOptions,
|
||||||
|
) => {
|
||||||
|
try {
|
||||||
|
const module = await loadOpenRouterImagesProviderModule();
|
||||||
|
return await module.generateImagesOpenRouter(model, context, options);
|
||||||
|
} catch (error) {
|
||||||
|
return createLazyLoadErrorImages(model, error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export function registerBuiltInImagesApiProviders(): void {
|
||||||
|
registerImagesApiProvider({
|
||||||
|
api: "openrouter-images",
|
||||||
|
generateImages: generateImagesOpenRouter,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
registerBuiltInImagesApiProviders();
|
||||||
@@ -16,6 +16,10 @@ export type KnownApi =
|
|||||||
|
|
||||||
export type Api = KnownApi | (string & {});
|
export type Api = KnownApi | (string & {});
|
||||||
|
|
||||||
|
export type KnownImagesApi = "openrouter-images";
|
||||||
|
|
||||||
|
export type ImagesApi = KnownImagesApi | (string & {});
|
||||||
|
|
||||||
export type KnownProvider =
|
export type KnownProvider =
|
||||||
| "amazon-bedrock"
|
| "amazon-bedrock"
|
||||||
| "anthropic"
|
| "anthropic"
|
||||||
@@ -50,6 +54,10 @@ export type KnownProvider =
|
|||||||
| "xiaomi-token-plan-sgp";
|
| "xiaomi-token-plan-sgp";
|
||||||
export type Provider = KnownProvider | string;
|
export type Provider = KnownProvider | string;
|
||||||
|
|
||||||
|
export type KnownImagesProvider = "openrouter";
|
||||||
|
|
||||||
|
export type ImagesProvider = KnownImagesProvider | string;
|
||||||
|
|
||||||
export type ThinkingLevel = "minimal" | "low" | "medium" | "high" | "xhigh";
|
export type ThinkingLevel = "minimal" | "low" | "medium" | "high" | "xhigh";
|
||||||
export type ModelThinkingLevel = "off" | ThinkingLevel;
|
export type ModelThinkingLevel = "off" | ThinkingLevel;
|
||||||
export type ThinkingLevelMap = Partial<Record<ModelThinkingLevel, string | null>>;
|
export type ThinkingLevelMap = Partial<Record<ModelThinkingLevel, string | null>>;
|
||||||
@@ -137,6 +145,48 @@ export interface StreamOptions {
|
|||||||
|
|
||||||
export type ProviderStreamOptions = StreamOptions & Record<string, unknown>;
|
export type ProviderStreamOptions = StreamOptions & Record<string, unknown>;
|
||||||
|
|
||||||
|
export interface ImagesOptions {
|
||||||
|
signal?: AbortSignal;
|
||||||
|
apiKey?: string;
|
||||||
|
/**
|
||||||
|
* Optional callback for inspecting or replacing provider payloads before sending.
|
||||||
|
* Return undefined to keep the payload unchanged.
|
||||||
|
*/
|
||||||
|
onPayload?: (payload: unknown, model: ImagesModel<ImagesApi>) => unknown | undefined | Promise<unknown | undefined>;
|
||||||
|
/**
|
||||||
|
* Optional callback invoked after an HTTP response is received.
|
||||||
|
*/
|
||||||
|
onResponse?: (response: ProviderResponse, model: ImagesModel<ImagesApi>) => void | Promise<void>;
|
||||||
|
/**
|
||||||
|
* Optional custom HTTP headers to include in API requests.
|
||||||
|
* Merged with provider defaults; can override default headers.
|
||||||
|
*/
|
||||||
|
headers?: Record<string, string>;
|
||||||
|
/**
|
||||||
|
* HTTP request timeout in milliseconds for providers/SDKs that support it.
|
||||||
|
*/
|
||||||
|
timeoutMs?: number;
|
||||||
|
/**
|
||||||
|
* Maximum retry attempts for providers/SDKs that support client-side retries.
|
||||||
|
*/
|
||||||
|
maxRetries?: number;
|
||||||
|
/**
|
||||||
|
* Maximum delay in milliseconds to wait for a retry when the server requests a long wait.
|
||||||
|
* If the server's requested delay exceeds this value, the request fails immediately
|
||||||
|
* with an error containing the requested delay, allowing higher-level retry logic
|
||||||
|
* to handle it with user visibility.
|
||||||
|
* Default: 60000 (60 seconds). Set to 0 to disable the cap.
|
||||||
|
*/
|
||||||
|
maxRetryDelayMs?: number;
|
||||||
|
/**
|
||||||
|
* Optional metadata to include in API requests.
|
||||||
|
* Providers extract the fields they understand and ignore the rest.
|
||||||
|
*/
|
||||||
|
metadata?: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ProviderImagesOptions = ImagesOptions & Record<string, unknown>;
|
||||||
|
|
||||||
// Unified options with reasoning passed to streamSimple() and completeSimple()
|
// Unified options with reasoning passed to streamSimple() and completeSimple()
|
||||||
export interface SimpleStreamOptions extends StreamOptions {
|
export interface SimpleStreamOptions extends StreamOptions {
|
||||||
reasoning?: ThinkingLevel;
|
reasoning?: ThinkingLevel;
|
||||||
@@ -158,6 +208,12 @@ export type StreamFunction<TApi extends Api = Api, TOptions extends StreamOption
|
|||||||
options?: TOptions,
|
options?: TOptions,
|
||||||
) => AssistantMessageEventStream;
|
) => AssistantMessageEventStream;
|
||||||
|
|
||||||
|
export type ImagesFunction<TApi extends ImagesApi = ImagesApi, TOptions extends ImagesOptions = ImagesOptions> = (
|
||||||
|
model: ImagesModel<TApi>,
|
||||||
|
context: ImagesContext,
|
||||||
|
options?: TOptions,
|
||||||
|
) => Promise<AssistantImages>;
|
||||||
|
|
||||||
export interface TextSignatureV1 {
|
export interface TextSignatureV1 {
|
||||||
v: 1;
|
v: 1;
|
||||||
id: string;
|
id: string;
|
||||||
@@ -244,6 +300,27 @@ export interface ToolResultMessage<TDetails = any> {
|
|||||||
|
|
||||||
export type Message = UserMessage | AssistantMessage | ToolResultMessage;
|
export type Message = UserMessage | AssistantMessage | ToolResultMessage;
|
||||||
|
|
||||||
|
export type ImagesInputContent = TextContent | ImageContent;
|
||||||
|
export type ImagesOutputContent = TextContent | ImageContent;
|
||||||
|
|
||||||
|
export interface ImagesContext {
|
||||||
|
input: ImagesInputContent[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ImagesStopReason = "stop" | "error" | "aborted";
|
||||||
|
|
||||||
|
export interface AssistantImages {
|
||||||
|
api: ImagesApi;
|
||||||
|
provider: ImagesProvider;
|
||||||
|
model: string;
|
||||||
|
output: ImagesOutputContent[];
|
||||||
|
responseId?: string;
|
||||||
|
usage?: Usage;
|
||||||
|
stopReason: ImagesStopReason;
|
||||||
|
errorMessage?: string;
|
||||||
|
timestamp: number; // Unix timestamp in milliseconds
|
||||||
|
}
|
||||||
|
|
||||||
import type { TSchema } from "typebox";
|
import type { TSchema } from "typebox";
|
||||||
|
|
||||||
export interface Tool<TParameters extends TSchema = TSchema> {
|
export interface Tool<TParameters extends TSchema = TSchema> {
|
||||||
@@ -462,3 +539,10 @@ export interface Model<TApi extends Api> {
|
|||||||
? AnthropicMessagesCompat
|
? AnthropicMessagesCompat
|
||||||
: never;
|
: never;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ImagesModel<TApi extends ImagesApi>
|
||||||
|
extends Omit<Model<Api>, "api" | "provider" | "reasoning" | "contextWindow" | "maxTokens" | "compat"> {
|
||||||
|
api: TApi;
|
||||||
|
provider: ImagesProvider;
|
||||||
|
output: ("text" | "image")[];
|
||||||
|
}
|
||||||
|
|||||||
90
packages/ai/test/images.test.ts
Normal file
90
packages/ai/test/images.test.ts
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
import { dirname, join } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { getImageModel } from "../src/image-models.js";
|
||||||
|
import { generateImages } from "../src/images.js";
|
||||||
|
import type { ImageContent, ImagesContext, ImagesModel, ProviderImagesOptions } from "../src/types.js";
|
||||||
|
|
||||||
|
const __filename = fileURLToPath(import.meta.url);
|
||||||
|
const __dirname = dirname(__filename);
|
||||||
|
|
||||||
|
type ImagesOptionsWithExtras = ProviderImagesOptions & Record<string, unknown>;
|
||||||
|
|
||||||
|
async function basicImageGeneration<TApi extends string>(model: ImagesModel<TApi>, options?: ImagesOptionsWithExtras) {
|
||||||
|
const context: ImagesContext = {
|
||||||
|
input: [{ type: "text", text: "Generate a simple red circle on a plain white background. No text." }],
|
||||||
|
};
|
||||||
|
|
||||||
|
const response = await generateImages(model, context, options);
|
||||||
|
|
||||||
|
expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop");
|
||||||
|
expect(response.errorMessage).toBeFalsy();
|
||||||
|
expect(response.output.some((item) => item.type === "image")).toBe(true);
|
||||||
|
expect(response.timestamp).toBeGreaterThan(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleTextAndImageOutput<TApi extends string>(
|
||||||
|
model: ImagesModel<TApi>,
|
||||||
|
options?: ImagesOptionsWithExtras,
|
||||||
|
) {
|
||||||
|
if (!model.output.includes("text")) {
|
||||||
|
console.log(`Skipping text+image output test - model ${model.id} doesn't support text output`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const context: ImagesContext = {
|
||||||
|
input: [{ type: "text", text: "Generate a red circle and include a brief description of the image." }],
|
||||||
|
};
|
||||||
|
|
||||||
|
const response = await generateImages(model, context, options);
|
||||||
|
|
||||||
|
expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop");
|
||||||
|
expect(response.output.some((item) => item.type === "image")).toBe(true);
|
||||||
|
expect(response.output.some((item) => item.type === "text" && item.text.trim().length > 0)).toBe(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleImageInput<TApi extends string>(model: ImagesModel<TApi>, options?: ImagesOptionsWithExtras) {
|
||||||
|
if (!model.input.includes("image")) {
|
||||||
|
console.log(`Skipping image input test - model ${model.id} doesn't support image input`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const imagePath = join(__dirname, "data", "red-circle.png");
|
||||||
|
const imageBuffer = readFileSync(imagePath);
|
||||||
|
const imageContent: ImageContent = {
|
||||||
|
type: "image",
|
||||||
|
data: imageBuffer.toString("base64"),
|
||||||
|
mimeType: "image/png",
|
||||||
|
};
|
||||||
|
|
||||||
|
const context: ImagesContext = {
|
||||||
|
input: [{ type: "text", text: "Create a variation of this image with a blue background." }, imageContent],
|
||||||
|
};
|
||||||
|
|
||||||
|
const response = await generateImages(model, context, options);
|
||||||
|
|
||||||
|
expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop");
|
||||||
|
expect(response.output.some((item) => item.type === "image")).toBe(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("Images E2E Tests", () => {
|
||||||
|
describe.skipIf(!process.env.OPENROUTER_API_KEY)(
|
||||||
|
"OpenRouter Images Provider (google/gemini-2.5-flash-image)",
|
||||||
|
() => {
|
||||||
|
const model = getImageModel("openrouter", "google/gemini-2.5-flash-image");
|
||||||
|
|
||||||
|
it("should generate a basic image", { retry: 3 }, async () => {
|
||||||
|
await basicImageGeneration(model);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle text plus image output", { retry: 3 }, async () => {
|
||||||
|
await handleTextAndImageOutput(model);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle image input", { retry: 3 }, async () => {
|
||||||
|
await handleImageInput(model);
|
||||||
|
});
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
140
packages/ai/test/openrouter-images.test.ts
Normal file
140
packages/ai/test/openrouter-images.test.ts
Normal file
@@ -0,0 +1,140 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { generateImages } from "../src/images.js";
|
||||||
|
import type { ImagesContext, ImagesModel } from "../src/types.js";
|
||||||
|
|
||||||
|
const mockState = vi.hoisted(() => ({
|
||||||
|
lastParams: undefined as unknown,
|
||||||
|
lastRequestOptions: undefined as unknown,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("openai", () => {
|
||||||
|
class FakeOpenAI {
|
||||||
|
chat = {
|
||||||
|
completions: {
|
||||||
|
create: (params: unknown, requestOptions?: unknown) => {
|
||||||
|
mockState.lastParams = params;
|
||||||
|
mockState.lastRequestOptions = requestOptions;
|
||||||
|
const signal = (requestOptions as { signal?: AbortSignal } | undefined)?.signal;
|
||||||
|
if (signal?.aborted) {
|
||||||
|
const error = new Error("Request aborted");
|
||||||
|
return {
|
||||||
|
withResponse: async () => {
|
||||||
|
throw error;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const response = {
|
||||||
|
id: "img-1",
|
||||||
|
usage: {
|
||||||
|
prompt_tokens: 12,
|
||||||
|
completion_tokens: 34,
|
||||||
|
prompt_tokens_details: { cached_tokens: 0 },
|
||||||
|
},
|
||||||
|
choices: [
|
||||||
|
{
|
||||||
|
message: {
|
||||||
|
content: "Here is your image.",
|
||||||
|
images: [{ image_url: "data:image/png;base64,ZmFrZS1wbmc=" }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const promise = Promise.resolve(response) as Promise<typeof response> & {
|
||||||
|
withResponse: () => Promise<{
|
||||||
|
data: typeof response;
|
||||||
|
response: { status: number; headers: Headers };
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
promise.withResponse = async () => ({
|
||||||
|
data: response,
|
||||||
|
response: { status: 200, headers: new Headers() },
|
||||||
|
});
|
||||||
|
return promise;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return { default: FakeOpenAI };
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("openrouter images", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
mockState.lastParams = undefined;
|
||||||
|
mockState.lastRequestOptions = undefined;
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns text plus images in final output", async () => {
|
||||||
|
const model: ImagesModel<"openrouter-images"> = {
|
||||||
|
id: "google/gemini-3.1-flash-image-preview",
|
||||||
|
name: "Gemini 3.1 Flash Image Preview",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["text", "image"],
|
||||||
|
output: ["text", "image"],
|
||||||
|
cost: { input: 0.015, output: 0.03, cacheRead: 0, cacheWrite: 0 },
|
||||||
|
headers: { "HTTP-Referer": "https://example.com" },
|
||||||
|
};
|
||||||
|
const context: ImagesContext = {
|
||||||
|
input: [{ type: "text", text: "Generate a dog" }],
|
||||||
|
};
|
||||||
|
|
||||||
|
const output = await generateImages(model, context, { apiKey: "test" });
|
||||||
|
expect(output.stopReason).toBe("stop");
|
||||||
|
expect(output.responseId).toBe("img-1");
|
||||||
|
expect(output.output[0]).toMatchObject({ type: "text", text: "Here is your image." });
|
||||||
|
expect(output.output[1]).toMatchObject({ type: "image", mimeType: "image/png", data: "ZmFrZS1wbmc=" });
|
||||||
|
|
||||||
|
const params = mockState.lastParams as {
|
||||||
|
stream?: boolean;
|
||||||
|
modalities?: string[];
|
||||||
|
messages?: [{ content?: [{ type: string; text?: string }] }];
|
||||||
|
};
|
||||||
|
expect(params.stream).toBe(false);
|
||||||
|
expect(params.modalities).toEqual(["image", "text"]);
|
||||||
|
expect(params.messages?.[0]?.content?.[0]).toMatchObject({ type: "text", text: "Generate a dog" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes through abort signal and returns aborted result", async () => {
|
||||||
|
const model: ImagesModel<"openrouter-images"> = {
|
||||||
|
id: "black-forest-labs/flux.2-pro",
|
||||||
|
name: "FLUX.2 Pro",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["text", "image"],
|
||||||
|
output: ["image"],
|
||||||
|
cost: { input: 0.015, output: 0.03, cacheRead: 0, cacheWrite: 0 },
|
||||||
|
};
|
||||||
|
const context: ImagesContext = {
|
||||||
|
input: [{ type: "text", text: "Generate a dog" }],
|
||||||
|
};
|
||||||
|
const controller = new AbortController();
|
||||||
|
controller.abort();
|
||||||
|
|
||||||
|
const output = await generateImages(model, context, { apiKey: "test", signal: controller.signal });
|
||||||
|
expect(output.stopReason).toBe("aborted");
|
||||||
|
expect(output.errorMessage).toBe("Request aborted");
|
||||||
|
expect(mockState.lastRequestOptions).toMatchObject({ signal: controller.signal });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("generateImages resolves the final assistant images result", async () => {
|
||||||
|
const model: ImagesModel<"openrouter-images"> = {
|
||||||
|
id: "black-forest-labs/flux.2-pro",
|
||||||
|
name: "FLUX.2 Pro",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["text", "image"],
|
||||||
|
output: ["image"],
|
||||||
|
cost: { input: 0.015, output: 0.03, cacheRead: 0, cacheWrite: 0 },
|
||||||
|
};
|
||||||
|
const context: ImagesContext = {
|
||||||
|
input: [{ type: "text", text: "Generate a dog" }],
|
||||||
|
};
|
||||||
|
|
||||||
|
const output = await generateImages(model, context, { apiKey: "test" });
|
||||||
|
expect(output.output.some((item) => item.type === "image")).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user