Merge pull request #3887 from cristinaponcela/feat/image-outputs
feat: image content
This commit is contained in:
264
packages/ai/src/image-models.generated.ts
Normal file
264
packages/ai/src/image-models.generated.ts
Normal file
@@ -0,0 +1,264 @@
|
||||
// This file is auto-generated by scripts/generate-image-models.ts
|
||||
// Do not edit manually - run 'npm run generate-image-models' to update
|
||||
|
||||
import type { ImagesApi, ImagesModel } from "./types.js";
|
||||
|
||||
export const IMAGE_MODELS = {
|
||||
openrouter: {
|
||||
"black-forest-labs/flux.2-flex": {
|
||||
id: "black-forest-labs/flux.2-flex",
|
||||
name: "Black Forest Labs: FLUX.2 Flex",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["text", "image"],
|
||||
output: ["image"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"black-forest-labs/flux.2-klein-4b": {
|
||||
id: "black-forest-labs/flux.2-klein-4b",
|
||||
name: "Black Forest Labs: FLUX.2 Klein 4B",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["text", "image"],
|
||||
output: ["image"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"black-forest-labs/flux.2-max": {
|
||||
id: "black-forest-labs/flux.2-max",
|
||||
name: "Black Forest Labs: FLUX.2 Max",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["text", "image"],
|
||||
output: ["image"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"black-forest-labs/flux.2-pro": {
|
||||
id: "black-forest-labs/flux.2-pro",
|
||||
name: "Black Forest Labs: FLUX.2 Pro",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["text", "image"],
|
||||
output: ["image"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"bytedance-seed/seedream-4.5": {
|
||||
id: "bytedance-seed/seedream-4.5",
|
||||
name: "ByteDance Seed: Seedream 4.5",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["image", "text"],
|
||||
output: ["image"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"google/gemini-2.5-flash-image": {
|
||||
id: "google/gemini-2.5-flash-image",
|
||||
name: "Google: Nano Banana (Gemini 2.5 Flash Image)",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["image", "text"],
|
||||
output: ["image", "text"],
|
||||
cost: {
|
||||
input: 0.3,
|
||||
output: 2.5,
|
||||
cacheRead: 0.03,
|
||||
cacheWrite: 0.08333333333333334,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"google/gemini-3-pro-image-preview": {
|
||||
id: "google/gemini-3-pro-image-preview",
|
||||
name: "Google: Nano Banana Pro (Gemini 3 Pro Image Preview)",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["image", "text"],
|
||||
output: ["image", "text"],
|
||||
cost: {
|
||||
input: 2,
|
||||
output: 12,
|
||||
cacheRead: 0.19999999999999998,
|
||||
cacheWrite: 0.375,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"google/gemini-3.1-flash-image-preview": {
|
||||
id: "google/gemini-3.1-flash-image-preview",
|
||||
name: "Google: Nano Banana 2 (Gemini 3.1 Flash Image Preview)",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["image", "text"],
|
||||
output: ["image", "text"],
|
||||
cost: {
|
||||
input: 0.5,
|
||||
output: 3,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"openai/gpt-5-image": {
|
||||
id: "openai/gpt-5-image",
|
||||
name: "OpenAI: GPT-5 Image",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["image", "text"],
|
||||
output: ["image", "text"],
|
||||
cost: {
|
||||
input: 10,
|
||||
output: 10,
|
||||
cacheRead: 1.25,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"openai/gpt-5-image-mini": {
|
||||
id: "openai/gpt-5-image-mini",
|
||||
name: "OpenAI: GPT-5 Image Mini",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["image", "text"],
|
||||
output: ["image", "text"],
|
||||
cost: {
|
||||
input: 2.5,
|
||||
output: 2,
|
||||
cacheRead: 0.25,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"openai/gpt-5.4-image-2": {
|
||||
id: "openai/gpt-5.4-image-2",
|
||||
name: "OpenAI: GPT-5.4 Image 2",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["image", "text"],
|
||||
output: ["image", "text"],
|
||||
cost: {
|
||||
input: 8,
|
||||
output: 15,
|
||||
cacheRead: 2,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"openrouter/auto": {
|
||||
id: "openrouter/auto",
|
||||
name: "Auto Router",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["text", "image"],
|
||||
output: ["text", "image"],
|
||||
cost: {
|
||||
input: -1000000,
|
||||
output: -1000000,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"sourceful/riverflow-v2-fast": {
|
||||
id: "sourceful/riverflow-v2-fast",
|
||||
name: "Sourceful: Riverflow V2 Fast",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["text", "image"],
|
||||
output: ["image"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"sourceful/riverflow-v2-fast-preview": {
|
||||
id: "sourceful/riverflow-v2-fast-preview",
|
||||
name: "Sourceful: Riverflow V2 Fast Preview",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["text", "image"],
|
||||
output: ["image"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"sourceful/riverflow-v2-max-preview": {
|
||||
id: "sourceful/riverflow-v2-max-preview",
|
||||
name: "Sourceful: Riverflow V2 Max Preview",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["text", "image"],
|
||||
output: ["image"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"sourceful/riverflow-v2-pro": {
|
||||
id: "sourceful/riverflow-v2-pro",
|
||||
name: "Sourceful: Riverflow V2 Pro",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["text", "image"],
|
||||
output: ["image"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
"sourceful/riverflow-v2-standard-preview": {
|
||||
id: "sourceful/riverflow-v2-standard-preview",
|
||||
name: "Sourceful: Riverflow V2 Standard Preview",
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
input: ["text", "image"],
|
||||
output: ["image"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
} satisfies ImagesModel<"openrouter-images">,
|
||||
},
|
||||
} as const satisfies Record<string, Record<string, ImagesModel<ImagesApi>>>;
|
||||
42
packages/ai/src/image-models.ts
Normal file
42
packages/ai/src/image-models.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import { IMAGE_MODELS } from "./image-models.generated.js";
|
||||
import type { ImagesApi, ImagesModel, KnownImagesProvider } from "./types.js";
|
||||
|
||||
const imageModelRegistry: Map<string, Map<string, ImagesModel<ImagesApi>>> = new Map();
|
||||
|
||||
for (const [provider, models] of Object.entries(IMAGE_MODELS)) {
|
||||
const providerModels = new Map<string, ImagesModel<ImagesApi>>();
|
||||
for (const [id, model] of Object.entries(models)) {
|
||||
providerModels.set(id, model as ImagesModel<ImagesApi>);
|
||||
}
|
||||
imageModelRegistry.set(provider, providerModels);
|
||||
}
|
||||
|
||||
type ImageModelApi<
|
||||
TProvider extends KnownImagesProvider,
|
||||
TModelId extends keyof (typeof IMAGE_MODELS)[TProvider],
|
||||
> = (typeof IMAGE_MODELS)[TProvider][TModelId] extends { api: infer TApi }
|
||||
? TApi extends ImagesApi
|
||||
? TApi
|
||||
: never
|
||||
: never;
|
||||
|
||||
export function getImageModel<
|
||||
TProvider extends KnownImagesProvider,
|
||||
TModelId extends keyof (typeof IMAGE_MODELS)[TProvider],
|
||||
>(provider: TProvider, modelId: TModelId): ImagesModel<ImageModelApi<TProvider, TModelId>> {
|
||||
const providerModels = imageModelRegistry.get(provider);
|
||||
return providerModels?.get(modelId as string) as ImagesModel<ImageModelApi<TProvider, TModelId>>;
|
||||
}
|
||||
|
||||
export function getImageProviders(): KnownImagesProvider[] {
|
||||
return Array.from(imageModelRegistry.keys()) as KnownImagesProvider[];
|
||||
}
|
||||
|
||||
export function getImageModels<TProvider extends KnownImagesProvider>(
|
||||
provider: TProvider,
|
||||
): ImagesModel<ImageModelApi<TProvider, keyof (typeof IMAGE_MODELS)[TProvider]>>[] {
|
||||
const models = imageModelRegistry.get(provider);
|
||||
return models
|
||||
? (Array.from(models.values()) as ImagesModel<ImageModelApi<TProvider, keyof (typeof IMAGE_MODELS)[TProvider]>>[])
|
||||
: [];
|
||||
}
|
||||
53
packages/ai/src/images-api-registry.ts
Normal file
53
packages/ai/src/images-api-registry.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
import type { AssistantImages, ImagesApi, ImagesContext, ImagesFunction, ImagesModel, ImagesOptions } from "./types.js";
|
||||
|
||||
export type ImagesApiFunction = (
|
||||
model: ImagesModel<ImagesApi>,
|
||||
context: ImagesContext,
|
||||
options?: ImagesOptions,
|
||||
) => Promise<AssistantImages>;
|
||||
|
||||
export interface ImagesApiProvider<TApi extends ImagesApi = ImagesApi, TOptions extends ImagesOptions = ImagesOptions> {
|
||||
api: TApi;
|
||||
generateImages: ImagesFunction<TApi, TOptions>;
|
||||
}
|
||||
|
||||
interface ImagesApiProviderInternal {
|
||||
api: ImagesApi;
|
||||
generateImages: ImagesApiFunction;
|
||||
}
|
||||
|
||||
type RegisteredImagesApiProvider = {
|
||||
provider: ImagesApiProviderInternal;
|
||||
sourceId?: string;
|
||||
};
|
||||
|
||||
const imagesApiProviderRegistry = new Map<string, RegisteredImagesApiProvider>();
|
||||
|
||||
function wrapGenerateImages<TApi extends ImagesApi, TOptions extends ImagesOptions>(
|
||||
api: TApi,
|
||||
generateImages: ImagesFunction<TApi, TOptions>,
|
||||
): ImagesApiFunction {
|
||||
return (model, context, options) => {
|
||||
if (model.api !== api) {
|
||||
throw new Error(`Mismatched api: ${model.api} expected ${api}`);
|
||||
}
|
||||
return generateImages(model as ImagesModel<TApi>, context, options as TOptions);
|
||||
};
|
||||
}
|
||||
|
||||
export function registerImagesApiProvider<TApi extends ImagesApi, TOptions extends ImagesOptions>(
|
||||
provider: ImagesApiProvider<TApi, TOptions>,
|
||||
sourceId?: string,
|
||||
): void {
|
||||
imagesApiProviderRegistry.set(provider.api, {
|
||||
provider: {
|
||||
api: provider.api,
|
||||
generateImages: wrapGenerateImages(provider.api, provider.generateImages),
|
||||
},
|
||||
sourceId,
|
||||
});
|
||||
}
|
||||
|
||||
export function getImagesApiProvider(api: ImagesApi): ImagesApiProviderInternal | undefined {
|
||||
return imagesApiProviderRegistry.get(api)?.provider;
|
||||
}
|
||||
21
packages/ai/src/images.ts
Normal file
21
packages/ai/src/images.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
import "./providers/images/register-builtins.js";
|
||||
|
||||
import { getImagesApiProvider } from "./images-api-registry.js";
|
||||
import type { AssistantImages, ImagesApi, ImagesContext, ImagesModel, ProviderImagesOptions } from "./types.js";
|
||||
|
||||
function resolveImagesApiProvider(api: ImagesApi) {
|
||||
const provider = getImagesApiProvider(api);
|
||||
if (!provider) {
|
||||
throw new Error(`No API provider registered for api: ${api}`);
|
||||
}
|
||||
return provider;
|
||||
}
|
||||
|
||||
export async function generateImages<TApi extends ImagesApi>(
|
||||
model: ImagesModel<TApi>,
|
||||
context: ImagesContext,
|
||||
options?: ProviderImagesOptions,
|
||||
): Promise<AssistantImages> {
|
||||
const provider = resolveImagesApiProvider(model.api);
|
||||
return provider.generateImages(model, context, options);
|
||||
}
|
||||
@@ -3,6 +3,9 @@ export { Type } from "typebox";
|
||||
|
||||
export * from "./api-registry.js";
|
||||
export * from "./env-api-keys.js";
|
||||
export * from "./image-models.js";
|
||||
export * from "./images.js";
|
||||
export * from "./images-api-registry.js";
|
||||
export * from "./models.js";
|
||||
export type { BedrockOptions, BedrockThinkingDisplay } from "./providers/amazon-bedrock.js";
|
||||
export type { AnthropicEffort, AnthropicOptions, AnthropicThinkingDisplay } from "./providers/anthropic.js";
|
||||
@@ -11,6 +14,7 @@ export * from "./providers/faux.js";
|
||||
export type { GoogleOptions } from "./providers/google.js";
|
||||
export type { GoogleThinkingLevel } from "./providers/google-shared.js";
|
||||
export type { GoogleVertexOptions } from "./providers/google-vertex.js";
|
||||
export * from "./providers/images/register-builtins.js";
|
||||
export type { MistralOptions } from "./providers/mistral.js";
|
||||
export type {
|
||||
OpenAICodexResponsesOptions,
|
||||
|
||||
187
packages/ai/src/providers/images/openrouter.ts
Normal file
187
packages/ai/src/providers/images/openrouter.ts
Normal file
@@ -0,0 +1,187 @@
|
||||
import OpenAI from "openai";
|
||||
import type {
|
||||
ChatCompletion,
|
||||
ChatCompletionContentPart,
|
||||
ChatCompletionContentPartImage,
|
||||
ChatCompletionContentPartText,
|
||||
ChatCompletionCreateParamsNonStreaming,
|
||||
} from "openai/resources/chat/completions.js";
|
||||
import { getEnvApiKey } from "../../env-api-keys.js";
|
||||
import type {
|
||||
AssistantImages,
|
||||
ImageContent,
|
||||
ImagesContext,
|
||||
ImagesFunction,
|
||||
ImagesModel,
|
||||
ImagesOptions,
|
||||
TextContent,
|
||||
} from "../../types.js";
|
||||
import { headersToRecord } from "../../utils/headers.js";
|
||||
import { sanitizeSurrogates } from "../../utils/sanitize-unicode.js";
|
||||
|
||||
interface OpenRouterGeneratedImage {
|
||||
image_url?: string | { url?: string };
|
||||
}
|
||||
|
||||
type OpenRouterImageGenerationMessage = ChatCompletion["choices"][number]["message"] & {
|
||||
images?: OpenRouterGeneratedImage[];
|
||||
};
|
||||
|
||||
type OpenRouterImageGenerationChoice = ChatCompletion["choices"][number] & {
|
||||
message: OpenRouterImageGenerationMessage;
|
||||
};
|
||||
|
||||
type OpenRouterImageGenerationResponse = ChatCompletion & {
|
||||
choices: OpenRouterImageGenerationChoice[];
|
||||
};
|
||||
|
||||
export const generateImagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = async (
|
||||
model: ImagesModel<"openrouter-images">,
|
||||
context: ImagesContext,
|
||||
options?: ImagesOptions,
|
||||
) => {
|
||||
const output: AssistantImages = {
|
||||
api: model.api,
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
output: [],
|
||||
stopReason: "stop",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
try {
|
||||
const apiKey = options?.apiKey || getEnvApiKey(model.provider);
|
||||
if (!apiKey) {
|
||||
throw new Error(`No API key available for provider: ${model.provider}`);
|
||||
}
|
||||
const client = createClient(model, apiKey, options?.headers);
|
||||
let params = buildParams(model, context);
|
||||
const nextParams = await options?.onPayload?.(params, model);
|
||||
if (nextParams !== undefined) {
|
||||
params = nextParams as typeof params;
|
||||
}
|
||||
const requestOptions = {
|
||||
...(options?.signal ? { signal: options.signal } : {}),
|
||||
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
|
||||
...(options?.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}),
|
||||
};
|
||||
const { data: response, response: rawResponse } = await client.chat.completions
|
||||
.create(params as unknown as ChatCompletionCreateParamsNonStreaming, requestOptions)
|
||||
.withResponse();
|
||||
await options?.onResponse?.({ status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, model);
|
||||
|
||||
const imageResponse = response as OpenRouterImageGenerationResponse;
|
||||
output.responseId = imageResponse.id;
|
||||
if (imageResponse.usage) {
|
||||
output.usage = parseUsage(imageResponse.usage, model);
|
||||
}
|
||||
|
||||
const choice = imageResponse.choices[0];
|
||||
if (choice) {
|
||||
const content = choice.message.content;
|
||||
if (typeof content === "string" && content.length > 0) {
|
||||
output.output.push({ type: "text", text: content } satisfies TextContent);
|
||||
}
|
||||
|
||||
for (const image of choice.message.images ?? []) {
|
||||
const imageUrl = typeof image.image_url === "string" ? image.image_url : image.image_url?.url;
|
||||
if (!imageUrl?.startsWith("data:")) continue;
|
||||
const matches = imageUrl.match(/^data:([^;]+);base64,(.+)$/);
|
||||
if (!matches) continue;
|
||||
output.output.push({
|
||||
type: "image",
|
||||
mimeType: matches[1],
|
||||
data: matches[2],
|
||||
} satisfies ImageContent);
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
} catch (error) {
|
||||
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
||||
output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
|
||||
return output;
|
||||
}
|
||||
};
|
||||
|
||||
function createClient(
|
||||
model: ImagesModel<"openrouter-images">,
|
||||
apiKey: string,
|
||||
optionsHeaders?: Record<string, string>,
|
||||
): OpenAI {
|
||||
return new OpenAI({
|
||||
apiKey,
|
||||
baseURL: model.baseUrl,
|
||||
dangerouslyAllowBrowser: true,
|
||||
defaultHeaders: {
|
||||
...model.headers,
|
||||
...optionsHeaders,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
type OpenRouterImagesCreateParams = Omit<ChatCompletionCreateParamsNonStreaming, "modalities"> & {
|
||||
modalities: Array<"image" | "text">;
|
||||
};
|
||||
|
||||
function buildParams(model: ImagesModel<"openrouter-images">, context: ImagesContext): OpenRouterImagesCreateParams {
|
||||
const content: ChatCompletionContentPart[] = context.input.map((item): ChatCompletionContentPart => {
|
||||
if (item.type === "text") {
|
||||
return {
|
||||
type: "text",
|
||||
text: sanitizeSurrogates(item.text),
|
||||
} satisfies ChatCompletionContentPartText;
|
||||
}
|
||||
return {
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: `data:${item.mimeType};base64,${item.data}`,
|
||||
},
|
||||
} satisfies ChatCompletionContentPartImage;
|
||||
});
|
||||
|
||||
return {
|
||||
model: model.id,
|
||||
messages: [
|
||||
{
|
||||
role: "user" as const,
|
||||
content,
|
||||
},
|
||||
],
|
||||
stream: false,
|
||||
modalities: model.output.includes("text") ? ["image", "text"] : ["image"],
|
||||
};
|
||||
}
|
||||
|
||||
function parseUsage(
|
||||
rawUsage: {
|
||||
prompt_tokens?: number;
|
||||
completion_tokens?: number;
|
||||
prompt_tokens_details?: { cached_tokens?: number; cache_write_tokens?: number };
|
||||
},
|
||||
model: ImagesModel<"openrouter-images">,
|
||||
) {
|
||||
const promptTokens = rawUsage.prompt_tokens || 0;
|
||||
const reportedCachedTokens = rawUsage.prompt_tokens_details?.cached_tokens || 0;
|
||||
const cacheWriteTokens = rawUsage.prompt_tokens_details?.cache_write_tokens || 0;
|
||||
const cacheReadTokens =
|
||||
cacheWriteTokens > 0 ? Math.max(0, reportedCachedTokens - cacheWriteTokens) : reportedCachedTokens;
|
||||
const input = Math.max(0, promptTokens - cacheReadTokens - cacheWriteTokens);
|
||||
const output = rawUsage.completion_tokens || 0;
|
||||
const usage = {
|
||||
input,
|
||||
output,
|
||||
cacheRead: cacheReadTokens,
|
||||
cacheWrite: cacheWriteTokens,
|
||||
totalTokens: input + output + cacheReadTokens + cacheWriteTokens,
|
||||
cost: {
|
||||
input: (model.cost.input / 1000000) * input,
|
||||
output: (model.cost.output / 1000000) * output,
|
||||
cacheRead: (model.cost.cacheRead / 1000000) * cacheReadTokens,
|
||||
cacheWrite: (model.cost.cacheWrite / 1000000) * cacheWriteTokens,
|
||||
total: 0,
|
||||
},
|
||||
};
|
||||
usage.cost.total = usage.cost.input + usage.cost.output + usage.cost.cacheRead + usage.cost.cacheWrite;
|
||||
return usage;
|
||||
}
|
||||
50
packages/ai/src/providers/images/register-builtins.ts
Normal file
50
packages/ai/src/providers/images/register-builtins.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
import { registerImagesApiProvider } from "../../images-api-registry.js";
|
||||
import type { AssistantImages, ImagesContext, ImagesFunction, ImagesModel, ImagesOptions } from "../../types.js";
|
||||
import type { generateImagesOpenRouter as generateImagesOpenRouterFunction } from "./openrouter.js";
|
||||
|
||||
interface OpenRouterImagesProviderModule {
|
||||
generateImagesOpenRouter: typeof generateImagesOpenRouterFunction;
|
||||
}
|
||||
|
||||
let openRouterImagesProviderModulePromise: Promise<OpenRouterImagesProviderModule> | undefined;
|
||||
|
||||
function createLazyLoadErrorImages(model: ImagesModel<"openrouter-images">, error: unknown): AssistantImages {
|
||||
return {
|
||||
api: model.api,
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
output: [],
|
||||
stopReason: "error",
|
||||
errorMessage: error instanceof Error ? error.message : String(error),
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function loadOpenRouterImagesProviderModule(): Promise<OpenRouterImagesProviderModule> {
|
||||
openRouterImagesProviderModulePromise ||= import("./openrouter.js").then(
|
||||
(module) => module as OpenRouterImagesProviderModule,
|
||||
);
|
||||
return openRouterImagesProviderModulePromise;
|
||||
}
|
||||
|
||||
export const generateImagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = async (
|
||||
model: ImagesModel<"openrouter-images">,
|
||||
context: ImagesContext,
|
||||
options?: ImagesOptions,
|
||||
) => {
|
||||
try {
|
||||
const module = await loadOpenRouterImagesProviderModule();
|
||||
return await module.generateImagesOpenRouter(model, context, options);
|
||||
} catch (error) {
|
||||
return createLazyLoadErrorImages(model, error);
|
||||
}
|
||||
};
|
||||
|
||||
export function registerBuiltInImagesApiProviders(): void {
|
||||
registerImagesApiProvider({
|
||||
api: "openrouter-images",
|
||||
generateImages: generateImagesOpenRouter,
|
||||
});
|
||||
}
|
||||
|
||||
registerBuiltInImagesApiProviders();
|
||||
@@ -16,6 +16,10 @@ export type KnownApi =
|
||||
|
||||
export type Api = KnownApi | (string & {});
|
||||
|
||||
export type KnownImagesApi = "openrouter-images";
|
||||
|
||||
export type ImagesApi = KnownImagesApi | (string & {});
|
||||
|
||||
export type KnownProvider =
|
||||
| "amazon-bedrock"
|
||||
| "anthropic"
|
||||
@@ -50,6 +54,10 @@ export type KnownProvider =
|
||||
| "xiaomi-token-plan-sgp";
|
||||
export type Provider = KnownProvider | string;
|
||||
|
||||
export type KnownImagesProvider = "openrouter";
|
||||
|
||||
export type ImagesProvider = KnownImagesProvider | string;
|
||||
|
||||
export type ThinkingLevel = "minimal" | "low" | "medium" | "high" | "xhigh";
|
||||
export type ModelThinkingLevel = "off" | ThinkingLevel;
|
||||
export type ThinkingLevelMap = Partial<Record<ModelThinkingLevel, string | null>>;
|
||||
@@ -137,6 +145,48 @@ export interface StreamOptions {
|
||||
|
||||
export type ProviderStreamOptions = StreamOptions & Record<string, unknown>;
|
||||
|
||||
export interface ImagesOptions {
|
||||
signal?: AbortSignal;
|
||||
apiKey?: string;
|
||||
/**
|
||||
* Optional callback for inspecting or replacing provider payloads before sending.
|
||||
* Return undefined to keep the payload unchanged.
|
||||
*/
|
||||
onPayload?: (payload: unknown, model: ImagesModel<ImagesApi>) => unknown | undefined | Promise<unknown | undefined>;
|
||||
/**
|
||||
* Optional callback invoked after an HTTP response is received.
|
||||
*/
|
||||
onResponse?: (response: ProviderResponse, model: ImagesModel<ImagesApi>) => void | Promise<void>;
|
||||
/**
|
||||
* Optional custom HTTP headers to include in API requests.
|
||||
* Merged with provider defaults; can override default headers.
|
||||
*/
|
||||
headers?: Record<string, string>;
|
||||
/**
|
||||
* HTTP request timeout in milliseconds for providers/SDKs that support it.
|
||||
*/
|
||||
timeoutMs?: number;
|
||||
/**
|
||||
* Maximum retry attempts for providers/SDKs that support client-side retries.
|
||||
*/
|
||||
maxRetries?: number;
|
||||
/**
|
||||
* Maximum delay in milliseconds to wait for a retry when the server requests a long wait.
|
||||
* If the server's requested delay exceeds this value, the request fails immediately
|
||||
* with an error containing the requested delay, allowing higher-level retry logic
|
||||
* to handle it with user visibility.
|
||||
* Default: 60000 (60 seconds). Set to 0 to disable the cap.
|
||||
*/
|
||||
maxRetryDelayMs?: number;
|
||||
/**
|
||||
* Optional metadata to include in API requests.
|
||||
* Providers extract the fields they understand and ignore the rest.
|
||||
*/
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export type ProviderImagesOptions = ImagesOptions & Record<string, unknown>;
|
||||
|
||||
// Unified options with reasoning passed to streamSimple() and completeSimple()
|
||||
export interface SimpleStreamOptions extends StreamOptions {
|
||||
reasoning?: ThinkingLevel;
|
||||
@@ -158,6 +208,12 @@ export type StreamFunction<TApi extends Api = Api, TOptions extends StreamOption
|
||||
options?: TOptions,
|
||||
) => AssistantMessageEventStream;
|
||||
|
||||
export type ImagesFunction<TApi extends ImagesApi = ImagesApi, TOptions extends ImagesOptions = ImagesOptions> = (
|
||||
model: ImagesModel<TApi>,
|
||||
context: ImagesContext,
|
||||
options?: TOptions,
|
||||
) => Promise<AssistantImages>;
|
||||
|
||||
export interface TextSignatureV1 {
|
||||
v: 1;
|
||||
id: string;
|
||||
@@ -244,6 +300,27 @@ export interface ToolResultMessage<TDetails = any> {
|
||||
|
||||
export type Message = UserMessage | AssistantMessage | ToolResultMessage;
|
||||
|
||||
export type ImagesInputContent = TextContent | ImageContent;
|
||||
export type ImagesOutputContent = TextContent | ImageContent;
|
||||
|
||||
export interface ImagesContext {
|
||||
input: ImagesInputContent[];
|
||||
}
|
||||
|
||||
export type ImagesStopReason = "stop" | "error" | "aborted";
|
||||
|
||||
export interface AssistantImages {
|
||||
api: ImagesApi;
|
||||
provider: ImagesProvider;
|
||||
model: string;
|
||||
output: ImagesOutputContent[];
|
||||
responseId?: string;
|
||||
usage?: Usage;
|
||||
stopReason: ImagesStopReason;
|
||||
errorMessage?: string;
|
||||
timestamp: number; // Unix timestamp in milliseconds
|
||||
}
|
||||
|
||||
import type { TSchema } from "typebox";
|
||||
|
||||
export interface Tool<TParameters extends TSchema = TSchema> {
|
||||
@@ -462,3 +539,10 @@ export interface Model<TApi extends Api> {
|
||||
? AnthropicMessagesCompat
|
||||
: never;
|
||||
}
|
||||
|
||||
export interface ImagesModel<TApi extends ImagesApi>
|
||||
extends Omit<Model<Api>, "api" | "provider" | "reasoning" | "contextWindow" | "maxTokens" | "compat"> {
|
||||
api: TApi;
|
||||
provider: ImagesProvider;
|
||||
output: ("text" | "image")[];
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user