From e3d066daa0e81ad74185e631dbd8a1c289e34903 Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Mon, 4 May 2026 16:39:38 +0200 Subject: [PATCH] feat: images api types --- packages/ai/src/types.ts | 98 ++++++++++++++++++++++++++- packages/ai/src/utils/event-stream.ts | 23 ++++++- 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 8ffd785f..babb0f89 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -1,6 +1,9 @@ -import type { AssistantMessageEventStream } from "./utils/event-stream.js"; +import type { AssistantImagesEventStream, AssistantMessageEventStream } from "./utils/event-stream.js"; -export type { AssistantMessageEventStream } from "./utils/event-stream.js"; +export type { + AssistantImagesEventStream, + AssistantMessageEventStream, +} from "./utils/event-stream.js"; export type KnownApi = | "openai-completions" @@ -16,6 +19,10 @@ export type KnownApi = export type Api = KnownApi | (string & {}); +export type KnownImagesApi = "openrouter-images"; + +export type ImagesApi = KnownImagesApi | (string & {}); + export type KnownProvider = | "amazon-bedrock" | "anthropic" @@ -45,6 +52,10 @@ export type KnownProvider = | "cloudflare-workers-ai"; export type Provider = KnownProvider | string; +export type KnownImagesProvider = "openrouter"; + +export type ImagesProvider = KnownImagesProvider | string; + export type ThinkingLevel = "minimal" | "low" | "medium" | "high" | "xhigh"; /** Token budgets for each thinking level (token-based providers only) */ @@ -130,6 +141,48 @@ export interface StreamOptions { export type ProviderStreamOptions = StreamOptions & Record; +export interface ImagesOptions { + signal?: AbortSignal; + apiKey?: string; + /** + * Optional callback for inspecting or replacing provider payloads before sending. + * Return undefined to keep the payload unchanged. + */ + onPayload?: (payload: unknown, model: ImagesModel) => unknown | undefined | Promise; + /** + * Optional callback invoked after an HTTP response is received. + */ + onResponse?: (response: ProviderResponse, model: ImagesModel) => void | Promise; + /** + * Optional custom HTTP headers to include in API requests. + * Merged with provider defaults; can override default headers. + */ + headers?: Record; + /** + * HTTP request timeout in milliseconds for providers/SDKs that support it. + */ + timeoutMs?: number; + /** + * Maximum retry attempts for providers/SDKs that support client-side retries. + */ + maxRetries?: number; + /** + * Maximum delay in milliseconds to wait for a retry when the server requests a long wait. + * If the server's requested delay exceeds this value, the request fails immediately + * with an error containing the requested delay, allowing higher-level retry logic + * to handle it with user visibility. + * Default: 60000 (60 seconds). Set to 0 to disable the cap. + */ + maxRetryDelayMs?: number; + /** + * Optional metadata to include in API requests. + * Providers extract the fields they understand and ignore the rest. + */ + metadata?: Record; +} + +export type ProviderImagesOptions = ImagesOptions & Record; + // Unified options with reasoning passed to streamSimple() and completeSimple() export interface SimpleStreamOptions extends StreamOptions { reasoning?: ThinkingLevel; @@ -151,6 +204,12 @@ export type StreamFunction AssistantMessageEventStream; +export type ImagesFunction = ( + model: ImagesModel, + context: ImagesContext, + options?: TOptions, +) => AssistantImagesEventStream; + export interface TextSignatureV1 { v: 1; id: string; @@ -235,6 +294,27 @@ export interface ToolResultMessage { export type Message = UserMessage | AssistantMessage | ToolResultMessage; +export type ImagesInputContent = TextContent | ImageContent; +export type ImagesOutputContent = TextContent | ImageContent; + +export interface ImagesContext { + input: ImagesInputContent[]; +} + +export type ImagesStopReason = "stop" | "error" | "aborted"; + +export interface AssistantImages { + api: ImagesApi; + provider: ImagesProvider; + model: string; + output: ImagesOutputContent[]; + responseId?: string; + usage?: Usage; + stopReason: ImagesStopReason; + errorMessage?: string; + timestamp: number; // Unix timestamp in milliseconds +} + import type { TSchema } from "typebox"; export interface Tool { @@ -271,6 +351,13 @@ export type AssistantMessageEvent = | { type: "done"; reason: Extract; message: AssistantMessage } | { type: "error"; reason: Extract; error: AssistantMessage }; +export type AssistantImagesEvent = + | { type: "start"; partial: AssistantImages } + | { type: "image_start"; contentIndex: number; partial: AssistantImages } + | { type: "image_end"; contentIndex: number; image: ImageContent; partial: AssistantImages } + | { type: "done"; reason: Extract; images: AssistantImages } + | { type: "error"; reason: Extract; error: AssistantImages }; + /** * Compatibility settings for OpenAI-compatible completions APIs. * Use this to override URL-based auto-detection for custom providers. @@ -450,3 +537,10 @@ export interface Model { ? AnthropicMessagesCompat : never; } + +export interface ImagesModel + extends Omit, "api" | "provider" | "reasoning" | "contextWindow" | "maxTokens" | "compat"> { + api: TApi; + provider: ImagesProvider; + output: ("text" | "image")[]; +} diff --git a/packages/ai/src/utils/event-stream.ts b/packages/ai/src/utils/event-stream.ts index f4a7ceba..9fb1c114 100644 --- a/packages/ai/src/utils/event-stream.ts +++ b/packages/ai/src/utils/event-stream.ts @@ -1,4 +1,4 @@ -import type { AssistantMessage, AssistantMessageEvent } from "../types.js"; +import type { AssistantImages, AssistantImagesEvent, AssistantMessage, AssistantMessageEvent } from "../types.js"; // Generic event stream class for async iteration export class EventStream implements AsyncIterable { @@ -81,7 +81,28 @@ export class AssistantMessageEventStream extends EventStream { + constructor() { + super( + (event) => event.type === "done" || event.type === "error", + (event) => { + if (event.type === "done") { + return event.images; + } else if (event.type === "error") { + return event.error; + } + throw new Error("Unexpected event type for final result"); + }, + ); + } +} + /** Factory function for AssistantMessageEventStream (for use in extensions) */ export function createAssistantMessageEventStream(): AssistantMessageEventStream { return new AssistantMessageEventStream(); } + +/** Factory function for AssistantImagesEventStream (for use in extensions) */ +export function createAssistantImagesEventStream(): AssistantImagesEventStream { + return new AssistantImagesEventStream(); +}