delete: images()
This commit is contained in:
@@ -18,7 +18,6 @@ Unified LLM API with automatic model discovery, provider configuration, token an
|
|||||||
- [Image Input](#image-input)
|
- [Image Input](#image-input)
|
||||||
- [Image Generation](#image-generation)
|
- [Image Generation](#image-generation)
|
||||||
- [Basic Image Generation](#basic-image-generation)
|
- [Basic Image Generation](#basic-image-generation)
|
||||||
- [Streaming Image Events](#streaming-image-events)
|
|
||||||
- [Notes and Limitations](#notes-and-limitations)
|
- [Notes and Limitations](#notes-and-limitations)
|
||||||
- [Thinking/Reasoning](#thinkingreasoning)
|
- [Thinking/Reasoning](#thinkingreasoning)
|
||||||
- [Unified Interface](#unified-interface-streamsimplecompletesimple)
|
- [Unified Interface](#unified-interface-streamsimplecompletesimple)
|
||||||
@@ -425,9 +424,9 @@ for (const block of response.content) {
|
|||||||
|
|
||||||
## Image Generation
|
## Image Generation
|
||||||
|
|
||||||
Image generation uses a separate API surface from text/chat generation. Use `getImageModel()` / `getImageModels()` / `getImageProviders()` to discover image-generation models, `images()` to stream events, and `generateImages()` to get the final result.
|
Image generation uses a separate API surface from text/chat generation. Use `getImageModel()` / `getImageModels()` / `getImageProviders()` to discover image-generation models, and `generateImages()` to get the final result.
|
||||||
|
|
||||||
Do not use `stream()` or `complete()` for image generation. Request options, error handling, and event/result flow follow the same overall pattern as the streaming APIs, but use image-specific types and events.
|
Do not use `stream()` or `complete()` for image generation. Image generation is a one-shot API: `generateImages()` waits for the provider response and returns the final `AssistantImages` result.
|
||||||
|
|
||||||
### Basic Image Generation
|
### Basic Image Generation
|
||||||
|
|
||||||
@@ -475,59 +474,12 @@ console.log(model.input); // ['text', 'image']
|
|||||||
console.log(model.output); // ['image'] or ['image', 'text']
|
console.log(model.output); // ['image'] or ['image', 'text']
|
||||||
```
|
```
|
||||||
|
|
||||||
### Streaming Image Events
|
|
||||||
|
|
||||||
`images()` returns an `AssistantImagesEventStream` with image-specific events:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { getImageModel, images } from '@mariozechner/pi-ai';
|
|
||||||
|
|
||||||
const model = getImageModel('openrouter', 'google/gemini-2.5-flash-image');
|
|
||||||
const s = images(model, {
|
|
||||||
input: [{ type: 'text', text: 'Generate a simple green triangle.' }]
|
|
||||||
}, {
|
|
||||||
apiKey: process.env.OPENROUTER_API_KEY
|
|
||||||
});
|
|
||||||
|
|
||||||
for await (const event of s) {
|
|
||||||
switch (event.type) {
|
|
||||||
case 'start':
|
|
||||||
console.log(`Starting image generation with ${event.partial.model}`);
|
|
||||||
break;
|
|
||||||
case 'image_start':
|
|
||||||
console.log(`Image block started at index ${event.contentIndex}`);
|
|
||||||
break;
|
|
||||||
case 'image_end':
|
|
||||||
console.log(`Image block complete: ${event.image.mimeType}`);
|
|
||||||
break;
|
|
||||||
case 'done':
|
|
||||||
console.log(`Finished: ${event.reason}`);
|
|
||||||
break;
|
|
||||||
case 'error':
|
|
||||||
console.error(`Error: ${event.error.errorMessage}`);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const finalResult = await s.result();
|
|
||||||
```
|
|
||||||
|
|
||||||
| Event Type | Description | Key Properties |
|
|
||||||
|------------|-------------|----------------|
|
|
||||||
| `start` | Stream begins | `partial`: Initial `AssistantImages` structure |
|
|
||||||
| `image_start` | Image block starts | `contentIndex`: Position in output array |
|
|
||||||
| `image_end` | Image block complete | `image`: Final `ImageContent`, `contentIndex`: Position |
|
|
||||||
| `done` | Stream complete | `reason`: Stop reason (`"stop"`), `images`: Final `AssistantImages` |
|
|
||||||
| `error` | Error occurred | `reason`: Error type (`"error"` or `"aborted"`), `error`: Final `AssistantImages` |
|
|
||||||
|
|
||||||
Text output, if the model returns any, is available on the final `AssistantImages.output` array.
|
|
||||||
|
|
||||||
### Notes and Limitations
|
### Notes and Limitations
|
||||||
|
|
||||||
- Use `getImageModel(...)`, not `getModel(...)`.
|
- Use `getImageModel(...)`, not `getModel(...)`.
|
||||||
- Use `images()` / `generateImages()`, not `stream()` / `complete()`.
|
- Use `generateImages()`, not `stream()` / `complete()`.
|
||||||
- Image-generation models do not participate in tool calling.
|
- Image-generation models do not participate in tool calling.
|
||||||
- Outputs are returned as base64-encoded `ImageContent` blocks in `AssistantImages.output`.
|
- Outputs are returned in `AssistantImages.output` and can include both base64-encoded `ImageContent` blocks and `TextContent` blocks.
|
||||||
- Some models return only images, others return images plus text. Check `model.output`.
|
- Some models return only images, others return images plus text. Check `model.output`.
|
||||||
- Some models accept image input, others are text-to-image only. Check `model.input`.
|
- Some models accept image input, others are text-to-image only. Check `model.input`.
|
||||||
- Like the streaming APIs, image generation supports options such as `apiKey`, `signal`, `headers`, `onPayload`, and `onResponse`, and results may include `stopReason`, `responseId`, and `usage`.
|
- Like the streaming APIs, image generation supports options such as `apiKey`, `signal`, `headers`, `onPayload`, and `onResponse`, and results may include `stopReason`, `responseId`, and `usage`.
|
||||||
|
|||||||
@@ -1,26 +1,19 @@
|
|||||||
import type {
|
import type { AssistantImages, ImagesApi, ImagesContext, ImagesFunction, ImagesModel, ImagesOptions } from "./types.js";
|
||||||
AssistantImagesEventStream,
|
|
||||||
ImagesApi,
|
|
||||||
ImagesContext,
|
|
||||||
ImagesFunction,
|
|
||||||
ImagesModel,
|
|
||||||
ImagesOptions,
|
|
||||||
} from "./types.js";
|
|
||||||
|
|
||||||
export type ImagesApiFunction = (
|
export type ImagesApiFunction = (
|
||||||
model: ImagesModel<ImagesApi>,
|
model: ImagesModel<ImagesApi>,
|
||||||
context: ImagesContext,
|
context: ImagesContext,
|
||||||
options?: ImagesOptions,
|
options?: ImagesOptions,
|
||||||
) => AssistantImagesEventStream;
|
) => Promise<AssistantImages>;
|
||||||
|
|
||||||
export interface ImagesApiProvider<TApi extends ImagesApi = ImagesApi, TOptions extends ImagesOptions = ImagesOptions> {
|
export interface ImagesApiProvider<TApi extends ImagesApi = ImagesApi, TOptions extends ImagesOptions = ImagesOptions> {
|
||||||
api: TApi;
|
api: TApi;
|
||||||
images: ImagesFunction<TApi, TOptions>;
|
generateImages: ImagesFunction<TApi, TOptions>;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface ImagesApiProviderInternal {
|
interface ImagesApiProviderInternal {
|
||||||
api: ImagesApi;
|
api: ImagesApi;
|
||||||
images: ImagesApiFunction;
|
generateImages: ImagesApiFunction;
|
||||||
}
|
}
|
||||||
|
|
||||||
type RegisteredImagesApiProvider = {
|
type RegisteredImagesApiProvider = {
|
||||||
@@ -30,15 +23,15 @@ type RegisteredImagesApiProvider = {
|
|||||||
|
|
||||||
const imagesApiProviderRegistry = new Map<string, RegisteredImagesApiProvider>();
|
const imagesApiProviderRegistry = new Map<string, RegisteredImagesApiProvider>();
|
||||||
|
|
||||||
function wrapImages<TApi extends ImagesApi, TOptions extends ImagesOptions>(
|
function wrapGenerateImages<TApi extends ImagesApi, TOptions extends ImagesOptions>(
|
||||||
api: TApi,
|
api: TApi,
|
||||||
images: ImagesFunction<TApi, TOptions>,
|
generateImages: ImagesFunction<TApi, TOptions>,
|
||||||
): ImagesApiFunction {
|
): ImagesApiFunction {
|
||||||
return (model, context, options) => {
|
return (model, context, options) => {
|
||||||
if (model.api !== api) {
|
if (model.api !== api) {
|
||||||
throw new Error(`Mismatched api: ${model.api} expected ${api}`);
|
throw new Error(`Mismatched api: ${model.api} expected ${api}`);
|
||||||
}
|
}
|
||||||
return images(model as ImagesModel<TApi>, context, options as TOptions);
|
return generateImages(model as ImagesModel<TApi>, context, options as TOptions);
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -49,7 +42,7 @@ export function registerImagesApiProvider<TApi extends ImagesApi, TOptions exten
|
|||||||
imagesApiProviderRegistry.set(provider.api, {
|
imagesApiProviderRegistry.set(provider.api, {
|
||||||
provider: {
|
provider: {
|
||||||
api: provider.api,
|
api: provider.api,
|
||||||
images: wrapImages(provider.api, provider.images),
|
generateImages: wrapGenerateImages(provider.api, provider.generateImages),
|
||||||
},
|
},
|
||||||
sourceId,
|
sourceId,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,15 +1,7 @@
|
|||||||
import "./providers/images/register-builtins.js";
|
import "./providers/images/register-builtins.js";
|
||||||
|
|
||||||
import { getImagesApiProvider } from "./images-api-registry.js";
|
import { getImagesApiProvider } from "./images-api-registry.js";
|
||||||
import type {
|
import type { AssistantImages, ImagesApi, ImagesContext, ImagesModel, ProviderImagesOptions } from "./types.js";
|
||||||
AssistantImages,
|
|
||||||
AssistantImagesEventStream,
|
|
||||||
ImagesApi,
|
|
||||||
ImagesContext,
|
|
||||||
ImagesModel,
|
|
||||||
ImagesOptions,
|
|
||||||
ProviderImagesOptions,
|
|
||||||
} from "./types.js";
|
|
||||||
|
|
||||||
function resolveImagesApiProvider(api: ImagesApi) {
|
function resolveImagesApiProvider(api: ImagesApi) {
|
||||||
const provider = getImagesApiProvider(api);
|
const provider = getImagesApiProvider(api);
|
||||||
@@ -19,20 +11,11 @@ function resolveImagesApiProvider(api: ImagesApi) {
|
|||||||
return provider;
|
return provider;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function images<TApi extends ImagesApi>(
|
|
||||||
model: ImagesModel<TApi>,
|
|
||||||
context: ImagesContext,
|
|
||||||
options?: ProviderImagesOptions,
|
|
||||||
): AssistantImagesEventStream {
|
|
||||||
const provider = resolveImagesApiProvider(model.api);
|
|
||||||
return provider.images(model, context, options as ImagesOptions);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function generateImages<TApi extends ImagesApi>(
|
export async function generateImages<TApi extends ImagesApi>(
|
||||||
model: ImagesModel<TApi>,
|
model: ImagesModel<TApi>,
|
||||||
context: ImagesContext,
|
context: ImagesContext,
|
||||||
options?: ProviderImagesOptions,
|
options?: ProviderImagesOptions,
|
||||||
): Promise<AssistantImages> {
|
): Promise<AssistantImages> {
|
||||||
const s = images(model, context, options);
|
const provider = resolveImagesApiProvider(model.api);
|
||||||
return s.result();
|
return provider.generateImages(model, context, options);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ import type {
|
|||||||
ImagesOptions,
|
ImagesOptions,
|
||||||
TextContent,
|
TextContent,
|
||||||
} from "../../types.js";
|
} from "../../types.js";
|
||||||
import { AssistantImagesEventStream } from "../../utils/event-stream.js";
|
|
||||||
import { headersToRecord } from "../../utils/headers.js";
|
import { headersToRecord } from "../../utils/headers.js";
|
||||||
import { sanitizeSurrogates } from "../../utils/sanitize-unicode.js";
|
import { sanitizeSurrogates } from "../../utils/sanitize-unicode.js";
|
||||||
|
|
||||||
@@ -36,87 +35,70 @@ type OpenRouterImageGenerationResponse = ChatCompletion & {
|
|||||||
choices: OpenRouterImageGenerationChoice[];
|
choices: OpenRouterImageGenerationChoice[];
|
||||||
};
|
};
|
||||||
|
|
||||||
export const imagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = (
|
export const generateImagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = async (
|
||||||
model: ImagesModel<"openrouter-images">,
|
model: ImagesModel<"openrouter-images">,
|
||||||
context: ImagesContext,
|
context: ImagesContext,
|
||||||
options?: ImagesOptions,
|
options?: ImagesOptions,
|
||||||
) => {
|
) => {
|
||||||
const stream = new AssistantImagesEventStream();
|
const output: AssistantImages = {
|
||||||
|
api: model.api,
|
||||||
|
provider: model.provider,
|
||||||
|
model: model.id,
|
||||||
|
output: [],
|
||||||
|
stopReason: "stop",
|
||||||
|
timestamp: Date.now(),
|
||||||
|
};
|
||||||
|
|
||||||
(async () => {
|
try {
|
||||||
const output: AssistantImages = {
|
const apiKey = options?.apiKey || getEnvApiKey(model.provider) || "";
|
||||||
api: model.api,
|
const client = createClient(model, apiKey, options?.headers);
|
||||||
provider: model.provider,
|
let params = buildParams(model, context);
|
||||||
model: model.id,
|
const nextParams = await options?.onPayload?.(params, model);
|
||||||
output: [],
|
if (nextParams !== undefined) {
|
||||||
stopReason: "stop",
|
params = nextParams as typeof params;
|
||||||
timestamp: Date.now(),
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
const apiKey = options?.apiKey || getEnvApiKey(model.provider) || "";
|
|
||||||
const client = createClient(model, apiKey, options?.headers);
|
|
||||||
let params = buildParams(model, context);
|
|
||||||
const nextParams = await options?.onPayload?.(params, model);
|
|
||||||
if (nextParams !== undefined) {
|
|
||||||
params = nextParams as typeof params;
|
|
||||||
}
|
|
||||||
const requestOptions = {
|
|
||||||
...(options?.signal ? { signal: options.signal } : {}),
|
|
||||||
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
|
|
||||||
...(options?.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}),
|
|
||||||
};
|
|
||||||
const { data: response, response: rawResponse } = await client.chat.completions
|
|
||||||
.create(params as unknown as ChatCompletionCreateParamsNonStreaming, requestOptions)
|
|
||||||
.withResponse();
|
|
||||||
await options?.onResponse?.(
|
|
||||||
{ status: rawResponse.status, headers: headersToRecord(rawResponse.headers) },
|
|
||||||
model,
|
|
||||||
);
|
|
||||||
|
|
||||||
stream.push({ type: "start", partial: output });
|
|
||||||
|
|
||||||
const imageResponse = response as OpenRouterImageGenerationResponse;
|
|
||||||
output.responseId = imageResponse.id;
|
|
||||||
if (imageResponse.usage) {
|
|
||||||
output.usage = parseUsage(imageResponse.usage, model);
|
|
||||||
}
|
|
||||||
|
|
||||||
const choice = imageResponse.choices[0];
|
|
||||||
if (choice) {
|
|
||||||
const content = choice.message.content;
|
|
||||||
if (typeof content === "string" && content.length > 0) {
|
|
||||||
output.output.push({ type: "text", text: content } satisfies TextContent);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const image of choice.message.images ?? []) {
|
|
||||||
const imageUrl = typeof image.image_url === "string" ? image.image_url : image.image_url?.url;
|
|
||||||
if (!imageUrl?.startsWith("data:")) continue;
|
|
||||||
const matches = imageUrl.match(/^data:([^;]+);base64,(.+)$/);
|
|
||||||
if (!matches) continue;
|
|
||||||
const imageBlock: ImageContent = {
|
|
||||||
type: "image",
|
|
||||||
mimeType: matches[1],
|
|
||||||
data: matches[2],
|
|
||||||
};
|
|
||||||
output.output.push(imageBlock);
|
|
||||||
const contentIndex = output.output.length - 1;
|
|
||||||
stream.push({ type: "image_start", contentIndex, partial: output });
|
|
||||||
stream.push({ type: "image_end", contentIndex, image: imageBlock, partial: output });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stream.push({ type: "done", reason: "stop", images: output });
|
|
||||||
stream.end();
|
|
||||||
} catch (error) {
|
|
||||||
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
|
||||||
output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
|
|
||||||
stream.push({ type: "error", reason: output.stopReason, error: output });
|
|
||||||
stream.end(output);
|
|
||||||
}
|
}
|
||||||
})();
|
const requestOptions = {
|
||||||
|
...(options?.signal ? { signal: options.signal } : {}),
|
||||||
|
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
|
||||||
|
...(options?.maxRetries !== undefined ? { maxRetries: options.maxRetries } : {}),
|
||||||
|
};
|
||||||
|
const { data: response, response: rawResponse } = await client.chat.completions
|
||||||
|
.create(params as unknown as ChatCompletionCreateParamsNonStreaming, requestOptions)
|
||||||
|
.withResponse();
|
||||||
|
await options?.onResponse?.({ status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, model);
|
||||||
|
|
||||||
return stream;
|
const imageResponse = response as OpenRouterImageGenerationResponse;
|
||||||
|
output.responseId = imageResponse.id;
|
||||||
|
if (imageResponse.usage) {
|
||||||
|
output.usage = parseUsage(imageResponse.usage, model);
|
||||||
|
}
|
||||||
|
|
||||||
|
const choice = imageResponse.choices[0];
|
||||||
|
if (choice) {
|
||||||
|
const content = choice.message.content;
|
||||||
|
if (typeof content === "string" && content.length > 0) {
|
||||||
|
output.output.push({ type: "text", text: content } satisfies TextContent);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const image of choice.message.images ?? []) {
|
||||||
|
const imageUrl = typeof image.image_url === "string" ? image.image_url : image.image_url?.url;
|
||||||
|
if (!imageUrl?.startsWith("data:")) continue;
|
||||||
|
const matches = imageUrl.match(/^data:([^;]+);base64,(.+)$/);
|
||||||
|
if (!matches) continue;
|
||||||
|
output.output.push({
|
||||||
|
type: "image",
|
||||||
|
mimeType: matches[1],
|
||||||
|
data: matches[2],
|
||||||
|
} satisfies ImageContent);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return output;
|
||||||
|
} catch (error) {
|
||||||
|
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
||||||
|
output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
|
||||||
|
return output;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
function createClient(
|
function createClient(
|
||||||
|
|||||||
@@ -1,74 +1,10 @@
|
|||||||
import { registerImagesApiProvider } from "../../images-api-registry.js";
|
import { registerImagesApiProvider } from "../../images-api-registry.js";
|
||||||
import type {
|
import { generateImagesOpenRouter } from "./openrouter.js";
|
||||||
AssistantImages,
|
|
||||||
AssistantImagesEvent,
|
|
||||||
ImagesContext,
|
|
||||||
ImagesFunction,
|
|
||||||
ImagesModel,
|
|
||||||
ImagesOptions,
|
|
||||||
} from "../../types.js";
|
|
||||||
import { AssistantImagesEventStream } from "../../utils/event-stream.js";
|
|
||||||
import type { imagesOpenRouter as imagesOpenRouterFunction } from "./openrouter.js";
|
|
||||||
|
|
||||||
interface OpenRouterImagesProviderModule {
|
|
||||||
imagesOpenRouter: typeof imagesOpenRouterFunction;
|
|
||||||
}
|
|
||||||
|
|
||||||
let openRouterImagesProviderModulePromise: Promise<OpenRouterImagesProviderModule> | undefined;
|
|
||||||
|
|
||||||
function forwardImagesStream(target: AssistantImagesEventStream, source: AsyncIterable<AssistantImagesEvent>): void {
|
|
||||||
(async () => {
|
|
||||||
for await (const event of source) {
|
|
||||||
target.push(event);
|
|
||||||
}
|
|
||||||
target.end();
|
|
||||||
})();
|
|
||||||
}
|
|
||||||
|
|
||||||
function createLazyLoadErrorImages(model: ImagesModel<"openrouter-images">, error: unknown): AssistantImages {
|
|
||||||
return {
|
|
||||||
api: model.api,
|
|
||||||
provider: model.provider,
|
|
||||||
model: model.id,
|
|
||||||
output: [],
|
|
||||||
stopReason: "error",
|
|
||||||
errorMessage: error instanceof Error ? error.message : String(error),
|
|
||||||
timestamp: Date.now(),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function loadOpenRouterImagesProviderModule(): Promise<OpenRouterImagesProviderModule> {
|
|
||||||
openRouterImagesProviderModulePromise ||= import("./openrouter.js").then(
|
|
||||||
(module) => module as OpenRouterImagesProviderModule,
|
|
||||||
);
|
|
||||||
return openRouterImagesProviderModulePromise;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const imagesOpenRouter: ImagesFunction<"openrouter-images", ImagesOptions> = (
|
|
||||||
model: ImagesModel<"openrouter-images">,
|
|
||||||
context: ImagesContext,
|
|
||||||
options?: ImagesOptions,
|
|
||||||
) => {
|
|
||||||
const outer = new AssistantImagesEventStream();
|
|
||||||
|
|
||||||
loadOpenRouterImagesProviderModule()
|
|
||||||
.then((module) => {
|
|
||||||
const inner = module.imagesOpenRouter(model, context, options);
|
|
||||||
forwardImagesStream(outer, inner);
|
|
||||||
})
|
|
||||||
.catch((error) => {
|
|
||||||
const images = createLazyLoadErrorImages(model, error);
|
|
||||||
outer.push({ type: "error", reason: "error", error: images });
|
|
||||||
outer.end(images);
|
|
||||||
});
|
|
||||||
|
|
||||||
return outer;
|
|
||||||
};
|
|
||||||
|
|
||||||
export function registerBuiltInImagesApiProviders(): void {
|
export function registerBuiltInImagesApiProviders(): void {
|
||||||
registerImagesApiProvider({
|
registerImagesApiProvider({
|
||||||
api: "openrouter-images",
|
api: "openrouter-images",
|
||||||
images: imagesOpenRouter,
|
generateImages: generateImagesOpenRouter,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,7 @@
|
|||||||
import type { AssistantMessageDiagnostic } from "./utils/diagnostics.js";
|
import type { AssistantMessageDiagnostic } from "./utils/diagnostics.js";
|
||||||
import type { AssistantImagesEventStream, AssistantMessageEventStream } from "./utils/event-stream.js";
|
import type { AssistantMessageEventStream } from "./utils/event-stream.js";
|
||||||
|
|
||||||
export type {
|
export type { AssistantMessageEventStream } from "./utils/event-stream.js";
|
||||||
AssistantImagesEventStream,
|
|
||||||
AssistantMessageEventStream,
|
|
||||||
} from "./utils/event-stream.js";
|
|
||||||
|
|
||||||
export type KnownApi =
|
export type KnownApi =
|
||||||
| "openai-completions"
|
| "openai-completions"
|
||||||
@@ -215,7 +212,7 @@ export type ImagesFunction<TApi extends ImagesApi = ImagesApi, TOptions extends
|
|||||||
model: ImagesModel<TApi>,
|
model: ImagesModel<TApi>,
|
||||||
context: ImagesContext,
|
context: ImagesContext,
|
||||||
options?: TOptions,
|
options?: TOptions,
|
||||||
) => AssistantImagesEventStream;
|
) => Promise<AssistantImages>;
|
||||||
|
|
||||||
export interface TextSignatureV1 {
|
export interface TextSignatureV1 {
|
||||||
v: 1;
|
v: 1;
|
||||||
@@ -360,13 +357,6 @@ export type AssistantMessageEvent =
|
|||||||
| { type: "done"; reason: Extract<StopReason, "stop" | "length" | "toolUse">; message: AssistantMessage }
|
| { type: "done"; reason: Extract<StopReason, "stop" | "length" | "toolUse">; message: AssistantMessage }
|
||||||
| { type: "error"; reason: Extract<StopReason, "aborted" | "error">; error: AssistantMessage };
|
| { type: "error"; reason: Extract<StopReason, "aborted" | "error">; error: AssistantMessage };
|
||||||
|
|
||||||
export type AssistantImagesEvent =
|
|
||||||
| { type: "start"; partial: AssistantImages }
|
|
||||||
| { type: "image_start"; contentIndex: number; partial: AssistantImages }
|
|
||||||
| { type: "image_end"; contentIndex: number; image: ImageContent; partial: AssistantImages }
|
|
||||||
| { type: "done"; reason: Extract<ImagesStopReason, "stop">; images: AssistantImages }
|
|
||||||
| { type: "error"; reason: Extract<ImagesStopReason, "aborted" | "error">; error: AssistantImages };
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compatibility settings for OpenAI-compatible completions APIs.
|
* Compatibility settings for OpenAI-compatible completions APIs.
|
||||||
* Use this to override URL-based auto-detection for custom providers.
|
* Use this to override URL-based auto-detection for custom providers.
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import type { AssistantImages, AssistantImagesEvent, AssistantMessage, AssistantMessageEvent } from "../types.js";
|
import type { AssistantMessage, AssistantMessageEvent } from "../types.js";
|
||||||
|
|
||||||
// Generic event stream class for async iteration
|
// Generic event stream class for async iteration
|
||||||
export class EventStream<T, R = T> implements AsyncIterable<T> {
|
export class EventStream<T, R = T> implements AsyncIterable<T> {
|
||||||
@@ -81,28 +81,7 @@ export class AssistantMessageEventStream extends EventStream<AssistantMessageEve
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export class AssistantImagesEventStream extends EventStream<AssistantImagesEvent, AssistantImages> {
|
|
||||||
constructor() {
|
|
||||||
super(
|
|
||||||
(event) => event.type === "done" || event.type === "error",
|
|
||||||
(event) => {
|
|
||||||
if (event.type === "done") {
|
|
||||||
return event.images;
|
|
||||||
} else if (event.type === "error") {
|
|
||||||
return event.error;
|
|
||||||
}
|
|
||||||
throw new Error("Unexpected event type for final result");
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Factory function for AssistantMessageEventStream (for use in extensions) */
|
/** Factory function for AssistantMessageEventStream (for use in extensions) */
|
||||||
export function createAssistantMessageEventStream(): AssistantMessageEventStream {
|
export function createAssistantMessageEventStream(): AssistantMessageEventStream {
|
||||||
return new AssistantMessageEventStream();
|
return new AssistantMessageEventStream();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Factory function for AssistantImagesEventStream (for use in extensions) */
|
|
||||||
export function createAssistantImagesEventStream(): AssistantImagesEventStream {
|
|
||||||
return new AssistantImagesEventStream();
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { dirname, join } from "node:path";
|
|||||||
import { fileURLToPath } from "node:url";
|
import { fileURLToPath } from "node:url";
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { getImageModel } from "../src/image-models.js";
|
import { getImageModel } from "../src/image-models.js";
|
||||||
import { generateImages, images } from "../src/images.js";
|
import { generateImages } from "../src/images.js";
|
||||||
import type { ImageContent, ImagesContext, ImagesModel, ProviderImagesOptions } from "../src/types.js";
|
import type { ImageContent, ImagesContext, ImagesModel, ProviderImagesOptions } from "../src/types.js";
|
||||||
|
|
||||||
const __filename = fileURLToPath(import.meta.url);
|
const __filename = fileURLToPath(import.meta.url);
|
||||||
@@ -24,36 +24,6 @@ async function basicImageGeneration<TApi extends string>(model: ImagesModel<TApi
|
|||||||
expect(response.timestamp).toBeGreaterThan(0);
|
expect(response.timestamp).toBeGreaterThan(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function handleStreaming<TApi extends string>(model: ImagesModel<TApi>, options?: ImagesOptionsWithExtras) {
|
|
||||||
const context: ImagesContext = {
|
|
||||||
input: [{ type: "text", text: "Generate a simple blue square on a plain white background. No text." }],
|
|
||||||
};
|
|
||||||
|
|
||||||
const s = images(model, context, options);
|
|
||||||
let started = false;
|
|
||||||
let imageStarted = false;
|
|
||||||
let imageCompleted = false;
|
|
||||||
|
|
||||||
for await (const event of s) {
|
|
||||||
if (event.type === "start") {
|
|
||||||
started = true;
|
|
||||||
} else if (event.type === "image_start") {
|
|
||||||
imageStarted = true;
|
|
||||||
} else if (event.type === "image_end") {
|
|
||||||
imageCompleted = true;
|
|
||||||
expect(event.image.type).toBe("image");
|
|
||||||
expect(event.image.data.length).toBeGreaterThan(0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await s.result();
|
|
||||||
expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop");
|
|
||||||
expect(started).toBe(true);
|
|
||||||
expect(imageStarted).toBe(true);
|
|
||||||
expect(imageCompleted).toBe(true);
|
|
||||||
expect(response.output.some((item) => item.type === "image")).toBe(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function handleTextAndImageOutput<TApi extends string>(
|
async function handleTextAndImageOutput<TApi extends string>(
|
||||||
model: ImagesModel<TApi>,
|
model: ImagesModel<TApi>,
|
||||||
options?: ImagesOptionsWithExtras,
|
options?: ImagesOptionsWithExtras,
|
||||||
@@ -108,10 +78,6 @@ describe("Images E2E Tests", () => {
|
|||||||
await basicImageGeneration(model);
|
await basicImageGeneration(model);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("should handle image streaming", { retry: 3 }, async () => {
|
|
||||||
await handleStreaming(model);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should handle text plus image output", { retry: 3 }, async () => {
|
it("should handle text plus image output", { retry: 3 }, async () => {
|
||||||
await handleTextAndImageOutput(model);
|
await handleTextAndImageOutput(model);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,17 +1,28 @@
|
|||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import { generateImages, images } from "../src/images.js";
|
import { generateImages } from "../src/images.js";
|
||||||
import type { ImagesContext, ImagesModel } from "../src/types.js";
|
import type { ImagesContext, ImagesModel } from "../src/types.js";
|
||||||
|
|
||||||
const mockState = vi.hoisted(() => ({
|
const mockState = vi.hoisted(() => ({
|
||||||
lastParams: undefined as unknown,
|
lastParams: undefined as unknown,
|
||||||
|
lastRequestOptions: undefined as unknown,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock("openai", () => {
|
vi.mock("openai", () => {
|
||||||
class FakeOpenAI {
|
class FakeOpenAI {
|
||||||
chat = {
|
chat = {
|
||||||
completions: {
|
completions: {
|
||||||
create: (params: unknown) => {
|
create: (params: unknown, requestOptions?: unknown) => {
|
||||||
mockState.lastParams = params;
|
mockState.lastParams = params;
|
||||||
|
mockState.lastRequestOptions = requestOptions;
|
||||||
|
const signal = (requestOptions as { signal?: AbortSignal } | undefined)?.signal;
|
||||||
|
if (signal?.aborted) {
|
||||||
|
const error = new Error("Request aborted");
|
||||||
|
return {
|
||||||
|
withResponse: async () => {
|
||||||
|
throw error;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
const response = {
|
const response = {
|
||||||
id: "img-1",
|
id: "img-1",
|
||||||
usage: {
|
usage: {
|
||||||
@@ -50,9 +61,10 @@ vi.mock("openai", () => {
|
|||||||
describe("openrouter images", () => {
|
describe("openrouter images", () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
mockState.lastParams = undefined;
|
mockState.lastParams = undefined;
|
||||||
|
mockState.lastRequestOptions = undefined;
|
||||||
});
|
});
|
||||||
|
|
||||||
it("emits image events and returns text plus images in final output", async () => {
|
it("returns text plus images in final output", async () => {
|
||||||
const model: ImagesModel<"openrouter-images"> = {
|
const model: ImagesModel<"openrouter-images"> = {
|
||||||
id: "google/gemini-3.1-flash-image-preview",
|
id: "google/gemini-3.1-flash-image-preview",
|
||||||
name: "Gemini 3.1 Flash Image Preview",
|
name: "Gemini 3.1 Flash Image Preview",
|
||||||
@@ -68,14 +80,8 @@ describe("openrouter images", () => {
|
|||||||
input: [{ type: "text", text: "Generate a dog" }],
|
input: [{ type: "text", text: "Generate a dog" }],
|
||||||
};
|
};
|
||||||
|
|
||||||
const result = images(model, context, { apiKey: "test" });
|
const output = await generateImages(model, context, { apiKey: "test" });
|
||||||
const eventTypes: string[] = [];
|
expect(output.stopReason).toBe("stop");
|
||||||
for await (const event of result) {
|
|
||||||
eventTypes.push(event.type);
|
|
||||||
}
|
|
||||||
|
|
||||||
const output = await result.result();
|
|
||||||
expect(eventTypes).toEqual(["start", "image_start", "image_end", "done"]);
|
|
||||||
expect(output.responseId).toBe("img-1");
|
expect(output.responseId).toBe("img-1");
|
||||||
expect(output.output[0]).toMatchObject({ type: "text", text: "Here is your image." });
|
expect(output.output[0]).toMatchObject({ type: "text", text: "Here is your image." });
|
||||||
expect(output.output[1]).toMatchObject({ type: "image", mimeType: "image/png", data: "ZmFrZS1wbmc=" });
|
expect(output.output[1]).toMatchObject({ type: "image", mimeType: "image/png", data: "ZmFrZS1wbmc=" });
|
||||||
@@ -90,6 +96,29 @@ describe("openrouter images", () => {
|
|||||||
expect(params.messages?.[0]?.content?.[0]).toMatchObject({ type: "text", text: "Generate a dog" });
|
expect(params.messages?.[0]?.content?.[0]).toMatchObject({ type: "text", text: "Generate a dog" });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("passes through abort signal and returns aborted result", async () => {
|
||||||
|
const model: ImagesModel<"openrouter-images"> = {
|
||||||
|
id: "black-forest-labs/flux.2-pro",
|
||||||
|
name: "FLUX.2 Pro",
|
||||||
|
api: "openrouter-images",
|
||||||
|
provider: "openrouter",
|
||||||
|
baseUrl: "https://openrouter.ai/api/v1",
|
||||||
|
input: ["text", "image"],
|
||||||
|
output: ["image"],
|
||||||
|
cost: { input: 0.015, output: 0.03, cacheRead: 0, cacheWrite: 0 },
|
||||||
|
};
|
||||||
|
const context: ImagesContext = {
|
||||||
|
input: [{ type: "text", text: "Generate a dog" }],
|
||||||
|
};
|
||||||
|
const controller = new AbortController();
|
||||||
|
controller.abort();
|
||||||
|
|
||||||
|
const output = await generateImages(model, context, { apiKey: "test", signal: controller.signal });
|
||||||
|
expect(output.stopReason).toBe("aborted");
|
||||||
|
expect(output.errorMessage).toBe("Request aborted");
|
||||||
|
expect(mockState.lastRequestOptions).toMatchObject({ signal: controller.signal });
|
||||||
|
});
|
||||||
|
|
||||||
it("generateImages resolves the final assistant images result", async () => {
|
it("generateImages resolves the final assistant images result", async () => {
|
||||||
const model: ImagesModel<"openrouter-images"> = {
|
const model: ImagesModel<"openrouter-images"> = {
|
||||||
id: "black-forest-labs/flux.2-pro",
|
id: "black-forest-labs/flux.2-pro",
|
||||||
|
|||||||
Reference in New Issue
Block a user