feat(ai): lazy-load provider modules for faster startup fixes #2297
This commit is contained in:
12
AGENTS.md
12
AGENTS.md
@@ -136,14 +136,16 @@ Adding a new provider requires changes across multiple files:
|
|||||||
### 2. Provider Implementation (`packages/ai/src/providers/`)
|
### 2. Provider Implementation (`packages/ai/src/providers/`)
|
||||||
Create provider file exporting:
|
Create provider file exporting:
|
||||||
- `stream<Provider>()` function returning `AssistantMessageEventStream`
|
- `stream<Provider>()` function returning `AssistantMessageEventStream`
|
||||||
|
- `streamSimple<Provider>()` for `SimpleStreamOptions` mapping
|
||||||
|
- Provider-specific options interface
|
||||||
- Message/tool conversion functions
|
- Message/tool conversion functions
|
||||||
- Response parsing emitting standardized events (`text`, `tool_call`, `thinking`, `usage`, `stop`)
|
- Response parsing emitting standardized events (`text`, `tool_call`, `thinking`, `usage`, `stop`)
|
||||||
|
|
||||||
### 3. Stream Integration (`packages/ai/src/stream.ts`)
|
### 3. Provider Exports and Lazy Registration
|
||||||
- Import provider's stream function and options type
|
- Add a package subpath export in `packages/ai/package.json` pointing at `./dist/providers/<provider>.js`
|
||||||
- Add credential detection in `getEnvApiKey()`
|
- Add `export type` re-exports in `packages/ai/src/index.ts` for provider option types that should remain available from the root entry
|
||||||
- Add case in `mapOptionsForApi()` for `SimpleStreamOptions` mapping
|
- Register the provider in `packages/ai/src/providers/register-builtins.ts` via lazy loader wrappers, do not statically import provider implementation modules there
|
||||||
- Add provider to `streamFunctions` map
|
- Add credential detection in `packages/ai/src/env-api-keys.ts`
|
||||||
|
|
||||||
### 4. Model Generation (`packages/ai/scripts/generate-models.ts`)
|
### 4. Model Generation (`packages/ai/scripts/generate-models.ts`)
|
||||||
- Add logic to fetch/parse models from provider source
|
- Add logic to fetch/parse models from provider source
|
||||||
|
|||||||
@@ -2,6 +2,10 @@
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- Lazy-load built-in provider modules and root provider wrappers so importing `@mariozechner/pi-ai` no longer eagerly loads provider SDKs, significantly reducing base startup cost without changing dependency installation footprint ([#2297](https://github.com/badlogic/pi-mono/issues/2297))
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
- Added provider-specific `responseId` support on `AssistantMessage` for providers that expose upstream response or message identifiers, including Anthropic, OpenAI, Google, Gemini CLI, and Mistral, and added end-to-end coverage for supported OAuth and API key providers ([#2245](https://github.com/badlogic/pi-mono/issues/2245))
|
- Added provider-specific `responseId` support on `AssistantMessage` for providers that expose upstream response or message identifiers, including Anthropic, OpenAI, Google, Gemini CLI, and Mistral, and added end-to-end coverage for supported OAuth and API key providers ([#2245](https://github.com/badlogic/pi-mono/issues/2245))
|
||||||
|
|||||||
@@ -1161,6 +1161,9 @@ Create a new provider file (for example `amazon-bedrock.ts`) that exports:
|
|||||||
#### 3. API Registry Integration (`src/providers/register-builtins.ts`)
|
#### 3. API Registry Integration (`src/providers/register-builtins.ts`)
|
||||||
|
|
||||||
- Register the API with `registerApiProvider()`
|
- Register the API with `registerApiProvider()`
|
||||||
|
- Add a package subpath export in `package.json` for the provider module (`./dist/providers/<provider>.js`)
|
||||||
|
- Add lazy loader wrappers in `src/providers/register-builtins.ts`, do not statically import provider implementation modules there
|
||||||
|
- Add any root-level `export type` re-exports in `src/index.ts` that should remain available from `@mariozechner/pi-ai`
|
||||||
- Add credential detection in `env-api-keys.ts` for the new provider
|
- Add credential detection in `env-api-keys.ts` for the new provider
|
||||||
- Ensure `streamSimple` handles auth lookup via `getEnvApiKey()` or provider-specific auth
|
- Ensure `streamSimple` handles auth lookup via `getEnvApiKey()` or provider-specific auth
|
||||||
|
|
||||||
|
|||||||
@@ -10,13 +10,49 @@
|
|||||||
"types": "./dist/index.d.ts",
|
"types": "./dist/index.d.ts",
|
||||||
"import": "./dist/index.js"
|
"import": "./dist/index.js"
|
||||||
},
|
},
|
||||||
|
"./anthropic": {
|
||||||
|
"types": "./dist/providers/anthropic.d.ts",
|
||||||
|
"import": "./dist/providers/anthropic.js"
|
||||||
|
},
|
||||||
|
"./azure-openai-responses": {
|
||||||
|
"types": "./dist/providers/azure-openai-responses.d.ts",
|
||||||
|
"import": "./dist/providers/azure-openai-responses.js"
|
||||||
|
},
|
||||||
|
"./google": {
|
||||||
|
"types": "./dist/providers/google.d.ts",
|
||||||
|
"import": "./dist/providers/google.js"
|
||||||
|
},
|
||||||
|
"./google-gemini-cli": {
|
||||||
|
"types": "./dist/providers/google-gemini-cli.d.ts",
|
||||||
|
"import": "./dist/providers/google-gemini-cli.js"
|
||||||
|
},
|
||||||
|
"./google-vertex": {
|
||||||
|
"types": "./dist/providers/google-vertex.d.ts",
|
||||||
|
"import": "./dist/providers/google-vertex.js"
|
||||||
|
},
|
||||||
|
"./mistral": {
|
||||||
|
"types": "./dist/providers/mistral.d.ts",
|
||||||
|
"import": "./dist/providers/mistral.js"
|
||||||
|
},
|
||||||
|
"./openai-codex-responses": {
|
||||||
|
"types": "./dist/providers/openai-codex-responses.d.ts",
|
||||||
|
"import": "./dist/providers/openai-codex-responses.js"
|
||||||
|
},
|
||||||
|
"./openai-completions": {
|
||||||
|
"types": "./dist/providers/openai-completions.d.ts",
|
||||||
|
"import": "./dist/providers/openai-completions.js"
|
||||||
|
},
|
||||||
|
"./openai-responses": {
|
||||||
|
"types": "./dist/providers/openai-responses.d.ts",
|
||||||
|
"import": "./dist/providers/openai-responses.js"
|
||||||
|
},
|
||||||
"./oauth": {
|
"./oauth": {
|
||||||
"types": "./dist/oauth.d.ts",
|
"types": "./dist/oauth.d.ts",
|
||||||
"import": "./dist/oauth.js"
|
"import": "./dist/oauth.js"
|
||||||
},
|
},
|
||||||
"./bedrock-provider": {
|
"./bedrock-provider": {
|
||||||
"types": "./bedrock-provider.d.ts",
|
"types": "./dist/bedrock-provider.d.ts",
|
||||||
"import": "./bedrock-provider.js"
|
"import": "./dist/bedrock-provider.js"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"bin": {
|
"bin": {
|
||||||
@@ -24,8 +60,6 @@
|
|||||||
},
|
},
|
||||||
"files": [
|
"files": [
|
||||||
"dist",
|
"dist",
|
||||||
"bedrock-provider.js",
|
|
||||||
"bedrock-provider.d.ts",
|
|
||||||
"README.md"
|
"README.md"
|
||||||
],
|
],
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -4,14 +4,15 @@ export { Type } from "@sinclair/typebox";
|
|||||||
export * from "./api-registry.js";
|
export * from "./api-registry.js";
|
||||||
export * from "./env-api-keys.js";
|
export * from "./env-api-keys.js";
|
||||||
export * from "./models.js";
|
export * from "./models.js";
|
||||||
export * from "./providers/anthropic.js";
|
export type { AnthropicOptions } from "./providers/anthropic.js";
|
||||||
export * from "./providers/azure-openai-responses.js";
|
export type { AzureOpenAIResponsesOptions } from "./providers/azure-openai-responses.js";
|
||||||
export * from "./providers/google.js";
|
export type { GoogleOptions } from "./providers/google.js";
|
||||||
export * from "./providers/google-gemini-cli.js";
|
export type { GoogleGeminiCliOptions, GoogleThinkingLevel } from "./providers/google-gemini-cli.js";
|
||||||
export * from "./providers/google-vertex.js";
|
export type { GoogleVertexOptions } from "./providers/google-vertex.js";
|
||||||
export * from "./providers/mistral.js";
|
export type { MistralOptions } from "./providers/mistral.js";
|
||||||
export * from "./providers/openai-completions.js";
|
export type { OpenAICodexResponsesOptions } from "./providers/openai-codex-responses.js";
|
||||||
export * from "./providers/openai-responses.js";
|
export type { OpenAICompletionsOptions } from "./providers/openai-completions.js";
|
||||||
|
export type { OpenAIResponsesOptions } from "./providers/openai-responses.js";
|
||||||
export * from "./providers/register-builtins.js";
|
export * from "./providers/register-builtins.js";
|
||||||
export * from "./stream.js";
|
export * from "./stream.js";
|
||||||
export * from "./types.js";
|
export * from "./types.js";
|
||||||
|
|||||||
@@ -1,16 +1,98 @@
|
|||||||
import { clearApiProviders, registerApiProvider } from "../api-registry.js";
|
import { clearApiProviders, registerApiProvider } from "../api-registry.js";
|
||||||
import type { AssistantMessage, AssistantMessageEvent, Context, Model, SimpleStreamOptions } from "../types.js";
|
import type {
|
||||||
|
Api,
|
||||||
|
AssistantMessage,
|
||||||
|
AssistantMessageEvent,
|
||||||
|
Context,
|
||||||
|
Model,
|
||||||
|
SimpleStreamOptions,
|
||||||
|
StreamFunction,
|
||||||
|
StreamOptions,
|
||||||
|
} from "../types.js";
|
||||||
import { AssistantMessageEventStream } from "../utils/event-stream.js";
|
import { AssistantMessageEventStream } from "../utils/event-stream.js";
|
||||||
import type { BedrockOptions } from "./amazon-bedrock.js";
|
import type { BedrockOptions } from "./amazon-bedrock.js";
|
||||||
import { streamAnthropic, streamSimpleAnthropic } from "./anthropic.js";
|
import type { AnthropicOptions } from "./anthropic.js";
|
||||||
import { streamAzureOpenAIResponses, streamSimpleAzureOpenAIResponses } from "./azure-openai-responses.js";
|
import type { AzureOpenAIResponsesOptions } from "./azure-openai-responses.js";
|
||||||
import { streamGoogle, streamSimpleGoogle } from "./google.js";
|
import type { GoogleOptions } from "./google.js";
|
||||||
import { streamGoogleGeminiCli, streamSimpleGoogleGeminiCli } from "./google-gemini-cli.js";
|
import type { GoogleGeminiCliOptions } from "./google-gemini-cli.js";
|
||||||
import { streamGoogleVertex, streamSimpleGoogleVertex } from "./google-vertex.js";
|
import type { GoogleVertexOptions } from "./google-vertex.js";
|
||||||
import { streamMistral, streamSimpleMistral } from "./mistral.js";
|
import type { MistralOptions } from "./mistral.js";
|
||||||
import { streamOpenAICodexResponses, streamSimpleOpenAICodexResponses } from "./openai-codex-responses.js";
|
import type { OpenAICodexResponsesOptions } from "./openai-codex-responses.js";
|
||||||
import { streamOpenAICompletions, streamSimpleOpenAICompletions } from "./openai-completions.js";
|
import type { OpenAICompletionsOptions } from "./openai-completions.js";
|
||||||
import { streamOpenAIResponses, streamSimpleOpenAIResponses } from "./openai-responses.js";
|
import type { OpenAIResponsesOptions } from "./openai-responses.js";
|
||||||
|
|
||||||
|
interface LazyProviderModule<
|
||||||
|
TApi extends Api,
|
||||||
|
TOptions extends StreamOptions,
|
||||||
|
TSimpleOptions extends SimpleStreamOptions,
|
||||||
|
> {
|
||||||
|
stream: (model: Model<TApi>, context: Context, options?: TOptions) => AsyncIterable<AssistantMessageEvent>;
|
||||||
|
streamSimple: (
|
||||||
|
model: Model<TApi>,
|
||||||
|
context: Context,
|
||||||
|
options?: TSimpleOptions,
|
||||||
|
) => AsyncIterable<AssistantMessageEvent>;
|
||||||
|
}
|
||||||
|
|
||||||
|
type DynamicImport = (specifier: string) => Promise<unknown>;
|
||||||
|
|
||||||
|
const dynamicImport: DynamicImport = (specifier) => import(specifier);
|
||||||
|
|
||||||
|
const ANTHROPIC_PROVIDER_SPECIFIER = "./anthropic.js";
|
||||||
|
const AZURE_OPENAI_RESPONSES_PROVIDER_SPECIFIER = "./azure-openai-responses.js";
|
||||||
|
const GOOGLE_PROVIDER_SPECIFIER = "./google.js";
|
||||||
|
const GOOGLE_GEMINI_CLI_PROVIDER_SPECIFIER = "./google-gemini-cli.js";
|
||||||
|
const GOOGLE_VERTEX_PROVIDER_SPECIFIER = "./google-vertex.js";
|
||||||
|
const MISTRAL_PROVIDER_SPECIFIER = "./mistral.js";
|
||||||
|
const OPENAI_CODEX_RESPONSES_PROVIDER_SPECIFIER = "./openai-codex-responses.js";
|
||||||
|
const OPENAI_COMPLETIONS_PROVIDER_SPECIFIER = "./openai-completions.js";
|
||||||
|
const OPENAI_RESPONSES_PROVIDER_SPECIFIER = "./openai-responses.js";
|
||||||
|
const BEDROCK_PROVIDER_SPECIFIER = "./amazon-" + "bedrock.js";
|
||||||
|
|
||||||
|
interface AnthropicProviderModule {
|
||||||
|
streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOptions>;
|
||||||
|
streamSimpleAnthropic: StreamFunction<"anthropic-messages", SimpleStreamOptions>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface AzureOpenAIResponsesProviderModule {
|
||||||
|
streamAzureOpenAIResponses: StreamFunction<"azure-openai-responses", AzureOpenAIResponsesOptions>;
|
||||||
|
streamSimpleAzureOpenAIResponses: StreamFunction<"azure-openai-responses", SimpleStreamOptions>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface GoogleProviderModule {
|
||||||
|
streamGoogle: StreamFunction<"google-generative-ai", GoogleOptions>;
|
||||||
|
streamSimpleGoogle: StreamFunction<"google-generative-ai", SimpleStreamOptions>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface GoogleGeminiCliProviderModule {
|
||||||
|
streamGoogleGeminiCli: StreamFunction<"google-gemini-cli", GoogleGeminiCliOptions>;
|
||||||
|
streamSimpleGoogleGeminiCli: StreamFunction<"google-gemini-cli", SimpleStreamOptions>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface GoogleVertexProviderModule {
|
||||||
|
streamGoogleVertex: StreamFunction<"google-vertex", GoogleVertexOptions>;
|
||||||
|
streamSimpleGoogleVertex: StreamFunction<"google-vertex", SimpleStreamOptions>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface MistralProviderModule {
|
||||||
|
streamMistral: StreamFunction<"mistral-conversations", MistralOptions>;
|
||||||
|
streamSimpleMistral: StreamFunction<"mistral-conversations", SimpleStreamOptions>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface OpenAICodexResponsesProviderModule {
|
||||||
|
streamOpenAICodexResponses: StreamFunction<"openai-codex-responses", OpenAICodexResponsesOptions>;
|
||||||
|
streamSimpleOpenAICodexResponses: StreamFunction<"openai-codex-responses", SimpleStreamOptions>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface OpenAICompletionsProviderModule {
|
||||||
|
streamOpenAICompletions: StreamFunction<"openai-completions", OpenAICompletionsOptions>;
|
||||||
|
streamSimpleOpenAICompletions: StreamFunction<"openai-completions", SimpleStreamOptions>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface OpenAIResponsesProviderModule {
|
||||||
|
streamOpenAIResponses: StreamFunction<"openai-responses", OpenAIResponsesOptions>;
|
||||||
|
streamSimpleOpenAIResponses: StreamFunction<"openai-responses", SimpleStreamOptions>;
|
||||||
|
}
|
||||||
|
|
||||||
interface BedrockProviderModule {
|
interface BedrockProviderModule {
|
||||||
streamBedrock: (
|
streamBedrock: (
|
||||||
@@ -25,23 +107,45 @@ interface BedrockProviderModule {
|
|||||||
) => AsyncIterable<AssistantMessageEvent>;
|
) => AsyncIterable<AssistantMessageEvent>;
|
||||||
}
|
}
|
||||||
|
|
||||||
type DynamicImport = (specifier: string) => Promise<unknown>;
|
let anthropicProviderModulePromise:
|
||||||
|
| Promise<LazyProviderModule<"anthropic-messages", AnthropicOptions, SimpleStreamOptions>>
|
||||||
const dynamicImport: DynamicImport = (specifier) => import(specifier);
|
| undefined;
|
||||||
const BEDROCK_PROVIDER_SPECIFIER = "./amazon-" + "bedrock.js";
|
let azureOpenAIResponsesProviderModulePromise:
|
||||||
|
| Promise<LazyProviderModule<"azure-openai-responses", AzureOpenAIResponsesOptions, SimpleStreamOptions>>
|
||||||
let bedrockProviderModuleOverride: BedrockProviderModule | undefined;
|
| undefined;
|
||||||
|
let googleProviderModulePromise:
|
||||||
|
| Promise<LazyProviderModule<"google-generative-ai", GoogleOptions, SimpleStreamOptions>>
|
||||||
|
| undefined;
|
||||||
|
let googleGeminiCliProviderModulePromise:
|
||||||
|
| Promise<LazyProviderModule<"google-gemini-cli", GoogleGeminiCliOptions, SimpleStreamOptions>>
|
||||||
|
| undefined;
|
||||||
|
let googleVertexProviderModulePromise:
|
||||||
|
| Promise<LazyProviderModule<"google-vertex", GoogleVertexOptions, SimpleStreamOptions>>
|
||||||
|
| undefined;
|
||||||
|
let mistralProviderModulePromise:
|
||||||
|
| Promise<LazyProviderModule<"mistral-conversations", MistralOptions, SimpleStreamOptions>>
|
||||||
|
| undefined;
|
||||||
|
let openAICodexResponsesProviderModulePromise:
|
||||||
|
| Promise<LazyProviderModule<"openai-codex-responses", OpenAICodexResponsesOptions, SimpleStreamOptions>>
|
||||||
|
| undefined;
|
||||||
|
let openAICompletionsProviderModulePromise:
|
||||||
|
| Promise<LazyProviderModule<"openai-completions", OpenAICompletionsOptions, SimpleStreamOptions>>
|
||||||
|
| undefined;
|
||||||
|
let openAIResponsesProviderModulePromise:
|
||||||
|
| Promise<LazyProviderModule<"openai-responses", OpenAIResponsesOptions, SimpleStreamOptions>>
|
||||||
|
| undefined;
|
||||||
|
let bedrockProviderModuleOverride:
|
||||||
|
| LazyProviderModule<"bedrock-converse-stream", BedrockOptions, SimpleStreamOptions>
|
||||||
|
| undefined;
|
||||||
|
let bedrockProviderModulePromise:
|
||||||
|
| Promise<LazyProviderModule<"bedrock-converse-stream", BedrockOptions, SimpleStreamOptions>>
|
||||||
|
| undefined;
|
||||||
|
|
||||||
export function setBedrockProviderModule(module: BedrockProviderModule): void {
|
export function setBedrockProviderModule(module: BedrockProviderModule): void {
|
||||||
bedrockProviderModuleOverride = module;
|
bedrockProviderModuleOverride = {
|
||||||
}
|
stream: module.streamBedrock,
|
||||||
|
streamSimple: module.streamSimpleBedrock,
|
||||||
async function loadBedrockProviderModule(): Promise<BedrockProviderModule> {
|
};
|
||||||
if (bedrockProviderModuleOverride) {
|
|
||||||
return bedrockProviderModuleOverride;
|
|
||||||
}
|
|
||||||
const module = await dynamicImport(BEDROCK_PROVIDER_SPECIFIER);
|
|
||||||
return module as BedrockProviderModule;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function forwardStream(target: AssistantMessageEventStream, source: AsyncIterable<AssistantMessageEvent>): void {
|
function forwardStream(target: AssistantMessageEventStream, source: AsyncIterable<AssistantMessageEvent>): void {
|
||||||
@@ -53,11 +157,11 @@ function forwardStream(target: AssistantMessageEventStream, source: AsyncIterabl
|
|||||||
})();
|
})();
|
||||||
}
|
}
|
||||||
|
|
||||||
function createLazyLoadErrorMessage(model: Model<"bedrock-converse-stream">, error: unknown): AssistantMessage {
|
function createLazyLoadErrorMessage<TApi extends Api>(model: Model<TApi>, error: unknown): AssistantMessage {
|
||||||
return {
|
return {
|
||||||
role: "assistant",
|
role: "assistant",
|
||||||
content: [],
|
content: [],
|
||||||
api: "bedrock-converse-stream",
|
api: model.api,
|
||||||
provider: model.provider,
|
provider: model.provider,
|
||||||
model: model.id,
|
model: model.id,
|
||||||
usage: {
|
usage: {
|
||||||
@@ -74,48 +178,208 @@ function createLazyLoadErrorMessage(model: Model<"bedrock-converse-stream">, err
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function streamBedrockLazy(
|
function createLazyStream<TApi extends Api, TOptions extends StreamOptions, TSimpleOptions extends SimpleStreamOptions>(
|
||||||
model: Model<"bedrock-converse-stream">,
|
loadModule: () => Promise<LazyProviderModule<TApi, TOptions, TSimpleOptions>>,
|
||||||
context: Context,
|
): StreamFunction<TApi, TOptions> {
|
||||||
options?: BedrockOptions,
|
return (model, context, options) => {
|
||||||
): AssistantMessageEventStream {
|
const outer = new AssistantMessageEventStream();
|
||||||
const outer = new AssistantMessageEventStream();
|
|
||||||
|
|
||||||
loadBedrockProviderModule()
|
loadModule()
|
||||||
.then((module) => {
|
.then((module) => {
|
||||||
const inner = module.streamBedrock(model, context, options);
|
const inner = module.stream(model, context, options);
|
||||||
forwardStream(outer, inner);
|
forwardStream(outer, inner);
|
||||||
})
|
})
|
||||||
.catch((error) => {
|
.catch((error) => {
|
||||||
const message = createLazyLoadErrorMessage(model, error);
|
const message = createLazyLoadErrorMessage(model, error);
|
||||||
outer.push({ type: "error", reason: "error", error: message });
|
outer.push({ type: "error", reason: "error", error: message });
|
||||||
outer.end(message);
|
outer.end(message);
|
||||||
});
|
});
|
||||||
|
|
||||||
return outer;
|
return outer;
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function streamSimpleBedrockLazy(
|
function createLazySimpleStream<
|
||||||
model: Model<"bedrock-converse-stream">,
|
TApi extends Api,
|
||||||
context: Context,
|
TOptions extends StreamOptions,
|
||||||
options?: SimpleStreamOptions,
|
TSimpleOptions extends SimpleStreamOptions,
|
||||||
): AssistantMessageEventStream {
|
>(loadModule: () => Promise<LazyProviderModule<TApi, TOptions, TSimpleOptions>>): StreamFunction<TApi, TSimpleOptions> {
|
||||||
const outer = new AssistantMessageEventStream();
|
return (model, context, options) => {
|
||||||
|
const outer = new AssistantMessageEventStream();
|
||||||
|
|
||||||
loadBedrockProviderModule()
|
loadModule()
|
||||||
.then((module) => {
|
.then((module) => {
|
||||||
const inner = module.streamSimpleBedrock(model, context, options);
|
const inner = module.streamSimple(model, context, options);
|
||||||
forwardStream(outer, inner);
|
forwardStream(outer, inner);
|
||||||
})
|
})
|
||||||
.catch((error) => {
|
.catch((error) => {
|
||||||
const message = createLazyLoadErrorMessage(model, error);
|
const message = createLazyLoadErrorMessage(model, error);
|
||||||
outer.push({ type: "error", reason: "error", error: message });
|
outer.push({ type: "error", reason: "error", error: message });
|
||||||
outer.end(message);
|
outer.end(message);
|
||||||
});
|
});
|
||||||
|
|
||||||
return outer;
|
return outer;
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function loadAnthropicProviderModule(): Promise<
|
||||||
|
LazyProviderModule<"anthropic-messages", AnthropicOptions, SimpleStreamOptions>
|
||||||
|
> {
|
||||||
|
anthropicProviderModulePromise ||= dynamicImport(ANTHROPIC_PROVIDER_SPECIFIER).then((module) => {
|
||||||
|
const provider = module as AnthropicProviderModule;
|
||||||
|
return {
|
||||||
|
stream: provider.streamAnthropic,
|
||||||
|
streamSimple: provider.streamSimpleAnthropic,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
return anthropicProviderModulePromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadAzureOpenAIResponsesProviderModule(): Promise<
|
||||||
|
LazyProviderModule<"azure-openai-responses", AzureOpenAIResponsesOptions, SimpleStreamOptions>
|
||||||
|
> {
|
||||||
|
azureOpenAIResponsesProviderModulePromise ||= dynamicImport(AZURE_OPENAI_RESPONSES_PROVIDER_SPECIFIER).then(
|
||||||
|
(module) => {
|
||||||
|
const provider = module as AzureOpenAIResponsesProviderModule;
|
||||||
|
return {
|
||||||
|
stream: provider.streamAzureOpenAIResponses,
|
||||||
|
streamSimple: provider.streamSimpleAzureOpenAIResponses,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
return azureOpenAIResponsesProviderModulePromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadGoogleProviderModule(): Promise<
|
||||||
|
LazyProviderModule<"google-generative-ai", GoogleOptions, SimpleStreamOptions>
|
||||||
|
> {
|
||||||
|
googleProviderModulePromise ||= dynamicImport(GOOGLE_PROVIDER_SPECIFIER).then((module) => {
|
||||||
|
const provider = module as GoogleProviderModule;
|
||||||
|
return {
|
||||||
|
stream: provider.streamGoogle,
|
||||||
|
streamSimple: provider.streamSimpleGoogle,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
return googleProviderModulePromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadGoogleGeminiCliProviderModule(): Promise<
|
||||||
|
LazyProviderModule<"google-gemini-cli", GoogleGeminiCliOptions, SimpleStreamOptions>
|
||||||
|
> {
|
||||||
|
googleGeminiCliProviderModulePromise ||= dynamicImport(GOOGLE_GEMINI_CLI_PROVIDER_SPECIFIER).then((module) => {
|
||||||
|
const provider = module as GoogleGeminiCliProviderModule;
|
||||||
|
return {
|
||||||
|
stream: provider.streamGoogleGeminiCli,
|
||||||
|
streamSimple: provider.streamSimpleGoogleGeminiCli,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
return googleGeminiCliProviderModulePromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadGoogleVertexProviderModule(): Promise<
|
||||||
|
LazyProviderModule<"google-vertex", GoogleVertexOptions, SimpleStreamOptions>
|
||||||
|
> {
|
||||||
|
googleVertexProviderModulePromise ||= dynamicImport(GOOGLE_VERTEX_PROVIDER_SPECIFIER).then((module) => {
|
||||||
|
const provider = module as GoogleVertexProviderModule;
|
||||||
|
return {
|
||||||
|
stream: provider.streamGoogleVertex,
|
||||||
|
streamSimple: provider.streamSimpleGoogleVertex,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
return googleVertexProviderModulePromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadMistralProviderModule(): Promise<
|
||||||
|
LazyProviderModule<"mistral-conversations", MistralOptions, SimpleStreamOptions>
|
||||||
|
> {
|
||||||
|
mistralProviderModulePromise ||= dynamicImport(MISTRAL_PROVIDER_SPECIFIER).then((module) => {
|
||||||
|
const provider = module as MistralProviderModule;
|
||||||
|
return {
|
||||||
|
stream: provider.streamMistral,
|
||||||
|
streamSimple: provider.streamSimpleMistral,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
return mistralProviderModulePromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadOpenAICodexResponsesProviderModule(): Promise<
|
||||||
|
LazyProviderModule<"openai-codex-responses", OpenAICodexResponsesOptions, SimpleStreamOptions>
|
||||||
|
> {
|
||||||
|
openAICodexResponsesProviderModulePromise ||= dynamicImport(OPENAI_CODEX_RESPONSES_PROVIDER_SPECIFIER).then(
|
||||||
|
(module) => {
|
||||||
|
const provider = module as OpenAICodexResponsesProviderModule;
|
||||||
|
return {
|
||||||
|
stream: provider.streamOpenAICodexResponses,
|
||||||
|
streamSimple: provider.streamSimpleOpenAICodexResponses,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
return openAICodexResponsesProviderModulePromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadOpenAICompletionsProviderModule(): Promise<
|
||||||
|
LazyProviderModule<"openai-completions", OpenAICompletionsOptions, SimpleStreamOptions>
|
||||||
|
> {
|
||||||
|
openAICompletionsProviderModulePromise ||= dynamicImport(OPENAI_COMPLETIONS_PROVIDER_SPECIFIER).then((module) => {
|
||||||
|
const provider = module as OpenAICompletionsProviderModule;
|
||||||
|
return {
|
||||||
|
stream: provider.streamOpenAICompletions,
|
||||||
|
streamSimple: provider.streamSimpleOpenAICompletions,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
return openAICompletionsProviderModulePromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadOpenAIResponsesProviderModule(): Promise<
|
||||||
|
LazyProviderModule<"openai-responses", OpenAIResponsesOptions, SimpleStreamOptions>
|
||||||
|
> {
|
||||||
|
openAIResponsesProviderModulePromise ||= dynamicImport(OPENAI_RESPONSES_PROVIDER_SPECIFIER).then((module) => {
|
||||||
|
const provider = module as OpenAIResponsesProviderModule;
|
||||||
|
return {
|
||||||
|
stream: provider.streamOpenAIResponses,
|
||||||
|
streamSimple: provider.streamSimpleOpenAIResponses,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
return openAIResponsesProviderModulePromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadBedrockProviderModule(): Promise<
|
||||||
|
LazyProviderModule<"bedrock-converse-stream", BedrockOptions, SimpleStreamOptions>
|
||||||
|
> {
|
||||||
|
if (bedrockProviderModuleOverride) {
|
||||||
|
return Promise.resolve(bedrockProviderModuleOverride);
|
||||||
|
}
|
||||||
|
bedrockProviderModulePromise ||= dynamicImport(BEDROCK_PROVIDER_SPECIFIER).then((module) => {
|
||||||
|
const provider = module as BedrockProviderModule;
|
||||||
|
return {
|
||||||
|
stream: provider.streamBedrock,
|
||||||
|
streamSimple: provider.streamSimpleBedrock,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
return bedrockProviderModulePromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const streamAnthropic = createLazyStream(loadAnthropicProviderModule);
|
||||||
|
export const streamSimpleAnthropic = createLazySimpleStream(loadAnthropicProviderModule);
|
||||||
|
export const streamAzureOpenAIResponses = createLazyStream(loadAzureOpenAIResponsesProviderModule);
|
||||||
|
export const streamSimpleAzureOpenAIResponses = createLazySimpleStream(loadAzureOpenAIResponsesProviderModule);
|
||||||
|
export const streamGoogle = createLazyStream(loadGoogleProviderModule);
|
||||||
|
export const streamSimpleGoogle = createLazySimpleStream(loadGoogleProviderModule);
|
||||||
|
export const streamGoogleGeminiCli = createLazyStream(loadGoogleGeminiCliProviderModule);
|
||||||
|
export const streamSimpleGoogleGeminiCli = createLazySimpleStream(loadGoogleGeminiCliProviderModule);
|
||||||
|
export const streamGoogleVertex = createLazyStream(loadGoogleVertexProviderModule);
|
||||||
|
export const streamSimpleGoogleVertex = createLazySimpleStream(loadGoogleVertexProviderModule);
|
||||||
|
export const streamMistral = createLazyStream(loadMistralProviderModule);
|
||||||
|
export const streamSimpleMistral = createLazySimpleStream(loadMistralProviderModule);
|
||||||
|
export const streamOpenAICodexResponses = createLazyStream(loadOpenAICodexResponsesProviderModule);
|
||||||
|
export const streamSimpleOpenAICodexResponses = createLazySimpleStream(loadOpenAICodexResponsesProviderModule);
|
||||||
|
export const streamOpenAICompletions = createLazyStream(loadOpenAICompletionsProviderModule);
|
||||||
|
export const streamSimpleOpenAICompletions = createLazySimpleStream(loadOpenAICompletionsProviderModule);
|
||||||
|
export const streamOpenAIResponses = createLazyStream(loadOpenAIResponsesProviderModule);
|
||||||
|
export const streamSimpleOpenAIResponses = createLazySimpleStream(loadOpenAIResponsesProviderModule);
|
||||||
|
const streamBedrockLazy = createLazyStream(loadBedrockProviderModule);
|
||||||
|
const streamSimpleBedrockLazy = createLazySimpleStream(loadBedrockProviderModule);
|
||||||
|
|
||||||
export function registerBuiltInApiProviders(): void {
|
export function registerBuiltInApiProviders(): void {
|
||||||
registerApiProvider({
|
registerApiProvider({
|
||||||
api: "anthropic-messages",
|
api: "anthropic-messages",
|
||||||
|
|||||||
102
packages/ai/test/lazy-module-load.test.ts
Normal file
102
packages/ai/test/lazy-module-load.test.ts
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
import { spawnSync } from "node:child_process";
|
||||||
|
import { createRequire } from "node:module";
|
||||||
|
import { dirname, resolve } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
|
const require = createRequire(import.meta.url);
|
||||||
|
const tsxLoader = require.resolve("tsx/esm");
|
||||||
|
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||||
|
const aiEntryUrl = new URL("../src/index.ts", import.meta.url).href;
|
||||||
|
|
||||||
|
const SDK_SPECIFIERS = [
|
||||||
|
"@anthropic-ai/sdk",
|
||||||
|
"openai",
|
||||||
|
"@google/genai",
|
||||||
|
"@mistralai/mistralai",
|
||||||
|
"@aws-sdk/client-bedrock-runtime",
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
type ProbeResult = {
|
||||||
|
loadedSpecifiers: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
function runProbe(action: string): ProbeResult {
|
||||||
|
const script = `
|
||||||
|
import { registerHooks } from "node:module";
|
||||||
|
|
||||||
|
const targets = new Set(${JSON.stringify(SDK_SPECIFIERS)});
|
||||||
|
const loaded = [];
|
||||||
|
|
||||||
|
registerHooks({
|
||||||
|
resolve(specifier, context, nextResolve) {
|
||||||
|
if (targets.has(specifier)) {
|
||||||
|
loaded.push(specifier);
|
||||||
|
}
|
||||||
|
return nextResolve(specifier, context);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const mod = await import(${JSON.stringify(aiEntryUrl)});
|
||||||
|
${action}
|
||||||
|
console.log(JSON.stringify({ loadedSpecifiers: [...new Set(loaded)] }));
|
||||||
|
`;
|
||||||
|
|
||||||
|
const result = spawnSync(process.execPath, ["--import", tsxLoader, "--input-type=module", "--eval", script], {
|
||||||
|
cwd: packageRoot,
|
||||||
|
encoding: "utf8",
|
||||||
|
});
|
||||||
|
|
||||||
|
if (result.status !== 0) {
|
||||||
|
throw new Error(`Probe failed (exit ${result.status})\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const stdoutLines = result.stdout
|
||||||
|
.split(/\r?\n/)
|
||||||
|
.map((line) => line.trim())
|
||||||
|
.filter((line) => line.length > 0);
|
||||||
|
const lastLine = stdoutLines.at(-1);
|
||||||
|
if (!lastLine) {
|
||||||
|
throw new Error(`Probe produced no output\nSTDERR:\n${result.stderr}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return JSON.parse(lastLine) as ProbeResult;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("lazy provider module loading", () => {
|
||||||
|
it("does not load provider SDKs when importing the root barrel", () => {
|
||||||
|
const result = runProbe("");
|
||||||
|
expect(result.loadedSpecifiers).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("loads only the Anthropic SDK when calling the root lazy wrapper", () => {
|
||||||
|
const result = runProbe(`
|
||||||
|
const model = {
|
||||||
|
id: "claude-sonnet-4-20250514",
|
||||||
|
name: "Claude Sonnet 4",
|
||||||
|
api: "anthropic-messages",
|
||||||
|
provider: "anthropic",
|
||||||
|
baseUrl: "https://api.anthropic.com",
|
||||||
|
reasoning: true,
|
||||||
|
input: ["text"],
|
||||||
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||||
|
contextWindow: 200000,
|
||||||
|
maxTokens: 8192,
|
||||||
|
};
|
||||||
|
const context = { messages: [{ role: "user", content: "hi" }] };
|
||||||
|
await mod.streamSimpleAnthropic(model, context).result();
|
||||||
|
`);
|
||||||
|
|
||||||
|
expect(result.loadedSpecifiers).toEqual(["@anthropic-ai/sdk"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("loads only the Anthropic SDK when dispatching through streamSimple", () => {
|
||||||
|
const result = runProbe(`
|
||||||
|
const model = mod.getModel("anthropic", "claude-sonnet-4-20250514");
|
||||||
|
const context = { messages: [{ role: "user", content: "hi" }] };
|
||||||
|
await mod.streamSimple(model, context).result();
|
||||||
|
`);
|
||||||
|
|
||||||
|
expect(result.loadedSpecifiers).toEqual(["@anthropic-ai/sdk"]);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user