fix(ai): support Anthropic eager tool streaming compat

closes #3575
This commit is contained in:
Mario Zechner
2026-04-23 23:12:45 +02:00
parent 6af10c9c7f
commit ffa0f31239
9 changed files with 412 additions and 16 deletions

View File

@@ -3,7 +3,13 @@
import { writeFileSync } from "fs";
import { join, dirname } from "path";
import { fileURLToPath } from "url";
import { Api, KnownProvider, Model, type OpenAICompletionsCompat } from "../src/types.js";
import {
Api,
type AnthropicMessagesCompat,
KnownProvider,
Model,
type OpenAICompletionsCompat,
} from "../src/types.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
@@ -60,6 +66,13 @@ const KIMI_STATIC_HEADERS = {
const AI_GATEWAY_MODELS_URL = "https://ai-gateway.vercel.sh/v1";
const AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh";
const ZAI_TOOL_STREAM_UNSUPPORTED_MODELS = new Set(["glm-4.5", "glm-4.5-air", "glm-4.5-flash", "glm-4.5v"]);
const EAGER_TOOL_INPUT_STREAMING_UNSUPPORTED_ANTHROPIC_MODELS = new Set(["github-copilot:claude-haiku-4.5"]);
function getAnthropicMessagesCompat(provider: string, modelId: string): AnthropicMessagesCompat | undefined {
return EAGER_TOOL_INPUT_STREAMING_UNSUPPORTED_ANTHROPIC_MODELS.has(`${provider}:${modelId}`)
? { supportsEagerToolInputStreaming: false }
: undefined;
}
function getBedrockBaseUrl(modelId: string): string {
return modelId.startsWith("eu.")
@@ -582,6 +595,9 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
? "openai-responses"
: "openai-completions";
const anthropicCompat =
api === "anthropic-messages" ? getAnthropicMessagesCompat("github-copilot", modelId) : undefined;
const copilotModel: Model<any> = {
id: modelId,
name: m.name || modelId,
@@ -599,6 +615,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
contextWindow: m.limit?.context || 128000,
maxTokens: m.limit?.output || 8192,
headers: { ...COPILOT_STATIC_HEADERS },
...(anthropicCompat ? { compat: anthropicCompat } : {}),
// compat only applies to openai-completions
...(api === "openai-completions" ? {
compat: {

View File

@@ -3014,6 +3014,7 @@ export const MODELS = {
provider: "github-copilot",
baseUrl: "https://api.individual.githubcopilot.com",
headers: {"User-Agent":"GitHubCopilotChat/0.35.0","Editor-Version":"vscode/1.107.0","Editor-Plugin-Version":"copilot-chat/0.35.0","Copilot-Integration-Id":"vscode-chat"},
compat: {"supportsEagerToolInputStreaming":false},
reasoning: true,
input: ["text", "image"],
cost: {

View File

@@ -9,6 +9,7 @@ import type {
import { getEnvApiKey } from "../env-api-keys.js";
import { calculateCost } from "../models.js";
import type {
AnthropicMessagesCompat,
Api,
AssistantMessage,
CacheRetention,
@@ -159,6 +160,15 @@ export type AnthropicEffort = "low" | "medium" | "high" | "xhigh" | "max";
export type AnthropicThinkingDisplay = "summarized" | "omitted";
const FINE_GRAINED_TOOL_STREAMING_BETA = "fine-grained-tool-streaming-2025-05-14";
const INTERLEAVED_THINKING_BETA = "interleaved-thinking-2025-05-14";
function getAnthropicCompat(model: Model<"anthropic-messages">): Required<AnthropicMessagesCompat> {
return {
supportsEagerToolInputStreaming: model.compat?.supportsEagerToolInputStreaming ?? true,
};
}
export interface AnthropicOptions extends StreamOptions {
/**
* Enable extended thinking.
@@ -433,6 +443,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
model,
apiKey,
options?.interleavedThinking ?? true,
shouldUseFineGrainedToolStreamingBeta(model, context),
options?.headers,
copilotDynamicHeaders,
);
@@ -728,6 +739,7 @@ function createClient(
model: Model<"anthropic-messages">,
apiKey: string,
interleavedThinking: boolean,
useFineGrainedToolStreamingBeta: boolean,
optionsHeaders?: Record<string, string>,
dynamicHeaders?: Record<string, string>,
): { client: Anthropic; isOAuthToken: boolean } {
@@ -735,11 +747,14 @@ function createClient(
// The beta header is deprecated on Opus 4.6 and redundant on Sonnet 4.6, so skip it.
const needsInterleavedBeta = interleavedThinking && !supportsAdaptiveThinking(model.id);
// Copilot: Bearer auth, selective betas (no fine-grained-tool-streaming)
// Copilot: Bearer auth, selective betas.
if (model.provider === "github-copilot") {
const betaFeatures: string[] = [];
if (useFineGrainedToolStreamingBeta) {
betaFeatures.push(FINE_GRAINED_TOOL_STREAMING_BETA);
}
if (needsInterleavedBeta) {
betaFeatures.push("interleaved-thinking-2025-05-14");
betaFeatures.push(INTERLEAVED_THINKING_BETA);
}
const client = new Anthropic({
@@ -763,8 +778,11 @@ function createClient(
}
const betaFeatures: string[] = [];
if (useFineGrainedToolStreamingBeta) {
betaFeatures.push(FINE_GRAINED_TOOL_STREAMING_BETA);
}
if (needsInterleavedBeta) {
betaFeatures.push("interleaved-thinking-2025-05-14");
betaFeatures.push(INTERLEAVED_THINKING_BETA);
}
// OAuth: Bearer auth, Claude Code identity headers
@@ -856,7 +874,12 @@ function buildParams(
}
if (context.tools) {
params.tools = convertTools(context.tools, isOAuthToken, cacheControl);
params.tools = convertTools(
context.tools,
isOAuthToken,
getAnthropicCompat(model).supportsEagerToolInputStreaming,
cacheControl,
);
}
// Configure thinking mode: adaptive (Opus 4.6+ and Sonnet 4.6),
@@ -1078,9 +1101,14 @@ function convertMessages(
return params;
}
function shouldUseFineGrainedToolStreamingBeta(model: Model<"anthropic-messages">, context: Context): boolean {
return !!context.tools?.length && !getAnthropicCompat(model).supportsEagerToolInputStreaming;
}
function convertTools(
tools: Tool[],
isOAuthToken: boolean,
supportsEagerToolInputStreaming: boolean,
cacheControl?: CacheControlEphemeral,
): Anthropic.Messages.Tool[] {
if (!tools) return [];
@@ -1091,7 +1119,7 @@ function convertTools(
return {
name: isOAuthToken ? toClaudeCodeName(tool.name) : tool.name,
description: tool.description,
eager_input_streaming: true,
...(supportsEagerToolInputStreaming ? { eager_input_streaming: true } : {}),
input_schema: {
type: "object",
properties: schema.properties ?? {},

View File

@@ -304,6 +304,18 @@ export interface OpenAIResponsesCompat {
sendSessionIdHeader?: boolean;
}
/** Compatibility settings for Anthropic Messages-compatible APIs. */
export interface AnthropicMessagesCompat {
/**
* Whether the provider accepts per-tool `eager_input_streaming`.
* When false, the Anthropic provider omits `tools[].eager_input_streaming`
* and sends the legacy `fine-grained-tool-streaming-2025-05-14` beta header
* for tool-enabled requests.
* Default: true.
*/
supportsEagerToolInputStreaming?: boolean;
}
/**
* OpenRouter provider routing preferences.
* Controls which upstream providers OpenRouter routes requests to.
@@ -414,5 +426,7 @@ export interface Model<TApi extends Api> {
? OpenAICompletionsCompat
: TApi extends "openai-responses"
? OpenAIResponsesCompat
: never;
: TApi extends "anthropic-messages"
? AnthropicMessagesCompat
: never;
}

View File

@@ -0,0 +1,122 @@
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import type { AddressInfo } from "node:net";
import { Type } from "typebox";
import { describe, expect, it } from "vitest";
import { streamAnthropic } from "../src/providers/anthropic.js";
import type { Context, Model, Tool } from "../src/types.js";
interface CapturedRequest {
headers: IncomingMessage["headers"];
body: Record<string, unknown>;
}
function createModel(baseUrl: string, compat?: Model<"anthropic-messages">["compat"]): Model<"anthropic-messages"> {
return {
id: "claude-opus-4-7",
name: "Claude Opus 4.7",
api: "anthropic-messages",
provider: "test-anthropic",
baseUrl,
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 200000,
maxTokens: 32000,
compat,
};
}
const tool: Tool = {
name: "lookup",
description: "Look up a value",
parameters: Type.Object({ value: Type.String() }),
};
function createContext(tools: Tool[] = [tool]): Context {
return {
messages: [{ role: "user", content: "Use the tool", timestamp: Date.now() }],
...(tools.length > 0 ? { tools } : {}),
};
}
async function readRequestBody(request: IncomingMessage): Promise<Record<string, unknown>> {
const chunks: Buffer[] = [];
for await (const chunk of request) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record<string, unknown>;
}
function writeEmptySseResponse(response: ServerResponse): void {
response.writeHead(200, { "content-type": "text/event-stream" });
response.end();
}
async function captureAnthropicRequest(
compat: Model<"anthropic-messages">["compat"],
context: Context,
): Promise<CapturedRequest> {
let capturedRequest: CapturedRequest | undefined;
const server = createServer(async (request, response) => {
capturedRequest = {
headers: request.headers,
body: await readRequestBody(request),
};
writeEmptySseResponse(response);
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const address = server.address() as AddressInfo;
try {
const stream = streamAnthropic(createModel(`http://127.0.0.1:${address.port}`, compat), context, {
apiKey: "test-key",
cacheRetention: "none",
});
for await (const event of stream) {
if (event.type === "done" || event.type === "error") break;
}
} finally {
await new Promise<void>((resolve, reject) => {
server.close((error) => (error ? reject(error) : resolve()));
});
}
if (!capturedRequest) {
throw new Error("Anthropic request was not captured");
}
return capturedRequest;
}
function getFirstTool(body: Record<string, unknown>): Record<string, unknown> {
const tools = body.tools;
if (!Array.isArray(tools) || typeof tools[0] !== "object" || tools[0] === null) {
throw new Error("Expected first tool in request body");
}
return tools[0] as Record<string, unknown>;
}
describe("Anthropic eager tool input streaming compatibility", () => {
it("sends per-tool eager_input_streaming by default", async () => {
const request = await captureAnthropicRequest(undefined, createContext());
expect(getFirstTool(request.body).eager_input_streaming).toBe(true);
expect(request.headers["anthropic-beta"]).toBeUndefined();
});
it("uses the legacy fine-grained tool streaming beta when eager tool input streaming is disabled", async () => {
const request = await captureAnthropicRequest({ supportsEagerToolInputStreaming: false }, createContext());
expect(getFirstTool(request.body).eager_input_streaming).toBeUndefined();
expect(request.headers["anthropic-beta"]).toBe("fine-grained-tool-streaming-2025-05-14");
});
it("does not send the legacy fine-grained tool streaming beta when there are no tools", async () => {
const request = await captureAnthropicRequest({ supportsEagerToolInputStreaming: false }, createContext([]));
expect(request.body.tools).toBeUndefined();
expect(request.headers["anthropic-beta"]).toBeUndefined();
});
});

View File

@@ -0,0 +1,143 @@
import { Type } from "typebox";
import { describe, expect, it } from "vitest";
import { getEnvApiKey } from "../src/env-api-keys.js";
import { getModels, getProviders } from "../src/models.js";
import { complete } from "../src/stream.js";
import type { Api, KnownProvider, Model, ProviderStreamOptions, Tool } from "../src/types.js";
import { resolveApiKey } from "./oauth.js";
const githubCopilotToken = await resolveApiKey("github-copilot");
const echoToolSchema = Type.Object({
value: Type.String({ description: "The value to echo" }),
});
const echoTool: Tool<typeof echoToolSchema> = {
name: "echo_value",
description: "Echo a string value",
parameters: echoToolSchema,
};
interface AnthropicEagerE2ECase {
name: string;
provider: KnownProvider;
model: Model<"anthropic-messages">;
apiKey: string | undefined;
}
function getE2EApiKey(provider: KnownProvider): string | undefined {
if (provider === "github-copilot") {
return githubCopilotToken;
}
return getEnvApiKey(provider);
}
function getAnthropicMessagesModels(provider: KnownProvider): Model<"anthropic-messages">[] {
const models = getModels(provider) as Model<Api>[];
return models.filter((model) => model.api === "anthropic-messages") as Model<"anthropic-messages">[];
}
const anthropicMessagesCases: AnthropicEagerE2ECase[] = getProviders().flatMap((provider) =>
getAnthropicMessagesModels(provider).map((model) => ({
name: `${provider}/${model.id}`,
provider,
model,
apiKey: getE2EApiKey(provider),
})),
);
function getProbePriority(model: Model<"anthropic-messages">): number {
const modelId = model.id.toLowerCase();
const cost = model.cost.input + model.cost.output;
let priority = cost;
// Prefer current Claude 4 Haiku routes when present: they are cheap and avoid
// stale Claude 3.x aliases that can remain in catalogs after upstream removal.
if (modelId.includes("haiku") && (modelId.includes("4-5") || modelId.includes("4.5"))) {
priority -= 1000;
} else if (modelId.includes("claude") && (modelId.includes("4-") || modelId.includes("4."))) {
priority -= 500;
}
return priority;
}
function selectOneCasePerProvider(cases: AnthropicEagerE2ECase[]): AnthropicEagerE2ECase[] {
const byProvider = new Map<KnownProvider, AnthropicEagerE2ECase[]>();
for (const testCase of cases) {
const providerCases = byProvider.get(testCase.provider) ?? [];
providerCases.push(testCase);
byProvider.set(testCase.provider, providerCases);
}
return Array.from(byProvider.values()).map(
(providerCases) =>
providerCases.sort(
(a, b) => getProbePriority(a.model) - getProbePriority(b.model) || a.model.id.localeCompare(b.model.id),
)[0],
);
}
const probeCases = selectOneCasePerProvider(anthropicMessagesCases);
function withEagerToolInputStreaming(model: Model<"anthropic-messages">): Model<"anthropic-messages"> {
return {
...model,
compat: {
...model.compat,
supportsEagerToolInputStreaming: true,
},
};
}
async function expectToolEnabledRequestAccepted(
model: Model<"anthropic-messages">,
apiKey: string | undefined,
): Promise<void> {
const options: ProviderStreamOptions = {
apiKey,
maxTokens: 128,
thinkingEnabled: false,
};
const response = await complete(
model,
{
systemPrompt: "You are a concise assistant. Use tools when useful.",
messages: [
{
role: "user",
content: "Call echo_value with value set to eager-input-streaming-compat.",
timestamp: Date.now(),
},
],
tools: [echoTool],
},
options,
);
expect(response.errorMessage, response.errorMessage).toBeFalsy();
expect(response.stopReason, response.errorMessage).not.toBe("error");
}
describe("Anthropic Messages eager tool input streaming E2E", () => {
it("covers every generated anthropic-messages model", () => {
const expectedModels = getProviders().flatMap((provider) =>
getAnthropicMessagesModels(provider).map((model) => `${provider}/${model.id}`),
);
expect(anthropicMessagesCases.map((testCase) => testCase.name).sort()).toEqual(expectedModels.sort());
});
describe("forced eager_input_streaming probe", () => {
for (const testCase of probeCases) {
const model = withEagerToolInputStreaming(testCase.model);
it.skipIf(!testCase.apiKey)(
`${testCase.name} accepts forced eager_input_streaming`,
{ retry: 2 },
async () => {
await expectToolEnabledRequestAccepted(model, testCase.apiKey);
},
);
}
});
});

View File

@@ -11,6 +11,7 @@ Add custom providers and models (Ollama, vLLM, LM Studio, proxies) via `~/.pi/ag
- [Model Configuration](#model-configuration)
- [Overriding Built-in Providers](#overriding-built-in-providers)
- [Per-model Overrides](#per-model-overrides)
- [Anthropic Messages Compatibility](#anthropic-messages-compatibility)
- [OpenAI Compatibility](#openai-compatibility)
## Minimal Example
@@ -195,7 +196,7 @@ If your command is slow, expensive, rate-limited, or should keep using a previou
| `contextWindow` | No | `128000` | Context window size in tokens |
| `maxTokens` | No | `16384` | Maximum output tokens |
| `cost` | No | all zeros | `{"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0}` (per million tokens) |
| `compat` | No | provider `compat` | OpenAI compatibility overrides. Merged with provider-level `compat` when both are set. |
| `compat` | No | provider `compat` | Provider compatibility overrides. Merged with provider-level `compat` when both are set. |
Current behavior:
- `/model` and `--list-models` list entries by model `id`.
@@ -269,6 +270,38 @@ Behavior notes:
- You can combine provider-level `baseUrl`/`headers` with `modelOverrides`.
- If `models` is also defined for a provider, custom models are merged after built-in overrides. A custom model with the same `id` replaces the overridden built-in model entry.
## Anthropic Messages Compatibility
For providers or proxies using `api: "anthropic-messages"`, use `compat.supportsEagerToolInputStreaming` to control Anthropic fine-grained tool streaming compatibility.
By default pi sends per-tool `eager_input_streaming: true`. If a proxy or Anthropic-compatible backend rejects that field, set `supportsEagerToolInputStreaming` to `false`. Pi will omit `tools[].eager_input_streaming` and send the legacy `fine-grained-tool-streaming-2025-05-14` beta header for tool-enabled requests instead.
```json
{
"providers": {
"anthropic-proxy": {
"baseUrl": "https://proxy.example.com",
"api": "anthropic-messages",
"apiKey": "ANTHROPIC_PROXY_KEY",
"compat": {
"supportsEagerToolInputStreaming": false
},
"models": [
{
"id": "claude-opus-4-7",
"reasoning": true,
"input": ["text", "image"]
}
]
}
}
}
```
| Field | Description |
|-------|-------------|
| `supportsEagerToolInputStreaming` | Whether the provider accepts per-tool `eager_input_streaming`. Default: `true`. Set to `false` to omit that field and use the legacy fine-grained tool streaming beta header on tool-enabled requests. |
## OpenAI Compatibility
For providers with partial OpenAI compatibility, use the `compat` field.

View File

@@ -3,6 +3,7 @@
*/
import {
type AnthropicMessagesCompat,
type Api,
type AssistantMessageEventStream,
type Context,
@@ -116,7 +117,15 @@ const OpenAIResponsesCompatSchema = Type.Object({
// Reserved for future use
});
const OpenAICompatSchema = Type.Union([OpenAICompletionsCompatSchema, OpenAIResponsesCompatSchema]);
const AnthropicMessagesCompatSchema = Type.Object({
supportsEagerToolInputStreaming: Type.Optional(Type.Boolean()),
});
const ProviderCompatSchema = Type.Union([
OpenAICompletionsCompatSchema,
OpenAIResponsesCompatSchema,
AnthropicMessagesCompatSchema,
]);
// Schema for custom model definition
// Most fields are optional with sensible defaults for local models (Ollama, LM Studio, etc.)
@@ -138,7 +147,7 @@ const ModelDefinitionSchema = Type.Object({
contextWindow: Type.Optional(Type.Number()),
maxTokens: Type.Optional(Type.Number()),
headers: Type.Optional(Type.Record(Type.String(), Type.String())),
compat: Type.Optional(OpenAICompatSchema),
compat: Type.Optional(ProviderCompatSchema),
});
// Schema for per-model overrides (all fields optional, merged with built-in model)
@@ -157,7 +166,7 @@ const ModelOverrideSchema = Type.Object({
contextWindow: Type.Optional(Type.Number()),
maxTokens: Type.Optional(Type.Number()),
headers: Type.Optional(Type.Record(Type.String(), Type.String())),
compat: Type.Optional(OpenAICompatSchema),
compat: Type.Optional(ProviderCompatSchema),
});
type ModelOverride = Static<typeof ModelOverrideSchema>;
@@ -167,7 +176,7 @@ const ProviderConfigSchema = Type.Object({
apiKey: Type.Optional(Type.String({ minLength: 1 })),
api: Type.Optional(Type.String({ minLength: 1 })),
headers: Type.Optional(Type.Record(Type.String(), Type.String())),
compat: Type.Optional(OpenAICompatSchema),
compat: Type.Optional(ProviderCompatSchema),
authHeader: Type.Optional(Type.Boolean()),
models: Type.Optional(Type.Array(ModelDefinitionSchema)),
modelOverrides: Type.Optional(Type.Record(Type.String(), ModelOverrideSchema)),
@@ -237,9 +246,9 @@ function mergeCompat(
): Model<Api>["compat"] | undefined {
if (!overrideCompat) return baseCompat;
const base = baseCompat as OpenAICompletionsCompat | OpenAIResponsesCompat | undefined;
const override = overrideCompat as OpenAICompletionsCompat | OpenAIResponsesCompat;
const merged = { ...base, ...override } as OpenAICompletionsCompat | OpenAIResponsesCompat;
const base = baseCompat as OpenAICompletionsCompat | OpenAIResponsesCompat | AnthropicMessagesCompat | undefined;
const override = overrideCompat as OpenAICompletionsCompat | OpenAIResponsesCompat | AnthropicMessagesCompat;
const merged = { ...base, ...override } as OpenAICompletionsCompat | OpenAIResponsesCompat | AnthropicMessagesCompat;
const baseCompletions = base as OpenAICompletionsCompat | undefined;
const overrideCompletions = override as OpenAICompletionsCompat;

View File

@@ -1,7 +1,7 @@
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { Api, Context, Model, OpenAICompletionsCompat } from "@mariozechner/pi-ai";
import type { AnthropicMessagesCompat, Api, Context, Model, OpenAICompletionsCompat } from "@mariozechner/pi-ai";
import { getApiProvider } from "@mariozechner/pi-ai";
import { getOAuthProvider } from "@mariozechner/pi-ai/oauth";
import { afterEach, beforeEach, describe, expect, test } from "vitest";
@@ -432,6 +432,35 @@ describe("ModelRegistry", () => {
expect(compat?.cacheControlFormat).toBe("anthropic");
});
test("compat schema accepts Anthropic eager tool input streaming flag", () => {
writeRawModelsJson({
demo: {
baseUrl: "https://example.com",
apiKey: "DEMO_KEY",
api: "anthropic-messages",
compat: {
supportsEagerToolInputStreaming: false,
},
models: [
{
id: "demo-model",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 1000,
maxTokens: 100,
},
],
},
});
const registry = ModelRegistry.create(authStorage, modelsJsonPath);
const compat = registry.find("demo", "demo-model")?.compat as AnthropicMessagesCompat | undefined;
expect(registry.getError()).toBeUndefined();
expect(compat?.supportsEagerToolInputStreaming).toBe(false);
});
test("model-level baseUrl overrides provider-level baseUrl for custom models", () => {
writeRawModelsJson({
"opencode-go": {