From 3054fd7a3be12870cb726e12e0b10d8270faf545 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 20 Apr 2026 17:12:05 +0200 Subject: [PATCH] fix(ai,coding-agent): support anthropic-style cache control for openai compatibles closes #3392 --- packages/ai/CHANGELOG.md | 1 + packages/ai/README.md | 3 +- packages/ai/scripts/generate-models.ts | 8 +- packages/ai/src/models.generated.ts | 50 ++---- .../ai/src/providers/openai-completions.ts | 165 +++++++++++++++--- packages/ai/src/types.ts | 2 + ...i-completions-cache-control-format.test.ts | 152 ++++++++++++++++ ...nai-completions-tool-result-images.test.ts | 1 + packages/coding-agent/CHANGELOG.md | 1 + packages/coding-agent/docs/custom-provider.md | 6 +- packages/coding-agent/docs/models.md | 3 + .../coding-agent/src/core/model-registry.ts | 1 + .../coding-agent/test/model-registry.test.ts | 4 +- 13 files changed, 326 insertions(+), 71 deletions(-) create mode 100644 packages/ai/test/openai-completions-cache-control-format.test.ts diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 7737311b..ff319426 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -8,6 +8,7 @@ - Fixed OpenRouter Meta tests by switching `meta-llama/llama-4-maverick` to `meta-llama/llama-4-scout` to avoid type-check failures from model-catalog drift. - Fixed direct OpenAI Chat Completions requests to map `sessionId` and `cacheRetention` to OpenAI prompt caching fields, sending `prompt_cache_key` when caching is enabled and `prompt_cache_retention: "24h"` for direct `api.openai.com` requests with long retention ([#3426](https://github.com/badlogic/pi-mono/issues/3426)) - Fixed OpenAI-compatible Chat Completions requests to optionally send aligned `session_id`, `x-client-request-id`, and `x-session-affinity` session-affinity headers from `sessionId` via `compat.sendSessionAffinityHeaders`, enabling cache-affinity routing for backends such as Fireworks ([#3430](https://github.com/badlogic/pi-mono/issues/3430)) +- Fixed OpenAI-compatible Chat Completions Anthropic-style prompt caching to apply `cache_control` markers to the system prompt, last tool definition, and last user/assistant text content via `compat.cacheControlFormat`, and enabled that compat for OpenCode/OpenCode Go Qwen 3.5/3.6 Plus models so prompt caching works there too ([#3392](https://github.com/badlogic/pi-mono/issues/3392)) ## [0.67.68] - 2026-04-17 diff --git a/packages/ai/README.md b/packages/ai/README.md index 8c1531c5..735cca13 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -856,7 +856,8 @@ interface OpenAICompletionsCompat { requiresToolResultName?: boolean; // Whether tool results require the `name` field (default: false) requiresAssistantAfterToolResult?: boolean; // Whether tool results must be followed by an assistant message (default: false) requiresThinkingAsText?: boolean; // Whether thinking blocks must be converted to text (default: false) - thinkingFormat?: 'openai' | 'zai' | 'qwen'; // Format for reasoning param: 'openai' uses reasoning_effort, 'zai' uses thinking: { type: "enabled" }, 'qwen' uses enable_thinking: boolean (default: openai) + thinkingFormat?: 'openai' | 'zai' | 'qwen' | 'qwen-chat-template'; // Format for reasoning param: 'openai' uses reasoning_effort, 'zai' uses thinking: { type: "enabled" }, 'qwen' uses enable_thinking: boolean, 'qwen-chat-template' uses chat_template_kwargs.enable_thinking (default: openai) + cacheControlFormat?: 'anthropic'; // Anthropic-style cache_control on system prompt, last tool, and last user/assistant text content openRouterRouting?: OpenRouterRouting; // OpenRouter routing preferences (default: {}) vercelGatewayRouting?: VercelGatewayRouting; // Vercel AI Gateway routing preferences (default: {}) } diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index ea5e14a1..cb0e6cdd 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -3,7 +3,7 @@ import { writeFileSync } from "fs"; import { join, dirname } from "path"; import { fileURLToPath } from "url"; -import { Api, KnownProvider, Model } from "../src/types.js"; +import { Api, KnownProvider, Model, type OpenAICompletionsCompat } from "../src/types.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -484,6 +484,7 @@ async function loadModelsDevData(): Promise[]> { const npm = m.provider?.npm; let api: Api; let baseUrl: string; + let compat: OpenAICompletionsCompat | undefined; if (npm === "@ai-sdk/openai") { api = "openai-responses"; @@ -495,6 +496,10 @@ async function loadModelsDevData(): Promise[]> { } else if (npm === "@ai-sdk/google") { api = "google-generative-ai"; baseUrl = `${variant.basePath}/v1`; + } else if (npm === "@ai-sdk/alibaba") { + api = "openai-completions"; + baseUrl = `${variant.basePath}/v1`; + compat = { cacheControlFormat: "anthropic" }; } else { // null, undefined, or @ai-sdk/openai-compatible api = "openai-completions"; @@ -515,6 +520,7 @@ async function loadModelsDevData(): Promise[]> { cacheRead: m.cost?.cache_read || 0, cacheWrite: m.cost?.cache_write || 0, }, + ...(compat ? { compat } : {}), contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); diff --git a/packages/ai/src/models.generated.ts b/packages/ai/src/models.generated.ts index 4ac72888..053e3018 100644 --- a/packages/ai/src/models.generated.ts +++ b/packages/ai/src/models.generated.ts @@ -3373,7 +3373,7 @@ export const MODELS = { cost: { input: 1.25, output: 10, - cacheRead: 0.31, + cacheRead: 0.125, cacheWrite: 0, }, contextWindow: 1048576, @@ -6710,6 +6710,7 @@ export const MODELS = { api: "openai-completions", provider: "opencode", baseUrl: "https://opencode.ai/zen/v1", + compat: {"cacheControlFormat":"anthropic"}, reasoning: true, input: ["text", "image"], cost: { @@ -6727,6 +6728,7 @@ export const MODELS = { api: "openai-completions", provider: "opencode", baseUrl: "https://opencode.ai/zen/v1", + compat: {"cacheControlFormat":"anthropic"}, reasoning: true, input: ["text", "image"], cost: { @@ -6865,6 +6867,7 @@ export const MODELS = { api: "openai-completions", provider: "opencode-go", baseUrl: "https://opencode.ai/zen/go/v1", + compat: {"cacheControlFormat":"anthropic"}, reasoning: true, input: ["text", "image"], cost: { @@ -6882,6 +6885,7 @@ export const MODELS = { api: "openai-completions", provider: "opencode-go", baseUrl: "https://opencode.ai/zen/go/v1", + compat: {"cacheControlFormat":"anthropic"}, reasoning: true, input: ["text", "image"], cost: { @@ -8009,13 +8013,13 @@ export const MODELS = { reasoning: false, input: ["text"], cost: { - input: 0.09999999999999999, - output: 0.32, + input: 0.12, + output: 0.38, cacheRead: 0, cacheWrite: 0, }, contextWindow: 131072, - maxTokens: 16384, + maxTokens: 131072, } satisfies Model<"openai-completions">, "meta-llama/llama-3.3-70b-instruct:free": { id: "meta-llama/llama-3.3-70b-instruct:free", @@ -8034,23 +8038,6 @@ export const MODELS = { contextWindow: 65536, maxTokens: 4096, } satisfies Model<"openai-completions">, - "meta-llama/llama-4-maverick": { - id: "meta-llama/llama-4-maverick", - name: "Meta: Llama 4 Maverick", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: false, - input: ["text", "image"], - cost: { - input: 0.15, - output: 0.6, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 1048576, - maxTokens: 16384, - } satisfies Model<"openai-completions">, "meta-llama/llama-4-scout": { id: "meta-llama/llama-4-scout", name: "Meta: Llama 4 Scout", @@ -8609,7 +8596,7 @@ export const MODELS = { cacheRead: 0.07, cacheWrite: 0, }, - contextWindow: 256000, + contextWindow: 262144, maxTokens: 4096, } satisfies Model<"openai-completions">, "nex-agi/deepseek-v3.1-nex-n1": { @@ -9088,23 +9075,6 @@ export const MODELS = { contextWindow: 128000, maxTokens: 16384, } satisfies Model<"openai-completions">, - "openai/gpt-4o:extended": { - id: "openai/gpt-4o:extended", - name: "OpenAI: GPT-4o (extended)", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: false, - input: ["text", "image"], - cost: { - input: 6, - output: 18, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 128000, - maxTokens: 64000, - } satisfies Model<"openai-completions">, "openai/gpt-5": { id: "openai/gpt-5", name: "OpenAI: GPT-5", @@ -9995,7 +9965,7 @@ export const MODELS = { api: "openai-completions", provider: "openrouter", baseUrl: "https://openrouter.ai/api/v1", - reasoning: true, + reasoning: false, input: ["text"], cost: { input: 0.071, diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index 56a49a53..654da6da 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -5,7 +5,9 @@ import type { ChatCompletionContentPart, ChatCompletionContentPartImage, ChatCompletionContentPartText, + ChatCompletionDeveloperMessageParam, ChatCompletionMessageParam, + ChatCompletionSystemMessageParam, ChatCompletionToolMessageParam, } from "openai/resources/chat/completions.js"; import { getEnvApiKey } from "../env-api-keys.js"; @@ -59,6 +61,25 @@ export interface OpenAICompletionsOptions extends StreamOptions { reasoningEffort?: "minimal" | "low" | "medium" | "high" | "xhigh"; } +interface OpenAICompatCacheControl { + type: "ephemeral"; + ttl?: string; +} + +type ResolvedOpenAICompletionsCompat = Omit, "cacheControlFormat"> & { + cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"]; +}; + +type ChatCompletionInstructionMessageParam = ChatCompletionDeveloperMessageParam | ChatCompletionSystemMessageParam; + +type ChatCompletionTextPartWithCacheControl = ChatCompletionContentPartText & { + cache_control?: OpenAICompatCacheControl; +}; + +type ChatCompletionToolWithCacheControl = OpenAI.Chat.Completions.ChatCompletionTool & { + cache_control?: OpenAICompatCacheControl; +}; + function resolveCacheRetention(cacheRetention?: CacheRetention): CacheRetention { if (cacheRetention) { return cacheRetention; @@ -354,7 +375,7 @@ function createClient( apiKey?: string, optionsHeaders?: Record, sessionId?: string, - compat: Required = getCompat(model), + compat: ResolvedOpenAICompletionsCompat = getCompat(model), ) { if (!apiKey) { if (!process.env.OPENAI_API_KEY) { @@ -398,11 +419,11 @@ function buildParams( model: Model<"openai-completions">, context: Context, options?: OpenAICompletionsOptions, - compat: Required = getCompat(model), + compat: ResolvedOpenAICompletionsCompat = getCompat(model), cacheRetention: CacheRetention = resolveCacheRetention(options?.cacheRetention), ) { const messages = convertMessages(model, context, compat); - maybeAddOpenRouterAnthropicCacheControl(model, messages); + const cacheControl = getCompatCacheControl(model, compat, cacheRetention); const params: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { model: model.id, @@ -443,6 +464,10 @@ function buildParams( params.tools = []; } + if (cacheControl) { + applyAnthropicCacheControl(messages, params.tools, cacheControl); + } + if (options?.toolChoice) { params.tool_choice = options.toolChoice; } @@ -497,43 +522,126 @@ function mapReasoningEffort( return reasoningEffortMap[effort] ?? effort; } -function maybeAddOpenRouterAnthropicCacheControl( +function getCompatCacheControl( model: Model<"openai-completions">, + compat: ResolvedOpenAICompletionsCompat, + cacheRetention: CacheRetention, +): OpenAICompatCacheControl | undefined { + if (compat.cacheControlFormat !== "anthropic" || cacheRetention === "none") { + return undefined; + } + + const ttl = cacheRetention === "long" && model.baseUrl.includes("api.anthropic.com") ? "1h" : undefined; + return { type: "ephemeral", ...(ttl ? { ttl } : {}) }; +} + +function applyAnthropicCacheControl( messages: ChatCompletionMessageParam[], + tools: OpenAI.Chat.Completions.ChatCompletionTool[] | undefined, + cacheControl: OpenAICompatCacheControl, ): void { - if (model.provider !== "openrouter" || !model.id.startsWith("anthropic/")) return; + addCacheControlToSystemPrompt(messages, cacheControl); + addCacheControlToLastTool(tools, cacheControl); + addCacheControlToLastConversationMessage(messages, cacheControl); +} - // Anthropic-style caching requires cache_control on a text part. Add a breakpoint - // on the last user/assistant message (walking backwards until we find text content). - for (let i = messages.length - 1; i >= 0; i--) { - const msg = messages[i]; - if (msg.role !== "user" && msg.role !== "assistant") continue; - - const content = msg.content; - if (typeof content === "string") { - msg.content = [ - Object.assign({ type: "text" as const, text: content }, { cache_control: { type: "ephemeral" } }), - ]; +function addCacheControlToSystemPrompt( + messages: ChatCompletionMessageParam[], + cacheControl: OpenAICompatCacheControl, +): void { + for (const message of messages) { + if (message.role === "system" || message.role === "developer") { + addCacheControlToInstructionMessage(message, cacheControl); return; } + } +} - if (!Array.isArray(content)) continue; - - // Find last text part and add cache_control - for (let j = content.length - 1; j >= 0; j--) { - const part = content[j]; - if (part?.type === "text") { - Object.assign(part, { cache_control: { type: "ephemeral" } }); +function addCacheControlToLastConversationMessage( + messages: ChatCompletionMessageParam[], + cacheControl: OpenAICompatCacheControl, +): void { + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i]; + if (message.role === "user" || message.role === "assistant") { + if (addCacheControlToMessage(message, cacheControl)) { return; } } } } +function addCacheControlToLastTool( + tools: OpenAI.Chat.Completions.ChatCompletionTool[] | undefined, + cacheControl: OpenAICompatCacheControl, +): void { + if (!tools || tools.length === 0) { + return; + } + + const lastTool = tools[tools.length - 1] as ChatCompletionToolWithCacheControl; + lastTool.cache_control = cacheControl; +} + +function addCacheControlToInstructionMessage( + message: ChatCompletionInstructionMessageParam, + cacheControl: OpenAICompatCacheControl, +): boolean { + return addCacheControlToTextContent(message, cacheControl); +} + +function addCacheControlToMessage( + message: ChatCompletionMessageParam, + cacheControl: OpenAICompatCacheControl, +): boolean { + if (message.role === "user" || message.role === "assistant") { + return addCacheControlToTextContent(message, cacheControl); + } + return false; +} + +function addCacheControlToTextContent( + message: + | ChatCompletionInstructionMessageParam + | ChatCompletionAssistantMessageParam + | Extract, + cacheControl: OpenAICompatCacheControl, +): boolean { + const content = message.content; + if (typeof content === "string") { + if (content.length === 0) { + return false; + } + message.content = [ + { + type: "text", + text: content, + cache_control: cacheControl, + }, + ] as ChatCompletionTextPartWithCacheControl[]; + return true; + } + + if (!Array.isArray(content)) { + return false; + } + + for (let i = content.length - 1; i >= 0; i--) { + const part = content[i]; + if (part?.type === "text") { + const textPart = part as ChatCompletionTextPartWithCacheControl; + textPart.cache_control = cacheControl; + return true; + } + } + + return false; +} + export function convertMessages( model: Model<"openai-completions">, context: Context, - compat: Required, + compat: ResolvedOpenAICompletionsCompat, ): ChatCompletionMessageParam[] { const params: ChatCompletionMessageParam[] = []; @@ -756,7 +864,7 @@ export function convertMessages( function convertTools( tools: Tool[], - compat: Required, + compat: ResolvedOpenAICompletionsCompat, ): OpenAI.Chat.Completions.ChatCompletionTool[] { return tools.map((tool) => ({ type: "function", @@ -839,7 +947,7 @@ function mapStopReason(reason: ChatCompletionChunk.Choice["finish_reason"] | str * Provider takes precedence over URL-based detection since it's explicitly configured. * Returns a fully resolved OpenAICompletionsCompat object with all fields set. */ -function detectCompat(model: Model<"openai-completions">): Required { +function detectCompat(model: Model<"openai-completions">): ResolvedOpenAICompletionsCompat { const provider = model.provider; const baseUrl = model.baseUrl; @@ -860,6 +968,7 @@ function detectCompat(model: Model<"openai-completions">): Required): Required): Required): Required { +function getCompat(model: Model<"openai-completions">): ResolvedOpenAICompletionsCompat { const detected = detectCompat(model); if (!model.compat) return detected; @@ -918,6 +1028,7 @@ function getCompat(model: Model<"openai-completions">): Required; + tools?: ToolWithCacheControl[]; +} + +const mockState = vi.hoisted(() => ({ + lastParams: undefined as CapturedParams | undefined, +})); + +vi.mock("openai", () => { + class FakeOpenAI { + chat = { + completions: { + create: (params: CapturedParams) => { + mockState.lastParams = params; + const stream = { + async *[Symbol.asyncIterator]() { + yield { + id: "chatcmpl-test", + choices: [{ delta: {}, finish_reason: "stop" }], + usage: { + prompt_tokens: 1, + completion_tokens: 1, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }; + }, + }; + const promise = Promise.resolve(stream) as Promise & { + withResponse: () => Promise<{ + data: typeof stream; + response: { status: number; headers: Headers }; + }>; + }; + promise.withResponse = async () => ({ + data: stream, + response: { status: 200, headers: new Headers() }, + }); + return promise; + }, + }, + }; + } + + return { default: FakeOpenAI }; +}); + +async function capturePayload( + model: Model<"openai-completions">, + options?: { cacheRetention?: "none" | "short" | "long" }, +): Promise { + const timestamp = Date.now(); + + await streamOpenAICompletions( + model, + { + systemPrompt: "System prompt", + messages: [{ role: "user", content: "Hello", timestamp }], + tools: [ + { + name: "read", + description: "Read a file", + parameters: Type.Object({ + path: Type.String(), + }), + }, + ], + }, + { apiKey: "test-key", ...options }, + ).result(); + + if (!mockState.lastParams) { + throw new Error("Expected payload to be captured"); + } + + return mockState.lastParams; +} + +function getInstructionMessage(params: CapturedParams) { + return params.messages.find((message) => message.role === "system" || message.role === "developer"); +} + +function expectAnthropicCacheMarkers(params: CapturedParams): void { + const instructionMessage = getInstructionMessage(params); + expect(instructionMessage).toBeDefined(); + expect(Array.isArray(instructionMessage?.content)).toBe(true); + expect((instructionMessage?.content as TextPart[])[0]?.cache_control).toEqual({ type: "ephemeral" }); + + expect(params.tools).toHaveLength(1); + expect(params.tools?.[0]?.cache_control).toEqual({ type: "ephemeral" }); + + const lastMessage = params.messages[params.messages.length - 1]; + expect(lastMessage.role).toBe("user"); + expect(Array.isArray(lastMessage.content)).toBe(true); + expect((lastMessage.content as TextPart[])[0]?.cache_control).toEqual({ type: "ephemeral" }); +} + +describe("openai-completions cacheControlFormat", () => { + beforeEach(() => { + mockState.lastParams = undefined; + }); + + it("applies Anthropic-style cache markers for built-in opencode-go Qwen models", async () => { + const model = getModel("opencode-go", "qwen3.5-plus"); + expect(model.compat?.cacheControlFormat).toBe("anthropic"); + + const params = await capturePayload(model); + expectAnthropicCacheMarkers(params); + }); + + it("preserves Anthropic-style cache markers for OpenRouter Anthropic models", async () => { + const model = getModel("openrouter", "anthropic/claude-sonnet-4"); + const params = await capturePayload(model); + expectAnthropicCacheMarkers(params); + }); + + it("omits Anthropic-style cache markers when cacheRetention is none", async () => { + const model = getModel("opencode-go", "qwen3.5-plus"); + const params = await capturePayload(model, { cacheRetention: "none" }); + const instructionMessage = getInstructionMessage(params); + + expect(Array.isArray(instructionMessage?.content)).toBe(false); + expect(params.tools?.[0]?.cache_control).toBeUndefined(); + expect(typeof params.messages[params.messages.length - 1]?.content).toBe("string"); + }); +}); diff --git a/packages/ai/test/openai-completions-tool-result-images.test.ts b/packages/ai/test/openai-completions-tool-result-images.test.ts index acab74c0..50fce94e 100644 --- a/packages/ai/test/openai-completions-tool-result-images.test.ts +++ b/packages/ai/test/openai-completions-tool-result-images.test.ts @@ -34,6 +34,7 @@ const compat: Required = { vercelGatewayRouting: {}, zaiToolStream: false, supportsStrictMode: true, + cacheControlFormat: "anthropic", sendSessionAffinityHeaders: false, }; diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index d2fcd7f5..fbd63786 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -19,6 +19,7 @@ - Fixed shared/exported plain-text tool output to preserve indentation instead of collapsing leading whitespace in the web share page ([#3440](https://github.com/badlogic/pi-mono/issues/3440)) - Fixed skill resolution to dedupe symlinked aliases by canonical path, so `pi config` no longer shows duplicate skill entries when `~/.pi/agent/skills` points to `~/.agents/skills` ([#3405](https://github.com/badlogic/pi-mono/issues/3405)) - Fixed OpenRouter request attribution to include Pi app headers (`HTTP-Referer: https://pi.dev`, `X-OpenRouter-Title: pi`, `X-OpenRouter-Categories: cli-agent`) when sessions are created through the coding-agent SDK and install telemetry is enabled ([#3414](https://github.com/badlogic/pi-mono/issues/3414)) +- Fixed custom-model `compat` schema/docs to support `cacheControlFormat: "anthropic"` for OpenAI-compatible providers that expose Anthropic-style prompt caching via `cache_control` markers ([#3392](https://github.com/badlogic/pi-mono/issues/3392)) ## [0.67.68] - 2026-04-17 diff --git a/packages/coding-agent/docs/custom-provider.md b/packages/coding-agent/docs/custom-provider.md index aff35489..be22acc9 100644 --- a/packages/coding-agent/docs/custom-provider.md +++ b/packages/coding-agent/docs/custom-provider.md @@ -183,12 +183,14 @@ models: [{ }, maxTokensField: "max_tokens", // instead of "max_completion_tokens" requiresToolResultName: true, // tool results need name field - thinkingFormat: "qwen" // top-level enable_thinking: true + thinkingFormat: "qwen", // top-level enable_thinking: true + cacheControlFormat: "anthropic" // Anthropic-style cache_control markers } }] ``` Use `qwen-chat-template` instead for local Qwen-compatible servers that read `chat_template_kwargs.enable_thinking`. +Use `cacheControlFormat: "anthropic"` for OpenAI-compatible providers that expose Anthropic-style prompt caching via `cache_control` on the system prompt, last tool definition, and last user/assistant text content. > Migration note: Mistral moved from `openai-completions` to `mistral-conversations`. > Use `mistral-conversations` for native Mistral models. @@ -589,8 +591,10 @@ interface ProviderModelConfig { requiresAssistantAfterToolResult?: boolean; requiresThinkingAsText?: boolean; thinkingFormat?: "openai" | "zai" | "qwen" | "qwen-chat-template"; + cacheControlFormat?: "anthropic"; }; } ``` `qwen` is for DashScope-style top-level `enable_thinking`. Use `qwen-chat-template` for local Qwen-compatible servers that read `chat_template_kwargs.enable_thinking`. +`cacheControlFormat: "anthropic"` applies Anthropic-style `cache_control` markers to the system prompt, last tool definition, and last user/assistant text content. diff --git a/packages/coding-agent/docs/models.md b/packages/coding-agent/docs/models.md index 8321d45d..844f2794 100644 --- a/packages/coding-agent/docs/models.md +++ b/packages/coding-agent/docs/models.md @@ -304,12 +304,15 @@ For providers with partial OpenAI compatibility, use the `compat` field. | `requiresAssistantAfterToolResult` | Insert an assistant message before a user message after tool results | | `requiresThinkingAsText` | Convert thinking blocks to plain text | | `thinkingFormat` | Use `reasoning_effort`, `zai`, `qwen`, or `qwen-chat-template` thinking parameters | +| `cacheControlFormat` | Use Anthropic-style `cache_control` markers on the system prompt, last tool definition, and last user/assistant text content. Currently only `anthropic` is supported. | | `supportsStrictMode` | Include the `strict` field in tool definitions | | `openRouterRouting` | OpenRouter provider routing preferences. This object is sent as-is in the `provider` field of the [OpenRouter API request](https://openrouter.ai/docs/guides/routing/provider-selection). | | `vercelGatewayRouting` | Vercel AI Gateway routing config for provider selection (`only`, `order`) | `qwen` uses top-level `enable_thinking`. Use `qwen-chat-template` for local Qwen-compatible servers that require `chat_template_kwargs.enable_thinking`. +`cacheControlFormat: "anthropic"` is for OpenAI-compatible providers that expose Anthropic-style prompt caching through `cache_control` markers on text content and tool definitions. + Example: ```json diff --git a/packages/coding-agent/src/core/model-registry.ts b/packages/coding-agent/src/core/model-registry.ts index 5a93b438..fc05c1c3 100644 --- a/packages/coding-agent/src/core/model-registry.ts +++ b/packages/coding-agent/src/core/model-registry.ts @@ -108,6 +108,7 @@ const OpenAICompletionsCompatSchema = Type.Object({ Type.Literal("qwen-chat-template"), ]), ), + cacheControlFormat: Type.Optional(Type.Literal("anthropic")), openRouterRouting: Type.Optional(OpenRouterRoutingSchema), vercelGatewayRouting: Type.Optional(VercelGatewayRoutingSchema), supportsStrictMode: Type.Optional(Type.Boolean()), diff --git a/packages/coding-agent/test/model-registry.test.ts b/packages/coding-agent/test/model-registry.test.ts index 15c25475..470ef926 100644 --- a/packages/coding-agent/test/model-registry.test.ts +++ b/packages/coding-agent/test/model-registry.test.ts @@ -374,7 +374,7 @@ describe("ModelRegistry", () => { } }); - test("compat schema accepts reasoningEffortMap and supportsStrictMode", () => { + test("compat schema accepts reasoningEffortMap, supportsStrictMode, and cacheControlFormat", () => { writeRawModelsJson({ demo: { baseUrl: "https://example.com/v1", @@ -394,6 +394,7 @@ describe("ModelRegistry", () => { high: "max", }, supportsStrictMode: false, + cacheControlFormat: "anthropic", }, }, ], @@ -406,6 +407,7 @@ describe("ModelRegistry", () => { expect(registry.getError()).toBeUndefined(); expect(compat?.reasoningEffortMap).toEqual({ minimal: "default", high: "max" }); expect(compat?.supportsStrictMode).toBe(false); + expect(compat?.cacheControlFormat).toBe("anthropic"); }); test("model-level baseUrl overrides provider-level baseUrl for custom models", () => {