shumengya/sproutclaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ai,coding-agent): support anthropic-style cache control for openai compatibles closes #3392

Browse Source
This commit is contained in:
Mario Zechner
2026-04-20 17:12:05 +02:00
parent 4b2caf43a8
commit 3054fd7a3b
13 changed files with 326 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
packages/ai/src/models.generated.ts
View File

@@ -3373,7 +3373,7 @@ export const MODELS = {
cost: {
input: 1.25,
output: 10,
cacheRead: 0.31,
cacheRead: 0.125,
cacheWrite: 0,
},
contextWindow: 1048576,
@@ -6710,6 +6710,7 @@ export const MODELS = {
api: "openai-completions",
provider: "opencode",
baseUrl: "https://opencode.ai/zen/v1",
compat: {"cacheControlFormat":"anthropic"},
reasoning: true,
input: ["text", "image"],
cost: {
@@ -6727,6 +6728,7 @@ export const MODELS = {
api: "openai-completions",
provider: "opencode",
baseUrl: "https://opencode.ai/zen/v1",
compat: {"cacheControlFormat":"anthropic"},
reasoning: true,
input: ["text", "image"],
cost: {
@@ -6865,6 +6867,7 @@ export const MODELS = {
api: "openai-completions",
provider: "opencode-go",
baseUrl: "https://opencode.ai/zen/go/v1",
compat: {"cacheControlFormat":"anthropic"},
reasoning: true,
input: ["text", "image"],
cost: {
@@ -6882,6 +6885,7 @@ export const MODELS = {
api: "openai-completions",
provider: "opencode-go",
baseUrl: "https://opencode.ai/zen/go/v1",
compat: {"cacheControlFormat":"anthropic"},
reasoning: true,
input: ["text", "image"],
cost: {
@@ -8009,13 +8013,13 @@ export const MODELS = {
reasoning: false,
input: ["text"],
cost: {
input: 0.09999999999999999,
output: 0.32,
input: 0.12,
output: 0.38,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 131072,
maxTokens: 16384,
maxTokens: 131072,
} satisfies Model<"openai-completions">,
"meta-llama/llama-3.3-70b-instruct:free": {
id: "meta-llama/llama-3.3-70b-instruct:free",
@@ -8034,23 +8038,6 @@ export const MODELS = {
contextWindow: 65536,
maxTokens: 4096,
} satisfies Model<"openai-completions">,
"meta-llama/llama-4-maverick": {
id: "meta-llama/llama-4-maverick",
name: "Meta: Llama 4 Maverick",
api: "openai-completions",
provider: "openrouter",
baseUrl: "https://openrouter.ai/api/v1",
reasoning: false,
input: ["text", "image"],
cost: {
input: 0.15,
output: 0.6,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 1048576,
maxTokens: 16384,
} satisfies Model<"openai-completions">,
"meta-llama/llama-4-scout": {
id: "meta-llama/llama-4-scout",
name: "Meta: Llama 4 Scout",
@@ -8609,7 +8596,7 @@ export const MODELS = {
cacheRead: 0.07,
cacheWrite: 0,
},
contextWindow: 256000,
contextWindow: 262144,
maxTokens: 4096,
} satisfies Model<"openai-completions">,
"nex-agi/deepseek-v3.1-nex-n1": {
@@ -9088,23 +9075,6 @@ export const MODELS = {
contextWindow: 128000,
maxTokens: 16384,
} satisfies Model<"openai-completions">,
"openai/gpt-4o:extended": {
id: "openai/gpt-4o:extended",
name: "OpenAI: GPT-4o (extended)",
api: "openai-completions",
provider: "openrouter",
baseUrl: "https://openrouter.ai/api/v1",
reasoning: false,
input: ["text", "image"],
cost: {
input: 6,
output: 18,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 128000,
maxTokens: 64000,
} satisfies Model<"openai-completions">,
"openai/gpt-5": {
id: "openai/gpt-5",
name: "OpenAI: GPT-5",
@@ -9995,7 +9965,7 @@ export const MODELS = {
api: "openai-completions",
provider: "openrouter",
baseUrl: "https://openrouter.ai/api/v1",
reasoning: true,
reasoning: false,
input: ["text"],
cost: {
input: 0.071,

165
packages/ai/src/providers/openai-completions.ts
View File

@@ -5,7 +5,9 @@ import type {
ChatCompletionContentPart,
ChatCompletionContentPartImage,
ChatCompletionContentPartText,
ChatCompletionDeveloperMessageParam,
ChatCompletionMessageParam,
ChatCompletionSystemMessageParam,
ChatCompletionToolMessageParam,
} from "openai/resources/chat/completions.js";
import { getEnvApiKey } from "../env-api-keys.js";
@@ -59,6 +61,25 @@ export interface OpenAICompletionsOptions extends StreamOptions {
reasoningEffort?: "minimal" | "low" | "medium" | "high" | "xhigh";
}
interface OpenAICompatCacheControl {
type: "ephemeral";
ttl?: string;
}
type ResolvedOpenAICompletionsCompat = Omit<Required<OpenAICompletionsCompat>, "cacheControlFormat"> & {
cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"];
};
type ChatCompletionInstructionMessageParam = ChatCompletionDeveloperMessageParam | ChatCompletionSystemMessageParam;
type ChatCompletionTextPartWithCacheControl = ChatCompletionContentPartText & {
cache_control?: OpenAICompatCacheControl;
};
type ChatCompletionToolWithCacheControl = OpenAI.Chat.Completions.ChatCompletionTool & {
cache_control?: OpenAICompatCacheControl;
};
function resolveCacheRetention(cacheRetention?: CacheRetention): CacheRetention {
if (cacheRetention) {
return cacheRetention;
@@ -354,7 +375,7 @@ function createClient(
apiKey?: string,
optionsHeaders?: Record<string, string>,
sessionId?: string,
compat: Required<OpenAICompletionsCompat> = getCompat(model),
compat: ResolvedOpenAICompletionsCompat = getCompat(model),
) {
if (!apiKey) {
if (!process.env.OPENAI_API_KEY) {
@@ -398,11 +419,11 @@ function buildParams(
model: Model<"openai-completions">,
context: Context,
options?: OpenAICompletionsOptions,
compat: Required<OpenAICompletionsCompat> = getCompat(model),
compat: ResolvedOpenAICompletionsCompat = getCompat(model),
cacheRetention: CacheRetention = resolveCacheRetention(options?.cacheRetention),
) {
const messages = convertMessages(model, context, compat);
maybeAddOpenRouterAnthropicCacheControl(model, messages);
const cacheControl = getCompatCacheControl(model, compat, cacheRetention);
const params: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = {
model: model.id,
@@ -443,6 +464,10 @@ function buildParams(
params.tools = [];
}
if (cacheControl) {
applyAnthropicCacheControl(messages, params.tools, cacheControl);
}
if (options?.toolChoice) {
params.tool_choice = options.toolChoice;
}
@@ -497,43 +522,126 @@ function mapReasoningEffort(
return reasoningEffortMap[effort] ?? effort;
}
function maybeAddOpenRouterAnthropicCacheControl(
function getCompatCacheControl(
model: Model<"openai-completions">,
compat: ResolvedOpenAICompletionsCompat,
cacheRetention: CacheRetention,
): OpenAICompatCacheControl | undefined {
if (compat.cacheControlFormat !== "anthropic" || cacheRetention === "none") {
return undefined;
}
const ttl = cacheRetention === "long" && model.baseUrl.includes("api.anthropic.com") ? "1h" : undefined;
return { type: "ephemeral", ...(ttl ? { ttl } : {}) };
}
function applyAnthropicCacheControl(
messages: ChatCompletionMessageParam[],
tools: OpenAI.Chat.Completions.ChatCompletionTool[] | undefined,
cacheControl: OpenAICompatCacheControl,
): void {
if (model.provider !== "openrouter" || !model.id.startsWith("anthropic/")) return;
addCacheControlToSystemPrompt(messages, cacheControl);
addCacheControlToLastTool(tools, cacheControl);
addCacheControlToLastConversationMessage(messages, cacheControl);
}
// Anthropic-style caching requires cache_control on a text part. Add a breakpoint
// on the last user/assistant message (walking backwards until we find text content).
for (let i = messages.length - 1; i >= 0; i--) {
const msg = messages[i];
if (msg.role !== "user" && msg.role !== "assistant") continue;
const content = msg.content;
if (typeof content === "string") {
msg.content = [
Object.assign({ type: "text" as const, text: content }, { cache_control: { type: "ephemeral" } }),
];
function addCacheControlToSystemPrompt(
messages: ChatCompletionMessageParam[],
cacheControl: OpenAICompatCacheControl,
): void {
for (const message of messages) {
if (message.role === "system" || message.role === "developer") {
addCacheControlToInstructionMessage(message, cacheControl);
return;
}
}
}
if (!Array.isArray(content)) continue;
// Find last text part and add cache_control
for (let j = content.length - 1; j >= 0; j--) {
const part = content[j];
if (part?.type === "text") {
Object.assign(part, { cache_control: { type: "ephemeral" } });
function addCacheControlToLastConversationMessage(
messages: ChatCompletionMessageParam[],
cacheControl: OpenAICompatCacheControl,
): void {
for (let i = messages.length - 1; i >= 0; i--) {
const message = messages[i];
if (message.role === "user" || message.role === "assistant") {
if (addCacheControlToMessage(message, cacheControl)) {
return;
}
}
}
}
function addCacheControlToLastTool(
tools: OpenAI.Chat.Completions.ChatCompletionTool[] | undefined,
cacheControl: OpenAICompatCacheControl,
): void {
if (!tools || tools.length === 0) {
return;
}
const lastTool = tools[tools.length - 1] as ChatCompletionToolWithCacheControl;
lastTool.cache_control = cacheControl;
}
function addCacheControlToInstructionMessage(
message: ChatCompletionInstructionMessageParam,
cacheControl: OpenAICompatCacheControl,
): boolean {
return addCacheControlToTextContent(message, cacheControl);
}
function addCacheControlToMessage(
message: ChatCompletionMessageParam,
cacheControl: OpenAICompatCacheControl,
): boolean {
if (message.role === "user" || message.role === "assistant") {
return addCacheControlToTextContent(message, cacheControl);
}
return false;
}
function addCacheControlToTextContent(
message:
| ChatCompletionInstructionMessageParam
| ChatCompletionAssistantMessageParam
| Extract<ChatCompletionMessageParam, { role: "user" }>,
cacheControl: OpenAICompatCacheControl,
): boolean {
const content = message.content;
if (typeof content === "string") {
if (content.length === 0) {
return false;
}
message.content = [
{
type: "text",
text: content,
cache_control: cacheControl,
},
] as ChatCompletionTextPartWithCacheControl[];
return true;
}
if (!Array.isArray(content)) {
return false;
}
for (let i = content.length - 1; i >= 0; i--) {
const part = content[i];
if (part?.type === "text") {
const textPart = part as ChatCompletionTextPartWithCacheControl;
textPart.cache_control = cacheControl;
return true;
}
}
return false;
}
export function convertMessages(
model: Model<"openai-completions">,
context: Context,
compat: Required<OpenAICompletionsCompat>,
compat: ResolvedOpenAICompletionsCompat,
): ChatCompletionMessageParam[] {
const params: ChatCompletionMessageParam[] = [];
@@ -756,7 +864,7 @@ export function convertMessages(
function convertTools(
tools: Tool[],
compat: Required<OpenAICompletionsCompat>,
compat: ResolvedOpenAICompletionsCompat,
): OpenAI.Chat.Completions.ChatCompletionTool[] {
return tools.map((tool) => ({
type: "function",
@@ -839,7 +947,7 @@ function mapStopReason(reason: ChatCompletionChunk.Choice["finish_reason"] | str
* Provider takes precedence over URL-based detection since it's explicitly configured.
* Returns a fully resolved OpenAICompletionsCompat object with all fields set.
*/
function detectCompat(model: Model<"openai-completions">): Required<OpenAICompletionsCompat> {
function detectCompat(model: Model<"openai-completions">): ResolvedOpenAICompletionsCompat {
const provider = model.provider;
const baseUrl = model.baseUrl;
@@ -860,6 +968,7 @@ function detectCompat(model: Model<"openai-completions">): Required<OpenAIComple
const isGrok = provider === "xai" || baseUrl.includes("api.x.ai");
const isGroq = provider === "groq" || baseUrl.includes("groq.com");
const cacheControlFormat = provider === "openrouter" && model.id.startsWith("anthropic/") ? "anthropic" : undefined;
const reasoningEffortMap =
isGroq && model.id === "qwen/qwen3-32b"
@@ -890,6 +999,7 @@ function detectCompat(model: Model<"openai-completions">): Required<OpenAIComple
vercelGatewayRouting: {},
zaiToolStream: false,
supportsStrictMode: true,
cacheControlFormat,
sendSessionAffinityHeaders: false,
};
}
@@ -898,7 +1008,7 @@ function detectCompat(model: Model<"openai-completions">): Required<OpenAIComple
* Get resolved compatibility settings for a model.
* Uses explicit model.compat if provided, otherwise auto-detects from provider/URL.
*/
function getCompat(model: Model<"openai-completions">): Required<OpenAICompletionsCompat> {
function getCompat(model: Model<"openai-completions">): ResolvedOpenAICompletionsCompat {
const detected = detectCompat(model);
if (!model.compat) return detected;
@@ -918,6 +1028,7 @@ function getCompat(model: Model<"openai-completions">): Required<OpenAICompletio
vercelGatewayRouting: model.compat.vercelGatewayRouting ?? detected.vercelGatewayRouting,
zaiToolStream: model.compat.zaiToolStream ?? detected.zaiToolStream,
supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode,
cacheControlFormat: model.compat.cacheControlFormat ?? detected.cacheControlFormat,
sendSessionAffinityHeaders: model.compat.sendSessionAffinityHeaders ?? detected.sendSessionAffinityHeaders,
};
}

2
packages/ai/src/types.ts
View File

@@ -291,6 +291,8 @@ export interface OpenAICompletionsCompat {
zaiToolStream?: boolean;
/** Whether the provider supports the `strict` field in tool definitions. Default: true. */
supportsStrictMode?: boolean;
/** Cache control convention for prompt caching. "anthropic" applies Anthropic-style `cache_control` markers to the system prompt, last tool definition, and last user/assistant text content. */
cacheControlFormat?: "anthropic";
/** Whether to send known session-affinity headers (`session_id`, `x-client-request-id`, `x-session-affinity`) from `options.sessionId` when caching is enabled. Default: false. */
sendSessionAffinityHeaders?: boolean;
}