diff --git a/packages/ai/src/providers/anthropic.ts b/packages/ai/src/providers/anthropic.ts index 3420c18a..053538f5 100644 --- a/packages/ai/src/providers/anthropic.ts +++ b/packages/ai/src/providers/anthropic.ts @@ -51,14 +51,14 @@ function resolveCacheRetention(cacheRetention?: CacheRetention): CacheRetention } function getCacheControl( - baseUrl: string, + model: Model<"anthropic-messages">, cacheRetention?: CacheRetention, ): { retention: CacheRetention; cacheControl?: CacheControlEphemeral } { const retention = resolveCacheRetention(cacheRetention); if (retention === "none") { return { retention }; } - const ttl = retention === "long" && baseUrl.includes("api.anthropic.com") ? "1h" : undefined; + const ttl = retention === "long" && getAnthropicCompat(model).supportsLongCacheRetention ? "1h" : undefined; return { retention, cacheControl: { type: "ephemeral", ...(ttl && { ttl }) }, @@ -166,6 +166,7 @@ const INTERLEAVED_THINKING_BETA = "interleaved-thinking-2025-05-14"; function getAnthropicCompat(model: Model<"anthropic-messages">): Required { return { supportsEagerToolInputStreaming: model.compat?.supportsEagerToolInputStreaming ?? true, + supportsLongCacheRetention: model.compat?.supportsLongCacheRetention ?? true, }; } @@ -833,7 +834,7 @@ function buildParams( isOAuthToken: boolean, options?: AnthropicOptions, ): MessageCreateParamsStreaming { - const { cacheControl } = getCacheControl(model.baseUrl, options?.cacheRetention); + const { cacheControl } = getCacheControl(model, options?.cacheRetention); const params: MessageCreateParamsStreaming = { model: model.id, messages: convertMessages(context.messages, model, isOAuthToken, cacheControl), diff --git a/packages/ai/src/providers/openai-responses.ts b/packages/ai/src/providers/openai-responses.ts index 135f57ef..8048d855 100644 --- a/packages/ai/src/providers/openai-responses.ts +++ b/packages/ai/src/providers/openai-responses.ts @@ -8,6 +8,7 @@ import type { CacheRetention, Context, Model, + OpenAIResponsesCompat, SimpleStreamOptions, StreamFunction, StreamOptions, @@ -35,18 +36,18 @@ function resolveCacheRetention(cacheRetention?: CacheRetention): CacheRetention return "short"; } -/** - * Get prompt cache retention based on cacheRetention and base URL. - * Only applies to direct OpenAI API calls (api.openai.com). - */ -function getPromptCacheRetention(baseUrl: string, cacheRetention: CacheRetention): "24h" | undefined { - if (cacheRetention !== "long") { - return undefined; - } - if (baseUrl.includes("api.openai.com")) { - return "24h"; - } - return undefined; +function getCompat(model: Model<"openai-responses">): Required { + return { + sendSessionIdHeader: model.compat?.sendSessionIdHeader ?? true, + supportsLongCacheRetention: model.compat?.supportsLongCacheRetention ?? true, + }; +} + +function getPromptCacheRetention( + compat: Required, + cacheRetention: CacheRetention, +): "24h" | undefined { + return cacheRetention === "long" && compat.supportsLongCacheRetention ? "24h" : undefined; } // OpenAI Responses-specific options @@ -169,6 +170,7 @@ function createClient( apiKey = process.env.OPENAI_API_KEY; } + const compat = getCompat(model); const headers = { ...model.headers }; if (model.provider === "github-copilot") { const hasImages = hasCopilotVisionInput(context.messages); @@ -180,7 +182,7 @@ function createClient( } if (sessionId) { - if (model.compat?.sendSessionIdHeader !== false) { + if (compat.sendSessionIdHeader) { headers.session_id = sessionId; } headers["x-client-request-id"] = sessionId; @@ -203,12 +205,13 @@ function buildParams(model: Model<"openai-responses">, context: Context, options const messages = convertResponsesMessages(model, context, OPENAI_TOOL_CALL_PROVIDERS); const cacheRetention = resolveCacheRetention(options?.cacheRetention); + const compat = getCompat(model); const params: ResponseCreateParamsStreaming = { model: model.id, input: messages, stream: true, prompt_cache_key: cacheRetention === "none" ? undefined : options?.sessionId, - prompt_cache_retention: getPromptCacheRetention(model.baseUrl, cacheRetention), + prompt_cache_retention: getPromptCacheRetention(compat, cacheRetention), store: false, }; diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index bb7950db..e69e87ed 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -296,12 +296,16 @@ export interface OpenAICompletionsCompat { cacheControlFormat?: "anthropic"; /** Whether to send known session-affinity headers (`session_id`, `x-client-request-id`, `x-session-affinity`) from `options.sessionId` when caching is enabled. Default: false. */ sendSessionAffinityHeaders?: boolean; + /** Whether the provider supports long prompt cache retention (`prompt_cache_retention: "24h"` or Anthropic-style `cache_control.ttl: "1h"`, depending on format). Default: true. */ + supportsLongCacheRetention?: boolean; } /** Compatibility settings for OpenAI Responses APIs. */ export interface OpenAIResponsesCompat { /** Whether to send the OpenAI `session_id` cache-affinity header from `options.sessionId` when caching is enabled. Default: true. */ sendSessionIdHeader?: boolean; + /** Whether the provider supports `prompt_cache_retention: "24h"`. Default: true. */ + supportsLongCacheRetention?: boolean; } /** Compatibility settings for Anthropic Messages-compatible APIs. */ @@ -314,6 +318,8 @@ export interface AnthropicMessagesCompat { * Default: true. */ supportsEagerToolInputStreaming?: boolean; + /** Whether the provider supports Anthropic long cache retention (`cache_control.ttl: "1h"`). Default: true. */ + supportsLongCacheRetention?: boolean; } /** diff --git a/packages/ai/test/anthropic-long-cache-retention-e2e.test.ts b/packages/ai/test/anthropic-long-cache-retention-e2e.test.ts new file mode 100644 index 00000000..d900aa71 --- /dev/null +++ b/packages/ai/test/anthropic-long-cache-retention-e2e.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from "vitest"; +import { getEnvApiKey } from "../src/env-api-keys.js"; +import { getModels, getProviders } from "../src/models.js"; +import { complete } from "../src/stream.js"; +import type { Api, KnownProvider, Model, ProviderStreamOptions } from "../src/types.js"; +import { resolveApiKey } from "./oauth.js"; + +const githubCopilotToken = await resolveApiKey("github-copilot"); + +interface AnthropicLongCacheRetentionE2ECase { + name: string; + provider: KnownProvider; + model: Model<"anthropic-messages">; + apiKey: string | undefined; +} + +function getE2EApiKey(provider: KnownProvider): string | undefined { + if (provider === "github-copilot") { + return githubCopilotToken; + } + return getEnvApiKey(provider); +} + +function getAnthropicMessagesModels(provider: KnownProvider): Model<"anthropic-messages">[] { + const models = getModels(provider) as Model[]; + return models.filter((model) => model.api === "anthropic-messages") as Model<"anthropic-messages">[]; +} + +const anthropicMessagesCases: AnthropicLongCacheRetentionE2ECase[] = getProviders().flatMap((provider) => + getAnthropicMessagesModels(provider).map((model) => ({ + name: `${provider}/${model.id}`, + provider, + model, + apiKey: getE2EApiKey(provider), + })), +); + +function getProbePriority(model: Model<"anthropic-messages">): number { + const modelId = model.id.toLowerCase(); + const cost = model.cost.input + model.cost.output; + let priority = cost; + + if (modelId.includes("haiku") && (modelId.includes("4-5") || modelId.includes("4.5"))) { + priority -= 1000; + } else if (modelId.includes("sonnet") && (modelId.includes("4-") || modelId.includes("4."))) { + priority -= 750; + } else if (modelId.includes("claude") && (modelId.includes("4-") || modelId.includes("4."))) { + priority -= 500; + } + + return priority; +} + +function selectOneCasePerProvider(cases: AnthropicLongCacheRetentionE2ECase[]): AnthropicLongCacheRetentionE2ECase[] { + const byProvider = new Map(); + for (const testCase of cases) { + const providerCases = byProvider.get(testCase.provider) ?? []; + providerCases.push(testCase); + byProvider.set(testCase.provider, providerCases); + } + + return Array.from(byProvider.values()).map( + (providerCases) => + providerCases.sort( + (a, b) => getProbePriority(a.model) - getProbePriority(b.model) || a.model.id.localeCompare(b.model.id), + )[0], + ); +} + +const probeCases = selectOneCasePerProvider(anthropicMessagesCases); + +function withLongCacheRetention(model: Model<"anthropic-messages">): Model<"anthropic-messages"> { + return { + ...model, + compat: { + ...model.compat, + supportsLongCacheRetention: true, + }, + }; +} + +async function expectLongCacheRetentionAccepted( + model: Model<"anthropic-messages">, + apiKey: string | undefined, +): Promise { + const options: ProviderStreamOptions = { + apiKey, + cacheRetention: "long", + maxTokens: 128, + thinkingEnabled: false, + }; + const response = await complete( + model, + { + systemPrompt: "You are a concise assistant.", + messages: [ + { + role: "user", + content: "Reply with exactly: long cache retention accepted", + timestamp: Date.now(), + }, + ], + }, + options, + ); + + expect(response.errorMessage, response.errorMessage).toBeFalsy(); + expect(response.stopReason, response.errorMessage).not.toBe("error"); +} + +describe("Anthropic Messages long cache retention E2E", () => { + it("covers every generated anthropic-messages model", () => { + const expectedModels = getProviders().flatMap((provider) => + getAnthropicMessagesModels(provider).map((model) => `${provider}/${model.id}`), + ); + expect(anthropicMessagesCases.map((testCase) => testCase.name).sort()).toEqual(expectedModels.sort()); + }); + + describe("forced long cache retention probe", () => { + for (const testCase of probeCases) { + const model = withLongCacheRetention(testCase.model); + + it.skipIf(!testCase.apiKey)(`${testCase.name} accepts long cache retention`, { retry: 2 }, async () => { + await expectLongCacheRetentionAccepted(model, testCase.apiKey); + }); + } + }); +}); diff --git a/packages/ai/test/cache-retention.test.ts b/packages/ai/test/cache-retention.test.ts index b517d0ba..2f234ab0 100644 --- a/packages/ai/test/cache-retention.test.ts +++ b/packages/ai/test/cache-retention.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { getModel } from "../src/models.js"; import { stream } from "../src/stream.js"; -import type { Context } from "../src/types.js"; +import type { Context, Model } from "../src/types.js"; describe("Cache Retention (PI_CACHE_RETENTION)", () => { const originalEnv = process.env.PI_CACHE_RETENTION; @@ -70,7 +70,7 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { expect(capturedPayload.system[0].cache_control).toEqual({ type: "ephemeral", ttl: "1h" }); }); - it("should not add ttl when baseUrl is not api.anthropic.com", async () => { + it("should add ttl for non-api.anthropic.com baseUrl by default", async () => { process.env.PI_CACHE_RETENTION = "long"; // Create a model with a different baseUrl (simulating a proxy) @@ -106,11 +106,39 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { // Expected to fail } - // The payload should have been captured before the error - if (capturedPayload) { - // System prompt should have cache_control WITHOUT ttl (proxy URL) - expect(capturedPayload.system[0].cache_control).toEqual({ type: "ephemeral" }); + expect(capturedPayload).not.toBeNull(); + expect(capturedPayload.system[0].cache_control).toEqual({ type: "ephemeral", ttl: "1h" }); + }); + + it("should omit ttl when supportsLongCacheRetention is false", async () => { + const baseModel = getModel("anthropic", "claude-haiku-4-5"); + const proxyModel = { + ...baseModel, + baseUrl: "https://my-proxy.example.com/v1", + compat: { supportsLongCacheRetention: false }, + }; + let capturedPayload: any = null; + + const { streamAnthropic } = await import("../src/providers/anthropic.js"); + + try { + const s = streamAnthropic(proxyModel, context, { + apiKey: "fake-key", + cacheRetention: "long", + onPayload: (payload) => { + capturedPayload = payload; + }, + }); + + for await (const event of s) { + if (event.type === "error") break; + } + } catch { + // Expected to fail } + + expect(capturedPayload).not.toBeNull(); + expect(capturedPayload.system[0].cache_control).toEqual({ type: "ephemeral" }); }); it("should omit cache_control when cacheRetention is none", async () => { @@ -240,7 +268,7 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { }, ); - it("should not set prompt_cache_retention when baseUrl is not api.openai.com", async () => { + it("should set prompt_cache_retention for non-api.openai.com baseUrl by default", async () => { process.env.PI_CACHE_RETENTION = "long"; // Create a model with a different baseUrl (simulating a proxy) @@ -270,10 +298,38 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { // Expected to fail } - // The payload should have been captured before the error - if (capturedPayload) { - expect(capturedPayload.prompt_cache_retention).toBeUndefined(); + expect(capturedPayload).not.toBeNull(); + expect(capturedPayload.prompt_cache_retention).toBe("24h"); + }); + + it("should omit prompt_cache_retention when supportsLongCacheRetention is false", async () => { + const model = { + ...getModel("openai", "gpt-4o-mini"), + compat: { supportsLongCacheRetention: false }, + }; + let capturedPayload: any = null; + + const { streamOpenAIResponses } = await import("../src/providers/openai-responses.js"); + + try { + const s = streamOpenAIResponses(model, context, { + apiKey: "fake-key", + cacheRetention: "long", + sessionId: "session-compat-false", + onPayload: (payload) => { + capturedPayload = payload; + }, + }); + + for await (const event of s) { + if (event.type === "error") break; + } + } catch { + // Expected to fail } + + expect(capturedPayload).not.toBeNull(); + expect(capturedPayload.prompt_cache_retention).toBeUndefined(); }); it("should omit prompt_cache_key when cacheRetention is none", async () => { @@ -332,4 +388,74 @@ describe("Cache Retention (PI_CACHE_RETENTION)", () => { expect(capturedPayload.prompt_cache_retention).toBe("24h"); }); }); + + describe("OpenAI Completions Provider", () => { + function createCompletionsModel(compat?: Model<"openai-completions">["compat"]): Model<"openai-completions"> { + return { + id: "test-model", + name: "Test Model", + api: "openai-completions", + provider: "test-openai-completions", + baseUrl: "https://my-proxy.example.com/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + compat, + }; + } + + it("should set prompt_cache_retention for non-api.openai.com baseUrl by default", async () => { + let capturedPayload: any = null; + const { streamOpenAICompletions } = await import("../src/providers/openai-completions.js"); + + try { + const s = streamOpenAICompletions(createCompletionsModel(), context, { + apiKey: "fake-key", + cacheRetention: "long", + sessionId: "session-completions", + onPayload: (payload) => { + capturedPayload = payload; + }, + }); + + for await (const event of s) { + if (event.type === "error") break; + } + } catch { + // Expected to fail + } + + expect(capturedPayload).not.toBeNull(); + expect(capturedPayload.prompt_cache_key).toBe("session-completions"); + expect(capturedPayload.prompt_cache_retention).toBe("24h"); + }); + + it("should omit prompt_cache_retention when supportsLongCacheRetention is false", async () => { + let capturedPayload: any = null; + const { streamOpenAICompletions } = await import("../src/providers/openai-completions.js"); + + try { + const s = streamOpenAICompletions(createCompletionsModel({ supportsLongCacheRetention: false }), context, { + apiKey: "fake-key", + cacheRetention: "long", + sessionId: "session-completions-false", + onPayload: (payload) => { + capturedPayload = payload; + }, + }); + + for await (const event of s) { + if (event.type === "error") break; + } + } catch { + // Expected to fail + } + + expect(capturedPayload).not.toBeNull(); + expect(capturedPayload.prompt_cache_key).toBeUndefined(); + expect(capturedPayload.prompt_cache_retention).toBeUndefined(); + }); + }); }); diff --git a/packages/ai/test/openai-completions-thinking-as-text.test.ts b/packages/ai/test/openai-completions-thinking-as-text.test.ts index 9ce40eef..f8d87b3f 100644 --- a/packages/ai/test/openai-completions-thinking-as-text.test.ts +++ b/packages/ai/test/openai-completions-thinking-as-text.test.ts @@ -38,6 +38,7 @@ const compat = { supportsStrictMode: true, cacheControlFormat: undefined, sendSessionAffinityHeaders: false, + supportsLongCacheRetention: true, } satisfies Required> & { cacheControlFormat?: OpenAICompletionsCompat["cacheControlFormat"]; }; diff --git a/packages/ai/test/openai-completions-tool-result-images.test.ts b/packages/ai/test/openai-completions-tool-result-images.test.ts index 4f912278..ac83b652 100644 --- a/packages/ai/test/openai-completions-tool-result-images.test.ts +++ b/packages/ai/test/openai-completions-tool-result-images.test.ts @@ -36,6 +36,7 @@ const compat: Required = { supportsStrictMode: true, cacheControlFormat: "anthropic", sendSessionAffinityHeaders: false, + supportsLongCacheRetention: true, }; function buildToolResult(toolCallId: string, timestamp: number): ToolResultMessage { diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 061f8de9..a843be21 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -24,6 +24,7 @@ ### Fixed +- Fixed `models.json` provider compatibility to accept `compat.supportsLongCacheRetention`, allowing proxies to opt out of long-retention cache fields when needed while long retention is enabled by default when requested ([#3543](https://github.com/badlogic/pi-mono/issues/3543)) - Fixed `--thinking xhigh` for `openai-codex` `gpt-5.5` so it is no longer downgraded to `high`. - Fixed git package installs with custom `npmCommand` values such as `pnpm` by avoiding npm-specific production flags in that compatibility path ([#3604](https://github.com/badlogic/pi-mono/issues/3604)) - Fixed first user messages rendering without spacing after existing notices such as compaction summaries or status messages ([#3613](https://github.com/badlogic/pi-mono/issues/3613)) diff --git a/packages/coding-agent/docs/models.md b/packages/coding-agent/docs/models.md index 43c0e45f..84d6695f 100644 --- a/packages/coding-agent/docs/models.md +++ b/packages/coding-agent/docs/models.md @@ -284,7 +284,8 @@ By default pi sends per-tool `eager_input_streaming: true`. If a proxy or Anthro "api": "anthropic-messages", "apiKey": "ANTHROPIC_PROXY_KEY", "compat": { - "supportsEagerToolInputStreaming": false + "supportsEagerToolInputStreaming": false, + "supportsLongCacheRetention": true }, "models": [ { @@ -301,6 +302,7 @@ By default pi sends per-tool `eager_input_streaming: true`. If a proxy or Anthro | Field | Description | |-------|-------------| | `supportsEagerToolInputStreaming` | Whether the provider accepts per-tool `eager_input_streaming`. Default: `true`. Set to `false` to omit that field and use the legacy fine-grained tool streaming beta header on tool-enabled requests. | +| `supportsLongCacheRetention` | Whether the provider accepts Anthropic long cache retention (`cache_control.ttl: "1h"`) when cache retention is `long`. Default: `true`. | ## OpenAI Compatibility @@ -339,6 +341,7 @@ For providers with partial OpenAI compatibility, use the `compat` field. | `thinkingFormat` | Use `reasoning_effort`, `zai`, `qwen`, or `qwen-chat-template` thinking parameters | | `cacheControlFormat` | Use Anthropic-style `cache_control` markers on the system prompt, last tool definition, and last user/assistant text content. Currently only `anthropic` is supported. | | `supportsStrictMode` | Include the `strict` field in tool definitions | +| `supportsLongCacheRetention` | Whether the provider accepts long cache retention when cache retention is `long`: `prompt_cache_retention: "24h"` for OpenAI prompt caching, or `cache_control.ttl: "1h"` when `cacheControlFormat` is `anthropic`. Default: `true`. | | `openRouterRouting` | OpenRouter provider routing preferences. This object is sent as-is in the `provider` field of the [OpenRouter API request](https://openrouter.ai/docs/guides/routing/provider-selection). | | `vercelGatewayRouting` | Vercel AI Gateway routing config for provider selection (`only`, `order`) | diff --git a/packages/coding-agent/src/core/model-registry.ts b/packages/coding-agent/src/core/model-registry.ts index 01955447..0b5848fc 100644 --- a/packages/coding-agent/src/core/model-registry.ts +++ b/packages/coding-agent/src/core/model-registry.ts @@ -111,14 +111,17 @@ const OpenAICompletionsCompatSchema = Type.Object({ openRouterRouting: Type.Optional(OpenRouterRoutingSchema), vercelGatewayRouting: Type.Optional(VercelGatewayRoutingSchema), supportsStrictMode: Type.Optional(Type.Boolean()), + supportsLongCacheRetention: Type.Optional(Type.Boolean()), }); const OpenAIResponsesCompatSchema = Type.Object({ - // Reserved for future use + sendSessionIdHeader: Type.Optional(Type.Boolean()), + supportsLongCacheRetention: Type.Optional(Type.Boolean()), }); const AnthropicMessagesCompatSchema = Type.Object({ supportsEagerToolInputStreaming: Type.Optional(Type.Boolean()), + supportsLongCacheRetention: Type.Optional(Type.Boolean()), }); const ProviderCompatSchema = Type.Union([ diff --git a/packages/coding-agent/test/model-registry.test.ts b/packages/coding-agent/test/model-registry.test.ts index c58016ba..9ce5f204 100644 --- a/packages/coding-agent/test/model-registry.test.ts +++ b/packages/coding-agent/test/model-registry.test.ts @@ -461,6 +461,35 @@ describe("ModelRegistry", () => { expect(compat?.supportsEagerToolInputStreaming).toBe(false); }); + test("compat schema accepts long cache retention flag", () => { + writeRawModelsJson({ + demo: { + baseUrl: "https://example.com", + apiKey: "DEMO_KEY", + api: "anthropic-messages", + compat: { + supportsLongCacheRetention: false, + }, + models: [ + { + id: "demo-model", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1000, + maxTokens: 100, + }, + ], + }, + }); + + const registry = ModelRegistry.create(authStorage, modelsJsonPath); + const compat = registry.find("demo", "demo-model")?.compat as AnthropicMessagesCompat | undefined; + + expect(registry.getError()).toBeUndefined(); + expect(compat?.supportsLongCacheRetention).toBe(false); + }); + test("model-level baseUrl overrides provider-level baseUrl for custom models", () => { writeRawModelsJson({ "opencode-go": {