fix(ai): add session affinity and compat fixes for Fireworks provider caching

Fireworks prompt caching is enabled by default (automatic prefix matching), but on serverless infrastructure, requests hit random replicas. Without session affinity, the per-replica cache misses, negating cache hit rates and the discounted cacheRead pricing. Changes: - Add sendSessionAffinityHeaders and supportsCacheControlOnTools to AnthropicMessagesCompat interface - Send x-session-affinity header for Fireworks (and Cloudflare AI Gateway Anthropic) when sessionId is available and caching is enabled - Omit cache_control on tool definitions for Fireworks (unsupported per https://docs.fireworks.ai/tools-sdks/anthropic-compatibility) - Default supportsEagerToolInputStreaming to false for Fireworks (unsupported field) - Default supportsLongCacheRetention to false for Fireworks (cache_control.ttl not supported) - Add compat settings to Fireworks models in generate-models.ts - Update generated models with Fireworks compat settings - Add integration tests for session affinity and tool compat Refs: https://docs.fireworks.ai/guides/prompt-caching Refs: https://docs.fireworks.ai/tools-sdks/anthropic-compatibility
2026-05-10 00:11:36 +02:00
parent fe6b85b32c
commit 99dc6fcec8
5 changed files with 358 additions and 4 deletions
--- a/packages/ai/src/models.generated.ts
+++ b/packages/ai/src/models.generated.ts
@@ -3614,6 +3614,12 @@ export const MODELS = {
 			},
 			contextWindow: 163840,
 			maxTokens: 163840,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/deepseek-v3p2": {
 			id: "accounts/fireworks/models/deepseek-v3p2",
@@ -3631,6 +3637,12 @@ export const MODELS = {
 			},
 			contextWindow: 160000,
 			maxTokens: 160000,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/deepseek-v4-pro": {
 			id: "accounts/fireworks/models/deepseek-v4-pro",
@@ -3648,6 +3660,12 @@ export const MODELS = {
 			},
 			contextWindow: 1000000,
 			maxTokens: 384000,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/glm-4p5": {
 			id: "accounts/fireworks/models/glm-4p5",
@@ -3665,6 +3683,12 @@ export const MODELS = {
 			},
 			contextWindow: 131072,
 			maxTokens: 131072,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/glm-4p5-air": {
 			id: "accounts/fireworks/models/glm-4p5-air",
@@ -3682,6 +3706,12 @@ export const MODELS = {
 			},
 			contextWindow: 131072,
 			maxTokens: 131072,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/glm-4p7": {
 			id: "accounts/fireworks/models/glm-4p7",
@@ -3699,6 +3729,12 @@ export const MODELS = {
 			},
 			contextWindow: 198000,
 			maxTokens: 198000,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/glm-5": {
 			id: "accounts/fireworks/models/glm-5",
@@ -3716,6 +3752,12 @@ export const MODELS = {
 			},
 			contextWindow: 202752,
 			maxTokens: 131072,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/glm-5p1": {
 			id: "accounts/fireworks/models/glm-5p1",
@@ -3733,6 +3775,12 @@ export const MODELS = {
 			},
 			contextWindow: 202800,
 			maxTokens: 131072,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/gpt-oss-120b": {
 			id: "accounts/fireworks/models/gpt-oss-120b",
@@ -3750,6 +3798,12 @@ export const MODELS = {
 			},
 			contextWindow: 131072,
 			maxTokens: 32768,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/gpt-oss-20b": {
 			id: "accounts/fireworks/models/gpt-oss-20b",
@@ -3767,6 +3821,12 @@ export const MODELS = {
 			},
 			contextWindow: 131072,
 			maxTokens: 32768,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/kimi-k2-instruct": {
 			id: "accounts/fireworks/models/kimi-k2-instruct",
@@ -3784,6 +3844,12 @@ export const MODELS = {
 			},
 			contextWindow: 128000,
 			maxTokens: 16384,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/kimi-k2-thinking": {
 			id: "accounts/fireworks/models/kimi-k2-thinking",
@@ -3801,6 +3867,12 @@ export const MODELS = {
 			},
 			contextWindow: 256000,
 			maxTokens: 256000,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/kimi-k2p5": {
 			id: "accounts/fireworks/models/kimi-k2p5",
@@ -3818,6 +3890,12 @@ export const MODELS = {
 			},
 			contextWindow: 256000,
 			maxTokens: 256000,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/kimi-k2p6": {
 			id: "accounts/fireworks/models/kimi-k2p6",
@@ -3835,6 +3913,12 @@ export const MODELS = {
 			},
 			contextWindow: 262000,
 			maxTokens: 262000,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/minimax-m2p1": {
 			id: "accounts/fireworks/models/minimax-m2p1",
@@ -3852,6 +3936,12 @@ export const MODELS = {
 			},
 			contextWindow: 200000,
 			maxTokens: 200000,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/minimax-m2p5": {
 			id: "accounts/fireworks/models/minimax-m2p5",
@@ -3869,6 +3959,12 @@ export const MODELS = {
 			},
 			contextWindow: 196608,
 			maxTokens: 196608,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/minimax-m2p7": {
 			id: "accounts/fireworks/models/minimax-m2p7",
@@ -3886,6 +3982,12 @@ export const MODELS = {
 			},
 			contextWindow: 196608,
 			maxTokens: 196608,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/models/qwen3p6-plus": {
 			id: "accounts/fireworks/models/qwen3p6-plus",
@@ -3903,6 +4005,12 @@ export const MODELS = {
 			},
 			contextWindow: 128000,
 			maxTokens: 8192,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 		"accounts/fireworks/routers/kimi-k2p5-turbo": {
 			id: "accounts/fireworks/routers/kimi-k2p5-turbo",
@@ -3920,6 +4028,12 @@ export const MODELS = {
 			},
 			contextWindow: 256000,
 			maxTokens: 256000,
+			compat: {
+				sendSessionAffinityHeaders: true,
+				supportsEagerToolInputStreaming: false,
+				supportsCacheControlOnTools: false,
+				supportsLongCacheRetention: false,
+			},
 		} satisfies Model<"anthropic-messages">,
 	},
 	"github-copilot": {
--- a/packages/ai/src/providers/anthropic.ts
+++ b/packages/ai/src/providers/anthropic.ts
@@ -165,9 +165,16 @@ const FINE_GRAINED_TOOL_STREAMING_BETA = "fine-grained-tool-streaming-2025-05-14
 const INTERLEAVED_THINKING_BETA = "interleaved-thinking-2025-05-14";

 function getAnthropicCompat(model: Model<"anthropic-messages">): Required<AnthropicMessagesCompat> {
+	// Auto-detect session affinity and cache control support from provider
+	const isFireworks = model.provider === "fireworks";
+	const isCloudflareAiGatewayAnthropic =
+		model.provider === "cloudflare-ai-gateway" && model.baseUrl.includes("anthropic");
 	return {
-		supportsEagerToolInputStreaming: model.compat?.supportsEagerToolInputStreaming ?? true,
-		supportsLongCacheRetention: model.compat?.supportsLongCacheRetention ?? true,
+		supportsEagerToolInputStreaming: model.compat?.supportsEagerToolInputStreaming ?? !isFireworks,
+		supportsLongCacheRetention: model.compat?.supportsLongCacheRetention ?? !isFireworks,
+		sendSessionAffinityHeaders:
+			model.compat?.sendSessionAffinityHeaders ?? !!(isFireworks || isCloudflareAiGatewayAnthropic),
+		supportsCacheControlOnTools: model.compat?.supportsCacheControlOnTools ?? !isFireworks,
 	};
 }

@@ -463,6 +470,9 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
 					});
 				}

+				const cacheRetention = options?.cacheRetention ?? resolveCacheRetention();
+				const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId;
+
 				const created = createClient(
 					model,
 					apiKey,
@@ -470,6 +480,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti
 					shouldUseFineGrainedToolStreamingBeta(model, context),
 					options?.headers,
 					copilotDynamicHeaders,
+					cacheSessionId,
 				);
 				client = created.client;
 				isOAuth = created.isOAuthToken;
@@ -766,6 +777,7 @@ function createClient(
 	useFineGrainedToolStreamingBeta: boolean,
 	optionsHeaders?: Record<string, string>,
 	dynamicHeaders?: Record<string, string>,
+	sessionId?: string,
 ): { client: Anthropic; isOAuthToken: boolean } {
 	// Adaptive thinking models (Opus 4.6, Sonnet 4.6) have interleaved thinking built-in.
 	// The beta header is deprecated on Opus 4.6 and redundant on Sonnet 4.6, so skip it.
@@ -847,6 +859,8 @@ function createClient(
 	}

 	// API key auth
+	const sessionAffinityHeaders: Record<string, string | null> =
+		sessionId && getAnthropicCompat(model).sendSessionAffinityHeaders ? { "x-session-affinity": sessionId } : {};
 	const client = new Anthropic({
 		apiKey,
 		baseURL: model.baseUrl,
@@ -857,6 +871,7 @@ function createClient(
 				"anthropic-dangerous-direct-browser-access": "true",
 				...(betaFeatures.length > 0 ? { "anthropic-beta": betaFeatures.join(",") } : {}),
 			},
+			sessionAffinityHeaders,
 			model.headers,
 			optionsHeaders,
 		),
@@ -912,11 +927,12 @@ function buildParams(
 	}

 	if (context.tools && context.tools.length > 0) {
+		const compat = getAnthropicCompat(model);
 		params.tools = convertTools(
 			context.tools,
 			isOAuthToken,
-			getAnthropicCompat(model).supportsEagerToolInputStreaming,
-			cacheControl,
+			compat.supportsEagerToolInputStreaming,
+			compat.supportsCacheControlOnTools ? cacheControl : undefined,
 		);
 	}

--- a/packages/ai/src/types.ts
+++ b/packages/ai/src/types.ts
@@ -419,6 +419,22 @@ export interface AnthropicMessagesCompat {
 	supportsEagerToolInputStreaming?: boolean;
 	/** Whether the provider supports Anthropic long cache retention (`cache_control.ttl: "1h"`). Default: true. */
 	supportsLongCacheRetention?: boolean;
+	/**
+	 * Whether to send the `x-session-affinity` header from `options.sessionId`
+	 * when caching is enabled. Required for providers like Fireworks that use
+	 * session affinity for prompt cache routing (requests to the same replica
+	 * maximize cache hits).
+	 * Default: false.
+	 */
+	sendSessionAffinityHeaders?: boolean;
+	/**
+	 * Whether the provider supports Anthropic-style `cache_control` markers on
+	 * tool definitions. When false, `cache_control` is omitted from tool params.
+	 * Some Anthropic-compatible providers (e.g., Fireworks) do not support this
+	 * field on tools and may reject or ignore it.
+	 * Default: true.
+	 */
+	supportsCacheControlOnTools?: boolean;
 }

 /**