From 2787b601d7cec86a82541e3c6b876748deca880d Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 19 May 2026 11:49:44 +0200 Subject: [PATCH] fix(ai): stop defaulting max token request caps closes #4675 --- packages/ai/CHANGELOG.md | 1 + packages/ai/src/providers/amazon-bedrock.ts | 4 +- packages/ai/src/providers/anthropic.ts | 6 ++- packages/ai/src/providers/simple-options.ts | 20 +++------- .../ai/test/github-copilot-anthropic.test.ts | 2 +- .../openai-completions-empty-tools.test.ts | 38 ++++++++++++++++++- 6 files changed, 51 insertions(+), 20 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 39a71698..001a10a2 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed +- Fixed OpenAI-compatible `streamSimple()` requests to stop sending model-derived default output token caps, avoiding context-window reservation failures on servers such as vLLM while preserving explicit `maxTokens` and required Anthropic `max_tokens` handling ([#4675](https://github.com/earendil-works/pi/issues/4675)). - Fixed OpenAI prompt cache keys to clamp session-derived values to the 64-character API limit across OpenAI Responses, Chat Completions, Codex Responses, and Azure OpenAI Responses ([#4720](https://github.com/earendil-works/pi/issues/4720)). ## [0.75.3] - 2026-05-18 diff --git a/packages/ai/src/providers/amazon-bedrock.ts b/packages/ai/src/providers/amazon-bedrock.ts index a48b5897..6e616759 100644 --- a/packages/ai/src/providers/amazon-bedrock.ts +++ b/packages/ai/src/providers/amazon-bedrock.ts @@ -314,8 +314,10 @@ export const streamSimpleBedrock: StreamFunction<"bedrock-converse-stream", Simp } satisfies BedrockOptions); } + // Undefined means the caller did not request an output cap; let the helper use the model cap. + // Do not coerce to 0 here, or the thinking budget would become the entire maxTokens value. const adjusted = adjustMaxTokensForThinking( - base.maxTokens || 0, + base.maxTokens, model.maxTokens, options.reasoning, options.thinkingBudgets, diff --git a/packages/ai/src/providers/anthropic.ts b/packages/ai/src/providers/anthropic.ts index 4079b149..29f51df9 100644 --- a/packages/ai/src/providers/anthropic.ts +++ b/packages/ai/src/providers/anthropic.ts @@ -751,8 +751,10 @@ export const streamSimpleAnthropic: StreamFunction<"anthropic-messages", SimpleS } satisfies AnthropicOptions); } + // Undefined means the caller did not request an output cap; let the helper use the model cap. + // Do not coerce to 0 here, or the thinking budget would become the entire max_tokens value. const adjusted = adjustMaxTokensForThinking( - base.maxTokens || 0, + base.maxTokens, model.maxTokens, options.reasoning, options.thinkingBudgets, @@ -891,7 +893,7 @@ function buildParams( const params: MessageCreateParamsStreaming = { model: model.id, messages: convertMessages(context.messages, model, isOAuthToken, cacheControl), - max_tokens: options?.maxTokens || (model.maxTokens / 3) | 0, + max_tokens: options?.maxTokens ?? model.maxTokens, stream: true, }; diff --git a/packages/ai/src/providers/simple-options.ts b/packages/ai/src/providers/simple-options.ts index d9dcc348..1822ebb1 100644 --- a/packages/ai/src/providers/simple-options.ts +++ b/packages/ai/src/providers/simple-options.ts @@ -1,19 +1,9 @@ import type { Api, Model, SimpleStreamOptions, StreamOptions, ThinkingBudgets, ThinkingLevel } from "../types.js"; -const DEFAULT_MAX_OUTPUT_TOKENS = 32000; -const CONTEXT_WINDOW_OUTPUT_TOLERANCE = 1024; - -export function buildBaseOptions(model: Model, options?: SimpleStreamOptions, apiKey?: string): StreamOptions { - const defaultMaxTokens = - model.maxTokens > 0 - ? model.maxTokens >= model.contextWindow - CONTEXT_WINDOW_OUTPUT_TOLERANCE - ? Math.min(model.maxTokens, DEFAULT_MAX_OUTPUT_TOKENS) - : model.maxTokens - : undefined; - +export function buildBaseOptions(_model: Model, options?: SimpleStreamOptions, apiKey?: string): StreamOptions { return { temperature: options?.temperature, - maxTokens: options?.maxTokens ?? defaultMaxTokens, + maxTokens: options?.maxTokens, signal: options?.signal, apiKey: apiKey || options?.apiKey, transport: options?.transport, @@ -34,7 +24,8 @@ export function clampReasoning(effort: ThinkingLevel | undefined): Exclude { const params = mockState.createParams!; expect(params.model).toBe("claude-sonnet-4.6"); expect(params.stream).toBe(true); - expect(params.max_tokens).toBeGreaterThan(0); + expect(params.max_tokens).toBe(model.maxTokens); expect(Array.isArray(params.messages)).toBe(true); }); diff --git a/packages/ai/test/openai-completions-empty-tools.test.ts b/packages/ai/test/openai-completions-empty-tools.test.ts index 8197efb0..8e5d49c7 100644 --- a/packages/ai/test/openai-completions-empty-tools.test.ts +++ b/packages/ai/test/openai-completions-empty-tools.test.ts @@ -93,6 +93,40 @@ describe("openai-completions empty tools handling", () => { expect("tools" in (params as object)).toBe(false); }); + it("does not send default max token fields", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }, + { apiKey: "test" }, + ).result(); + + const params = mockState.lastParams as { max_tokens?: number; max_completion_tokens?: number }; + expect(params.max_tokens).toBeUndefined(); + expect(params.max_completion_tokens).toBeUndefined(); + }); + + it("sends explicit maxTokens", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + + await streamSimple( + model, + { + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }, + { apiKey: "test", maxTokens: 1234 }, + ).result(); + + const params = mockState.lastParams as { max_tokens?: number; max_completion_tokens?: number }; + expect(params.max_tokens).toBeUndefined(); + expect(params.max_completion_tokens).toBe(1234); + }); + it("uses conservative OpenAI-compatible fields for Cloudflare AI Gateway /compat models", async () => { process.env.CLOUDFLARE_ACCOUNT_ID = "account-id"; process.env.CLOUDFLARE_GATEWAY_ID = "gateway-id"; @@ -104,7 +138,7 @@ describe("openai-completions empty tools handling", () => { systemPrompt: "You are helpful.", messages: [{ role: "user", content: "hi", timestamp: Date.now() }], }, - { apiKey: "test", reasoning: "high" }, + { apiKey: "test", maxTokens: 1234, reasoning: "high" }, ).result(); const params = mockState.lastParams as { @@ -115,7 +149,7 @@ describe("openai-completions empty tools handling", () => { store?: boolean; }; expect(params.messages[0].role).toBe("system"); - expect(params.max_tokens).toBeDefined(); + expect(params.max_tokens).toBe(1234); expect(params.max_completion_tokens).toBeUndefined(); expect(params.reasoning_effort).toBeUndefined(); expect(params.store).toBeUndefined();