From a5fac1ef09179afa54939311819c63da490dfdaa Mon Sep 17 00:00:00 2001 From: wirjo Date: Sun, 19 Apr 2026 16:48:43 +1000 Subject: [PATCH] feat(amazon-bedrock): conditionally omit maxTokens from inferenceConfig (#3400) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When model.maxTokens is 0 (unset/unknown), omit maxTokens from the Bedrock ConverseStream inferenceConfig instead of sending 0. Bedrock's inferenceConfig.maxTokens is optional — when omitted, the model uses its own internal default. This is optimal for Bedrock's token quota management: - At request start, Bedrock reserves input_tokens + max_tokens from your TPM quota - For Claude 3.7+ models, output tokens have a 5x burndown rate - Sending an unnecessarily high maxTokens wastes TPM capacity during the reservation window, reducing concurrent request throughput - Omitting it lets Bedrock use the model default (~4096 for Claude), matching expected output sizes and maximizing quota utilization Also applies the same conditional pattern to temperature — only include it in inferenceConfig when explicitly set. Changes: - simple-options.ts: Use ?? instead of || so explicit 0 is preserved; return undefined when model.maxTokens is 0 (unset) - amazon-bedrock.ts: Spread maxTokens and temperature into inferenceConfig only when defined Fixes #3399 --- packages/ai/src/providers/amazon-bedrock.ts | 5 ++++- packages/ai/src/providers/simple-options.ts | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/ai/src/providers/amazon-bedrock.ts b/packages/ai/src/providers/amazon-bedrock.ts index c6aa377e..dd8d6123 100644 --- a/packages/ai/src/providers/amazon-bedrock.ts +++ b/packages/ai/src/providers/amazon-bedrock.ts @@ -188,7 +188,10 @@ export const streamBedrock: StreamFunction<"bedrock-converse-stream", BedrockOpt modelId: model.id, messages: convertMessages(context, model, cacheRetention), system: buildSystemPrompt(context.systemPrompt, model, cacheRetention), - inferenceConfig: { maxTokens: options.maxTokens, temperature: options.temperature }, + inferenceConfig: { + ...(options.maxTokens !== undefined && { maxTokens: options.maxTokens }), + ...(options.temperature !== undefined && { temperature: options.temperature }), + }, toolConfig: convertToolConfig(context.tools, options.toolChoice), additionalModelRequestFields: buildAdditionalModelRequestFields(model, options), ...(options.requestMetadata !== undefined && { requestMetadata: options.requestMetadata }), diff --git a/packages/ai/src/providers/simple-options.ts b/packages/ai/src/providers/simple-options.ts index 6e478e64..35524aa8 100644 --- a/packages/ai/src/providers/simple-options.ts +++ b/packages/ai/src/providers/simple-options.ts @@ -3,7 +3,7 @@ import type { Api, Model, SimpleStreamOptions, StreamOptions, ThinkingBudgets, T export function buildBaseOptions(model: Model, options?: SimpleStreamOptions, apiKey?: string): StreamOptions { return { temperature: options?.temperature, - maxTokens: options?.maxTokens || Math.min(model.maxTokens, 32000), + maxTokens: options?.maxTokens ?? (model.maxTokens > 0 ? Math.min(model.maxTokens, 32000) : undefined), signal: options?.signal, apiKey: apiKey || options?.apiKey, cacheRetention: options?.cacheRetention,