From 758ede4da037d51e9e1eb9f716e9e97ba6491282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kao=20F=C3=A9lix?= Date: Tue, 31 Mar 2026 13:28:24 +0100 Subject: [PATCH] Enable tool streaming for newer Z.ai models (#2732) --- packages/ai/scripts/generate-models.ts | 42 ++--- .../ai/src/providers/openai-completions.ts | 5 + packages/ai/src/types.ts | 2 + .../openai-completions-tool-choice.test.ts | 153 ++++++++++++++++++ ...nai-completions-tool-result-images.test.ts | 1 + 5 files changed, 183 insertions(+), 20 deletions(-) diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 468ab44e..fc908ccf 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -55,6 +55,7 @@ const COPILOT_STATIC_HEADERS = { const AI_GATEWAY_MODELS_URL = "https://ai-gateway.vercel.sh/v1"; const AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh"; +const ZAI_TOOL_STREAM_UNSUPPORTED_MODELS = new Set(["glm-4.5", "glm-4.5-air", "glm-4.5-flash", "glm-4.5v"]); async function fetchOpenRouterModels(): Promise[]> { try { @@ -379,28 +380,29 @@ async function loadModelsDevData(): Promise[]> { for (const [modelId, model] of Object.entries(data.zai.models)) { const m = model as ModelsDevModel; if (m.tool_call !== true) continue; - const supportsImage = m.modalities?.input?.includes("image") + const supportsImage = m.modalities?.input?.includes("image"); models.push({ - id: modelId, - name: m.name || modelId, - api: "openai-completions", - provider: "zai", - baseUrl: "https://api.z.ai/api/coding/paas/v4", - reasoning: m.reasoning === true, - input: supportsImage ? ["text", "image"] : ["text"], - cost: { - input: m.cost?.input || 0, - output: m.cost?.output || 0, - cacheRead: m.cost?.cache_read || 0, - cacheWrite: m.cost?.cache_write || 0, - }, - compat: { - supportsDeveloperRole: false, - thinkingFormat: "zai", - }, - contextWindow: m.limit?.context || 4096, - maxTokens: m.limit?.output || 4096, + id: modelId, + name: m.name || modelId, + api: "openai-completions", + provider: "zai", + baseUrl: "https://api.z.ai/api/coding/paas/v4", + reasoning: m.reasoning === true, + input: supportsImage ? ["text", "image"] : ["text"], + cost: { + input: m.cost?.input || 0, + output: m.cost?.output || 0, + cacheRead: m.cost?.cache_read || 0, + cacheWrite: m.cost?.cache_write || 0, + }, + compat: { + supportsDeveloperRole: false, + thinkingFormat: "zai", + ...(!ZAI_TOOL_STREAM_UNSUPPORTED_MODELS.has(modelId) ? { zaiToolStream: true } : {}), + }, + contextWindow: m.limit?.context || 4096, + maxTokens: m.limit?.output || 4096, }); } } diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index fc95f717..79446578 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -395,6 +395,9 @@ function buildParams(model: Model<"openai-completions">, context: Context, optio if (context.tools) { params.tools = convertTools(context.tools, compat); + if (compat.zaiToolStream) { + (params as any).tool_stream = true; + } } else if (hasToolHistory(context.messages)) { // Anthropic (via LiteLLM/proxy) requires tools param when conversation has tool_calls/tool_results params.tools = []; @@ -835,6 +838,7 @@ function detectCompat(model: Model<"openai-completions">): Required): Required { expect(params.reasoning_effort).toBe("medium"); }); + it("enables tool_stream for supported z.ai models with tools", async () => { + const model = getModel("zai", "glm-5")!; + const tools: Tool[] = [ + { + name: "ping", + description: "Ping tool", + parameters: Type.Object({ + ok: Type.Boolean(), + }), + }, + ]; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Call ping with ok=true", + timestamp: Date.now(), + }, + ], + tools, + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { tool_stream?: boolean }; + expect(params.tool_stream).toBe(true); + }); + + it("stores z.ai tool_stream support in model compat metadata", () => { + expect(getModel("zai", "glm-5")?.compat?.zaiToolStream).toBe(true); + expect(getModel("zai", "glm-4.7")?.compat?.zaiToolStream).toBe(true); + expect(getModel("zai", "glm-4.7-flash")?.compat?.zaiToolStream).toBe(true); + expect(getModel("zai", "glm-4.6v")?.compat?.zaiToolStream).toBe(true); + expect(getModel("zai", "glm-4.5-air")?.compat?.zaiToolStream).toBeUndefined(); + }); + + it("omits tool_stream for unsupported z.ai models", async () => { + const model = getModel("zai", "glm-4.5-air")!; + const tools: Tool[] = [ + { + name: "ping", + description: "Ping tool", + parameters: Type.Object({ + ok: Type.Boolean(), + }), + }, + ]; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Call ping with ok=true", + timestamp: Date.now(), + }, + ], + tools, + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { tool_stream?: boolean }; + expect(params.tool_stream).toBeUndefined(); + }); + + it("respects explicit z.ai tool_stream compat override", async () => { + const baseModel = getModel("zai", "glm-4.5-air")!; + const model = { + ...baseModel, + compat: { + ...baseModel.compat, + zaiToolStream: true, + }, + } as const; + const tools: Tool[] = [ + { + name: "ping", + description: "Ping tool", + parameters: Type.Object({ + ok: Type.Boolean(), + }), + }, + ]; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Call ping with ok=true", + timestamp: Date.now(), + }, + ], + tools, + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { tool_stream?: boolean }; + expect(params.tool_stream).toBe(true); + }); + + it("omits tool_stream when no tools are provided", async () => { + const model = getModel("zai", "glm-5")!; + let payload: unknown; + + await streamSimple( + model, + { + messages: [ + { + role: "user", + content: "Hi", + timestamp: Date.now(), + }, + ], + }, + { + apiKey: "test", + onPayload: (params: unknown) => { + payload = params; + }, + }, + ).result(); + + const params = (payload ?? mockState.lastParams) as { tool_stream?: boolean }; + expect(params.tool_stream).toBeUndefined(); + }); + it("maps non-standard provider finish_reason values to stopReason error", async () => { mockState.chunks = [ { diff --git a/packages/ai/test/openai-completions-tool-result-images.test.ts b/packages/ai/test/openai-completions-tool-result-images.test.ts index 5161a139..6100547a 100644 --- a/packages/ai/test/openai-completions-tool-result-images.test.ts +++ b/packages/ai/test/openai-completions-tool-result-images.test.ts @@ -32,6 +32,7 @@ const compat: Required = { thinkingFormat: "openai", openRouterRouting: {}, vercelGatewayRouting: {}, + zaiToolStream: false, supportsStrictMode: true, };