Armin Ronacher
@@ -2,6 +2,10 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- Added inherited configurable `chat-template` thinking support for OpenAI-compatible providers that use `chat_template_kwargs`, such as DeepSeek models behind vLLM ([#5673](https://github.com/earendil-works/pi/issues/5673)).
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed fuzzy `edit` matches to preserve untouched line blocks instead of rewriting the whole file through normalized content ([#5899](https://github.com/earendil-works/pi/issues/5899)).
|
||||
|
||||
@@ -229,7 +229,7 @@ models: [{
|
||||
}]
|
||||
```
|
||||
|
||||
Use `openrouter` for OpenRouter-style `reasoning: { effort }` controls. Use `together` for Together-style `reasoning: { enabled }` controls; with `supportsReasoningEffort`, it also sends `reasoning_effort`. Use `qwen-chat-template` instead for local Qwen-compatible servers that read `chat_template_kwargs.enable_thinking`.
|
||||
Use `openrouter` for OpenRouter-style `reasoning: { effort }` controls. Use `together` for Together-style `reasoning: { enabled }` controls; with `supportsReasoningEffort`, it also sends `reasoning_effort`. Use `qwen-chat-template` for local Qwen-compatible servers that read `chat_template_kwargs.enable_thinking` and need `preserve_thinking`.
|
||||
Use `cacheControlFormat: "anthropic"` for OpenAI-compatible providers that expose Anthropic-style prompt caching via `cache_control` on the system prompt, last tool definition, and last user/assistant text content.
|
||||
|
||||
For Anthropic-compatible providers using `api: "anthropic-messages"`, set `compat.forceAdaptiveThinking: true` on models or providers whose upstream model requires adaptive thinking (`thinking.type: "adaptive"` plus `output_config.effort`). Built-in adaptive Claude models set this automatically. Set `compat.allowEmptySignature: true` only for providers that emit empty thinking signatures and expect `signature: ""` on replay.
|
||||
@@ -718,7 +718,8 @@ interface ProviderModelConfig {
|
||||
requiresAssistantAfterToolResult?: boolean;
|
||||
requiresThinkingAsText?: boolean;
|
||||
requiresReasoningContentOnAssistantMessages?: boolean;
|
||||
thinkingFormat?: "openai" | "openrouter" | "deepseek" | "together" | "zai" | "qwen" | "qwen-chat-template";
|
||||
thinkingFormat?: "openai" | "openrouter" | "deepseek" | "together" | "zai" | "qwen" | "chat-template" | "qwen-chat-template" | "string-thinking" | "ant-ling";
|
||||
chatTemplateKwargs?: Record<string, string | number | boolean | null | { "$var": "thinking.enabled" | "thinking.effort"; omitWhenOff?: boolean }>;
|
||||
cacheControlFormat?: "anthropic";
|
||||
|
||||
// anthropic-messages
|
||||
@@ -732,5 +733,5 @@ interface ProviderModelConfig {
|
||||
}
|
||||
```
|
||||
|
||||
`openrouter` sends `reasoning: { effort }`. `deepseek` sends `thinking: { type: "enabled" | "disabled" }` and `reasoning_effort` when enabled. `together` sends `reasoning: { enabled }` and also `reasoning_effort` when `supportsReasoningEffort` is enabled. `qwen` is for DashScope-style top-level `enable_thinking`. Use `qwen-chat-template` for local Qwen-compatible servers that read `chat_template_kwargs.enable_thinking`.
|
||||
`openrouter` sends `reasoning: { effort }`. `deepseek` sends `thinking: { type: "enabled" | "disabled" }` and `reasoning_effort` when enabled. `together` sends `reasoning: { enabled }` and also `reasoning_effort` when `supportsReasoningEffort` is enabled. `qwen` is for DashScope-style top-level `enable_thinking`. Use `qwen-chat-template` for local Qwen-compatible servers that read `chat_template_kwargs.enable_thinking` and need `preserve_thinking`. Use `chat-template` for configurable `chat_template_kwargs`, for example DeepSeek V3.x behind vLLM with `chatTemplateKwargs: { "thinking": { "$var": "thinking.enabled" } }`.
|
||||
`cacheControlFormat: "anthropic"` applies Anthropic-style `cache_control` markers to the system prompt, last tool definition, and last user/assistant text content.
|
||||
|
||||
@@ -399,14 +399,15 @@ For providers with partial OpenAI compatibility, use the `compat` field.
|
||||
| `requiresAssistantAfterToolResult` | Insert an assistant message before a user message after tool results |
|
||||
| `requiresThinkingAsText` | Convert thinking blocks to plain text |
|
||||
| `requiresReasoningContentOnAssistantMessages` | Include empty `reasoning_content` on all replayed assistant messages when reasoning is enabled |
|
||||
| `thinkingFormat` | Use `reasoning_effort`, `openrouter`, `deepseek`, `together`, `zai`, `qwen`, or `qwen-chat-template` thinking parameters |
|
||||
| `thinkingFormat` | Use `reasoning_effort`, `openrouter`, `deepseek`, `together`, `zai`, `qwen`, `chat-template`, or `qwen-chat-template` thinking parameters |
|
||||
| `chatTemplateKwargs` | `chat_template_kwargs` values for `thinkingFormat: "chat-template"`; use `{ "$var": "thinking.enabled" }` or `{ "$var": "thinking.effort" }` for pi-controlled thinking values |
|
||||
| `cacheControlFormat` | Use Anthropic-style `cache_control` markers on the system prompt, last tool definition, and last user/assistant text content. Currently only `anthropic` is supported. |
|
||||
| `supportsStrictMode` | Include the `strict` field in tool definitions |
|
||||
| `supportsLongCacheRetention` | Whether the provider accepts long cache retention when cache retention is `long`: `prompt_cache_retention: "24h"` for OpenAI prompt caching, or `cache_control.ttl: "1h"` when `cacheControlFormat` is `anthropic`. Default: `true`. |
|
||||
| `openRouterRouting` | OpenRouter provider routing preferences. This object is sent as-is in the `provider` field of the [OpenRouter API request](https://openrouter.ai/docs/guides/routing/provider-selection). |
|
||||
| `vercelGatewayRouting` | Vercel AI Gateway routing config for provider selection (`only`, `order`) |
|
||||
|
||||
`openrouter` uses `reasoning: { effort }`. `together` uses `reasoning: { enabled }` and also `reasoning_effort` when `supportsReasoningEffort` is enabled. `qwen` uses top-level `enable_thinking`. Use `qwen-chat-template` for local Qwen-compatible servers that require `chat_template_kwargs.enable_thinking`.
|
||||
`openrouter` uses `reasoning: { effort }`. `together` uses `reasoning: { enabled }` and also `reasoning_effort` when `supportsReasoningEffort` is enabled. `qwen` uses top-level `enable_thinking`. Use `qwen-chat-template` for local Qwen-compatible servers that require `chat_template_kwargs.enable_thinking` and `preserve_thinking`. Use `chat-template` for vLLM/Hugging Face chat templates that need configurable `chat_template_kwargs`, such as `chatTemplateKwargs: { "thinking": { "$var": "thinking.enabled" } }` for DeepSeek V3.x templates.
|
||||
|
||||
`cacheControlFormat: "anthropic"` is for OpenAI-compatible providers that expose Anthropic-style prompt caching through `cache_control` markers on text content and tool definitions.
|
||||
|
||||
|
||||
@@ -96,6 +96,13 @@ const ThinkingLevelMapSchema = Type.Object({
|
||||
xhigh: Type.Optional(ThinkingLevelMapValueSchema),
|
||||
});
|
||||
|
||||
const ChatTemplateKwargScalarSchema = Type.Union([Type.String(), Type.Number(), Type.Boolean(), Type.Null()]);
|
||||
const ChatTemplateKwargVariableSchema = Type.Object({
|
||||
$var: Type.Union([Type.Literal("thinking.enabled"), Type.Literal("thinking.effort")]),
|
||||
omitWhenOff: Type.Optional(Type.Boolean()),
|
||||
});
|
||||
const ChatTemplateKwargSchema = Type.Union([ChatTemplateKwargScalarSchema, ChatTemplateKwargVariableSchema]);
|
||||
|
||||
const OpenAICompletionsCompatSchema = Type.Object({
|
||||
supportsStore: Type.Optional(Type.Boolean()),
|
||||
supportsDeveloperRole: Type.Optional(Type.Boolean()),
|
||||
@@ -114,9 +121,13 @@ const OpenAICompletionsCompatSchema = Type.Object({
|
||||
Type.Literal("deepseek"),
|
||||
Type.Literal("zai"),
|
||||
Type.Literal("qwen"),
|
||||
Type.Literal("chat-template"),
|
||||
Type.Literal("qwen-chat-template"),
|
||||
Type.Literal("string-thinking"),
|
||||
Type.Literal("ant-ling"),
|
||||
]),
|
||||
),
|
||||
chatTemplateKwargs: Type.Optional(Type.Record(Type.String(), ChatTemplateKwargSchema)),
|
||||
cacheControlFormat: Type.Optional(Type.Literal("anthropic")),
|
||||
openRouterRouting: Type.Optional(OpenRouterRoutingSchema),
|
||||
vercelGatewayRouting: Type.Optional(VercelGatewayRoutingSchema),
|
||||
@@ -289,6 +300,13 @@ function mergeCompat(
|
||||
};
|
||||
}
|
||||
|
||||
if (baseCompletions?.chatTemplateKwargs || overrideCompletions.chatTemplateKwargs) {
|
||||
mergedCompletions.chatTemplateKwargs = {
|
||||
...baseCompletions?.chatTemplateKwargs,
|
||||
...overrideCompletions.chatTemplateKwargs,
|
||||
};
|
||||
}
|
||||
|
||||
return merged as Model<Api>["compat"];
|
||||
}
|
||||
|
||||
|
||||
@@ -434,6 +434,43 @@ describe("ModelRegistry", () => {
|
||||
expect(compat?.cacheControlFormat).toBe("anthropic");
|
||||
});
|
||||
|
||||
test("compat schema accepts chat template thinking configuration", () => {
|
||||
writeRawModelsJson({
|
||||
demo: {
|
||||
baseUrl: "https://example.com/v1",
|
||||
apiKey: "DEMO_KEY",
|
||||
api: "openai-completions",
|
||||
models: [
|
||||
{
|
||||
id: "demo-model",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 1000,
|
||||
maxTokens: 100,
|
||||
compat: {
|
||||
thinkingFormat: "chat-template",
|
||||
chatTemplateKwargs: {
|
||||
preserve_thinking: true,
|
||||
thinking: { $var: "thinking.enabled" },
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const registry = ModelRegistry.create(authStorage, modelsJsonPath);
|
||||
const compat = registry.find("demo", "demo-model")?.compat as OpenAICompletionsCompat | undefined;
|
||||
|
||||
expect(registry.getError()).toBeUndefined();
|
||||
expect(compat?.thinkingFormat).toBe("chat-template");
|
||||
expect(compat?.chatTemplateKwargs).toEqual({
|
||||
preserve_thinking: true,
|
||||
thinking: { $var: "thinking.enabled" },
|
||||
});
|
||||
});
|
||||
|
||||
test("compat schema accepts Anthropic eager tool input streaming flag", () => {
|
||||
writeRawModelsJson({
|
||||
demo: {
|
||||
|
||||
Reference in New Issue
Block a user