From b63d26332f35b9b8456f4abd3bc03e2d4bf2cc3e Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Thu, 28 May 2026 12:34:02 +0200 Subject: [PATCH] Finish harness tool registry semantics --- packages/agent/docs/agent-harness.md | 9 +- packages/agent/docs/durable-harness.md | 37 +++++- packages/agent/src/harness/agent-harness.ts | 107 ++++++++++++++---- .../compaction/branch-summarization.ts | 1 + .../src/harness/compaction/compaction.ts | 1 + packages/agent/src/harness/session/session.ts | 16 ++- packages/agent/src/harness/types.ts | 36 ++++-- .../agent/test/harness/agent-harness.test.ts | 106 ++++++++++++++++- 8 files changed, 272 insertions(+), 41 deletions(-) diff --git a/packages/agent/docs/agent-harness.md b/packages/agent/docs/agent-harness.md index 65828889..64a8baec 100644 --- a/packages/agent/docs/agent-harness.md +++ b/packages/agent/docs/agent-harness.md @@ -258,13 +258,14 @@ Done: - Exported `QueueMode` from core types. - Added `AgentHarnessOptions.steeringMode` and `followUpMode`. - Added live `getSteeringMode()` / `setSteeringMode()` and `getFollowUpMode()` / `setFollowUpMode()`. +- Added `getTools()` and `getActiveTools()`. +- Added `tools_update` observability events, including active-tool-only updates. +- Active tool changes are persisted as branch-scoped `active_tools_change` entries. +- Duplicate tool names and duplicate active tool names reject. Remaining: -- Add `getTools()` semantics. -- Add `getActiveTools()` semantics. -- Decide and implement tool update observability events. -- Include active-tool-only updates in the runtime config observability plan. +- None. Notes: diff --git a/packages/agent/docs/durable-harness.md b/packages/agent/docs/durable-harness.md index 64e313df..ce5aa46d 100644 --- a/packages/agent/docs/durable-harness.md +++ b/packages/agent/docs/durable-harness.md @@ -14,6 +14,8 @@ A fully durable `AgentHarness` is not realistic by itself because important depe - resource loaders - system-prompt callbacks/modifiers +Tool registries are runtime dependencies. The harness should persist serializable tool configuration, such as active tool names, but not concrete tool implementations. + The practical target is a semi-durable harness: - session is the durable append-only state tree @@ -29,6 +31,7 @@ Existing session state already includes harness state: - model changes - thinking-level changes +- active-tool changes - leaf entries - labels - compactions and branch summaries @@ -50,10 +53,38 @@ The app must recreate compatible runtime dependencies: Harness can validate stable IDs/versions/hashes when available, but it cannot serialize these dependencies itself. +## Runtime configuration and restore + +Constructor options remain explicit runtime configuration and do not read session state. Hidden async restore in a constructor would make failure handling ambiguous. + +A future async builder/factory should own durable restore: + +```ts +const harness = await AgentHarness.builder() + .env(env) + .session(session) + .model(defaultModel) + .tools(runtimeTools) + .defaultActiveTools(["read", "edit"]) + .restore({ missingActiveTools: "fail" }); +``` + +`restore()` should read the active branch, reduce durable harness configuration, apply defaults for missing entries, validate against app-supplied runtime dependencies, construct the harness, and optionally emit `source: "restore"` update events after construction. + +For active tools: + +- `active_tools_change` entries are branch-scoped durable config. +- If no `active_tools_change` exists on the branch, restore uses builder defaults, or all registered tools if no default active names were supplied. +- Active tool names must be unique. +- Tool registry names must be unique. +- Missing restored active tool names should fail restore by default; permissive drop/disable policies can be added explicitly later. +- Concrete tools are never restored from session; the host app must provide compatible tools. + ## What harness should persist Minimum useful durability entries: +- branch-scoped active tool names - queued steer/followUp/nextTurn messages - queue consumption tied to a turn - pending session writes accepted during active operations @@ -93,11 +124,11 @@ On startup: 3. Harness reduces session entries into: - current leaf - conversation branch - - harness config + - harness config, including active tool names - queues - pending writes - active operation/turn/tool state -4. Harness validates required runtime dependencies. +4. Harness validates required runtime dependencies, including restored active tool names against the app-provided tool registry. 5. Harness reconciles unfinished operation state. Provider streams are not resumable. Recovery can only retry from a durable boundary or mark the operation interrupted. @@ -173,7 +204,7 @@ recovery: "mark_interrupted" | "retry_unfinished" ## Open questions -- Which harness config entries should move into session first: tools, active tools, resources, stream options, system prompt refs? +- Which remaining harness config entries should move into session first: resources, stream options, system prompt refs? - Should resolved system prompt text be snapshotted per turn for audit/debug? - Do we require strict dependency ID/version matching on resume? - How much provider request data should be journaled? diff --git a/packages/agent/src/harness/agent-harness.ts b/packages/agent/src/harness/agent-harness.ts index 80535e8e..96563465 100644 --- a/packages/agent/src/harness/agent-harness.ts +++ b/packages/agent/src/harness/agent-harness.ts @@ -86,6 +86,16 @@ function mergeHeaders(...headers: Array | undefined>): Re return hasHeaders ? merged : undefined; } +function findDuplicateNames(names: string[]): string[] { + const seen = new Set(); + const duplicates = new Set(); + for (const name of names) { + if (seen.has(name)) duplicates.add(name); + seen.add(name); + } + return [...duplicates]; +} + function applyStreamOptionsPatch( base: AgentHarnessStreamOptions, patch?: AgentHarnessStreamOptionsPatch, @@ -194,12 +204,20 @@ export class AgentHarness< this.streamOptions = cloneStreamOptions(options.streamOptions); this.systemPrompt = options.systemPrompt; this.getApiKeyAndHeaders = options.getApiKeyAndHeaders; + this.validateUniqueNames( + (options.tools ?? []).map((tool) => tool.name), + "Duplicate tool name(s)", + ); for (const tool of options.tools ?? []) { this.tools.set(tool.name, tool); } this.model = options.model; this.thinkingLevel = options.thinkingLevel ?? "off"; - this.activeToolNames = options.activeToolNames ?? (options.tools ?? []).map((tool) => tool.name); + this.activeToolNames = options.activeToolNames + ? [...options.activeToolNames] + : (options.tools ?? []).map((tool) => tool.name); + this.validateUniqueNames(this.activeToolNames, "Duplicate active tool name(s)"); + this.validateToolNames(this.activeToolNames); this.steeringQueueMode = options.steeringMode ?? "one-at-a-time"; this.followUpQueueMode = options.followUpMode ?? "one-at-a-time"; } @@ -451,7 +469,14 @@ export class AgentHarness< }; } + private validateUniqueNames(names: string[], message: string): void { + const duplicates = findDuplicateNames(names); + if (duplicates.length > 0) + throw new AgentHarnessError("invalid_argument", `${message}: ${duplicates.join(", ")}`); + } + private validateToolNames(toolNames: string[], tools: Map = this.tools): void { + this.validateUniqueNames(toolNames, "Duplicate active tool name(s)"); const missing = toolNames.filter((name) => !tools.has(name)); if (missing.length > 0) throw new AgentHarnessError("invalid_argument", `Unknown tool(s): ${missing.join(", ")}`); } @@ -465,6 +490,8 @@ export class AgentHarness< await this.session.appendModelChange(write.provider, write.modelId); } else if (write.type === "thinking_level_change") { await this.session.appendThinkingLevelChange(write.thinkingLevel); + } else if (write.type === "active_tools_change") { + await this.session.appendActiveToolsChange(write.activeToolNames); } else if (write.type === "custom") { await this.session.appendCustomEntry(write.customType, write.data); } else if (write.type === "custom_message") { @@ -838,10 +865,6 @@ export class AgentHarness< return this.model; } - getThinkingLevel(): ThinkingLevel { - return this.thinkingLevel; - } - async setModel(model: Model): Promise { try { const previousModel = this.model; @@ -851,12 +874,16 @@ export class AgentHarness< this.pendingSessionWrites.push({ type: "model_change", provider: model.provider, modelId: model.id }); } this.model = model; - await this.emitOwn({ type: "model_select", model, previousModel, source: "set" }); + await this.emitOwn({ type: "model_update", model, previousModel, source: "set" }); } catch (error) { throw normalizeHarnessError(error, "session"); } } + getThinkingLevel(): ThinkingLevel { + return this.thinkingLevel; + } + async setThinkingLevel(level: ThinkingLevel): Promise { try { const previousLevel = this.thinkingLevel; @@ -866,16 +893,70 @@ export class AgentHarness< this.pendingSessionWrites.push({ type: "thinking_level_change", thinkingLevel: level }); } this.thinkingLevel = level; - await this.emitOwn({ type: "thinking_level_select", level, previousLevel }); + await this.emitOwn({ type: "thinking_level_update", level, previousLevel }); } catch (error) { throw normalizeHarnessError(error, "session"); } } + getTools(): TTool[] { + return [...this.tools.values()]; + } + + async setTools(tools: TTool[], activeToolNames?: string[]): Promise { + try { + this.validateUniqueNames( + tools.map((tool) => tool.name), + "Duplicate tool name(s)", + ); + const nextTools = new Map(tools.map((tool) => [tool.name, tool])); + const nextActiveToolNames = activeToolNames ? [...activeToolNames] : this.activeToolNames; + this.validateToolNames(nextActiveToolNames, nextTools); + const previousToolNames = [...this.tools.keys()]; + const previousActiveToolNames = [...this.activeToolNames]; + if (this.phase === "idle") { + await this.session.appendActiveToolsChange(nextActiveToolNames); + } else { + this.pendingSessionWrites.push({ type: "active_tools_change", activeToolNames: [...nextActiveToolNames] }); + } + this.tools = nextTools; + this.activeToolNames = [...nextActiveToolNames]; + await this.emitOwn({ + type: "tools_update", + toolNames: [...this.tools.keys()], + previousToolNames, + activeToolNames: [...this.activeToolNames], + previousActiveToolNames, + source: "set", + }); + } catch (error) { + throw normalizeHarnessError(error, "invalid_argument"); + } + } + + getActiveTools(): TTool[] { + return this.activeToolNames.map((name) => this.tools.get(name)!); + } + async setActiveTools(toolNames: string[]): Promise { try { this.validateToolNames(toolNames); + const previousToolNames = [...this.tools.keys()]; + const previousActiveToolNames = [...this.activeToolNames]; + if (this.phase === "idle") { + await this.session.appendActiveToolsChange(toolNames); + } else { + this.pendingSessionWrites.push({ type: "active_tools_change", activeToolNames: [...toolNames] }); + } this.activeToolNames = [...toolNames]; + await this.emitOwn({ + type: "tools_update", + toolNames: [...this.tools.keys()], + previousToolNames, + activeToolNames: [...this.activeToolNames], + previousActiveToolNames, + source: "set", + }); } catch (error) { throw normalizeHarnessError(error, "invalid_argument"); } @@ -921,18 +1002,6 @@ export class AgentHarness< this.streamOptions = cloneStreamOptions(streamOptions); } - async setTools(tools: TTool[], activeToolNames?: string[]): Promise { - try { - const nextTools = new Map(tools.map((tool) => [tool.name, tool])); - const nextActiveToolNames = activeToolNames ? [...activeToolNames] : this.activeToolNames; - this.validateToolNames(nextActiveToolNames, nextTools); - this.tools = nextTools; - this.activeToolNames = [...nextActiveToolNames]; - } catch (error) { - throw normalizeHarnessError(error, "invalid_argument"); - } - } - async abort(): Promise { const clearedSteer = [...this.steerQueue]; const clearedFollowUp = [...this.followUpQueue]; diff --git a/packages/agent/src/harness/compaction/branch-summarization.ts b/packages/agent/src/harness/compaction/branch-summarization.ts index 6ca4a430..c1824ebf 100644 --- a/packages/agent/src/harness/compaction/branch-summarization.ts +++ b/packages/agent/src/harness/compaction/branch-summarization.ts @@ -112,6 +112,7 @@ function getMessageFromEntry(entry: SessionTreeEntry): AgentMessage | undefined return createCompactionSummaryMessage(entry.summary, entry.tokensBefore, entry.timestamp); case "thinking_level_change": case "model_change": + case "active_tools_change": case "custom": case "label": case "session_info": diff --git a/packages/agent/src/harness/compaction/compaction.ts b/packages/agent/src/harness/compaction/compaction.ts index a7ae33e9..e3a9f114 100644 --- a/packages/agent/src/harness/compaction/compaction.ts +++ b/packages/agent/src/harness/compaction/compaction.ts @@ -281,6 +281,7 @@ function findValidCutPoints(entries: SessionTreeEntry[], startIndex: number, end } case "thinking_level_change": case "model_change": + case "active_tools_change": case "compaction": case "branch_summary": case "custom": diff --git a/packages/agent/src/harness/session/session.ts b/packages/agent/src/harness/session/session.ts index 4baf50ec..6f136208 100644 --- a/packages/agent/src/harness/session/session.ts +++ b/packages/agent/src/harness/session/session.ts @@ -2,6 +2,7 @@ import type { ImageContent, TextContent } from "@earendil-works/pi-ai"; import type { AgentMessage } from "../../types.ts"; import { createBranchSummaryMessage, createCompactionSummaryMessage, createCustomMessage } from "../messages.ts"; import type { + ActiveToolsChangeEntry, BranchSummaryEntry, CompactionEntry, CustomEntry, @@ -21,6 +22,7 @@ import { SessionError } from "../types.ts"; export function buildSessionContext(pathEntries: SessionTreeEntry[]): SessionContext { let thinkingLevel = "off"; let model: { provider: string; modelId: string } | null = null; + let activeToolNames: string[] | null = null; let compaction: CompactionEntry | null = null; for (const entry of pathEntries) { @@ -30,6 +32,8 @@ export function buildSessionContext(pathEntries: SessionTreeEntry[]): SessionCon model = { provider: entry.provider, modelId: entry.modelId }; } else if (entry.type === "message" && entry.message.role === "assistant") { model = { provider: entry.message.provider, modelId: entry.message.model }; + } else if (entry.type === "active_tools_change") { + activeToolNames = [...entry.activeToolNames]; } else if (entry.type === "compaction") { compaction = entry; } @@ -72,7 +76,7 @@ export function buildSessionContext(pathEntries: SessionTreeEntry[]): SessionCon } } - return { messages, thinkingLevel, model }; + return { messages, thinkingLevel, model, activeToolNames }; } export class Session { @@ -156,6 +160,16 @@ export class Session { } satisfies ModelChangeEntry); } + async appendActiveToolsChange(activeToolNames: string[]): Promise { + return this.appendTypedEntry({ + type: "active_tools_change", + id: await this.storage.createEntryId(), + parentId: await this.storage.getLeafId(), + timestamp: new Date().toISOString(), + activeToolNames: [...activeToolNames], + } satisfies ActiveToolsChangeEntry); + } + async appendCompaction( summary: string, firstKeptEntryId: string, diff --git a/packages/agent/src/harness/types.ts b/packages/agent/src/harness/types.ts index 2d9096c4..4756ca84 100644 --- a/packages/agent/src/harness/types.ts +++ b/packages/agent/src/harness/types.ts @@ -354,6 +354,11 @@ export interface ModelChangeEntry extends SessionTreeEntryBase { modelId: string; } +export interface ActiveToolsChangeEntry extends SessionTreeEntryBase { + type: "active_tools_change"; + activeToolNames: string[]; +} + export interface CompactionEntry extends SessionTreeEntryBase { type: "compaction"; summary: string; @@ -405,6 +410,7 @@ export type SessionTreeEntry = | MessageEntry | ThinkingLevelChangeEntry | ModelChangeEntry + | ActiveToolsChangeEntry | CompactionEntry | BranchSummaryEntry | CustomEntry @@ -417,6 +423,7 @@ export interface SessionContext { messages: AgentMessage[]; thinkingLevel: string; model: { provider: string; modelId: string } | null; + activeToolNames: string[] | null; } export interface SessionMetadata { @@ -593,19 +600,28 @@ export interface SessionTreeEvent { fromHook?: boolean; } -export interface ModelSelectEvent { - type: "model_select"; +export interface ModelUpdateEvent { + type: "model_update"; model: Model; previousModel: Model | undefined; source: "set" | "restore"; } -export interface ThinkingLevelSelectEvent { - type: "thinking_level_select"; +export interface ThinkingLevelUpdateEvent { + type: "thinking_level_update"; level: ThinkingLevel; previousLevel: ThinkingLevel; } +export interface ToolsUpdateEvent { + type: "tools_update"; + toolNames: string[]; + previousToolNames: string[]; + activeToolNames: string[]; + previousActiveToolNames: string[]; + source: "set" | "restore"; +} + export interface ResourcesUpdateEvent< TSkill extends Skill = Skill, TPromptTemplate extends PromptTemplate = PromptTemplate, @@ -634,9 +650,10 @@ export type AgentHarnessOwnEvent< | SessionCompactEvent | SessionBeforeTreeEvent | SessionTreeEvent - | ModelSelectEvent - | ThinkingLevelSelectEvent - | ResourcesUpdateEvent; + | ModelUpdateEvent + | ThinkingLevelUpdateEvent + | ResourcesUpdateEvent + | ToolsUpdateEvent; export type AgentHarnessEvent = | AgentEvent @@ -696,9 +713,10 @@ export type AgentHarnessEventResultMap = { session_compact: undefined; session_before_tree: SessionBeforeTreeResult | undefined; session_tree: undefined; - model_select: undefined; - thinking_level_select: undefined; + model_update: undefined; + thinking_level_update: undefined; resources_update: undefined; + tools_update: undefined; queue_update: undefined; save_point: undefined; abort: undefined; diff --git a/packages/agent/test/harness/agent-harness.test.ts b/packages/agent/test/harness/agent-harness.test.ts index 8a395532..1d24eb4c 100644 --- a/packages/agent/test/harness/agent-harness.test.ts +++ b/packages/agent/test/harness/agent-harness.test.ts @@ -17,10 +17,6 @@ interface AppPromptTemplate extends PromptTemplate { source: "project" | "user"; } -interface AppTool extends AgentTool { - source: "builtin" | "extension"; -} - const registrations: Array<{ unregister(): void }> = []; function textFromUserMessages(messages: Array<{ role: string; content: unknown }>): string[] { @@ -458,11 +454,111 @@ describe("AgentHarness", () => { }); }); + it("preserves app tool types for getters and update events", async () => { + const session = new Session(new InMemorySessionStorage()); + const env = new NodeExecutionEnv({ cwd: process.cwd() }); + const model = getModel("anthropic", "claude-sonnet-4-5"); + type AppTool = AgentTool & { source: "builtin" | "extension" }; + const inspectTool: AppTool = { ...calculateTool, name: "inspect", source: "builtin" }; + const searchTool: AppTool = { ...calculateTool, name: "search", source: "extension" }; + const harness = new AgentHarness({ + env, + session, + model, + tools: [inspectTool, searchTool], + activeToolNames: ["inspect"], + }); + const updates: Array<{ + toolNames: string[]; + previousToolNames: string[]; + activeToolNames: string[]; + previousActiveToolNames: string[]; + source: "set" | "restore"; + }> = []; + harness.subscribe((event) => { + if (event.type === "tools_update") { + updates.push({ + toolNames: event.toolNames, + previousToolNames: event.previousToolNames, + activeToolNames: event.activeToolNames, + previousActiveToolNames: event.previousActiveToolNames, + source: event.source, + }); + expect(harness.getActiveTools().map((tool) => tool.name)).toEqual(event.activeToolNames); + } + }); + + const tools = harness.getTools(); + const activeTools = harness.getActiveTools(); + tools.pop(); + activeTools.pop(); + expect(harness.getTools().map((tool) => tool.name)).toEqual(["inspect", "search"]); + expect(harness.getActiveTools().map((tool) => tool.source)).toEqual(["builtin"]); + + await harness.setActiveTools(["search"]); + await harness.setTools([searchTool], ["search"]); + await expect(harness.setActiveTools(["missing"])).rejects.toMatchObject({ code: "invalid_argument" }); + await expect(harness.setActiveTools(["search", "search"])).rejects.toMatchObject({ code: "invalid_argument" }); + await expect(harness.setTools([inspectTool])).rejects.toMatchObject({ code: "invalid_argument" }); + await expect(harness.setTools([inspectTool, inspectTool], ["inspect"])).rejects.toMatchObject({ + code: "invalid_argument", + }); + + expect(updates).toEqual([ + { + toolNames: ["inspect", "search"], + previousToolNames: ["inspect", "search"], + activeToolNames: ["search"], + previousActiveToolNames: ["inspect"], + source: "set", + }, + { + toolNames: ["search"], + previousToolNames: ["inspect", "search"], + activeToolNames: ["search"], + previousActiveToolNames: ["search"], + source: "set", + }, + ]); + expect(harness.getTools().map((tool) => tool.source)).toEqual(["extension"]); + expect(harness.getActiveTools().map((tool) => tool.name)).toEqual(["search"]); + expect((await session.buildContext()).activeToolNames).toEqual(["search"]); + }); + + it("validates constructor tool names", () => { + const session = new Session(new InMemorySessionStorage()); + const env = new NodeExecutionEnv({ cwd: process.cwd() }); + const model = getModel("anthropic", "claude-sonnet-4-5"); + expect( + () => new AgentHarness({ env, session, model, tools: [calculateTool], activeToolNames: ["missing"] }), + ).toThrow(/Unknown tool/); + expect( + () => + new AgentHarness({ + env, + session, + model, + tools: [calculateTool, calculateTool], + activeToolNames: [calculateTool.name], + }), + ).toThrow(/Duplicate tool/); + expect( + () => + new AgentHarness({ + env, + session, + model, + tools: [calculateTool], + activeToolNames: [calculateTool.name, calculateTool.name], + }), + ).toThrow(/Duplicate active tool/); + }); + it("preserves app resource types for getters and update events", async () => { const session = new Session(new InMemorySessionStorage()); const env = new NodeExecutionEnv({ cwd: process.cwd() }); const model = getModel("anthropic", "claude-sonnet-4-5"); - const harness = new AgentHarness({ env, session, model }); + const harness = new AgentHarness({ env, session, model }); const skill: AppSkill = { name: "inspect", description: "Inspect things",