diff --git a/packages/coding-agent/test/test-harness.test.ts b/packages/coding-agent/test/test-harness.test.ts new file mode 100644 index 00000000..0c7f19f2 --- /dev/null +++ b/packages/coding-agent/test/test-harness.test.ts @@ -0,0 +1,269 @@ +/** + * Tests for the test harness itself. + * Validates that the faux provider and session factory work correctly. + */ + +import type { AgentTool } from "@mariozechner/pi-agent-core"; +import type { AssistantMessage } from "@mariozechner/pi-ai"; +import { Type } from "@sinclair/typebox"; +import { afterEach, describe, expect, it } from "vitest"; +import { createHarness, type Harness } from "./test-harness.js"; + +describe("test harness", () => { + let harness: Harness; + + afterEach(() => { + harness?.cleanup(); + }); + + it("simple text response", async () => { + harness = createHarness({ responses: ["hello world"] }); + + await harness.session.prompt("hi"); + + expect(harness.faux.callCount).toBe(1); + + const assistantMessages = harness.session.messages.filter((m) => m.role === "assistant"); + expect(assistantMessages).toHaveLength(1); + + const msg = assistantMessages[0] as AssistantMessage; + expect(msg.content).toEqual([{ type: "text", text: "hello world" }]); + expect(msg.stopReason).toBe("stop"); + }); + + it("response sequence", async () => { + harness = createHarness({ responses: ["first", "second", "third"] }); + + await harness.session.prompt("a"); + await harness.session.prompt("b"); + await harness.session.prompt("c"); + + expect(harness.faux.callCount).toBe(3); + + const assistantTexts = harness.session.messages + .filter((m): m is AssistantMessage => m.role === "assistant") + .map((m) => m.content.find((c) => c.type === "text")?.text); + + expect(assistantTexts).toEqual(["first", "second", "third"]); + }); + + it("tool call response triggers tool execution", async () => { + let toolExecuted = false; + const echoTool: AgentTool = { + name: "echo", + label: "Echo", + description: "Echo back", + parameters: Type.Object({ text: Type.String() }), + execute: async () => { + toolExecuted = true; + return { content: [{ type: "text", text: "echoed" }], details: {} }; + }, + }; + + harness = createHarness({ + responses: [{ toolCalls: [{ name: "echo", args: { text: "hi" } }] }, "done after tool"], + tools: [echoTool], + baseToolsOverride: { echo: echoTool }, + }); + + await harness.session.prompt("use the tool"); + + expect(toolExecuted).toBe(true); + expect(harness.faux.callCount).toBe(2); + + const toolResults = harness.session.messages.filter((m) => m.role === "toolResult"); + expect(toolResults).toHaveLength(1); + }); + + it("error response", async () => { + harness = createHarness({ + responses: [{ error: "something broke" }], + }); + + await harness.session.prompt("hi"); + + const assistantMessages = harness.session.messages.filter((m): m is AssistantMessage => m.role === "assistant"); + expect(assistantMessages).toHaveLength(1); + expect(assistantMessages[0].stopReason).toBe("error"); + expect(assistantMessages[0].errorMessage).toBe("something broke"); + }); + + it("retry on transient error", async () => { + harness = createHarness({ + responses: [{ error: "overloaded_error" }, "recovered"], + settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } }, + }); + + await harness.session.prompt("hi"); + + expect(harness.faux.callCount).toBe(2); + + const retryStarts = harness.eventsOfType("auto_retry_start"); + expect(retryStarts).toHaveLength(1); + + const retryEnds = harness.eventsOfType("auto_retry_end"); + expect(retryEnds).toHaveLength(1); + expect(retryEnds[0].success).toBe(true); + }); + + it("custom usage numbers", async () => { + harness = createHarness({ + responses: [{ text: "big response", usage: { input: 100000, output: 5000 } }], + }); + + await harness.session.prompt("hi"); + + const msg = harness.session.messages.find((m): m is AssistantMessage => m.role === "assistant")!; + expect(msg.usage.input).toBe(100000); + expect(msg.usage.output).toBe(5000); + }); + + it("event capture", async () => { + harness = createHarness({ responses: ["hello"] }); + + await harness.session.prompt("hi"); + + const agentStarts = harness.eventsOfType("agent_start"); + expect(agentStarts).toHaveLength(1); + + const agentEnds = harness.eventsOfType("agent_end"); + expect(agentEnds).toHaveLength(1); + + const messageEnds = harness.eventsOfType("message_end"); + expect(messageEnds.length).toBeGreaterThanOrEqual(2); // user + assistant + }); + + it("context capture", async () => { + harness = createHarness({ responses: ["reply"] }); + + await harness.session.prompt("my question"); + + expect(harness.faux.contexts).toHaveLength(1); + const ctx = harness.faux.contexts[0]; + const userMsg = ctx.messages.find((m) => m.role === "user"); + expect(userMsg).toBeDefined(); + }); + + it("wraps around when more calls than responses", async () => { + harness = createHarness({ responses: ["a", "b"] }); + + await harness.session.prompt("1"); + await harness.session.prompt("2"); + await harness.session.prompt("3"); + + expect(harness.faux.callCount).toBe(3); + + const texts = harness.session.messages + .filter((m): m is AssistantMessage => m.role === "assistant") + .map((m) => m.content.find((c) => c.type === "text")?.text); + + expect(texts).toEqual(["a", "b", "a"]); + }); + + it("streams text deltas", async () => { + harness = createHarness({ responses: ["hello world"] }); + + await harness.session.prompt("hi"); + + const updates = harness.eventsOfType("message_update"); + const textDeltas = updates.filter((e) => e.assistantMessageEvent.type === "text_delta"); + expect(textDeltas.length).toBeGreaterThan(0); + + // Deltas should reconstruct the full text + const reconstructed = textDeltas.map((e) => (e.assistantMessageEvent as { delta: string }).delta).join(""); + expect(reconstructed).toBe("hello world"); + }); + + it("streams thinking deltas", async () => { + harness = createHarness({ + responses: [{ thinking: "let me think about this", text: "answer" }], + }); + + await harness.session.prompt("hi"); + + const updates = harness.eventsOfType("message_update"); + const thinkingStarts = updates.filter((e) => e.assistantMessageEvent.type === "thinking_start"); + const thinkingDeltas = updates.filter((e) => e.assistantMessageEvent.type === "thinking_delta"); + const thinkingEnds = updates.filter((e) => e.assistantMessageEvent.type === "thinking_end"); + + expect(thinkingStarts).toHaveLength(1); + expect(thinkingDeltas.length).toBeGreaterThan(0); + expect(thinkingEnds).toHaveLength(1); + + const reconstructed = thinkingDeltas.map((e) => (e.assistantMessageEvent as { delta: string }).delta).join(""); + expect(reconstructed).toBe("let me think about this"); + }); + + it("streams tool call deltas", async () => { + const echoTool: AgentTool = { + name: "echo", + label: "Echo", + description: "Echo back", + parameters: Type.Object({ text: Type.String() }), + execute: async () => ({ content: [{ type: "text", text: "echoed" }], details: {} }), + }; + + harness = createHarness({ + responses: [{ toolCalls: [{ name: "echo", args: { text: "hi" } }] }, "done"], + tools: [echoTool], + baseToolsOverride: { echo: echoTool }, + }); + + await harness.session.prompt("use tool"); + + const updates = harness.eventsOfType("message_update"); + const toolcallStarts = updates.filter((e) => e.assistantMessageEvent.type === "toolcall_start"); + const toolcallDeltas = updates.filter((e) => e.assistantMessageEvent.type === "toolcall_delta"); + const toolcallEnds = updates.filter((e) => e.assistantMessageEvent.type === "toolcall_end"); + + expect(toolcallStarts).toHaveLength(1); + expect(toolcallDeltas.length).toBeGreaterThan(0); + expect(toolcallEnds).toHaveLength(1); + }); + + it("streams thinking then text then tool call in order", async () => { + const echoTool: AgentTool = { + name: "echo", + label: "Echo", + description: "Echo back", + parameters: Type.Object({ text: Type.String() }), + execute: async () => ({ content: [{ type: "text", text: "echoed" }], details: {} }), + }; + + harness = createHarness({ + responses: [ + { + thinking: "hmm", + text: "I will call a tool", + toolCalls: [{ name: "echo", args: { text: "x" } }], + }, + "final", + ], + tools: [echoTool], + baseToolsOverride: { echo: echoTool }, + }); + + await harness.session.prompt("do it"); + + const updates = harness.eventsOfType("message_update"); + const streamTypes = updates.map((e) => e.assistantMessageEvent.type); + + // Thinking events should come before text events, text before toolcall + const firstThinking = streamTypes.indexOf("thinking_start"); + const firstText = streamTypes.indexOf("text_start"); + const firstToolcall = streamTypes.indexOf("toolcall_start"); + + expect(firstThinking).toBeLessThan(firstText); + expect(firstText).toBeLessThan(firstToolcall); + }); + + it("session persistence works", async () => { + harness = createHarness({ responses: ["persisted"] }); + + await harness.session.prompt("hi"); + + const entries = harness.sessionManager.getEntries(); + const messageEntries = entries.filter((e) => e.type === "message"); + expect(messageEntries.length).toBeGreaterThanOrEqual(2); // user + assistant + }); +}); diff --git a/packages/coding-agent/test/test-harness.ts b/packages/coding-agent/test/test-harness.ts new file mode 100644 index 00000000..0c428a8e --- /dev/null +++ b/packages/coding-agent/test/test-harness.ts @@ -0,0 +1,414 @@ +/** + * Test harness for AgentSession runtime testing. + * + * Provides: + * - A faux stream function with declarative response sequencing + * - A one-call factory for a fully wired AgentSession with real in-memory dependencies + * - Event capture for assertions + */ + +import { existsSync, mkdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AgentTool } from "@mariozechner/pi-agent-core"; +import { Agent } from "@mariozechner/pi-agent-core"; +import type { + AssistantMessage, + AssistantMessageEvent, + AssistantMessageEventStream, + Context, + Model, + SimpleStreamOptions, + StopReason, + TextContent, + ThinkingContent, + ToolCall, + Usage, +} from "@mariozechner/pi-ai"; +import { createAssistantMessageEventStream } from "@mariozechner/pi-ai"; +import { AgentSession, type AgentSessionEvent } from "../src/core/agent-session.js"; +import { AuthStorage } from "../src/core/auth-storage.js"; +import { ModelRegistry } from "../src/core/model-registry.js"; +import { SessionManager } from "../src/core/session-manager.js"; +import type { Settings } from "../src/core/settings-manager.js"; +import { SettingsManager } from "../src/core/settings-manager.js"; +import { createTestResourceLoader } from "./utilities.js"; + +// ============================================================================ +// Faux model +// ============================================================================ + +const FAUX_PROVIDER = "faux"; +const FAUX_MODEL_ID = "faux-1"; +const FAUX_API = "anthropic-messages" as const; + +export const fauxModel: Model = { + id: FAUX_MODEL_ID, + name: "Faux Model", + api: FAUX_API, + provider: FAUX_PROVIDER, + baseUrl: "http://localhost:0", + reasoning: false, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 16384, +}; + +// ============================================================================ +// Response description +// ============================================================================ + +export interface FauxResponse { + /** Text content blocks. String shorthand becomes a single text block. */ + text?: string; + /** Tool calls to include in the response. */ + toolCalls?: Array<{ id?: string; name: string; args: Record }>; + /** Thinking content. */ + thinking?: string; + /** Stop reason. Defaults to "stop", or "toolUse" if toolCalls are present, or "error" if error is set. */ + stopReason?: StopReason; + /** Error message. Sets stopReason to "error" if not explicitly set. */ + error?: string; + /** Usage numbers. Merged with defaults (input: 100, output: 50). */ + usage?: Partial; + /** Delay in ms before the response starts. */ + delayMs?: number; + /** Model overrides (provider, model id) for responses that should look like they came from a different model. */ + model?: { provider?: string; id?: string }; +} + +/** Shorthand: a string becomes a simple text response. */ +export type FauxResponseInput = FauxResponse | string; + +// ============================================================================ +// Faux stream function +// ============================================================================ + +function normalizeResponse(input: FauxResponseInput): FauxResponse { + if (typeof input === "string") { + return { text: input }; + } + return input; +} + +function buildUsage(partial?: Partial): Usage { + const input = partial?.input ?? 100; + const output = partial?.output ?? 50; + const cacheRead = partial?.cacheRead ?? 0; + const cacheWrite = partial?.cacheWrite ?? 0; + return { + input, + output, + cacheRead, + cacheWrite, + totalTokens: partial?.totalTokens ?? input + output + cacheRead + cacheWrite, + cost: partial?.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }; +} + +let toolCallIdCounter = 0; + +function buildAssistantMessage(resp: FauxResponse): AssistantMessage { + const content: (TextContent | ThinkingContent | ToolCall)[] = []; + + if (resp.thinking) { + content.push({ type: "thinking", thinking: resp.thinking }); + } + if (resp.text !== undefined) { + content.push({ type: "text", text: resp.text }); + } + if (resp.toolCalls) { + for (const tc of resp.toolCalls) { + content.push({ + type: "toolCall", + id: tc.id ?? `faux_tc_${++toolCallIdCounter}`, + name: tc.name, + arguments: tc.args, + }); + } + } + + // If no content was added at all, add empty text + if (content.length === 0 && !resp.error) { + content.push({ type: "text", text: "" }); + } + + let stopReason: StopReason; + if (resp.stopReason) { + stopReason = resp.stopReason; + } else if (resp.error) { + stopReason = "error"; + } else if (resp.toolCalls && resp.toolCalls.length > 0) { + stopReason = "toolUse"; + } else { + stopReason = "stop"; + } + + return { + role: "assistant", + content, + api: FAUX_API, + provider: resp.model?.provider ?? FAUX_PROVIDER, + model: resp.model?.id ?? FAUX_MODEL_ID, + usage: buildUsage(resp.usage), + stopReason, + errorMessage: resp.error, + timestamp: Date.now(), + }; +} + +// ============================================================================ +// Token-level streaming +// ============================================================================ + +/** Split a string into chunks of varying size (3-5 chars) for simulating token-by-token streaming. */ +function chunkString(text: string): string[] { + const chunks: string[] = []; + let i = 0; + while (i < text.length) { + const size = 3 + Math.floor(Math.random() * 3); // 3, 4, or 5 + chunks.push(text.slice(i, i + size)); + i += size; + } + return chunks.length > 0 ? chunks : [""]; +} + +/** + * Stream a complete AssistantMessage through an EventStream with realistic + * intermediate delta events for each content block. + */ +function streamWithDeltas(stream: AssistantMessageEventStream, message: AssistantMessage): void { + const isError = message.stopReason === "error" || message.stopReason === "aborted"; + + // Build partial progressively as we stream content blocks + const partial: AssistantMessage = { ...message, content: [] }; + stream.push({ type: "start", partial: { ...partial } }); + + for (let i = 0; i < message.content.length; i++) { + const block = message.content[i]; + + if (block.type === "thinking") { + partial.content = [...partial.content, { type: "thinking", thinking: "" }]; + stream.push({ type: "thinking_start", contentIndex: i, partial: { ...partial } }); + + for (const chunk of chunkString(block.thinking)) { + (partial.content[i] as ThinkingContent).thinking += chunk; + stream.push(makeEvent("thinking_delta", i, chunk, partial)); + } + + stream.push({ + type: "thinking_end", + contentIndex: i, + content: block.thinking, + partial: { ...partial }, + }); + } else if (block.type === "text") { + partial.content = [...partial.content, { type: "text", text: "" }]; + stream.push({ type: "text_start", contentIndex: i, partial: { ...partial } }); + + for (const chunk of chunkString(block.text)) { + (partial.content[i] as TextContent).text += chunk; + stream.push(makeEvent("text_delta", i, chunk, partial)); + } + + stream.push({ + type: "text_end", + contentIndex: i, + content: block.text, + partial: { ...partial }, + }); + } else if (block.type === "toolCall") { + const argsJson = JSON.stringify(block.arguments); + partial.content = [...partial.content, { type: "toolCall", id: block.id, name: block.name, arguments: {} }]; + stream.push({ type: "toolcall_start", contentIndex: i, partial: { ...partial } }); + + for (const chunk of chunkString(argsJson)) { + stream.push(makeEvent("toolcall_delta", i, chunk, partial)); + } + + // Final toolcall has the real parsed arguments + (partial.content[i] as ToolCall).arguments = block.arguments; + stream.push({ + type: "toolcall_end", + contentIndex: i, + toolCall: block, + partial: { ...partial }, + }); + } + } + + if (isError) { + stream.push({ type: "error", reason: message.stopReason as "error" | "aborted", error: message }); + } else { + stream.push({ type: "done", reason: message.stopReason as "stop" | "length" | "toolUse", message }); + } +} + +function makeEvent( + type: "text_delta" | "thinking_delta" | "toolcall_delta", + contentIndex: number, + delta: string, + partial: AssistantMessage, +): AssistantMessageEvent { + return { type, contentIndex, delta, partial: { ...partial } }; +} + +// ============================================================================ +// Stream function factory +// ============================================================================ + +export interface FauxStreamFnState { + /** Number of times the stream function has been called. */ + callCount: number; + /** The context passed to each call, in order. */ + contexts: Context[]; +} + +/** + * Create a faux stream function from a sequence of response descriptions. + * + * The function cycles through responses in order. If more calls are made than + * responses provided, it wraps around. + * + * Returns the stream function and a state object for inspection. + */ +export function createFauxStreamFn(responses: FauxResponseInput[]): { + streamFn: (model: Model, context: Context, options?: SimpleStreamOptions) => AssistantMessageEventStream; + state: FauxStreamFnState; +} { + if (responses.length === 0) { + throw new Error("createFauxStreamFn requires at least one response"); + } + + const state: FauxStreamFnState = { callCount: 0, contexts: [] }; + + const streamFn = (_model: Model, context: Context, _options?: SimpleStreamOptions) => { + const index = state.callCount % responses.length; + state.callCount++; + state.contexts.push(context); + + const resp = normalizeResponse(responses[index]); + const message = buildAssistantMessage(resp); + const stream = createAssistantMessageEventStream(); + + const emit = () => { + streamWithDeltas(stream, message); + }; + + if (resp.delayMs && resp.delayMs > 0) { + setTimeout(emit, resp.delayMs); + } else { + queueMicrotask(emit); + } + + return stream; + }; + + return { streamFn, state }; +} + +// ============================================================================ +// Session harness +// ============================================================================ + +export interface HarnessOptions { + /** Response sequence for the faux provider. Default: single "ok" response. */ + responses?: FauxResponseInput[]; + /** Model to use. Default: fauxModel. */ + model?: Model; + /** Context window override (applied to the model). */ + contextWindow?: number; + /** Settings overrides (retry, compaction, etc.). */ + settings?: Partial; + /** System prompt. Default: "You are a test assistant." */ + systemPrompt?: string; + /** Custom tools to register on the agent. */ + tools?: AgentTool[]; + /** Base tools override (replaces built-in read/bash/edit/write). */ + baseToolsOverride?: Record; +} + +export interface Harness { + session: AgentSession; + agent: Agent; + sessionManager: SessionManager; + settingsManager: SettingsManager; + /** Faux stream function state (call count, captured contexts). */ + faux: FauxStreamFnState; + /** All events emitted by the session, in order. */ + events: AgentSessionEvent[]; + /** Filter captured events by type. */ + eventsOfType(type: T): Extract[]; + /** Temp directory (cleaned up by cleanup()). */ + tempDir: string; + /** Dispose session and remove temp directory. */ + cleanup: () => void; +} + +export function createHarness(options: HarnessOptions = {}): Harness { + const tempDir = join(tmpdir(), `pi-harness-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(tempDir, { recursive: true }); + + const baseModel = options.model ?? fauxModel; + const model: Model = options.contextWindow ? { ...baseModel, contextWindow: options.contextWindow } : baseModel; + + const { streamFn, state: fauxState } = createFauxStreamFn(options.responses ?? ["ok"]); + + const agent = new Agent({ + getApiKey: () => "faux-key", + initialState: { + model, + systemPrompt: options.systemPrompt ?? "You are a test assistant.", + tools: options.tools ?? [], + }, + streamFn, + }); + + const sessionManager = SessionManager.inMemory(); + const settingsManager = SettingsManager.create(tempDir, tempDir); + + if (options.settings) { + settingsManager.applyOverrides(options.settings); + } + + const authStorage = AuthStorage.create(join(tempDir, "auth.json")); + authStorage.setRuntimeApiKey(model.provider, "faux-key"); + const modelRegistry = new ModelRegistry(authStorage, tempDir); + + const session = new AgentSession({ + agent, + sessionManager, + settingsManager, + cwd: tempDir, + modelRegistry, + resourceLoader: createTestResourceLoader(), + baseToolsOverride: options.baseToolsOverride, + }); + + const events: AgentSessionEvent[] = []; + session.subscribe((event) => { + events.push(event); + }); + + const cleanup = () => { + session.dispose(); + if (existsSync(tempDir)) { + rmSync(tempDir, { recursive: true }); + } + }; + + return { + session, + agent, + sessionManager, + settingsManager, + faux: fauxState, + events, + eventsOfType(type: T) { + return events.filter((e): e is Extract => e.type === type); + }, + tempDir, + cleanup, + }; +}