test(coding-agent): add AgentSession test harness with faux provider
Adds a reusable test harness for AgentSession runtime testing: - Faux stream function with declarative response sequencing (text, tool calls, thinking, errors, custom usage, delays) - Token-level streaming with realistic delta events (3-5 char chunks) - One-call session factory with real in-memory dependencies - Event capture with type-filtered access - 14 self-tests covering text, tool calls, errors, retry, streaming deltas, thinking, event ordering, persistence ref #2023
This commit is contained in:
Mario Zechner
269
packages/coding-agent/test/test-harness.test.ts
Normal file
269
packages/coding-agent/test/test-harness.test.ts
Normal file
@@ -0,0 +1,269 @@
|
||||
/**
|
||||
* Tests for the test harness itself.
|
||||
* Validates that the faux provider and session factory work correctly.
|
||||
*/
|
||||
|
||||
import type { AgentTool } from "@mariozechner/pi-agent-core";
|
||||
import type { AssistantMessage } from "@mariozechner/pi-ai";
|
||||
import { Type } from "@sinclair/typebox";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { createHarness, type Harness } from "./test-harness.js";
|
||||
|
||||
describe("test harness", () => {
|
||||
let harness: Harness;
|
||||
|
||||
afterEach(() => {
|
||||
harness?.cleanup();
|
||||
});
|
||||
|
||||
it("simple text response", async () => {
|
||||
harness = createHarness({ responses: ["hello world"] });
|
||||
|
||||
await harness.session.prompt("hi");
|
||||
|
||||
expect(harness.faux.callCount).toBe(1);
|
||||
|
||||
const assistantMessages = harness.session.messages.filter((m) => m.role === "assistant");
|
||||
expect(assistantMessages).toHaveLength(1);
|
||||
|
||||
const msg = assistantMessages[0] as AssistantMessage;
|
||||
expect(msg.content).toEqual([{ type: "text", text: "hello world" }]);
|
||||
expect(msg.stopReason).toBe("stop");
|
||||
});
|
||||
|
||||
it("response sequence", async () => {
|
||||
harness = createHarness({ responses: ["first", "second", "third"] });
|
||||
|
||||
await harness.session.prompt("a");
|
||||
await harness.session.prompt("b");
|
||||
await harness.session.prompt("c");
|
||||
|
||||
expect(harness.faux.callCount).toBe(3);
|
||||
|
||||
const assistantTexts = harness.session.messages
|
||||
.filter((m): m is AssistantMessage => m.role === "assistant")
|
||||
.map((m) => m.content.find((c) => c.type === "text")?.text);
|
||||
|
||||
expect(assistantTexts).toEqual(["first", "second", "third"]);
|
||||
});
|
||||
|
||||
it("tool call response triggers tool execution", async () => {
|
||||
let toolExecuted = false;
|
||||
const echoTool: AgentTool = {
|
||||
name: "echo",
|
||||
label: "Echo",
|
||||
description: "Echo back",
|
||||
parameters: Type.Object({ text: Type.String() }),
|
||||
execute: async () => {
|
||||
toolExecuted = true;
|
||||
return { content: [{ type: "text", text: "echoed" }], details: {} };
|
||||
},
|
||||
};
|
||||
|
||||
harness = createHarness({
|
||||
responses: [{ toolCalls: [{ name: "echo", args: { text: "hi" } }] }, "done after tool"],
|
||||
tools: [echoTool],
|
||||
baseToolsOverride: { echo: echoTool },
|
||||
});
|
||||
|
||||
await harness.session.prompt("use the tool");
|
||||
|
||||
expect(toolExecuted).toBe(true);
|
||||
expect(harness.faux.callCount).toBe(2);
|
||||
|
||||
const toolResults = harness.session.messages.filter((m) => m.role === "toolResult");
|
||||
expect(toolResults).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("error response", async () => {
|
||||
harness = createHarness({
|
||||
responses: [{ error: "something broke" }],
|
||||
});
|
||||
|
||||
await harness.session.prompt("hi");
|
||||
|
||||
const assistantMessages = harness.session.messages.filter((m): m is AssistantMessage => m.role === "assistant");
|
||||
expect(assistantMessages).toHaveLength(1);
|
||||
expect(assistantMessages[0].stopReason).toBe("error");
|
||||
expect(assistantMessages[0].errorMessage).toBe("something broke");
|
||||
});
|
||||
|
||||
it("retry on transient error", async () => {
|
||||
harness = createHarness({
|
||||
responses: [{ error: "overloaded_error" }, "recovered"],
|
||||
settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } },
|
||||
});
|
||||
|
||||
await harness.session.prompt("hi");
|
||||
|
||||
expect(harness.faux.callCount).toBe(2);
|
||||
|
||||
const retryStarts = harness.eventsOfType("auto_retry_start");
|
||||
expect(retryStarts).toHaveLength(1);
|
||||
|
||||
const retryEnds = harness.eventsOfType("auto_retry_end");
|
||||
expect(retryEnds).toHaveLength(1);
|
||||
expect(retryEnds[0].success).toBe(true);
|
||||
});
|
||||
|
||||
it("custom usage numbers", async () => {
|
||||
harness = createHarness({
|
||||
responses: [{ text: "big response", usage: { input: 100000, output: 5000 } }],
|
||||
});
|
||||
|
||||
await harness.session.prompt("hi");
|
||||
|
||||
const msg = harness.session.messages.find((m): m is AssistantMessage => m.role === "assistant")!;
|
||||
expect(msg.usage.input).toBe(100000);
|
||||
expect(msg.usage.output).toBe(5000);
|
||||
});
|
||||
|
||||
it("event capture", async () => {
|
||||
harness = createHarness({ responses: ["hello"] });
|
||||
|
||||
await harness.session.prompt("hi");
|
||||
|
||||
const agentStarts = harness.eventsOfType("agent_start");
|
||||
expect(agentStarts).toHaveLength(1);
|
||||
|
||||
const agentEnds = harness.eventsOfType("agent_end");
|
||||
expect(agentEnds).toHaveLength(1);
|
||||
|
||||
const messageEnds = harness.eventsOfType("message_end");
|
||||
expect(messageEnds.length).toBeGreaterThanOrEqual(2); // user + assistant
|
||||
});
|
||||
|
||||
it("context capture", async () => {
|
||||
harness = createHarness({ responses: ["reply"] });
|
||||
|
||||
await harness.session.prompt("my question");
|
||||
|
||||
expect(harness.faux.contexts).toHaveLength(1);
|
||||
const ctx = harness.faux.contexts[0];
|
||||
const userMsg = ctx.messages.find((m) => m.role === "user");
|
||||
expect(userMsg).toBeDefined();
|
||||
});
|
||||
|
||||
it("wraps around when more calls than responses", async () => {
|
||||
harness = createHarness({ responses: ["a", "b"] });
|
||||
|
||||
await harness.session.prompt("1");
|
||||
await harness.session.prompt("2");
|
||||
await harness.session.prompt("3");
|
||||
|
||||
expect(harness.faux.callCount).toBe(3);
|
||||
|
||||
const texts = harness.session.messages
|
||||
.filter((m): m is AssistantMessage => m.role === "assistant")
|
||||
.map((m) => m.content.find((c) => c.type === "text")?.text);
|
||||
|
||||
expect(texts).toEqual(["a", "b", "a"]);
|
||||
});
|
||||
|
||||
it("streams text deltas", async () => {
|
||||
harness = createHarness({ responses: ["hello world"] });
|
||||
|
||||
await harness.session.prompt("hi");
|
||||
|
||||
const updates = harness.eventsOfType("message_update");
|
||||
const textDeltas = updates.filter((e) => e.assistantMessageEvent.type === "text_delta");
|
||||
expect(textDeltas.length).toBeGreaterThan(0);
|
||||
|
||||
// Deltas should reconstruct the full text
|
||||
const reconstructed = textDeltas.map((e) => (e.assistantMessageEvent as { delta: string }).delta).join("");
|
||||
expect(reconstructed).toBe("hello world");
|
||||
});
|
||||
|
||||
it("streams thinking deltas", async () => {
|
||||
harness = createHarness({
|
||||
responses: [{ thinking: "let me think about this", text: "answer" }],
|
||||
});
|
||||
|
||||
await harness.session.prompt("hi");
|
||||
|
||||
const updates = harness.eventsOfType("message_update");
|
||||
const thinkingStarts = updates.filter((e) => e.assistantMessageEvent.type === "thinking_start");
|
||||
const thinkingDeltas = updates.filter((e) => e.assistantMessageEvent.type === "thinking_delta");
|
||||
const thinkingEnds = updates.filter((e) => e.assistantMessageEvent.type === "thinking_end");
|
||||
|
||||
expect(thinkingStarts).toHaveLength(1);
|
||||
expect(thinkingDeltas.length).toBeGreaterThan(0);
|
||||
expect(thinkingEnds).toHaveLength(1);
|
||||
|
||||
const reconstructed = thinkingDeltas.map((e) => (e.assistantMessageEvent as { delta: string }).delta).join("");
|
||||
expect(reconstructed).toBe("let me think about this");
|
||||
});
|
||||
|
||||
it("streams tool call deltas", async () => {
|
||||
const echoTool: AgentTool = {
|
||||
name: "echo",
|
||||
label: "Echo",
|
||||
description: "Echo back",
|
||||
parameters: Type.Object({ text: Type.String() }),
|
||||
execute: async () => ({ content: [{ type: "text", text: "echoed" }], details: {} }),
|
||||
};
|
||||
|
||||
harness = createHarness({
|
||||
responses: [{ toolCalls: [{ name: "echo", args: { text: "hi" } }] }, "done"],
|
||||
tools: [echoTool],
|
||||
baseToolsOverride: { echo: echoTool },
|
||||
});
|
||||
|
||||
await harness.session.prompt("use tool");
|
||||
|
||||
const updates = harness.eventsOfType("message_update");
|
||||
const toolcallStarts = updates.filter((e) => e.assistantMessageEvent.type === "toolcall_start");
|
||||
const toolcallDeltas = updates.filter((e) => e.assistantMessageEvent.type === "toolcall_delta");
|
||||
const toolcallEnds = updates.filter((e) => e.assistantMessageEvent.type === "toolcall_end");
|
||||
|
||||
expect(toolcallStarts).toHaveLength(1);
|
||||
expect(toolcallDeltas.length).toBeGreaterThan(0);
|
||||
expect(toolcallEnds).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("streams thinking then text then tool call in order", async () => {
|
||||
const echoTool: AgentTool = {
|
||||
name: "echo",
|
||||
label: "Echo",
|
||||
description: "Echo back",
|
||||
parameters: Type.Object({ text: Type.String() }),
|
||||
execute: async () => ({ content: [{ type: "text", text: "echoed" }], details: {} }),
|
||||
};
|
||||
|
||||
harness = createHarness({
|
||||
responses: [
|
||||
{
|
||||
thinking: "hmm",
|
||||
text: "I will call a tool",
|
||||
toolCalls: [{ name: "echo", args: { text: "x" } }],
|
||||
},
|
||||
"final",
|
||||
],
|
||||
tools: [echoTool],
|
||||
baseToolsOverride: { echo: echoTool },
|
||||
});
|
||||
|
||||
await harness.session.prompt("do it");
|
||||
|
||||
const updates = harness.eventsOfType("message_update");
|
||||
const streamTypes = updates.map((e) => e.assistantMessageEvent.type);
|
||||
|
||||
// Thinking events should come before text events, text before toolcall
|
||||
const firstThinking = streamTypes.indexOf("thinking_start");
|
||||
const firstText = streamTypes.indexOf("text_start");
|
||||
const firstToolcall = streamTypes.indexOf("toolcall_start");
|
||||
|
||||
expect(firstThinking).toBeLessThan(firstText);
|
||||
expect(firstText).toBeLessThan(firstToolcall);
|
||||
});
|
||||
|
||||
it("session persistence works", async () => {
|
||||
harness = createHarness({ responses: ["persisted"] });
|
||||
|
||||
await harness.session.prompt("hi");
|
||||
|
||||
const entries = harness.sessionManager.getEntries();
|
||||
const messageEntries = entries.filter((e) => e.type === "message");
|
||||
expect(messageEntries.length).toBeGreaterThanOrEqual(2); // user + assistant
|
||||
});
|
||||
});
|
||||
414
packages/coding-agent/test/test-harness.ts
Normal file
414
packages/coding-agent/test/test-harness.ts
Normal file
@@ -0,0 +1,414 @@
|
||||
/**
|
||||
* Test harness for AgentSession runtime testing.
|
||||
*
|
||||
* Provides:
|
||||
* - A faux stream function with declarative response sequencing
|
||||
* - A one-call factory for a fully wired AgentSession with real in-memory dependencies
|
||||
* - Event capture for assertions
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AgentTool } from "@mariozechner/pi-agent-core";
|
||||
import { Agent } from "@mariozechner/pi-agent-core";
|
||||
import type {
|
||||
AssistantMessage,
|
||||
AssistantMessageEvent,
|
||||
AssistantMessageEventStream,
|
||||
Context,
|
||||
Model,
|
||||
SimpleStreamOptions,
|
||||
StopReason,
|
||||
TextContent,
|
||||
ThinkingContent,
|
||||
ToolCall,
|
||||
Usage,
|
||||
} from "@mariozechner/pi-ai";
|
||||
import { createAssistantMessageEventStream } from "@mariozechner/pi-ai";
|
||||
import { AgentSession, type AgentSessionEvent } from "../src/core/agent-session.js";
|
||||
import { AuthStorage } from "../src/core/auth-storage.js";
|
||||
import { ModelRegistry } from "../src/core/model-registry.js";
|
||||
import { SessionManager } from "../src/core/session-manager.js";
|
||||
import type { Settings } from "../src/core/settings-manager.js";
|
||||
import { SettingsManager } from "../src/core/settings-manager.js";
|
||||
import { createTestResourceLoader } from "./utilities.js";
|
||||
|
||||
// ============================================================================
|
||||
// Faux model
|
||||
// ============================================================================
|
||||
|
||||
const FAUX_PROVIDER = "faux";
|
||||
const FAUX_MODEL_ID = "faux-1";
|
||||
const FAUX_API = "anthropic-messages" as const;
|
||||
|
||||
export const fauxModel: Model<typeof FAUX_API> = {
|
||||
id: FAUX_MODEL_ID,
|
||||
name: "Faux Model",
|
||||
api: FAUX_API,
|
||||
provider: FAUX_PROVIDER,
|
||||
baseUrl: "http://localhost:0",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 16384,
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Response description
|
||||
// ============================================================================
|
||||
|
||||
export interface FauxResponse {
|
||||
/** Text content blocks. String shorthand becomes a single text block. */
|
||||
text?: string;
|
||||
/** Tool calls to include in the response. */
|
||||
toolCalls?: Array<{ id?: string; name: string; args: Record<string, unknown> }>;
|
||||
/** Thinking content. */
|
||||
thinking?: string;
|
||||
/** Stop reason. Defaults to "stop", or "toolUse" if toolCalls are present, or "error" if error is set. */
|
||||
stopReason?: StopReason;
|
||||
/** Error message. Sets stopReason to "error" if not explicitly set. */
|
||||
error?: string;
|
||||
/** Usage numbers. Merged with defaults (input: 100, output: 50). */
|
||||
usage?: Partial<Usage>;
|
||||
/** Delay in ms before the response starts. */
|
||||
delayMs?: number;
|
||||
/** Model overrides (provider, model id) for responses that should look like they came from a different model. */
|
||||
model?: { provider?: string; id?: string };
|
||||
}
|
||||
|
||||
/** Shorthand: a string becomes a simple text response. */
|
||||
export type FauxResponseInput = FauxResponse | string;
|
||||
|
||||
// ============================================================================
|
||||
// Faux stream function
|
||||
// ============================================================================
|
||||
|
||||
function normalizeResponse(input: FauxResponseInput): FauxResponse {
|
||||
if (typeof input === "string") {
|
||||
return { text: input };
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
function buildUsage(partial?: Partial<Usage>): Usage {
|
||||
const input = partial?.input ?? 100;
|
||||
const output = partial?.output ?? 50;
|
||||
const cacheRead = partial?.cacheRead ?? 0;
|
||||
const cacheWrite = partial?.cacheWrite ?? 0;
|
||||
return {
|
||||
input,
|
||||
output,
|
||||
cacheRead,
|
||||
cacheWrite,
|
||||
totalTokens: partial?.totalTokens ?? input + output + cacheRead + cacheWrite,
|
||||
cost: partial?.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
let toolCallIdCounter = 0;
|
||||
|
||||
function buildAssistantMessage(resp: FauxResponse): AssistantMessage {
|
||||
const content: (TextContent | ThinkingContent | ToolCall)[] = [];
|
||||
|
||||
if (resp.thinking) {
|
||||
content.push({ type: "thinking", thinking: resp.thinking });
|
||||
}
|
||||
if (resp.text !== undefined) {
|
||||
content.push({ type: "text", text: resp.text });
|
||||
}
|
||||
if (resp.toolCalls) {
|
||||
for (const tc of resp.toolCalls) {
|
||||
content.push({
|
||||
type: "toolCall",
|
||||
id: tc.id ?? `faux_tc_${++toolCallIdCounter}`,
|
||||
name: tc.name,
|
||||
arguments: tc.args,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// If no content was added at all, add empty text
|
||||
if (content.length === 0 && !resp.error) {
|
||||
content.push({ type: "text", text: "" });
|
||||
}
|
||||
|
||||
let stopReason: StopReason;
|
||||
if (resp.stopReason) {
|
||||
stopReason = resp.stopReason;
|
||||
} else if (resp.error) {
|
||||
stopReason = "error";
|
||||
} else if (resp.toolCalls && resp.toolCalls.length > 0) {
|
||||
stopReason = "toolUse";
|
||||
} else {
|
||||
stopReason = "stop";
|
||||
}
|
||||
|
||||
return {
|
||||
role: "assistant",
|
||||
content,
|
||||
api: FAUX_API,
|
||||
provider: resp.model?.provider ?? FAUX_PROVIDER,
|
||||
model: resp.model?.id ?? FAUX_MODEL_ID,
|
||||
usage: buildUsage(resp.usage),
|
||||
stopReason,
|
||||
errorMessage: resp.error,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Token-level streaming
|
||||
// ============================================================================
|
||||
|
||||
/** Split a string into chunks of varying size (3-5 chars) for simulating token-by-token streaming. */
|
||||
function chunkString(text: string): string[] {
|
||||
const chunks: string[] = [];
|
||||
let i = 0;
|
||||
while (i < text.length) {
|
||||
const size = 3 + Math.floor(Math.random() * 3); // 3, 4, or 5
|
||||
chunks.push(text.slice(i, i + size));
|
||||
i += size;
|
||||
}
|
||||
return chunks.length > 0 ? chunks : [""];
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream a complete AssistantMessage through an EventStream with realistic
|
||||
* intermediate delta events for each content block.
|
||||
*/
|
||||
function streamWithDeltas(stream: AssistantMessageEventStream, message: AssistantMessage): void {
|
||||
const isError = message.stopReason === "error" || message.stopReason === "aborted";
|
||||
|
||||
// Build partial progressively as we stream content blocks
|
||||
const partial: AssistantMessage = { ...message, content: [] };
|
||||
stream.push({ type: "start", partial: { ...partial } });
|
||||
|
||||
for (let i = 0; i < message.content.length; i++) {
|
||||
const block = message.content[i];
|
||||
|
||||
if (block.type === "thinking") {
|
||||
partial.content = [...partial.content, { type: "thinking", thinking: "" }];
|
||||
stream.push({ type: "thinking_start", contentIndex: i, partial: { ...partial } });
|
||||
|
||||
for (const chunk of chunkString(block.thinking)) {
|
||||
(partial.content[i] as ThinkingContent).thinking += chunk;
|
||||
stream.push(makeEvent("thinking_delta", i, chunk, partial));
|
||||
}
|
||||
|
||||
stream.push({
|
||||
type: "thinking_end",
|
||||
contentIndex: i,
|
||||
content: block.thinking,
|
||||
partial: { ...partial },
|
||||
});
|
||||
} else if (block.type === "text") {
|
||||
partial.content = [...partial.content, { type: "text", text: "" }];
|
||||
stream.push({ type: "text_start", contentIndex: i, partial: { ...partial } });
|
||||
|
||||
for (const chunk of chunkString(block.text)) {
|
||||
(partial.content[i] as TextContent).text += chunk;
|
||||
stream.push(makeEvent("text_delta", i, chunk, partial));
|
||||
}
|
||||
|
||||
stream.push({
|
||||
type: "text_end",
|
||||
contentIndex: i,
|
||||
content: block.text,
|
||||
partial: { ...partial },
|
||||
});
|
||||
} else if (block.type === "toolCall") {
|
||||
const argsJson = JSON.stringify(block.arguments);
|
||||
partial.content = [...partial.content, { type: "toolCall", id: block.id, name: block.name, arguments: {} }];
|
||||
stream.push({ type: "toolcall_start", contentIndex: i, partial: { ...partial } });
|
||||
|
||||
for (const chunk of chunkString(argsJson)) {
|
||||
stream.push(makeEvent("toolcall_delta", i, chunk, partial));
|
||||
}
|
||||
|
||||
// Final toolcall has the real parsed arguments
|
||||
(partial.content[i] as ToolCall).arguments = block.arguments;
|
||||
stream.push({
|
||||
type: "toolcall_end",
|
||||
contentIndex: i,
|
||||
toolCall: block,
|
||||
partial: { ...partial },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (isError) {
|
||||
stream.push({ type: "error", reason: message.stopReason as "error" | "aborted", error: message });
|
||||
} else {
|
||||
stream.push({ type: "done", reason: message.stopReason as "stop" | "length" | "toolUse", message });
|
||||
}
|
||||
}
|
||||
|
||||
function makeEvent(
|
||||
type: "text_delta" | "thinking_delta" | "toolcall_delta",
|
||||
contentIndex: number,
|
||||
delta: string,
|
||||
partial: AssistantMessage,
|
||||
): AssistantMessageEvent {
|
||||
return { type, contentIndex, delta, partial: { ...partial } };
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Stream function factory
|
||||
// ============================================================================
|
||||
|
||||
export interface FauxStreamFnState {
|
||||
/** Number of times the stream function has been called. */
|
||||
callCount: number;
|
||||
/** The context passed to each call, in order. */
|
||||
contexts: Context[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a faux stream function from a sequence of response descriptions.
|
||||
*
|
||||
* The function cycles through responses in order. If more calls are made than
|
||||
* responses provided, it wraps around.
|
||||
*
|
||||
* Returns the stream function and a state object for inspection.
|
||||
*/
|
||||
export function createFauxStreamFn(responses: FauxResponseInput[]): {
|
||||
streamFn: (model: Model<any>, context: Context, options?: SimpleStreamOptions) => AssistantMessageEventStream;
|
||||
state: FauxStreamFnState;
|
||||
} {
|
||||
if (responses.length === 0) {
|
||||
throw new Error("createFauxStreamFn requires at least one response");
|
||||
}
|
||||
|
||||
const state: FauxStreamFnState = { callCount: 0, contexts: [] };
|
||||
|
||||
const streamFn = (_model: Model<any>, context: Context, _options?: SimpleStreamOptions) => {
|
||||
const index = state.callCount % responses.length;
|
||||
state.callCount++;
|
||||
state.contexts.push(context);
|
||||
|
||||
const resp = normalizeResponse(responses[index]);
|
||||
const message = buildAssistantMessage(resp);
|
||||
const stream = createAssistantMessageEventStream();
|
||||
|
||||
const emit = () => {
|
||||
streamWithDeltas(stream, message);
|
||||
};
|
||||
|
||||
if (resp.delayMs && resp.delayMs > 0) {
|
||||
setTimeout(emit, resp.delayMs);
|
||||
} else {
|
||||
queueMicrotask(emit);
|
||||
}
|
||||
|
||||
return stream;
|
||||
};
|
||||
|
||||
return { streamFn, state };
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Session harness
|
||||
// ============================================================================
|
||||
|
||||
export interface HarnessOptions {
|
||||
/** Response sequence for the faux provider. Default: single "ok" response. */
|
||||
responses?: FauxResponseInput[];
|
||||
/** Model to use. Default: fauxModel. */
|
||||
model?: Model<any>;
|
||||
/** Context window override (applied to the model). */
|
||||
contextWindow?: number;
|
||||
/** Settings overrides (retry, compaction, etc.). */
|
||||
settings?: Partial<Settings>;
|
||||
/** System prompt. Default: "You are a test assistant." */
|
||||
systemPrompt?: string;
|
||||
/** Custom tools to register on the agent. */
|
||||
tools?: AgentTool[];
|
||||
/** Base tools override (replaces built-in read/bash/edit/write). */
|
||||
baseToolsOverride?: Record<string, AgentTool>;
|
||||
}
|
||||
|
||||
export interface Harness {
|
||||
session: AgentSession;
|
||||
agent: Agent;
|
||||
sessionManager: SessionManager;
|
||||
settingsManager: SettingsManager;
|
||||
/** Faux stream function state (call count, captured contexts). */
|
||||
faux: FauxStreamFnState;
|
||||
/** All events emitted by the session, in order. */
|
||||
events: AgentSessionEvent[];
|
||||
/** Filter captured events by type. */
|
||||
eventsOfType<T extends AgentSessionEvent["type"]>(type: T): Extract<AgentSessionEvent, { type: T }>[];
|
||||
/** Temp directory (cleaned up by cleanup()). */
|
||||
tempDir: string;
|
||||
/** Dispose session and remove temp directory. */
|
||||
cleanup: () => void;
|
||||
}
|
||||
|
||||
export function createHarness(options: HarnessOptions = {}): Harness {
|
||||
const tempDir = join(tmpdir(), `pi-harness-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
const baseModel = options.model ?? fauxModel;
|
||||
const model: Model<any> = options.contextWindow ? { ...baseModel, contextWindow: options.contextWindow } : baseModel;
|
||||
|
||||
const { streamFn, state: fauxState } = createFauxStreamFn(options.responses ?? ["ok"]);
|
||||
|
||||
const agent = new Agent({
|
||||
getApiKey: () => "faux-key",
|
||||
initialState: {
|
||||
model,
|
||||
systemPrompt: options.systemPrompt ?? "You are a test assistant.",
|
||||
tools: options.tools ?? [],
|
||||
},
|
||||
streamFn,
|
||||
});
|
||||
|
||||
const sessionManager = SessionManager.inMemory();
|
||||
const settingsManager = SettingsManager.create(tempDir, tempDir);
|
||||
|
||||
if (options.settings) {
|
||||
settingsManager.applyOverrides(options.settings);
|
||||
}
|
||||
|
||||
const authStorage = AuthStorage.create(join(tempDir, "auth.json"));
|
||||
authStorage.setRuntimeApiKey(model.provider, "faux-key");
|
||||
const modelRegistry = new ModelRegistry(authStorage, tempDir);
|
||||
|
||||
const session = new AgentSession({
|
||||
agent,
|
||||
sessionManager,
|
||||
settingsManager,
|
||||
cwd: tempDir,
|
||||
modelRegistry,
|
||||
resourceLoader: createTestResourceLoader(),
|
||||
baseToolsOverride: options.baseToolsOverride,
|
||||
});
|
||||
|
||||
const events: AgentSessionEvent[] = [];
|
||||
session.subscribe((event) => {
|
||||
events.push(event);
|
||||
});
|
||||
|
||||
const cleanup = () => {
|
||||
session.dispose();
|
||||
if (existsSync(tempDir)) {
|
||||
rmSync(tempDir, { recursive: true });
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
session,
|
||||
agent,
|
||||
sessionManager,
|
||||
settingsManager,
|
||||
faux: fauxState,
|
||||
events,
|
||||
eventsOfType<T extends AgentSessionEvent["type"]>(type: T) {
|
||||
return events.filter((e): e is Extract<AgentSessionEvent, { type: T }> => e.type === type);
|
||||
},
|
||||
tempDir,
|
||||
cleanup,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user