From f348a06294f12792858c083dac2bcc462988a3d3 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sun, 10 May 2026 19:10:54 +0200 Subject: [PATCH] test(agent): cover harness stream configuration --- packages/agent/docs/agent-harness.md | 55 ++++- .../test/harness/agent-harness-stream.test.ts | 206 ++++++++++++++++++ 2 files changed, 251 insertions(+), 10 deletions(-) create mode 100644 packages/agent/test/harness/agent-harness-stream.test.ts diff --git a/packages/agent/docs/agent-harness.md b/packages/agent/docs/agent-harness.md index b3a57377..5baccdff 100644 --- a/packages/agent/docs/agent-harness.md +++ b/packages/agent/docs/agent-harness.md @@ -193,11 +193,46 @@ Branch summary generation is part of the tree navigation operation. Auto-compaction and retry decision points are not implemented in `AgentHarness` yet. +## Test organization + +Harness tests should stay focused by area instead of growing one large catch-all file. + +Current structure: + +- `packages/agent/test/harness/agent-harness.test.ts`: basic construction/API smoke tests. +- `packages/agent/test/harness/agent-harness-stream.test.ts`: stream options and provider hook semantics. + +Preferred future structure: + +- `agent-harness-resources.test.ts`: resource snapshot/loading semantics. +- `agent-harness-tools.test.ts`: tool registry getters, active-tool semantics, and update events. +- `agent-harness-lifecycle.test.ts`: phase/save-point/settled/reentrancy behavior. + +Use the `pi-ai` faux provider (`registerFauxProvider`, `fauxAssistantMessage`) for deterministic harness/provider tests. Faux response factories can inspect `StreamOptions`, invoke `options.onPayload`, and return scripted assistant messages without real provider APIs or network access. + ## Implementation todo This list tracks the remaining work before treating `AgentHarness` as migration-ready. -### 1. Finish curated provider/stream configuration +### 1. Remove `Agent` dependency from `AgentHarness` + +New top priority. + +`AgentHarness` should likely call `agentLoop` / `agentLoopContinue` directly instead of owning an internal `Agent` instance. The harness already owns session persistence, runtime config snapshots, queues, provider stream configuration, hooks/events, phase semantics, and abort semantics. Keeping `Agent` in the middle creates duplicated state and adapter seams. + +Still needed: + +- Replace internal `new Agent(...)` with direct low-level loop calls. +- Move active run/abort-controller lifecycle into `AgentHarness`. +- Move queue draining into `AgentHarness` only, removing duplicated low-level `Agent` queues. +- Reduce low-level `AgentEvent` state directly in the harness where needed. +- Preserve current public behavior for `prompt`, `skill`, `promptFromTemplate`, `steer`, `followUp`, `nextTurn`, `abort`, and `waitForIdle`. +- Preserve provider hook behavior implemented by the harness stream wrapper. +- Preserve save-point snapshot refresh semantics without side-effecting through `Agent.prepareNextTurn`. +- Decide whether `AgentHarness.agent` remains temporarily for compatibility or is removed before migration. +- Add tests covering parity with the current harness behavior before and after the refactor. + +### 2. Finish curated provider/stream configuration Implemented so far: @@ -226,12 +261,12 @@ Implemented provider hook behavior: 3. `before_provider_request` patches, in hook registration order - `before_provider_request` does not patch `reasoning`; add that only if a concrete use case appears. -Still needed: +Implemented validation: -- Add tests proving stream options are snapshotted per turn and changes while busy affect only future provider requests/save-point snapshots. -- Add tests proving provider hook patch/deletion/chaining semantics. +- `packages/agent/test/harness/agent-harness-stream.test.ts` uses the `pi-ai` faux provider. +- Tests cover stream option forwarding, auth header merge, request hook patching, request hook deletion semantics, request hook chaining, payload hook chaining, and busy/save-point snapshot behavior. -### 2. Design per-`AgentHarness` model registry +### 3. Design per-`AgentHarness` model registry Not started. @@ -243,7 +278,7 @@ Still needed: - Define model change semantics during active turns and save points. - Preserve current `setModel()` behavior until the registry model is designed. -### 3. Design generic hook/event extension mechanism +### 4. Design generic hook/event extension mechanism Current cleanup already done: @@ -266,7 +301,7 @@ Still needed: - possibly `tool_result`: each hook receives the result fields produced by previous hooks. - Do not chain hooks where semantics are policy-based or ambiguous until explicitly designed, such as `tool_call`, `session_before_compact`, `session_before_tree`, and `before_agent_start`. -### 4. Add explicit tool registry read/update semantics +### 5. Add explicit tool registry read/update semantics Implemented so far: @@ -286,7 +321,7 @@ Still needed: - Decide and implement tool update observability events. - Include active-tool-only updates in the uniform runtime config observability plan. -### 5. Full `AgentHarness` lifecycle/state pass +### 6. Full `AgentHarness` lifecycle/state pass Implemented so far: @@ -320,7 +355,7 @@ Still needed: - Document or change timing for model/thinking/stream-option events that may fire before queued session entries persist while busy. - Audit `abort()` barrier semantics. -### 6. Later coding-agent migration plan +### 7. Later coding-agent migration plan Not started. @@ -332,7 +367,7 @@ Still needed: - Preserve UI/session behavior outside core. - Move coding-agent stream/auth/retry/header behavior onto the harness stream configuration and provider hooks. -### 7. Final lifecycle hardening suite +### 8. Final lifecycle hardening suite Before treating `AgentHarness` as migration-ready, add a broad test suite that exercises listeners and hooks closing over the harness and calling public APIs during every relevant event. diff --git a/packages/agent/test/harness/agent-harness-stream.test.ts b/packages/agent/test/harness/agent-harness-stream.test.ts new file mode 100644 index 00000000..e014b377 --- /dev/null +++ b/packages/agent/test/harness/agent-harness-stream.test.ts @@ -0,0 +1,206 @@ +import { fauxAssistantMessage, fauxToolCall, registerFauxProvider, type StreamOptions } from "@earendil-works/pi-ai"; +import { afterEach, describe, expect, it } from "vitest"; +import { AgentHarness } from "../../src/harness/agent-harness.js"; +import { NodeExecutionEnv } from "../../src/harness/execution-env.js"; +import { Session } from "../../src/harness/session/session.js"; +import { InMemorySessionStorage } from "../../src/harness/session/storage/memory.js"; +import { calculateTool } from "../utils/calculate.js"; + +const registrations: Array<{ unregister(): void }> = []; + +afterEach(() => { + for (const registration of registrations.splice(0)) { + registration.unregister(); + } +}); + +function createHarness(options: ConstructorParameters[0]): AgentHarness { + return new AgentHarness(options); +} + +function captureOptions(options: StreamOptions | undefined): StreamOptions { + return { + ...options, + headers: options?.headers ? { ...options.headers } : undefined, + metadata: options?.metadata ? { ...options.metadata } : undefined, + }; +} + +describe("AgentHarness stream configuration", () => { + it("snapshots stream options and merges auth headers before provider request hooks", async () => { + let capturedOptions: StreamOptions | undefined; + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([ + (_context, options) => { + capturedOptions = options; + return fauxAssistantMessage("ok"); + }, + ]); + + const session = new Session(new InMemorySessionStorage({ metadata: { id: "session-1", createdAt: "now" } })); + const harness = createHarness({ + env: new NodeExecutionEnv({ cwd: process.cwd() }), + session, + model: registration.getModel(), + streamOptions: { + timeoutMs: 1000, + maxRetries: 2, + maxRetryDelayMs: 3000, + headers: { "x-base": "base" }, + metadata: { base: true }, + cacheRetention: "none", + }, + getApiKeyAndHeaders: async () => ({ apiKey: "secret", headers: { "x-auth": "auth" } }), + }); + + harness.on("before_provider_request", (event) => { + expect(event.sessionId).toBe("session-1"); + expect(event.streamOptions.headers).toEqual({ "x-base": "base", "x-auth": "auth" }); + return { + streamOptions: { + headers: { "x-hook": "hook" }, + metadata: { hook: true }, + }, + }; + }); + + await harness.prompt("hello"); + + expect(capturedOptions).toMatchObject({ + apiKey: "secret", + timeoutMs: 1000, + maxRetries: 2, + maxRetryDelayMs: 3000, + sessionId: "session-1", + cacheRetention: "none", + }); + expect(capturedOptions?.headers).toEqual({ "x-base": "base", "x-auth": "auth", "x-hook": "hook" }); + expect(capturedOptions?.metadata).toEqual({ base: true, hook: true }); + }); + + it("chains provider request patches and supports deletion semantics", async () => { + let capturedOptions: StreamOptions | undefined; + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([ + (_context, options) => { + capturedOptions = options; + return fauxAssistantMessage("ok"); + }, + ]); + + const harness = createHarness({ + env: new NodeExecutionEnv({ cwd: process.cwd() }), + session: new Session(new InMemorySessionStorage()), + model: registration.getModel(), + streamOptions: { + timeoutMs: 1000, + maxRetries: 2, + headers: { keep: "base", remove: "base" }, + metadata: { keep: "base", remove: "base" }, + }, + }); + + harness.on("before_provider_request", (event) => { + expect(event.streamOptions.headers).toEqual({ keep: "base", remove: "base" }); + return { + streamOptions: { + headers: { first: "1", remove: undefined }, + metadata: { first: 1, remove: undefined }, + }, + }; + }); + harness.on("before_provider_request", (event) => { + expect(event.streamOptions.headers).toEqual({ keep: "base", first: "1" }); + expect(event.streamOptions.metadata).toEqual({ keep: "base", first: 1 }); + return { + streamOptions: { + timeoutMs: undefined, + headers: { second: "2" }, + metadata: undefined, + }, + }; + }); + + await harness.prompt("hello"); + + expect(capturedOptions?.timeoutMs).toBeUndefined(); + expect(capturedOptions?.maxRetries).toBe(2); + expect(capturedOptions?.headers).toEqual({ keep: "base", first: "1", second: "2" }); + expect(capturedOptions?.metadata).toBeUndefined(); + }); + + it("uses updated stream options for save-point snapshots without mutating the active request", async () => { + const capturedOptions: StreamOptions[] = []; + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([ + (_context, options) => { + capturedOptions.push(captureOptions(options)); + return fauxAssistantMessage(fauxToolCall("calculate", { expression: "1 + 1" }, { id: "call-1" }), { + stopReason: "toolUse", + }); + }, + (_context, options) => { + capturedOptions.push(captureOptions(options)); + return fauxAssistantMessage("done"); + }, + ]); + + const harness = createHarness({ + env: new NodeExecutionEnv({ cwd: process.cwd() }), + session: new Session(new InMemorySessionStorage()), + model: registration.getModel(), + tools: [calculateTool], + streamOptions: { timeoutMs: 1000, headers: { turn: "first" } }, + }); + + harness.subscribe((event) => { + if (event.type === "tool_execution_start") { + harness.setStreamOptions({ timeoutMs: 2000, headers: { turn: "second" } }); + } + }); + + await harness.prompt("hello"); + + expect(capturedOptions).toHaveLength(2); + expect(capturedOptions[0].timeoutMs).toBe(1000); + expect(capturedOptions[0].headers).toEqual({ turn: "first" }); + expect(capturedOptions[1].timeoutMs).toBe(2000); + expect(capturedOptions[1].headers).toEqual({ turn: "second" }); + }); + + it("chains provider payload hooks", async () => { + const seenPayloads: unknown[] = []; + let finalPayload: unknown; + const registration = registerFauxProvider(); + registrations.push(registration); + registration.setResponses([ + async (_context, options, _state, model) => { + finalPayload = await options?.onPayload?.({ steps: ["provider"] }, model); + return fauxAssistantMessage("ok"); + }, + ]); + + const harness = createHarness({ + env: new NodeExecutionEnv({ cwd: process.cwd() }), + session: new Session(new InMemorySessionStorage()), + model: registration.getModel(), + }); + + harness.on("before_provider_payload", (event) => { + seenPayloads.push(event.payload); + return { payload: { steps: ["provider", "first"] } }; + }); + harness.on("before_provider_payload", (event) => { + seenPayloads.push(event.payload); + return { payload: { steps: ["provider", "first", "second"] } }; + }); + + await harness.prompt("hello"); + + expect(seenPayloads).toEqual([{ steps: ["provider"] }, { steps: ["provider", "first"] }]); + expect(finalPayload).toEqual({ steps: ["provider", "first", "second"] }); + }); +});