From 957f21cea8fe914aca2c3ef6fcc81b9b1a6d7f20 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 1 Apr 2026 14:31:02 +0200 Subject: [PATCH] test(coding-agent): add queue characterization coverage --- AGENTS.md | 2 + packages/coding-agent/test/suite/README.md | 16 ++ .../test/suite/agent-session-queue.test.ts | 192 ++++++++++++++++++ 3 files changed, 210 insertions(+) create mode 100644 packages/coding-agent/test/suite/README.md create mode 100644 packages/coding-agent/test/suite/agent-session-queue.test.ts diff --git a/AGENTS.md b/AGENTS.md index 0ce6db2e..c18d9977 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,8 @@ read README.md, then ask which module(s) to work on. Based on the answer, read t - Run tests from the package root, not the repo root. - If you create or modify a test file, you MUST run that test file and iterate until it passes. - When writing tests, run them, identify issues in either the test or implementation, and iterate until fixed. +- For `packages/coding-agent/test/suite/`, use `test/suite/harness.ts` plus the faux provider. Do not use real provider APIs, real API keys, or paid tokens. +- Put issue-specific regressions under `packages/coding-agent/test/suite/regressions/` and name them `-.test.ts`. - NEVER commit unless user asks ## GitHub Issues diff --git a/packages/coding-agent/test/suite/README.md b/packages/coding-agent/test/suite/README.md new file mode 100644 index 00000000..6017634d --- /dev/null +++ b/packages/coding-agent/test/suite/README.md @@ -0,0 +1,16 @@ +# Coding agent suite tests + +Use `test/suite/` for the new harness-based test suite around `AgentSession` and `AgentSessionRuntimeHost`. + +Rules: +- Use `test/suite/harness.ts` +- Use the faux provider from `packages/ai/src/providers/faux.ts` +- Do not use real provider APIs, real API keys, network calls, or paid tokens +- Keep these tests CI-safe and deterministic +- Do not use or extend the legacy `test/test-harness.ts` path unless a missing capability forces it + +Organization: +- Put broad lifecycle and characterization tests directly under `test/suite/` +- Put issue-specific regression tests under `test/suite/regressions/` +- Name regression tests as `-.test.ts` +- Example: `test/suite/regressions/2023-queued-slash-command-followup.test.ts` diff --git a/packages/coding-agent/test/suite/agent-session-queue.test.ts b/packages/coding-agent/test/suite/agent-session-queue.test.ts new file mode 100644 index 00000000..226227aa --- /dev/null +++ b/packages/coding-agent/test/suite/agent-session-queue.test.ts @@ -0,0 +1,192 @@ +import type { AgentTool } from "@mariozechner/pi-agent-core"; +import { fauxAssistantMessage, fauxToolCall } from "@mariozechner/pi-ai"; +import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; +import { Type } from "@sinclair/typebox"; +import { afterEach, describe, expect, it } from "vitest"; +import { createHarness, type Harness } from "./harness.js"; + +function getUserTexts(harness: Harness): string[] { + return harness.session.messages + .filter((message) => message.role === "user") + .map((message) => { + const content = message.content; + if (typeof content === "string") { + return content; + } + return content + .filter((part): part is { type: "text"; text: string } => part.type === "text") + .map((part) => part.text) + .join("\n"); + }); +} + +function getAssistantTexts(harness: Harness): string[] { + return harness.session.messages + .filter((message) => message.role === "assistant") + .map((message) => + message.content + .filter((part): part is { type: "text"; text: string } => part.type === "text") + .map((part) => part.text) + .join("\n"), + ); +} + +describe("AgentSession queue characterization", () => { + const harnesses: Harness[] = []; + + afterEach(() => { + while (harnesses.length > 0) { + harnesses.pop()?.cleanup(); + } + }); + + it("dispatches extension commands immediately when prompted while idle", async () => { + const commandRuns: string[] = []; + const harness = await createHarness({ + extensionFactories: [ + (pi) => { + pi.registerCommand("testcmd", { + description: "Test command", + handler: async (args) => { + commandRuns.push(args); + }, + }); + }, + ], + }); + harnesses.push(harness); + + await harness.session.prompt("/testcmd hello world"); + + expect(commandRuns).toEqual(["hello world"]); + expect(harness.getPendingResponseCount()).toBe(0); + expect(harness.session.messages).toEqual([]); + }); + + it("delivers extension-origin steering messages before the next LLM call", async () => { + let extensionApi: ExtensionAPI | undefined; + let releaseToolExecution: (() => void) | undefined; + const toolStarted = new Promise((resolve) => { + releaseToolExecution = resolve; + }); + const waitTool: AgentTool = { + name: "wait", + label: "Wait", + description: "Wait for the test to release execution", + parameters: Type.Object({}), + execute: async () => { + await toolStarted; + return { + content: [{ type: "text", text: "released" }], + details: {}, + }; + }, + }; + const harness = await createHarness({ + tools: [waitTool], + extensionFactories: [ + (pi) => { + extensionApi = pi; + }, + ], + }); + harnesses.push(harness); + + harness.setResponses([ + fauxAssistantMessage(fauxToolCall("wait", {}), { stopReason: "toolUse" }), + (context) => { + const sawSteer = context.messages.some( + (message) => + message.role === "user" && + typeof message.content !== "string" && + message.content.some((part) => part.type === "text" && part.text === "steer now"), + ); + return fauxAssistantMessage(sawSteer ? "saw steer" : "missing steer"); + }, + ]); + + const sawToolStart = new Promise((resolve) => { + const unsubscribe = harness.session.subscribe((event) => { + if (event.type === "tool_execution_start" && event.toolName === "wait") { + unsubscribe(); + resolve(); + } + }); + }); + + const promptPromise = harness.session.prompt("start"); + await sawToolStart; + await new Promise((resolve) => setTimeout(resolve, 0)); + + extensionApi?.sendUserMessage("steer now", { deliverAs: "steer" }); + releaseToolExecution?.(); + await promptPromise; + + expect(getUserTexts(harness)).toEqual(["start", "steer now"]); + expect(getAssistantTexts(harness)).toContain("saw steer"); + }); + + it("treats queued slash-command follow-ups as raw user text instead of dispatching the command", async () => { + let extensionApi: ExtensionAPI | undefined; + const commandRuns: string[] = []; + let releaseToolExecution: (() => void) | undefined; + const toolRelease = new Promise((resolve) => { + releaseToolExecution = resolve; + }); + const waitTool: AgentTool = { + name: "wait", + label: "Wait", + description: "Wait for the test to release execution", + parameters: Type.Object({}), + execute: async () => { + await toolRelease; + return { + content: [{ type: "text", text: "released" }], + details: {}, + }; + }, + }; + const harness = await createHarness({ + tools: [waitTool], + extensionFactories: [ + (pi) => { + extensionApi = pi; + pi.registerCommand("testcmd", { + description: "Test command", + handler: async (args) => { + commandRuns.push(args); + }, + }); + }, + ], + }); + harnesses.push(harness); + + harness.setResponses([ + fauxAssistantMessage(fauxToolCall("wait", {}), { stopReason: "toolUse" }), + fauxAssistantMessage("first turn complete"), + fauxAssistantMessage("queued follow-up handled by model"), + ]); + + const sawToolStart = new Promise((resolve) => { + const unsubscribe = harness.session.subscribe((event) => { + if (event.type === "tool_execution_start" && event.toolName === "wait") { + unsubscribe(); + resolve(); + } + }); + }); + + const promptPromise = harness.session.prompt("start"); + await sawToolStart; + await new Promise((resolve) => setTimeout(resolve, 0)); + + extensionApi?.sendUserMessage("/testcmd queued", { deliverAs: "followUp" }); + releaseToolExecution?.(); + await promptPromise; + + expect(commandRuns).toEqual([]); + expect(getUserTexts(harness)).toEqual(["start", "/testcmd queued"]); + expect(getAssistantTexts(harness)).toContain("queued follow-up handled by model"); + }); +});