test(coding-agent): add session lifecycle characterization suite
This commit is contained in:
Mario Zechner
323
packages/coding-agent/test/suite/agent-session-prompt.test.ts
Normal file
323
packages/coding-agent/test/suite/agent-session-prompt.test.ts
Normal file
@@ -0,0 +1,323 @@
|
||||
import { mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AgentTool } from "@mariozechner/pi-agent-core";
|
||||
import { fauxAssistantMessage, fauxToolCall, type Model } from "@mariozechner/pi-ai";
|
||||
import { Type } from "@sinclair/typebox";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import type { PromptTemplate } from "../../src/core/prompt-templates.js";
|
||||
import { createSyntheticSourceInfo } from "../../src/core/source-info.js";
|
||||
import { createTestResourceLoader } from "../utilities.js";
|
||||
import { createHarness, getMessageText, type Harness } from "./harness.js";
|
||||
|
||||
describe("AgentSession prompt characterization", () => {
|
||||
const harnesses: Harness[] = [];
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
while (harnesses.length > 0) {
|
||||
harnesses.pop()?.cleanup();
|
||||
}
|
||||
while (tempDirs.length > 0) {
|
||||
const tempDir = tempDirs.pop();
|
||||
if (tempDir) {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("prompts while idle and records a single text response", async () => {
|
||||
const harness = await createHarness();
|
||||
harnesses.push(harness);
|
||||
|
||||
harness.setResponses([fauxAssistantMessage("hello")]);
|
||||
|
||||
await harness.session.prompt("hi");
|
||||
|
||||
expect(harness.session.messages.map((message) => message.role)).toEqual(["user", "assistant"]);
|
||||
expect(getMessageText(harness.session.messages[0]!)).toBe("hi");
|
||||
expect(harness.getPendingResponseCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("handles a tool call turn and waits for the follow-up LLM response", async () => {
|
||||
const toolRuns: string[] = [];
|
||||
const echoTool: AgentTool = {
|
||||
name: "echo",
|
||||
label: "Echo",
|
||||
description: "Echo text back",
|
||||
parameters: Type.Object({ text: Type.String() }),
|
||||
execute: async (_toolCallId, params) => {
|
||||
const text = typeof params === "object" && params !== null && "text" in params ? String(params.text) : "";
|
||||
toolRuns.push(text);
|
||||
return {
|
||||
content: [{ type: "text", text: `echo:${text}` }],
|
||||
details: { text },
|
||||
};
|
||||
},
|
||||
};
|
||||
const harness = await createHarness({ tools: [echoTool] });
|
||||
harnesses.push(harness);
|
||||
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage(fauxToolCall("echo", { text: "hello" }), { stopReason: "toolUse" }),
|
||||
fauxAssistantMessage("done"),
|
||||
]);
|
||||
|
||||
await harness.session.prompt("start");
|
||||
|
||||
expect(toolRuns).toEqual(["hello"]);
|
||||
expect(harness.session.messages.map((message) => message.role)).toEqual([
|
||||
"user",
|
||||
"assistant",
|
||||
"toolResult",
|
||||
"assistant",
|
||||
]);
|
||||
expect(harness.session.messages[2]?.role).toBe("toolResult");
|
||||
expect(harness.session.messages[3]?.role).toBe("assistant");
|
||||
});
|
||||
|
||||
it("executes multiple tool calls from one response and continues with a single follow-up response", async () => {
|
||||
const toolRuns: string[] = [];
|
||||
const makeTool = (name: string, delayMs: number): AgentTool => ({
|
||||
name,
|
||||
label: name,
|
||||
description: `${name} tool`,
|
||||
parameters: Type.Object({ value: Type.String() }),
|
||||
execute: async (_toolCallId, params) => {
|
||||
const value =
|
||||
typeof params === "object" && params !== null && "value" in params ? String(params.value) : "";
|
||||
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||
toolRuns.push(`${name}:${value}`);
|
||||
return {
|
||||
content: [{ type: "text", text: `${name}:${value}` }],
|
||||
details: { value },
|
||||
};
|
||||
},
|
||||
});
|
||||
const harness = await createHarness({ tools: [makeTool("slow", 25), makeTool("fast", 0)] });
|
||||
harnesses.push(harness);
|
||||
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage([fauxToolCall("slow", { value: "a" }), fauxToolCall("fast", { value: "b" })], {
|
||||
stopReason: "toolUse",
|
||||
}),
|
||||
(context) => {
|
||||
const toolResults = context.messages.filter((message) => message.role === "toolResult");
|
||||
return fauxAssistantMessage(`tool results: ${toolResults.length}`);
|
||||
},
|
||||
]);
|
||||
|
||||
await harness.session.prompt("run tools");
|
||||
|
||||
expect(toolRuns.sort()).toEqual(["fast:b", "slow:a"]);
|
||||
expect(harness.session.messages.filter((message) => message.role === "toolResult")).toHaveLength(2);
|
||||
expect(harness.session.messages[harness.session.messages.length - 1]?.role).toBe("assistant");
|
||||
});
|
||||
|
||||
it("preserves image attachments in the provider context", async () => {
|
||||
const harness = await createHarness();
|
||||
harnesses.push(harness);
|
||||
let sawImage = false;
|
||||
|
||||
harness.setResponses([
|
||||
(context) => {
|
||||
const user = context.messages.find((message) => message.role === "user");
|
||||
sawImage =
|
||||
user?.role === "user" &&
|
||||
typeof user.content !== "string" &&
|
||||
user.content.some((part) => part.type === "image");
|
||||
return fauxAssistantMessage("ok");
|
||||
},
|
||||
]);
|
||||
|
||||
await harness.session.prompt("describe", {
|
||||
images: [
|
||||
{
|
||||
type: "image",
|
||||
mimeType: "image/png",
|
||||
data: "ZmFrZQ==",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(sawImage).toBe(true);
|
||||
});
|
||||
|
||||
it("expands skill commands before sending the prompt", async () => {
|
||||
const tempDir = join(tmpdir(), `pi-skill-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
tempDirs.push(tempDir);
|
||||
const skillPath = join(tempDir, "test-skill.md");
|
||||
writeFileSync(skillPath, "# Test Skill\n\nUse the skill body.");
|
||||
|
||||
const resourceLoader = {
|
||||
...createTestResourceLoader(),
|
||||
getSkills: () => ({
|
||||
skills: [
|
||||
{
|
||||
name: "test",
|
||||
description: "Test skill",
|
||||
filePath: skillPath,
|
||||
disableModelInvocation: false,
|
||||
baseDir: tempDir,
|
||||
sourceInfo: createSyntheticSourceInfo(skillPath, {
|
||||
source: "local",
|
||||
scope: "project",
|
||||
origin: "top-level",
|
||||
baseDir: tempDir,
|
||||
}),
|
||||
},
|
||||
],
|
||||
diagnostics: [],
|
||||
}),
|
||||
};
|
||||
const harness = await createHarness({ resourceLoader });
|
||||
harnesses.push(harness);
|
||||
let expandedPrompt = "";
|
||||
|
||||
harness.setResponses([
|
||||
(context) => {
|
||||
const user = context.messages.find((message) => message.role === "user");
|
||||
expandedPrompt = user ? getMessageText(user) : "";
|
||||
return fauxAssistantMessage("ok");
|
||||
},
|
||||
]);
|
||||
|
||||
await harness.session.prompt("/skill:test explain this");
|
||||
|
||||
expect(expandedPrompt).toContain('<skill name="test" location="');
|
||||
expect(expandedPrompt).toContain("Use the skill body.");
|
||||
expect(expandedPrompt).toContain("explain this");
|
||||
});
|
||||
|
||||
it("expands prompt templates before sending the prompt", async () => {
|
||||
const template: PromptTemplate = {
|
||||
name: "review",
|
||||
description: "Review template",
|
||||
content: "Review this code: $1",
|
||||
filePath: "/virtual/review.md",
|
||||
sourceInfo: createSyntheticSourceInfo("/virtual/review.md", {
|
||||
source: "local",
|
||||
scope: "temporary",
|
||||
origin: "top-level",
|
||||
}),
|
||||
};
|
||||
const resourceLoader = {
|
||||
...createTestResourceLoader(),
|
||||
getPrompts: () => ({ prompts: [template], diagnostics: [] }),
|
||||
};
|
||||
const harness = await createHarness({ resourceLoader });
|
||||
harnesses.push(harness);
|
||||
let expandedPrompt = "";
|
||||
|
||||
harness.setResponses([
|
||||
(context) => {
|
||||
const user = context.messages.find((message) => message.role === "user");
|
||||
expandedPrompt = user ? getMessageText(user) : "";
|
||||
return fauxAssistantMessage("ok");
|
||||
},
|
||||
]);
|
||||
|
||||
await harness.session.prompt("/review src/index.ts");
|
||||
|
||||
expect(expandedPrompt).toBe("Review this code: src/index.ts");
|
||||
});
|
||||
|
||||
it("dispatches extension commands without consuming a provider response", async () => {
|
||||
const commandRuns: string[] = [];
|
||||
const harness = await createHarness({
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.registerCommand("testcmd", {
|
||||
description: "Test command",
|
||||
handler: async (args) => {
|
||||
commandRuns.push(args);
|
||||
},
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
harness.setResponses([fauxAssistantMessage("should stay queued")]);
|
||||
|
||||
await harness.session.prompt("/testcmd hello world");
|
||||
|
||||
expect(commandRuns).toEqual(["hello world"]);
|
||||
expect(harness.session.messages).toEqual([]);
|
||||
expect(harness.getPendingResponseCount()).toBe(1);
|
||||
});
|
||||
|
||||
it("sendUserMessage while idle triggers a turn", async () => {
|
||||
const harness = await createHarness();
|
||||
harnesses.push(harness);
|
||||
|
||||
harness.setResponses([fauxAssistantMessage("response")]);
|
||||
|
||||
await harness.session.sendUserMessage("from extension");
|
||||
|
||||
expect(harness.session.messages.map((message) => message.role)).toEqual(["user", "assistant"]);
|
||||
expect(getMessageText(harness.session.messages[0]!)).toBe("from extension");
|
||||
});
|
||||
|
||||
it("throws when prompted during streaming without a streamingBehavior", async () => {
|
||||
let releaseToolExecution: (() => void) | undefined;
|
||||
const toolRelease = new Promise<void>((resolve) => {
|
||||
releaseToolExecution = resolve;
|
||||
});
|
||||
const waitTool: AgentTool = {
|
||||
name: "wait",
|
||||
label: "Wait",
|
||||
description: "Wait for release",
|
||||
parameters: Type.Object({}),
|
||||
execute: async () => {
|
||||
await toolRelease;
|
||||
return {
|
||||
content: [{ type: "text", text: "released" }],
|
||||
details: {},
|
||||
};
|
||||
},
|
||||
};
|
||||
const harness = await createHarness({ tools: [waitTool] });
|
||||
harnesses.push(harness);
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage(fauxToolCall("wait", {}), { stopReason: "toolUse" }),
|
||||
fauxAssistantMessage("done"),
|
||||
]);
|
||||
|
||||
const sawToolStart = new Promise<void>((resolve) => {
|
||||
const unsubscribe = harness.session.subscribe((event) => {
|
||||
if (event.type === "tool_execution_start") {
|
||||
unsubscribe();
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const promptPromise = harness.session.prompt("start");
|
||||
await sawToolStart;
|
||||
|
||||
await expect(harness.session.prompt("second")).rejects.toThrow(
|
||||
"Agent is already processing. Specify streamingBehavior ('steer' or 'followUp') to queue the message.",
|
||||
);
|
||||
|
||||
releaseToolExecution?.();
|
||||
await promptPromise;
|
||||
});
|
||||
|
||||
it("throws when prompting without a model", async () => {
|
||||
const harness = await createHarness();
|
||||
harnesses.push(harness);
|
||||
harness.session.agent.state.model = undefined as unknown as Model<any>;
|
||||
|
||||
await expect(harness.session.prompt("hi")).rejects.toThrow("No model selected.");
|
||||
});
|
||||
|
||||
it("throws when prompting without configured auth", async () => {
|
||||
const harness = await createHarness({ withConfiguredAuth: false });
|
||||
harnesses.push(harness);
|
||||
|
||||
await expect(harness.session.prompt("hi")).rejects.toThrow(
|
||||
`No API key found for ${harness.getModel().provider}.`,
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user