feat(agent): return diagnostics from resource loaders

packages/agent/src/harness/prompt-templates.ts

@@ -1,47 +1,108 @@
 import { parse } from "yaml";
 import type { ExecutionEnv, FileInfo, PromptTemplate } from "./types.js";
 
+/** Warning produced while loading prompt templates. */
+export interface PromptTemplateDiagnostic {
+	/** Diagnostic severity. Currently only warnings are emitted. */
+	type: "warning";
+	/** Human-readable diagnostic message. */
+	message: string;
+	/** Path associated with the diagnostic. */
+	path: string;
+}
+
 interface PromptTemplateFrontmatter {
 	description?: string;
 	"argument-hint"?: string;
 	[key: string]: unknown;
 }
 
-export async function loadPromptTemplates(env: ExecutionEnv, paths: string | string[]): Promise<PromptTemplate[]> {
+/**
-	const templates: PromptTemplate[] = [];
+ * Load prompt templates from one or more paths.
+ *
+ * Directory inputs load direct `.md` children non-recursively. File inputs load explicit `.md` files. Missing paths and
+ * non-markdown files are skipped. Read and parse failures are returned as diagnostics.
+ */
+export async function loadPromptTemplates(
+	env: ExecutionEnv,
+	paths: string | string[],
+): Promise<{ promptTemplates: PromptTemplate[]; diagnostics: PromptTemplateDiagnostic[] }> {
+	const promptTemplates: PromptTemplate[] = [];
+	const diagnostics: PromptTemplateDiagnostic[] = [];
 	for (const path of Array.isArray(paths) ? paths : [paths]) {
 		const info = await safeFileInfo(env, path);
 		if (!info) continue;
 		const kind = await resolveKind(env, info);
 		if (kind === "directory") {
-			templates.push(...(await loadTemplatesFromDir(env, info.path)));
+			const result = await loadTemplatesFromDir(env, info.path);
+			promptTemplates.push(...result.promptTemplates);
+			diagnostics.push(...result.diagnostics);
 		} else if (kind === "file" && info.name.endsWith(".md")) {
-			const template = await loadTemplateFromFile(env, info.path);
+			const result = await loadTemplateFromFile(env, info.path);
-			if (template) templates.push(template);
+			if (result.promptTemplate) promptTemplates.push(result.promptTemplate);
+			diagnostics.push(...result.diagnostics);
 		}
 	}
-	return templates;
+	return { promptTemplates, diagnostics };
 }
 
-async function loadTemplatesFromDir(env: ExecutionEnv, dir: string): Promise<PromptTemplate[]> {
+/**
-	const templates: PromptTemplate[] = [];
+ * Load prompt templates from source-tagged paths.
+ *
+ * Source values are preserved exactly and attached to every loaded prompt template and diagnostic. The agent package does
+ * not interpret source values; applications define their own provenance shape.
+ */
+export async function loadSourcedPromptTemplates<TSource>(
+	env: ExecutionEnv,
+	inputs: Array<{ path: string; source: TSource }>,
+): Promise<{
+	promptTemplates: Array<{ promptTemplate: PromptTemplate; source: TSource }>;
+	diagnostics: Array<PromptTemplateDiagnostic & { source: TSource }>;
+}> {
+	const promptTemplates: Array<{ promptTemplate: PromptTemplate; source: TSource }> = [];
+	const diagnostics: Array<PromptTemplateDiagnostic & { source: TSource }> = [];
+	for (const input of inputs) {
+		const result = await loadPromptTemplates(env, input.path);
+		for (const promptTemplate of result.promptTemplates)
+			promptTemplates.push({ promptTemplate, source: input.source });
+		for (const diagnostic of result.diagnostics) diagnostics.push({ ...diagnostic, source: input.source });
+	}
+	return { promptTemplates, diagnostics };
+}
+
+async function loadTemplatesFromDir(
+	env: ExecutionEnv,
+	dir: string,
+): Promise<{ promptTemplates: PromptTemplate[]; diagnostics: PromptTemplateDiagnostic[] }> {
+	const promptTemplates: PromptTemplate[] = [];
+	const diagnostics: PromptTemplateDiagnostic[] = [];
 	let entries: FileInfo[];
 	try {
 		entries = await env.listDir(dir);
-	} catch {
+	} catch (error) {
-		return templates;
+		diagnostics.push({
+			type: "warning",
+			message: errorMessage(error, "failed to list prompt template directory"),
+			path: dir,
+		});
+		return { promptTemplates, diagnostics };
 	}
 
 	for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
 		const kind = await resolveKind(env, entry);
 		if (kind !== "file" || !entry.name.endsWith(".md")) continue;
-		const template = await loadTemplateFromFile(env, entry.path);
+		const result = await loadTemplateFromFile(env, entry.path);
-		if (template) templates.push(template);
+		if (result.promptTemplate) promptTemplates.push(result.promptTemplate);
+		diagnostics.push(...result.diagnostics);
 	}
-	return templates;
+	return { promptTemplates, diagnostics };
 }
 
-async function loadTemplateFromFile(env: ExecutionEnv, filePath: string): Promise<PromptTemplate | null> {
+async function loadTemplateFromFile(
+	env: ExecutionEnv,
+	filePath: string,
+): Promise<{ promptTemplate: PromptTemplate | null; diagnostics: PromptTemplateDiagnostic[] }> {
+	const diagnostics: PromptTemplateDiagnostic[] = [];
 	try {
 		const rawContent = await env.readTextFile(filePath);
 		const { frontmatter, body } = parseFrontmatter<PromptTemplateFrontmatter>(rawContent);
@@ -52,12 +113,20 @@ async function loadTemplateFromFile(env: ExecutionEnv, filePath: string): Promis
 			if (firstLine.length > 60) description += "...";
 		}
 		return {
-			name: basenameEnvPath(filePath).replace(/\.md$/i, ""),
+			promptTemplate: {
-			description,
+				name: basenameEnvPath(filePath).replace(/\.md$/i, ""),
-			content: body,
+				description,
+				content: body,
+			},
+			diagnostics,
 		};
-	} catch {
+	} catch (error) {
-		return null;
+		diagnostics.push({
+			type: "warning",
+			message: errorMessage(error, "failed to load prompt template"),
+			path: filePath,
+		});
+		return { promptTemplate: null, diagnostics };
 	}
 }
 
@@ -96,6 +165,11 @@ function basenameEnvPath(path: string): string {
 	return slashIndex === -1 ? normalized : normalized.slice(slashIndex + 1);
 }
 
+function errorMessage(error: unknown, fallback: string): string {
+	return error instanceof Error ? error.message : fallback;
+}
+
+/** Parse slash-command arguments using simple shell-style single and double quotes. */
 export function parseCommandArgs(argsString: string): string[] {
 	const args: string[] = [];
 	let current = "";
@@ -121,6 +195,7 @@ export function parseCommandArgs(argsString: string): string[] {
 	return args;
 }
 
+/** Substitute prompt template placeholders (`$1`, `$@`, `$ARGUMENTS`, `${@:N}`, `${@:N:L}`) with command arguments. */
 export function substituteArgs(content: string, args: string[]): string {
 	let result = content;
 	result = result.replace(/\$(\d+)/g, (_, num: string) => args[parseInt(num, 10) - 1] ?? "");
@@ -136,6 +211,7 @@ export function substituteArgs(content: string, args: string[]): string {
 	return result;
 }
 
+/** Expand `/template args` text using the matching prompt template, or return the original text when no template matches. */
 export function expandPromptTemplate(text: string, templates: PromptTemplate[]): string {
 	if (!text.startsWith("/")) return text;
 	const spaceIndex = text.indexOf(" ");

packages/agent/src/harness/skills.ts

@@ -8,9 +8,13 @@ const IGNORE_FILE_NAMES = [".gitignore", ".ignore", ".fdignore"];
 
 type IgnoreMatcher = ReturnType<typeof ignore>;
 
+/** Warning produced while loading skills. */
 export interface SkillDiagnostic {
+	/** Diagnostic severity. Currently only warnings are emitted. */
 	type: "warning";
+	/** Human-readable diagnostic message. */
 	message: string;
+	/** Path associated with the diagnostic. */
 	path: string;
 }
 
@@ -21,11 +25,13 @@ interface SkillFrontmatter {
 	[key: string]: unknown;
 }
 
-export async function loadSkills(env: ExecutionEnv, dirs: string | string[]): Promise<Skill[]> {
+/**
-	return (await loadSkillsWithDiagnostics(env, dirs)).skills;
+ * Load skills from one or more directories.
-}
+ *
-
+ * Traverses directories recursively, loads `SKILL.md` files, loads direct root `.md` files as skills, honors ignore files,
-export async function loadSkillsWithDiagnostics(
+ * and returns diagnostics for invalid skill files. Missing input directories are skipped.
+ */
+export async function loadSkills(
 	env: ExecutionEnv,
 	dirs: string | string[],
 ): Promise<{ skills: Skill[]; diagnostics: SkillDiagnostic[] }> {
@@ -41,6 +47,29 @@ export async function loadSkillsWithDiagnostics(
 	return { skills, diagnostics };
 }
 
+/**
+ * Load skills from source-tagged directories.
+ *
+ * Source values are preserved exactly and attached to every loaded skill and diagnostic. The agent package does not
+ * interpret source values; applications define their own provenance shape.
+ */
+export async function loadSourcedSkills<TSource>(
+	env: ExecutionEnv,
+	inputs: Array<{ path: string; source: TSource }>,
+): Promise<{
+	skills: Array<{ skill: Skill; source: TSource }>;
+	diagnostics: Array<SkillDiagnostic & { source: TSource }>;
+}> {
+	const skills: Array<{ skill: Skill; source: TSource }> = [];
+	const diagnostics: Array<SkillDiagnostic & { source: TSource }> = [];
+	for (const input of inputs) {
+		const result = await loadSkills(env, input.path);
+		for (const skill of result.skills) skills.push({ skill, source: input.source });
+		for (const diagnostic of result.diagnostics) diagnostics.push({ ...diagnostic, source: input.source });
+	}
+	return { skills, diagnostics };
+}
+
 async function loadSkillsFromDirInternal(
 	env: ExecutionEnv,
 	dir: string,

packages/agent/test/harness/prompt-templates.test.ts

@@ -2,7 +2,11 @@ import { symlink } from "node:fs/promises";
 import { join } from "node:path";
 import { describe, expect, it } from "vitest";
 import { NodeExecutionEnv } from "../../src/harness/execution-env.js";
-import { expandPromptTemplate, loadPromptTemplates } from "../../src/harness/prompt-templates.js";
+import {
+	expandPromptTemplate,
+	loadPromptTemplates,
+	loadSourcedPromptTemplates,
+} from "../../src/harness/prompt-templates.js";
 import { createTempDir } from "./session-test-utils.js";
 
 describe("loadPromptTemplates", () => {
@@ -15,21 +19,61 @@ describe("loadPromptTemplates", () => {
 		await env.writeFile("a/nested/ignored.md", "Ignored");
 		await env.writeFile("b/two.md", "First line description\nBody");
 
-		const templates = await loadPromptTemplates(env, ["a", "b"]);
+		const { promptTemplates, diagnostics } = await loadPromptTemplates(env, ["a", "b"]);
 
-		expect(templates).toEqual([
+		expect(diagnostics).toEqual([]);
+		expect(promptTemplates).toEqual([
 			{ name: "one", description: "One template", content: "Hello $1" },
 			{ name: "two", description: "First line description", content: "First line description\nBody" },
 		]);
 	});
 
+	it("preserves source info for sourced prompt templates", async () => {
+		const root = createTempDir();
+		const env = new NodeExecutionEnv({ cwd: root });
+		await env.createDir("prompts", { recursive: true });
+		await env.writeFile("prompts/example.md", "---\ndescription: Example\n---\nExample body");
+
+		const { promptTemplates, diagnostics } = await loadSourcedPromptTemplates(env, [
+			{ path: "prompts", source: { type: "project" as const } },
+		]);
+
+		expect(diagnostics).toEqual([]);
+		expect(promptTemplates).toEqual([
+			{
+				promptTemplate: { name: "example", description: "Example", content: "Example body" },
+				source: { type: "project" },
+			},
+		]);
+	});
+
+	it("attaches source info to diagnostics", async () => {
+		const root = createTempDir();
+		const env = new NodeExecutionEnv({ cwd: root });
+		await env.writeFile("broken.md", "---\ndescription: [unterminated\n---\nBody");
+
+		const { promptTemplates, diagnostics } = await loadSourcedPromptTemplates(env, [
+			{ path: "broken.md", source: { type: "user" as const } },
+		]);
+
+		expect(promptTemplates).toEqual([]);
+		expect(diagnostics).toHaveLength(1);
+		expect(diagnostics[0]).toMatchObject({
+			type: "warning",
+			path: join(root, "broken.md"),
+			source: { type: "user" },
+		});
+	});
+
 	it("loads explicit markdown files and symlinked files", async () => {
 		const root = createTempDir();
 		const env = new NodeExecutionEnv({ cwd: root });
 		await env.writeFile("target.md", "---\ndescription: Target\n---\nTarget body");
 		await symlink(join(root, "target.md"), join(root, "link.md"));
 
-		expect(await loadPromptTemplates(env, ["target.md", "link.md"])).toEqual([
+		const { promptTemplates } = await loadPromptTemplates(env, ["target.md", "link.md"]);
+
+		expect(promptTemplates).toEqual([
 			{ name: "target", description: "Target", content: "Target body" },
 			{ name: "link", description: "Target", content: "Target body" },
 		]);

packages/agent/test/harness/skills.test.ts

@@ -2,7 +2,7 @@ import { symlink } from "node:fs/promises";
 import { join } from "node:path";
 import { describe, expect, it } from "vitest";
 import { NodeExecutionEnv } from "../../src/harness/execution-env.js";
-import { loadSkills } from "../../src/harness/skills.js";
+import { loadSkills, loadSourcedSkills } from "../../src/harness/skills.js";
 import { createTempDir } from "./session-test-utils.js";
 
 describe("loadSkills", () => {
@@ -21,8 +21,9 @@ Use this skill.
 `,
 		);
 
-		const skills = await loadSkills(env, ".agents/skills");
+		const { skills, diagnostics } = await loadSkills(env, ".agents/skills");
 
+		expect(diagnostics).toEqual([]);
 		expect(skills).toEqual([
 			{
 				name: "example",
@@ -44,12 +45,61 @@ Use this skill.
 		);
 		await symlink(join(root, "actual"), join(root, "skills-link"));
 
-		const skills = await loadSkills(env, "skills-link");
+		const { skills } = await loadSkills(env, "skills-link");
 
 		expect(skills.map((skill) => skill.name)).toEqual(["example"]);
 		expect(skills[0]?.filePath).toBe(join(root, "skills-link/example/SKILL.md"));
 	});
 
+	it("preserves source info for sourced skills", async () => {
+		const root = createTempDir();
+		const env = new NodeExecutionEnv({ cwd: root });
+		await env.createDir("user/example", { recursive: true });
+		await env.writeFile(
+			"user/example/SKILL.md",
+			"---\nname: example\ndescription: Example skill\n---\nUse this skill.",
+		);
+
+		const { skills, diagnostics } = await loadSourcedSkills(env, [
+			{ path: "user", source: { type: "user" as const } },
+		]);
+
+		expect(diagnostics).toEqual([]);
+		expect(skills).toEqual([
+			{
+				skill: {
+					name: "example",
+					description: "Example skill",
+					content: "Use this skill.",
+					filePath: join(root, "user/example/SKILL.md"),
+					disableModelInvocation: false,
+				},
+				source: { type: "user" },
+			},
+		]);
+	});
+
+	it("attaches source info to diagnostics", async () => {
+		const root = createTempDir();
+		const env = new NodeExecutionEnv({ cwd: root });
+		await env.createDir("user/broken", { recursive: true });
+		await env.writeFile("user/broken/SKILL.md", "---\nname: broken\n---\nMissing description.");
+
+		const { skills, diagnostics } = await loadSourcedSkills(env, [
+			{ path: "user", source: { type: "user" as const } },
+		]);
+
+		expect(skills).toEqual([]);
+		expect(diagnostics).toEqual([
+			{
+				type: "warning",
+				message: "description is required",
+				path: join(root, "user/broken/SKILL.md"),
+				source: { type: "user" },
+			},
+		]);
+	});
+
 	it("loads direct markdown children only from the root directory", async () => {
 		const root = createTempDir();
 		const env = new NodeExecutionEnv({ cwd: root });
@@ -57,7 +107,7 @@ Use this skill.
 		await env.writeFile("skills/root.md", "---\ndescription: Root skill\n---\nRoot content");
 		await env.writeFile("skills/nested/ignored.md", "---\ndescription: Ignored\n---\nIgnored content");
 
-		const skills = await loadSkills(env, "skills");
+		const { skills } = await loadSkills(env, "skills");
 
 		expect(skills.map((skill) => skill.name)).toEqual(["skills"]);
 		expect(skills[0]?.content).toBe("Root content");

packages/agent/test/scratch/simple.ts

@@ -1,31 +1,51 @@
+import { homedir } from "node:os";
+import { join } from "node:path";
 import { getModel } from "@mariozechner/pi-ai";
 import { InMemorySessionStorage } from "../../src/harness/session/storage/memory.js";
 import {
 	createAgentHarness,
 	formatSkillsForSystemPrompt,
-	loadSkills,
+	loadSourcedPromptTemplates,
+	loadSourcedSkills,
 	NodeExecutionEnv,
 	Session,
 } from "../../src/index.js";
 
+type Source = { type: "project" | "user" | "path"; dir: string };
+
 const env = new NodeExecutionEnv({ cwd: process.cwd() });
-const skills = await loadSkills(env, "/Users/badlogic/.pi/agent/skills");
+const source = (type: Source["type"], dir: string) => ({ path: dir, source: { type, dir } });
+const { skills: sourcedSkills } = await loadSourcedSkills<Source>(env, [
+	source("project", join(env.cwd, ".pi/skills")),
+	source("user", join(homedir(), ".pi/agent/skills")),
+	source("path", join(env.cwd, "../../../pi-skills")),
+]);
+const { promptTemplates: sourcedPromptTemplates } = await loadSourcedPromptTemplates<Source>(env, [
+	source("project", join(env.cwd, ".pi/prompts")),
+	source("user", join(homedir(), ".pi/agent/prompts")),
+]);
+
 const session = new Session(new InMemorySessionStorage());
 const agent = createAgentHarness({
 	env,
 	session,
 	model: getModel("openai", "gpt-5.5"),
 	thinkingLevel: "low",
-	systemPrompt: ({ env, resources }) =>
+	systemPrompt: ({ env, resources }) => {
-		[
+		console.log("Building system prompt");
+		return [
 			`You are a helpful assistant.`,
 			formatSkillsForSystemPrompt(resources.skills ?? []),
 			`Current working directory: ${env.cwd}`,
 		]
 			.filter((part) => part.length > 0)
-			.join("\n\n"),
+			.join("\n\n");
-	resources: { skills },
+	},
+	resources: {
+		promptTemplates: sourcedPromptTemplates.map((entry) => entry.promptTemplate),
+		skills: sourcedSkills.map((entry) => entry.skill),
+	},
 });
 
-const response = await agent.prompt("What skills do you have?");
+const response = await agent.prompt("What skills do you have? Any duplicates?");
 console.log(response);