diff --git a/docs/superpowers/specs/2026-10-07-source-citations-design.md b/docs/superpowers/specs/2026-10-07-source-citations-design.md index 67aac29..70dc1ae 100644 --- a/docs/superpowers/specs/2026-10-07-source-citations-design.md +++ b/docs/superpowers/specs/2026-10-07-source-citations-design.md @@ -174,13 +174,15 @@ The answer then gains a list, after the body and before the `_Tools:_` line: ``` _Sources_ -- [[1]](https://example.com/page) Result title -- [[2]](https://example.com/page-two) Page title +- [\[1\]](https://example.com/page) Result title +- [\[2\]](https://example.com/page-two) Page title ``` The marker is the link, matching the inline marker character for character, so -the same token means the same thing in both places. An entry whose URL cannot be -linked keeps its title and shows no marker link. +the same token means the same thing in both places. The brackets are +escaped because `markdownLink` escapes them: an unescaped `]` would end the link +text early, and the renderer draws the escaped form as a plain `[1]`. An entry +whose URL cannot be linked keeps its title and shows no marker link. ### Prompt diff --git a/src/__tests__/agent.test.ts b/src/__tests__/agent.test.ts index da505a7..2df7ff4 100644 --- a/src/__tests__/agent.test.ts +++ b/src/__tests__/agent.test.ts @@ -21,7 +21,18 @@ function ndjsonResponse(events: unknown[], status = 200): Response { } function toolCall(name: string, args: Record = {}): Record { - return { message: { role: "assistant", content: "", tool_calls: [{ function: { name, arguments: args } }] }, done: true }; + return { + message: { role: "assistant", content: "", tool_calls: [{ function: { name, arguments: args } }] }, + done: true, + }; +} + +/** The shape the two Ollama web endpoints answer with. */ +function jsonResponse(payload: unknown): Response { + return new Response(JSON.stringify(payload), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); } function answer(text: string): Record[] { @@ -136,6 +147,33 @@ describe("runAgent", () => { expect(toolMessage?.content).toContain("2026"); }); + it("numbers a turn's sources once, and keeps each call's own on its record", async () => { + // Two searches in one turn. The second has to continue the first's numbering, + // or one marker in the answer could mean two different pages; and each record + // has to hold only what its own call found, or the list would repeat a page. + fetchMock + .mockResolvedValueOnce(ndjsonResponse([toolCall("web_search", { query: "one" })])) + .mockResolvedValueOnce( + jsonResponse({ results: [{ title: "First", url: "https://first.example", content: "A" }] }), + ) + .mockResolvedValueOnce(ndjsonResponse([toolCall("web_search", { query: "two" })])) + .mockResolvedValueOnce( + jsonResponse({ + results: [{ title: "Second", url: "https://second.example", content: "B" }], + }), + ) + .mockResolvedValueOnce(ndjsonResponse(answer("Both [1][2] agree."))); + + const result = await runAgent({ + ...baseOptions([{ role: "user", content: "hi" }]), + tools: toolsFor({ webEnabled: true }), + }); + + expect(result.toolCalls.map((call) => call.sources?.map((source) => source.index))).toEqual([[1], [2]]); + expect(result.toolCalls[0].result).toContain("[1] First"); + expect(result.toolCalls[1].result).toContain("[2] Second"); + }); + it("reports an unknown tool as an error without aborting the turn", async () => { fetchMock .mockResolvedValueOnce(ndjsonResponse([toolCall("no_such_tool")])) @@ -229,7 +267,9 @@ describe("runAgent", () => { for (let i = 0; i < DEFAULT_MAX_TOOL_ROUNDS; i++) { fetchMock.mockResolvedValueOnce(ndjsonResponse([preamble(i)])); } - fetchMock.mockResolvedValueOnce(ndjsonResponse([{ message: { role: "assistant", content: "" }, done: true }])); + fetchMock.mockResolvedValueOnce( + ndjsonResponse([{ message: { role: "assistant", content: "" }, done: true }]), + ); const result = await runAgent(baseOptions([{ role: "user", content: "hi" }])); @@ -261,137 +301,137 @@ describe("runAgent", () => { }); }); - /** Queues `count` rounds that all ask for the identical call. */ - function queueRepeats(count: number): void { - for (let i = 0; i < count; i++) { - fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })])); - } +/** Queues `count` rounds that all ask for the identical call. */ +function queueRepeats(count: number): void { + for (let i = 0; i < count; i++) { + fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })])); } +} - it("stops when the same call comes round a third time instead of running it again", async () => { - // The rut this guards against: the model decides the first search did not - // answer the question and asks the identical question again, and again. - // Two executions is a retry; a third is not going to learn anything new. - queueRepeats(3); - fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("It is 03:04 UTC."))); +it("stops when the same call comes round a third time instead of running it again", async () => { + // The rut this guards against: the model decides the first search did not + // answer the question and asks the identical question again, and again. + // Two executions is a retry; a third is not going to learn anything new. + queueRepeats(3); + fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("It is 03:04 UTC."))); - const result = await runAgent(baseOptions([{ role: "user", content: "hi" }])); + const result = await runAgent(baseOptions([{ role: "user", content: "hi" }])); - // Two executions, plus the refusal that ended the turn. - expect(result.toolCalls).toHaveLength(3); - expect(result.toolCalls.filter((call) => call.repeated)).toHaveLength(1); - expect(result.content).toBe("It is 03:04 UTC."); - // Two tool rounds, the refusal round, and the closing round. - expect(fetchMock).toHaveBeenCalledTimes(4); + // Two executions, plus the refusal that ended the turn. + expect(result.toolCalls).toHaveLength(3); + expect(result.toolCalls.filter((call) => call.repeated)).toHaveLength(1); + expect(result.content).toBe("It is 03:04 UTC."); + // Two tool rounds, the refusal round, and the closing round. + expect(fetchMock).toHaveBeenCalledTimes(4); +}); + +it("does not count a different call with the same tool as a repeat", async () => { + // Searching for three different things is a model doing its job, not a loop. + fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })])); + fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "CET" })])); + fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "JST" })])); + fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("Three zones."))); + + const result = await runAgent(baseOptions([{ role: "user", content: "hi" }])); + + expect(result.toolCalls).toHaveLength(3); + expect(result.toolCalls.every((call) => !call.repeated)).toBe(true); +}); + +it("treats reordered arguments as the same call", async () => { + // Models are not consistent about key order, and a guard that only catches + // byte-identical calls would miss the ordinary shape of a repeat. + fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC", unit: "h" })])); + fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { unit: "h", tz: "UTC" })])); + fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC", unit: "h" })])); + fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done"))); + + const result = await runAgent(baseOptions([{ role: "user", content: "hi" }])); + + expect(result.toolCalls.filter((call) => call.repeated)).toHaveLength(1); +}); + +it("tells the model the call was repeating rather than that a budget ran out", async () => { + queueRepeats(3); + fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("ok"))); + + await runAgent(baseOptions([{ role: "user", content: "hi" }])); + + const last = JSON.parse(fetchMock.mock.calls[3][1].body as string) as { messages: ApiMessage[] }; + const nudge = last.messages[last.messages.length - 1]; + + expect(nudge.role).toBe("user"); + expect(nudge.content).toContain("identical arguments 3 times"); + expect(nudge.content).not.toContain("reached the limit"); +}); + +it("answers from what it gathered when a repeat ends the turn", async () => { + queueRepeats(3); + fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("I could not get past the time check."))); + + const result = await runAgent(baseOptions([{ role: "user", content: "hi" }])); + + expect(result.content).toBe("I could not get past the time check."); + expect(result.content).not.toContain("kept repeating"); +}); + +it("says the turn was cut short when a repeat leaves the closing round speechless", async () => { + queueRepeats(3); + fetchMock.mockResolvedValueOnce( + ndjsonResponse([{ message: { role: "assistant", content: "" }, done: true }]), + ); + + const result = await runAgent(baseOptions([{ role: "user", content: "hi" }])); + + expect(result.content).toContain("kept repeating the same call"); +}); + +it("honours a caller's round cap in place of the default", async () => { + for (let i = 0; i < 4; i++) { + fetchMock.mockResolvedValueOnce(ndjsonResponse([distinctCall(i)])); + } + fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done"))); + + const result = await runAgent({ ...baseOptions([{ role: "user", content: "hi" }]), maxToolRounds: 4 }); + + expect(result.toolCalls).toHaveLength(4); + expect(result.content).toBe("done"); +}); + +it("runs past the default cap when the caller asks for no cap at all", async () => { + // The preference can be switched to Unlimited, and the only difference that + // should make is that the cap stops arriving. + const rounds = DEFAULT_MAX_TOOL_ROUNDS + 3; + for (let i = 0; i < rounds; i++) { + fetchMock.mockResolvedValueOnce(ndjsonResponse([distinctCall(i)])); + } + fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done"))); + + const result = await runAgent({ ...baseOptions([{ role: "user", content: "hi" }]), maxToolRounds: null }); + + expect(result.toolCalls).toHaveLength(rounds); + expect(result.content).toBe("done"); +}); + +it("reports each tool start with how many times that call has come round", async () => { + // This is what the transcript shows while a turn is in flight, so a repeat + // is visible as it happens and not only in the record afterwards. + const started: { name: string; label: string; count: number }[] = []; + + fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })])); + fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })])); + fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done"))); + + await runAgent({ + ...baseOptions([{ role: "user", content: "hi" }]), + events: { onToolStart: (name, label, count) => started.push({ name, label, count }) }, }); - it("does not count a different call with the same tool as a repeat", async () => { - // Searching for three different things is a model doing its job, not a loop. - fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })])); - fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "CET" })])); - fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "JST" })])); - fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("Three zones."))); - - const result = await runAgent(baseOptions([{ role: "user", content: "hi" }])); - - expect(result.toolCalls).toHaveLength(3); - expect(result.toolCalls.every((call) => !call.repeated)).toBe(true); - }); - - it("treats reordered arguments as the same call", async () => { - // Models are not consistent about key order, and a guard that only catches - // byte-identical calls would miss the ordinary shape of a repeat. - fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC", unit: "h" })])); - fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { unit: "h", tz: "UTC" })])); - fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC", unit: "h" })])); - fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done"))); - - const result = await runAgent(baseOptions([{ role: "user", content: "hi" }])); - - expect(result.toolCalls.filter((call) => call.repeated)).toHaveLength(1); - }); - - it("tells the model the call was repeating rather than that a budget ran out", async () => { - queueRepeats(3); - fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("ok"))); - - await runAgent(baseOptions([{ role: "user", content: "hi" }])); - - const last = JSON.parse(fetchMock.mock.calls[3][1].body as string) as { messages: ApiMessage[] }; - const nudge = last.messages[last.messages.length - 1]; - - expect(nudge.role).toBe("user"); - expect(nudge.content).toContain("identical arguments 3 times"); - expect(nudge.content).not.toContain("reached the limit"); - }); - - it("answers from what it gathered when a repeat ends the turn", async () => { - queueRepeats(3); - fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("I could not get past the time check."))); - - const result = await runAgent(baseOptions([{ role: "user", content: "hi" }])); - - expect(result.content).toBe("I could not get past the time check."); - expect(result.content).not.toContain("kept repeating"); - }); - - it("says the turn was cut short when a repeat leaves the closing round speechless", async () => { - queueRepeats(3); - fetchMock.mockResolvedValueOnce( - ndjsonResponse([{ message: { role: "assistant", content: "" }, done: true }]), - ); - - const result = await runAgent(baseOptions([{ role: "user", content: "hi" }])); - - expect(result.content).toContain("kept repeating the same call"); - }); - - it("honours a caller's round cap in place of the default", async () => { - for (let i = 0; i < 4; i++) { - fetchMock.mockResolvedValueOnce(ndjsonResponse([distinctCall(i)])); - } - fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done"))); - - const result = await runAgent({ ...baseOptions([{ role: "user", content: "hi" }]), maxToolRounds: 4 }); - - expect(result.toolCalls).toHaveLength(4); - expect(result.content).toBe("done"); - }); - - it("runs past the default cap when the caller asks for no cap at all", async () => { - // The preference can be switched to Unlimited, and the only difference that - // should make is that the cap stops arriving. - const rounds = DEFAULT_MAX_TOOL_ROUNDS + 3; - for (let i = 0; i < rounds; i++) { - fetchMock.mockResolvedValueOnce(ndjsonResponse([distinctCall(i)])); - } - fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done"))); - - const result = await runAgent({ ...baseOptions([{ role: "user", content: "hi" }]), maxToolRounds: null }); - - expect(result.toolCalls).toHaveLength(rounds); - expect(result.content).toBe("done"); - }); - - it("reports each tool start with how many times that call has come round", async () => { - // This is what the transcript shows while a turn is in flight, so a repeat - // is visible as it happens and not only in the record afterwards. - const started: { name: string; label: string; count: number }[] = []; - - fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })])); - fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })])); - fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done"))); - - await runAgent({ - ...baseOptions([{ role: "user", content: "hi" }]), - events: { onToolStart: (name, label, count) => started.push({ name, label, count }) }, - }); - - expect(started).toEqual([ - { name: "get_datetime", label: "Checking the time", count: 1 }, - { name: "get_datetime", label: "Checking the time", count: 2 }, - ]); - }); + expect(started).toEqual([ + { name: "get_datetime", label: "Checking the time", count: 1 }, + { name: "get_datetime", label: "Checking the time", count: 2 }, + ]); +}); describe("module wiring", () => { it("keeps the request, tool, and agent layers free of Vicinae imports", () => { diff --git a/src/lib/agent.ts b/src/lib/agent.ts index b757ba5..d6d2e8a 100644 --- a/src/lib/agent.ts +++ b/src/lib/agent.ts @@ -1,3 +1,4 @@ +import { createSourceList } from "./citations"; import { DEFAULT_MAX_TOOL_ROUNDS, MAX_REPEATED_TOOL_CALLS } from "./constants"; import { streamChat } from "./ollama"; import { executeTool, toolLabel, type ToolContext } from "./tools"; @@ -137,6 +138,14 @@ function exhaustedNotice(reason: StopReason): string { export async function runAgent(options: AgentOptions): Promise { const conversation: ApiMessage[] = [...options.messages]; const toolCalls: ToolCallRecord[] = []; + /** + * The turn's sources, so one number means one page for the whole turn. + * + * Created here rather than passed in: it is scoped to a single run of this loop, + * and every call the loop makes shares it, which is what keeps three searches in + * one answer from each starting again at 1. + */ + const sources = createSourceList(); /** `Infinity` needs no special case: the loop simply never reaches its own end. */ const rounds = options.maxToolRounds === null ? Infinity : (options.maxToolRounds ?? DEFAULT_MAX_TOOL_ROUNDS); @@ -239,6 +248,7 @@ export async function runAgent(options: AgentOptions): Promise { place: options.toolContext.place, now: options.toolContext.now, signal: options.signal, + sources, }); const record: ToolCallRecord = { @@ -246,6 +256,9 @@ export async function runAgent(options: AgentOptions): Promise { arguments: args, result: result.content, isError: result.isError, + // Only what this call added. The turn's whole list is the flattening of + // its records, so the running total is deliberately not repeated here. + sources: result.sources, }; toolCalls.push(record); options.events?.onToolCall?.(record); diff --git a/src/lib/schemas.ts b/src/lib/schemas.ts index f552461..8893011 100644 --- a/src/lib/schemas.ts +++ b/src/lib/schemas.ts @@ -7,12 +7,23 @@ import { z } from "zod"; * older or hand-edited payload gets rejected instead of crashing a render. */ +export const SourceSchema = z.object({ + index: z.number(), + title: z.string(), + url: z.string(), +}); + export const ToolCallRecordSchema = z.object({ name: z.string(), arguments: z.record(z.string(), z.unknown()), result: z.string(), isError: z.boolean(), repeated: z.boolean().optional(), + // Absent on turns stored before sources were numbered, which is honest: those + // answers never cited a number, so there is nothing to draw for them. zod drops + // unknown keys, so without this line the field would be lost on the first + // reload and a reopened conversation would lose every link. + sources: z.array(SourceSchema).optional(), }); export const AttachmentSchema = z.object({