feat: number a turn's sources once and store them with the records
runAgent owns the list, so every call in one turn shares its numbering, and each record carries only the entries its own call added: the turn's whole list is the flattening of its records. The schema accepts the field as optional, because zod drops unknown keys and a reopened conversation would otherwise lose every link.
This commit is contained in:
+167
-127
@@ -21,7 +21,18 @@ function ndjsonResponse(events: unknown[], status = 200): Response {
|
||||
}
|
||||
|
||||
function toolCall(name: string, args: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
return { message: { role: "assistant", content: "", tool_calls: [{ function: { name, arguments: args } }] }, done: true };
|
||||
return {
|
||||
message: { role: "assistant", content: "", tool_calls: [{ function: { name, arguments: args } }] },
|
||||
done: true,
|
||||
};
|
||||
}
|
||||
|
||||
/** The shape the two Ollama web endpoints answer with. */
|
||||
function jsonResponse(payload: unknown): Response {
|
||||
return new Response(JSON.stringify(payload), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
function answer(text: string): Record<string, unknown>[] {
|
||||
@@ -136,6 +147,33 @@ describe("runAgent", () => {
|
||||
expect(toolMessage?.content).toContain("2026");
|
||||
});
|
||||
|
||||
it("numbers a turn's sources once, and keeps each call's own on its record", async () => {
|
||||
// Two searches in one turn. The second has to continue the first's numbering,
|
||||
// or one marker in the answer could mean two different pages; and each record
|
||||
// has to hold only what its own call found, or the list would repeat a page.
|
||||
fetchMock
|
||||
.mockResolvedValueOnce(ndjsonResponse([toolCall("web_search", { query: "one" })]))
|
||||
.mockResolvedValueOnce(
|
||||
jsonResponse({ results: [{ title: "First", url: "https://first.example", content: "A" }] }),
|
||||
)
|
||||
.mockResolvedValueOnce(ndjsonResponse([toolCall("web_search", { query: "two" })]))
|
||||
.mockResolvedValueOnce(
|
||||
jsonResponse({
|
||||
results: [{ title: "Second", url: "https://second.example", content: "B" }],
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(ndjsonResponse(answer("Both [1][2] agree.")));
|
||||
|
||||
const result = await runAgent({
|
||||
...baseOptions([{ role: "user", content: "hi" }]),
|
||||
tools: toolsFor({ webEnabled: true }),
|
||||
});
|
||||
|
||||
expect(result.toolCalls.map((call) => call.sources?.map((source) => source.index))).toEqual([[1], [2]]);
|
||||
expect(result.toolCalls[0].result).toContain("[1] First");
|
||||
expect(result.toolCalls[1].result).toContain("[2] Second");
|
||||
});
|
||||
|
||||
it("reports an unknown tool as an error without aborting the turn", async () => {
|
||||
fetchMock
|
||||
.mockResolvedValueOnce(ndjsonResponse([toolCall("no_such_tool")]))
|
||||
@@ -229,7 +267,9 @@ describe("runAgent", () => {
|
||||
for (let i = 0; i < DEFAULT_MAX_TOOL_ROUNDS; i++) {
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([preamble(i)]));
|
||||
}
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([{ message: { role: "assistant", content: "" }, done: true }]));
|
||||
fetchMock.mockResolvedValueOnce(
|
||||
ndjsonResponse([{ message: { role: "assistant", content: "" }, done: true }]),
|
||||
);
|
||||
|
||||
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
||||
|
||||
@@ -261,137 +301,137 @@ describe("runAgent", () => {
|
||||
});
|
||||
});
|
||||
|
||||
/** Queues `count` rounds that all ask for the identical call. */
|
||||
function queueRepeats(count: number): void {
|
||||
for (let i = 0; i < count; i++) {
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
||||
}
|
||||
/** Queues `count` rounds that all ask for the identical call. */
|
||||
function queueRepeats(count: number): void {
|
||||
for (let i = 0; i < count; i++) {
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
||||
}
|
||||
}
|
||||
|
||||
it("stops when the same call comes round a third time instead of running it again", async () => {
|
||||
// The rut this guards against: the model decides the first search did not
|
||||
// answer the question and asks the identical question again, and again.
|
||||
// Two executions is a retry; a third is not going to learn anything new.
|
||||
queueRepeats(3);
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("It is 03:04 UTC.")));
|
||||
it("stops when the same call comes round a third time instead of running it again", async () => {
|
||||
// The rut this guards against: the model decides the first search did not
|
||||
// answer the question and asks the identical question again, and again.
|
||||
// Two executions is a retry; a third is not going to learn anything new.
|
||||
queueRepeats(3);
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("It is 03:04 UTC.")));
|
||||
|
||||
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
||||
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
||||
|
||||
// Two executions, plus the refusal that ended the turn.
|
||||
expect(result.toolCalls).toHaveLength(3);
|
||||
expect(result.toolCalls.filter((call) => call.repeated)).toHaveLength(1);
|
||||
expect(result.content).toBe("It is 03:04 UTC.");
|
||||
// Two tool rounds, the refusal round, and the closing round.
|
||||
expect(fetchMock).toHaveBeenCalledTimes(4);
|
||||
// Two executions, plus the refusal that ended the turn.
|
||||
expect(result.toolCalls).toHaveLength(3);
|
||||
expect(result.toolCalls.filter((call) => call.repeated)).toHaveLength(1);
|
||||
expect(result.content).toBe("It is 03:04 UTC.");
|
||||
// Two tool rounds, the refusal round, and the closing round.
|
||||
expect(fetchMock).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
it("does not count a different call with the same tool as a repeat", async () => {
|
||||
// Searching for three different things is a model doing its job, not a loop.
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "CET" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "JST" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("Three zones.")));
|
||||
|
||||
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
||||
|
||||
expect(result.toolCalls).toHaveLength(3);
|
||||
expect(result.toolCalls.every((call) => !call.repeated)).toBe(true);
|
||||
});
|
||||
|
||||
it("treats reordered arguments as the same call", async () => {
|
||||
// Models are not consistent about key order, and a guard that only catches
|
||||
// byte-identical calls would miss the ordinary shape of a repeat.
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC", unit: "h" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { unit: "h", tz: "UTC" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC", unit: "h" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
||||
|
||||
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
||||
|
||||
expect(result.toolCalls.filter((call) => call.repeated)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("tells the model the call was repeating rather than that a budget ran out", async () => {
|
||||
queueRepeats(3);
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("ok")));
|
||||
|
||||
await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
||||
|
||||
const last = JSON.parse(fetchMock.mock.calls[3][1].body as string) as { messages: ApiMessage[] };
|
||||
const nudge = last.messages[last.messages.length - 1];
|
||||
|
||||
expect(nudge.role).toBe("user");
|
||||
expect(nudge.content).toContain("identical arguments 3 times");
|
||||
expect(nudge.content).not.toContain("reached the limit");
|
||||
});
|
||||
|
||||
it("answers from what it gathered when a repeat ends the turn", async () => {
|
||||
queueRepeats(3);
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("I could not get past the time check.")));
|
||||
|
||||
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
||||
|
||||
expect(result.content).toBe("I could not get past the time check.");
|
||||
expect(result.content).not.toContain("kept repeating");
|
||||
});
|
||||
|
||||
it("says the turn was cut short when a repeat leaves the closing round speechless", async () => {
|
||||
queueRepeats(3);
|
||||
fetchMock.mockResolvedValueOnce(
|
||||
ndjsonResponse([{ message: { role: "assistant", content: "" }, done: true }]),
|
||||
);
|
||||
|
||||
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
||||
|
||||
expect(result.content).toContain("kept repeating the same call");
|
||||
});
|
||||
|
||||
it("honours a caller's round cap in place of the default", async () => {
|
||||
for (let i = 0; i < 4; i++) {
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([distinctCall(i)]));
|
||||
}
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
||||
|
||||
const result = await runAgent({ ...baseOptions([{ role: "user", content: "hi" }]), maxToolRounds: 4 });
|
||||
|
||||
expect(result.toolCalls).toHaveLength(4);
|
||||
expect(result.content).toBe("done");
|
||||
});
|
||||
|
||||
it("runs past the default cap when the caller asks for no cap at all", async () => {
|
||||
// The preference can be switched to Unlimited, and the only difference that
|
||||
// should make is that the cap stops arriving.
|
||||
const rounds = DEFAULT_MAX_TOOL_ROUNDS + 3;
|
||||
for (let i = 0; i < rounds; i++) {
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([distinctCall(i)]));
|
||||
}
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
||||
|
||||
const result = await runAgent({ ...baseOptions([{ role: "user", content: "hi" }]), maxToolRounds: null });
|
||||
|
||||
expect(result.toolCalls).toHaveLength(rounds);
|
||||
expect(result.content).toBe("done");
|
||||
});
|
||||
|
||||
it("reports each tool start with how many times that call has come round", async () => {
|
||||
// This is what the transcript shows while a turn is in flight, so a repeat
|
||||
// is visible as it happens and not only in the record afterwards.
|
||||
const started: { name: string; label: string; count: number }[] = [];
|
||||
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
||||
|
||||
await runAgent({
|
||||
...baseOptions([{ role: "user", content: "hi" }]),
|
||||
events: { onToolStart: (name, label, count) => started.push({ name, label, count }) },
|
||||
});
|
||||
|
||||
it("does not count a different call with the same tool as a repeat", async () => {
|
||||
// Searching for three different things is a model doing its job, not a loop.
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "CET" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "JST" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("Three zones.")));
|
||||
|
||||
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
||||
|
||||
expect(result.toolCalls).toHaveLength(3);
|
||||
expect(result.toolCalls.every((call) => !call.repeated)).toBe(true);
|
||||
});
|
||||
|
||||
it("treats reordered arguments as the same call", async () => {
|
||||
// Models are not consistent about key order, and a guard that only catches
|
||||
// byte-identical calls would miss the ordinary shape of a repeat.
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC", unit: "h" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { unit: "h", tz: "UTC" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC", unit: "h" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
||||
|
||||
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
||||
|
||||
expect(result.toolCalls.filter((call) => call.repeated)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("tells the model the call was repeating rather than that a budget ran out", async () => {
|
||||
queueRepeats(3);
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("ok")));
|
||||
|
||||
await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
||||
|
||||
const last = JSON.parse(fetchMock.mock.calls[3][1].body as string) as { messages: ApiMessage[] };
|
||||
const nudge = last.messages[last.messages.length - 1];
|
||||
|
||||
expect(nudge.role).toBe("user");
|
||||
expect(nudge.content).toContain("identical arguments 3 times");
|
||||
expect(nudge.content).not.toContain("reached the limit");
|
||||
});
|
||||
|
||||
it("answers from what it gathered when a repeat ends the turn", async () => {
|
||||
queueRepeats(3);
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("I could not get past the time check.")));
|
||||
|
||||
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
||||
|
||||
expect(result.content).toBe("I could not get past the time check.");
|
||||
expect(result.content).not.toContain("kept repeating");
|
||||
});
|
||||
|
||||
it("says the turn was cut short when a repeat leaves the closing round speechless", async () => {
|
||||
queueRepeats(3);
|
||||
fetchMock.mockResolvedValueOnce(
|
||||
ndjsonResponse([{ message: { role: "assistant", content: "" }, done: true }]),
|
||||
);
|
||||
|
||||
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
||||
|
||||
expect(result.content).toContain("kept repeating the same call");
|
||||
});
|
||||
|
||||
it("honours a caller's round cap in place of the default", async () => {
|
||||
for (let i = 0; i < 4; i++) {
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([distinctCall(i)]));
|
||||
}
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
||||
|
||||
const result = await runAgent({ ...baseOptions([{ role: "user", content: "hi" }]), maxToolRounds: 4 });
|
||||
|
||||
expect(result.toolCalls).toHaveLength(4);
|
||||
expect(result.content).toBe("done");
|
||||
});
|
||||
|
||||
it("runs past the default cap when the caller asks for no cap at all", async () => {
|
||||
// The preference can be switched to Unlimited, and the only difference that
|
||||
// should make is that the cap stops arriving.
|
||||
const rounds = DEFAULT_MAX_TOOL_ROUNDS + 3;
|
||||
for (let i = 0; i < rounds; i++) {
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([distinctCall(i)]));
|
||||
}
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
||||
|
||||
const result = await runAgent({ ...baseOptions([{ role: "user", content: "hi" }]), maxToolRounds: null });
|
||||
|
||||
expect(result.toolCalls).toHaveLength(rounds);
|
||||
expect(result.content).toBe("done");
|
||||
});
|
||||
|
||||
it("reports each tool start with how many times that call has come round", async () => {
|
||||
// This is what the transcript shows while a turn is in flight, so a repeat
|
||||
// is visible as it happens and not only in the record afterwards.
|
||||
const started: { name: string; label: string; count: number }[] = [];
|
||||
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
||||
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
||||
|
||||
await runAgent({
|
||||
...baseOptions([{ role: "user", content: "hi" }]),
|
||||
events: { onToolStart: (name, label, count) => started.push({ name, label, count }) },
|
||||
});
|
||||
|
||||
expect(started).toEqual([
|
||||
{ name: "get_datetime", label: "Checking the time", count: 1 },
|
||||
{ name: "get_datetime", label: "Checking the time", count: 2 },
|
||||
]);
|
||||
});
|
||||
expect(started).toEqual([
|
||||
{ name: "get_datetime", label: "Checking the time", count: 1 },
|
||||
{ name: "get_datetime", label: "Checking the time", count: 2 },
|
||||
]);
|
||||
});
|
||||
|
||||
describe("module wiring", () => {
|
||||
it("keeps the request, tool, and agent layers free of Vicinae imports", () => {
|
||||
|
||||
Reference in New Issue
Block a user