|
|
|
@@ -21,7 +21,18 @@ function ndjsonResponse(events: unknown[], status = 200): Response {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function toolCall(name: string, args: Record<string, unknown> = {}): Record<string, unknown> {
|
|
|
|
|
return { message: { role: "assistant", content: "", tool_calls: [{ function: { name, arguments: args } }] }, done: true };
|
|
|
|
|
return {
|
|
|
|
|
message: { role: "assistant", content: "", tool_calls: [{ function: { name, arguments: args } }] },
|
|
|
|
|
done: true,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** The shape the two Ollama web endpoints answer with. */
|
|
|
|
|
function jsonResponse(payload: unknown): Response {
|
|
|
|
|
return new Response(JSON.stringify(payload), {
|
|
|
|
|
status: 200,
|
|
|
|
|
headers: { "Content-Type": "application/json" },
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function answer(text: string): Record<string, unknown>[] {
|
|
|
|
@@ -136,6 +147,33 @@ describe("runAgent", () => {
|
|
|
|
|
expect(toolMessage?.content).toContain("2026");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("numbers a turn's sources once, and keeps each call's own on its record", async () => {
|
|
|
|
|
// Two searches in one turn. The second has to continue the first's numbering,
|
|
|
|
|
// or one marker in the answer could mean two different pages; and each record
|
|
|
|
|
// has to hold only what its own call found, or the list would repeat a page.
|
|
|
|
|
fetchMock
|
|
|
|
|
.mockResolvedValueOnce(ndjsonResponse([toolCall("web_search", { query: "one" })]))
|
|
|
|
|
.mockResolvedValueOnce(
|
|
|
|
|
jsonResponse({ results: [{ title: "First", url: "https://first.example", content: "A" }] }),
|
|
|
|
|
)
|
|
|
|
|
.mockResolvedValueOnce(ndjsonResponse([toolCall("web_search", { query: "two" })]))
|
|
|
|
|
.mockResolvedValueOnce(
|
|
|
|
|
jsonResponse({
|
|
|
|
|
results: [{ title: "Second", url: "https://second.example", content: "B" }],
|
|
|
|
|
}),
|
|
|
|
|
)
|
|
|
|
|
.mockResolvedValueOnce(ndjsonResponse(answer("Both [1][2] agree.")));
|
|
|
|
|
|
|
|
|
|
const result = await runAgent({
|
|
|
|
|
...baseOptions([{ role: "user", content: "hi" }]),
|
|
|
|
|
tools: toolsFor({ webEnabled: true }),
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(result.toolCalls.map((call) => call.sources?.map((source) => source.index))).toEqual([[1], [2]]);
|
|
|
|
|
expect(result.toolCalls[0].result).toContain("[1] First");
|
|
|
|
|
expect(result.toolCalls[1].result).toContain("[2] Second");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("reports an unknown tool as an error without aborting the turn", async () => {
|
|
|
|
|
fetchMock
|
|
|
|
|
.mockResolvedValueOnce(ndjsonResponse([toolCall("no_such_tool")]))
|
|
|
|
@@ -229,7 +267,9 @@ describe("runAgent", () => {
|
|
|
|
|
for (let i = 0; i < DEFAULT_MAX_TOOL_ROUNDS; i++) {
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([preamble(i)]));
|
|
|
|
|
}
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([{ message: { role: "assistant", content: "" }, done: true }]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(
|
|
|
|
|
ndjsonResponse([{ message: { role: "assistant", content: "" }, done: true }]),
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
|
|
|
|
|
|
|
|
@@ -261,137 +301,137 @@ describe("runAgent", () => {
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
/** Queues `count` rounds that all ask for the identical call. */
|
|
|
|
|
function queueRepeats(count: number): void {
|
|
|
|
|
for (let i = 0; i < count; i++) {
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
|
|
|
|
}
|
|
|
|
|
/** Queues `count` rounds that all ask for the identical call. */
|
|
|
|
|
function queueRepeats(count: number): void {
|
|
|
|
|
for (let i = 0; i < count; i++) {
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
it("stops when the same call comes round a third time instead of running it again", async () => {
|
|
|
|
|
// The rut this guards against: the model decides the first search did not
|
|
|
|
|
// answer the question and asks the identical question again, and again.
|
|
|
|
|
// Two executions is a retry; a third is not going to learn anything new.
|
|
|
|
|
queueRepeats(3);
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("It is 03:04 UTC.")));
|
|
|
|
|
it("stops when the same call comes round a third time instead of running it again", async () => {
|
|
|
|
|
// The rut this guards against: the model decides the first search did not
|
|
|
|
|
// answer the question and asks the identical question again, and again.
|
|
|
|
|
// Two executions is a retry; a third is not going to learn anything new.
|
|
|
|
|
queueRepeats(3);
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("It is 03:04 UTC.")));
|
|
|
|
|
|
|
|
|
|
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
|
|
|
|
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
|
|
|
|
|
|
|
|
|
// Two executions, plus the refusal that ended the turn.
|
|
|
|
|
expect(result.toolCalls).toHaveLength(3);
|
|
|
|
|
expect(result.toolCalls.filter((call) => call.repeated)).toHaveLength(1);
|
|
|
|
|
expect(result.content).toBe("It is 03:04 UTC.");
|
|
|
|
|
// Two tool rounds, the refusal round, and the closing round.
|
|
|
|
|
expect(fetchMock).toHaveBeenCalledTimes(4);
|
|
|
|
|
// Two executions, plus the refusal that ended the turn.
|
|
|
|
|
expect(result.toolCalls).toHaveLength(3);
|
|
|
|
|
expect(result.toolCalls.filter((call) => call.repeated)).toHaveLength(1);
|
|
|
|
|
expect(result.content).toBe("It is 03:04 UTC.");
|
|
|
|
|
// Two tool rounds, the refusal round, and the closing round.
|
|
|
|
|
expect(fetchMock).toHaveBeenCalledTimes(4);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("does not count a different call with the same tool as a repeat", async () => {
|
|
|
|
|
// Searching for three different things is a model doing its job, not a loop.
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "CET" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "JST" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("Three zones.")));
|
|
|
|
|
|
|
|
|
|
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
|
|
|
|
|
|
|
|
|
expect(result.toolCalls).toHaveLength(3);
|
|
|
|
|
expect(result.toolCalls.every((call) => !call.repeated)).toBe(true);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("treats reordered arguments as the same call", async () => {
|
|
|
|
|
// Models are not consistent about key order, and a guard that only catches
|
|
|
|
|
// byte-identical calls would miss the ordinary shape of a repeat.
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC", unit: "h" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { unit: "h", tz: "UTC" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC", unit: "h" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
|
|
|
|
|
|
|
|
|
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
|
|
|
|
|
|
|
|
|
expect(result.toolCalls.filter((call) => call.repeated)).toHaveLength(1);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("tells the model the call was repeating rather than that a budget ran out", async () => {
|
|
|
|
|
queueRepeats(3);
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("ok")));
|
|
|
|
|
|
|
|
|
|
await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
|
|
|
|
|
|
|
|
|
const last = JSON.parse(fetchMock.mock.calls[3][1].body as string) as { messages: ApiMessage[] };
|
|
|
|
|
const nudge = last.messages[last.messages.length - 1];
|
|
|
|
|
|
|
|
|
|
expect(nudge.role).toBe("user");
|
|
|
|
|
expect(nudge.content).toContain("identical arguments 3 times");
|
|
|
|
|
expect(nudge.content).not.toContain("reached the limit");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("answers from what it gathered when a repeat ends the turn", async () => {
|
|
|
|
|
queueRepeats(3);
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("I could not get past the time check.")));
|
|
|
|
|
|
|
|
|
|
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
|
|
|
|
|
|
|
|
|
expect(result.content).toBe("I could not get past the time check.");
|
|
|
|
|
expect(result.content).not.toContain("kept repeating");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("says the turn was cut short when a repeat leaves the closing round speechless", async () => {
|
|
|
|
|
queueRepeats(3);
|
|
|
|
|
fetchMock.mockResolvedValueOnce(
|
|
|
|
|
ndjsonResponse([{ message: { role: "assistant", content: "" }, done: true }]),
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
|
|
|
|
|
|
|
|
|
expect(result.content).toContain("kept repeating the same call");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("honours a caller's round cap in place of the default", async () => {
|
|
|
|
|
for (let i = 0; i < 4; i++) {
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([distinctCall(i)]));
|
|
|
|
|
}
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
|
|
|
|
|
|
|
|
|
const result = await runAgent({ ...baseOptions([{ role: "user", content: "hi" }]), maxToolRounds: 4 });
|
|
|
|
|
|
|
|
|
|
expect(result.toolCalls).toHaveLength(4);
|
|
|
|
|
expect(result.content).toBe("done");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("runs past the default cap when the caller asks for no cap at all", async () => {
|
|
|
|
|
// The preference can be switched to Unlimited, and the only difference that
|
|
|
|
|
// should make is that the cap stops arriving.
|
|
|
|
|
const rounds = DEFAULT_MAX_TOOL_ROUNDS + 3;
|
|
|
|
|
for (let i = 0; i < rounds; i++) {
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([distinctCall(i)]));
|
|
|
|
|
}
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
|
|
|
|
|
|
|
|
|
const result = await runAgent({ ...baseOptions([{ role: "user", content: "hi" }]), maxToolRounds: null });
|
|
|
|
|
|
|
|
|
|
expect(result.toolCalls).toHaveLength(rounds);
|
|
|
|
|
expect(result.content).toBe("done");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("reports each tool start with how many times that call has come round", async () => {
|
|
|
|
|
// This is what the transcript shows while a turn is in flight, so a repeat
|
|
|
|
|
// is visible as it happens and not only in the record afterwards.
|
|
|
|
|
const started: { name: string; label: string; count: number }[] = [];
|
|
|
|
|
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
|
|
|
|
|
|
|
|
|
await runAgent({
|
|
|
|
|
...baseOptions([{ role: "user", content: "hi" }]),
|
|
|
|
|
events: { onToolStart: (name, label, count) => started.push({ name, label, count }) },
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("does not count a different call with the same tool as a repeat", async () => {
|
|
|
|
|
// Searching for three different things is a model doing its job, not a loop.
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "CET" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "JST" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("Three zones.")));
|
|
|
|
|
|
|
|
|
|
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
|
|
|
|
|
|
|
|
|
expect(result.toolCalls).toHaveLength(3);
|
|
|
|
|
expect(result.toolCalls.every((call) => !call.repeated)).toBe(true);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("treats reordered arguments as the same call", async () => {
|
|
|
|
|
// Models are not consistent about key order, and a guard that only catches
|
|
|
|
|
// byte-identical calls would miss the ordinary shape of a repeat.
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC", unit: "h" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { unit: "h", tz: "UTC" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC", unit: "h" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
|
|
|
|
|
|
|
|
|
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
|
|
|
|
|
|
|
|
|
expect(result.toolCalls.filter((call) => call.repeated)).toHaveLength(1);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("tells the model the call was repeating rather than that a budget ran out", async () => {
|
|
|
|
|
queueRepeats(3);
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("ok")));
|
|
|
|
|
|
|
|
|
|
await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
|
|
|
|
|
|
|
|
|
const last = JSON.parse(fetchMock.mock.calls[3][1].body as string) as { messages: ApiMessage[] };
|
|
|
|
|
const nudge = last.messages[last.messages.length - 1];
|
|
|
|
|
|
|
|
|
|
expect(nudge.role).toBe("user");
|
|
|
|
|
expect(nudge.content).toContain("identical arguments 3 times");
|
|
|
|
|
expect(nudge.content).not.toContain("reached the limit");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("answers from what it gathered when a repeat ends the turn", async () => {
|
|
|
|
|
queueRepeats(3);
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("I could not get past the time check.")));
|
|
|
|
|
|
|
|
|
|
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
|
|
|
|
|
|
|
|
|
expect(result.content).toBe("I could not get past the time check.");
|
|
|
|
|
expect(result.content).not.toContain("kept repeating");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("says the turn was cut short when a repeat leaves the closing round speechless", async () => {
|
|
|
|
|
queueRepeats(3);
|
|
|
|
|
fetchMock.mockResolvedValueOnce(
|
|
|
|
|
ndjsonResponse([{ message: { role: "assistant", content: "" }, done: true }]),
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
const result = await runAgent(baseOptions([{ role: "user", content: "hi" }]));
|
|
|
|
|
|
|
|
|
|
expect(result.content).toContain("kept repeating the same call");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("honours a caller's round cap in place of the default", async () => {
|
|
|
|
|
for (let i = 0; i < 4; i++) {
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([distinctCall(i)]));
|
|
|
|
|
}
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
|
|
|
|
|
|
|
|
|
const result = await runAgent({ ...baseOptions([{ role: "user", content: "hi" }]), maxToolRounds: 4 });
|
|
|
|
|
|
|
|
|
|
expect(result.toolCalls).toHaveLength(4);
|
|
|
|
|
expect(result.content).toBe("done");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("runs past the default cap when the caller asks for no cap at all", async () => {
|
|
|
|
|
// The preference can be switched to Unlimited, and the only difference that
|
|
|
|
|
// should make is that the cap stops arriving.
|
|
|
|
|
const rounds = DEFAULT_MAX_TOOL_ROUNDS + 3;
|
|
|
|
|
for (let i = 0; i < rounds; i++) {
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([distinctCall(i)]));
|
|
|
|
|
}
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
|
|
|
|
|
|
|
|
|
const result = await runAgent({ ...baseOptions([{ role: "user", content: "hi" }]), maxToolRounds: null });
|
|
|
|
|
|
|
|
|
|
expect(result.toolCalls).toHaveLength(rounds);
|
|
|
|
|
expect(result.content).toBe("done");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("reports each tool start with how many times that call has come round", async () => {
|
|
|
|
|
// This is what the transcript shows while a turn is in flight, so a repeat
|
|
|
|
|
// is visible as it happens and not only in the record afterwards.
|
|
|
|
|
const started: { name: string; label: string; count: number }[] = [];
|
|
|
|
|
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse([toolCall("get_datetime", { tz: "UTC" })]));
|
|
|
|
|
fetchMock.mockResolvedValueOnce(ndjsonResponse(answer("done")));
|
|
|
|
|
|
|
|
|
|
await runAgent({
|
|
|
|
|
...baseOptions([{ role: "user", content: "hi" }]),
|
|
|
|
|
events: { onToolStart: (name, label, count) => started.push({ name, label, count }) },
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(started).toEqual([
|
|
|
|
|
{ name: "get_datetime", label: "Checking the time", count: 1 },
|
|
|
|
|
{ name: "get_datetime", label: "Checking the time", count: 2 },
|
|
|
|
|
]);
|
|
|
|
|
});
|
|
|
|
|
expect(started).toEqual([
|
|
|
|
|
{ name: "get_datetime", label: "Checking the time", count: 1 },
|
|
|
|
|
{ name: "get_datetime", label: "Checking the time", count: 2 },
|
|
|
|
|
]);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
describe("module wiring", () => {
|
|
|
|
|
it("keeps the request, tool, and agent layers free of Vicinae imports", () => {
|
|
|
|
|