diff --git a/src/__tests__/render.test.ts b/src/__tests__/render.test.ts index b94a69b..aeec779 100644 --- a/src/__tests__/render.test.ts +++ b/src/__tests__/render.test.ts @@ -1,93 +1,102 @@ import { describe, expect, it } from "vitest"; import { renderInlineContent, renderModelOutput } from "../lib/render"; +import type { Source } from "../lib/types"; + +/** + * The drawn text, for the many cases here that are about what the pane shows + * rather than about which citations were linked. + */ +function rendered(markdown: string, sources?: Source[]): string { + return renderModelOutput(markdown, sources ? { sources } : {}).text; +} describe("renderModelOutput", () => { it("renders inline maths in a sentence", () => { - expect(renderModelOutput("The answer is $x^2 + 1$, obviously.")).toBe("The answer is x² + 1, obviously."); - expect(renderModelOutput("So \\(x^2\\) holds.")).toBe("So x² holds."); + expect(rendered("The answer is $x^2 + 1$, obviously.")).toBe("The answer is x² + 1, obviously."); + expect(rendered("So \\(x^2\\) holds.")).toBe("So x² holds."); }); it("leaves money and shell variables alone", () => { - expect(renderModelOutput("Costs $5 and $10 today.")).toBe("Costs $5 and $10 today."); - expect(renderModelOutput("cd $HOME && echo $PATH")).toBe("cd $HOME && echo $PATH"); + expect(rendered("Costs $5 and $10 today.")).toBe("Costs $5 and $10 today."); + expect(rendered("cd $HOME && echo $PATH")).toBe("cd $HOME && echo $PATH"); }); it("puts a display formula on a paragraph of its own", () => { - expect(renderModelOutput("$$\\frac{a}{b} = c$$")).toBe("\na/b = c\n"); - expect(renderModelOutput("$$\nx = 1\n$$")).toBe("\nx = 1\n"); - expect(renderModelOutput("\\[\\sum_{i=1}^{n} i\\]")).toBe("\n∑ᵢ₌₁ⁿ i\n"); + expect(rendered("$$\\frac{a}{b} = c$$")).toBe("\na/b = c\n"); + expect(rendered("$$\nx = 1\n$$")).toBe("\nx = 1\n"); + expect(rendered("\\[\\sum_{i=1}^{n} i\\]")).toBe("\n∑ᵢ₌₁ⁿ i\n"); }); it("does not let a stray marker swallow the rest of the answer", () => { // An opening `$$` with no closer only counts when what follows reads as a // formula, so a price is not a formula and the paragraphs after it survive. - expect(renderModelOutput("The price is\n$$5 per unit.\n\nThat is cheap.")).toBe( + expect(rendered("The price is\n$$5 per unit.\n\nThat is cheap.")).toBe( "The price is\n$$5 per unit.\n\nThat is cheap.", ); }); it("renders a maths fence as a formula rather than as code", () => { - expect(renderModelOutput("```math\n\\sum_{i=1}^{n} i\n```")).toBe("\n∑ᵢ₌₁ⁿ i\n"); - expect(renderModelOutput("```latex\nx^2\n```")).toBe("\nx²\n"); + expect(rendered("```math\n\\sum_{i=1}^{n} i\n```")).toBe("\n∑ᵢ₌₁ⁿ i\n"); + expect(rendered("```latex\nx^2\n```")).toBe("\nx²\n"); }); it("leaves maths inside a fence alone", () => { const markdown = "```ts\nconst x = $a$;\n```"; - expect(renderModelOutput(markdown)).toBe(markdown); + expect(rendered(markdown)).toBe(markdown); }); it("leaves a dollar inside a code span alone", () => { - expect(renderModelOutput("`$x^2$` stays.")).toBe("`$x^2$` stays."); + expect(rendered("`$x^2$` stays.")).toBe("`$x^2$` stays."); }); it("renames a fence language the renderer would print but not colour", () => { - expect(renderModelOutput("```console\nls\n```")).toBe("```bash\nls\n```"); - expect(renderModelOutput("```terminal\nls\n```")).toBe("```bash\nls\n```"); + expect(rendered("```console\nls\n```")).toBe("```bash\nls\n```"); + expect(rendered("```terminal\nls\n```")).toBe("```bash\nls\n```"); }); it("drops a fence label that carries no information", () => { - expect(renderModelOutput("```text\nplain\n```")).toBe("```\nplain\n```"); + expect(rendered("```text\nplain\n```")).toBe("```\nplain\n```"); }); it("leaves a label that does resolve exactly as written", () => { const markdown = "```python\nprint(1)\n```"; - expect(renderModelOutput(markdown)).toBe(markdown); + expect(rendered(markdown)).toBe(markdown); }); it("fences a diagram so its alignment survives", () => { - expect(renderModelOutput("src/\n├── a.ts\n└── b.ts")).toBe("```\nsrc/\n├── a.ts\n└── b.ts\n```"); + expect(rendered("src/\n├── a.ts\n└── b.ts")).toBe("```\nsrc/\n├── a.ts\n└── b.ts\n```"); }); it("leaves a real table to the table renderer", () => { const table = "| a | b |\n| --- | --- |\n| 1 | 2 |"; - expect(renderModelOutput(table)).toBe(table); + expect(rendered(table)).toBe(table); }); it("leaves a diagram inside a fence alone", () => { const markdown = "```text\n├── a.ts\n└── b.ts\n```"; - expect(renderModelOutput(markdown)).toBe("```\n├── a.ts\n└── b.ts\n```"); + expect(rendered(markdown)).toBe("```\n├── a.ts\n└── b.ts\n```"); }); it("filters HTML in the flow of an answer", () => { - expect(renderModelOutput("still here")).toBe("still here"); - expect(renderModelOutput('x')).toBe( + expect(rendered("still here")).toBe("still here"); + expect(rendered('x')).toBe( 'x', ); - expect(renderModelOutput('Read the docs.')).toBe( + expect(rendered('Read the docs.')).toBe( "Read [the docs](https://example.com).", ); }); it("leaves the markup inside a fence as the code it is", () => { const markdown = "```html\nx\n```"; - expect(renderModelOutput(markdown)).toBe(markdown); + expect(rendered(markdown)).toBe(markdown); }); it("hands a block of HTML to the engine as a paragraph", () => { // A line that starts with a block tag is an HTML *block*, and Vicinae's block // path keeps only the text of one. Wrapped in a span it is a paragraph, which // is the path that keeps raw HTML. - expect(renderModelOutput("
ab
")).toBe( + expect(rendered("
ab
")).toBe( "
ab
", ); }); @@ -95,35 +104,31 @@ describe("renderModelOutput", () => { it("joins a block of HTML onto one line", () => { // `class` is dropped, because there is no stylesheet for it to name, and the // style attribute is the only thing the renderer reads. - expect(renderModelOutput('
\nbold\n
')).toBe( - "
bold
", - ); + expect(rendered('
\nbold\n
')).toBe("
bold
"); }); it("leaves markup that does not balance where the text it carries still shows", () => { - expect(renderModelOutput("
hello")).toBe("
hello"); + expect(rendered("
hello")).toBe("
hello"); }); it("leaves a block holding an image alone, because that path loads it", () => { - expect(renderModelOutput('
')).toBe( - '
', - ); + expect(rendered('
')).toBe('
'); }); it("leaves a lone inline tag alone, because it is not a block", () => { const markdown = "Try bold for that."; - expect(renderModelOutput(markdown)).toBe(markdown); + expect(rendered(markdown)).toBe(markdown); }); it("keeps an answer that has nothing to render exactly as written", () => { const plain = "Just a sentence, with a [link](https://example.com) and `code`."; - expect(renderModelOutput(plain)).toBe(plain); + expect(rendered(plain)).toBe(plain); }); it("repairs fences in model output but not in what a person typed", () => { // A lone command is a fence when a model writes it, and a question about it // is left as the person typed it. - expect(renderModelOutput("`pwd`")).toBe("```bash\npwd\n```"); + expect(rendered("`pwd`")).toBe("```bash\npwd\n```"); expect(renderInlineContent("`pwd`")).toBe("`pwd`"); }); @@ -133,3 +138,31 @@ describe("renderModelOutput", () => { expect(renderInlineContent("```ts\nconst x = $a$;\n```")).toBe("```ts\nconst x = $a$;\n```"); }); }); + +describe("citations in the pipeline", () => { + const sources = [{ index: 1, title: "First", url: "https://first.example" }]; + + it("reports which citations it linked", () => { + const result = renderModelOutput("Rain [1].", { sources }); + + expect(result.text).toBe("Rain [\\[1\\]](https://first.example)."); + expect([...result.cited]).toEqual([1]); + }); + + it("links a marker that has to survive the maths and markup passes", () => { + expect(rendered("Let $x^2$ be [1], and nothing else.", sources)).toContain( + "x² be [\\[1\\]](https://first.example)", + ); + }); + + it("leaves a marker inside a fence for the reader to copy", () => { + const text = rendered("```bash\ncurl [1]\n```\n\nSee [1].", sources); + + expect(text).toContain("curl [1]"); + expect(text).toContain("[\\[1\\]](https://first.example)."); + }); + + it("changes nothing when no sources are given", () => { + expect(rendered("Rain [1].")).toBe("Rain [1]."); + }); +}); diff --git a/src/lib/markdown.ts b/src/lib/markdown.ts index 6cdce72..8a01e31 100644 --- a/src/lib/markdown.ts +++ b/src/lib/markdown.ts @@ -154,7 +154,7 @@ function renderMessage(message: ThreadMessage, options: TranscriptOptions): stri ? // One pass, in one place: fences get their languages and links recovered, // then TeX becomes something drawable and the HTML around it is filtered to // the subset the renderer acts on. See `render.ts` for the order and why. - renderModelOutput(answered) + renderModelOutput(answered).text : !message.isStreaming ? "_No response._" : activityLine diff --git a/src/lib/render.ts b/src/lib/render.ts index 0c23bab..a7164e7 100644 --- a/src/lib/render.ts +++ b/src/lib/render.ts @@ -1,6 +1,8 @@ +import { linkCitations } from "./citations"; import { CLOSING_FENCE, OPENING_FENCE, mapOutsideCodeSpans, readCodeSpan, repairCodeBlocks } from "./code"; import { sanitizeInlineHtml } from "./html"; import { looksLikeMath, renderLatex } from "./latex"; +import type { Source } from "./types"; /** * The rendering pass for model output. @@ -490,6 +492,30 @@ export function renderInlineContent(markdown: string): string { return flatten(htmlPass(mathPass(splitSegments(markdown)))); } +/** What the pane will draw, and what the answer cited so the list can filter. */ +export interface RenderedMarkdown { + text: string; + /** + * The numbers this answer cited that became links. + * + * Reported rather than worked out again by the caller: the pass already had to + * decide which markers were real and which were inside code, and a second reader + * of the same text could only disagree with it. + */ + cited: Set; +} + +/** Optional input to the pass. */ +export interface RenderOptions { + /** + * The turn's sources, so a marker can become a link to the page it names. + * + * Absent for a question, and absent for an answer in a turn that consulted + * nothing, in which case the citation pass is the identity function. + */ + sources?: Source[]; +} + /** * The full rendering pass, from what the model wrote to what the pane draws. * @@ -497,6 +523,28 @@ export function renderInlineContent(markdown: string): string { * markdown around each message (a blockquote for a question, a rule between * exchanges) and none of that should be read as model output. */ -export function renderModelOutput(markdown: string): string { - return renderInlineContent(repairCodeBlocks(markdown)); +export function renderModelOutput(markdown: string, options: RenderOptions = {}): RenderedMarkdown { + const sources = options.sources ?? []; + const cited = new Set(); + + /** + * Maths first, then citations, then markup. + * + * The order is forced by one collision. A link label has to escape its brackets + * or the parser ends the link text early, and `[\[1\]]` is exactly the shape of + * LaTeX's own `\[...\]` display delimiter, which the maths pass would eat and + * hand back as a bare `1`. So the model's maths is converted before any marker + * becomes a link: the citation pass writes final form markdown, which is not + * input for anything but the markup pass, and that one only reads angle brackets. + */ + const segments = mathPass(splitSegments(repairCodeBlocks(markdown))).map((segment) => { + if (segment.fence || sources.length === 0) return segment; + + const result = linkCitations(segment.lines.join("\n"), sources); + for (const index of result.cited) cited.add(index); + + return { fence: false, language: "", lines: result.text.split("\n") }; + }); + + return { text: flatten(htmlPass(segments)), cited }; }