src/__tests__/render.test.ts

@@ -1,93 +1,102 @@
 import { describe, expect, it } from "vitest";
 import { renderInlineContent, renderModelOutput } from "../lib/render";
+import type { Source } from "../lib/types";
+
+/**
+ * The drawn text, for the many cases here that are about what the pane shows
+ * rather than about which citations were linked.
+ */
+function rendered(markdown: string, sources?: Source[]): string {
+  return renderModelOutput(markdown, sources ? { sources } : {}).text;
+}
 
 describe("renderModelOutput", () => {
   it("renders inline maths in a sentence", () => {
-    expect(renderModelOutput("The answer is $x^2 + 1$, obviously.")).toBe("The answer is x² + 1, obviously.");
+    expect(rendered("The answer is $x^2 + 1$, obviously.")).toBe("The answer is x² + 1, obviously.");
-    expect(renderModelOutput("So \\(x^2\\) holds.")).toBe("So x² holds.");
+    expect(rendered("So \\(x^2\\) holds.")).toBe("So x² holds.");
   });
 
   it("leaves money and shell variables alone", () => {
-    expect(renderModelOutput("Costs $5 and $10 today.")).toBe("Costs $5 and $10 today.");
+    expect(rendered("Costs $5 and $10 today.")).toBe("Costs $5 and $10 today.");
-    expect(renderModelOutput("cd $HOME && echo $PATH")).toBe("cd $HOME && echo $PATH");
+    expect(rendered("cd $HOME && echo $PATH")).toBe("cd $HOME && echo $PATH");
   });
 
   it("puts a display formula on a paragraph of its own", () => {
-    expect(renderModelOutput("$$\\frac{a}{b} = c$$")).toBe("\na/b = c\n");
+    expect(rendered("$$\\frac{a}{b} = c$$")).toBe("\na/b = c\n");
-    expect(renderModelOutput("$$\nx = 1\n$$")).toBe("\nx = 1\n");
+    expect(rendered("$$\nx = 1\n$$")).toBe("\nx = 1\n");
-    expect(renderModelOutput("\\[\\sum_{i=1}^{n} i\\]")).toBe("\n∑ᵢ₌₁ⁿ i\n");
+    expect(rendered("\\[\\sum_{i=1}^{n} i\\]")).toBe("\n∑ᵢ₌₁ⁿ i\n");
   });
 
   it("does not let a stray marker swallow the rest of the answer", () => {
     // An opening `$$` with no closer only counts when what follows reads as a
     // formula, so a price is not a formula and the paragraphs after it survive.
-    expect(renderModelOutput("The price is\n$$5 per unit.\n\nThat is cheap.")).toBe(
+    expect(rendered("The price is\n$$5 per unit.\n\nThat is cheap.")).toBe(
       "The price is\n$$5 per unit.\n\nThat is cheap.",
     );
   });
 
   it("renders a maths fence as a formula rather than as code", () => {
-    expect(renderModelOutput("```math\n\\sum_{i=1}^{n} i\n```")).toBe("\n∑ᵢ₌₁ⁿ i\n");
+    expect(rendered("```math\n\\sum_{i=1}^{n} i\n```")).toBe("\n∑ᵢ₌₁ⁿ i\n");
-    expect(renderModelOutput("```latex\nx^2\n```")).toBe("\nx²\n");
+    expect(rendered("```latex\nx^2\n```")).toBe("\nx²\n");
   });
 
   it("leaves maths inside a fence alone", () => {
     const markdown = "```ts\nconst x = $a$;\n```";
-    expect(renderModelOutput(markdown)).toBe(markdown);
+    expect(rendered(markdown)).toBe(markdown);
   });
 
   it("leaves a dollar inside a code span alone", () => {
-    expect(renderModelOutput("`$x^2$` stays.")).toBe("`$x^2$` stays.");
+    expect(rendered("`$x^2$` stays.")).toBe("`$x^2$` stays.");
   });
 
   it("renames a fence language the renderer would print but not colour", () => {
-    expect(renderModelOutput("```console\nls\n```")).toBe("```bash\nls\n```");
+    expect(rendered("```console\nls\n```")).toBe("```bash\nls\n```");
-    expect(renderModelOutput("```terminal\nls\n```")).toBe("```bash\nls\n```");
+    expect(rendered("```terminal\nls\n```")).toBe("```bash\nls\n```");
   });
 
   it("drops a fence label that carries no information", () => {
-    expect(renderModelOutput("```text\nplain\n```")).toBe("```\nplain\n```");
+    expect(rendered("```text\nplain\n```")).toBe("```\nplain\n```");
   });
 
   it("leaves a label that does resolve exactly as written", () => {
     const markdown = "```python\nprint(1)\n```";
-    expect(renderModelOutput(markdown)).toBe(markdown);
+    expect(rendered(markdown)).toBe(markdown);
   });
 
   it("fences a diagram so its alignment survives", () => {
-    expect(renderModelOutput("src/\n├── a.ts\n└── b.ts")).toBe("```\nsrc/\n├── a.ts\n└── b.ts\n```");
+    expect(rendered("src/\n├── a.ts\n└── b.ts")).toBe("```\nsrc/\n├── a.ts\n└── b.ts\n```");
   });
 
   it("leaves a real table to the table renderer", () => {
     const table = "| a | b |\n| --- | --- |\n| 1 | 2 |";
-    expect(renderModelOutput(table)).toBe(table);
+    expect(rendered(table)).toBe(table);
   });
 
   it("leaves a diagram inside a fence alone", () => {
     const markdown = "```text\n├── a.ts\n└── b.ts\n```";
-    expect(renderModelOutput(markdown)).toBe("```\n├── a.ts\n└── b.ts\n```");
+    expect(rendered(markdown)).toBe("```\n├── a.ts\n└── b.ts\n```");
   });
 
   it("filters HTML in the flow of an answer", () => {
-    expect(renderModelOutput("<script>alert(1)</script>still here")).toBe("still here");
+    expect(rendered("<script>alert(1)</script>still here")).toBe("still here");
-    expect(renderModelOutput('<span style="color:#ff0000;position:fixed">x</span>')).toBe(
+    expect(rendered('<span style="color:#ff0000;position:fixed">x</span>')).toBe(
       '<span style="color:#ff0000">x</span>',
     );
-    expect(renderModelOutput('Read <a href="https://example.com">the docs</a>.')).toBe(
+    expect(rendered('Read <a href="https://example.com">the docs</a>.')).toBe(
       "Read [the docs](https://example.com).",
     );
   });
 
   it("leaves the markup inside a fence as the code it is", () => {
     const markdown = "```html\n<b>x</b>\n```";
-    expect(renderModelOutput(markdown)).toBe(markdown);
+    expect(rendered(markdown)).toBe(markdown);
   });
 
   it("hands a block of HTML to the engine as a paragraph", () => {
     // A line that starts with a block tag is an HTML *block*, and Vicinae's block
     // path keeps only the text of one. Wrapped in a span it is a paragraph, which
     // is the path that keeps raw HTML.
-    expect(renderModelOutput("<table><tr><td>a</td><td>b</td></tr></table>")).toBe(
+    expect(rendered("<table><tr><td>a</td><td>b</td></tr></table>")).toBe(
       "<span><table><tr><td>a</td><td>b</td></tr></table></span>",
     );
   });
@@ -95,35 +104,31 @@ describe("renderModelOutput", () => {
   it("joins a block of HTML onto one line", () => {
     // `class` is dropped, because there is no stylesheet for it to name, and the
     // style attribute is the only thing the renderer reads.
-    expect(renderModelOutput('<div class="card">\n<b>bold</b>\n</div>')).toBe(
+    expect(rendered('<div class="card">\n<b>bold</b>\n</div>')).toBe("<span><div> <b>bold</b> </div></span>");
-      "<span><div> <b>bold</b> </div></span>",
-    );
   });
 
   it("leaves markup that does not balance where the text it carries still shows", () => {
-    expect(renderModelOutput("<div>hello")).toBe("<div>hello");
+    expect(rendered("<div>hello")).toBe("<div>hello");
   });
 
   it("leaves a block holding an image alone, because that path loads it", () => {
-    expect(renderModelOutput('<div><img src="https://x/y.png"></div>')).toBe(
+    expect(rendered('<div><img src="https://x/y.png"></div>')).toBe('<div><img src="https://x/y.png"></div>');
-      '<div><img src="https://x/y.png"></div>',
-    );
   });
 
   it("leaves a lone inline tag alone, because it is not a block", () => {
     const markdown = "Try <b>bold</b> for that.";
-    expect(renderModelOutput(markdown)).toBe(markdown);
+    expect(rendered(markdown)).toBe(markdown);
   });
 
   it("keeps an answer that has nothing to render exactly as written", () => {
     const plain = "Just a sentence, with a [link](https://example.com) and `code`.";
-    expect(renderModelOutput(plain)).toBe(plain);
+    expect(rendered(plain)).toBe(plain);
   });
 
   it("repairs fences in model output but not in what a person typed", () => {
     // A lone command is a fence when a model writes it, and a question about it
     // is left as the person typed it.
-    expect(renderModelOutput("`pwd`")).toBe("```bash\npwd\n```");
+    expect(rendered("`pwd`")).toBe("```bash\npwd\n```");
     expect(renderInlineContent("`pwd`")).toBe("`pwd`");
   });
 
@@ -133,3 +138,31 @@ describe("renderModelOutput", () => {
     expect(renderInlineContent("```ts\nconst x = $a$;\n```")).toBe("```ts\nconst x = $a$;\n```");
   });
 });
+
+describe("citations in the pipeline", () => {
+  const sources = [{ index: 1, title: "First", url: "https://first.example" }];
+
+  it("reports which citations it linked", () => {
+    const result = renderModelOutput("Rain [1].", { sources });
+
+    expect(result.text).toBe("Rain [\\[1\\]](https://first.example).");
+    expect([...result.cited]).toEqual([1]);
+  });
+
+  it("links a marker that has to survive the maths and markup passes", () => {
+    expect(rendered("Let $x^2$ be [1], and nothing else.", sources)).toContain(
+      "x² be [\\[1\\]](https://first.example)",
+    );
+  });
+
+  it("leaves a marker inside a fence for the reader to copy", () => {
+    const text = rendered("```bash\ncurl [1]\n```\n\nSee [1].", sources);
+
+    expect(text).toContain("curl [1]");
+    expect(text).toContain("[\\[1\\]](https://first.example).");
+  });
+
+  it("changes nothing when no sources are given", () => {
+    expect(rendered("Rain [1].")).toBe("Rain [1].");
+  });
+});

src/lib/markdown.ts

@@ -154,7 +154,7 @@ function renderMessage(message: ThreadMessage, options: TranscriptOptions): stri
     ? // One pass, in one place: fences get their languages and links recovered,
       // then TeX becomes something drawable and the HTML around it is filtered to
       // the subset the renderer acts on. See `render.ts` for the order and why.
-      renderModelOutput(answered)
+      renderModelOutput(answered).text
     : !message.isStreaming
       ? "_No response._"
       : activityLine

src/lib/render.ts

@@ -1,6 +1,8 @@
+import { linkCitations } from "./citations";
 import { CLOSING_FENCE, OPENING_FENCE, mapOutsideCodeSpans, readCodeSpan, repairCodeBlocks } from "./code";
 import { sanitizeInlineHtml } from "./html";
 import { looksLikeMath, renderLatex } from "./latex";
+import type { Source } from "./types";
 
 /**
  * The rendering pass for model output.
@@ -490,6 +492,30 @@ export function renderInlineContent(markdown: string): string {
   return flatten(htmlPass(mathPass(splitSegments(markdown))));
 }
 
+/** What the pane will draw, and what the answer cited so the list can filter. */
+export interface RenderedMarkdown {
+  text: string;
+  /**
+   * The numbers this answer cited that became links.
+   *
+   * Reported rather than worked out again by the caller: the pass already had to
+   * decide which markers were real and which were inside code, and a second reader
+   * of the same text could only disagree with it.
+   */
+  cited: Set<number>;
+}
+
+/** Optional input to the pass. */
+export interface RenderOptions {
+  /**
+   * The turn's sources, so a marker can become a link to the page it names.
+   *
+   * Absent for a question, and absent for an answer in a turn that consulted
+   * nothing, in which case the citation pass is the identity function.
+   */
+  sources?: Source[];
+}
+
 /**
  * The full rendering pass, from what the model wrote to what the pane draws.
  *
@@ -497,6 +523,28 @@ export function renderInlineContent(markdown: string): string {
  * markdown around each message (a blockquote for a question, a rule between
  * exchanges) and none of that should be read as model output.
  */
-export function renderModelOutput(markdown: string): string {
+export function renderModelOutput(markdown: string, options: RenderOptions = {}): RenderedMarkdown {
-  return renderInlineContent(repairCodeBlocks(markdown));
+  const sources = options.sources ?? [];
+  const cited = new Set<number>();
+
+  /**
+   * Maths first, then citations, then markup.
+   *
+   * The order is forced by one collision. A link label has to escape its brackets
+   * or the parser ends the link text early, and `[\[1\]]` is exactly the shape of
+   * LaTeX's own `\[...\]` display delimiter, which the maths pass would eat and
+   * hand back as a bare `1`. So the model's maths is converted before any marker
+   * becomes a link: the citation pass writes final form markdown, which is not
+   * input for anything but the markup pass, and that one only reads angle brackets.
+   */
+  const segments = mathPass(splitSegments(repairCodeBlocks(markdown))).map((segment) => {
+    if (segment.fence || sources.length === 0) return segment;
+
+    const result = linkCitations(segment.lines.join("\n"), sources);
+    for (const index of result.cited) cited.add(index);
+
+    return { fence: false, language: "", lines: result.text.split("\n") };
+  });
+
+  return { text: flatten(htmlPass(segments)), cited };
 }