fix: local Ollama serves web tools on a different path

src/__tests__/history.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from "vitest";
 import { DEFAULT_SYSTEM_PROMPT, RECENCY_HINT } from "../lib/constants";
-import { buildMessages, composeSystemPrompt } from "../lib/history";
+import { buildMessages, composeSystemPrompt, describeRuntime, type RuntimeContext } from "../lib/history";
 import type { ChatTurn } from "../lib/types";
 
 function turn(prompt: string, response: string): ChatTurn {
@@ -52,18 +52,97 @@ describe("buildMessages", () => {
   });
 });
 
+/** The setup a test is composing for: the user's own machine, which is the case that exposed the gaps. */
+function context(overrides: Partial<RuntimeContext> = {}): RuntimeContext {
+  return {
+    model: "ornith-1.5:9b",
+    host: "http://localhost:11434",
+    selfHosted: true,
+    webTools: true,
+    vision: "yes",
+    ...overrides,
+  };
+}
+
 describe("composeSystemPrompt", () => {
   it("uses the default prompt when the user has not set one", () => {
-    expect(composeSystemPrompt("", false)).toBe(DEFAULT_SYSTEM_PROMPT);
+    expect(composeSystemPrompt("", context())).toContain(DEFAULT_SYSTEM_PROMPT);
   });
 
-  it("prefers the user's prompt when set", () => {
-    expect(composeSystemPrompt("Be terse.", false)).toBe("Be terse.");
+  it("prefers the user's prompt when set, and still says what the model is", () => {
+    // The runtime facts are not a matter of style, so a custom prompt must not
+    // drop them. This is the case that would otherwise silently lose them.
+    const composed = composeSystemPrompt("Be terse.", context());
+
+    expect(composed).toContain("Be terse.");
+    expect(composed).not.toContain(DEFAULT_SYSTEM_PROMPT);
+    expect(composed).toContain("You are Lam");
   });
 
   it("appends the recency hint only when web tools are on", () => {
-    expect(composeSystemPrompt("Be terse.", true)).toBe(`Be terse.\n\n${RECENCY_HINT}`);
-    expect(composeSystemPrompt("Be terse.", false)).not.toContain(RECENCY_HINT);
+    expect(composeSystemPrompt("Be terse.", context({ webTools: true }))).toContain(RECENCY_HINT);
+    expect(composeSystemPrompt("Be terse.", context({ webTools: false }))).not.toContain(RECENCY_HINT);
+  });
+});
+
+describe("describeRuntime", () => {
+  it("names the model and the server for a local setup", () => {
+    const text = describeRuntime(context());
+
+    expect(text).toContain("`ornith-1.5:9b`");
+    expect(text).toContain("http://localhost:11434");
+    expect(text).toContain("reachable on this machine");
+  });
+
+  it("says Ollama Cloud when that is the host", () => {
+    const text = describeRuntime(
+      context({ host: "https://ollama.com", selfHosted: false, model: "gpt-oss:120b" }),
+    );
+
+    expect(text).toContain("Ollama Cloud");
+    expect(text).not.toContain("reachable on this machine");
+  });
+
+  it("explains the :cloud convention only where it applies", () => {
+    // A local daemon proxying a cloud model is the case where a model that
+    // assumed it was local would tell the user their conversation stays put.
+    expect(describeRuntime(context({ selfHosted: true }))).toContain("`:cloud`");
+    expect(describeRuntime(context({ selfHosted: false }))).not.toContain("leaves this machine");
+  });
+
+  it("tells it to say so when there are no web tools, rather than offering", () => {
+    expect(describeRuntime(context({ webTools: true }))).toContain("You have `web_search`");
+    expect(describeRuntime(context({ webTools: false }))).toContain("no web tools in this session");
+  });
+
+  it("uses the confirmed answer about images when there is one", () => {
+    expect(describeRuntime(context({ vision: "yes" }))).toContain("You can read images");
+    expect(describeRuntime(context({ vision: "no" }))).toContain("You cannot read images");
+  });
+
+  it("tells a model whose eyes are unconfirmed to describe what it sees rather than reason about it", () => {
+    // The reported failure: handed a screenshot and an unconfirmed answer, the
+    // model wrote two paragraphs about whether it could see instead of looking.
+    const text = describeRuntime(context({ vision: "unknown" }));
+
+    expect(text).toContain("has not been confirmed");
+    expect(text).toContain("describe what you see");
+  });
+
+  it("always denies access it does not have", () => {
+    const text = describeRuntime(context());
+
+    expect(text).toContain("You see only what the user types or attaches");
+    expect(text).toContain("cannot read their screen, files, clipboard");
+  });
+
+  it("never claims the model is Vicinae, which was the actual confusion", () => {
+    // The reported answer: "Vicinae is a desktop launcher for Ollama". Vicinae is
+    // the launcher; Ollama is what serves the model, and they are not the same.
+    const text = describeRuntime(context());
+
+    expect(text).toContain("running inside the Vicinae launcher");
+    expect(text).not.toMatch(/You are Vicinae/);
   });
 });
 

src/__tests__/ollama.test.ts

@@ -9,6 +9,8 @@ import {
   parseNdjsonStream,
   pullModel,
   supportsVision,
+  webFetch,
+  webSearch,
 } from "../lib/ollama";
 
 /**
@@ -403,3 +405,88 @@ describe("describePullProgress", () => {
     expect(describePullProgress({ status: "downloading", total: 100, completed: 140 })).toContain("100%");
   });
 });
+
+describe("web tool paths", () => {
+  const fetchMock = vi.fn();
+
+  beforeEach(() => {
+    fetchMock.mockReset();
+    vi.stubGlobal("fetch", fetchMock);
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+  });
+
+  function json(payload: unknown, status = 200): Response {
+    return new Response(JSON.stringify(payload), { status });
+  }
+
+  it("uses the plain path on Ollama Cloud", async () => {
+    fetchMock.mockResolvedValueOnce(json({ results: [] }));
+
+    await webSearch({ host: "https://ollama.com", apiKey: "k", query: "q" });
+
+    expect(fetchMock.mock.calls[0][0]).toBe("https://ollama.com/api/web_search");
+  });
+
+  it("uses the experimental path on a local server, which is the one it serves", async () => {
+    // The reported bug: this used to post to /api/web_search on every host, so
+    // pointing the extension at a local server made every web call a 404 and the
+    // model reported that its web tools were not configured.
+    fetchMock.mockResolvedValueOnce(json({ results: [] }));
+
+    await webSearch({ host: "http://localhost:11434", apiKey: "k", query: "q" });
+
+    expect(fetchMock.mock.calls[0][0]).toBe("http://localhost:11434/api/experimental/web_search");
+  });
+
+  it("does the same for web_fetch", async () => {
+    fetchMock.mockResolvedValueOnce(json({ title: "T", content: "C", links: null }));
+
+    await expect(
+      webFetch({ host: "http://localhost:11434", apiKey: "k", url: "https://example.com" }),
+    ).resolves.toEqual({ title: "T", content: "C", links: [] });
+
+    expect(fetchMock.mock.calls[0][0]).toBe("http://localhost:11434/api/experimental/web_fetch");
+  });
+
+  it("falls back to the other path when the preferred one is missing", async () => {
+    // A server built the other way round, or a future rename: a 404 means the
+    // route is absent, so trying the other one cannot double-send anything.
+    fetchMock
+      .mockResolvedValueOnce(new Response("404 page not found", { status: 404 }))
+      .mockResolvedValueOnce(json({ results: [{ title: "t", url: "u", content: "c" }] }));
+
+    await expect(webSearch({ host: "http://localhost:11434", apiKey: "k", query: "q" })).resolves.toHaveLength(
+      1,
+    );
+
+    expect(fetchMock.mock.calls.map((call) => call[0])).toEqual([
+      "http://localhost:11434/api/experimental/web_search",
+      "http://localhost:11434/api/web_search",
+    ]);
+  });
+
+  it("does not fall back on anything other than a 404", async () => {
+    // A 401 or a 500 is the server answering, and retrying the other path would
+    // just send the same refused request twice.
+    fetchMock.mockResolvedValueOnce(new Response("unauthorized", { status: 401 }));
+
+    await expect(webSearch({ host: "https://ollama.com", apiKey: "k", query: "q" })).rejects.toThrow(
+      /rejected the API key/,
+    );
+
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+
+  it("reports the failure from the path it actually tried last", async () => {
+    fetchMock
+      .mockResolvedValueOnce(new Response("404 page not found", { status: 404 }))
+      .mockResolvedValueOnce(new Response("404 page not found", { status: 404 }));
+
+    await expect(webSearch({ host: "http://localhost:11434", apiKey: "k", query: "q" })).rejects.toThrow(
+      /404/,
+    );
+  });
+});

src/lib/components/ChatView.tsx

@@ -131,9 +131,13 @@ function ChatSession({
   initialAttachmentPath,
   launchNote,
 }: ChatSessionProps) {
-  const chat = useChat({ preferences, initialSession });
+  // The capability check is resolved before the chat exists, because its answer
+  // is part of the system prompt: a model that knows it can read images describes
+  // the one it is sent, and one that does not know reasons about itself instead.
+  const vision = useVisionSupport(preferences);
+  const chat = useChat({ preferences, initialSession, vision: vision.support });
   const { replaceSession } = chat;
-  const { canSeeImages } = useVisionSupport(preferences);
+  const { canSeeImages } = vision;
   const { activity } = chat;
 
   const [input, setInput] = useState(initialPrompt ?? "");

src/lib/history.ts

@@ -1,5 +1,5 @@
 import { DEFAULT_SYSTEM_PROMPT, RECENCY_HINT } from "./constants";
-import type { ApiMessage, ChatTurn } from "./types";
+import type { ApiMessage, ChatTurn, VisionSupport } from "./types";
 
 /**
  * Turns stored turns into the message array sent to the model.
@@ -34,9 +34,83 @@ export function buildMessages(
   return messages;
 }
 
-/** Uses the user's prompt if they set one, otherwise Lam's default. */
-export function composeSystemPrompt(customPrompt: string, webEnabled: boolean): string {
+/**
+ * What Lam knows about the setup it is running in.
+ *
+ * Gathered from preferences and the capability check rather than asked for, so
+ * the model can answer questions about itself instead of guessing. The guesses
+ * were the problem: with none of this in the prompt, a local model described
+ * Vicinae as "a desktop launcher for Ollama", reported its own web tools as not
+ * configured, and reasoned out loud about whether it could see an image it had
+ * just been sent.
+ */
+export interface RuntimeContext {
+  model: string;
+  host: string;
+  /** True when the host is a server the user runs rather than ollama.com. */
+  selfHosted: boolean;
+  webTools: boolean;
+  vision: VisionSupport;
+}
+
+/**
+ * The paragraph that tells the model what it is and what it can do.
+ *
+ * Kept factual and short. Every claim here is one the extension can stand
+ * behind, because a model that is told something wrong about itself will state
+ * it confidently, and the user has no way to tell that apart from an answer.
+ *
+ * The `:cloud` convention is stated as a convention rather than as a lookup. The
+ * authoritative field is `remote_host` on `GET /api/tags`, which the Models
+ * command reads and shows as a `cloud` tag; fetching it here would mean a second
+ * request on every model change for one sentence. The convention is Ollama's own
+ * and holds on this machine, where both `:cloud` entries report
+ * `remote_host: https://ollama.com` and the local model reports nothing.
+ */
+export function describeRuntime(context: RuntimeContext): string {
+  const served = context.selfHosted
+    ? `the Ollama server at ${context.host}, which is reachable on this machine`
+    : "Ollama Cloud";
+
+  const lines = [
+    "What you are, for questions about your own setup. Do not speculate beyond this:",
+    `- You are Lam, running inside the Vicinae launcher on the user's desktop, and answering as \`${context.model}\` served by ${served}.`,
+    "- You see only what the user types or attaches. You cannot read their screen, files, clipboard, calendar or anything else on the machine, and you cannot act on it.",
+  ];
+
+  if (context.selfHosted) {
+    lines.push(
+      "- A model name ending in `:cloud` is proxied by that server to ollama.com, so the conversation leaves this machine when it is used. Anything else is running locally.",
+    );
+  }
+
+  lines.push(
+    context.webTools
+      ? "- You have `web_search` and `web_fetch`. Use them rather than guessing about anything recent, and say so if they fail."
+      : "- You have no web tools in this session. Say that plainly rather than offering to look something up.",
+  );
+
+  lines.push(
+    context.vision === "yes"
+      ? "- You can read images. A screenshot attached to a question is yours to describe, so describe it."
+      : context.vision === "no"
+        ? "- You cannot read images. If a question depends on one, say so plainly."
+        : "- Whether you can read images has not been confirmed. If one is attached with a question, describe what you see rather than reasoning about whether you are able to.",
+  );
+
+  return lines.join("\n");
+}
+
+/**
+ * Uses the user's prompt if they set one, otherwise Lam's default.
+ *
+ * The runtime paragraph is appended either way. It is not a matter of style, so
+ * replacing the default prompt should not silently remove it, and a user writing
+ * their own prompt has even less reason to expect the model to know what it is.
+ */
+export function composeSystemPrompt(customPrompt: string, context: RuntimeContext): string {
   const parts = [customPrompt.trim() || DEFAULT_SYSTEM_PROMPT];
-  if (webEnabled) parts.push(RECENCY_HINT);
+  if (context.webTools) parts.push(RECENCY_HINT);
+  parts.push(describeRuntime(context));
   return parts.join("\n\n");
 }

src/lib/hooks/useChat.ts

@@ -3,11 +3,12 @@ import { runAgent } from "../agent";
 import { toStoredAttachment } from "../attachments";
 import { ipPlace } from "../geo";
 import { buildMessages, composeSystemPrompt } from "../history";
+import { isSelfHosted } from "../ollama";
 import { OllamaError } from "../ollama";
 import type { ResolvedPreferences } from "../preferences";
 import { createSession, createTurn, saveSession, withTitleFromFirstTurn } from "../sessions";
 import { toolsFor } from "../tools";
-import type { PendingAttachment, Session } from "../types";
+import type { PendingAttachment, Session, VisionSupport } from "../types";
 import { errorMessage, isAbortError } from "../utils";
 
 /**
@@ -33,6 +34,14 @@ export interface UseChatOptions {
   preferences: ResolvedPreferences;
   /** Restores an existing conversation, e.g. when reopening from the sessions command. */
   initialSession?: Session;
+  /**
+   * Whether the model can read images, for the system prompt.
+   *
+   * Passed in rather than checked here because the answer is already known to the
+   * view: the snip action is gated on the same value, and asking twice would mean
+   * two capability requests per model change.
+   */
+  vision?: VisionSupport;
 }
 
 /** Which tool is running right now, and how stale it is getting. */
@@ -85,6 +94,11 @@ export function useChat(options: UseChatOptions): UseChat {
 
   const sessionRef = useRef(session);
   const preferencesRef = useRef(options.preferences);
+  /**
+   * Read through a ref for the same reason as the preferences: `ask` is a stable
+   * callback, and the capability check lands after the first render.
+   */
+  const visionRef = useRef<VisionSupport>(options.vision ?? "unknown");
   const abortRef = useRef<AbortController | null>(null);
   const liveRef = useRef(false);
   const responseRef = useRef("");
@@ -93,6 +107,9 @@ export function useChat(options: UseChatOptions): UseChat {
   const flushTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
 
   preferencesRef.current = options.preferences;
+  // Only ever forwards. The check starts as "unknown", and a value arriving is
+  // news; going back to "unknown" would be the view telling us less than before.
+  if (options.vision) visionRef.current = options.vision;
 
   /** Updates state and the ref together, so `ask` never reads a stale session. */
   const commit = useCallback((next: Session) => {
@@ -181,7 +198,13 @@ export function useChat(options: UseChatOptions): UseChat {
       abortRef.current = controller;
 
       const messages = buildMessages(
-        composeSystemPrompt(preferences.systemPrompt, preferences.webEnabled),
+        composeSystemPrompt(preferences.systemPrompt, {
+          model: preferences.model,
+          host: preferences.host,
+          selfHosted: isSelfHosted(preferences.host),
+          webTools: preferences.webEnabled,
+          vision: visionRef.current,
+        }),
         base.turns,
         prompt,
         attachment ? [attachment.base64] : undefined,

src/lib/hooks/useVision.ts

@@ -1,5 +1,6 @@
 import { useEffect, useState } from "react";
 import type { ResolvedPreferences } from "../preferences";
+import type { VisionSupport } from "../types";
 import { visionSupport } from "../vision";
 
 /**
@@ -17,12 +18,22 @@ import { visionSupport } from "../vision";
 export interface VisionState {
   /** True only once Ollama has confirmed the model reports the `vision` capability. */
   canSeeImages: boolean;
+  /**
+   * The same answer without the fail-closed collapse.
+   *
+   * Carried because the system prompt needs the difference: telling a model it
+   * cannot see when the check merely failed is a false claim about the user's
+   * setup, and telling it nothing is what made it reason out loud about its own
+   * capability instead of describing the image.
+   */
+  support: VisionSupport;
   /** True while an answer for this model is still outstanding. */
   isChecking: boolean;
 }
 
 export function useVisionSupport(preferences: ResolvedPreferences): VisionState {
   const [canSeeImages, setCanSeeImages] = useState(false);
+  const [support, setSupport] = useState<VisionSupport>("unknown");
   const [isChecking, setIsChecking] = useState(true);
 
   useEffect(() => {
@@ -37,8 +48,10 @@ export function useVisionSupport(preferences: ResolvedPreferences): VisionState
       model: preferences.model,
       signal: controller.signal,
     })
-      .then((support) => {
-        if (!cancelled) setCanSeeImages(support === "yes");
+      .then((answer) => {
+        if (cancelled) return;
+        setSupport(answer);
+        setCanSeeImages(answer === "yes");
       })
       .finally(() => {
         if (!cancelled) setIsChecking(false);
@@ -50,5 +63,5 @@ export function useVisionSupport(preferences: ResolvedPreferences): VisionState
     };
   }, [preferences.host, preferences.apiKey, preferences.model]);
 
-  return { canSeeImages, isChecking };
+  return { canSeeImages, support, isChecking };
 }

src/lib/ollama.ts

@@ -72,7 +72,7 @@ function describeHttpError(status: number, bodyText: string, host: string): Olla
     case 404:
       return new OllamaError(`Ollama returned 404 from ${host}.`, {
         status,
-        hint: "The model name may be wrong, or this host may not expose that endpoint. Local Ollama servers do not provide the web tools.",
+        hint: "The model name may be wrong, or this host may not expose that endpoint.",
       });
     case 429:
       return new OllamaError("Ollama is rate limiting this key.", {
@@ -417,6 +417,53 @@ export function supportsVision(capabilities: string[]): boolean {
   return capabilities.includes("vision");
 }
 
+/**
+ * Posts to a web tool, at the path this kind of host actually serves it from.
+ *
+ * The two kinds of host expose the same two operations at different paths, and
+ * there is no path that works for both. Measured, not looked up:
+ *
+ * - `https://ollama.com/api/web_search` answers 401 without a key, so the route
+ *   exists, and its `/api/experimental/` alias is a 404.
+ * - A local 0.35.1 daemon answers 404 to that path, and serves
+ *   `/api/experimental/web_search` with real results.
+ *
+ * This was a live bug: pointing the extension at a local server made every web
+ * tool call a 404, and the model, handed a failure and no explanation, told the
+ * user its web tools simply were not configured.
+ *
+ * The other path is tried when the first answers 404, because that is a server
+ * built the other way round rather than a request being refused. A 404 body is
+ * drained before the retry so the connection is not left hanging.
+ */
+async function postWebTool(options: {
+  host: string;
+  apiKey: string;
+  kind: "search" | "fetch";
+  body: Record<string, unknown>;
+  signal?: AbortSignal;
+}): Promise<Response> {
+  const host = normalizeHost(options.host);
+
+  const [preferred, fallback] = isSelfHosted(host)
+    ? [`/api/experimental/web_${options.kind}`, `/api/web_${options.kind}`]
+    : [`/api/web_${options.kind}`, `/api/experimental/web_${options.kind}`];
+
+  const post = (path: string) =>
+    fetch(`${host}${path}`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", ...authHeaders(options.apiKey) },
+      body: JSON.stringify(options.body),
+      signal: options.signal,
+    });
+
+  const response = await post(preferred);
+  if (response.status !== 404) return response;
+
+  await response.text().catch(() => "");
+  return post(fallback);
+}
+
 export interface WebSearchResult {
   title: string;
   url: string;
@@ -432,20 +479,19 @@ export interface WebSearchOptions {
 }
 
 export async function webSearch(options: WebSearchOptions): Promise<WebSearchResult[]> {
-  const host = normalizeHost(options.host);
-
   const body: Record<string, unknown> = { query: options.query };
   if (typeof options.maxResults === "number") body.max_results = options.maxResults;
 
-  const response = await fetch(`${host}/api/web_search`, {
-    method: "POST",
-    headers: { "Content-Type": "application/json", ...authHeaders(options.apiKey) },
-    body: JSON.stringify(body),
+  const response = await postWebTool({
+    host: options.host,
+    apiKey: options.apiKey,
+    kind: "search",
+    body,
     signal: options.signal,
   });
 
   if (!response.ok) {
-    throw describeHttpError(response.status, await safeText(response), host);
+    throw describeHttpError(response.status, await safeText(response), normalizeHost(options.host));
   }
 
   const payload = (await response.json()) as { results?: WebSearchResult[] };
@@ -466,19 +512,21 @@ export interface WebFetchOptions {
 }
 
 export async function webFetch(options: WebFetchOptions): Promise<WebFetchResult> {
-  const host = normalizeHost(options.host);
-
-  const response = await fetch(`${host}/api/web_fetch`, {
-    method: "POST",
-    headers: { "Content-Type": "application/json", ...authHeaders(options.apiKey) },
-    body: JSON.stringify({ url: options.url }),
+  const response = await postWebTool({
+    host: options.host,
+    apiKey: options.apiKey,
+    kind: "fetch",
+    body: { url: options.url },
     signal: options.signal,
   });
 
   if (!response.ok) {
-    throw describeHttpError(response.status, await safeText(response), host);
+    throw describeHttpError(response.status, await safeText(response), normalizeHost(options.host));
   }
 
+  // `links` comes back as null from a local daemon rather than as an empty list,
+  // which `??` covers. (Probed directly: a local `web_fetch` answered
+  // `{"title":...,"content":...,"links":null}`.)
   const payload = (await response.json()) as Partial<WebFetchResult>;
   return {
     title: payload.title ?? options.url,

src/lib/tools.ts

@@ -169,7 +169,7 @@ export async function executeTool(
     }
   } catch (error) {
     const suffix = isWebTool(name)
-      ? " The web tools require an Ollama Cloud host and an API key with web search enabled."
+      ? " Web search needs Ollama Cloud, or a local Ollama server new enough to serve it (0.35 and later)."
       : "";
     return { content: `${errorMessage(error)}${suffix}`, isError: true };
   }

src/lib/types.ts

@@ -148,6 +148,16 @@ export interface SessionSummary {
   attachmentPaths?: string[];
 }
 
+/**
+ * Whether the model in use can read images, as far as the capability check knows.
+ *
+ * `"unknown"` is deliberately distinct from `"no"`: a failed request is not
+ * evidence that a model cannot see, and reporting it as one would be a false
+ * claim about the user's setup. Lives here rather than in `vision.ts` so that
+ * modules which must stay free of Vicinae imports can name it.
+ */
+export type VisionSupport = "yes" | "no" | "unknown";
+
 /** A model as reported by `GET /api/tags`. */
 export interface ModelInfo {
   name: string;

src/lib/vision.ts

@@ -1,5 +1,6 @@
 import { LocalStorage } from "@vicinae/api";
 import { modelCapabilities, supportsVision } from "./ollama";
+import type { VisionSupport } from "./types";
 
 /**
  * The cached answer to "can this model see images?".
@@ -17,7 +18,11 @@ import { modelCapabilities, supportsVision } from "./ollama";
 /** One key per host and model, so switching model invalidates by construction. */
 const VISION_CACHE_PREFIX = "lam.vision.";
 
-export type VisionSupport = "yes" | "no" | "unknown";
+// Re-exported so callers can keep naming the type from here, where it belongs
+// conceptually. The definition moved to `types.ts` so that `history.ts`, which
+// must not import a Vicinae module, can describe the same tri-state in the
+// system prompt.
+export type { VisionSupport };
 
 export interface VisionCheck {
   host: string;