From e9386b3d46f6f6b2bb9adace5ba39b21250d81ac Mon Sep 17 00:00:00 2001 From: Adrian Bonpin Date: Mon, 5 Oct 2026 17:41:33 +0800 Subject: [PATCH] feat: unload a loaded model from the models list --- README.md | 9 +++ package.json | 2 +- src/__tests__/ollama.test.ts | 110 +++++++++++++++++++++++++++++++++++ src/lib/ollama.ts | 102 ++++++++++++++++++++++++++++++++ src/models.tsx | 75 ++++++++++++++++++++++-- 5 files changed, 291 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 051e188..c2cda82 100644 --- a/README.md +++ b/README.md @@ -144,10 +144,19 @@ vicinae 'vicinae://launch/lam-agent/chat?fallbackText=summarise%20this' | Action | Notes | | --- | --- | | **Use This Model** | The primary action. Immediate, and survives restarts. | +| **Unload Model** | Only on a row that is loaded. No confirmation, because nothing is deleted. | | **Copy Model Name** | For pasting into the `Model` preference. | | **Delete Model** | Confirmed first. A server you run only. | | **Pull a Model…** | Confirmed by the download itself. A server you run only. | +### Loaded models + +Installed and loaded are different states, and the list shows both. `GET /api/ps` reports what is holding memory right now, and those rows get a green `loaded` tag, so "why is my GPU busy" has an answer in the same place as "what do I have". + +**Unload** frees the memory without deleting anything, which is why it needs no confirmation: the worst case is that the next question pays to load the model again. It is `POST /api/generate` with `keep_alive: 0` and no prompt, which is how Ollama expresses "release this now", and what its own `ollama stop` does. The reply reports `done_reason: "unload"`, and the toast says how much was freed. + +A host that does not serve `/api/ps` (older builds answer 404) simply shows no loaded tags. That check resolving to nothing is not an error, and it must not take the model list down with it. + ### Why a pick is not written to the preference It cannot be. `@vicinae/api` exposes `getPreferenceValues` and nothing that sets one, so the only way to change the preference would be to write Vicinae's own config file. That does not work: preferences are read once at startup and held in memory, so an edit to `~/.config/vicinae/settings.json` is invisible until Vicinae restarts. This was measured rather than assumed. Pointing `ollamaHost` at a local listener that logged every request, then launching a command, showed the requests still going to the old host. diff --git a/package.json b/package.json index aff373c..8ba5147 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "name": "models", "title": "Ollama Models", "subtitle": "Lam Agent", - "description": "Browse the models your Ollama host can reach, pick the one Lam should use, and pull or delete models on a server you run. A pick takes effect immediately; a pull keeps downloading until it finishes, so leave the launcher open.", + "description": "Browse the models your Ollama host can reach, pick the one Lam should use, unload one that is holding memory, and pull or delete models on a server you run. A pick takes effect immediately; a pull keeps downloading until it finishes, so leave the launcher open.", "mode": "view" }, { diff --git a/src/__tests__/ollama.test.ts b/src/__tests__/ollama.test.ts index d27f572..75bca9b 100644 --- a/src/__tests__/ollama.test.ts +++ b/src/__tests__/ollama.test.ts @@ -4,11 +4,13 @@ import { describePullProgress, isSelfHosted, listModels, + listRunningModels, modelCapabilities, normalizeHost, parseNdjsonStream, pullModel, supportsVision, + unloadModel, webFetch, webSearch, } from "../lib/ollama"; @@ -490,3 +492,111 @@ describe("web tool paths", () => { ); }); }); + +describe("listRunningModels", () => { + const fetchMock = vi.fn(); + + beforeEach(() => { + fetchMock.mockReset(); + vi.stubGlobal("fetch", fetchMock); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + /** Trimmed from a real `GET /api/ps` on Ollama 0.35.1 with one model in VRAM. */ + const realPayload = { + models: [ + { + name: "ornith-1.5:9b", + model: "ornith-1.5:9b", + size: 10413733314, + size_vram: 10413733314, + expires_at: "2026-10-05T17:44:11.125356+08:00", + context_length: 131072, + details: { parameter_size: "9.0B" }, + }, + ], + }; + + it("reads what is in memory, including when it will go on its own", async () => { + fetchMock.mockResolvedValueOnce(new Response(JSON.stringify(realPayload), { status: 200 })); + + const running = await listRunningModels({ host: "http://localhost:11434", apiKey: "k" }); + + expect(running).toEqual([ + { + name: "ornith-1.5:9b", + parameterSize: "9.0B", + sizeVram: 10413733314, + expiresAt: Date.parse("2026-10-05T17:44:11.125356+08:00"), + }, + ]); + + expect(fetchMock.mock.calls[0][0]).toBe("http://localhost:11434/api/ps"); + }); + + it("is empty, not an error, when the server does not serve /api/ps", async () => { + // Older builds answer 404. Taking the whole model list down because the + // *running* list is unavailable would be a bad trade. + fetchMock.mockResolvedValueOnce(new Response("404 page not found", { status: 404 })); + + await expect(listRunningModels({ host: "http://localhost:11434", apiKey: "k" })).resolves.toEqual([]); + }); + + it("is empty when the request fails outright", async () => { + fetchMock.mockRejectedValueOnce(new Error("connection refused")); + + await expect(listRunningModels({ host: "http://localhost:11434", apiKey: "k" })).resolves.toEqual([]); + }); + + it("copes with a model holding no VRAM figure, as a CPU-offloaded one would", async () => { + fetchMock.mockResolvedValueOnce( + new Response(JSON.stringify({ models: [{ name: "cpu-model:1b", size_vram: 0 }] }), { status: 200 }), + ); + + await expect(listRunningModels({ host: "http://localhost:11434", apiKey: "k" })).resolves.toEqual([ + { name: "cpu-model:1b", parameterSize: undefined, sizeVram: 0, expiresAt: undefined }, + ]); + }); +}); + +describe("unloadModel", () => { + const fetchMock = vi.fn(); + + beforeEach(() => { + fetchMock.mockReset(); + vi.stubGlobal("fetch", fetchMock); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("posts keep_alive 0 with no prompt, which is how Ollama is told to release a model", async () => { + fetchMock.mockResolvedValueOnce( + new Response(JSON.stringify({ done: true, done_reason: "unload" }), { status: 200 }), + ); + + await unloadModel({ host: "http://localhost:11434", apiKey: "k", model: "ornith-1.5:9b" }); + + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe("http://localhost:11434/api/generate"); + expect(init.method).toBe("POST"); + // No `prompt` key at all, so this can never generate a completion. + expect(JSON.parse(init.body)).toEqual({ + model: "ornith-1.5:9b", + keep_alive: 0, + stream: false, + }); + }); + + it("surfaces a refusal rather than pretending the memory came back", async () => { + fetchMock.mockResolvedValueOnce(new Response("nope", { status: 500 })); + + await expect( + unloadModel({ host: "http://localhost:11434", apiKey: "k", model: "x:1" }), + ).rejects.toThrow(/HTTP 500/); + }); +}); diff --git a/src/lib/ollama.ts b/src/lib/ollama.ts index f7de3eb..99afe30 100644 --- a/src/lib/ollama.ts +++ b/src/lib/ollama.ts @@ -263,6 +263,108 @@ export interface DeleteModelOptions { signal?: AbortSignal; } +/** + * A model the server has in memory right now, as reported by `GET /api/ps`. + * + * Distinct from {@link ModelInfo}, which is what is on disk. A model can be + * installed without being loaded, and loaded models are the ones holding VRAM. + */ +export interface RunningModel { + name: string; + parameterSize?: string; + /** + * Bytes held in memory. Equal to the on-disk size when the model is fully + * offloaded to the GPU, and less when part of it is running on the CPU. + */ + sizeVram?: number; + /** When the server will drop it on its own, as a Unix timestamp. */ + expiresAt?: number; +} + +export interface ListRunningModelsOptions { + host: string; + apiKey: string; + signal?: AbortSignal; +} + +/** + * The models holding memory on this server. + * + * Resolves to an empty list rather than throwing. Nothing here is worth failing + * over: a server that does not serve `/api/ps` (older builds answer 404) simply + * has nothing known to be loaded, and taking the whole model list down because + * the *running* list is unavailable would be a bad trade. + */ +export async function listRunningModels(options: ListRunningModelsOptions): Promise { + const host = normalizeHost(options.host); + + try { + const response = await fetch(`${host}/api/ps`, { + headers: authHeaders(options.apiKey), + signal: options.signal, + }); + + if (!response.ok) return []; + + const payload = (await response.json()) as { + models?: Array<{ + name?: string; + model?: string; + size_vram?: number; + expires_at?: string; + details?: { parameter_size?: string }; + }>; + }; + + return (payload.models ?? []) + .map((model) => ({ + name: model.name ?? model.model ?? "", + parameterSize: model.details?.parameter_size, + sizeVram: typeof model.size_vram === "number" ? model.size_vram : undefined, + expiresAt: model.expires_at ? Date.parse(model.expires_at) : undefined, + })) + .filter((model) => model.name.length > 0); + } catch { + return []; + } +} + +export interface UnloadModelOptions { + host: string; + apiKey: string; + model: string; + signal?: AbortSignal; +} + +/** + * Drops a model out of memory, freeing the VRAM it was holding. + * + * `POST /api/generate` with `keep_alive: 0` and no prompt is how Ollama says + * "release this now"; its own `ollama stop` does the same thing. Probed against + * 0.35.1: the reply is `{"done":true,"done_reason":"unload"}` and `/api/ps` + * comes back empty. The `DELETE` form of the same route answers 405, so there is + * no delete-shaped way to do this. + * + * Nothing is deleted, so a mistake costs a reload. That is why this needs no + * confirmation while `deleteModel` does. + */ +export async function unloadModel(options: UnloadModelOptions): Promise { + const host = normalizeHost(options.host); + + const response = await fetch(`${host}/api/generate`, { + method: "POST", + headers: { "Content-Type": "application/json", ...authHeaders(options.apiKey) }, + // No `prompt`, so this never generates anything, and `stream: false` because + // there is exactly one reply to read. + body: JSON.stringify({ model: options.model, keep_alive: 0, stream: false }), + signal: options.signal, + }); + + if (!response.ok) { + throw describeHttpError(response.status, await safeText(response), host); + } +} + /** * Removes a model from a server the user runs. * diff --git a/src/models.tsx b/src/models.tsx index 39cd349..e2d8cf5 100644 --- a/src/models.tsx +++ b/src/models.tsx @@ -12,7 +12,16 @@ import { } from "@vicinae/api"; import { useCallback, useEffect, useState } from "react"; import { usePreferences } from "./lib/hooks/usePreferences"; -import { deleteModel, describePullProgress, isSelfHosted, listModels, pullModel } from "./lib/ollama"; +import { + deleteModel, + describePullProgress, + isSelfHosted, + listModels, + listRunningModels, + pullModel, + unloadModel, + type RunningModel, +} from "./lib/ollama"; import { clearModelSelection, selectModel, type ResolvedPreferences } from "./lib/preferences"; import type { ModelInfo } from "./lib/types"; import { errorMessage, formatBytes } from "./lib/utils"; @@ -20,10 +29,12 @@ import { errorMessage, formatBytes } from "./lib/utils"; /** * Browse, pick, and manage the models the configured Ollama host can reach. * - * Three jobs in one command, because they are three answers to the same question - * ("what models do I have?"): pick one, delete one, pull a new one. The last two - * only appear on a host the user runs, since there is nothing to pull onto - * Ollama Cloud and nothing to delete from it. + * Four jobs in one command, because they are four answers to the same question + * ("what models do I have?"): pick one, unload one, delete one, pull a new one. + * Unloading is the odd one out and the reason a row needs two states rather than + * one: a model can be installed without being loaded, and only the loaded ones + * are holding memory. Delete and pull only appear on a host the user runs, since + * there is nothing to pull onto Ollama Cloud and nothing to delete from it. * * ## Where a pick is stored * @@ -70,6 +81,8 @@ interface ModelsScreenProps { function ModelsScreen({ preferences, reloadPreferences }: ModelsScreenProps) { const [models, setModels] = useState(undefined); + /** By name, so a row can ask whether it is the one holding memory. */ + const [running, setRunning] = useState>(new Map()); const [error, setError] = useState(undefined); const [isLoading, setIsLoading] = useState(true); const [view, setView] = useState("list"); @@ -81,10 +94,19 @@ function ModelsScreen({ preferences, reloadPreferences }: ModelsScreenProps) { setIsLoading(true); try { - setModels(await listModels({ host: preferences.host, apiKey: preferences.apiKey })); + // Both at once: what is installed and what is loaded are different + // questions, and the second must not hold up the first. + const [installed, loaded] = await Promise.all([ + listModels({ host: preferences.host, apiKey: preferences.apiKey }), + listRunningModels({ host: preferences.host, apiKey: preferences.apiKey }), + ]); + + setModels(installed); + setRunning(new Map(loaded.map((model) => [model.name, model]))); setError(undefined); } catch (thrown) { setModels([]); + setRunning(new Map()); setError(errorMessage(thrown)); } finally { setIsLoading(false); @@ -95,6 +117,38 @@ function ModelsScreen({ preferences, reloadPreferences }: ModelsScreenProps) { void load(); }, [load]); + /** + * Releases a model from memory. + * + * No confirmation, unlike deleting: nothing is removed from disk, and the + * worst case is that the next question pays to load it again. The toast reports + * what was freed, because that is the whole reason to do this. + */ + const unload = useCallback( + async (model: ModelInfo, loaded: RunningModel) => { + try { + await unloadModel({ host: preferences.host, apiKey: preferences.apiKey, model: model.name }); + } catch (thrown) { + await showToast({ + style: Toast.Style.Failure, + title: `Could not unload ${model.name}`, + message: errorMessage(thrown), + }); + return; + } + + const freed = formatBytes(loaded.sizeVram); + await showToast({ + style: Toast.Style.Success, + title: `Unloaded ${model.name}`, + message: freed ? `${freed} of memory freed` : undefined, + }); + + void load(); + }, + [load, preferences.host, preferences.apiKey], + ); + const choose = useCallback( async (name: string) => { if (name === preferences.model) { @@ -238,6 +292,7 @@ function ModelsScreen({ preferences, reloadPreferences }: ModelsScreenProps) { {loaded.map((model) => { const isCurrent = model.name === preferences.model; const size = model.remoteHost ? undefined : formatBytes(model.size); + const inMemory = running.get(model.name); return ( void choose(model.name)} /> + {inMemory ? ( + void unload(model, inMemory)} + /> + ) : null} {selfHosted ? (