feat: snip a region into chat, and make links clickable (#14)
This commit was merged in pull request #14.
This commit is contained in:
+73
-6
@@ -59,10 +59,36 @@ function normalizeToolCall(call: ToolCall): { name: string; arguments: Record<st
|
||||
return { name, arguments: args as Record<string, unknown> };
|
||||
}
|
||||
|
||||
/**
|
||||
* The last word, when the tool budget runs out.
|
||||
*
|
||||
* The loop used to end by throwing away every preamble the model had written and
|
||||
* returning a notice about itself, so a long research question produced no answer
|
||||
* at all. One closing round, with the tools taken away, turns that into an answer
|
||||
* built from whatever was already gathered, with the gaps named. The instruction
|
||||
* asks for the gaps because an answer that knows what it is missing is worth more
|
||||
* than a confident half of one.
|
||||
*/
|
||||
const CLOSING_INSTRUCTION =
|
||||
"You have reached the limit on tool calls for this turn, so no more tools are available. Answer now using what you have already gathered. If something is missing, say plainly which parts you could not confirm.";
|
||||
|
||||
/** Only reachable if the closing round comes back empty too. */
|
||||
function exhaustedNotice(rounds: number): string {
|
||||
return `_Ran out of tool calls after ${rounds} rounds and could not write an answer. Try asking something narrower._`;
|
||||
}
|
||||
|
||||
export async function runAgent(options: AgentOptions): Promise<AgentResult> {
|
||||
const conversation: ApiMessage[] = [...options.messages];
|
||||
const toolCalls: ToolCallRecord[] = [];
|
||||
let thinking = "";
|
||||
/**
|
||||
* The most recent preamble the model wrote.
|
||||
*
|
||||
* Not normally shown, because a preamble is what it says before finding out
|
||||
* anything. Kept anyway, so that a turn which runs out of budget still has
|
||||
* something to fall back on if the closing round produces nothing.
|
||||
*/
|
||||
let lastPreamble = "";
|
||||
|
||||
for (let iteration = 0; iteration < MAX_TOOL_ITERATIONS; iteration++) {
|
||||
let content = "";
|
||||
@@ -91,20 +117,25 @@ export async function runAgent(options: AgentOptions): Promise<AgentResult> {
|
||||
|
||||
thinking += streamedThinking;
|
||||
|
||||
// No tool calls means this was the model's answer. Earlier iterations only
|
||||
// produced a preamble, which is intentionally discarded rather than shown
|
||||
// as a half-answer.
|
||||
// No tool calls means this was the model's answer, rather than something it
|
||||
// said on the way to finding out.
|
||||
if (!pending || pending.length === 0) {
|
||||
return { content, thinking, toolCalls };
|
||||
}
|
||||
|
||||
if (content.trim()) lastPreamble = content;
|
||||
|
||||
conversation.push({ role: "assistant", content, tool_calls: pending });
|
||||
|
||||
for (const rawCall of pending) {
|
||||
const { name, arguments: args } = normalizeToolCall(rawCall);
|
||||
|
||||
if (!name) {
|
||||
conversation.push({ role: "tool", content: "Malformed tool call: missing function name.", tool_name: "unknown" });
|
||||
conversation.push({
|
||||
role: "tool",
|
||||
content: "Malformed tool call: missing function name.",
|
||||
tool_name: "unknown",
|
||||
});
|
||||
toolCalls.push({ name: "unknown", arguments: {}, result: "Malformed tool call.", isError: true });
|
||||
continue;
|
||||
}
|
||||
@@ -114,11 +145,17 @@ export async function runAgent(options: AgentOptions): Promise<AgentResult> {
|
||||
const result = await executeTool(name, args, {
|
||||
host: options.toolContext.host,
|
||||
apiKey: options.toolContext.apiKey,
|
||||
place: options.toolContext.place,
|
||||
now: options.toolContext.now,
|
||||
signal: options.signal,
|
||||
});
|
||||
|
||||
const record: ToolCallRecord = { name, arguments: args, result: result.content, isError: result.isError };
|
||||
const record: ToolCallRecord = {
|
||||
name,
|
||||
arguments: args,
|
||||
result: result.content,
|
||||
isError: result.isError,
|
||||
};
|
||||
toolCalls.push(record);
|
||||
options.events?.onToolCall?.(record);
|
||||
|
||||
@@ -126,8 +163,38 @@ export async function runAgent(options: AgentOptions): Promise<AgentResult> {
|
||||
}
|
||||
}
|
||||
|
||||
// Budget spent, so the turn gets one closing round with the tools taken away.
|
||||
// Ending on an answer drawn from what was gathered beats ending on a notice
|
||||
// about the loop itself, and the model is the only one that can say what it
|
||||
// did and did not manage to confirm.
|
||||
options.events?.onStatus?.("Writing the answer");
|
||||
|
||||
conversation.push({ role: "user", content: CLOSING_INSTRUCTION });
|
||||
|
||||
let closing = "";
|
||||
|
||||
for await (const chunk of streamChat({
|
||||
host: options.host,
|
||||
apiKey: options.apiKey,
|
||||
model: options.model,
|
||||
messages: conversation,
|
||||
// Deliberately no tools: the answer has to be written from what is already here.
|
||||
tools: undefined,
|
||||
think: options.think,
|
||||
signal: options.signal,
|
||||
})) {
|
||||
if (chunk.thinking) {
|
||||
thinking += chunk.thinking;
|
||||
options.events?.onThinkingDelta?.(chunk.thinking);
|
||||
}
|
||||
if (chunk.content) {
|
||||
closing += chunk.content;
|
||||
options.events?.onContentDelta?.(chunk.content);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
content: `_Stopped after ${MAX_TOOL_ITERATIONS} rounds of tool calls without a final answer._`,
|
||||
content: closing.trim() || lastPreamble.trim() || exhaustedNotice(MAX_TOOL_ITERATIONS),
|
||||
thinking,
|
||||
toolCalls,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { extname } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import type { Attachment, PendingAttachment } from "./types";
|
||||
|
||||
/**
|
||||
* Reading attachments off disk, and describing them to the renderer.
|
||||
*
|
||||
* Kept separate from `capture.ts` because the two answer different questions:
|
||||
* capture is about getting an image *onto* the disk, this is about getting it
|
||||
* back off again and telling Vicinae how to draw it.
|
||||
*
|
||||
* Imports no Vicinae module, so it unit tests with no runtime.
|
||||
*/
|
||||
|
||||
/** Extensions Ollama accepts. A snip is always PNG; the rest are here for a future picker. */
|
||||
const MIME_TYPES: Record<string, string> = {
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".webp": "image/webp",
|
||||
".gif": "image/gif",
|
||||
};
|
||||
|
||||
/**
|
||||
* The box a preview is fitted into.
|
||||
*
|
||||
* The height is not a preference: `MdImage.qml` clamps rendered height to 200px
|
||||
* whatever is asked for (`Math.min(_rawH, 200, maxImageHeight)`), so a taller
|
||||
* request is silently ignored. The width is here to keep a very wide capture
|
||||
* from being scaled up past the detail pane it sits in.
|
||||
*/
|
||||
const PREVIEW_MAX_WIDTH = 400;
|
||||
const PREVIEW_MAX_HEIGHT = 200;
|
||||
|
||||
export function mimeForPath(path: string): string {
|
||||
return MIME_TYPES[extname(path).toLowerCase()] ?? "application/octet-stream";
|
||||
}
|
||||
|
||||
/** The eight bytes every PNG starts with. */
|
||||
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
|
||||
|
||||
/**
|
||||
* Pixel dimensions from a PNG's `IHDR` chunk, or `undefined` for anything else.
|
||||
*
|
||||
* Only PNG is handled, because that is the only thing a snip produces and
|
||||
* guessing at other formats would be a lie. The header is 24 bytes, so this
|
||||
* costs nothing on top of the read that is already happening.
|
||||
*/
|
||||
export function pngSize(bytes: Buffer): { width: number; height: number } | undefined {
|
||||
if (bytes.length < 24) return undefined;
|
||||
if (!PNG_SIGNATURE.every((byte, index) => bytes[index] === byte)) return undefined;
|
||||
if (bytes.toString("ascii", 12, 16) !== "IHDR") return undefined;
|
||||
|
||||
const width = bytes.readUInt32BE(16);
|
||||
const height = bytes.readUInt32BE(20);
|
||||
|
||||
return width > 0 && height > 0 ? { width, height } : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* The exact box to ask the renderer for, or `undefined` when the size is unknown.
|
||||
*
|
||||
* This is the whole point of reading the header. With no hint, `MdImage.qml`
|
||||
* sizes its container to the pane width and 200px tall and then aspect-fits the
|
||||
* image inside it, so a wide screenshot sits in a letterbox with dead space
|
||||
* above and below it. Asking for a box that already matches the image's aspect
|
||||
* ratio leaves none.
|
||||
*
|
||||
* The scale is capped at 1 so a small capture is not blown up into a blurry one.
|
||||
*/
|
||||
export function previewBox(
|
||||
width: number | undefined,
|
||||
height: number | undefined,
|
||||
): { width: number; height: number } | undefined {
|
||||
if (!width || !height || width <= 0 || height <= 0) return undefined;
|
||||
|
||||
const scale = Math.min(1, PREVIEW_MAX_WIDTH / width, PREVIEW_MAX_HEIGHT / height);
|
||||
|
||||
return {
|
||||
width: Math.max(1, Math.round(width * scale)),
|
||||
height: Math.max(1, Math.round(height * scale)),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A markdown image source for a local file.
|
||||
*
|
||||
* A `file://` URL rather than a bare path, because Vicinae's markdown renderer
|
||||
* only resolves a scheme it recognises: `imageProviderUrl` in
|
||||
* `markdown-model.cpp` routes `file://`, and a scheme-less string, through
|
||||
* `ImageURL::local`, and anything it does not recognise it returns as empty.
|
||||
* `pathToFileURL` does the escaping, so a capture under a directory with a space
|
||||
* in its name still resolves.
|
||||
*
|
||||
* The query is read from the URL before it is resolved to a path, so the size
|
||||
* hints travel here without ever reaching the file name. They are spelled the
|
||||
* way Raycast spelled them, which is what the parser looks for.
|
||||
*/
|
||||
export function imageSource(attachment: Pick<Attachment, "path" | "width" | "height">): string {
|
||||
const url = pathToFileURL(attachment.path);
|
||||
const box = previewBox(attachment.width, attachment.height);
|
||||
|
||||
if (box) {
|
||||
url.searchParams.set("raycast-width", String(box.width));
|
||||
url.searchParams.set("raycast-height", String(box.height));
|
||||
}
|
||||
|
||||
return url.href;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads an image into the shape a request needs.
|
||||
*
|
||||
* Ollama wants base64 with no `data:` URL prefix, which is why the payload is
|
||||
* built here rather than handed straight to the renderer. The dimensions are
|
||||
* kept for the same reason the path is: the transcript needs them long after
|
||||
* this call is over, and the preview layout depends on them.
|
||||
*/
|
||||
export async function readAttachment(path: string): Promise<PendingAttachment> {
|
||||
const bytes = await readFile(path);
|
||||
const size = pngSize(bytes);
|
||||
|
||||
return {
|
||||
path,
|
||||
mime: mimeForPath(path),
|
||||
base64: bytes.toString("base64"),
|
||||
width: size?.width,
|
||||
height: size?.height,
|
||||
};
|
||||
}
|
||||
|
||||
/** The persistable half of a pending attachment: everything except the base64 payload. */
|
||||
export function toStoredAttachment(attachment: PendingAttachment): Attachment {
|
||||
const { path, mime, width, height } = attachment;
|
||||
return { path, mime, width, height };
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdir, stat, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
/**
|
||||
* Screen capture for "snip to chat".
|
||||
*
|
||||
* Vicinae has no screenshot API. There is no capture call anywhere in
|
||||
* `@vicinae/api`, and `Clipboard` moves text, html, urls and a file path and
|
||||
* nothing binary, so a bitmap can never be read back *out* of the clipboard
|
||||
* either. The only way to get a region of the screen is to shell out.
|
||||
*
|
||||
* That is acceptable here because Vicinae extensions run under Node ("all
|
||||
* standard Node.js APIs are available"), which is why `node:child_process` is
|
||||
* not a workaround so much as the supported route. The important consequence is
|
||||
* that the capture is *owned* rather than borrowed: a tool writes a PNG to a
|
||||
* path this module picks, and that path is what the chat stores. Nothing has to
|
||||
* guess what some other application left on the clipboard.
|
||||
*
|
||||
* Every function here takes its environment as an argument and imports no
|
||||
* Vicinae module, so the selection logic unit tests with no runtime and no
|
||||
* installed tools.
|
||||
*/
|
||||
|
||||
/** A PNG piped through stdout (flameshot) is easily several megabytes. */
|
||||
const MAX_CAPTURE_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
/** Long enough to pick a region unhurried, short enough that a wedged tool cannot hang the view forever. */
|
||||
const CAPTURE_TIMEOUT_MS = 5 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* What macOS says when Screen Recording permission is missing.
|
||||
*
|
||||
* This one pattern is why the module reads stderr at all. The message is the
|
||||
* *only* difference between "the system refused to let us look" and "the user
|
||||
* pressed Escape": both produce no file, and `screencapture -i` has been
|
||||
* reported to exit `0` in the refusal case. Treating the refusal as a cancel
|
||||
* leaves the user with a feature that silently does nothing.
|
||||
*/
|
||||
const SCREEN_RECORDING_DENIED = /could not create image from (rect|display|window)/i;
|
||||
|
||||
export interface RunResult {
|
||||
stdout: Buffer;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
/** Runs a command. Injected so tests never spawn anything. */
|
||||
export type RunCommand = (
|
||||
tool: string,
|
||||
args: string[],
|
||||
options?: { signal?: AbortSignal },
|
||||
) => Promise<RunResult>;
|
||||
|
||||
export interface CaptureBackend {
|
||||
/** Shown to the user when a capture fails, so they know what to install. */
|
||||
name: string;
|
||||
/** Every one of these has to resolve on `PATH` for the backend to be usable. */
|
||||
requires: string[];
|
||||
/** Produces the image at `output`, or resolves without one when the user cancels. */
|
||||
capture: (context: CaptureContext) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface CaptureContext {
|
||||
output: string;
|
||||
signal?: AbortSignal;
|
||||
run: RunCommand;
|
||||
}
|
||||
|
||||
export interface CaptureResult {
|
||||
/** Absolute path of the image that was written. */
|
||||
path: string;
|
||||
bytes: number;
|
||||
/** The tool that produced it, for the transcript and for error messages. */
|
||||
backend: string;
|
||||
}
|
||||
|
||||
export interface CaptureOptions {
|
||||
/** Where images are kept. The caller supplies `environment.supportPath`, which keeps this module Vicinae-free. */
|
||||
outputDir: string;
|
||||
platform?: NodeJS.Platform;
|
||||
signal?: AbortSignal;
|
||||
now?: Date;
|
||||
/** Overrides for tests. */
|
||||
hasCommand?: (tool: string) => Promise<boolean>;
|
||||
run?: RunCommand;
|
||||
}
|
||||
|
||||
/** Raised when the machine has no capture tool at all, with something actionable to do about it. */
|
||||
export class NoCaptureToolError extends Error {
|
||||
readonly hint: string;
|
||||
|
||||
constructor(platform: NodeJS.Platform) {
|
||||
super("No screen capture tool is installed.");
|
||||
this.name = "NoCaptureToolError";
|
||||
this.hint =
|
||||
platform === "darwin"
|
||||
? "macOS ships screencapture, so this should not happen. Check that it is on PATH."
|
||||
: "Install one of: grim + slurp (Wayland), grimblast (Hyprland), spectacle, maim + slop (X11), scrot, or flameshot.";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Raised when the system refuses to hand over the screen.
|
||||
*
|
||||
* Separate from a generic tool failure because it is the one capture error the
|
||||
* user can actually fix, and because on macOS the fix is not obvious: the
|
||||
* permission belongs to the *launcher*, since TCC attributes the request to the
|
||||
* app responsible for spawning the extension runtime rather than to the
|
||||
* extension itself.
|
||||
*/
|
||||
export class CapturePermissionError extends Error {
|
||||
readonly hint: string;
|
||||
|
||||
constructor(platform: NodeJS.Platform) {
|
||||
super("The system denied screen capture.");
|
||||
this.name = "CapturePermissionError";
|
||||
this.hint =
|
||||
platform === "darwin"
|
||||
? "Give Vicinae Screen Recording permission in System Settings under Privacy & Security, then quit Vicinae and open it again. The permission belongs to Vicinae rather than to the extension, because macOS attributes the request to the app that launched the extension runtime."
|
||||
: "Give the launcher permission to record the screen, then restart it.";
|
||||
}
|
||||
}
|
||||
|
||||
/** A capture tool that ran and failed for a reason it could explain. */
|
||||
export class CaptureToolError extends Error {
|
||||
readonly tool: string;
|
||||
readonly stderr: string;
|
||||
|
||||
constructor(tool: string, stderr: string, cause?: unknown) {
|
||||
const detail = stderr.trim().split("\n")[0] ?? "";
|
||||
super(`${tool} failed.${detail ? ` ${detail}` : ""}`, cause === undefined ? undefined : { cause });
|
||||
this.name = "CaptureToolError";
|
||||
this.tool = tool;
|
||||
this.stderr = stderr;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* `command -v` rather than spawning the tool itself: several of these block or
|
||||
* open a window when run bare, and `--version` is not reliably supported.
|
||||
*
|
||||
* The tool names are literals from the table below and never come from the
|
||||
* user, so there is nothing to escape.
|
||||
*/
|
||||
async function commandExists(tool: string): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
execFile("sh", ["-c", `command -v ${tool}`], { timeout: 5_000 }, (error) => resolve(!error));
|
||||
});
|
||||
}
|
||||
|
||||
function runCommand(
|
||||
tool: string,
|
||||
args: string[],
|
||||
options: { signal?: AbortSignal } = {},
|
||||
): Promise<RunResult> {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile(
|
||||
tool,
|
||||
args,
|
||||
{
|
||||
signal: options.signal,
|
||||
timeout: CAPTURE_TIMEOUT_MS,
|
||||
maxBuffer: MAX_CAPTURE_BYTES,
|
||||
encoding: "buffer",
|
||||
},
|
||||
(error, stdout, stderr) => {
|
||||
const text = stderr?.toString("utf8") ?? "";
|
||||
|
||||
if (error) {
|
||||
reject(new CaptureToolError(tool, text, error));
|
||||
return;
|
||||
}
|
||||
|
||||
resolve({ stdout: stdout as Buffer, stderr: text });
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* `grim -g "$(slurp)"`, expressed as two argv calls rather than a shell pipeline.
|
||||
*
|
||||
* slurp prints the chosen geometry and exits non-zero when the selection is
|
||||
* cancelled, so an empty or absent geometry is the cancel signal.
|
||||
*/
|
||||
const grimSlurp: CaptureBackend = {
|
||||
name: "grim + slurp",
|
||||
requires: ["grim", "slurp"],
|
||||
async capture({ output, signal, run }) {
|
||||
const geometry = (await run("slurp", [], { signal })).stdout.toString("utf8").trim();
|
||||
if (!geometry) return;
|
||||
await run("grim", ["-g", geometry, output], { signal });
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* flameshot is the only backend that will not take an output path: `gui --raw`
|
||||
* writes the PNG to stdout, so it is captured and written here instead.
|
||||
*/
|
||||
const flameshot: CaptureBackend = {
|
||||
name: "flameshot",
|
||||
requires: ["flameshot"],
|
||||
async capture({ output, signal, run }) {
|
||||
const png = (await run("flameshot", ["gui", "--raw"], { signal })).stdout;
|
||||
if (png.length === 0) return;
|
||||
await writeFile(output, png);
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Backends in preference order.
|
||||
*
|
||||
* Wayland first on Linux, because that is what the launcher's own users are on
|
||||
* (the README's shortcut examples are Hyprland and niri), then the X11 tools.
|
||||
* macOS needs only the built-in tool, which is why it is tried on its own.
|
||||
*/
|
||||
export const CAPTURE_BACKENDS: CaptureBackend[] = [
|
||||
{
|
||||
name: "grimblast",
|
||||
requires: ["grimblast"],
|
||||
async capture({ output, signal, run }) {
|
||||
await run("grimblast", ["save", "area", output], { signal });
|
||||
},
|
||||
},
|
||||
grimSlurp,
|
||||
{
|
||||
name: "spectacle",
|
||||
requires: ["spectacle"],
|
||||
async capture({ output, signal, run }) {
|
||||
// -r region, -b background, -n no notification, -o output file.
|
||||
await run("spectacle", ["-r", "-b", "-n", "-o", output], { signal });
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "maim",
|
||||
requires: ["maim"],
|
||||
async capture({ output, signal, run }) {
|
||||
await run("maim", ["-s", output], { signal });
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "scrot",
|
||||
requires: ["scrot"],
|
||||
async capture({ output, signal, run }) {
|
||||
await run("scrot", ["-s", output], { signal });
|
||||
},
|
||||
},
|
||||
flameshot,
|
||||
{
|
||||
name: "screencapture",
|
||||
requires: ["screencapture"],
|
||||
async capture({ output, signal, run }) {
|
||||
// -i interactive region, -x no shutter sound.
|
||||
await run("screencapture", ["-i", "-x", output], { signal });
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
* The first backend whose tools are all installed, or `undefined`.
|
||||
*
|
||||
* `hasCommand` is a parameter rather than a call so the preference order can be
|
||||
* tested against a fake `PATH`, which is the part most likely to break on a
|
||||
* platform nobody here runs.
|
||||
*/
|
||||
export async function selectBackend(
|
||||
hasCommand: (tool: string) => Promise<boolean>,
|
||||
backends: CaptureBackend[] = CAPTURE_BACKENDS,
|
||||
): Promise<CaptureBackend | undefined> {
|
||||
for (const backend of backends) {
|
||||
const checks = await Promise.all(backend.requires.map((tool) => hasCommand(tool)));
|
||||
if (checks.every(Boolean)) return backend;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Sortable, filesystem-safe, and obvious when found lying in the support directory. */
|
||||
export function snipFileName(now: Date): string {
|
||||
const pad = (value: number) => String(value).padStart(2, "0");
|
||||
|
||||
const date = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}`;
|
||||
const time = `${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
|
||||
|
||||
return `snip-${date}-${time}.png`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Captures a region and returns the file it wrote, or `null` when the user
|
||||
* backed out.
|
||||
*
|
||||
* Success is decided by the file, not by an exit code: the tools disagree about
|
||||
* what a cancelled selection looks like, and at least one reports success
|
||||
* anyway. Failure is then triaged from what the tool said on stderr, which is
|
||||
* the only channel that separates a refused permission from a cancel.
|
||||
*/
|
||||
export async function captureRegion(options: CaptureOptions): Promise<CaptureResult | null> {
|
||||
const platform = options.platform ?? process.platform;
|
||||
const hasCommand = options.hasCommand ?? commandExists;
|
||||
const baseRun = options.run ?? runCommand;
|
||||
|
||||
const backend = await selectBackend(
|
||||
hasCommand,
|
||||
platform === "darwin" ? CAPTURE_BACKENDS.filter((b) => b.name === "screencapture") : CAPTURE_BACKENDS,
|
||||
);
|
||||
if (!backend) throw new NoCaptureToolError(platform);
|
||||
|
||||
await mkdir(options.outputDir, { recursive: true });
|
||||
const output = join(options.outputDir, snipFileName(options.now ?? new Date()));
|
||||
|
||||
/** Everything the tools said, whether or not they exited cleanly. */
|
||||
const diagnostics: string[] = [];
|
||||
const run: RunCommand = async (tool, args, runOptions) => {
|
||||
try {
|
||||
const result = await baseRun(tool, args, runOptions);
|
||||
if (result.stderr.trim()) diagnostics.push(result.stderr.trim());
|
||||
return result;
|
||||
} catch (thrown) {
|
||||
const stderr = thrown instanceof CaptureToolError ? thrown.stderr : "";
|
||||
if (stderr.trim()) diagnostics.push(stderr.trim());
|
||||
throw thrown;
|
||||
}
|
||||
};
|
||||
|
||||
let failure: unknown;
|
||||
try {
|
||||
await backend.capture({ output, signal: options.signal, run });
|
||||
} catch (thrown) {
|
||||
failure = thrown;
|
||||
}
|
||||
|
||||
const written = await stat(output).catch(() => null);
|
||||
if (written && written.size > 0) {
|
||||
return { path: output, bytes: written.size, backend: backend.name };
|
||||
}
|
||||
|
||||
const detail = diagnostics.join("\n");
|
||||
|
||||
if (SCREEN_RECORDING_DENIED.test(detail)) throw new CapturePermissionError(platform);
|
||||
|
||||
// A tool that explained itself is a failure worth reporting. Silence with no
|
||||
// image is the user pressing Escape, which is the common case and says nothing.
|
||||
if (detail) throw new CaptureToolError(backend.requires[0], detail, failure);
|
||||
|
||||
return null;
|
||||
}
|
||||
+190
-3
@@ -88,6 +88,35 @@ const OPENING_FENCE = /^(\s*)(`{3,}|~{3,})\s*([^\s`~]*)\s*$/;
|
||||
const CLOSING_FENCE = /^\s*(`{3,}|~{3,})\s*$/;
|
||||
const STANDALONE_CODE_SPAN = /^\s*`([^`\n]+)`\s*[.!]?\s*$/;
|
||||
|
||||
/**
|
||||
* A line that is nothing but a bare http(s) URL, give or take a bullet.
|
||||
*
|
||||
* Vicinae only makes a URL clickable when markdown presents it as a link. Its
|
||||
* parser attaches the `table` and `strikethrough` GFM extensions and nothing
|
||||
* else, so the `autolink` extension is off: a bare URL renders as plain dead
|
||||
* text. `MarkdownModel` turns `CMARK_NODE_LINK` into `<a href=...>`, and the
|
||||
* selection controller opens that anchor on release, so `[label](url)` is the
|
||||
* one form the user can actually click.
|
||||
*/
|
||||
const BARE_URL = /^https?:\/\/\S+$/;
|
||||
/** Sentence punctuation a model appends to a URL that is not part of it. */
|
||||
const TRAILING_SENTENCE_PUNCTUATION = /[.,;:!?]+$/;
|
||||
/** One inline code span that holds nothing but a URL. */
|
||||
const STANDALONE_URL_SPAN = /^`([^`\n]+)`[.!]?$/;
|
||||
/** A bullet or numbered-list marker, kept so a list of links stays a list. */
|
||||
const LIST_MARKER = /^([ \t]*(?:[-*+]|\d+[.)])[ \t]+)(.*)$/;
|
||||
|
||||
/** A GFM table row: it opens and closes with a pipe. */
|
||||
const TABLE_ROW = /^\s*\|.*\|\s*$/;
|
||||
/**
|
||||
* The delimiter row under a header.
|
||||
*
|
||||
* The test is not how many dashes there are, because GFM wants only one per
|
||||
* column. It is that the line could be nothing else: pipes, dashes, colons and
|
||||
* spaces, with at least one dash to tell it apart from a row of empty cells.
|
||||
*/
|
||||
const TABLE_DELIMITER_ROW = /^\s*\|?[\s:|-]*-[\s:|-]*\|?\s*$/;
|
||||
|
||||
function significantLines(code: string): string[] {
|
||||
return code
|
||||
.split("\n")
|
||||
@@ -272,12 +301,170 @@ export function labelCodeFences(markdown: string): string {
|
||||
return out.join("\n");
|
||||
}
|
||||
|
||||
/** A URL with the sentence punctuation after it removed. */
|
||||
function asBareUrl(candidate: string): string | undefined {
|
||||
const trimmed = candidate.trim();
|
||||
if (!BARE_URL.test(trimmed)) return undefined;
|
||||
return trimmed.replace(TRAILING_SENTENCE_PUNCTUATION, "");
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a URL as a markdown link, using the address itself as the label so the
|
||||
* destination stays readable rather than turning into a generic word.
|
||||
*
|
||||
* A destination containing parentheses (plenty of Wikipedia articles) has to
|
||||
* be wrapped in pointy brackets or the parser reads the first `)` as the end of
|
||||
* the destination and leaves the rest as trailing prose.
|
||||
*/
|
||||
function asLink(url: string): string {
|
||||
// Square brackets in the *label* would end the link text early, which is why
|
||||
// an IPv6 literal such as `http://[::1]/` needs them escaped. The destination
|
||||
// tolerates brackets, but not parentheses, which is what the pointy form is
|
||||
// for.
|
||||
const label = url.replace(/[[\]]/g, "\\$&");
|
||||
const destination = /[()]/.test(url) ? `<${url}>` : url;
|
||||
return `[${label}](${destination})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The URLs of a fence whose every non-empty line is one, or `undefined` when
|
||||
* the block is real code.
|
||||
*
|
||||
* Deliberately narrow: one line that is not a URL disqualifies the whole block,
|
||||
* so a `curl` invocation that happens to contain an address is never touched.
|
||||
*/
|
||||
function urlOnlyBlock(lines: string[]): string[] | undefined {
|
||||
const urls: string[] = [];
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length === 0) continue;
|
||||
|
||||
const url = asBareUrl(trimmed);
|
||||
if (!url) return undefined;
|
||||
urls.push(url);
|
||||
}
|
||||
|
||||
return urls.length > 0 ? urls : undefined;
|
||||
}
|
||||
|
||||
/** Rewrites a line that is only a URL, or only a span holding one, into a link. */
|
||||
function linkifyLine(line: string): string {
|
||||
const marker = LIST_MARKER.exec(line);
|
||||
const prefix = marker ? marker[1] : (line.match(/^[ \t]*/)?.[0] ?? "");
|
||||
const rest = marker ? marker[2] : line.slice(prefix.length);
|
||||
|
||||
const span = STANDALONE_URL_SPAN.exec(rest.trim());
|
||||
const url = asBareUrl(span ? span[1] : rest);
|
||||
|
||||
return url ? `${prefix}${asLink(url)}` : line;
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns URLs the model wrote as code back into links.
|
||||
*
|
||||
* This is the reported bug: asked for a few GitHub projects, the model answers
|
||||
* with a fenced block of bare addresses. A fence renders through
|
||||
* `MdCodeBlock.qml` as a dark panel with a Copy button and no link handling at
|
||||
* all, so the one thing the user wants to do with a URL is the one thing they
|
||||
* cannot. The same applies to a lone `` `https://...` `` span.
|
||||
*
|
||||
* Three shapes are recovered, and only these:
|
||||
*
|
||||
* 1. A fence whose entire body is URLs.
|
||||
* 2. A line that is exactly one inline span holding a URL.
|
||||
* 3. A line that is exactly a URL, bulleted or not.
|
||||
*
|
||||
* Existing fences holding anything else are copied through untouched, so real
|
||||
* code is never rewritten.
|
||||
*/
|
||||
export function restoreUrlLinks(markdown: string): string {
|
||||
const lines = markdown.split("\n");
|
||||
const out: string[] = [];
|
||||
let index = 0;
|
||||
|
||||
while (index < lines.length) {
|
||||
if (!OPENING_FENCE.test(lines[index])) {
|
||||
out.push(linkifyLine(lines[index]));
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const body: string[] = [];
|
||||
let cursor = index + 1;
|
||||
while (cursor < lines.length && !CLOSING_FENCE.test(lines[cursor])) {
|
||||
body.push(lines[cursor]);
|
||||
cursor += 1;
|
||||
}
|
||||
|
||||
const closing = cursor < lines.length;
|
||||
const urls = urlOnlyBlock(body);
|
||||
|
||||
if (urls) out.push(...urls.map(asLink));
|
||||
else {
|
||||
out.push(lines[index], ...body);
|
||||
if (closing) out.push(lines[cursor]);
|
||||
}
|
||||
|
||||
index = cursor + (closing ? 1 : 0);
|
||||
}
|
||||
|
||||
return out.join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts the blank line a table needs to start at all.
|
||||
*
|
||||
* A GFM table cannot interrupt a paragraph, so a table written straight after a
|
||||
* line of prose is not a table: it renders as a paragraph of pipes and dashes,
|
||||
* which is how a perfectly good answer turns into noise. The blank line looks
|
||||
* unnecessary, so models leave it out.
|
||||
*
|
||||
* Only a real table start is touched, which is a row followed by a delimiter
|
||||
* row, so prose that happens to contain a pipe is left alone. Fences are copied
|
||||
* through untouched for the same reason, since a table shown as example code
|
||||
* must stay exactly as written.
|
||||
*/
|
||||
export function separateTables(markdown: string): string {
|
||||
const lines = markdown.split("\n");
|
||||
const out: string[] = [];
|
||||
let index = 0;
|
||||
|
||||
while (index < lines.length) {
|
||||
if (OPENING_FENCE.test(lines[index])) {
|
||||
index = copyFence(lines, out, index);
|
||||
continue;
|
||||
}
|
||||
|
||||
const startsTable =
|
||||
TABLE_ROW.test(lines[index]) && index + 1 < lines.length && TABLE_DELIMITER_ROW.test(lines[index + 1]);
|
||||
|
||||
// Only when something would otherwise swallow it. A table already at a block
|
||||
// boundary, after a blank line or after a fenced block, needs nothing: the
|
||||
// fence has already ended the preceding block.
|
||||
const previous = out[out.length - 1];
|
||||
const atBlockBoundary =
|
||||
previous === undefined || previous.trim().length === 0 || CLOSING_FENCE.test(previous);
|
||||
|
||||
if (startsTable && !atBlockBoundary) out.push("");
|
||||
|
||||
out.push(lines[index]);
|
||||
index += 1;
|
||||
}
|
||||
|
||||
return out.join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* The full pre-render pass, in order.
|
||||
*
|
||||
* Promotion runs first so a promoted block is seen as an ordinary fence by the
|
||||
* labeller, which is also harmless when promotion already inferred a language.
|
||||
* Tables are separated first because they change block boundaries: a blank line
|
||||
* has to exist before anything else looks at the line. Links are restored next
|
||||
* so a recovered URL leaves no span for the promoter to fence again. Promotion
|
||||
* then runs before labelling, so a promoted block is seen as an ordinary fence
|
||||
* by the labeller, which is also harmless when promotion already inferred a
|
||||
* language.
|
||||
*/
|
||||
export function repairCodeBlocks(markdown: string): string {
|
||||
return labelCodeFences(promoteStandaloneCodeSpans(markdown));
|
||||
return labelCodeFences(promoteStandaloneCodeSpans(restoreUrlLinks(separateTables(markdown))));
|
||||
}
|
||||
|
||||
+151
-38
@@ -1,10 +1,14 @@
|
||||
import { Action, ActionPanel, Icon, List } from "@vicinae/api";
|
||||
import { Action, ActionPanel, Icon, LaunchType, List, launchCommand } from "@vicinae/api";
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { readAttachment } from "../attachments";
|
||||
import { useChat } from "../hooks/useChat";
|
||||
import { buildThreadMarkdown, type TranscriptOptions } from "../markdown";
|
||||
import { useVisionSupport } from "../hooks/useVision";
|
||||
import { buildThreadMarkdown, renderAttachments, type TranscriptOptions } from "../markdown";
|
||||
import { getLamPreferences } from "../preferences";
|
||||
import { deleteSession, listSessions, loadSession, summarizeSession } from "../sessions";
|
||||
import { buildThread } from "../thread";import type { Session, SessionSummary } from "../types";
|
||||
import { resolveStartup } from "../startup";
|
||||
import { buildThread } from "../thread";
|
||||
import type { PendingAttachment, Session, SessionSummary } from "../types";
|
||||
import { recencyGroup, truncate } from "../utils";
|
||||
|
||||
/**
|
||||
@@ -44,15 +48,30 @@ import { recencyGroup, truncate } from "../utils";
|
||||
* selection to the first row on every model change
|
||||
* (`GenericListView.selectFirstOnReset`), and streaming changes the model on
|
||||
* every flush, so ordering by recency keeps the chat in progress near the top.
|
||||
* - Capturing a region does **not** happen here. It is delegated to the `snip`
|
||||
* command, because this view owns a window and the window would be in the
|
||||
* shot. See {@link ChatViewProps}.
|
||||
*/
|
||||
|
||||
const EMPTY_TRANSCRIPT = "**Ask Lam anything**\n\nType a question in the search bar and press Return.";
|
||||
|
||||
/** Sent when a snip is attached and Return is pressed before anything is typed. */
|
||||
const DEFAULT_IMAGE_PROMPT = "What is in this image?";
|
||||
|
||||
/** The reminder shown while a snip is attached but not yet sent. */
|
||||
const PENDING_NOTE = "_Ask a question about it, or press Return to send it as it is._";
|
||||
|
||||
export interface ChatViewProps {
|
||||
/** Opens a specific conversation instead of resuming the latest one. */
|
||||
initialSession?: Session;
|
||||
/** Text handed over by a deeplink or fallback command. */
|
||||
initialPrompt?: string;
|
||||
/** A conversation to reload by id, which is how the snip command hands one back. */
|
||||
initialSessionId?: string;
|
||||
/** A captured image to read in, which is how the snip command hands one over. */
|
||||
initialAttachmentPath?: string;
|
||||
/** Something the launching command needs to say, such as a failed capture. */
|
||||
launchNote?: string;
|
||||
}
|
||||
|
||||
interface SessionRow {
|
||||
@@ -62,14 +81,26 @@ interface SessionRow {
|
||||
isActive: boolean;
|
||||
}
|
||||
|
||||
export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
|
||||
export function ChatView({
|
||||
initialSession,
|
||||
initialPrompt,
|
||||
initialSessionId,
|
||||
initialAttachmentPath,
|
||||
launchNote,
|
||||
}: ChatViewProps) {
|
||||
const preferences = useMemo(() => getLamPreferences(), []);
|
||||
const chat = useChat({ preferences, initialSession });
|
||||
const { replaceSession } = chat;
|
||||
const { canSeeImages } = useVisionSupport(preferences);
|
||||
|
||||
const [input, setInput] = useState(initialPrompt ?? "");
|
||||
const [summaries, setSummaries] = useState<SessionSummary[] | undefined>(undefined);
|
||||
const [resumed, setResumed] = useState(Boolean(initialSession));
|
||||
// One flag, not two. See the startup effect below for why that matters.
|
||||
const [startup, setStartup] = useState<"loading" | "ready">(initialSession ? "ready" : "loading");
|
||||
// A captured image is held here until the user says what they want to know
|
||||
// about it. Nothing is sent to the model on capture alone.
|
||||
const [pending, setPending] = useState<PendingAttachment | undefined>(undefined);
|
||||
const [snipNote, setSnipNote] = useState<string | undefined>(undefined);
|
||||
|
||||
const activeId = chat.session.id;
|
||||
|
||||
@@ -85,35 +116,47 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
|
||||
}, [reload, chat.session.updatedAt]);
|
||||
|
||||
/**
|
||||
* Reopening the command continues the most recent conversation, but only if it
|
||||
* was used recently. Past the configured idle window it starts a fresh chat
|
||||
* instead. The previous conversation is never removed, it just drops down the
|
||||
* sidebar. Setting the preference to "Never" keeps the last chat open forever.
|
||||
* Everything that has to happen before the first render, in one place.
|
||||
*
|
||||
* This used to be two effects guarding two booleans, and a hand-off launch
|
||||
* satisfied the guard of *both* while setting neither, so `resumed` stayed
|
||||
* false and the view sat on "Loading conversations…" forever. Anything that
|
||||
* blocks the first render belongs in one effect with one flag, and the flag is
|
||||
* cleared in a `finally` so that no branch, including a thrown one, can leave
|
||||
* the view spinning.
|
||||
*/
|
||||
useEffect(() => {
|
||||
if (initialSession) return;
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const [latest] = await listSessions();
|
||||
if (!latest) return;
|
||||
const outcome = await resolveStartup(
|
||||
{
|
||||
hasInitialSession: Boolean(initialSession),
|
||||
initialSessionId,
|
||||
initialAttachmentPath,
|
||||
resumeTimeoutMs: preferences.resumeTimeoutMs,
|
||||
now: Date.now(),
|
||||
},
|
||||
{ loadSession, listSessions, readAttachment },
|
||||
);
|
||||
|
||||
const idleMs = Date.now() - latest.updatedAt;
|
||||
const { resumeTimeoutMs } = preferences;
|
||||
if (resumeTimeoutMs !== undefined && idleMs > resumeTimeoutMs) return;
|
||||
if (cancelled) return;
|
||||
|
||||
const session = await loadSession(latest.id);
|
||||
if (!cancelled && session) replaceSession(session);
|
||||
if (outcome.session) replaceSession(outcome.session);
|
||||
if (outcome.attachment) setPending(outcome.attachment);
|
||||
if (outcome.attachmentError) {
|
||||
setSnipNote(`**Could not open the screenshot:** ${outcome.attachmentError}`);
|
||||
}
|
||||
} finally {
|
||||
if (!cancelled) setResumed(true);
|
||||
if (!cancelled) setStartup("ready");
|
||||
}
|
||||
})();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [initialSession, preferences, replaceSession]);
|
||||
}, [initialSession, initialSessionId, initialAttachmentPath, preferences.resumeTimeoutMs, replaceSession]);
|
||||
|
||||
const thread = useMemo(
|
||||
() =>
|
||||
@@ -131,11 +174,31 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
|
||||
[preferences.showToolCalls, preferences.showReasoning],
|
||||
);
|
||||
|
||||
const threadMarkdown = useMemo(
|
||||
() =>
|
||||
thread.length === 0 ? EMPTY_TRANSCRIPT : buildThreadMarkdown(thread, transcriptOptions),
|
||||
[thread, transcriptOptions],
|
||||
);
|
||||
/**
|
||||
* The transcript, preceded by any notes about this view's own state.
|
||||
*
|
||||
* Notes go first because the transcript reads newest-first: the top of the
|
||||
* panel is the text directly under the search bar, which is where the user is
|
||||
* already looking.
|
||||
*/
|
||||
const threadMarkdown = useMemo(() => {
|
||||
const notes = [
|
||||
launchNote,
|
||||
snipNote,
|
||||
// The preview comes first so the captured image is visible before the user
|
||||
// decides what to ask about it, rather than only after sending.
|
||||
pending ? `${renderAttachments([pending])}\n\n${PENDING_NOTE}` : undefined,
|
||||
].filter((note): note is string => Boolean(note));
|
||||
|
||||
// The placeholder is what tells a user that the search bar is the composer.
|
||||
// When a note above already says that, printing both put the same
|
||||
// instruction twice, one line under the other.
|
||||
const empty = notes.length === 0 ? EMPTY_TRANSCRIPT : "";
|
||||
const transcript =
|
||||
thread.length === 0 ? empty : buildThreadMarkdown(thread, transcriptOptions);
|
||||
|
||||
return [...notes, transcript].filter((part) => part.length > 0).join("\n\n");
|
||||
}, [launchNote, snipNote, pending, thread, transcriptOptions]);
|
||||
|
||||
const rows: SessionRow[] = useMemo(() => {
|
||||
const base = summaries ?? [];
|
||||
@@ -151,10 +214,7 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
|
||||
if (base.some((summary) => summary.id === activeId)) return mapped;
|
||||
|
||||
const active = summarizeSession(chat.session);
|
||||
return [
|
||||
{ id: active.id, title: active.title, updatedAt: active.updatedAt, isActive: true },
|
||||
...mapped,
|
||||
];
|
||||
return [{ id: active.id, title: active.title, updatedAt: active.updatedAt, isActive: true }, ...mapped];
|
||||
}, [summaries, activeId, chat.session]);
|
||||
|
||||
/**
|
||||
@@ -194,15 +254,28 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
|
||||
[activeId, chat, reload],
|
||||
);
|
||||
|
||||
/** The snip a "show me the screenshot" action would open: the pending one, else the newest. */
|
||||
const latestAttachment =
|
||||
pending ?? [...chat.session.turns].reverse().find((turn) => turn.attachments.length > 0)?.attachments[0];
|
||||
|
||||
const send = useCallback(
|
||||
(text: string) => {
|
||||
chat.ask(text);
|
||||
const typed = text.trim();
|
||||
// With a snip attached, Return has to send even before anything is typed,
|
||||
// otherwise the only way to ask about an image is to type out a question
|
||||
// the image already implies.
|
||||
const prompt = typed || (pending ? DEFAULT_IMAGE_PROMPT : "");
|
||||
if (!prompt) return;
|
||||
|
||||
chat.ask(prompt, pending);
|
||||
setPending(undefined);
|
||||
setSnipNote(undefined);
|
||||
setInput("");
|
||||
},
|
||||
[chat.ask],
|
||||
[chat.ask, pending],
|
||||
);
|
||||
|
||||
const hasPrompt = input.trim().length > 0;
|
||||
const hasPrompt = input.trim().length > 0 || pending !== undefined;
|
||||
|
||||
const stopAction = <Action title="Stop Generating" icon={Icon.Stop} onAction={chat.cancel} />;
|
||||
const sendAction = <Action title="Send" icon={Icon.SpeechBubble} onAction={() => send(input)} />;
|
||||
@@ -216,10 +289,46 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
|
||||
/>
|
||||
);
|
||||
|
||||
/**
|
||||
* Hands the capture to the `snip` command instead of doing it here.
|
||||
*
|
||||
* Two reasons, both structural. The window has to be out of the way, and only
|
||||
* the snip command is positioned to close it before the selector appears. And
|
||||
* launching a different command is what makes the hand-off safe: Vicinae's
|
||||
* `activateEntrypoint` treats a launch of the *currently active* view command
|
||||
* as a toggle and closes the window instead of reopening it, so relaying
|
||||
* through `snip` avoids ever asking for `chat` while `chat` is active.
|
||||
*
|
||||
* Absent while streaming, because the hand-off unloads this view and would
|
||||
* abandon an answer mid-sentence.
|
||||
*/
|
||||
const snipAction =
|
||||
canSeeImages && !chat.isStreaming ? (
|
||||
<Action
|
||||
title="Snip a Region"
|
||||
icon={Icon.Screenshot}
|
||||
shortcut={{ key: "s", modifiers: ["cmd", "shift"] }}
|
||||
onAction={() =>
|
||||
void launchCommand({
|
||||
name: "snip",
|
||||
type: LaunchType.UserInitiated,
|
||||
context: { sessionId: chat.session.id },
|
||||
})
|
||||
}
|
||||
/>
|
||||
) : null;
|
||||
|
||||
// A screenshot is a file, so it opens in the system image viewer. Without this
|
||||
// the only way to see what was sent would be to find it on disk by hand.
|
||||
const openSnipAction = latestAttachment ? (
|
||||
<Action.Open title="Open Screenshot" icon={Icon.Image} target={latestAttachment.path} />
|
||||
) : null;
|
||||
|
||||
/**
|
||||
* The primary action adapts to the search bar. With text in it, Return sends;
|
||||
* with it empty (where sending would be a no-op anyway), Return opens the
|
||||
* highlighted conversation instead.
|
||||
* highlighted conversation instead. A pending snip counts as text, so Return
|
||||
* sends it rather than navigating away.
|
||||
*/
|
||||
const rowActions = (row: SessionRow) => {
|
||||
const openAction = (
|
||||
@@ -231,6 +340,8 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
|
||||
{chat.isStreaming ? stopAction : hasPrompt ? sendAction : openAction}
|
||||
{hasPrompt ? openAction : null}
|
||||
{newChatAction}
|
||||
{snipAction}
|
||||
{openSnipAction}
|
||||
<Action
|
||||
title="Delete Conversation"
|
||||
icon={Icon.Trash}
|
||||
@@ -244,13 +355,13 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
|
||||
);
|
||||
};
|
||||
|
||||
if (!resumed || summaries === undefined) {
|
||||
if (startup === "loading" || summaries === undefined) {
|
||||
return <List isLoading navigationTitle="Lam" searchBarPlaceholder="Loading conversations…" />;
|
||||
}
|
||||
|
||||
const placeholder = chat.status
|
||||
? `${chat.status}. Press ⏎ to stop`
|
||||
: "Lam is thinking. Press ⏎ to stop";
|
||||
const placeholder = chat.status ? `${chat.status}. Press ⏎ to stop` : "Lam is thinking. Press ⏎ to stop";
|
||||
|
||||
const idlePlaceholder = pending ? "Ask about the screenshot…" : "Ask Lam anything…";
|
||||
|
||||
return (
|
||||
<List
|
||||
@@ -262,11 +373,13 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
|
||||
// Raycast puts the conversation title in a header; Vicinae's nearest
|
||||
// equivalent is the status-bar title, so the open chat is named there too.
|
||||
navigationTitle={truncate(chat.session.title, 40)}
|
||||
searchBarPlaceholder={chat.isStreaming ? placeholder : "Ask Lam anything…"}
|
||||
searchBarPlaceholder={chat.isStreaming ? placeholder : idlePlaceholder}
|
||||
actions={
|
||||
<ActionPanel>
|
||||
{chat.isStreaming ? stopAction : sendAction}
|
||||
{newChatAction}
|
||||
{snipAction}
|
||||
{openSnipAction}
|
||||
</ActionPanel>
|
||||
}
|
||||
>
|
||||
|
||||
+16
-3
@@ -7,8 +7,17 @@ export const DEFAULT_MODEL = "gpt-oss:120b";
|
||||
export const STORAGE_INDEX_KEY = "lam.sessions.index";
|
||||
export const STORAGE_SESSION_PREFIX = "lam.session.";
|
||||
|
||||
/** How many times the model may call tools before Lam gives up and reports back. */
|
||||
export const MAX_TOOL_ITERATIONS = 5;
|
||||
/**
|
||||
* How many rounds of tool calls one turn may spend before the model is told to
|
||||
* answer with what it has.
|
||||
*
|
||||
* Five was too few for a question that needs several searches and a few pages,
|
||||
* and the failure mode was the worst one available: the budget ran out, every
|
||||
* preamble the model had written was discarded, and the turn ended with a notice
|
||||
* about the limit rather than an answer. The closing round in `agent.ts` now
|
||||
* turns that into an answer, and this is the room it gets to gather first.
|
||||
*/
|
||||
export const MAX_TOOL_ITERATIONS = 8;
|
||||
|
||||
/** Oldest sessions past this count are evicted, along with their storage keys. */
|
||||
export const MAX_SESSIONS = 100;
|
||||
@@ -24,7 +33,9 @@ export const DEFAULT_SYSTEM_PROMPT = `You are Lam, a friendly assistant in the V
|
||||
|
||||
Sound like a person, not a document. Use contractions, talk about "you" and "I", and a short natural remark is welcome ("Sure, easy one", "Ah, that trips people up"). Performed enthusiasm and filler added to seem helpful are not.
|
||||
|
||||
Lead with the answer, then the detail. This renders in a narrow window, so stay brief: a tight sentence or two, or a short list, then stop. Keep paragraphs short and skip large headings and wide tables.
|
||||
Lead with the answer, then the detail. This renders in a narrow window, so stay brief: a tight sentence or two, or a short list, then stop. Keep paragraphs short.
|
||||
|
||||
Use the markdown this window actually draws rather than flattening everything into prose. It renders tables, bulleted and numbered lists, headings, blockquotes, bold, italics, inline code and fenced code, so pick the shape that fits. A table is the right shape for comparing three or more things across the same few attributes, so write the table itself: never hand back a table as a bulleted list of rows, and never describe a table you could simply write. Keep it to about four narrow columns so it fits, give it a header row, and leave a blank line before it. A list is for steps or for independent items. Reach for a heading only when an answer genuinely has sections, since at this size a heading usually costs a line and buys nothing.
|
||||
|
||||
Write plainly:
|
||||
- Never use an em dash. Use a comma, a colon, or a full stop.
|
||||
@@ -36,6 +47,8 @@ Write plainly:
|
||||
|
||||
Fence every command and every snippet, and always name the language: bash for shell and terminal commands, then json, sql, yaml, or the language for code (ts, js, python, rust, go). Never leave code inline in a sentence, and never open a fence without a language. Put no $ or shell prompt inside the block so it can be copied and run as-is. Ignore the formatting of earlier answers in this conversation: if one put a command in inline code, fence it anyway.
|
||||
|
||||
A web address is a link, never code. A markdown link, written as [label](https://example.com), is clickable in this window, while a URL inside backticks or a fenced block is dead text the user cannot open. So never fence a URL, never put one in inline code, and never answer "here are the links" with a snippet. Label a link with what it is ("OpenClickly on GitHub"), and when you give several, put each on its own bullet with its own link.
|
||||
|
||||
A fenced block is never a whole answer on its own. Say in one plain sentence what the code does or when to reach for it, so the reply reads like something a person wrote rather than a bare snippet.
|
||||
|
||||
Call a tool when it makes the answer more accurate. Anything time-sensitive or past your cutoff should be checked rather than guessed, and never invent tool output. If a tool fails, say so and answer with what you know.
|
||||
|
||||
+240
@@ -0,0 +1,240 @@
|
||||
import { LocalStorage } from "@vicinae/api";
|
||||
import { z } from "zod";
|
||||
import { errorMessage } from "./utils";
|
||||
|
||||
/**
|
||||
* IP geolocation, opt-in.
|
||||
*
|
||||
* This is the one place in the extension that tells a third party anything about
|
||||
* the user. The README said no geolocation service ever saw the user's address,
|
||||
* and this is what makes that conditional, so it is off unless asked for, it is
|
||||
* skipped entirely when the user has already said where they are, and the tool
|
||||
* output names the provider so the model can qualify what it repeats.
|
||||
*
|
||||
* How accurate is it? Less than the name suggests, and the honest answer is that
|
||||
* an IP address locates the point where the traffic left the ISP, not the person.
|
||||
* Measured from one machine on Globe Telecom, an IPv4 query resolved to Cebu City
|
||||
* and an IPv6 query, moments later, to Talisay in Calabarzon: about 500km apart.
|
||||
* The cache is therefore not just a privacy measure, and the tool output calls
|
||||
* the result a rough guess rather than a city.
|
||||
*
|
||||
* The answer is cached for hours rather than looked up per call. A city does not
|
||||
* change between questions, and every lookup hands the provider another copy of
|
||||
* the user's IP address.
|
||||
*
|
||||
* Imports Vicinae for `LocalStorage` only, which is the same footing as
|
||||
* `sessions.ts` and `vision.ts`. Everything else about it is plain `fetch`, so
|
||||
* the parsing and the cache are unit tested without a runtime.
|
||||
*/
|
||||
|
||||
/** Where the lookup goes. Swapping providers means changing this and the parser below. */
|
||||
export const GEO_PROVIDER = "https://ipwho.is/";
|
||||
export const GEO_PROVIDER_NAME = "ipwho.is";
|
||||
|
||||
/** How long a resolved place is trusted. Long enough to not re-leak the IP on every question. */
|
||||
export const GEO_TTL_MS = 6 * 60 * 60 * 1000;
|
||||
|
||||
/** An IP lookup is a nice-to-have, so it must never hold up an answer. */
|
||||
const GEO_TIMEOUT_MS = 8_000;
|
||||
|
||||
const STORAGE_KEY = "lam.geo";
|
||||
|
||||
export interface GeoPlace {
|
||||
/** Ready to show, e.g. "Cebu City, Central Visayas, Philippines". */
|
||||
place: string;
|
||||
city?: string;
|
||||
region?: string;
|
||||
country?: string;
|
||||
latitude?: number;
|
||||
longitude?: number;
|
||||
timezone?: string;
|
||||
/** Named in the tool output, so the model can say where this came from. */
|
||||
source: string;
|
||||
fetchedAt: number;
|
||||
}
|
||||
|
||||
export type GeoLookup = { status: "ok"; place: GeoPlace } | { status: "failed"; message: string };
|
||||
|
||||
export interface GeoOptions {
|
||||
signal?: AbortSignal;
|
||||
/** Injected so the cache is deterministic under test. */
|
||||
now?: number;
|
||||
/** Ignores a cached answer. */
|
||||
refresh?: boolean;
|
||||
}
|
||||
|
||||
const CachedSchema = z.object({
|
||||
place: z.string(),
|
||||
city: z.string().optional(),
|
||||
region: z.string().optional(),
|
||||
country: z.string().optional(),
|
||||
latitude: z.number().optional(),
|
||||
longitude: z.number().optional(),
|
||||
timezone: z.string().optional(),
|
||||
source: z.string(),
|
||||
fetchedAt: z.number(),
|
||||
});
|
||||
|
||||
/** The subset of the provider's response this needs, with every field optional so a shape change degrades instead of throwing. */
|
||||
const ResponseSchema = z.object({
|
||||
success: z.boolean().optional(),
|
||||
message: z.string().optional(),
|
||||
city: z.string().nullable().optional(),
|
||||
region: z.string().nullable().optional(),
|
||||
country: z.string().nullable().optional(),
|
||||
latitude: z.number().nullable().optional(),
|
||||
longitude: z.number().nullable().optional(),
|
||||
// ipwho.is nests the timezone; some providers hand back a plain string.
|
||||
timezone: z
|
||||
.union([z.string(), z.object({ id: z.string().optional() })])
|
||||
.nullable()
|
||||
.optional(),
|
||||
});
|
||||
|
||||
/**
|
||||
* Joins the parts of a place, dropping duplicates.
|
||||
*
|
||||
* Providers often return the city twice (once as `city`, once as `region`) for
|
||||
* city-states and single-city regions, and "Singapore, Singapore, Singapore"
|
||||
* reads like a bug.
|
||||
*/
|
||||
export function buildPlace(parts: { city?: string; region?: string; country?: string }): string {
|
||||
const seen = new Set<string>();
|
||||
const kept: string[] = [];
|
||||
|
||||
for (const part of [parts.city, parts.region, parts.country]) {
|
||||
const trimmed = part?.trim();
|
||||
const key = trimmed?.toLowerCase();
|
||||
if (!trimmed || !key || seen.has(key)) continue;
|
||||
|
||||
seen.add(key);
|
||||
kept.push(trimmed);
|
||||
}
|
||||
|
||||
return kept.join(", ");
|
||||
}
|
||||
|
||||
/** Aborts on either the caller's signal or a timeout, whichever comes first. */
|
||||
async function fetchWithTimeout(
|
||||
url: string,
|
||||
signal: AbortSignal | undefined,
|
||||
timeoutMs: number,
|
||||
): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
const forward = () => controller.abort();
|
||||
|
||||
signal?.addEventListener("abort", forward, { once: true });
|
||||
|
||||
try {
|
||||
return await fetch(url, {
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
// Named rather than anonymous. Node's default UA is bare, and "who is
|
||||
// asking" is the one thing a geolocation provider has a fair interest in.
|
||||
"User-Agent": "lam-agent-vicinae",
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", forward);
|
||||
}
|
||||
}
|
||||
|
||||
async function readCache(now: number): Promise<GeoPlace | undefined> {
|
||||
try {
|
||||
const raw = await LocalStorage.getItem<string>(STORAGE_KEY);
|
||||
if (!raw) return undefined;
|
||||
|
||||
const parsed = CachedSchema.safeParse(JSON.parse(raw));
|
||||
if (!parsed.success) return undefined;
|
||||
if (now - parsed.data.fetchedAt > GEO_TTL_MS) return undefined;
|
||||
|
||||
return parsed.data;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async function writeCache(place: GeoPlace): Promise<void> {
|
||||
await LocalStorage.setItem(STORAGE_KEY, JSON.stringify(place)).catch(() => undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the user's rough place from their IP address.
|
||||
*
|
||||
* Never throws: a lookup failure leaves the region-level answer intact, and the
|
||||
* message is passed on so the model can say what went wrong rather than quietly
|
||||
* presenting a coarser answer as though it were the same thing.
|
||||
*/
|
||||
export async function ipPlace(options: GeoOptions = {}): Promise<GeoLookup> {
|
||||
const now = options.now ?? Date.now();
|
||||
|
||||
if (!options.refresh) {
|
||||
const cached = await readCache(now);
|
||||
if (cached) return { status: "ok", place: cached };
|
||||
}
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetchWithTimeout(GEO_PROVIDER, options.signal, GEO_TIMEOUT_MS);
|
||||
} catch (thrown) {
|
||||
return { status: "failed", message: `could not reach ${GEO_PROVIDER_NAME} (${errorMessage(thrown)})` };
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
return { status: "failed", message: `${GEO_PROVIDER_NAME} answered HTTP ${response.status}` };
|
||||
}
|
||||
|
||||
let payload: unknown;
|
||||
try {
|
||||
payload = await response.json();
|
||||
} catch {
|
||||
return { status: "failed", message: `${GEO_PROVIDER_NAME} did not answer with JSON` };
|
||||
}
|
||||
|
||||
const parsed = ResponseSchema.safeParse(payload);
|
||||
if (!parsed.success) {
|
||||
return { status: "failed", message: `${GEO_PROVIDER_NAME} answered in an unexpected shape` };
|
||||
}
|
||||
|
||||
const data = parsed.data;
|
||||
if (data.success === false) {
|
||||
return {
|
||||
status: "failed",
|
||||
message: data.message ?? `${GEO_PROVIDER_NAME} refused the lookup`,
|
||||
};
|
||||
}
|
||||
|
||||
const city = data.city ?? undefined;
|
||||
const region = data.region ?? undefined;
|
||||
const country = data.country ?? undefined;
|
||||
const place = buildPlace({ city, region, country });
|
||||
|
||||
if (!place) {
|
||||
return { status: "failed", message: `${GEO_PROVIDER_NAME} returned no usable place` };
|
||||
}
|
||||
|
||||
const timezone = typeof data.timezone === "string" ? data.timezone : data.timezone?.id;
|
||||
|
||||
const resolved: GeoPlace = {
|
||||
place,
|
||||
city,
|
||||
region,
|
||||
country,
|
||||
latitude: data.latitude ?? undefined,
|
||||
longitude: data.longitude ?? undefined,
|
||||
timezone: timezone ?? undefined,
|
||||
source: GEO_PROVIDER_NAME,
|
||||
fetchedAt: now,
|
||||
};
|
||||
|
||||
await writeCache(resolved);
|
||||
return { status: "ok", place: resolved };
|
||||
}
|
||||
|
||||
/** Drops the cached place, so a move or a provider change is not pinned for hours. */
|
||||
export async function forgetGeoPlace(): Promise<void> {
|
||||
await LocalStorage.removeItem(STORAGE_KEY).catch(() => undefined);
|
||||
}
|
||||
+14
-2
@@ -8,8 +8,17 @@ import type { ApiMessage, ChatTurn } from "./types";
|
||||
* messages from a previous turn are intentionally dropped: they are noise once
|
||||
* the answer is written, and replaying them would re-send large page contents
|
||||
* on every subsequent request.
|
||||
*
|
||||
* Attached images are treated the same way, and for the same reason. They are
|
||||
* the largest thing a request can carry, so only the question being asked now
|
||||
* brings one. A follow-up about a screenshot needs the image attached again.
|
||||
*/
|
||||
export function buildMessages(systemPrompt: string, turns: ChatTurn[], prompt: string): ApiMessage[] {
|
||||
export function buildMessages(
|
||||
systemPrompt: string,
|
||||
turns: ChatTurn[],
|
||||
prompt: string,
|
||||
images?: string[],
|
||||
): ApiMessage[] {
|
||||
const messages: ApiMessage[] = [{ role: "system", content: systemPrompt }];
|
||||
|
||||
for (const turn of turns) {
|
||||
@@ -18,7 +27,10 @@ export function buildMessages(systemPrompt: string, turns: ChatTurn[], prompt: s
|
||||
messages.push({ role: "assistant", content: turn.response });
|
||||
}
|
||||
|
||||
messages.push({ role: "user", content: prompt });
|
||||
const question: ApiMessage = { role: "user", content: prompt };
|
||||
if (images && images.length > 0) question.images = images;
|
||||
messages.push(question);
|
||||
|
||||
return messages;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { runAgent } from "../agent";
|
||||
import { toStoredAttachment } from "../attachments";
|
||||
import { ipPlace } from "../geo";
|
||||
import { buildMessages, composeSystemPrompt } from "../history";
|
||||
import { OllamaError } from "../ollama";
|
||||
import type { ResolvedPreferences } from "../preferences";
|
||||
import { createSession, createTurn, saveSession, withTitleFromFirstTurn } from "../sessions";
|
||||
import { toolsFor } from "../tools";
|
||||
import type { Session } from "../types";
|
||||
import type { PendingAttachment, Session } from "../types";
|
||||
import { errorMessage, isAbortError } from "../utils";
|
||||
|
||||
/**
|
||||
@@ -41,7 +43,7 @@ export interface UseChat {
|
||||
/** Short activity line such as "Searching the web", shown while a turn is in flight. */
|
||||
status?: string;
|
||||
error?: string;
|
||||
ask: (prompt: string) => void;
|
||||
ask: (prompt: string, attachment?: PendingAttachment) => void;
|
||||
cancel: () => void;
|
||||
replaceSession: (session: Session) => void;
|
||||
newSession: () => void;
|
||||
@@ -119,16 +121,22 @@ export function useChat(options: UseChatOptions): UseChat {
|
||||
}, []);
|
||||
|
||||
const ask = useCallback(
|
||||
(rawPrompt: string) => {
|
||||
(rawPrompt: string, attachment?: PendingAttachment) => {
|
||||
const prompt = rawPrompt.trim();
|
||||
if (!prompt || liveRef.current) return;
|
||||
|
||||
const preferences = preferencesRef.current;
|
||||
const base = sessionRef.current;
|
||||
|
||||
// Only the path, mime type and dimensions are kept on the turn. The base64
|
||||
// payload is used for this one request and then dropped, so it never
|
||||
// reaches storage. The dimensions have to survive because the transcript
|
||||
// lays the preview out long after the payload is gone.
|
||||
const attachments = attachment ? [toStoredAttachment(attachment)] : [];
|
||||
|
||||
// The turn is inserted immediately with an empty response so the UI shows
|
||||
// the question while the answer streams in beneath it.
|
||||
const turn = createTurn(prompt, preferences.model);
|
||||
const turn = createTurn(prompt, preferences.model, Date.now(), attachments);
|
||||
const started = withTitleFromFirstTurn({
|
||||
...base,
|
||||
model: preferences.model,
|
||||
@@ -152,6 +160,7 @@ export function useChat(options: UseChatOptions): UseChat {
|
||||
composeSystemPrompt(preferences.systemPrompt, preferences.webEnabled),
|
||||
base.turns,
|
||||
prompt,
|
||||
attachment ? [attachment.base64] : undefined,
|
||||
);
|
||||
|
||||
void (async () => {
|
||||
@@ -166,7 +175,14 @@ export function useChat(options: UseChatOptions): UseChat {
|
||||
think: preferences.think,
|
||||
messages,
|
||||
tools: toolsFor({ webEnabled: preferences.webEnabled }),
|
||||
toolContext: { host: preferences.host, apiKey: preferences.apiKey },
|
||||
toolContext: {
|
||||
host: preferences.host,
|
||||
apiKey: preferences.apiKey,
|
||||
place: preferences.place,
|
||||
// Only wired up when the user switched it on, so an off setting
|
||||
// cannot reach the network for a location at all.
|
||||
ipPlace: preferences.ipGeolocation ? ipPlace : undefined,
|
||||
},
|
||||
signal: controller.signal,
|
||||
events: {
|
||||
onContentDelta: (delta) => {
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import type { ResolvedPreferences } from "../preferences";
|
||||
import { visionSupport } from "../vision";
|
||||
|
||||
/**
|
||||
* Whether the current model accepts images, for gating the snip action.
|
||||
*
|
||||
* Fails closed: only a confirmed `"yes"` turns the feature on. A model that
|
||||
* cannot see would otherwise accept a screenshot and answer as though it had,
|
||||
* which is worse than the action not being offered.
|
||||
*
|
||||
* The wording for a *failed* check lives in the snip command rather than here,
|
||||
* because that is the only place with somewhere to say it (a HUD). This hook
|
||||
* only decides whether to show an action.
|
||||
*/
|
||||
|
||||
export interface VisionState {
|
||||
/** True only once Ollama has confirmed the model reports the `vision` capability. */
|
||||
canSeeImages: boolean;
|
||||
/** True while an answer for this model is still outstanding. */
|
||||
isChecking: boolean;
|
||||
}
|
||||
|
||||
export function useVisionSupport(preferences: ResolvedPreferences): VisionState {
|
||||
const [canSeeImages, setCanSeeImages] = useState(false);
|
||||
const [isChecking, setIsChecking] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const controller = new AbortController();
|
||||
|
||||
setIsChecking(true);
|
||||
|
||||
void visionSupport({
|
||||
host: preferences.host,
|
||||
apiKey: preferences.apiKey,
|
||||
model: preferences.model,
|
||||
signal: controller.signal,
|
||||
})
|
||||
.then((support) => {
|
||||
if (!cancelled) setCanSeeImages(support === "yes");
|
||||
})
|
||||
.finally(() => {
|
||||
if (!cancelled) setIsChecking(false);
|
||||
});
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
controller.abort();
|
||||
};
|
||||
}, [preferences.host, preferences.apiKey, preferences.model]);
|
||||
|
||||
return { canSeeImages, isChecking };
|
||||
}
|
||||
+69
-13
@@ -1,30 +1,76 @@
|
||||
/**
|
||||
* Time and place, derived locally.
|
||||
*
|
||||
* `get_location` is intentionally timezone-derived rather than a GPS or IP
|
||||
* lookup. It answers the questions people actually ask a launcher ("what time
|
||||
* is it there", "is it late in London") without shipping the user's IP address
|
||||
* to a third-party geolocation service on every tool call. It is also pure, so
|
||||
* it is cheap to test.
|
||||
* `get_location` is timezone-derived rather than a GPS or IP lookup, so the
|
||||
* user's IP is never shipped to a third-party geolocation service on every tool
|
||||
* call. It is also pure, so it is cheap to test.
|
||||
*
|
||||
* The correction that matters here is that a timezone name is **not** a city.
|
||||
* The IANA database names a zone after one representative place, and a single
|
||||
* zone can cover a whole country: "Asia/Manila" is the Philippines,
|
||||
* "Asia/Kolkata" is India, "Europe/Oslo" is Norway, "Australia/Brisbane" is
|
||||
* Queensland. Reporting "Manila" as the user's city is simply wrong for anyone
|
||||
* in Cebu, and being confidently wrong is worse than being vague, because a
|
||||
* model asked about the weather will use it without hesitating. So the zone's
|
||||
* label is still reported, but named for what it is instead of being dressed up
|
||||
* as the user's city.
|
||||
*
|
||||
* The only exact answer available without asking a third party is the Location
|
||||
* preference, which the user fills in themselves. It is reported verbatim and
|
||||
* preferred over anything inferred.
|
||||
*/
|
||||
|
||||
export interface LocationContext {
|
||||
timezone: string;
|
||||
utcOffset: string;
|
||||
locale: string;
|
||||
/** Country code from the locale, e.g. "PH". The user's own setting rather than an inference. */
|
||||
regionCode?: string;
|
||||
/** Readable name for that region, e.g. "Philippines". */
|
||||
region?: string;
|
||||
city?: string;
|
||||
/**
|
||||
* The timezone's own area label, e.g. "Manila".
|
||||
*
|
||||
* Deliberately not called a city. See the note at the top of this file.
|
||||
*/
|
||||
timezoneArea?: string;
|
||||
/** The user's configured Location preference, verbatim. The only exact value here. */
|
||||
place?: string;
|
||||
}
|
||||
|
||||
/** Splits an IANA zone like "Asia/Manila" or "America/Argentina/Salta" into readable parts. */
|
||||
export function describeTimezone(timeZone: string): { region?: string; city?: string } {
|
||||
/**
|
||||
* Splits an IANA zone like "Asia/Manila" or "America/Argentina/Salta".
|
||||
*
|
||||
* Only the last segment is kept, and the continent grouping is dropped: "Asia"
|
||||
* next to a locale region of "Philippines" reads as two answers to the same
|
||||
* question, and the zone's grouping answers a question nobody asked.
|
||||
*/
|
||||
export function describeTimezoneArea(timeZone: string): { area?: string } {
|
||||
const parts = timeZone.split("/").filter(Boolean);
|
||||
if (parts.length < 2) return {};
|
||||
|
||||
const region = parts[0];
|
||||
const city = parts[parts.length - 1].replace(/_/g, " ");
|
||||
return { area: parts[parts.length - 1].replace(/_/g, " ") || undefined };
|
||||
}
|
||||
|
||||
return { region, city: city || undefined };
|
||||
/**
|
||||
* The region from the locale subtag.
|
||||
*
|
||||
* This is a declaration rather than an inference: a locale of "en-PH" says the
|
||||
* user set their region to the Philippines. That makes it a better answer than
|
||||
* anything read out of a zone name, and it costs nothing.
|
||||
*/
|
||||
export function localeRegion(locale: string): { code: string; name: string } | undefined {
|
||||
try {
|
||||
const code = new Intl.Locale(locale).region;
|
||||
if (!code) return undefined;
|
||||
|
||||
const name = new Intl.DisplayNames(["en"], { type: "region" }).of(code);
|
||||
return { code, name: name ?? code };
|
||||
} catch {
|
||||
// A malformed locale, or an ICU build without region display names. Not
|
||||
// worth failing the whole answer over.
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** "GMT+08:00" on a modern V8; falls back to a short name if `longOffset` is unavailable. */
|
||||
@@ -56,7 +102,11 @@ function resolveTimeZone(timeZone?: string): string {
|
||||
}
|
||||
}
|
||||
|
||||
export function getLocationContext(date: Date = new Date(), timeZone?: string): LocationContext {
|
||||
export function getLocationContext(
|
||||
date: Date = new Date(),
|
||||
timeZone?: string,
|
||||
place?: string,
|
||||
): LocationContext {
|
||||
const zone = resolveTimeZone(timeZone);
|
||||
|
||||
let locale = "en-US";
|
||||
@@ -66,11 +116,17 @@ export function getLocationContext(date: Date = new Date(), timeZone?: string):
|
||||
// Keep the fallback locale.
|
||||
}
|
||||
|
||||
const region = localeRegion(locale);
|
||||
const configured = place?.trim();
|
||||
|
||||
return {
|
||||
timezone: zone,
|
||||
utcOffset: utcOffsetLabel(date, zone),
|
||||
locale,
|
||||
...describeTimezone(zone),
|
||||
regionCode: region?.code,
|
||||
region: region?.name,
|
||||
timezoneArea: describeTimezoneArea(zone).area,
|
||||
place: configured && configured.length > 0 ? configured : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+28
-3
@@ -1,5 +1,6 @@
|
||||
import { imageSource } from "./attachments";
|
||||
import { repairCodeBlocks } from "./code";
|
||||
import type { ChatTurn, ThreadMessage } from "./types";
|
||||
import type { Attachment, ChatTurn, ThreadMessage } from "./types";
|
||||
|
||||
/**
|
||||
* Markdown rendering for the detail panel.
|
||||
@@ -22,6 +23,24 @@ import type { ChatTurn, ThreadMessage } from "./types";
|
||||
|
||||
export const STREAMING_CURSOR = "▍";
|
||||
|
||||
/**
|
||||
* Renders attachments as markdown images.
|
||||
*
|
||||
* Vicinae does resolve a local file: `MdImage.qml` draws it block-level and
|
||||
* centred, uses the alt text as an italic caption underneath, and caps the
|
||||
* height at 200px no matter what is asked for. So this is a thumbnail rather
|
||||
* than something readable, which is why the action panel also offers Open
|
||||
* Screenshot for the full image.
|
||||
*
|
||||
* The alt text is short on purpose. Whatever is here is drawn centred under the
|
||||
* image, so a sentence reads as a stray line of prose floating in the middle of
|
||||
* the pane. It is also the only thing left if the file has since been deleted,
|
||||
* which is why it is not empty.
|
||||
*/
|
||||
export function renderAttachments(attachments: Attachment[]): string {
|
||||
return attachments.map((attachment) => `})`).join("\n\n");
|
||||
}
|
||||
|
||||
/** Which optional sections the transcript includes. Driven by preferences. */
|
||||
export interface TranscriptOptions {
|
||||
showToolCalls: boolean;
|
||||
@@ -49,7 +68,12 @@ function renderTools(message: ThreadMessage): string {
|
||||
|
||||
function renderMessage(message: ThreadMessage, options: TranscriptOptions): string {
|
||||
// The question is identified by its styling alone.
|
||||
if (message.role === "user") return quote(message.text);
|
||||
if (message.role === "user") {
|
||||
const images = renderAttachments(message.attachments);
|
||||
const asked = quote(message.text);
|
||||
|
||||
return images ? `${asked}\n\n${images}` : asked;
|
||||
}
|
||||
|
||||
// While a tool is running there is no answer text yet, so the activity line
|
||||
// stands in and the user can see what Lam is actually doing.
|
||||
@@ -138,9 +162,10 @@ export function sessionToPlainText(turns: ChatTurn[]): string {
|
||||
return turns
|
||||
.map((turn) => {
|
||||
const question = `**You**\n\n${turn.prompt}`;
|
||||
const attached = turn.attachments.length > 0 ? "\n\n_(screenshot attached)_" : "";
|
||||
const answer = turn.response.trim() || (turn.error ? "" : "_(no response)_");
|
||||
const failure = turn.error ? `\n\n**[error]** ${turn.error}` : "";
|
||||
return `${question}\n\n**Lam**\n\n${answer}${failure}`;
|
||||
return `${question}${attached}\n\n**Lam**\n\n${answer}${failure}`;
|
||||
})
|
||||
.join("\n\n---\n\n");
|
||||
}
|
||||
|
||||
@@ -219,6 +219,44 @@ export async function listModels(options: ListModelsOptions): Promise<ModelInfo[
|
||||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
export interface ModelCapabilitiesOptions {
|
||||
host: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* The features a model reports for itself, straight from `POST /api/show`.
|
||||
*
|
||||
* `GET /api/tags` is not enough here: it returns names, sizes and quantization
|
||||
* levels and says nothing about what the model can actually do. The capability
|
||||
* list is what turns "image input" from a guess about the model's name into a
|
||||
* fact, so it is worth the extra request.
|
||||
*/
|
||||
export async function modelCapabilities(options: ModelCapabilitiesOptions): Promise<string[]> {
|
||||
const host = normalizeHost(options.host);
|
||||
|
||||
const response = await fetch(`${host}/api/show`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", ...authHeaders(options.apiKey) },
|
||||
body: JSON.stringify({ model: options.model }),
|
||||
signal: options.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw describeHttpError(response.status, await safeText(response), host);
|
||||
}
|
||||
|
||||
const payload = (await response.json()) as { capabilities?: string[] };
|
||||
return payload.capabilities ?? [];
|
||||
}
|
||||
|
||||
/** Ollama reports image input as the `vision` capability. */
|
||||
export function supportsVision(capabilities: string[]): boolean {
|
||||
return capabilities.includes("vision");
|
||||
}
|
||||
|
||||
export interface WebSearchResult {
|
||||
title: string;
|
||||
url: string;
|
||||
|
||||
@@ -15,6 +15,8 @@ export interface LamPreferences {
|
||||
model?: string;
|
||||
ollamaHost?: string;
|
||||
systemPrompt?: string;
|
||||
location?: string;
|
||||
ipGeolocation?: boolean;
|
||||
webSearchEnabled?: boolean;
|
||||
thinking?: string;
|
||||
showToolCalls?: boolean;
|
||||
@@ -28,6 +30,16 @@ export interface ResolvedPreferences {
|
||||
model: string;
|
||||
host: string;
|
||||
systemPrompt: string;
|
||||
/**
|
||||
* The place the user says they are, verbatim, or an empty string.
|
||||
*
|
||||
* The only exact location this extension has. Everything else is inferred from
|
||||
* the device timezone and locale, which is accurate to a region rather than a
|
||||
* city, and a timezone name is not a city at all.
|
||||
*/
|
||||
place: string;
|
||||
/** Whether to resolve a rough place from the IP address. Off unless asked for. */
|
||||
ipGeolocation: boolean;
|
||||
webEnabled: boolean;
|
||||
/** Undefined when reasoning is off, which is what the API client expects. */
|
||||
think?: string;
|
||||
@@ -78,6 +90,10 @@ export function resolvePreferences(raw: LamPreferences): ResolvedPreferences {
|
||||
model: (raw.model ?? "").trim() || DEFAULT_MODEL,
|
||||
host: (raw.ollamaHost ?? "").trim() || DEFAULT_HOST,
|
||||
systemPrompt: (raw.systemPrompt ?? "").trim(),
|
||||
place: (raw.location ?? "").trim(),
|
||||
// Defaults to off: this is the one setting that sends something about the
|
||||
// user to a third party, so it is a deliberate choice rather than a default.
|
||||
ipGeolocation: raw.ipGeolocation ?? false,
|
||||
// Defaults to on: the preference is a checkbox the user has to actively untick.
|
||||
webEnabled: raw.webSearchEnabled ?? true,
|
||||
think: thinking && thinking !== "off" ? thinking : undefined,
|
||||
|
||||
@@ -14,6 +14,15 @@ export const ToolCallRecordSchema = z.object({
|
||||
isError: z.boolean(),
|
||||
});
|
||||
|
||||
export const AttachmentSchema = z.object({
|
||||
path: z.string(),
|
||||
mime: z.string().default("image/png"),
|
||||
// Absent on sessions written before the header was read, which is fine: a
|
||||
// preview without dimensions falls back to the renderer's own sizing.
|
||||
width: z.number().optional(),
|
||||
height: z.number().optional(),
|
||||
});
|
||||
|
||||
export const ChatTurnSchema = z.object({
|
||||
id: z.string(),
|
||||
prompt: z.string(),
|
||||
@@ -23,6 +32,9 @@ export const ChatTurnSchema = z.object({
|
||||
createdAt: z.number(),
|
||||
error: z.string().optional(),
|
||||
toolCalls: z.array(ToolCallRecordSchema).default([]),
|
||||
// Sessions written before image input existed have no attachments key, so the
|
||||
// default is what keeps them loading rather than being rejected as corrupt.
|
||||
attachments: z.array(AttachmentSchema).default([]),
|
||||
});
|
||||
|
||||
export const SessionSchema = z.object({
|
||||
|
||||
+8
-2
@@ -1,7 +1,7 @@
|
||||
import { LocalStorage } from "@vicinae/api";
|
||||
import { MAX_SESSIONS, MAX_STORED_CHARS_PER_TURN, MAX_TURNS_PER_SESSION, STORAGE_INDEX_KEY, STORAGE_SESSION_PREFIX, TITLE_MAX_LENGTH } from "./constants";
|
||||
import { SessionIndexSchema, SessionSchema } from "./schemas";
|
||||
import type { ChatTurn, Session, SessionSummary } from "./types";
|
||||
import type { Attachment, ChatTurn, Session, SessionSummary } from "./types";
|
||||
import { titleFromPrompt, truncate } from "./utils";
|
||||
|
||||
/**
|
||||
@@ -56,7 +56,12 @@ export function createSession(model: string, now: number = Date.now()): Session
|
||||
}
|
||||
|
||||
/** Creates the turn record for a prompt that is about to be sent. */
|
||||
export function createTurn(prompt: string, model: string, now: number = Date.now()): ChatTurn {
|
||||
export function createTurn(
|
||||
prompt: string,
|
||||
model: string,
|
||||
now: number = Date.now(),
|
||||
attachments: Attachment[] = [],
|
||||
): ChatTurn {
|
||||
return {
|
||||
id: newId(),
|
||||
prompt,
|
||||
@@ -65,6 +70,7 @@ export function createTurn(prompt: string, model: string, now: number = Date.now
|
||||
model,
|
||||
createdAt: now,
|
||||
toolCalls: [],
|
||||
attachments,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import { environment } from "@vicinae/api";
|
||||
import { join } from "node:path";
|
||||
import { CapturePermissionError, NoCaptureToolError } from "./capture";
|
||||
import { errorMessage } from "./utils";
|
||||
|
||||
/**
|
||||
* The seam between the snip command and the chat command.
|
||||
*
|
||||
* The capture cannot happen inside the chat view. Vicinae shows the launcher
|
||||
* window when it activates a command, so a region selector started from a view
|
||||
* is dragging around a panel parked in the middle of the screen. Capturing from
|
||||
* a `no-view` command and handing the file over is what keeps the window out of
|
||||
* the way, which means the two commands need a small agreed contract.
|
||||
*
|
||||
* Imports Vicinae for `environment.supportPath` only, and nothing else in the
|
||||
* logic layer depends on it.
|
||||
*/
|
||||
|
||||
/** What the snip command passes to the chat command through `launchCommand`. */
|
||||
export interface SnipLaunchContext {
|
||||
/** Absolute path of the captured PNG. Absent when the capture failed. */
|
||||
snipPath?: string;
|
||||
/** Why a capture failed, so the chat can explain itself rather than look empty. */
|
||||
snipError?: string;
|
||||
/** The conversation the snip was started from, restored so it is not lost. */
|
||||
sessionId?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Snips live in the extension's own support directory rather than a temp
|
||||
* directory: a session stores the path, and a path that survives a reboot is the
|
||||
* difference between an old conversation showing its screenshot and pointing at
|
||||
* nothing.
|
||||
*/
|
||||
export function snipDirectory(): string {
|
||||
return join(environment.supportPath, "snips");
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the handoff payload.
|
||||
*
|
||||
* A launch context is whatever the launching command put there, so every field
|
||||
* is checked rather than trusted: a wrong type would otherwise reach the file
|
||||
* system as a path.
|
||||
*/
|
||||
export function readSnipContext(launchContext: unknown): SnipLaunchContext {
|
||||
if (!launchContext || typeof launchContext !== "object") return {};
|
||||
|
||||
const { snipPath, snipError, sessionId } = launchContext as Record<string, unknown>;
|
||||
const asString = (value: unknown) => (typeof value === "string" && value.length > 0 ? value : undefined);
|
||||
|
||||
return {
|
||||
snipPath: asString(snipPath),
|
||||
snipError: asString(snipError),
|
||||
sessionId: asString(sessionId),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns a capture failure into something worth showing.
|
||||
*
|
||||
* The two failures that carry a hint are the only two the user can do anything
|
||||
* about, and the hints are useless if they are dropped: "no capture tool is
|
||||
* installed" and "the system is blocking screen capture" need completely
|
||||
* different responses from completely different places.
|
||||
*/
|
||||
export function describeCaptureFailure(thrown: unknown): string {
|
||||
const actionable =
|
||||
thrown instanceof CapturePermissionError || thrown instanceof NoCaptureToolError
|
||||
? thrown.hint
|
||||
: undefined;
|
||||
|
||||
return actionable ? `${errorMessage(thrown)} ${actionable}` : errorMessage(thrown);
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import type { PendingAttachment, Session, SessionSummary } from "./types";
|
||||
import { errorMessage } from "./utils";
|
||||
|
||||
/**
|
||||
* Everything the chat view resolves before its first render.
|
||||
*
|
||||
* Pulled out of the component and made pure because this is the part that keeps
|
||||
* going wrong. Three things can happen on open, and all of them are
|
||||
* asynchronous: a launch hands over a captured file, it may also name the
|
||||
* conversation to return to, and an ordinary open resumes the most recent
|
||||
* conversation if it is fresh enough. Every path has to end in a rendered chat.
|
||||
*
|
||||
* As a plain function it is testable without mounting anything, which matters
|
||||
* because the alternative is finding out from a screenshot that a view is
|
||||
* spinning forever.
|
||||
*
|
||||
* Imports nothing from Vicinae, so it belongs to the logic layer.
|
||||
*/
|
||||
|
||||
export interface StartupInput {
|
||||
/** A conversation already in hand, which is how the sessions command pushes this view. */
|
||||
hasInitialSession: boolean;
|
||||
/** A conversation to reopen by id, handed over by the snip command. */
|
||||
initialSessionId?: string;
|
||||
/** A captured image to read in. */
|
||||
initialAttachmentPath?: string;
|
||||
/** Idle time past which an ordinary open starts fresh. Undefined means never time out. */
|
||||
resumeTimeoutMs?: number;
|
||||
/** Injected so the resume window is deterministic under test. */
|
||||
now: number;
|
||||
}
|
||||
|
||||
/** The side effects, injected so nothing here touches storage or the file system directly. */
|
||||
export interface StartupPorts {
|
||||
loadSession: (id: string) => Promise<Session | undefined>;
|
||||
listSessions: () => Promise<SessionSummary[]>;
|
||||
readAttachment: (path: string) => Promise<PendingAttachment>;
|
||||
}
|
||||
|
||||
export interface StartupResult {
|
||||
/** The conversation to show, when one was restored. */
|
||||
session?: Session;
|
||||
/** The image that was handed over, when one could be read. */
|
||||
attachment?: PendingAttachment;
|
||||
/** Why a handed-over image could not be read, so the view can say so. */
|
||||
attachmentError?: string;
|
||||
}
|
||||
|
||||
/** True when the launch handed something over, which also means "do not resume". */
|
||||
export function isHandoff(input: Pick<StartupInput, "initialSessionId" | "initialAttachmentPath">): boolean {
|
||||
return Boolean(input.initialSessionId || input.initialAttachmentPath);
|
||||
}
|
||||
|
||||
export async function resolveStartup(input: StartupInput, ports: StartupPorts): Promise<StartupResult> {
|
||||
const result: StartupResult = {};
|
||||
|
||||
if (input.initialSessionId) {
|
||||
// A hand-off names the conversation to return to, so that one is restored
|
||||
// rather than whichever chat happens to be most recent. By id matters
|
||||
// because taking a snip is not a turn, so nothing was saved in between.
|
||||
// A session that has since been deleted just means starting fresh.
|
||||
result.session = await ports.loadSession(input.initialSessionId).catch(() => undefined);
|
||||
}
|
||||
|
||||
if (input.initialAttachmentPath) {
|
||||
try {
|
||||
result.attachment = await ports.readAttachment(input.initialAttachmentPath);
|
||||
} catch (thrown) {
|
||||
// A missing file leaves an empty chat with an explanation, which beats
|
||||
// refusing to open at all.
|
||||
result.attachmentError = errorMessage(thrown);
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing was handed over, so the idle-resume rule applies: reopening the
|
||||
// command continues the most recent conversation, but only if it was used
|
||||
// recently. The previous conversation is never removed either way, it just
|
||||
// drops down the sidebar.
|
||||
if (!input.hasInitialSession && !isHandoff(input)) {
|
||||
const latest = await ports
|
||||
.listSessions()
|
||||
.then((sessions) => sessions[0])
|
||||
.catch(() => undefined);
|
||||
|
||||
if (latest) {
|
||||
const idleMs = input.now - latest.updatedAt;
|
||||
const fresh = input.resumeTimeoutMs === undefined || idleMs <= input.resumeTimeoutMs;
|
||||
|
||||
if (fresh) {
|
||||
result.session = await ports.loadSession(latest.id).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -43,6 +43,8 @@ export function buildThread(turns: ChatTurn[], live: LiveState): ThreadMessage[]
|
||||
// Only while there is nothing to read yet. Once tokens land, the
|
||||
// placeholder it feeds is gone anyway.
|
||||
status: isLive && text.length === 0 ? live.status : undefined,
|
||||
// Only a question carries attachments, so the answer never has one.
|
||||
attachments: [],
|
||||
});
|
||||
|
||||
messages.push({
|
||||
@@ -53,6 +55,7 @@ export function buildThread(turns: ChatTurn[], live: LiveState): ThreadMessage[]
|
||||
isStreaming: false,
|
||||
thinking: "",
|
||||
toolCalls: [],
|
||||
attachments: turn.attachments,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+76
-10
@@ -1,5 +1,6 @@
|
||||
import { formatNow, getLocationContext, weekdayName } from "./location";
|
||||
import { webFetch, webSearch } from "./ollama";
|
||||
import type { GeoLookup } from "./geo";
|
||||
import type { ToolDefinition } from "./types";
|
||||
import { errorMessage } from "./utils";
|
||||
|
||||
@@ -19,6 +20,21 @@ export interface ToolContext {
|
||||
signal?: AbortSignal;
|
||||
/** Injectable clock so tests are deterministic. */
|
||||
now?: Date;
|
||||
/**
|
||||
* The user's Location preference, if they set one.
|
||||
*
|
||||
* Injected rather than read here, because this module is deliberately free of
|
||||
* Vicinae imports and preferences arrive through that API.
|
||||
*/
|
||||
place?: string;
|
||||
/**
|
||||
* Resolves a rough place from the user's IP address, when that is switched on.
|
||||
*
|
||||
* Injected for the same reason as `place`, and absent when IP geolocation is
|
||||
* off, which is what keeps this module from being able to reach the network
|
||||
* for a location on its own.
|
||||
*/
|
||||
ipPlace?: (options: { signal?: AbortSignal }) => Promise<GeoLookup>;
|
||||
}
|
||||
|
||||
export interface ToolResult {
|
||||
@@ -45,7 +61,7 @@ export const TOOL_DEFINITIONS: ToolDefinition[] = [
|
||||
function: {
|
||||
name: "get_location",
|
||||
description:
|
||||
"Get the user's approximate location context: timezone, UTC offset, locale, region, and city. Use it for local questions such as weather, business hours, or travel timing. This is derived from the device timezone, not GPS, so treat it as a region rather than an address.",
|
||||
"Get where the user is, for local questions such as weather, business hours, or travel timing. Returns the place they configured in preferences, plus the timezone, UTC offset, locale and region. Without a configured place this is region-level, not an address, and the timezone area it reports is the timezone's own name rather than the user's city, so never treat it as one.",
|
||||
parameters: { type: "object", properties: {}, required: [] },
|
||||
},
|
||||
},
|
||||
@@ -140,7 +156,7 @@ export async function executeTool(
|
||||
return { content: describeNow(context.now), isError: false };
|
||||
|
||||
case "get_location":
|
||||
return { content: describeLocation(context.now), isError: false };
|
||||
return { content: await describeLocation(context), isError: false };
|
||||
|
||||
case "web_search":
|
||||
return await runWebSearch(args, context);
|
||||
@@ -172,18 +188,68 @@ function describeNow(now?: Date): string {
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function describeLocation(now?: Date): string {
|
||||
const date = now ?? new Date();
|
||||
const location = getLocationContext(date);
|
||||
async function describeLocation(context: ToolContext): Promise<string> {
|
||||
const date = context.now ?? new Date();
|
||||
const configured = context.place?.trim() || undefined;
|
||||
|
||||
const lines = [
|
||||
// A place the user typed wins outright, and while it is set there is no reason
|
||||
// to hand their IP address to anyone.
|
||||
const lookup =
|
||||
configured || !context.ipPlace ? undefined : await context.ipPlace({ signal: context.signal });
|
||||
const ip = lookup?.status === "ok" ? lookup.place : undefined;
|
||||
|
||||
const location = getLocationContext(date, undefined, configured ?? ip?.place);
|
||||
|
||||
const lines: string[] = [];
|
||||
|
||||
// The place comes first because it is the only answer that is about where the
|
||||
// user actually is, rather than about which zone the clock is set to.
|
||||
if (location.place) lines.push(`Place: ${location.place}`);
|
||||
|
||||
lines.push(
|
||||
`Timezone: ${location.timezone}`,
|
||||
`UTC offset: ${location.utcOffset}`,
|
||||
`Locale: ${location.locale}`,
|
||||
];
|
||||
if (location.city) lines.push(`City: ${location.city}`);
|
||||
if (location.region) lines.push(`Region: ${location.region}`);
|
||||
lines.push("Note: derived from the device timezone. It indicates region, not a precise address.");
|
||||
);
|
||||
|
||||
if (location.region) {
|
||||
lines.push(`Region: ${location.region}${location.regionCode ? ` (${location.regionCode})` : ""}`);
|
||||
}
|
||||
if (location.timezoneArea) {
|
||||
lines.push(`Timezone area: ${location.timezoneArea}`);
|
||||
}
|
||||
|
||||
// A mismatch between the clock and the IP is worth surfacing: it usually means
|
||||
// a VPN or a device still set to somewhere the user has left.
|
||||
if (ip?.timezone && ip.timezone !== location.timezone) {
|
||||
lines.push(`IP-reported timezone: ${ip.timezone}, which differs from the device timezone`);
|
||||
}
|
||||
|
||||
if (configured) {
|
||||
lines.push(
|
||||
"Note: Place is the user's own setting, so prefer it for anything that depends on exactly where they are.",
|
||||
);
|
||||
} else if (ip) {
|
||||
lines.push(
|
||||
`Note: Place came from an IP address lookup via ${ip.source}, so treat it as a rough guess rather than the user's city. An IP address locates where the request left the ISP's network, which can be hundreds of kilometres away, and it can be out of date if they have moved since.`,
|
||||
);
|
||||
} else if (lookup?.status === "failed") {
|
||||
// Deliberately not the "no Location is set" wording below. A failed lookup
|
||||
// and an unset preference are different problems, and reporting the wrong
|
||||
// one sends the user looking in the wrong place. This is also the only
|
||||
// channel that reaches them: the model paraphrases, so it is told to say so.
|
||||
lines.push(
|
||||
`Note: IP geolocation is switched on but the lookup failed (${lookup.message}), so no place could be resolved. Tell the user this plainly. Setting Location in preferences avoids the lookup and the network entirely. Everything below is derived from the device timezone and locale, so it is region-level.`,
|
||||
);
|
||||
} else {
|
||||
lines.push(
|
||||
"Note: no Location is set in preferences. Everything above is derived from the device timezone and locale, so it is accurate to a region and not to a city. Ask which city they are in when an exact place matters.",
|
||||
);
|
||||
}
|
||||
|
||||
if (location.timezoneArea) {
|
||||
lines.push("The timezone area is the timezone's own name, not the user's city.");
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
@@ -37,8 +37,53 @@ export interface ApiMessage {
|
||||
tool_calls?: ToolCall[];
|
||||
/** Only set on `role: "tool"` messages, naming the tool that produced this content. */
|
||||
tool_name?: string;
|
||||
/**
|
||||
* Base64 image data, for vision models.
|
||||
*
|
||||
* Ollama only reads these on a `user` message (`gpt-oss` style tool roles have
|
||||
* no images). They are the one genuinely large thing in a request, so they are
|
||||
* attached to the question they belong to and never replayed later, the same
|
||||
* way tool output is not replayed.
|
||||
*/
|
||||
images?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* A file the user attached to a question.
|
||||
*
|
||||
* Only the path is stored. A screenshot runs to megabytes, and
|
||||
* `MAX_STORED_CHARS_PER_TURN` exists precisely so one turn cannot dominate
|
||||
* LocalStorage, so the bytes stay on disk and are read when the turn is sent.
|
||||
*/
|
||||
export interface Attachment {
|
||||
/** Absolute path to the image on disk. */
|
||||
path: string;
|
||||
/** Sent to Ollama next to the base64 payload. */
|
||||
mime: string;
|
||||
/**
|
||||
* Pixel dimensions, read from the file's header.
|
||||
*
|
||||
* Kept because the transcript has to lay the preview out long after the file
|
||||
* was read, and Vicinae sizes an image's container to the request rather than
|
||||
* to the image, which leaves dead space unless the request has the right
|
||||
* aspect ratio. Absent for a format whose header we do not parse.
|
||||
*/
|
||||
width?: number;
|
||||
height?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* An attachment that has been read and is ready to send, held in the view while
|
||||
* the user types their question.
|
||||
*
|
||||
* The base64 payload lives here and nowhere else. It is needed for exactly one
|
||||
* request, so it is deliberately not part of `Attachment` and never reaches
|
||||
* storage.
|
||||
*/
|
||||
export interface PendingAttachment extends Attachment {
|
||||
/** Base64 of the image, with no `data:` URL prefix. Ollama wants it bare. */
|
||||
base64: string;
|
||||
}
|
||||
/** What a tool actually returned during a turn, kept for display and debugging. */
|
||||
export interface ToolCallRecord {
|
||||
name: string;
|
||||
@@ -58,6 +103,8 @@ export interface ChatTurn {
|
||||
/** Set when the turn failed, so a stored session can show what went wrong. */
|
||||
error?: string;
|
||||
toolCalls: ToolCallRecord[];
|
||||
/** Images the user attached, kept for the transcript. The bytes are not stored. */
|
||||
attachments: Attachment[];
|
||||
}
|
||||
|
||||
export interface Session {
|
||||
@@ -104,4 +151,6 @@ export interface ThreadMessage {
|
||||
error?: string;
|
||||
/** Activity line shown while this message is streaming, e.g. "Searching the web". */
|
||||
status?: string;
|
||||
/** Images attached to this question. Only ever non-empty on a user message. */
|
||||
attachments: Attachment[];
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import { LocalStorage } from "@vicinae/api";
|
||||
import { modelCapabilities, supportsVision } from "./ollama";
|
||||
|
||||
/**
|
||||
* The cached answer to "can this model see images?".
|
||||
*
|
||||
* `POST /api/show` is a round trip, and the answer only changes when the user
|
||||
* switches model. Without a cache the chat view would ask again on every mount
|
||||
* and the snip action would flicker in and out of the action panel while the
|
||||
* request was in flight.
|
||||
*
|
||||
* `"unknown"` is deliberately distinct from `"no"`. A failed request is not
|
||||
* evidence that a model cannot see, so the caller decides how much to trust it
|
||||
* and a failure is never written to the cache.
|
||||
*/
|
||||
|
||||
/** One key per host and model, so switching model invalidates by construction. */
|
||||
const VISION_CACHE_PREFIX = "lam.vision.";
|
||||
|
||||
export type VisionSupport = "yes" | "no" | "unknown";
|
||||
|
||||
export interface VisionCheck {
|
||||
host: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
signal?: AbortSignal;
|
||||
/** Ignores the cache. Used when the answer might have gone stale. */
|
||||
refresh?: boolean;
|
||||
}
|
||||
|
||||
function cacheKey(host: string, model: string): string {
|
||||
return `${VISION_CACHE_PREFIX}${host}|${model}`;
|
||||
}
|
||||
|
||||
export async function visionSupport(check: VisionCheck): Promise<VisionSupport> {
|
||||
const key = cacheKey(check.host, check.model);
|
||||
|
||||
if (!check.refresh) {
|
||||
const cached = await LocalStorage.getItem<boolean>(key).catch(() => undefined);
|
||||
if (cached !== undefined) return cached ? "yes" : "no";
|
||||
}
|
||||
|
||||
try {
|
||||
const capable = supportsVision(
|
||||
await modelCapabilities({
|
||||
host: check.host,
|
||||
apiKey: check.apiKey,
|
||||
model: check.model,
|
||||
signal: check.signal,
|
||||
}),
|
||||
);
|
||||
|
||||
await LocalStorage.setItem(key, capable).catch(() => undefined);
|
||||
return capable ? "yes" : "no";
|
||||
} catch {
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Drops every cached answer.
|
||||
*
|
||||
* A model can gain vision without its name changing (a re-pull, a different
|
||||
* tag on the same account), so `Set Up Lam Agent` calls this rather than
|
||||
* leaving the user pinned to a stale "no".
|
||||
*/
|
||||
export async function forgetVisionSupport(): Promise<void> {
|
||||
const items = await LocalStorage.allItems().catch(() => ({}));
|
||||
|
||||
await Promise.all(
|
||||
Object.keys(items)
|
||||
.filter((key) => key.startsWith(VISION_CACHE_PREFIX))
|
||||
.map((key) => LocalStorage.removeItem(key).catch(() => undefined)),
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user