adrianbonpin/lam-agent-vicinae
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: snip a region into chat, and make links clickable

Browse Source 
Two changes: screenshot input gated on the model really reporting vision,
and a fix for URLs that arrived as code.

Links
- A fence whose body is all URLs, a lone `https://...` inline span, and a
  line that is only a URL are rewritten into [url](url) markdown links.
  Vicinae attaches only the table and strikethrough GFM extensions, so
  autolink is off and a bare URL is dead text; clickability comes from
  MarkdownModel turning CMARK_NODE_LINK into an <a href>. A fenced URL was
  a dark panel with a Copy button and no way to open it.
- promoteStandaloneCodeSpans was itself promoting a lone URL span into a
  fence, so it made this worse. Link restoration now runs first.
- The default prompt says never to fence a URL.
- Anything that is not one of those three shapes is copied through, so a
  curl invocation containing an address is never rewritten.

Screenshots
- capture.ts shells out to an auto-detected tool: grimblast, grim + slurp,
  spectacle, maim, scrot, flameshot, or screencapture on macOS. There is
  no screenshot call in @vicinae/api and Clipboard moves no binary, so the
  extension owns the capture and reads the file it wrote rather than
  guessing at the clipboard. Cancel is detected by whether an image landed
  on disk, because the tools disagree on exit codes.
- In chat: "Snip a Region" in the action panel (cmd+shift+S) attaches to
  the current conversation. Standalone: "Snip to Lam" starts a new one.
- vision.ts caches the /api/show capability answer per host and model. A
  failed check is "unknown", never "no", so a dropped connection is not
  reported as a fact about the user's model. Actions still fail closed.
- Only the path is stored, base64 is read at send time, and the image goes
  on its own turn only. Capture is never a model-callable tool.

Verified: 244 tests, tsc, eslint, and vici build (both entrypoints).
This commit is contained in:
adrianbonpin
2026-10-04 15:13:15 +08:00
parent f5d1ced4d6
commit 5e083d8142
29 changed files with 1744 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/components/ChatView.tsx
+162 -17
View File
@@ -1,10 +1,13 @@
import { Action, ActionPanel, Icon, List } from "@vicinae/api";
import { useCallback, useEffect, useMemo, useState } from "react";
import { useChat } from "../hooks/useChat";
import { useSnip } from "../hooks/useSnip";
import { useVisionSupport } from "../hooks/useVision";
import { buildThreadMarkdown, type TranscriptOptions } from "../markdown";
import { getLamPreferences } from "../preferences";
import { deleteSession, listSessions, loadSession, summarizeSession } from "../sessions";
import { buildThread } from "../thread";import type { Session, SessionSummary } from "../types";
import { buildThread } from "../thread";
import type { PendingAttachment, Session, SessionSummary } from "../types";
import { recencyGroup, truncate } from "../utils";
/**
@@ -48,11 +51,21 @@ import { recencyGroup, truncate } from "../utils";
const EMPTY_TRANSCRIPT = "**Ask Lam anything**\n\nType a question in the search bar and press Return.";
/** Sent when a snip is attached and Return is pressed before anything is typed. */
const DEFAULT_IMAGE_PROMPT = "What is in this image?";
/** The reminder shown while a snip is captured but not yet sent. */
const PENDING_NOTE = "_Screenshot attached. Ask a question about it, or press Return to send it as it is._";
export interface ChatViewProps {
/** Opens a specific conversation instead of resuming the latest one. */
initialSession?: Session;
/** Text handed over by a deeplink or fallback command. */
initialPrompt?: string;
/** A snip that has already been captured, used when a view is pushed with one. */
initialAttachment?: PendingAttachment;
/** Captures a snip on mount, which is what the standalone command does. */
startWithSnip?: boolean;
}
interface SessionRow {
@@ -62,14 +75,28 @@ interface SessionRow {
isActive: boolean;
}
export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
export function ChatView({ initialSession, initialPrompt, initialAttachment, startWithSnip }: ChatViewProps) {
const preferences = useMemo(() => getLamPreferences(), []);
const chat = useChat({ preferences, initialSession });
const { replaceSession } = chat;
const { isSniping, snip } = useSnip();
const {
support: vision,
canSeeImages,
isChecking: isCheckingVision,
recheck,
} = useVisionSupport(preferences);
const [input, setInput] = useState(initialPrompt ?? "");
const [summaries, setSummaries] = useState<SessionSummary[] | undefined>(undefined);
const [resumed, setResumed] = useState(Boolean(initialSession));
// A snip command starts a new chat on purpose, so the idle-resume below is
// skipped and the view must not wait for it.
const [resumed, setResumed] = useState(Boolean(initialSession) || Boolean(startWithSnip));
// A captured image is held here until the user says what they want to know
// about it. Nothing is sent to the model on capture alone.
const [pending, setPending] = useState<PendingAttachment | undefined>(initialAttachment);
const [snipNote, setSnipNote] = useState<string | undefined>(undefined);
const activeId = chat.session.id;
@@ -89,9 +116,13 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
* was used recently. Past the configured idle window it starts a fresh chat
* instead. The previous conversation is never removed, it just drops down the
* sidebar. Setting the preference to "Never" keeps the last chat open forever.
*
* The snip command opts out entirely: it exists to start a new chat about one
* screenshot, and quietly continuing the previous conversation would bury the
* image in an unrelated thread.
*/
useEffect(() => {
if (initialSession) return;
if (initialSession || startWithSnip) return;
let cancelled = false;
void (async () => {
@@ -113,7 +144,7 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
return () => {
cancelled = true;
};
}, [initialSession, preferences, replaceSession]);
}, [initialSession, startWithSnip, preferences, replaceSession]);
const thread = useMemo(
() =>
@@ -131,11 +162,45 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
[preferences.showToolCalls, preferences.showReasoning],
);
const threadMarkdown = useMemo(
() =>
thread.length === 0 ? EMPTY_TRANSCRIPT : buildThreadMarkdown(thread, transcriptOptions),
[thread, transcriptOptions],
);
/**
* The transcript, preceded by any notes about this view's own state.
*
* Notes go first because the transcript reads newest-first: the top of the
* panel is the text directly under the search bar, which is where the user is
* already looking.
*/
const threadMarkdown = useMemo(() => {
const notes: string[] = [];
// Only the standalone command has to explain a model that cannot see; inside
// the chat the action is simply absent. A failed check is worded separately
// from a text-only model, because saying "your model cannot see" when the
// request merely failed is a claim about the user's setup that is not true.
if (startWithSnip && !isCheckingVision && vision !== "yes") {
notes.push(
vision === "no"
? `**${preferences.model} cannot see images.** Choose a model with vision support in preferences, and this command becomes a one-key snip.`
: `**Could not check whether ${preferences.model} can see images.** Use Check Again below, or snip from the chat's action panel once the connection is back.`,
);
}
if (snipNote) notes.push(snipNote);
if (pending) notes.push(PENDING_NOTE);
const transcript =
thread.length === 0 ? EMPTY_TRANSCRIPT : buildThreadMarkdown(thread, transcriptOptions);
return [...notes, transcript].join("\n\n");
}, [
startWithSnip,
isCheckingVision,
vision,
preferences.model,
snipNote,
pending,
thread,
transcriptOptions,
]);
const rows: SessionRow[] = useMemo(() => {
const base = summaries ?? [];
@@ -194,15 +259,53 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
[activeId, chat, reload],
);
/**
* Captures a region and holds it until a question is sent with it.
*
* A cancelled selection deliberately says nothing: the user pressed Escape and
* telling them so is noise. A failed capture has to explain itself, because
* the usual cause is a capture tool that is not installed.
*/
const takeSnip = useCallback(async () => {
setSnipNote(undefined);
const outcome = await snip();
if (outcome.status === "captured") setPending(outcome.attachment);
else if (outcome.status === "failed")
setSnipNote(`**Could not take a screenshot:** ${outcome.message}`);
}, [snip]);
// The standalone command captures on arrival. Waiting for the capability check
// matters here: capturing a screenshot only to be told the model cannot read
// it would throw away the selection the user just made.
useEffect(() => {
if (!startWithSnip || !canSeeImages || initialAttachment) return;
void takeSnip();
}, [startWithSnip, canSeeImages, initialAttachment, takeSnip]);
/** The snip a "show me the screenshot" action would open: the pending one, else the newest. */
const latestAttachment =
pending ??
[...chat.session.turns].reverse().find((turn) => turn.attachments.length > 0)?.attachments[0];
const send = useCallback(
(text: string) => {
chat.ask(text);
const typed = text.trim();
// With a snip attached, Return has to send even before anything is typed,
// otherwise the only way to ask about an image is to type out a question
// the image already implies.
const prompt = typed || (pending ? DEFAULT_IMAGE_PROMPT : "");
if (!prompt) return;
chat.ask(prompt, pending);
setPending(undefined);
setSnipNote(undefined);
setInput("");
},
[chat.ask],
[chat.ask, pending],
);
const hasPrompt = input.trim().length > 0;
const hasPrompt = input.trim().length > 0 || pending !== undefined;
const stopAction = <Action title="Stop Generating" icon={Icon.Stop} onAction={chat.cancel} />;
const sendAction = <Action title="Send" icon={Icon.SpeechBubble} onAction={() => send(input)} />;
@@ -216,10 +319,40 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
/>
);
/**
* Present only when Ollama has confirmed the model reports `vision`.
*
* Offering it otherwise would let the user frame a careful screenshot and
* then have a model that cannot see answer as though it had, which is a worse
* failure than the action not being there.
*/
const snipAction = canSeeImages ? (
<Action
title="Snip a Region"
icon={Icon.Screenshot}
shortcut={{ key: "s", modifiers: ["cmd", "shift"] }}
onAction={() => void takeSnip()}
/>
) : null;
// A screenshot is a file, so it opens in the system image viewer. Without this
// the only way to see what was sent would be to find it on disk by hand.
const openSnipAction = latestAttachment ? (
<Action.Open title="Open Screenshot" icon={Icon.Image} target={latestAttachment.path} />
) : null;
// Only the standalone command reaches a state with nowhere to go: a failed
// capability check leaves it with no snip and no action that would start one.
const checkAgainAction =
startWithSnip && !isCheckingVision && vision === "unknown" ? (
<Action title="Check Again" icon={Icon.ArrowClockwise} onAction={recheck} />
) : null;
/**
* The primary action adapts to the search bar. With text in it, Return sends;
* with it empty (where sending would be a no-op anyway), Return opens the
* highlighted conversation instead.
* highlighted conversation instead. A pending snip counts as text, so Return
* sends it rather than navigating away.
*/
const rowActions = (row: SessionRow) => {
const openAction = (
@@ -231,6 +364,9 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
{chat.isStreaming ? stopAction : hasPrompt ? sendAction : openAction}
{hasPrompt ? openAction : null}
{newChatAction}
{snipAction}
{openSnipAction}
{checkAgainAction}
<Action
title="Delete Conversation"
icon={Icon.Trash}
@@ -248,13 +384,17 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
return <List isLoading navigationTitle="Lam" searchBarPlaceholder="Loading conversations…" />;
}
const busy = chat.isStreaming || isSniping;
const placeholder = chat.status
? `${chat.status}. Press ⏎ to stop`
: "Lam is thinking. Press ⏎ to stop";
const idlePlaceholder = pending ? "Ask about the screenshot…" : "Ask Lam anything…";
return (
<List
isLoading={chat.isStreaming}
isLoading={busy}
isShowingDetail
filtering={false}
searchText={input}
@@ -262,11 +402,16 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
// Raycast puts the conversation title in a header; Vicinae's nearest
// equivalent is the status-bar title, so the open chat is named there too.
navigationTitle={truncate(chat.session.title, 40)}
searchBarPlaceholder={chat.isStreaming ? placeholder : "Ask Lam anything…"}
searchBarPlaceholder={
chat.isStreaming ? placeholder : isSniping ? "Select a region…" : idlePlaceholder
}
actions={
<ActionPanel>
{chat.isStreaming ? stopAction : sendAction}
{newChatAction}
{snipAction}
{openSnipAction}
{checkAgainAction}
</ActionPanel>
}
>
@@ -281,7 +426,7 @@ export function ChatView({ initialSession, initialPrompt }: ChatViewProps) {
// Title only. Raycast's sidebar shows nothing else, and stacking a
// subtitle plus a timestamp on a 38px row elided the title down to
// a few characters. The check icon is what marks the open chat.
detail={<List.Item.Detail markdown={threadMarkdown} isLoading={chat.isStreaming} />}
detail={<List.Item.Detail markdown={threadMarkdown} isLoading={busy} />}
actions={rowActions(row)}
/>
))}