diff --git a/resources/mcp-bridge.js b/resources/mcp-bridge.js index 305b7440..9f5d5fc2 100644 --- a/resources/mcp-bridge.js +++ b/resources/mcp-bridge.js @@ -350,7 +350,7 @@ const TOOLS = [ { name: 'screenshot_tab', description: - "Take a screenshot of a browser tab in this worktree at the viewport's CSS-pixel dimensions — so screenshot coords can be passed straight to click_tab. Returns a JPEG (quality 70) by default for context-efficiency; ask for PNG only when lossless matters. Screenshots are for visual verification, not for finding click targets — prefer get_tab_clickables for interaction. Tab id comes from list_browser_tabs.", + "Take a screenshot of a browser tab in this worktree at the viewport's CSS-pixel dimensions — so screenshot coords can be passed straight to click_tab. Returns a JPEG (quality 70) by default for context-efficiency; ask for PNG only when lossless matters. Screenshots are for visual verification, not for finding click targets — prefer get_tab_clickables for interaction. Also returns the path the capture was saved to: embed that path as markdown (`![](path)`) in your reply and Ness renders the screenshot inline in the chat, which is the clearest way to show the user a UI change you just made. Set the display size with a spec after a pipe in the alt text — `![login form|500](path)` for 500px wide, `![x|50%](path)` to fill half the column, `![x|400x300](path)` for both dimensions; with no spec it renders at a default size. Tab id comes from list_browser_tabs.", inputSchema: { type: 'object', properties: { @@ -851,9 +851,25 @@ async function handleToolCall(name, args) { if (!data) throw new Error(r && r.error ? r.error : 'screenshot failed') const mimeType = r.mimeType || (r.format === 'jpeg' ? 'image/jpeg' : 'image/png') - return { - content: [{ type: 'image', data, mimeType }] + const content = [{ type: 'image', data, mimeType }] + if (r.path) { + // One terse line, not a paragraph: this rides along with every + // screenshot (context cost + it shows in the tool card). How to use + // the path lives in the tool description, which is in the system + // prompt anyway. + content.push({ + type: 'text', + text: + 'Saved to ' + + r.path + + ' — embed as ![](' + + r.path + + ') to show it in your reply, or ![|600](' + + r.path + + ') to set its width.' + }) } + return { content } } if (name === 'get_tab_dom') { if (!args || !args.tab_id) throw new Error('tab_id is required') diff --git a/src/main/control-server.test.ts b/src/main/control-server.test.ts index f724f643..ca837aad 100644 --- a/src/main/control-server.test.ts +++ b/src/main/control-server.test.ts @@ -549,6 +549,14 @@ describe('control-server /browser/screenshot endpoint', () => { expect(r.json).toMatchObject({ data: 'AAAA', format: 'jpeg', mimeType: 'image/jpeg' }) }) + it('returns the on-disk path, hash-keyed so a repeat capture reuses the file', async () => { + captureResult = { data: 'AAAA', format: 'jpeg' } + const first = await get(`/browser/screenshot?tabId=${BROWSER_TAB}`) + expect(first.json.path).toMatch(/harness-attachments\/result-[0-9a-f]+\.jpg$/) + const second = await get(`/browser/screenshot?tabId=${BROWSER_TAB}`) + expect(second.json.path).toBe(first.json.path) + }) + it('surfaces the capture failure reason verbatim as a 500', async () => { captureResult = { error: 'tab viewport is 0x0 — nothing to capture' } const r = await get(`/browser/screenshot?tabId=${BROWSER_TAB}`) diff --git a/src/main/control-server.ts b/src/main/control-server.ts index 0cb6e50d..45d86864 100644 --- a/src/main/control-server.ts +++ b/src/main/control-server.ts @@ -11,6 +11,7 @@ import type { PRStatus } from '../shared/state/prs' import type { ChatDeliveryResult } from './chat-delivery' import type { CaptureResult } from './browser-manager-types' import { wrapAutomatedMessage } from '../shared/state/json-claude' +import { writeResultImage } from './json-claude-attachments' import { log } from './debug' export interface BrowserTabSummary { @@ -619,10 +620,23 @@ async function handleRequest( error: result?.error ?? 'capture failed: tab is no longer available' }) } + const mimeType = result.format === 'png' ? 'image/png' : 'image/jpeg' + // Spill the capture to disk and hand the path back, so the agent can + // embed the screenshot in its reply (`![](path)`) to show the user + // what it just verified. The write is content-hash keyed, so this is + // the same file the chat transcript's tool-result extractor derives + // from the image block — one copy, not two. + let path: string | undefined + try { + path = writeResultImage(result.data, mimeType) + } catch (err) { + log('browser', `screenshot spill-to-disk failed: ${String(err)}`) + } return sendJson(res, 200, { data: result.data, format: result.format, - mimeType: result.format === 'png' ? 'image/png' : 'image/jpeg', + mimeType, + path, // Kept for older MCP bridge versions. New bridges read `data`+`format`. pngBase64: result.format === 'png' ? result.data : undefined }) diff --git a/src/main/persistence.ts b/src/main/persistence.ts index 457d0113..37178c92 100644 --- a/src/main/persistence.ts +++ b/src/main/persistence.ts @@ -440,13 +440,15 @@ Ness also exposes embedded browser tabs — you can open a browser alongside the - navigate_tab, back_tab, forward_tab, reload_tab: drive the tab. - get_tab_clickables: returns a compact JSON snapshot of in-viewport interactive elements (buttons, links, inputs, [role=button|link|tab|menuitem|checkbox|radio|switch|option|combobox|searchbox|textbox], [tabindex], [contenteditable], [onclick]) — including elements inside open shadow roots. Each entry is {role, name, cx, cy, w, h} with the click center already computed. - click_tab, type_tab, scroll_tab, show_cursor: interact with the page — click at (x, y), type into the focused field, scroll, or just move the visible cursor overlay so the user can see what you're about to do. -- screenshot_tab: visual verification only. Returns JPEG quality 70 by default for context-efficiency; ask for format:'png' only when lossless matters. Screenshot dimensions match the CSS viewport, so coords observed in a screenshot can be passed straight to click_tab. +- screenshot_tab: visual verification. Returns JPEG quality 70 by default for context-efficiency; ask for format:'png' only when lossless matters. Screenshot dimensions match the CSS viewport, so coords observed in a screenshot can be passed straight to click_tab. The result also carries the path the capture was saved to. - set_tab_viewport: emulate a viewport size — this is how you check a page's mobile layout. The page lays out at exactly the size you ask for regardless of the user's pane size, and screenshots + clickables come back in that size. 600px wide or less also turns on mobile emulation (viewport meta tag, touch events, an iPhone user agent so UA-sniffing sites serve their mobile page). Call it with no width/height to go back to the pane size. Click targeting workflow: **prefer get_tab_clickables → match by role + name → call click_tab(cx, cy) for anything you want to click**. It's far cheaper than a screenshot + vision and far more reliable for real DOM targets. Reserve screenshot_tab for confirming a click had the visual effect you wanted, or for targets without accessible names (canvas/SVG/images). The clickables snapshot is in-viewport only and capped at 500 items — if the target isn't there, scroll_tab first, then re-snapshot. To type into a field: click_tab on it first to focus, then type_tab. type_tab also accepts a \`key\` argument (Enter, Tab, Backspace, ArrowDown, …) for submitting forms or navigating menus. Prefer these over blind curl/fetch — or shelling out to \`open \`, which launches the user's default browser outside Ness where you can't see the result — when you need to verify rendered UI, inspect a dev server, debug a page the user is looking at, or confirm your changes actually work in the browser. +Show your work with screenshots. Every screenshot_tab result comes back with the path it was saved to, and embedding an absolute image path as markdown (\`![](/path/to/shot.png)\`) renders it inline in the chat — so when you've changed something visual, end your reply with the screenshot that proves it rather than only describing it. This works for any image file on this machine, not just screenshot_tab captures. Size it with a spec after a pipe in the alt text: \`![caption|500](path)\` is 500px wide, \`![caption|50%](path)\` fills half the column, \`![caption|400x300](path)\` sets both dimensions, and no spec means a default size. Be selective: embed the one or two shots that show the result, not every frame you took while finding your way around. + Ness also exposes shell tabs for long-running processes — anything that wouldn't naturally exit within a few seconds (dev servers, watchers, \`tail -f\`, REPL-style tools, long builds). Drive them via the ness-control shell tools (scoped to this worktree only): - create_shell: spawn a shell tab, optionally with a command to run (\`zsh -ilc \`). Returns an id — keep it for later reads. diff --git a/src/renderer/components/JsonModeChat.tsx b/src/renderer/components/JsonModeChat.tsx index 0f82607f..d9a799d3 100644 --- a/src/renderer/components/JsonModeChat.tsx +++ b/src/renderer/components/JsonModeChat.tsx @@ -10,7 +10,7 @@ import { type MouseEvent as ReactMouseEvent, type ReactNode } from 'react' -import ReactMarkdown, { type Components } from 'react-markdown' +import ReactMarkdown, { defaultUrlTransform, type Components } from 'react-markdown' import { rehypeHighlightShared } from '../rehype-highlight-shared' import remarkGfm from 'remark-gfm' import { remarkInsight } from '../remark-insight' @@ -58,6 +58,11 @@ import { import { fuzzyMatch } from '../fuzzy' import { worktreeHandle } from '../../shared/state/worktrees' import { CLAUDE_MODELS } from '../../shared/agent-registry' +import { + parseImageAlt, + parseImageTitle, + resolveLocalImageSrc +} from '../../shared/chat-image-src' import { QUESTION_TOOL_NAME, type JsonClaudeAutomationSource, @@ -79,10 +84,79 @@ import { const REMARK_PLUGINS = [remarkGfm, remarkInsight] const REHYPE_PLUGINS = [rehypeHighlightShared, rehypeColorHex] +/** Markdown `![alt](…)` in a message. An agent that just took a browser + * screenshot can embed it in its prose to show the thing it built — the + * file is on the machine running the session, so it loads over the + * attachment-read IPC rather than as a page-origin URL. Remote srcs are + * left to the browser (react-markdown's urlTransform has already + * sanitized the protocol by the time we see them). + * + * Display size comes from a spec in the alt (`![panel|400](…)`) or the + * title (`![panel](… "=400x300")`) — see shared/chat-image-src. */ +function MarkdownImage({ + src, + alt, + title +}: { + src?: string + alt?: string + title?: string +}): JSX.Element | null { + const parsedAlt = parseImageAlt(alt) + const parsedTitle = parseImageTitle(title) + const local = resolveLocalImageSrc(src) + const dimensions = parsedAlt.size ?? parsedTitle.size ?? local?.size ?? undefined + if (local) { + return ( + + ) + } + if (!src) return null + return ( + {parsedAlt.alt} + ) +} + // react-markdown v10's Components type only lists known HTML tag names, // but the runtime accepts any string tag; cast to register our custom // element. -const MARKDOWN_COMPONENTS = { 'insight-card': InsightCard } as unknown as Components +const MARKDOWN_COMPONENTS = { + 'insight-card': InsightCard, + img: MarkdownImage +} as unknown as Components + +// react-markdown's default transform blanks any src whose protocol isn't +// http(s)/mailto/xmpp/irc, which includes the file:// form of a local +// screenshot. Let those through — MarkdownImage decides what's local, and +// a bare absolute path (the form we tell agents to use) already survives +// the default transform untouched. +function chatUrlTransform(url: string): string { + return /^file:\/\//i.test(url) ? url : defaultUrlTransform(url) +} // Matches color literals we want to swatch: // - #RRGGBB and #RRGGBBAA hex. 3-hex (#fff) and 4-hex (#ffff) are @@ -190,6 +264,7 @@ const MarkdownWithFind = memo(function MarkdownWithFind({ remarkPlugins={REMARK_PLUGINS} rehypePlugins={rehypePlugins} components={MARKDOWN_COMPONENTS} + urlTransform={chatUrlTransform} > {children} diff --git a/src/renderer/components/JsonModeChatImageThumb.tsx b/src/renderer/components/JsonModeChatImageThumb.tsx index bcba3d18..96210197 100644 --- a/src/renderer/components/JsonModeChatImageThumb.tsx +++ b/src/renderer/components/JsonModeChatImageThumb.tsx @@ -35,20 +35,30 @@ interface Props { /** 'square' crops to a 64px tile — right for pasted attachments, where * the thumbnail is an affordance rather than something to read. * 'wide' keeps the aspect ratio at 128px tall, for browser - * screenshots where a centre-crop would throw away the page. */ - shape?: 'square' | 'wide' + * screenshots where a centre-crop would throw away the page. + * 'inline' is for a screenshot the agent embedded in its prose: as + * large as the column allows, because the point is to look at it. */ + shape?: 'square' | 'wide' | 'inline' /** 'sm' is half the linear size of 'base' — used for the strip that * rides along in a collapsed tool card / tool group header, where the * thumbnail is a "there's a screenshot here" marker rather than - * something to read. */ + * something to read. Ignored for shape 'inline'. */ size?: 'base' | 'sm' + /** Alt text, when the embedder has one (markdown `![alt](path)`). */ + alt?: string + /** Explicit display size from the embed's size spec. Only honoured for + * shape 'inline' — the other shapes are fixed-size chrome. Width still + * can't exceed the column. */ + dimensions?: { width?: number; height?: number; widthPercent?: number } } export function JsonModeChatImageThumb({ path, mediaType, shape = 'square', - size = 'base' + size = 'base', + alt, + dimensions }: Props): JSX.Element { const [dataUrl, setDataUrl] = useState( CACHE.has(path) ? CACHE.get(path)! : null @@ -83,27 +93,57 @@ export function JsonModeChatImageThumb({ return () => window.removeEventListener('keydown', onKey) }, [showFull]) - const name = path.split('/').pop() || path + const name = alt || path.split('/').pop() || path const sm = size === 'sm' + // An explicit size replaces the inline default rather than adding to it, + // so `![x|800](…)` can go wider than the 24rem default cap. max-width + // still wins, so nothing overflows the column. + const sized = shape === 'inline' && dimensions ? dimensions : null + const sizedStyle = sized + ? { + width: sized.widthPercent + ? `${sized.widthPercent}%` + : sized.width + ? `${sized.width}px` + : undefined, + height: sized.height ? `${sized.height}px` : undefined + } + : undefined const boxClass = - shape === 'wide' - ? sm - ? 'h-16 w-24' - : 'h-32 w-48' - : sm - ? 'h-8 w-8' - : 'h-16 w-16' + shape === 'inline' + ? sized + ? 'max-w-full' + : 'h-40 w-full max-w-md' + : shape === 'wide' + ? sm + ? 'h-16 w-24' + : 'h-32 w-48' + : sm + ? 'h-8 w-8' + : 'h-16 w-16' const imgClass = - shape === 'wide' - ? `${sm ? 'h-16' : 'h-32'} w-auto max-w-full object-contain bg-app` - : sm - ? 'h-8 w-8 object-cover' - : 'h-16 w-16 object-cover' + shape === 'inline' + ? sized + ? 'max-w-full object-contain bg-app' + : // Unsized default: a glanceable 16rem-tall preview, click to + // zoom. Bigger than this and one screenshot owns the viewport. + 'max-h-64 max-w-lg w-auto object-contain bg-app' + : shape === 'wide' + ? `${sm ? 'h-16' : 'h-32'} w-auto max-w-full object-contain bg-app` + : sm + ? 'h-8 w-8 object-cover' + : 'h-16 w-16 object-cover' + // Reserve the requested box while loading / when the file is gone, so the + // transcript doesn't reflow once the bytes arrive. + const placeholderStyle = sized + ? { ...sizedStyle, height: sizedStyle?.height ?? '10rem' } + : undefined if (pending) { return (
) @@ -111,10 +151,14 @@ export function JsonModeChatImageThumb({ if (!dataUrl) { return (
+ {shape === 'inline' && ( + {path} + )}
) } @@ -129,6 +173,7 @@ export function JsonModeChatImageThumb({ {name} diff --git a/src/shared/chat-image-src.test.ts b/src/shared/chat-image-src.test.ts new file mode 100644 index 00000000..78348822 --- /dev/null +++ b/src/shared/chat-image-src.test.ts @@ -0,0 +1,180 @@ +import { describe, it, expect } from 'vitest' +import { + parseImageAlt, + parseImageSizeSpec, + parseImageTitle, + resolveLocalImageSrc +} from './chat-image-src' + +describe('resolveLocalImageSrc', () => { + it('resolves an absolute posix path by extension', () => { + expect(resolveLocalImageSrc('/tmp/harness-attachments/result-abc.jpg')).toEqual({ + path: '/tmp/harness-attachments/result-abc.jpg', + mediaType: 'image/jpeg' + }) + expect(resolveLocalImageSrc('/tmp/a.PNG')).toEqual({ + path: '/tmp/a.PNG', + mediaType: 'image/png' + }) + expect(resolveLocalImageSrc('/tmp/a.svg')?.mediaType).toBe('image/svg+xml') + }) + + it('trims surrounding whitespace', () => { + expect(resolveLocalImageSrc(' /tmp/a.png ')?.path).toBe('/tmp/a.png') + }) + + it('strips a file:// prefix and percent-decodes', () => { + expect(resolveLocalImageSrc('file:///tmp/my%20shot.png')).toEqual({ + path: '/tmp/my shot.png', + mediaType: 'image/png' + }) + expect(resolveLocalImageSrc('file://localhost/tmp/a.webp')).toEqual({ + path: '/tmp/a.webp', + mediaType: 'image/webp' + }) + }) + + it('leaves a malformed percent-escape alone rather than throwing', () => { + expect(resolveLocalImageSrc('file:///tmp/100%.png')?.path).toBe('/tmp/100%.png') + }) + + it('resolves windows drive paths', () => { + expect(resolveLocalImageSrc('C:/Users/x/shot.png')?.path).toBe('C:/Users/x/shot.png') + expect(resolveLocalImageSrc('C:\\Users\\x\\shot.png')?.mediaType).toBe('image/png') + }) + + it('declines remote and inline srcs', () => { + expect(resolveLocalImageSrc('https://example.com/a.png')).toBeNull() + expect(resolveLocalImageSrc('http://example.com/a.png')).toBeNull() + expect(resolveLocalImageSrc('data:image/png;base64,AAAA')).toBeNull() + expect(resolveLocalImageSrc('//example.com/a.png')).toBeNull() + }) + + it('declines relative paths — a transcript may be read off-machine', () => { + expect(resolveLocalImageSrc('shot.png')).toBeNull() + expect(resolveLocalImageSrc('./docs/shot.png')).toBeNull() + expect(resolveLocalImageSrc('../shot.png')).toBeNull() + }) + + it('declines paths without a recognized image extension', () => { + expect(resolveLocalImageSrc('/tmp/notes.txt')).toBeNull() + expect(resolveLocalImageSrc('/tmp/noext')).toBeNull() + expect(resolveLocalImageSrc('/tmp/')).toBeNull() + }) + + it('declines empty input', () => { + expect(resolveLocalImageSrc('')).toBeNull() + expect(resolveLocalImageSrc(' ')).toBeNull() + expect(resolveLocalImageSrc(null)).toBeNull() + expect(resolveLocalImageSrc(undefined)).toBeNull() + }) +}) + +describe('parseImageSizeSpec', () => { + it('reads a bare width as pixels', () => { + expect(parseImageSizeSpec('400')).toEqual({ width: 400 }) + expect(parseImageSizeSpec('400px')).toEqual({ width: 400 }) + expect(parseImageSizeSpec(' =400 ')).toEqual({ width: 400 }) + }) + + it('reads a width x height pair, either half optional', () => { + expect(parseImageSizeSpec('400x300')).toEqual({ width: 400, height: 300 }) + expect(parseImageSizeSpec('400X300')).toEqual({ width: 400, height: 300 }) + expect(parseImageSizeSpec('400×300')).toEqual({ width: 400, height: 300 }) + expect(parseImageSizeSpec('400x')).toEqual({ width: 400 }) + expect(parseImageSizeSpec('x300')).toEqual({ height: 300 }) + }) + + it('reads a percentage as a column fraction', () => { + expect(parseImageSizeSpec('50%')).toEqual({ widthPercent: 50 }) + }) + + it('clamps out-of-range values instead of rejecting them', () => { + expect(parseImageSizeSpec('99999')).toEqual({ width: 2000 }) + expect(parseImageSizeSpec('2')).toEqual({ width: 16 }) + expect(parseImageSizeSpec('999%')).toEqual({ widthPercent: 100 }) + expect(parseImageSizeSpec('1%')).toEqual({ widthPercent: 5 }) + }) + + it('declines anything that is not a size', () => { + expect(parseImageSizeSpec('x')).toBeNull() + expect(parseImageSizeSpec('wide')).toBeNull() + expect(parseImageSizeSpec('400em')).toBeNull() + expect(parseImageSizeSpec('')).toBeNull() + expect(parseImageSizeSpec(null)).toBeNull() + }) +}) + +describe('parseImageAlt', () => { + it('splits a size spec off the end', () => { + expect(parseImageAlt('the new panel|400')).toEqual({ + alt: 'the new panel', + size: { width: 400 } + }) + expect(parseImageAlt('panel | 50%')).toEqual({ + alt: 'panel', + size: { widthPercent: 50 } + }) + expect(parseImageAlt('|400')).toEqual({ alt: '', size: { width: 400 } }) + }) + + it('leaves alt text whole when the tail is not a size', () => { + expect(parseImageAlt('a | b | c')).toEqual({ alt: 'a | b | c', size: null }) + expect(parseImageAlt('no pipe here')).toEqual({ alt: 'no pipe here', size: null }) + expect(parseImageAlt(undefined)).toEqual({ alt: '', size: null }) + }) +}) + +describe('parseImageTitle', () => { + it('reads a leading-= title as a size and drops the tooltip', () => { + expect(parseImageTitle('=400x300')).toEqual({ + title: undefined, + size: { width: 400, height: 300 } + }) + }) + + it('keeps an ordinary title as a tooltip, numbers included', () => { + expect(parseImageTitle('Settings pane')).toEqual({ + title: 'Settings pane', + size: null + }) + expect(parseImageTitle('400')).toEqual({ title: '400', size: null }) + expect(parseImageTitle(undefined)).toEqual({ title: undefined, size: null }) + }) + + it('keeps a leading-= title that is not a valid size, rather than eating it', () => { + expect(parseImageTitle('=wide')).toEqual({ title: '=wide', size: null }) + }) +}) + +describe('resolveLocalImageSrc — size fragment', () => { + it('reads a size off a trailing fragment', () => { + expect(resolveLocalImageSrc('/tmp/a.png#w=200')).toEqual({ + path: '/tmp/a.png', + mediaType: 'image/png', + size: { width: 200 } + }) + expect(resolveLocalImageSrc('/tmp/a.png#200')?.size).toEqual({ width: 200 }) + expect(resolveLocalImageSrc('/tmp/a.png#width=400x300')?.size).toEqual({ + width: 400, + height: 300 + }) + }) + + it('leaves a # that is not a size alone', () => { + // No size to parse, so the fragment stays part of the name — and that + // name has no image extension, so it is not a local image. + expect(resolveLocalImageSrc('/tmp/a.png#section')).toBeNull() + expect(resolveLocalImageSrc('/tmp/weird#name.png')).toEqual({ + path: '/tmp/weird#name.png', + mediaType: 'image/png' + }) + }) + + it('omits size when there is no fragment', () => { + expect(resolveLocalImageSrc('/tmp/a.png')).toEqual({ + path: '/tmp/a.png', + mediaType: 'image/png' + }) + }) +}) diff --git a/src/shared/chat-image-src.ts b/src/shared/chat-image-src.ts new file mode 100644 index 00000000..82ae417b --- /dev/null +++ b/src/shared/chat-image-src.ts @@ -0,0 +1,174 @@ +// Markdown in a chat message can point at an image that lives on the +// machine running the agent rather than at a URL — a screenshot the agent +// just took, say. A plain can't load those: the renderer's +// origin is app:// (http://localhost in dev), so an absolute filesystem +// path resolves against the wrong root, and react-markdown's default URL +// transform strips file:// outright. So srcs that look like local files are +// routed through the readAttachmentImage IPC (which hands back base64 for a +// data URL) and everything else is left to render as an ordinary . + +const MEDIA_TYPE_BY_EXT: Record = { + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + gif: 'image/gif', + webp: 'image/webp', + bmp: 'image/bmp', + svg: 'image/svg+xml' +} + +export interface LocalImageSrc { + path: string + mediaType: string + /** Size parsed off a `#…` fragment on the src, e.g. + * `/tmp/a.png#w=200`. An alt or title spec outranks it. */ + size?: ImageSizeHint +} + +function isAbsolutePath(path: string): boolean { + return path.startsWith('/') || /^[a-z]:[\\/]/i.test(path) +} + +/** Resolve a markdown image src to an on-disk path, or null when it isn't + * one (http(s), data:, protocol-relative, bare relative paths, or a file + * extension we don't recognize as an image). + * + * Relative paths are deliberately NOT resolved against the worktree root. + * A chat transcript is rendered by clients that may not share a + * filesystem with the session, so only a src the author made explicitly + * absolute is treated as a file reference. */ +export function resolveLocalImageSrc(src: string | null | undefined): LocalImageSrc | null { + if (!src) return null + let path = src.trim() + if (!path) return null + + if (/^file:\/\//i.test(path)) { + const rest = path.replace(/^file:\/\/(localhost)?/i, '') + try { + path = decodeURIComponent(rest) + } catch { + path = rest + } + } else if ( + // The drive-letter check comes first: `C:/…` would otherwise read as a + // one-letter URL scheme. + !/^[a-z]:[\\/]/i.test(path) && + (/^[a-z][a-z0-9+.-]*:/i.test(path) || path.startsWith('//')) + ) { + // http:, https:, data:, protocol-relative — someone else's to load. + return null + } + + if (!isAbsolutePath(path)) return null + + // A trailing `#…` is only treated as a size when stripping it leaves a + // real image extension AND the fragment parses as a size — otherwise a + // filename that genuinely contains `#` keeps working. Without this, + // `/tmp/a.png#w=200` resolved to nothing and fell through to a broken + // that rendered as blank space, which is the worst of the + // available outcomes. + let size: ImageSizeHint | undefined + const hash = path.lastIndexOf('#') + if (hash > 0) { + const head = path.slice(0, hash) + if (imageMediaType(head)) { + const parsed = parseImageSizeSpec(path.slice(hash + 1)) + if (parsed) { + path = head + size = parsed + } + } + } + + const mediaType = imageMediaType(path) + if (!mediaType) return null + return { path, mediaType, ...(size ? { size } : {}) } +} + +function imageMediaType(path: string): string | undefined { + const ext = /\.([a-z0-9]+)$/i.exec(path)?.[1]?.toLowerCase() + return ext ? MEDIA_TYPE_BY_EXT[ext] : undefined +} + +// Display size. CommonMark has no syntax for it and raw HTML is off in chat +// (rehype-raw would let any model-authored markup into the renderer), so the +// size rides in the two places an image already carries free text: the alt +// after a pipe (`![panel|400](…)`, Obsidian's spelling) and the title when +// it starts with `=` (`![panel](… "=400x300")`, markdown-it-imsize's). Both +// are prior art, so whichever an agent reaches for works. + +/** Width is px unless `widthPercent` is set, in which case it's a + * percentage of the available column. Height is always px. */ +export interface ImageSizeHint { + width?: number + height?: number + widthPercent?: number +} + +// Below ~16px there's nothing to look at; above 2000 the column clamps it +// anyway. Bounds exist so a bogus spec can't blow out the layout. +const MIN_IMAGE_PX = 16 +const MAX_IMAGE_PX = 2000 + +function clampPx(n: number): number | undefined { + if (!Number.isFinite(n)) return undefined + return Math.min(MAX_IMAGE_PX, Math.max(MIN_IMAGE_PX, Math.round(n))) +} + +/** Parse a size spec: `400`, `400x300`, `x300` (height only), `50%`. A + * leading `=` is tolerated. Returns null when it isn't one. */ +export function parseImageSizeSpec(spec: string | null | undefined): ImageSizeHint | null { + if (!spec) return null + // `=400`, `w=400`, `width=400` all mean the same thing — agents reach for + // each of them. + const s = spec.trim().replace(/^(?:w|width)?=/i, '').trim() + if (!s) return null + + const pct = /^(\d{1,3})%$/.exec(s) + if (pct) { + const p = Math.min(100, Math.max(5, parseInt(pct[1], 10))) + return { widthPercent: p } + } + const pair = /^(\d{1,6})?\s*[x×]\s*(\d{1,6})?$/i.exec(s) + if (pair && (pair[1] || pair[2])) { + const width = pair[1] ? clampPx(parseInt(pair[1], 10)) : undefined + const height = pair[2] ? clampPx(parseInt(pair[2], 10)) : undefined + if (width === undefined && height === undefined) return null + return { ...(width !== undefined ? { width } : {}), ...(height !== undefined ? { height } : {}) } + } + const only = /^(\d{1,6})(?:px)?$/i.exec(s) + if (only) { + const width = clampPx(parseInt(only[1], 10)) + return width === undefined ? null : { width } + } + return null +} + +/** Split a size spec off the end of alt text: `panel|400` → alt `panel`, + * size 400px wide. Alt text that merely contains a pipe (a table row + * pasted into alt, say) is left whole. */ +export function parseImageAlt(alt: string | null | undefined): { + alt: string + size: ImageSizeHint | null +} { + const raw = alt ?? '' + const pipe = raw.lastIndexOf('|') + if (pipe === -1) return { alt: raw, size: null } + const size = parseImageSizeSpec(raw.slice(pipe + 1)) + if (!size) return { alt: raw, size: null } + return { alt: raw.slice(0, pipe).trim(), size } +} + +/** A title is only read as a size when it starts with `=`, so ordinary + * tooltip titles — including numeric ones — keep working. Returns the + * size and the title that should still render as a tooltip. */ +export function parseImageTitle(title: string | null | undefined): { + title: string | undefined + size: ImageSizeHint | null +} { + const raw = title ?? '' + if (!raw.trim().startsWith('=')) return { title: raw || undefined, size: null } + const size = parseImageSizeSpec(raw) + if (!size) return { title: raw, size: null } + return { title: undefined, size } +}