diff --git a/src/agents/pi-embedded-subscribe.tools.ts b/src/agents/pi-embedded-subscribe.tools.ts index ea39833..e40f617 100644 --- a/src/agents/pi-embedded-subscribe.tools.ts +++ b/src/agents/pi-embedded-subscribe.tools.ts @@ -104,9 +104,10 @@ export function sanitizeToolResult(result: unknown): unknown { if (type === "image") { const data = typeof entry.data === "string" ? entry.data : undefined; const bytes = data ? data.length : undefined; - const cleaned = { ...entry }; - delete cleaned.data; - return { ...cleaned, bytes, omitted: true }; + // Retain `data` so the TUI can render images inline. The `omitted: true` + // flag tells the model layer to skip this block when building the prompt, + // so the base64 never inflates the token context. + return { ...entry, bytes, omitted: true }; } return entry; }); diff --git a/src/tui/clipboard-image.ts b/src/tui/clipboard-image.ts index edefa66..ff4c1fc 100644 --- a/src/tui/clipboard-image.ts +++ b/src/tui/clipboard-image.ts @@ -13,15 +13,23 @@ export type ClipboardImage = { * Attempts to capture an image from the system clipboard. * Returns the image as base64 + mimeType, or null if no image is on the clipboard. * - * Supports macOS (pbpaste/osascript) and Linux (xclip). + * Supports macOS (osascript), Linux (xclip or wl-paste on Wayland), and + * Windows (PowerShell + System.Windows.Forms). */ export function captureClipboardImage(): ClipboardImage | null { if (process.platform === "darwin") { return captureMacOS(); } if (process.platform === "linux") { + // Wayland: prefer wl-paste when WAYLAND_DISPLAY is set. + if (process.env.WAYLAND_DISPLAY) { + return captureWayland() ?? captureLinux(); + } return captureLinux(); } + if (process.platform === "win32") { + return captureWindows(); + } return null; } @@ -124,3 +132,70 @@ function captureLinux(): ClipboardImage | null { return null; } } + +/** Capture clipboard image on Wayland via wl-paste. */ +function captureWayland(): ClipboardImage | null { + try { + // Check available MIME types + const types = execSync("wl-paste -l", { + encoding: "utf-8", + timeout: 3000, + stdio: ["pipe", "pipe", "pipe"], + }); + if (!types.includes("image/png")) { + return null; + } + } catch { + return null; + } + + try { + const buf = execSync("wl-paste -t image/png", { + timeout: 5000, + stdio: ["pipe", "pipe", "pipe"], + maxBuffer: 50 * 1024 * 1024, + }); + if (buf.length === 0) { + return null; + } + return { + base64: buf.toString("base64"), + mimeType: "image/png", + }; + } catch { + return null; + } +} + +/** Capture clipboard image on Windows via PowerShell + System.Windows.Forms. */ +function captureWindows(): ClipboardImage | null { + // PowerShell script: check if clipboard has an image, get it as PNG bytes, + // and write base64 to stdout. + const script = [ + "Add-Type -AssemblyName System.Windows.Forms", + "Add-Type -AssemblyName System.Drawing", + "$img = [System.Windows.Forms.Clipboard]::GetImage()", + "if ($null -eq $img) { exit 1 }", + "$ms = New-Object System.IO.MemoryStream", + "$img.Save($ms, [System.Drawing.Imaging.ImageFormat]::Png)", + "[Console]::OpenStandardOutput().Write($ms.ToArray(), 0, $ms.Length)", + ].join("; "); + + try { + const buf = execSync(`powershell -NoProfile -Command "${script}"`, { + timeout: 5000, + stdio: ["pipe", "pipe", "pipe"], + maxBuffer: 50 * 1024 * 1024, + windowsHide: true, + }); + if (buf.length === 0) { + return null; + } + return { + base64: buf.toString("base64"), + mimeType: "image/png", + }; + } catch { + return null; + } +} diff --git a/src/tui/components/tool-execution.ts b/src/tui/components/tool-execution.ts index df48c41..200c679 100644 --- a/src/tui/components/tool-execution.ts +++ b/src/tui/components/tool-execution.ts @@ -1,4 +1,4 @@ -import { Box, Container, Markdown, Spacer, Text } from "@earendil-works/pi-tui"; +import { Box, Container, Image, Markdown, Spacer, Text } from "@earendil-works/pi-tui"; import { formatToolDetail, resolveToolDisplay } from "../../agents/tool-display.js"; import { renderDiff, parseDiffStats, formatDiffStatsLine } from "../diff-renderer.js"; import { linkifyFilePaths } from "../linkify-paths.js"; @@ -11,6 +11,8 @@ type ToolResultContent = { mimeType?: string; bytes?: number; omitted?: boolean; + /** Base64-encoded image data, retained for inline rendering. */ + data?: string; }; type ToolResult = { @@ -61,6 +63,7 @@ export class ToolExecutionComponent extends Container { private header: Text; private argsLine: Text; private output: Markdown; + private imageContainer: Container; private toolName: string; private args: unknown; private result?: ToolResult; @@ -78,11 +81,13 @@ export class ToolExecutionComponent extends Container { this.output = new Markdown("", 0, 0, markdownTheme, { color: (line) => theme.toolOutput(line), }); + this.imageContainer = new Container(); this.addChild(new Spacer(1)); this.addChild(this.box); this.box.addChild(this.header); this.box.addChild(this.argsLine); this.box.addChild(this.output); + this.box.addChild(this.imageContainer); this.refresh(); } @@ -145,9 +150,29 @@ export class ToolExecutionComponent extends Container { // Hidden: only the header line, no output body. if (this.sectionState === "hidden") { this.output.setText(""); + this.imageContainer.clear(); return; } + // Render inline images from image content blocks that carry base64 data. + // The Image component handles terminal capability detection internally + // (Kitty/iTerm2) and falls back to a text placeholder when unsupported. + this.imageContainer.clear(); + if (this.result?.content && !this.isPartial) { + for (const entry of this.result.content) { + if (entry.type === "image" && entry.data) { + const mime = entry.mimeType ?? "image/png"; + try { + const img = new Image(entry.data, mime, { fallbackColor: theme.dim }); + this.imageContainer.addChild(img); + } catch { + // If Image construction fails, the text placeholder from + // extractText already covers this block. + } + } + } + } + const isExpanded = this.sectionState === "expanded"; if (isDiff) { diff --git a/src/tui/tui-command-handlers.ts b/src/tui/tui-command-handlers.ts index 3cbb69e..c5e830e 100644 --- a/src/tui/tui-command-handlers.ts +++ b/src/tui/tui-command-handlers.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; import type { Component, TUI } from "@earendil-works/pi-tui"; +import { copyToClipboard } from "../infra/clipboard.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; const logger = createSubsystemLogger("tui"); @@ -12,7 +13,7 @@ import { expandMarkdownCommand, findMarkdownCommand } from "../commands/markdown import type { SessionsPatchResult } from "../gateway/protocol/index.js"; import { formatRelativeTimestamp } from "../infra/format-time/format-relative.ts"; import { normalizeAgentId } from "../routing/session-key.js"; -import { execSync, spawn } from "node:child_process"; +import { execSync } from "node:child_process"; import { writeFileSync } from "node:fs"; import { helpText, parseCommand } from "./commands.js"; import { undo, listUndoEntries } from "./undo-manager.js"; @@ -888,14 +889,10 @@ export function createCommandHandlers(context: CommandHandlerContext) { break; } try { - const proc = spawn( - process.platform === "darwin" ? "pbcopy" : "xclip", - process.platform === "darwin" ? [] : ["-selection", "clipboard"], - { stdio: ["pipe", "ignore", "ignore"] }, + const ok = await copyToClipboard(lastText); + chatLog.addSystem( + ok ? "last response copied to clipboard" : "copy failed: no clipboard tool found", ); - proc.stdin?.write(lastText); - proc.stdin?.end(); - chatLog.addSystem("last response copied to clipboard"); } catch (err) { chatLog.addSystem(`copy failed: ${String(err)}`); } @@ -1238,14 +1235,14 @@ export function createCommandHandlers(context: CommandHandlerContext) { } const token = exportSession(payload); - // Copy to clipboard + // Copy to clipboard using the cross-platform clipboard helper + // (pbcopy / xclip / wl-copy / clip.exe / PowerShell fallback). try { - execSync( - `echo -n "${token}" | pbcopy 2>/dev/null || echo -n "${token}" | xclip -sel clipboard 2>/dev/null || echo -n "${token}" | xsel --clipboard 2>/dev/null`, - { encoding: "utf-8" }, - ); + const ok = await copyToClipboard(token); chatLog.addSystem( - `Session exported to clipboard (${token.length} chars, ${messages.length} messages). Share this token to import on another device.`, + ok + ? `Session exported to clipboard (${token.length} chars, ${messages.length} messages). Share this token to import on another device.` + : `Session token (no clipboard tool found):\n${token}`, ); } catch { chatLog.addSystem(`Session token:\n${token}`); diff --git a/src/tui/tui-helpers.ts b/src/tui/tui-helpers.ts new file mode 100644 index 0000000..2af978f --- /dev/null +++ b/src/tui/tui-helpers.ts @@ -0,0 +1,228 @@ +/** + * Pure helper functions extracted from tui.ts. + * + * These functions have no dependency on the TUI instance, the gateway client, + * or any mutable state — they are pure factories and resolvers tested in + * tui.test.ts. Extracting them keeps tui.ts focused on wiring. + */ + +import { Key, matchesKey } from "@earendil-works/pi-tui"; +import { buildAgentMainSessionKey } from "../routing/session-key.js"; +import type { SessionScope } from "./tui-types.js"; + +export function createEditorSubmitHandler(params: { + editor: { + setText: (value: string) => void; + addToHistory: (value: string) => void; + }; + handleCommand: (value: string) => Promise | void; + sendMessage: (value: string) => Promise | void; + handleBangLine: (value: string) => Promise | void; +}) { + return (text: string) => { + const raw = text; + const value = raw.trim(); + params.editor.setText(""); + + // Keep previous behavior: ignore empty/whitespace-only submissions. + if (!value) { + return; + } + + // Bash mode: only if the very first character is '!' and it's not just '!'. + // IMPORTANT: use the raw (untrimmed) text so leading spaces do NOT trigger. + // Per requirement: a lone '!' should be treated as a normal message. + if (raw.startsWith("!") && raw !== "!") { + params.editor.addToHistory(raw); + void params.handleBangLine(raw); + return; + } + + // Enable built-in editor prompt history navigation (up/down). + params.editor.addToHistory(value); + + if (value.startsWith("/")) { + void params.handleCommand(value); + return; + } + + void params.sendMessage(value); + }; +} + +export function shouldEnableWindowsGitBashPasteFallback(params?: { + platform?: string; + env?: NodeJS.ProcessEnv; +}): boolean { + const platform = params?.platform ?? process.platform; + if (platform !== "win32") { + return false; + } + const env = params?.env ?? process.env; + const msystem = (env.MSYSTEM ?? "").toUpperCase(); + const shell = env.SHELL ?? ""; + const termProgram = (env.TERM_PROGRAM ?? "").toLowerCase(); + if (msystem.startsWith("MINGW") || msystem.startsWith("MSYS")) { + return true; + } + if (shell.toLowerCase().includes("bash")) { + return true; + } + return termProgram.includes("mintty"); +} + +export function createSubmitBurstCoalescer(params: { + submit: (value: string) => void; + enabled: boolean; + burstWindowMs?: number; + now?: () => number; + setTimer?: typeof setTimeout; + clearTimer?: typeof clearTimeout; +}) { + const windowMs = Math.max(1, params.burstWindowMs ?? 50); + const now = params.now ?? (() => Date.now()); + const setTimer = params.setTimer ?? setTimeout; + const clearTimer = params.clearTimer ?? clearTimeout; + let pending: string | null = null; + let pendingAt = 0; + let flushTimer: ReturnType | null = null; + + const clearFlushTimer = () => { + if (!flushTimer) { + return; + } + clearTimer(flushTimer); + flushTimer = null; + }; + + const flushPending = () => { + if (pending === null) { + return; + } + const value = pending; + pending = null; + pendingAt = 0; + clearFlushTimer(); + params.submit(value); + }; + + const scheduleFlush = () => { + clearFlushTimer(); + flushTimer = setTimer(() => { + flushPending(); + }, windowMs); + }; + + return (value: string) => { + if (!params.enabled) { + params.submit(value); + return; + } + if (value.includes("\n")) { + flushPending(); + params.submit(value); + return; + } + const ts = now(); + if (pending === null) { + pending = value; + pendingAt = ts; + scheduleFlush(); + return; + } + if (ts - pendingAt <= windowMs) { + pending = `${pending}\n${value}`; + pendingAt = ts; + scheduleFlush(); + return; + } + flushPending(); + pending = value; + pendingAt = ts; + scheduleFlush(); + }; +} + +export function resolveTuiSessionKey(params: { + raw?: string; + sessionScope: SessionScope; + currentAgentId: string; + sessionMainKey: string; +}) { + const trimmed = (params.raw ?? "").trim(); + if (!trimmed) { + if (params.sessionScope === "global") { + return "global"; + } + return buildAgentMainSessionKey({ + agentId: params.currentAgentId, + mainKey: params.sessionMainKey, + }); + } + if (trimmed === "global" || trimmed === "unknown") { + return trimmed; + } + if (trimmed.startsWith("agent:")) { + return trimmed; + } + return `agent:${params.currentAgentId}:${trimmed}`; +} + +export async function tryInlinePairingApproval(): Promise { + try { + const { listDevicePairing, approveDevicePairing } = await import("../infra/device-pairing.js"); + const list = await listDevicePairing(); + if (!list.pending.length) return false; + const latest = list.pending.reduce((a, b) => (b.ts > a.ts ? b : a)); + return (await approveDevicePairing(latest.requestId)) !== null; + } catch { + return false; + } +} + +export function resolveGatewayDisconnectState(reason?: string): { + connectionStatus: string; + activityStatus: string; + pairingHint?: string; + gatewayDownHint?: string; +} { + const reasonLabel = reason?.trim() ? reason.trim() : "closed"; + if (/pairing required/i.test(reasonLabel)) { + return { + connectionStatus: `gateway disconnected: ${reasonLabel}`, + activityStatus: "pairing required: run mayros devices list", + pairingHint: + "Pairing required. Run `mayros devices list`, approve your request ID, then reconnect.", + }; + } + if (/ECONNREFUSED|connect failed/i.test(reasonLabel)) { + return { + connectionStatus: "gateway not running", + activityStatus: "start gateway with: mayros gateway run", + gatewayDownHint: + "Gateway is not responding. Start it with `mayros gateway run` or `mayros onboard`.", + }; + } + return { + connectionStatus: `gateway disconnected: ${reasonLabel}`, + activityStatus: "idle", + }; +} + +export function createBackspaceDeduper(params?: { dedupeWindowMs?: number; now?: () => number }) { + const dedupeWindowMs = Math.max(0, Math.floor(params?.dedupeWindowMs ?? 8)); + const now = params?.now ?? (() => Date.now()); + let lastBackspaceAt = -1; + + return (data: string): string => { + if (!matchesKey(data, Key.backspace)) { + return data; + } + const ts = now(); + if (lastBackspaceAt >= 0 && ts - lastBackspaceAt <= dedupeWindowMs) { + return ""; + } + lastBackspaceAt = ts; + return data; + }; +} diff --git a/src/tui/tui.ts b/src/tui/tui.ts index 031f2b8..5718f8b 100644 --- a/src/tui/tui.ts +++ b/src/tui/tui.ts @@ -1,9 +1,7 @@ import { CombinedAutocompleteProvider, Container, - Key, Loader, - matchesKey, ProcessTerminal, Text, TUI, @@ -13,7 +11,6 @@ import { VERSION } from "../version.js"; import { loadConfig } from "../config/config.js"; import { isLoopbackHost } from "../gateway/net.js"; import { - buildAgentMainSessionKey, normalizeAgentId, normalizeMainKey, parseAgentSessionKey, @@ -53,223 +50,23 @@ import { buildWaitingStatusMessage, defaultWaitingPhrases } from "./tui-waiting. export { resolveFinalAssistantText } from "./tui-formatters.js"; export type { TuiOptions } from "./tui-types.js"; - -export function createEditorSubmitHandler(params: { - editor: { - setText: (value: string) => void; - addToHistory: (value: string) => void; - }; - handleCommand: (value: string) => Promise | void; - sendMessage: (value: string) => Promise | void; - handleBangLine: (value: string) => Promise | void; -}) { - return (text: string) => { - const raw = text; - const value = raw.trim(); - params.editor.setText(""); - - // Keep previous behavior: ignore empty/whitespace-only submissions. - if (!value) { - return; - } - - // Bash mode: only if the very first character is '!' and it's not just '!'. - // IMPORTANT: use the raw (untrimmed) text so leading spaces do NOT trigger. - // Per requirement: a lone '!' should be treated as a normal message. - if (raw.startsWith("!") && raw !== "!") { - params.editor.addToHistory(raw); - void params.handleBangLine(raw); - return; - } - - // Enable built-in editor prompt history navigation (up/down). - params.editor.addToHistory(value); - - if (value.startsWith("/")) { - void params.handleCommand(value); - return; - } - - void params.sendMessage(value); - }; -} - -export function shouldEnableWindowsGitBashPasteFallback(params?: { - platform?: string; - env?: NodeJS.ProcessEnv; -}): boolean { - const platform = params?.platform ?? process.platform; - if (platform !== "win32") { - return false; - } - const env = params?.env ?? process.env; - const msystem = (env.MSYSTEM ?? "").toUpperCase(); - const shell = env.SHELL ?? ""; - const termProgram = (env.TERM_PROGRAM ?? "").toLowerCase(); - if (msystem.startsWith("MINGW") || msystem.startsWith("MSYS")) { - return true; - } - if (shell.toLowerCase().includes("bash")) { - return true; - } - return termProgram.includes("mintty"); -} - -export function createSubmitBurstCoalescer(params: { - submit: (value: string) => void; - enabled: boolean; - burstWindowMs?: number; - now?: () => number; - setTimer?: typeof setTimeout; - clearTimer?: typeof clearTimeout; -}) { - const windowMs = Math.max(1, params.burstWindowMs ?? 50); - const now = params.now ?? (() => Date.now()); - const setTimer = params.setTimer ?? setTimeout; - const clearTimer = params.clearTimer ?? clearTimeout; - let pending: string | null = null; - let pendingAt = 0; - let flushTimer: ReturnType | null = null; - - const clearFlushTimer = () => { - if (!flushTimer) { - return; - } - clearTimer(flushTimer); - flushTimer = null; - }; - - const flushPending = () => { - if (pending === null) { - return; - } - const value = pending; - pending = null; - pendingAt = 0; - clearFlushTimer(); - params.submit(value); - }; - - const scheduleFlush = () => { - clearFlushTimer(); - flushTimer = setTimer(() => { - flushPending(); - }, windowMs); - }; - - return (value: string) => { - if (!params.enabled) { - params.submit(value); - return; - } - if (value.includes("\n")) { - flushPending(); - params.submit(value); - return; - } - const ts = now(); - if (pending === null) { - pending = value; - pendingAt = ts; - scheduleFlush(); - return; - } - if (ts - pendingAt <= windowMs) { - pending = `${pending}\n${value}`; - pendingAt = ts; - scheduleFlush(); - return; - } - flushPending(); - pending = value; - pendingAt = ts; - scheduleFlush(); - }; -} - -export function resolveTuiSessionKey(params: { - raw?: string; - sessionScope: SessionScope; - currentAgentId: string; - sessionMainKey: string; -}) { - const trimmed = (params.raw ?? "").trim(); - if (!trimmed) { - if (params.sessionScope === "global") { - return "global"; - } - return buildAgentMainSessionKey({ - agentId: params.currentAgentId, - mainKey: params.sessionMainKey, - }); - } - if (trimmed === "global" || trimmed === "unknown") { - return trimmed; - } - if (trimmed.startsWith("agent:")) { - return trimmed; - } - return `agent:${params.currentAgentId}:${trimmed}`; -} - -async function tryInlinePairingApproval(): Promise { - try { - const { listDevicePairing, approveDevicePairing } = await import("../infra/device-pairing.js"); - const list = await listDevicePairing(); - if (!list.pending.length) return false; - const latest = list.pending.reduce((a, b) => (b.ts > a.ts ? b : a)); - return (await approveDevicePairing(latest.requestId)) !== null; - } catch { - return false; - } -} - -export function resolveGatewayDisconnectState(reason?: string): { - connectionStatus: string; - activityStatus: string; - pairingHint?: string; - gatewayDownHint?: string; -} { - const reasonLabel = reason?.trim() ? reason.trim() : "closed"; - if (/pairing required/i.test(reasonLabel)) { - return { - connectionStatus: `gateway disconnected: ${reasonLabel}`, - activityStatus: "pairing required: run mayros devices list", - pairingHint: - "Pairing required. Run `mayros devices list`, approve your request ID, then reconnect.", - }; - } - if (/ECONNREFUSED|connect failed/i.test(reasonLabel)) { - return { - connectionStatus: "gateway not running", - activityStatus: "start gateway with: mayros gateway run", - gatewayDownHint: - "Gateway is not responding. Start it with `mayros gateway run` or `mayros onboard`.", - }; - } - return { - connectionStatus: `gateway disconnected: ${reasonLabel}`, - activityStatus: "idle", - }; -} - -export function createBackspaceDeduper(params?: { dedupeWindowMs?: number; now?: () => number }) { - const dedupeWindowMs = Math.max(0, Math.floor(params?.dedupeWindowMs ?? 8)); - const now = params?.now ?? (() => Date.now()); - let lastBackspaceAt = -1; - - return (data: string): string => { - if (!matchesKey(data, Key.backspace)) { - return data; - } - const ts = now(); - if (lastBackspaceAt >= 0 && ts - lastBackspaceAt <= dedupeWindowMs) { - return ""; - } - lastBackspaceAt = ts; - return data; - }; -} +export { + createEditorSubmitHandler, + shouldEnableWindowsGitBashPasteFallback, + createSubmitBurstCoalescer, + resolveTuiSessionKey, + resolveGatewayDisconnectState, + createBackspaceDeduper, +} from "./tui-helpers.js"; +import { + createEditorSubmitHandler, + shouldEnableWindowsGitBashPasteFallback, + createSubmitBurstCoalescer, + resolveTuiSessionKey, + resolveGatewayDisconnectState, + createBackspaceDeduper, + tryInlinePairingApproval, +} from "./tui-helpers.js"; export async function runTui(opts: TuiOptions) { const config = loadConfig();