From 2e955a2538ed185b41f82b92f160debbb95b6e75 Mon Sep 17 00:00:00 2001 From: sungl <81428141+Sun-GLiang@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:38:47 +0800 Subject: [PATCH 01/35] feat(desktop): automatically render and archive chat images Resolve assistant Markdown images through a session-scoped Host service, save bounded replay copies without rewriting message text, and provide lazy previews, zoom and retry through independent UI capabilities. Share archived image payloads across deliveries and conversation copies, reuse Read filesystem boundaries, and retain PublishImage for explicit publication before temporary sources are removed. Generated-by: Codex (GPT-6) --- .../check-renderer-architecture.test.mjs | 10 +- .../scripts/vite-renderer-entry-contract.ts | 9 +- .../main/runtime-host-artifacts-ipc-main.ts | 1 + apps/desktop/src/main/runtime-host-client.ts | 4 + apps/desktop/src/preload/bridge-contract.d.ts | 1 + apps/desktop/src/preload/preload.ts | 1 + .../src/renderer/chat-message-surface.tsx | 1 + .../features/conversation/staging-services.ts | 1 + .../ui/staged-quote-chat-view.tsx | 6 +- .../src/renderer/features/workbar/ports.ts | 1 + .../tools/side-chat/quote-companion-panel.tsx | 2 +- .../src/renderer/features/workhub/ports.ts | 1 + .../features/workhub/ui/workhub-root.tsx | 2 +- apps/desktop/src/renderer/index.html | 3 +- .../desktop/create-workhub-services.ts | 1 + package-lock.json | 3 + packages/core/package.json | 1 + packages/core/src/artifacts.ts | 3 + packages/core/src/image-delivery.ts | 81 ++++ .../core/src/interaction-permission-review.ts | 8 +- packages/core/src/permission.ts | 1 + packages/runtime-host/package.json | 1 + .../src/__tests__/chat-image-delivery.test.ts | 348 ++++++++++++++++++ .../src/__tests__/execution-artifacts.test.ts | 93 +++++ .../__tests__/execution-composition.test.ts | 98 +++++ .../runtime-host/src/protocol/artifact.ts | 2 + .../src/protocol/image-delivery.ts | 98 +++++ packages/runtime-host/src/protocol/index.ts | 5 +- .../runtime-host/src/protocol/operations.ts | 1 + .../src/server/artifact-coordinator.ts | 9 + .../src/server/chat-image-delivery.ts | 339 +++++++++++++++++ .../src/server/chat-image-markdown.ts | 38 ++ .../src/server/chat-image-source.ts | 157 ++++++++ .../src/server/execution-artifacts.ts | 46 ++- .../src/server/execution-composition.ts | 87 +++++ .../src/server/root-turn-coordinator.ts | 18 + packages/runtime/package.json | 1 + .../builtin-tools-file-worker.test.ts | 62 ++++ .../src/__tests__/tool-availability.test.ts | 6 +- packages/runtime/src/builtin-tools.ts | 68 +++- packages/runtime/src/image-file-reader.ts | 44 +++ .../src/system-prompt/main-session-prompt.ts | 3 + packages/runtime/src/tool-availability.ts | 1 + .../storage/src/artifact-image-storage.ts | 74 ++++ .../storage/src/artifact-metadata-codec.ts | 3 + packages/storage/src/artifact-store.ts | 85 ++++- packages/storage/src/artifact-stores.ts | 8 + .../storage/src/sqlite-artifact-metadata.ts | 22 ++ .../storage/src/sqlite-artifact-schema.ts | 7 +- packages/ui/package.json | 5 +- .../scripts/markdown-images.browser.test.mjs | 192 ++++++++++ .../src/__tests__/attachment-image.test.tsx | 32 +- .../ui/src/__tests__/markdown-body.test.ts | 31 +- packages/ui/src/attachment-image.tsx | 59 ++- packages/ui/src/chat-turn.tsx | 13 +- packages/ui/src/chat-view.tsx | 4 + packages/ui/src/image-delivery.tsx | 74 ++++ packages/ui/src/markdown-body.tsx | 116 +++--- packages/ui/src/markdown-image.tsx | 94 +++++ packages/ui/src/markdown.tsx | 2 + packages/ui/src/shared-ui-copy.ts | 36 ++ packages/ui/src/styles.css | 45 +++ 62 files changed, 2444 insertions(+), 124 deletions(-) create mode 100644 packages/core/src/image-delivery.ts create mode 100644 packages/runtime-host/src/__tests__/chat-image-delivery.test.ts create mode 100644 packages/runtime-host/src/protocol/image-delivery.ts create mode 100644 packages/runtime-host/src/server/chat-image-delivery.ts create mode 100644 packages/runtime-host/src/server/chat-image-markdown.ts create mode 100644 packages/runtime-host/src/server/chat-image-source.ts create mode 100644 packages/runtime/src/image-file-reader.ts create mode 100644 packages/storage/src/artifact-image-storage.ts create mode 100644 packages/ui/scripts/markdown-images.browser.test.mjs create mode 100644 packages/ui/src/image-delivery.tsx create mode 100644 packages/ui/src/markdown-image.tsx diff --git a/apps/desktop/scripts/check-renderer-architecture.test.mjs b/apps/desktop/scripts/check-renderer-architecture.test.mjs index 2fe7328ae7c..b1236071e23 100644 --- a/apps/desktop/scripts/check-renderer-architecture.test.mjs +++ b/apps/desktop/scripts/check-renderer-architecture.test.mjs @@ -314,9 +314,10 @@ function canonicalRendererEntryHtml( + Maka @@ -1778,6 +1779,13 @@ describe('renderer architecture checker fixtures', () => { ); }); + it('rejects an HTML transform that changes the no-referrer policy', () => { + assert.throws( + () => assertRendererEntryHtml(canonicalRendererEntryHtml().replace('content="no-referrer"', 'content="unsafe-url"')), + /renderer entry HTML contract forbids transformed executable or navigation surfaces/u, + ); + }); + it('rejects executable HTML injected around the canonical module entry', () => { assert.doesNotThrow(() => assertRendererEntryHtml(canonicalRendererEntryHtml())); const emittedHtml = canonicalRendererEntryHtml().replace( diff --git a/apps/desktop/scripts/vite-renderer-entry-contract.ts b/apps/desktop/scripts/vite-renderer-entry-contract.ts index a3327fd28b9..4e9ed2c98af 100644 --- a/apps/desktop/scripts/vite-renderer-entry-contract.ts +++ b/apps/desktop/scripts/vite-renderer-entry-contract.ts @@ -40,7 +40,7 @@ const ALLOWED_HTML_TAGS = new Set([ 'title', ]); const CONTENT_SECURITY_POLICY = - "default-src 'self'; script-src 'self' maka-client-plugin:; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; connect-src 'self'"; + "default-src 'self'; script-src 'self' maka-client-plugin:; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: http: https:; connect-src 'self'"; function normalizePath(path: string): string { return path.split(sep).join('/'); @@ -88,8 +88,13 @@ export function assertRendererEntryHtml(html: string, expectedScriptSource?: str htmlAttribute(attributes, 'http-equiv')?.toLowerCase() === 'content-security-policy' && htmlAttribute(attributes, 'content')?.trim().replace(/\s+/gu, ' ') === CONTENT_SECURITY_POLICY, ); + const referrerMetas = metas.filter( + (attributes) => htmlAttribute(attributes, 'name')?.toLowerCase() === 'referrer' && + htmlAttribute(attributes, 'content') === 'no-referrer', + ); const validMetas = - metas.length === 3 && + metas.length === 4 && + referrerMetas.length === 1 && charsetMetas.length === 1 && viewportMetas.length === 1 && policyMetas.length === 1; diff --git a/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts b/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts index 2f52baf456e..faa7b7161b2 100644 --- a/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts +++ b/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts @@ -89,6 +89,7 @@ export function registerRuntimeHostArtifactsIpc( return result; }, ); + deps.ipcMain.handle('attachments:resolveImage', (_event, sessionId: string, request: import('@maka/core/image-delivery').ImageDeliveryRequest) => deps.client.resolveImageDelivery(sessionId, request)); registerRuntimeHostAttachmentPreviewIpc(deps); const materializePresentationArtifact = async ( sessionId: string, diff --git a/apps/desktop/src/main/runtime-host-client.ts b/apps/desktop/src/main/runtime-host-client.ts index f7471d228d3..7272fe6d271 100644 --- a/apps/desktop/src/main/runtime-host-client.ts +++ b/apps/desktop/src/main/runtime-host-client.ts @@ -867,6 +867,10 @@ export class DesktopRuntimeHostClient { return result.project; } + resolveImageDelivery(sessionId: string, request: import('@maka/core/image-delivery').ImageDeliveryRequest): Promise { + return this.request('artifact.image.resolve', { sessionId, ...request }); + } + async listArtifacts(sessionId: string): Promise { for (let attempt = 0; attempt < MAX_OPTIMISTIC_ATTEMPTS; attempt += 1) { const first = await this.request("artifact.query", { diff --git a/apps/desktop/src/preload/bridge-contract.d.ts b/apps/desktop/src/preload/bridge-contract.d.ts index 5287b9e34f0..2b293523965 100644 --- a/apps/desktop/src/preload/bridge-contract.d.ts +++ b/apps/desktop/src/preload/bridge-contract.d.ts @@ -1745,6 +1745,7 @@ export interface MakaBridge { | { ok: true; base64: string; mimeType: string } | { ok: false; reason: string } >; + resolveImage(sessionId: string, request: import('@maka/core/image-delivery').ImageDeliveryRequest): Promise; readBytes(sessionId: string, artifactId: string): Promise; }; search: { diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts index aba743dc2cd..be090c01638 100644 --- a/apps/desktop/src/preload/preload.ts +++ b/apps/desktop/src/preload/preload.ts @@ -3422,6 +3422,7 @@ const makaBridge = { }, }, attachments: { + resolveImage: (sessionId, request) => invokeSessionRuntimeHost('attachments:resolveImage', sessionId, request), pickDirectory: () => invokeWhenReady('directories:pick'), // The renderer hands over the dropped or pasted File objects, never paths: // only a File backed by something the user dropped or pasted has a path, diff --git a/apps/desktop/src/renderer/chat-message-surface.tsx b/apps/desktop/src/renderer/chat-message-surface.tsx index eb4e3c5801e..163e62d2142 100644 --- a/apps/desktop/src/renderer/chat-message-surface.tsx +++ b/apps/desktop/src/renderer/chat-message-surface.tsx @@ -58,6 +58,7 @@ interface ChatMessageSurfaceProps extends Omit< | 'pendingQuotes' | 'onQuoteAnnotationSubmit' | 'onReadAttachmentBytes' + | 'onResolveImageDelivery' > { /** * #1985: the live projection and the shell-run records are the only session diff --git a/apps/desktop/src/renderer/features/conversation/staging-services.ts b/apps/desktop/src/renderer/features/conversation/staging-services.ts index 98002392b88..d8bfc678748 100644 --- a/apps/desktop/src/renderer/features/conversation/staging-services.ts +++ b/apps/desktop/src/renderer/features/conversation/staging-services.ts @@ -23,6 +23,7 @@ import type { ArtifactBinaryReadResult } from '@maka/core/artifacts'; /** One attachment port: the Composer stages through it and the transcript reads image bytes through it. */ export interface ComposerStagingServices extends ComposerAttachmentService { + resolveImage?: (sessionId: string, request: import('@maka/core/image-delivery').ImageDeliveryRequest) => Promise; readBytes(sessionId: string, artifactId: string): Promise; } const context = createServicesContext('ComposerStagingServicesProvider'); diff --git a/apps/desktop/src/renderer/features/conversation/ui/staged-quote-chat-view.tsx b/apps/desktop/src/renderer/features/conversation/ui/staged-quote-chat-view.tsx index 12c55d2a565..b81e97d9297 100644 --- a/apps/desktop/src/renderer/features/conversation/ui/staged-quote-chat-view.tsx +++ b/apps/desktop/src/renderer/features/conversation/ui/staged-quote-chat-view.tsx @@ -24,8 +24,8 @@ import { useComposerStagingServices } from '../staging-services.js'; /** Transcript attachment reads come from the injected attachment port, never from the caller. */ export function StagedQuoteChatView(props: Omit, - 'handleRef' | 'pendingQuotes' | 'onQuoteAnnotationSubmit' | 'onReadAttachmentBytes'>) { + 'handleRef' | 'pendingQuotes' | 'onQuoteAnnotationSubmit' | 'onReadAttachmentBytes' | 'onResolveImageDelivery'>) { const staging = useComposerStaging(); - const { readBytes } = useComposerStagingServices(); - return ; + const { readBytes, resolveImage } = useComposerStagingServices(); + return ; } diff --git a/apps/desktop/src/renderer/features/workbar/ports.ts b/apps/desktop/src/renderer/features/workbar/ports.ts index f74b022e461..8e8ed1af9bf 100644 --- a/apps/desktop/src/renderer/features/workbar/ports.ts +++ b/apps/desktop/src/renderer/features/workbar/ports.ts @@ -165,6 +165,7 @@ export interface WorkbarArtifactsService { } export interface WorkbarAttachmentsService { + resolveImage?: (sessionId: string, request: import('@maka/core/image-delivery').ImageDeliveryRequest) => Promise; readBytes(sessionId: string, artifactId: string): Promise; pickFiles(): Promise< | { diff --git a/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx b/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx index 29b7a102300..2e278a21bf8 100644 --- a/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx +++ b/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx @@ -404,7 +404,7 @@ export function QuoteCompanionPanel(props: { liveTurns={companion.liveTurns} activeTurn={companion.activeTurn} activeSession={companion.companionSession} - onReadAttachmentBytes={attachments.readBytes} + onReadAttachmentBytes={attachments.readBytes} onResolveImageDelivery={attachments.resolveImage} deriveTurnPresentation={deriveTurnPresentation} onEditUserMessage={(turnId) => { const message = companion.messages.find( diff --git a/apps/desktop/src/renderer/features/workhub/ports.ts b/apps/desktop/src/renderer/features/workhub/ports.ts index 59a81bcf963..93f17748310 100644 --- a/apps/desktop/src/renderer/features/workhub/ports.ts +++ b/apps/desktop/src/renderer/features/workhub/ports.ts @@ -56,6 +56,7 @@ export interface WorkHubServices extends WorkHubWorkspaceServices { model: string; }): Promise; readonly attachments: ComposerAttachmentService; + resolveImageDelivery?: (sessionId: string, request: import('@maka/core/image-delivery').ImageDeliveryRequest) => Promise; readAttachmentBytes(sessionId: string, artifactId: string): Promise; prepareAttachments(sessionId: string, items: Array<{ approvalId: string; name: string; mimeType?: string } | { file: File }>): Promise; listActiveInteractions(sessionId: string): Promise; diff --git a/apps/desktop/src/renderer/features/workhub/ui/workhub-root.tsx b/apps/desktop/src/renderer/features/workhub/ui/workhub-root.tsx index e6eb0d74eae..f87510db540 100644 --- a/apps/desktop/src/renderer/features/workhub/ui/workhub-root.tsx +++ b/apps/desktop/src/renderer/features/workhub/ui/workhub-root.tsx @@ -408,7 +408,7 @@ export function WorkHubRoot() {
call(services.presentation.openSession(id))} scrollBehavior="auto" onNew={() => composer.current?.focus()} diff --git a/apps/desktop/src/renderer/index.html b/apps/desktop/src/renderer/index.html index 19950c7a2df..aa66e09e36c 100644 --- a/apps/desktop/src/renderer/index.html +++ b/apps/desktop/src/renderer/index.html @@ -21,10 +21,11 @@ + Maka diff --git a/apps/desktop/scripts/vite-renderer-entry-contract.ts b/apps/desktop/scripts/vite-renderer-entry-contract.ts index 4e9ed2c98af..0a1403043ab 100644 --- a/apps/desktop/scripts/vite-renderer-entry-contract.ts +++ b/apps/desktop/scripts/vite-renderer-entry-contract.ts @@ -40,7 +40,7 @@ const ALLOWED_HTML_TAGS = new Set([ 'title', ]); const CONTENT_SECURITY_POLICY = - "default-src 'self'; script-src 'self' maka-client-plugin:; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: http: https:; connect-src 'self'"; + "default-src 'self'; script-src 'self' maka-client-plugin:; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; connect-src 'self'"; function normalizePath(path: string): string { return path.split(sep).join('/'); diff --git a/apps/desktop/src/main/__tests__/runtime-host-artifacts-ipc-main.test.ts b/apps/desktop/src/main/__tests__/runtime-host-artifacts-ipc-main.test.ts index 420765c7e0d..5d7619bb042 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-artifacts-ipc-main.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-artifacts-ipc-main.test.ts @@ -26,6 +26,28 @@ import { test } from "node:test"; import { registerRuntimeHostArtifactsIpc } from "../runtime-host-artifacts-ipc-main.js"; import { ManagedArtifactPreview } from '../managed-artifact-preview.js'; +test('image resolution validates renderer requests before forwarding them', async () => { + const handlers = new Map(); + const forwarded: unknown[] = []; + registerRuntimeHostArtifactsIpc({ + uiLocale: () => 'en', + ipcMain: { handle: (channel, handler) => handlers.set(channel, handler as Handler) }, + client: { hostEpoch: 'host', resolveImageDelivery: async (...args: unknown[]) => { + forwarded.push(args); return { status: 'requires_confirmation' }; + } } as never, + mainWindowController: {} as never, + showItemInFolder() {}, + }); + const resolve = handlers.get('attachments:resolveImage')!; + const request = { turnId: 'turn', messageId: 'message', source: 'https://example.invalid/image.png', loadRemote: true }; + await resolve({}, 'session', request); + assert.deepEqual(forwarded, [['session', request]]); + for (const invalid of [null, { ...request, sessionId: 'other' }, { ...request, loadRemote: 'yes' }, { ...request, source: 'bad\nurl' }]) { + await assert.rejects(async () => resolve({}, 'session', invalid), /Invalid image delivery request/); + } + assert.equal(forwarded.length, 1); +}); + for (const launchFails of [false, true]) { test(`HTML external open uses the managed endpoint and reports launch failure=${launchFails}`, async () => { const service = new ManagedArtifactPreview(); diff --git a/apps/desktop/src/main/__tests__/runtime-host-client-operations.test.ts b/apps/desktop/src/main/__tests__/runtime-host-client-operations.test.ts index d852e9f8d1f..c577a7cfd36 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-client-operations.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-client-operations.test.ts @@ -34,6 +34,18 @@ import { DesktopRuntimeHostClientError, } from '../runtime-host-client.js'; +test('image resolution keeps the routed session even if a request carries another id', async () => { + const { client, requests } = clientWithResponses([{ status: 'requires_confirmation' }]); + await client.resolveImageDelivery('routed-session', { + sessionId: 'injected-session', turnId: 'turn', messageId: 'message', + source: 'https://example.invalid/image.png', loadRemote: true, + } as import('@maka/core/image-delivery').ImageDeliveryRequest); + assert.deepEqual(requests, [{ operation: 'artifact.image.resolve', input: { + sessionId: 'routed-session', turnId: 'turn', messageId: 'message', + source: 'https://example.invalid/image.png', loadRemote: true, + } }]); +}); + test('restarts a paginated catalog read instead of mixing revisions', async () => { const revisionOne = catalogRevision('1'); const revisionTwo = catalogRevision('2'); diff --git a/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts b/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts index faa7b7161b2..203daf8ac33 100644 --- a/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts +++ b/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts @@ -18,6 +18,7 @@ */ import type { UiCatalog, UiLocale } from '@maka/core/ui-locale'; +import { isImageDeliveryRequest } from '@maka/core/image-delivery'; import { randomUUID } from "node:crypto"; import { open, mkdir, rename, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -89,7 +90,10 @@ export function registerRuntimeHostArtifactsIpc( return result; }, ); - deps.ipcMain.handle('attachments:resolveImage', (_event, sessionId: string, request: import('@maka/core/image-delivery').ImageDeliveryRequest) => deps.client.resolveImageDelivery(sessionId, request)); + deps.ipcMain.handle('attachments:resolveImage', (_event, sessionId: string, request: unknown) => { + if (!isImageDeliveryRequest(request)) throw new Error('Invalid image delivery request'); + return deps.client.resolveImageDelivery(sessionId, request); + }); registerRuntimeHostAttachmentPreviewIpc(deps); const materializePresentationArtifact = async ( sessionId: string, diff --git a/apps/desktop/src/main/runtime-host-client.ts b/apps/desktop/src/main/runtime-host-client.ts index 7272fe6d271..a90cba26e82 100644 --- a/apps/desktop/src/main/runtime-host-client.ts +++ b/apps/desktop/src/main/runtime-host-client.ts @@ -868,7 +868,7 @@ export class DesktopRuntimeHostClient { } resolveImageDelivery(sessionId: string, request: import('@maka/core/image-delivery').ImageDeliveryRequest): Promise { - return this.request('artifact.image.resolve', { sessionId, ...request }); + return this.request('artifact.image.resolve', { ...request, sessionId }); } async listArtifacts(sessionId: string): Promise { diff --git a/apps/desktop/src/renderer/index.html b/apps/desktop/src/renderer/index.html index aa66e09e36c..ad98a582beb 100644 --- a/apps/desktop/src/renderer/index.html +++ b/apps/desktop/src/renderer/index.html @@ -25,7 +25,7 @@ Maka