diff --git a/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt b/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt index 7e62f8d604c..dbe751bc9b9 100644 --- a/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt +++ b/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt @@ -12696,6 +12696,59 @@ This software is provided by the copyright holders and contributors “as is” ================================================================================ +Package: marked@18.0.11 +Declared license: MIT +Selected license: MIT +Repository: git+https://github.com/markedjs/marked.git + +--- LICENSE --- +# License information + +## Contribution License Agreement + +If you contribute code to this project, you are implicitly allowing your code +to be distributed under the MIT license. You are also implicitly verifying that +all code is your original work. `` + +## Marked + +Copyright (c) 2018+, MarkedJS (https://github.com/markedjs/) +Copyright (c) 2011-2018, Christopher Jeffrey (https://github.com/chjj/) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +## Markdown + +Copyright © 2004, John Gruber +http://daringfireball.net/ +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +* Neither the name “Markdown” nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +This software is provided by the copyright holders and contributors “as is” and any express or implied warranties, including, but not limited to, the implied warranties of merchantability and fitness for a particular purpose are disclaimed. In no event shall the copyright owner or contributors be liable for any direct, indirect, incidental, special, exemplary, or consequential damages (including, but not limited to, procurement of substitute goods or services; loss of use, data, or profits; or business interruption) however caused and on any theory of liability, whether in contract, strict liability, or tort (including negligence or otherwise) arising in any way out of the use of this software, even if advised of the possibility of such damage. + +================================================================================ + Package: math-intrinsics@1.1.0 Declared license: MIT Selected license: MIT diff --git a/apps/desktop/scripts/check-renderer-architecture.test.mjs b/apps/desktop/scripts/check-renderer-architecture.test.mjs index 2fe7328ae7c..2295af95303 100644 --- a/apps/desktop/scripts/check-renderer-architecture.test.mjs +++ b/apps/desktop/scripts/check-renderer-architecture.test.mjs @@ -314,6 +314,7 @@ function canonicalRendererEntryHtml( + { ); }); + it('rejects an HTML transform that changes the no-referrer policy', () => { + assert.throws( + () => assertRendererEntryHtml(canonicalRendererEntryHtml().replace('content="no-referrer"', 'content="unsafe-url"')), + /renderer entry HTML contract forbids transformed executable or navigation surfaces/u, + ); + }); + it('rejects executable HTML injected around the canonical module entry', () => { assert.doesNotThrow(() => assertRendererEntryHtml(canonicalRendererEntryHtml())); const emittedHtml = canonicalRendererEntryHtml().replace( diff --git a/apps/desktop/scripts/vite-renderer-entry-contract.ts b/apps/desktop/scripts/vite-renderer-entry-contract.ts index a3327fd28b9..0a1403043ab 100644 --- a/apps/desktop/scripts/vite-renderer-entry-contract.ts +++ b/apps/desktop/scripts/vite-renderer-entry-contract.ts @@ -88,8 +88,13 @@ export function assertRendererEntryHtml(html: string, expectedScriptSource?: str htmlAttribute(attributes, 'http-equiv')?.toLowerCase() === 'content-security-policy' && htmlAttribute(attributes, 'content')?.trim().replace(/\s+/gu, ' ') === CONTENT_SECURITY_POLICY, ); + const referrerMetas = metas.filter( + (attributes) => htmlAttribute(attributes, 'name')?.toLowerCase() === 'referrer' && + htmlAttribute(attributes, 'content') === 'no-referrer', + ); const validMetas = - metas.length === 3 && + metas.length === 4 && + referrerMetas.length === 1 && charsetMetas.length === 1 && viewportMetas.length === 1 && policyMetas.length === 1; diff --git a/apps/desktop/src/main/__tests__/runtime-host-artifacts-ipc-main.test.ts b/apps/desktop/src/main/__tests__/runtime-host-artifacts-ipc-main.test.ts index 420765c7e0d..5d7619bb042 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-artifacts-ipc-main.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-artifacts-ipc-main.test.ts @@ -26,6 +26,28 @@ import { test } from "node:test"; import { registerRuntimeHostArtifactsIpc } from "../runtime-host-artifacts-ipc-main.js"; import { ManagedArtifactPreview } from '../managed-artifact-preview.js'; +test('image resolution validates renderer requests before forwarding them', async () => { + const handlers = new Map(); + const forwarded: unknown[] = []; + registerRuntimeHostArtifactsIpc({ + uiLocale: () => 'en', + ipcMain: { handle: (channel, handler) => handlers.set(channel, handler as Handler) }, + client: { hostEpoch: 'host', resolveImageDelivery: async (...args: unknown[]) => { + forwarded.push(args); return { status: 'requires_confirmation' }; + } } as never, + mainWindowController: {} as never, + showItemInFolder() {}, + }); + const resolve = handlers.get('attachments:resolveImage')!; + const request = { turnId: 'turn', messageId: 'message', source: 'https://example.invalid/image.png', loadRemote: true }; + await resolve({}, 'session', request); + assert.deepEqual(forwarded, [['session', request]]); + for (const invalid of [null, { ...request, sessionId: 'other' }, { ...request, loadRemote: 'yes' }, { ...request, source: 'bad\nurl' }]) { + await assert.rejects(async () => resolve({}, 'session', invalid), /Invalid image delivery request/); + } + assert.equal(forwarded.length, 1); +}); + for (const launchFails of [false, true]) { test(`HTML external open uses the managed endpoint and reports launch failure=${launchFails}`, async () => { const service = new ManagedArtifactPreview(); diff --git a/apps/desktop/src/main/__tests__/runtime-host-client-operations.test.ts b/apps/desktop/src/main/__tests__/runtime-host-client-operations.test.ts index d852e9f8d1f..c577a7cfd36 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-client-operations.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-client-operations.test.ts @@ -34,6 +34,18 @@ import { DesktopRuntimeHostClientError, } from '../runtime-host-client.js'; +test('image resolution keeps the routed session even if a request carries another id', async () => { + const { client, requests } = clientWithResponses([{ status: 'requires_confirmation' }]); + await client.resolveImageDelivery('routed-session', { + sessionId: 'injected-session', turnId: 'turn', messageId: 'message', + source: 'https://example.invalid/image.png', loadRemote: true, + } as import('@maka/core/image-delivery').ImageDeliveryRequest); + assert.deepEqual(requests, [{ operation: 'artifact.image.resolve', input: { + sessionId: 'routed-session', turnId: 'turn', messageId: 'message', + source: 'https://example.invalid/image.png', loadRemote: true, + } }]); +}); + test('restarts a paginated catalog read instead of mixing revisions', async () => { const revisionOne = catalogRevision('1'); const revisionTwo = catalogRevision('2'); diff --git a/apps/desktop/src/main/__tests__/workhub-coordination-transcript-preload.test.ts b/apps/desktop/src/main/__tests__/workhub-coordination-transcript-preload.test.ts index 3545fcf3987..3b33ac1c8e4 100644 --- a/apps/desktop/src/main/__tests__/workhub-coordination-transcript-preload.test.ts +++ b/apps/desktop/src/main/__tests__/workhub-coordination-transcript-preload.test.ts @@ -84,6 +84,12 @@ test('WorkHub upload references round-trip through idle answers, both queue mode return { ok: true, disposition: args[2] === 'current_turn' ? 'steering' : 'followup', attachments: [uploaded] }; } if (channel === 'attachments:readBytes') return { ok: true, base64: 'aGVsbG8=' }; + if (channel === 'attachments:resolveImage') { + assert.deepEqual(structuredClone(args[2]), { + turnId: 'image-turn', messageId: 'image-message', source: '/tmp/image.png', + }); + return { status: 'ready', artifactId: 'image-1' }; + } throw new Error(`Unexpected channel: ${channel}`); }, }, @@ -109,6 +115,10 @@ test('WorkHub upload references round-trip through idle answers, both queue mode } assert.deepEqual(structuredClone(sent.map(({ attachments }) => attachments)), [[uploaded], [uploaded], [uploaded]]); assert.equal((await services.readAttachmentBytes(sessionId, 'brief.txt')).ok, true); + assert.ok(services.resolveImageDelivery); + assert.deepEqual(await services.resolveImageDelivery(sessionId, { + turnId: 'image-turn', messageId: 'image-message', source: '/tmp/image.png', + }), { status: 'ready', artifactId: 'image-1' }); preparationResult = { ok: false, code: 'item_too_large' }; await assert.rejects( services.prepareAttachments(sessionId, [ diff --git a/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts b/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts index 2f52baf456e..203daf8ac33 100644 --- a/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts +++ b/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts @@ -18,6 +18,7 @@ */ import type { UiCatalog, UiLocale } from '@maka/core/ui-locale'; +import { isImageDeliveryRequest } from '@maka/core/image-delivery'; import { randomUUID } from "node:crypto"; import { open, mkdir, rename, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -89,6 +90,10 @@ export function registerRuntimeHostArtifactsIpc( return result; }, ); + deps.ipcMain.handle('attachments:resolveImage', (_event, sessionId: string, request: unknown) => { + if (!isImageDeliveryRequest(request)) throw new Error('Invalid image delivery request'); + return deps.client.resolveImageDelivery(sessionId, request); + }); registerRuntimeHostAttachmentPreviewIpc(deps); const materializePresentationArtifact = async ( sessionId: string, diff --git a/apps/desktop/src/main/runtime-host-client.ts b/apps/desktop/src/main/runtime-host-client.ts index f7471d228d3..a90cba26e82 100644 --- a/apps/desktop/src/main/runtime-host-client.ts +++ b/apps/desktop/src/main/runtime-host-client.ts @@ -867,6 +867,10 @@ export class DesktopRuntimeHostClient { return result.project; } + resolveImageDelivery(sessionId: string, request: import('@maka/core/image-delivery').ImageDeliveryRequest): Promise { + return this.request('artifact.image.resolve', { ...request, sessionId }); + } + async listArtifacts(sessionId: string): Promise { for (let attempt = 0; attempt < MAX_OPTIMISTIC_ATTEMPTS; attempt += 1) { const first = await this.request("artifact.query", { diff --git a/apps/desktop/src/preload/bridge-contract.d.ts b/apps/desktop/src/preload/bridge-contract.d.ts index 5287b9e34f0..9a99b44739f 100644 --- a/apps/desktop/src/preload/bridge-contract.d.ts +++ b/apps/desktop/src/preload/bridge-contract.d.ts @@ -1745,6 +1745,7 @@ export interface MakaBridge { | { ok: true; base64: string; mimeType: string } | { ok: false; reason: string } >; + resolveImageDelivery: import('@maka/core/image-delivery').ResolveImageDelivery; readBytes(sessionId: string, artifactId: string): Promise; }; search: { diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts index aba743dc2cd..1b978bf0546 100644 --- a/apps/desktop/src/preload/preload.ts +++ b/apps/desktop/src/preload/preload.ts @@ -3422,6 +3422,7 @@ const makaBridge = { }, }, attachments: { + resolveImageDelivery: (sessionId, request) => invokeSessionRuntimeHost('attachments:resolveImage', sessionId, request), pickDirectory: () => invokeWhenReady('directories:pick'), // The renderer hands over the dropped or pasted File objects, never paths: // only a File backed by something the user dropped or pasted has a path, diff --git a/apps/desktop/src/renderer/chat-message-surface.tsx b/apps/desktop/src/renderer/chat-message-surface.tsx index eb4e3c5801e..163e62d2142 100644 --- a/apps/desktop/src/renderer/chat-message-surface.tsx +++ b/apps/desktop/src/renderer/chat-message-surface.tsx @@ -58,6 +58,7 @@ interface ChatMessageSurfaceProps extends Omit< | 'pendingQuotes' | 'onQuoteAnnotationSubmit' | 'onReadAttachmentBytes' + | 'onResolveImageDelivery' > { /** * #1985: the live projection and the shell-run records are the only session diff --git a/apps/desktop/src/renderer/features/conversation/staging-services.ts b/apps/desktop/src/renderer/features/conversation/staging-services.ts index 98002392b88..9cbfb55c767 100644 --- a/apps/desktop/src/renderer/features/conversation/staging-services.ts +++ b/apps/desktop/src/renderer/features/conversation/staging-services.ts @@ -20,10 +20,12 @@ import { createServicesContext } from '../../application/contracts/feature-services.js'; import type { ComposerAttachmentService } from '@maka/ui/use-composer-attachments'; import type { ArtifactBinaryReadResult } from '@maka/core/artifacts'; +import type { ResolveImageDelivery } from '@maka/core/image-delivery'; /** One attachment port: the Composer stages through it and the transcript reads image bytes through it. */ export interface ComposerStagingServices extends ComposerAttachmentService { readBytes(sessionId: string, artifactId: string): Promise; + resolveImageDelivery?: ResolveImageDelivery; } const context = createServicesContext('ComposerStagingServicesProvider'); export const ComposerStagingServicesProvider = context.Provider; diff --git a/apps/desktop/src/renderer/features/conversation/ui/staged-quote-chat-view.tsx b/apps/desktop/src/renderer/features/conversation/ui/staged-quote-chat-view.tsx index 12c55d2a565..78366070d5e 100644 --- a/apps/desktop/src/renderer/features/conversation/ui/staged-quote-chat-view.tsx +++ b/apps/desktop/src/renderer/features/conversation/ui/staged-quote-chat-view.tsx @@ -24,8 +24,8 @@ import { useComposerStagingServices } from '../staging-services.js'; /** Transcript attachment reads come from the injected attachment port, never from the caller. */ export function StagedQuoteChatView(props: Omit, - 'handleRef' | 'pendingQuotes' | 'onQuoteAnnotationSubmit' | 'onReadAttachmentBytes'>) { + 'handleRef' | 'pendingQuotes' | 'onQuoteAnnotationSubmit' | 'onReadAttachmentBytes' | 'onResolveImageDelivery'>) { const staging = useComposerStaging(); - const { readBytes } = useComposerStagingServices(); - return ; + const { readBytes, resolveImageDelivery } = useComposerStagingServices(); + return ; } diff --git a/apps/desktop/src/renderer/features/workbar/ports.ts b/apps/desktop/src/renderer/features/workbar/ports.ts index f74b022e461..1c5aa70672e 100644 --- a/apps/desktop/src/renderer/features/workbar/ports.ts +++ b/apps/desktop/src/renderer/features/workbar/ports.ts @@ -17,6 +17,7 @@ * under the License. */ +import type { ResolveImageDelivery } from '@maka/core/image-delivery'; import type { SessionInspectorService } from '../../application/contracts/session-inspector/service.js'; export type { SessionInspectorService, SessionTracePage, SessionUsageSummary } from '../../application/contracts/session-inspector/service.js'; @@ -166,6 +167,7 @@ export interface WorkbarArtifactsService { export interface WorkbarAttachmentsService { readBytes(sessionId: string, artifactId: string): Promise; + resolveImageDelivery?: ResolveImageDelivery; pickFiles(): Promise< | { ok: true; diff --git a/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx b/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx index 29b7a102300..0a4b8e8af21 100644 --- a/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx +++ b/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx @@ -404,7 +404,7 @@ export function QuoteCompanionPanel(props: { liveTurns={companion.liveTurns} activeTurn={companion.activeTurn} activeSession={companion.companionSession} - onReadAttachmentBytes={attachments.readBytes} + onReadAttachmentBytes={attachments.readBytes} onResolveImageDelivery={attachments.resolveImageDelivery} deriveTurnPresentation={deriveTurnPresentation} onEditUserMessage={(turnId) => { const message = companion.messages.find( diff --git a/apps/desktop/src/renderer/features/workhub/ports.ts b/apps/desktop/src/renderer/features/workhub/ports.ts index 59a81bcf963..62017d11b8d 100644 --- a/apps/desktop/src/renderer/features/workhub/ports.ts +++ b/apps/desktop/src/renderer/features/workhub/ports.ts @@ -18,6 +18,7 @@ */ import type { ArtifactBinaryReadResult } from '@maka/core/artifacts'; +import type { ResolveImageDelivery } from '@maka/core/image-delivery'; import type { ChatModelChoice } from '@maka/core/chat-model-choice'; import type { UiLocale } from '@maka/core/ui-locale'; import type { StoredMessage, SessionSummary } from '@maka/core/session'; @@ -57,6 +58,7 @@ export interface WorkHubServices extends WorkHubWorkspaceServices { }): Promise; readonly attachments: ComposerAttachmentService; readAttachmentBytes(sessionId: string, artifactId: string): Promise; + resolveImageDelivery?: ResolveImageDelivery; prepareAttachments(sessionId: string, items: Array<{ approvalId: string; name: string; mimeType?: string } | { file: File }>): Promise; listActiveInteractions(sessionId: string): Promise; subscribeActiveInteractions(handler: (event: { sessionId: string; interactions: import('@maka/core/events').ActiveInteractionRequestEvent[] }) => void): () => void; diff --git a/apps/desktop/src/renderer/features/workhub/ui/workhub-root.tsx b/apps/desktop/src/renderer/features/workhub/ui/workhub-root.tsx index e6eb0d74eae..f87510db540 100644 --- a/apps/desktop/src/renderer/features/workhub/ui/workhub-root.tsx +++ b/apps/desktop/src/renderer/features/workhub/ui/workhub-root.tsx @@ -408,7 +408,7 @@ export function WorkHubRoot() {
call(services.presentation.openSession(id))} scrollBehavior="auto" onNew={() => composer.current?.focus()} diff --git a/apps/desktop/src/renderer/index.html b/apps/desktop/src/renderer/index.html index 19950c7a2df..ad98a582beb 100644 --- a/apps/desktop/src/renderer/index.html +++ b/apps/desktop/src/renderer/index.html @@ -21,6 +21,7 @@ + { const result = await bridge.workHub.prepareAttachments(sessionId, items); if (!result.ok) throw new AttachmentIngestBlockedError(result.code); diff --git a/docs/astryx-surface-file-inventory.md b/docs/astryx-surface-file-inventory.md index 592f4e7b079..dec60a166d3 100644 --- a/docs/astryx-surface-file-inventory.md +++ b/docs/astryx-surface-file-inventory.md @@ -6,7 +6,7 @@ Generated against `@astryxdesign/core@0.6.3` (195 component exports). Wiki bar: Design Conventions · API Use-the-System · Theming · Container Padding. -**Totals:** 334 files — blocker 0, reimplementation 0, polish 4, aligned 330. +**Totals:** 336 files — blocker 0, reimplementation 0, polish 4, aligned 332. ## Exclusions (explicit) @@ -308,11 +308,13 @@ Wiki bar: Design Conventions · API Use-the-System · Theming · Container Paddi | `packages/ui/src/executor-model-picker.tsx` | ui-composition | Button, Popover | aligned — uses Astryx (Button, Popover) | aligned | | `packages/ui/src/form-interaction-prompt.tsx` | ui-composition | Button, CheckboxInput, RadioList, RadioListItem, Selector, Text, TextInput | aligned — uses Astryx (Button, CheckboxInput, RadioList, RadioListItem, Selector, Text, TextInput) | aligned | | `packages/ui/src/icons.tsx` | ui-composition | none | aligned — no raw controls; no Astryx JSX usage | aligned | +| `packages/ui/src/image-delivery.tsx` | ui-composition | none | aligned — no raw controls; no Astryx JSX usage | aligned | | `packages/ui/src/inline-reference.tsx` | ui-composition | ChatTokenizedText | aligned — uses Astryx (ChatTokenizedText) | aligned | | `packages/ui/src/inline-rename-input.tsx` | ui-composition | TextInput | aligned — uses Astryx (TextInput) | aligned | | `packages/ui/src/locale-context.tsx` | ui-composition | none | aligned — no raw controls; no Astryx JSX usage | aligned | | `packages/ui/src/maka-wordmark.tsx` | ui-composition | none | aligned — no raw controls; no Astryx JSX usage | aligned | | `packages/ui/src/markdown-body.tsx` | ui-composition | CodeBlock, Link, Markdown | aligned — uses Astryx (CodeBlock, Link, Markdown) | aligned | +| `packages/ui/src/markdown-image.tsx` | ui-composition | Button, IconButton, Link, Spinner, Tooltip | aligned — uses Astryx (Button, IconButton, Link, Spinner, Tooltip) | aligned | | `packages/ui/src/markdown-math.tsx` | ui-composition | none | aligned — no raw controls; no Astryx JSX usage | aligned | | `packages/ui/src/markdown.tsx` | ui-composition | none | aligned — no raw controls; no Astryx JSX usage | aligned | | `packages/ui/src/mermaid-diagram.tsx` | ui-composition | Button, CodeBlock, Collapsible, Dialog, IconButton, Toolbar | aligned — uses Astryx (Button, CodeBlock, Collapsible, Dialog, IconButton, Toolbar) | aligned | diff --git a/docs/astryx-surface-file-inventory.paths b/docs/astryx-surface-file-inventory.paths index 6fab66f4bd7..ebae26ff537 100644 --- a/docs/astryx-surface-file-inventory.paths +++ b/docs/astryx-surface-file-inventory.paths @@ -278,11 +278,13 @@ packages/ui/src/directory-reference-chip.tsx packages/ui/src/executor-model-picker.tsx packages/ui/src/form-interaction-prompt.tsx packages/ui/src/icons.tsx +packages/ui/src/image-delivery.tsx packages/ui/src/inline-reference.tsx packages/ui/src/inline-rename-input.tsx packages/ui/src/locale-context.tsx packages/ui/src/maka-wordmark.tsx packages/ui/src/markdown-body.tsx +packages/ui/src/markdown-image.tsx packages/ui/src/markdown-math.tsx packages/ui/src/markdown.tsx packages/ui/src/mermaid-diagram.tsx diff --git a/docs/windows-test-inventory.md b/docs/windows-test-inventory.md index 7e7ff70fade..8b6330adaed 100644 --- a/docs/windows-test-inventory.md +++ b/docs/windows-test-inventory.md @@ -17,9 +17,9 @@ Locations intentionally omit line numbers so unrelated edits do not invalidate t |---|---:| | windows-backend-gap | 27 | | portable-candidate | 57 | -| platform-contract | 48 | +| platform-contract | 49 | -Total Windows-excluded declarations: **132** +Total Windows-excluded declarations: **133** ## Inventory @@ -87,6 +87,7 @@ Total Windows-excluded declarations: **132** | platform-contract | `packages/runtime/src/__tests__/filesystem-worker-process-runner.test.ts` filesystem worker rejects boundedly when a detached descendant retains stdout | `process.platform === 'win32' ? 'POSIX detached process-group semantics required' : false` | | platform-contract | `packages/runtime/src/__tests__/filesystem-worker-smoke.test.ts` macOS filesystem worker smoke | `process.platform !== 'darwin'` | | platform-contract | `packages/runtime/src/__tests__/glob-search.test.ts` both Glob paths report permission failures and recover after permissions are restored | `process.platform === 'win32' \|\| process.getuid?.() === 0` | +| platform-contract | `packages/runtime/src/__tests__/image-file.test.ts` chat reads reject FIFOs without waiting for a writer | `process.platform === 'win32'` | | portable-candidate | `packages/runtime/src/__tests__/node-pty-write-lifecycle.test.ts` does not carry queued Unix PTY writes past native exit | `process.platform === 'win32' ? 'Unix PTY file-descriptor lifecycle only' : false` | | platform-contract | `packages/runtime/src/__tests__/owned-child-process.test.ts` POSIX cancellation delivers SIGTERM exactly once to the owned command | `process.platform === 'win32' ? 'POSIX detached process-group semantics required' : false` | | platform-contract | `packages/runtime/src/__tests__/owned-child-process.test.ts` supervisor mirrors a command that dies from ${signal} | `process.platform === 'win32' ? 'POSIX detached process-group semantics required' : false` | diff --git a/package-lock.json b/package-lock.json index fc228a1b70c..31e6e65c54b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12546,9 +12546,9 @@ "license": "MIT" }, "node_modules/micromark-util-types": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", - "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.3.tgz", + "integrity": "sha512-oxB2Ik03hI0gv+VNn9tnh1t1YEe9MDPptViAEgfdf3YQHsn0pzGTgCdlSCJXcwhqm8phaEuM7zeEu3QQzVBrPg==", "dev": true, "funding": [ { @@ -17304,7 +17304,8 @@ "version": "0.1.0", "license": "Apache-2.0", "dependencies": { - "intl-messageformat": "11.2.15" + "intl-messageformat": "11.2.15", + "marked": "18.0.11" } }, "packages/core/node_modules/@formatjs/icu-messageformat-parser": { @@ -17516,8 +17517,10 @@ "virtua": "0.52.7" }, "devDependencies": { + "@playwright/test": "^1.63.0", "@types/react": "^19.3.0", "@types/react-dom": "^19.3.0", + "esbuild": "^0.28.2", "linkedom": "^0.18.13" } }, diff --git a/packages/core/package.json b/packages/core/package.json index 85bda2bad27..cdfa2444699 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -64,6 +64,8 @@ "./context-offload": "./dist/context-offload.js", "./tool-result-archive-evidence": "./dist/tool-result-archive-evidence.js", "./attachments": "./dist/attachments.js", + "./image-delivery": "./dist/image-delivery.js", + "./image-markdown": "./dist/image-markdown.js", "./artifacts": "./dist/artifacts.js", "./pet": "./dist/pet.js", "./skill-invocation": "./dist/skill-invocation.js", @@ -179,6 +181,7 @@ "test:dist": "node --test \"dist/**/*.test.js\"" }, "dependencies": { - "intl-messageformat": "11.2.15" + "intl-messageformat": "11.2.15", + "marked": "18.0.11" } } diff --git a/packages/core/src/__tests__/image-markdown.test.ts b/packages/core/src/__tests__/image-markdown.test.ts new file mode 100644 index 00000000000..f65102d253f --- /dev/null +++ b/packages/core/src/__tests__/image-markdown.test.ts @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { markdownImageSources, parseMarkdownImageDestination } from '../image-markdown.js'; +import { IMAGE_MARKDOWN_MAX_LENGTH } from '../image-delivery.js'; + +test('canonical destinations retain document order, references and explicit attachments', () => { + assert.deepEqual( + markdownImageSources( + [ + '![space]( "Title")', + '![reference][pic]', + '![duplicate]()', + '![attachment](maka://runtime/attachments/image-1 "Preview")', + String.raw`![escaped](/tmp/a\(1\).png)`, + '', + '[pic]: ', + '`![code](/tmp/secret.png)`', + '```md', + '![fenced](/tmp/secret2.png)', + '```', + '', + '![incomplete](', + ].join('\n'), + ), + [ + '/tmp/a (1).png', + '/tmp/my "image".png', + 'maka://runtime/attachments/image-1', + '/tmp/a(1).png', + ], + ); +}); + +test('document budget is measured in UTF-16 code units and accepts the boundary', () => { + const image = '![x](/tmp/a.png)'; + const text = image + '\n\n' + '中'.repeat(IMAGE_MARKDOWN_MAX_LENGTH - image.length - 2); + assert.equal(text.length, IMAGE_MARKDOWN_MAX_LENGTH); + assert.deepEqual(markdownImageSources(text), ['/tmp/a.png']); + assert.deepEqual(markdownImageSources(text + '中'), []); +}); + +test('inline destinations require one complete image token with no trailing syntax', () => { + assert.equal(parseMarkdownImageDestination(' "Title"'), '/tmp/my image.png'); + assert.equal(parseMarkdownImageDestination(String.raw`/tmp/a\(1\).png`), '/tmp/a(1).png'); + for (const source of [ + ' { + for (const text of ['https://example.com/![x](/tmp/a.png)', 'www.example.com/![x](/tmp/a.png)']) { + assert.deepEqual(markdownImageSources(text), []); + } + const text = '![x](https://example.com/a?token=secret&v=1)'; + assert.deepEqual(markdownImageSources(text), ['https://example.com/a?token=secret&v=1']); +}); diff --git a/packages/core/src/artifacts.ts b/packages/core/src/artifacts.ts index 171a8b0c66a..1cd87adc5c0 100644 --- a/packages/core/src/artifacts.ts +++ b/packages/core/src/artifacts.ts @@ -117,6 +117,8 @@ export function isArtifactTurnKey(value: unknown): value is string { ); } +import type { ImageDeliveryMetadata } from './image-delivery.js'; + export interface ArtifactDescriptor { id: string; sessionId: string; @@ -129,6 +131,7 @@ export interface ArtifactDescriptor { mimeType?: string; source: ArtifactSource; summary?: string; + imageDelivery?: ImageDeliveryMetadata; } export interface ArtifactRecord extends ArtifactDescriptor { diff --git a/packages/core/src/image-delivery.ts b/packages/core/src/image-delivery.ts new file mode 100644 index 00000000000..c89a0944176 --- /dev/null +++ b/packages/core/src/image-delivery.ts @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** Client-safe delivery contract. No filesystem or networking capabilities live here. */ +export const IMAGE_DELIVERY_IDENTITY_MAX_LENGTH = 512; +export const IMAGE_DELIVERY_SOURCE_MAX_LENGTH = 4096; +/** JavaScript string length, shared by stream capture and Markdown consumers. */ +export const IMAGE_MARKDOWN_MAX_LENGTH = 1024 * 1024; +export function isImageDeliverySource(value: unknown): value is string { + return ( + typeof value === 'string' && + value.length > 0 && + value.length <= IMAGE_DELIVERY_SOURCE_MAX_LENGTH && + !/[\x00-\x1f\x7f]/.test(value) + ); +} +export function isRemoteImageSource(source: string): boolean { + return /^https?:/i.test(source); +} +export function isImageDeliveryRequest(value: unknown): value is ImageDeliveryRequest { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const v = value as Record; + const identity = (s: unknown) => + typeof s === 'string' && + s.length > 0 && + s.length <= IMAGE_DELIVERY_IDENTITY_MAX_LENGTH && + !/[\x00-\x1f\x7f]/.test(s); + return ( + Object.keys(v).every((k) => + ['turnId', 'messageId', 'source', 'retry', 'loadRemote'].includes(k), + ) && + identity(v.turnId) && + identity(v.messageId) && + isImageDeliverySource(v.source) && + (v.retry === undefined || typeof v.retry === 'boolean') && + (v.loadRemote === undefined || typeof v.loadRemote === 'boolean') + ); +} +export const IMAGE_DELIVERY_FAILURES = [ + 'not_found', + 'not_allowed', + 'too_large', + 'unsupported_mime', + 'quota_exceeded', + 'download_failed', + 'read_failed', + 'queue_full', +] as const; +export type ImageDeliveryFailure = (typeof IMAGE_DELIVERY_FAILURES)[number]; +export interface ImageDeliveryRequest { + readonly turnId: string; + readonly messageId: string; + readonly source: string; + readonly retry?: boolean; + /** Client permits remote media loading; the Host also checks application outbound policy. */ + readonly loadRemote?: boolean; +} +/** Stable source identity, independent of retry or display authority. */ +export interface ImageDeliveryIdentity { + readonly sessionId: string; + readonly turnId: string; + readonly messageId: string; + readonly source: string; +} +/** Attempts are workflow state; only successful captures become Artifacts. */ +export type ImageDeliveryAttempt = + | { readonly status: 'pending' } + | { readonly status: 'failed'; readonly reason: ImageDeliveryFailure }; +export function isImageDeliveryAttempt(value: unknown): value is ImageDeliveryAttempt { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const v = value as Record; + return ( + Object.keys(v).every((k) => ['status', 'reason'].includes(k)) && + (v.status === 'pending' + ? v.reason === undefined + : v.status === 'failed' && IMAGE_DELIVERY_FAILURES.includes(v.reason as ImageDeliveryFailure)) + ); +} +export type ImageDeliveryResult = + // Legacy wire spelling: the client has not granted remote display authority. + | { readonly status: 'requires_confirmation' } + | { readonly status: 'pending' } + | { readonly status: 'ready'; readonly artifactId: string } + | { readonly status: 'failed'; readonly reason: ImageDeliveryFailure } + | { readonly status: 'unavailable' }; + +export type ResolveImageDelivery = ( + sessionId: string, + request: ImageDeliveryRequest, +) => Promise; +/** Stored with its Artifact, so session copy/export carries both provenance and bytes. */ +export interface ImageDeliveryMetadata { + readonly messageId: string; + readonly source: string; + readonly status: 'ready'; + readonly contentSha256: string; +} +export interface ImageArchiveLimits { + readonly sessionBytes: number; + readonly workspaceBytes: number; +} +export const DEFAULT_IMAGE_ARCHIVE_LIMITS: ImageArchiveLimits = Object.freeze({ + sessionBytes: 100 * 1024 * 1024, + workspaceBytes: 1024 * 1024 * 1024, +}); +export function isImageDeliveryMetadata(value: unknown): value is ImageDeliveryMetadata { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const v = value as Record; + return ( + Object.keys(v).every((k) => ['messageId', 'source', 'status', 'contentSha256'].includes(k)) && + typeof v.messageId === 'string' && + v.messageId.length > 0 && + v.messageId.length <= IMAGE_DELIVERY_IDENTITY_MAX_LENGTH && + isImageDeliverySource(v.source) && + v.status === 'ready' && + typeof v.contentSha256 === 'string' && + /^[a-f0-9]{64}$/.test(v.contentSha256) + ); +} diff --git a/packages/core/src/image-markdown.ts b/packages/core/src/image-markdown.ts new file mode 100644 index 00000000000..5d823080c69 --- /dev/null +++ b/packages/core/src/image-markdown.ts @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Lexer, Marked } from 'marked'; +import { IMAGE_MARKDOWN_MAX_LENGTH } from './image-delivery.js'; + +const parser = new Marked({ gfm: true }); + +/** Canonical image destinations in real Markdown nodes, including attachment refs. + * Consumers own their source policy; parsing grants no filesystem or network access. + */ +export function markdownImageSources(text: string): string[] { + return [...new Set(markdownImages(text).map((image) => image.source))]; +} + +/** Original token spelling and canonical destination, including reference images. */ +export function markdownImages(text: string): { raw: string; source: string }[] { + if (!text.includes('![') || text.length > IMAGE_MARKDOWN_MAX_LENGTH) return []; + const images: { raw: string; source: string }[] = []; + parser.walkTokens(parser.lexer(text), (token) => { + if (token.type === 'image') images.push({ raw: token.raw, source: token.href }); + }); + return images; +} + +/** Resolve a complete inline destination, rejecting trailing or partial syntax. */ +export function parseMarkdownImageDestination(source: string): string | undefined { + const markdown = `![](${source})`; + const [token] = Lexer.lexInline(markdown, { gfm: true }); + return token?.type === 'image' && token.raw === markdown ? token.href : undefined; +} diff --git a/packages/runtime-host/src/__tests__/chat-image-delivery.test.ts b/packages/runtime-host/src/__tests__/chat-image-delivery.test.ts new file mode 100644 index 00000000000..f7b7c325dab --- /dev/null +++ b/packages/runtime-host/src/__tests__/chat-image-delivery.test.ts @@ -0,0 +1,935 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { test, type TestContext } from 'node:test'; +import { mkdtemp, mkdir, readFile, writeFile, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { createServer } from 'node:http'; +import net from 'node:net'; +import tls from 'node:tls'; +import dns from 'node:dns/promises'; +import { syncBuiltinESMExports } from 'node:module'; +import { createHash } from 'node:crypto'; +import { createImageFileReader, ImageFileReadError } from '@maka/runtime/image-file-reader'; +import { FilesystemWorkerClientError } from '@maka/runtime/filesystem-worker'; +import { openInteractiveArtifactStoreForWrite } from '@maka/storage/artifact-stores'; +import { resolveStorageRoot, tryAcquireInteractiveRootOwner } from '@maka/storage/root-authority'; +import { + ChatImageDeliveryService, + type ChatImageDeliveryPorts, +} from '../server/chat-image-delivery.js'; +import { chatImageSources } from '../server/chat-image-markdown.js'; +import { checkedChatImage, downloadChatImage } from '../server/chat-image-source.js'; +import { IMAGE_DELIVERY_OPERATION_SPECS } from '../protocol/image-delivery.js'; +import { SessionAdmissionGate } from '../server/session-admission-gate.js'; +import { isImageDeliveryMetadata, isImageDeliverySource } from '@maka/core/image-delivery'; +const PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==', + 'base64', +); +// Mock only Node's DNS/transport boundary. The production downloader still +// checks public addresses, pins DNS, follows redirects and validates HTTP bytes. +// Top-level tests in this file run serially; restore the ESM bindings at teardown. +function publicImageOrigin( + t: TestContext, + localSource: string, + protocol = 'http:', + address = '93.184.216.34', +): string { + const local = new URL(localSource); + const source = new URL(localSource); + source.hostname = 'image.example'; + source.protocol = protocol; + const connect = net.connect; + t.mock.method(dns, 'lookup', async (host: string) => { + assert.equal(host, source.hostname); + return [{ address, family: address.includes(':') ? 6 : 4 }]; + }); + const transport = (options: net.TcpNetConnectOpts & { servername?: string }) => { + assert.equal(options.host, source.hostname); + assert.ok(options.lookup); + options.lookup(source.hostname, { all: true }, (error, addresses) => { + assert.equal(error, null); + assert.deepEqual(addresses, [{ address, family: address.includes(':') ? 6 : 4 }]); + }); + const socket = connect({ host: local.hostname, port: Number(local.port) }); + if (protocol === 'https:') { + assert.equal(options.servername, source.hostname); + // This HTTP fixture exercises HTTPS redirect policy, not TLS verification. + socket.once('connect', () => socket.emit('secureConnect')); + } + return socket; + }; + t.mock.method(net, 'connect', transport); + t.mock.method(tls, 'connect', transport); + syncBuiltinESMExports(); + t.after(() => { + t.mock.restoreAll(); + syncBuiltinESMExports(); + }); + return source.href; +} + +const REQUEST = { + sessionId: 'session-1', + turnId: 'turn-1', + messageId: 'message-1', + source: '/tmp/image.png', +}; +async function fixture( + limits?: { sessionBytes: number; workspaceBytes: number }, + readLocalImage?: ChatImageDeliveryPorts['readLocalImage'], + beforeCreate?: (input: Parameters[0]) => void, + download?: ChatImageDeliveryPorts['download'], +) { + const root = await mkdtemp(join(tmpdir(), 'maka-image-delivery-')); + const owner = await tryAcquireInteractiveRootOwner( + await resolveStorageRoot({ path: root, kind: 'interactive' }), + ); + assert.ok(owner); + const store = await openInteractiveArtifactStoreForWrite(owner.lease); + const authority = { store, close: () => store.close() }; + const errors: unknown[] = []; + const presentationErrors: unknown[] = []; + let remoteAllowed = true; + let reads = 0; + let leases = 0; + const messages = new Map(); + const localReader = createImageFileReader(); + let present = true; + const service = new ChatImageDeliveryService({ + artifacts: { + create: async (input) => { + beforeCreate?.(input); + return authority.store.create(input); + }, + findImageDelivery: authority.store.findImageDelivery, + setImageDeliveryAttempt: authority.store.setImageDeliveryAttempt, + }, + admission: new SessionAdmissionGate(), + limits, + sourceReadTimeoutMs: 1000, + canLoadRemote: async () => remoteAllowed, + download: download ?? downloadChatImage, + isPresent: async () => present, + readMessage: async (i) => messages.get(i.messageId), + readLocalImage: async (sessionId, path, signal) => { + reads++; + if (readLocalImage) return readLocalImage(sessionId, path, signal); + return localReader({ path, cwd: root, abortSignal: signal }); + }, + acquireResidency: () => { + leases++; + return { + release: () => { + leases--; + }, + }; + }, + persistenceFailed: (error) => errors.push(error), + presentationFailed: (error) => presentationErrors.push(error), + }); + return { + root, + owner, + authority, + service, + messages, + errors, + presentationErrors, + allowRemote(value: boolean) { + remoteAllowed = value; + }, + get reads() { + return reads; + }, + get leases() { + return leases; + }, + removeSession: () => { + present = false; + }, + async close() { + await service.close(); + authority.close(); + await owner.close(); + await rm(owner.controlDirectory, { recursive: true, force: true }); + await rm(root, { recursive: true, force: true }); + }, + }; +} + +test('Host drain releases both capture slots even when a source reader ignores cancellation', { + timeout: 2000, +}, async () => { + const f = await fixture(undefined, () => new Promise(() => {})); + try { + observe(f.service, '/tmp/blocked-1.png'); + observe(f.service, '/tmp/blocked-2.png', 'session-1', 'message-2'); + while (f.reads < 2) await new Promise((resolve) => setImmediate(resolve)); + await f.service.close(); + assert.equal(f.leases, 0); + assert.deepEqual(f.errors, []); + } finally { + await f.close(); + } +}); +test('a stalled local source expires within its read budget and can be retried', { + timeout: 15000, +}, async () => { + let reads = 0; + const f = await fixture(undefined, async () => { + if (++reads === 1) return new Promise(() => {}); + return checkedChatImage(PNG); + }); + try { + observe(f.service, REQUEST.source); + await f.service.waitForIdle(); + assert.deepEqual(await f.service.resolve(REQUEST), { status: 'failed', reason: 'read_failed' }); + assert.equal(f.leases, 0); + assert.equal((await f.service.resolve({ ...REQUEST, retry: true })).status, 'pending'); + await f.service.waitForIdle(); + assert.equal((await f.service.resolve(REQUEST)).status, 'ready'); + assert.deepEqual(f.errors, []); + } finally { + await f.close(); + } +}); +function observe( + service: ChatImageDeliveryService, + source: string, + sessionId = REQUEST.sessionId, + messageId = REQUEST.messageId, +) { + service.observe(sessionId, { + id: 'text-event', + type: 'text_complete', + turnId: REQUEST.turnId, + ts: 1, + messageId, + text: `![Screenshot](<${source}>)`, + }); +} +test('parses real Markdown image nodes, including references and balanced destinations, without scanning code or HTML', () => { + const text = [ + '![one]()', + '![two][pic]', + '![titled](https://example.com/titled.png "Screenshot title")', + String.raw`![escaped](/tmp/a\(1\).png)`, + '', + '[pic]: https://example.com/a.png', + '`![code](/tmp/secret.png)`', + '```md', + '![fenced](/tmp/secret2.png)', + '```', + '', + '![incomplete](https://example.com/', + ].join('\n'); + assert.deepEqual(chatImageSources(text), [ + '/tmp/a (1).png', + 'https://example.com/a.png', + 'https://example.com/titled.png', + '/tmp/a(1).png', + ]); +}); +test('automatically saves a local delivery without any UI request; deletion and ArtifactStore reopen do not break replay', async () => { + const f = await fixture(); + try { + const source = join(f.root, 'original.png'); + await writeFile(source, PNG); + observe(f.service, source); + await f.service.waitForIdle(); + assert.equal(f.reads, 1); + assert.equal(f.leases, 0); + assert.deepEqual(f.errors, []); + await rm(source); + const input = { ...REQUEST, source }; + const result = await f.service.resolve(input); + assert.equal(result.status, 'ready'); + if (result.status !== 'ready') return; + f.authority.close(); + const reopenedStore = await openInteractiveArtifactStoreForWrite(f.owner.lease); + const reopened = { store: reopenedStore, close: () => reopenedStore.close() }; + try { + const record = await reopened.store.findImageDelivery( + input.sessionId, + input.turnId, + input.messageId, + source, + ); + assert.equal(record?.status, 'ready'); + const bytes = await reopened.store.readBinaryInSession(input.sessionId, result.artifactId); + assert.equal(bytes.ok, true); + if (bytes.ok) assert.equal(bytes.base64, PNG.toString('base64')); + } finally { + reopened.close(); + } + } finally { + await f.close(); + } +}); + +test('literal percent filenames retain precedence and file URLs are decoded only once', async () => { + const paths: string[] = []; + const reader = createImageFileReader(); + const f = await fixture(undefined, (_session, path, abortSignal) => { + paths.push(path); + return reader({ path, cwd: f.root, abortSignal }); + }); + try { + for (const name of ['literal%20name.png', '100%.png', 'invalid%E4.png']) { + const path = join(f.root, name); + await writeFile(path, PNG); + if (name.includes('%20')) await writeFile(path.replace('%20', ' '), 'wrong source'); + for (const source of [path, pathToFileURL(path).href]) { + paths.length = 0; + observe(f.service, source); + await f.service.waitForIdle(); + assert.equal((await f.service.resolve({ ...REQUEST, source })).status, 'ready', source); + assert.deepEqual(paths, [path]); + } + } + assert.deepEqual(f.errors, []); + } finally { + await f.close(); + } +}); + +test('remote delivery is downloaded once, remains replayable after the origin server disappears, and sends no credentials/referrer', async (t) => { + const f = await fixture(); + let requests = 0; + const server = createServer((req, res) => { + requests++; + assert.equal(req.headers.referer, undefined); + assert.equal(req.headers.cookie, undefined); + assert.equal(req.headers.authorization, undefined); + res.setHeader('Content-Type', 'image/png'); + res.end(PNG); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const source = publicImageOrigin( + t, + `http://127.0.0.1:${(server.address() as import('node:net').AddressInfo).port}/image.png`, + ); + try { + f.messages.set(REQUEST.messageId, `![Screenshot](${source})`); + observe(f.service, source); + await f.service.waitForIdle(); + assert.equal(requests, 0); + assert.deepEqual(await f.service.resolve({ ...REQUEST, source }), { + status: 'requires_confirmation', + }); + assert.equal( + (await f.service.resolve({ ...REQUEST, source, loadRemote: true })).status, + 'pending', + ); + await f.service.waitForIdle(); + assert.deepEqual(f.errors, []); + assert.equal((await f.service.resolve({ ...REQUEST, source })).status, 'ready'); + assert.equal(requests, 1); + await new Promise((resolve) => server.close(() => resolve())); + assert.equal((await f.service.resolve({ ...REQUEST, source })).status, 'ready'); + assert.equal(requests, 1); + } finally { + server.close(); + await f.close(); + } +}); +test('a supplied path is never a read grant; sources inside code cannot trigger archival', async () => { + const f = await fixture(); + try { + f.messages.set(REQUEST.messageId, '`![secret](/tmp/image.png)`'); + assert.deepEqual(await f.service.resolve(REQUEST), { status: 'unavailable' }); + f.messages.set(REQUEST.messageId, 'plain text'); + assert.deepEqual(await f.service.resolve(REQUEST), { status: 'unavailable' }); + assert.equal(f.reads, 0); + } finally { + await f.close(); + } +}); +test('transient failure is persisted and is retried only on an explicit request', async () => { + const f = await fixture(); + try { + const source = join(f.root, 'later.png'); + observe(f.service, source); + await f.service.waitForIdle(); + assert.deepEqual(await f.service.resolve({ ...REQUEST, source }), { + status: 'failed', + reason: 'not_found', + }); + await writeFile(source, PNG); + assert.equal((await f.service.resolve({ ...REQUEST, source })).status, 'failed'); + assert.equal(f.reads, 1); + assert.equal((await f.service.resolve({ ...REQUEST, source, retry: true })).status, 'pending'); + await f.service.waitForIdle(); + assert.equal((await f.service.resolve({ ...REQUEST, source })).status, 'ready'); + assert.equal(f.reads, 2); + } finally { + await f.close(); + } +}); + +test('truncated HTTP images are rejected, and an explicit retry archives the repaired source', async (t) => { + const f = await fixture(); + let content = PNG.subarray(0, 8); + let requests = 0; + const server = createServer((_req, res) => { + requests++; + res.setHeader('Content-Type', 'image/png'); + res.end(content); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const source = publicImageOrigin( + t, + `http://127.0.0.1:${(server.address() as import('node:net').AddressInfo).port}/image.png`, + ); + const identity = { ...REQUEST, source, loadRemote: true }; + try { + f.messages.set(REQUEST.messageId, `![Screenshot](${source})`); + await f.service.resolve(identity); + await f.service.waitForIdle(); + assert.deepEqual(await f.service.resolve(identity), { + status: 'failed', + reason: 'unsupported_mime', + }); + assert.equal(requests, 1); + const failed = await f.authority.store.listPage(REQUEST.sessionId, { offset: 0, limit: 10 }); + assert.equal(failed.total, 0, 'failed delivery is not a file'); + content = PNG; + assert.equal((await f.service.resolve({ ...identity, retry: true })).status, 'pending'); + await f.service.waitForIdle(); + const ready = await f.service.resolve(identity); + assert.equal(ready.status, 'ready'); + assert.equal(requests, 2); + if (ready.status === 'ready') { + const bytes = await f.authority.store.readBinaryInSession( + identity.sessionId, + ready.artifactId, + ); + assert.ok(bytes.ok); + if (bytes.ok) assert.equal(bytes.base64, PNG.toString('base64')); + } + assert.equal( + (await f.authority.store.listPage(identity.sessionId, { offset: 0, limit: 10 })).total, + 1, + ); + assert.deepEqual(f.errors, []); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + await f.close(); + } +}); + +test('ready image retries retain the only saved copy after its source is deleted', async () => { + const f = await fixture(); + const source = join(f.root, 'temporary.png'); + try { + await writeFile(source, PNG); + observe(f.service, source); + await f.service.waitForIdle(); + const identity = { ...REQUEST, source }; + const ready = await f.service.resolve(identity); + assert.equal(ready.status, 'ready'); + await rm(source); + const retries = await Promise.all([ + f.service.resolve({ ...identity, retry: true }), + f.service.resolve({ ...identity, retry: true }), + ]); + assert.deepEqual(retries, [ready, ready]); + assert.equal(f.reads, 1); + assert.equal( + (await f.authority.store.listPage(REQUEST.sessionId, { offset: 0, limit: 10 })).total, + 1, + ); + if (ready.status === 'ready') { + const bytes = await f.authority.store.readBinaryInSession( + REQUEST.sessionId, + ready.artifactId, + ); + assert.ok(bytes.ok); + if (bytes.ok) assert.equal(bytes.base64, PNG.toString('base64')); + } + assert.deepEqual(f.errors, []); + } finally { + await f.close(); + } +}); + +test('Worker read errors retain their image delivery reasons in persisted failures', async () => { + for (const [reason, expected] of [ + ['not_found', 'not_found'], + ['filesystem_denied', 'not_allowed'], + ['image_too_large', 'too_large'], + ['invalid_image', 'unsupported_mime'], + ] as const) { + const reader = createImageFileReader({ + filesystemWorker: { + execute: async () => { + throw new FilesystemWorkerClientError({ + reason, + stage: 'operation', + message: '读取失败', + }); + }, + }, + }); + const f = await fixture(undefined, (_session, path, abortSignal) => + reader({ path, cwd: process.cwd(), abortSignal }), + ); + try { + observe(f.service, REQUEST.source); + await f.service.waitForIdle(); + assert.deepEqual(await f.service.resolve(REQUEST), { status: 'failed', reason: expected }); + assert.deepEqual(f.errors, []); + } finally { + await f.close(); + } + } +}); + +test('pending capture survives persistence failure and is removed only after a durable terminal record', async () => { + let failPublication = true; + const f = await fixture( + undefined, + async () => checkedChatImage(PNG), + (input) => { + if (failPublication && input.imageDelivery?.status === 'ready') + throw new Error('publication failed'); + }, + ); + try { + observe(f.service, REQUEST.source); + await f.service.waitForIdle(); + const pending = await f.authority.store.listPage(REQUEST.sessionId, { offset: 0, limit: 10 }); + assert.equal(pending.total, 0, 'pending capture is not an artifact'); + assert.deepEqual( + await f.authority.store.findImageDelivery( + REQUEST.sessionId, + REQUEST.turnId, + REQUEST.messageId, + REQUEST.source, + ), + { status: 'pending' }, + ); + assert.equal(f.errors.length, 1); + failPublication = false; + assert.equal((await f.service.resolve(REQUEST)).status, 'pending'); + await f.service.waitForIdle(); + const ready = await f.authority.store.listPage(REQUEST.sessionId, { offset: 0, limit: 10 }); + assert.equal(ready.total, 1); + assert.equal(ready.records[0]?.imageDelivery?.status, 'ready'); + assert.equal(f.errors.length, 1); + } finally { + await f.close(); + } +}); + +test('quota admission is atomic under concurrent jobs and identical content shares bytes across sessions', async () => { + const f = await fixture({ sessionBytes: PNG.length, workspaceBytes: PNG.length }); + try { + const source = join(f.root, 'same.png'); + await writeFile(source, PNG); + observe(f.service, source); + observe(f.service, source, 'session-2'); + await f.service.waitForIdle(); + const a = await f.service.resolve({ ...REQUEST, source }); + const b = await f.service.resolve({ ...REQUEST, sessionId: 'session-2', source }); + assert.equal(a.status, 'ready'); + assert.equal(b.status, 'ready'); + if (a.status !== 'ready' || b.status !== 'ready') return; + const ra = (await f.authority.store.getInSession('session-1', a.artifactId)).record!; + const rb = (await f.authority.store.getInSession('session-2', b.artifactId)).record!; + assert.equal( + (await stat(join(f.root, 'artifacts', ra.relativePath))).ino, + (await stat(join(f.root, 'artifacts', rb.relativePath))).ino, + ); + const different = join(f.root, 'different.png'); + await writeFile(different, Buffer.concat([PNG, Buffer.from('different')])); + observe(f.service, different, 'session-1', 'message-2'); + await f.service.waitForIdle(); + assert.deepEqual( + await f.service.resolve({ ...REQUEST, source: different, messageId: 'message-2' }), + { status: 'failed', reason: 'quota_exceeded' }, + ); + await f.authority.store.purgeSessionArtifacts('session-1'); + assert.equal((await f.authority.store.readBinaryInSession('session-2', b.artifactId)).ok, true); + assert.deepEqual(f.errors, []); + } finally { + await f.close(); + } +}); +test('copying a conversation carries delivery provenance and saved bytes; it does not read the original source', async () => { + const f = await fixture(); + try { + const source = join(f.root, 'original.png'); + await writeFile(source, PNG); + observe(f.service, source); + await f.service.waitForIdle(); + await rm(source); + await f.authority.store.copyConversationArtifacts({ + sourceSessionId: 'session-1', + targetSessionId: 'session-copy', + turnIds: ['turn-1'], + }); + const original = await f.service.resolve({ ...REQUEST, source }); + const copied = await f.service.resolve({ ...REQUEST, sessionId: 'session-copy', source }); + assert.equal(copied.status, 'ready'); + if (copied.status === 'ready' && original.status === 'ready') { + const sourceRecord = (await f.authority.store.getInSession('session-1', original.artifactId)) + .record!; + const targetRecord = (await f.authority.store.getInSession('session-copy', copied.artifactId)) + .record!; + assert.equal( + (await stat(join(f.root, 'artifacts', sourceRecord.relativePath))).ino, + (await stat(join(f.root, 'artifacts', targetRecord.relativePath))).ino, + ); + await f.authority.store.purgeSessionArtifacts('session-1'); + assert.equal( + (await f.authority.store.readBinaryInSession('session-copy', copied.artifactId)).ok, + true, + ); + } + assert.equal(f.reads, 1); + } finally { + await f.close(); + } +}); +test('source readers reject unsafe MIME and redirects to private networks', async (t) => { + assert.throws(() => checkedChatImage(Buffer.from('')), /unsupported_mime/); + const server = createServer((_req, res) => { + res.writeHead(302, { location: 'http://169.254.169.254/latest/meta-data/' }); + res.end(); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const source = publicImageOrigin( + t, + `http://127.0.0.1:${(server.address() as import('node:net').AddressInfo).port}/redirect`, + ); + await assert.rejects(downloadChatImage(source, AbortSignal.timeout(2000)), /not_allowed/); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } +}); +test('HTTP download enforces header/stream limits, redirect budget and cancellation', async (t) => { + const server = createServer((req, res) => { + if (req.url === '/length') { + res.writeHead(200, { 'content-length': 2 * 1024 * 1024 + 1 }); + res.end(); + } else if (req.url === '/stream') { + res.writeHead(200); + res.write(Buffer.alloc(2 * 1024 * 1024 + 1)); + res.end(); + } else if (req.url === '/redirect') { + res.writeHead(302, { location: '/redirect' }); + res.end(); + } else if (req.url === '/stalled') { + // Keep the response open until the downloader's cancellation closes it. + } else { + res.writeHead(503); + res.end(); + } + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const source = publicImageOrigin( + t, + `http://127.0.0.1:${(server.address() as import('node:net').AddressInfo).port}/`, + ); + try { + for (const [path, reason] of [ + ['length', 'too_large'], + ['stream', 'too_large'], + ['redirect', 'download_failed'], + ['unavailable', 'download_failed'], + ]) { + await assert.rejects( + downloadChatImage(source + path, AbortSignal.timeout(2000)), + new RegExp(reason), + ); + } + await assert.rejects(downloadChatImage(source + 'stalled', AbortSignal.timeout(50)), { + name: 'TimeoutError', + }); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); + +test('HTTPS download cannot redirect to HTTP', async (t) => { + let requests = 0; + const server = createServer((_req, res) => { + requests++; + res.writeHead(302, { location: 'http://image.example/image.png' }); + res.end(); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const source = publicImageOrigin( + t, + `http://127.0.0.1:${(server.address() as import('node:net').AddressInfo).port}/`, + 'https:', + ); + try { + await assert.rejects(downloadChatImage(source, AbortSignal.timeout(2000)), /not_allowed/); + assert.equal(requests, 1); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } +}); + +test('resolver codec rejects excess fields and noncanonical artifact identities', () => { + const spec = IMAGE_DELIVERY_OPERATION_SPECS['artifact.image.resolve']; + assert.deepEqual(spec.decodeInput(REQUEST), REQUEST); + assert.throws(() => spec.decodeInput({ ...REQUEST, arbitraryRead: true })); + assert.throws(() => spec.decodeOutput({ status: 'ready', artifactId: '../private' })); + assert.deepEqual(spec.decodeInput({ ...REQUEST, loadRemote: true }), { + ...REQUEST, + loadRemote: true, + }); + assert.deepEqual(spec.decodeOutput({ status: 'requires_confirmation' }), { + status: 'requires_confirmation', + }); + assert.throws(() => spec.decodeInput({ ...REQUEST, loadRemote: 'true' })); + assert.throws(() => spec.decodeOutput({ status: 'requires_confirmation', source: '/private' })); +}); + +test('remote capture requires client display authority and application network access, including on retry', async () => { + let downloads = 0; + const f = await fixture(undefined, undefined, undefined, async () => { + downloads++; + return checkedChatImage(PNG); + }); + const source = 'https://example.invalid/image.png?data=secret'; + const identity = { ...REQUEST, source }; + f.messages.set(REQUEST.messageId, `![](${source})`); + try { + observe(f.service, source); + await f.service.waitForIdle(); + assert.equal(downloads, 0); + assert.deepEqual(await f.service.resolve(identity), { status: 'requires_confirmation' }); + f.allowRemote(false); + assert.deepEqual(await f.service.resolve(identity), { + status: 'failed', + reason: 'not_allowed', + }); + assert.deepEqual(await f.service.resolve({ ...identity, loadRemote: true }), { + status: 'failed', + reason: 'not_allowed', + }); + assert.equal(downloads, 0); + assert.equal( + (await f.authority.store.listPage(REQUEST.sessionId, { offset: 0, limit: 10 })).total, + 0, + ); + f.allowRemote(true); + assert.deepEqual(await f.service.resolve({ ...identity, loadRemote: true }), { + status: 'pending', + }); + await f.service.waitForIdle(); + assert.equal(downloads, 1); + const ready = await f.service.resolve(identity); + assert.equal(ready.status, 'ready'); + f.allowRemote(false); + assert.deepEqual(await f.service.resolve({ ...identity, retry: true }), ready); + assert.equal(downloads, 1, 'saved replay/retry never contacts origin'); + assert.deepEqual( + await f.service.resolve({ + ...identity, + source: 'https://example.invalid/forged', + loadRemote: true, + }), + { status: 'unavailable' }, + ); + } finally { + await f.close(); + } +}); + +test('failed remote downloads issue one request and require a client retry', async () => { + let downloads = 0; + const f = await fixture(undefined, undefined, undefined, async () => { + downloads++; + throw new Error('offline'); + }); + const source = 'https://example.invalid/image.png'; + const identity = { ...REQUEST, source }; + f.messages.set(REQUEST.messageId, `![](${source})`); + try { + await f.service.resolve({ ...identity, loadRemote: true }); + await f.service.waitForIdle(); + assert.equal(downloads, 1); + assert.deepEqual(await f.service.resolve({ ...identity, retry: true }), { + status: 'requires_confirmation', + }); + assert.equal(downloads, 1); + await f.service.resolve({ ...identity, retry: true, loadRemote: true }); + await f.service.waitForIdle(); + assert.equal(downloads, 2); + } finally { + await f.close(); + } +}); + +test('a Markdown parser failure stays in presentation and subsequent images still capture', async () => { + const f = await fixture(undefined, async () => checkedChatImage(PNG)); + try { + f.service.observe(REQUEST.sessionId, { + type: 'text_complete', + id: 'deep-text', + ts: 1, + turnId: REQUEST.turnId, + messageId: REQUEST.messageId, + text: '> '.repeat(5000) + '![](/tmp/image.png)', + }); + assert.equal(f.presentationErrors.length, 1); + assert.ok(f.presentationErrors[0] instanceof RangeError); + assert.deepEqual(f.errors, []); + observe(f.service, REQUEST.source); + await f.service.waitForIdle(); + assert.equal((await f.service.resolve(REQUEST)).status, 'ready'); + } finally { + await f.close(); + } +}); + +test('loopback is denied before GET, including redirects from a public origin', async (t) => { + let hits = 0; + const target = createServer((_req, res) => { + hits++; + res.end(PNG); + }); + await new Promise((resolve) => target.listen(0, '127.0.0.1', resolve)); + const targetUrl = `http://127.0.0.1:${(target.address() as import('node:net').AddressInfo).port}/image.png`; + const origin = createServer((_req, res) => { + res.writeHead(302, { location: targetUrl }); + res.end(); + }); + await new Promise((resolve) => origin.listen(0, '127.0.0.1', resolve)); + try { + const source = publicImageOrigin( + t, + `http://127.0.0.1:${(origin.address() as import('node:net').AddressInfo).port}/redirect`, + ); + for (const url of [targetUrl, 'http://[::1]:80/image.png']) + await assert.rejects(downloadChatImage(url, AbortSignal.timeout(1000)), /not_allowed/); + await assert.rejects(downloadChatImage(source, AbortSignal.timeout(1000)), /not_allowed/); + assert.equal(hits, 0); + } finally { + await Promise.all( + [target, origin].map( + (server) => new Promise((resolve) => server.close(() => resolve())), + ), + ); + } +}); + +test('capture, wire and stored metadata share source limits while retaining boundary policies', () => { + const decode = IMAGE_DELIVERY_OPERATION_SPECS['artifact.image.resolve'].decodeInput; + const source = '/' + 'a'.repeat(4095); + assert.deepEqual(chatImageSources(`![x](<${source}>)`), [source]); + assert.equal(decode({ ...REQUEST, source, messageId: 'm'.repeat(512) }).source, source); + assert.throws(() => decode({ ...REQUEST, messageId: 'm'.repeat(513) })); + for (const source of ['', '/' + 'a'.repeat(4096), '/tmp/a\x01.png', '/tmp/a\x7f.png']) { + assert.equal(isImageDeliverySource(source), false); + assert.equal( + isImageDeliveryMetadata({ + messageId: 'm', + source, + status: 'ready', + contentSha256: 'a'.repeat(64), + }), + false, + ); + assert.throws(() => decode({ ...REQUEST, source })); + assert.deepEqual(chatImageSources(`![x](<${source}>)`), []); + } + // Explicit attachment refs belong to UI resolution, not automatic source capture. + assert.deepEqual(chatImageSources('![x](maka://runtime/attachments/image-1)'), []); + assert.equal( + isImageDeliveryMetadata({ + messageId: 'm\x01', + source: REQUEST.source, + status: 'ready', + contentSha256: 'a'.repeat(64), + }), + true, + ); + assert.throws(() => decode({ ...REQUEST, messageId: 'm\x01' })); + assert.equal( + chatImageSources(Array.from({ length: 70 }, (_, i) => `![x](/tmp/${i}.png)`).join('\n')).length, + 64, + ); +}); + +test('local and downloaded invalid image bytes retain identical failure reasons', async () => { + const f = await fixture(); + try { + for (const [bytes, reason] of [ + [PNG.subarray(0, 8), 'unsupported_mime'], + [new Uint8Array(2 * 1024 * 1024 + 1), 'too_large'], + ] as const) { + assert.throws( + () => checkedChatImage(bytes), + (error: unknown) => error instanceof Error && 'reason' in error && error.reason === reason, + ); + const source = join(f.root, 'invalid.png'); + await writeFile(source, bytes); + await assert.rejects( + createImageFileReader()({ path: source, cwd: f.root }), + (error: unknown) => error instanceof Error && 'reason' in error && error.reason === reason, + ); + } + } finally { + await f.close(); + } +}); + +test('a queued retry reports pending instead of replaying the previous failure', async () => { + let release!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + let fail = true; + const f = await fixture(undefined, async (_sessionId, path) => { + if (path !== REQUEST.source) await blocked; + else if (fail) throw new ImageFileReadError('not_found'); + return checkedChatImage(PNG); + }); + try { + observe(f.service, REQUEST.source); + await f.service.waitForIdle(); + assert.deepEqual(await f.service.resolve(REQUEST), { status: 'failed', reason: 'not_found' }); + observe(f.service, '/tmp/blocker-one.png'); + observe(f.service, '/tmp/blocker-two.png'); + while (f.reads < 3) await new Promise((resolve) => setImmediate(resolve)); + fail = false; + assert.deepEqual(await f.service.resolve({ ...REQUEST, retry: true }), { status: 'pending' }); + assert.deepEqual(await f.service.resolve(REQUEST), { status: 'pending' }); + release(); + await f.service.waitForIdle(); + assert.equal((await f.service.resolve(REQUEST)).status, 'ready'); + assert.deepEqual(f.errors, []); + } finally { + release(); + await f.close(); + } +}); diff --git a/packages/runtime-host/src/__tests__/execution-composition.test.ts b/packages/runtime-host/src/__tests__/execution-composition.test.ts index 501ae0b66ba..8cfb027ac1c 100644 --- a/packages/runtime-host/src/__tests__/execution-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-composition.test.ts @@ -21,6 +21,9 @@ import assert from 'node:assert/strict'; import { execFileSync } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { createServer } from 'node:http'; +import net from 'node:net'; +import dns from 'node:dns/promises'; +import { syncBuiltinESMExports } from 'node:module'; import { WORKHUB_COORDINATION_SESSION_ID } from '@maka/core/session'; import { TOOL_BOUNDARY_PROTOCOL_V1, @@ -4629,3 +4632,311 @@ async function actWorkHub( await desktop.close(); } } + +test('production Host automatically archives assistant Markdown images and serves them after source deletion', async () => { + await withCompositionRoot(async ({ root, owner }) => { + const source = join(root, 'delivery.png'); + const png = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==', + 'base64', + ); + await writeFile(source, png); + const captured = await createCapturedExecutionComposition(owner, { + primaryBackendFactory: (context) => + new (class extends FakeBackend { + override async *send(input: BackendSendInput): AsyncIterable { + yield { + type: 'text_complete', + id: 'delivery-text-event', + turnId: input.turnId, + ts: Date.now(), + messageId: 'delivered-message', + text: `![Screenshot](<${source}>)`, + }; + yield { + type: 'complete', + id: 'delivery-complete-event', + turnId: input.turnId, + ts: Date.now(), + stopReason: 'end_turn', + }; + } + })(context), + context: { + retainUntilProcessExit: () => undefined, + requestDrain: () => assert.fail('image delivery must not drain Host'), + }, + }); + try { + const session = await captured.manager.createSession({ + cwd: root, + llmConnectionId: FAKE_CONNECTION_ID, + llmConnectionSlug: 'fake', + model: 'fake-model', + permissionMode: 'bypass', + }); + const context = { + hostEpoch: 'execution-composition-test', + connectionId: 'image-client', + principal: 'local_os_user', + acquireResidency: () => ({ release() {} }), + }; + const started = await captured.composition.handlers['turn.start']( + { + sessionId: session.id, + turnId: 'image-delivery-turn', + content: { text: 'deliver screenshot' }, + }, + context, + ); + assert.equal(started.ok, true, JSON.stringify(started)); + const request = { + sessionId: session.id, + turnId: 'image-delivery-turn', + messageId: 'delivered-message', + source, + }; + let artifactId: string | undefined; + await waitFor(async () => { + const result = await captured.composition.handlers['artifact.image.resolve']( + request, + context, + ); + assert.equal(result.ok, true, JSON.stringify(result)); + if (result.ok && result.result.status === 'ready') artifactId = result.result.artifactId; + return !!artifactId; + }, 5000); + await rm(source); + const replay = await captured.composition.handlers['artifact.image.resolve']( + request, + context, + ); + assert.deepEqual(replay, { ok: true, result: { status: 'ready', artifactId } }); + const binary = await captured.composition.handlers['artifact.query']( + { sessionId: session.id, kind: 'read_binary', artifactId: artifactId! }, + context, + ); + assert.equal(binary.ok, true, JSON.stringify(binary)); + if (binary.ok && binary.result.kind === 'binary' && binary.result.preview.ok) + assert.equal(binary.result.preview.base64, png.toString('base64')); + const forged = await captured.composition.handlers['artifact.image.resolve']( + { ...request, source: '/tmp/private.png' }, + context, + ); + assert.deepEqual(forged, { ok: true, result: { status: 'unavailable' } }); + } finally { + captured.composition.beginDrain(); + await captured.composition.close(); + } + }); +}); + +test('production Host rejects loopback image destinations even when a restricted-session client permits media loading', async () => { + await withCompositionRoot(async ({ root, owner }) => { + let requests = 0; + const server = createServer((_request, response) => { + requests++; + response.end(); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const source = `http://127.0.0.1:${(server.address() as import('node:net').AddressInfo).port}/secret.png`; + const captured = await createCapturedExecutionComposition(owner, { + primaryBackendFactory: (context) => + new (class extends FakeBackend { + override async *send(input: BackendSendInput): AsyncIterable { + yield { + type: 'text_complete', + id: `text-${input.turnId}`, + turnId: input.turnId, + ts: Date.now(), + messageId: `message-${input.turnId}`, + text: `![](${source})`, + }; + yield { + type: 'complete', + id: `complete-${input.turnId}`, + turnId: input.turnId, + ts: Date.now(), + stopReason: 'end_turn', + }; + } + })(context), + context: { + retainUntilProcessExit: () => undefined, + requestDrain: () => assert.fail('remote images must not drain Host'), + }, + }); + try { + for (const permissionMode of ['explore', 'ask'] as const) { + const session = await captured.manager.createSession({ + cwd: root, + llmConnectionId: FAKE_CONNECTION_ID, + llmConnectionSlug: 'fake', + model: 'fake-model', + permissionMode, + }); + const context = { + hostEpoch: 'execution-composition-test', + connectionId: 'image-client', + principal: 'local_os_user', + acquireResidency: () => ({ release() {} }), + }; + const turnId = `remote-${permissionMode}`; + assert.equal( + ( + await captured.composition.handlers['turn.start']( + { sessionId: session.id, turnId, content: { text: 'deliver image' } }, + context, + ) + ).ok, + true, + ); + const request = { + sessionId: session.id, + turnId, + messageId: `message-${turnId}`, + source, + loadRemote: true, + }; + await waitFor(async () => { + const result = await captured.composition.handlers['artifact.image.resolve']( + request, + context, + ); + return ( + result.ok && result.result.status === 'failed' && result.result.reason === 'not_allowed' + ); + }, 5000); + assert.deepEqual( + await captured.composition.handlers['artifact.image.resolve']( + { ...request, loadRemote: true }, + context, + ), + { ok: true, result: { status: 'failed', reason: 'not_allowed' } }, + ); + } + assert.equal(requests, 0); + } finally { + captured.composition.beginDrain(); + await captured.composition.close(); + await new Promise((resolve) => server.close(() => resolve())); + } + }); +}); + +test('production Host loads visible remote media independently of agent sandbox networking and respects app privacy', async (t) => { + const png = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==', + 'base64', + ); + let requests = 0; + const server = createServer((_request, response) => { + requests++; + response.end(png); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const origin = `http://127.0.0.1:${(server.address() as import('node:net').AddressInfo).port}`; + const realConnect = net.connect; + t.mock.method(dns, 'lookup', async (host: string) => { + assert.equal(host, 'image.example'); + return [{ address: '93.184.216.34', family: 4 }]; + }); + t.mock.method(net, 'connect', (options: net.TcpNetConnectOpts) => { + assert.equal(options.host, 'image.example'); + assert.ok(options.lookup); + options.lookup(options.host, { all: true }, (error, addresses) => { + assert.equal(error, null); + assert.deepEqual(addresses, [{ address: '93.184.216.34', family: 4 }]); + }); + return realConnect({ host: '127.0.0.1', port: Number(new URL(origin).port) }); + }); + syncBuiltinESMExports(); + try { + await withCompositionRoot(async ({ root, owner }) => { + const captured = await createCapturedExecutionComposition(owner, { + primaryBackendFactory: (context) => + new (class extends FakeBackend { + override async *send(input: BackendSendInput): AsyncIterable { + yield { + type: 'text_complete', + id: `text-${input.turnId}`, + turnId: input.turnId, + ts: Date.now(), + messageId: `message-${input.turnId}`, + text: `![](http://image.example/${input.turnId}.png)`, + }; + yield { + type: 'complete', + id: `complete-${input.turnId}`, + turnId: input.turnId, + ts: Date.now(), + stopReason: 'end_turn', + }; + } + })(context), + }); + const context = { + hostEpoch: 'execution-composition-test', + connectionId: 'image-client', + principal: 'local_os_user', + acquireResidency: () => ({ release() {} }), + }; + try { + for (const [permissionMode, privacy] of [ + ['explore', false], + ['ask', false], + ['bypass', true], + ] as const) { + const policy = await captured.composition.handlers['runtime.policy.query']({}, context); + assert.ok(policy.ok); + const changed = await captured.composition.handlers['runtime.policy.mutate']( + { + expectedRevision: policy.result.revision, + operation: { kind: 'set_privacy', value: { incognitoActive: privacy } }, + }, + context, + ); + assert.ok(changed.ok); + const session = await captured.manager.createSession({ + cwd: root, + llmConnectionId: FAKE_CONNECTION_ID, + llmConnectionSlug: 'fake', + model: 'fake-model', + permissionMode, + }); + const turnId = `media-${permissionMode}`; + const started = await captured.composition.handlers['turn.start']( + { sessionId: session.id, turnId, content: { text: 'display image' } }, + context, + ); + assert.ok(started.ok, JSON.stringify(started)); + const request = { + sessionId: session.id, + turnId, + messageId: `message-${turnId}`, + source: `http://image.example/${turnId}.png`, + loadRemote: true, + }; + await waitFor(async () => { + const result = await captured.composition.handlers['artifact.image.resolve']( + request, + context, + ); + assert.ok(result.ok); + return privacy + ? result.result.status === 'failed' && result.result.reason === 'not_allowed' + : result.result.status === 'ready'; + }, 5000); + } + assert.equal(requests, 2, 'privacy mode must not issue a third media request'); + } finally { + captured.composition.beginDrain(); + await captured.composition.close(); + } + }); + } finally { + t.mock.restoreAll(); + syncBuiltinESMExports(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); diff --git a/packages/runtime-host/src/__tests__/protocol.test.ts b/packages/runtime-host/src/__tests__/protocol.test.ts index bce0ba4d4aa..db6b5115544 100644 --- a/packages/runtime-host/src/__tests__/protocol.test.ts +++ b/packages/runtime-host/src/__tests__/protocol.test.ts @@ -133,6 +133,9 @@ describe('Runtime Host bootstrap protocol', () => { // field, so mixed-version peers must fail during the handshake instead. assert.ok(RUNTIME_HOST_COMPATIBILITY_EPOCH > 22); }); + test('rejects epoch-220 peers predating image resolution', () => { + assert.ok(RUNTIME_HOST_COMPATIBILITY_EPOCH > 220); + }); test('publishes a new compatibility epoch for Artifact invalidation frames', () => { assert.ok(RUNTIME_HOST_COMPATIBILITY_EPOCH > 196); diff --git a/packages/runtime-host/src/protocol/artifact.ts b/packages/runtime-host/src/protocol/artifact.ts index 08e45820c06..a5f2c254ce2 100644 --- a/packages/runtime-host/src/protocol/artifact.ts +++ b/packages/runtime-host/src/protocol/artifact.ts @@ -17,6 +17,7 @@ * under the License. */ +import { IMAGE_DELIVERY_OPERATION_SPECS } from './image-delivery.js'; import { ARTIFACT_KINDS, ARTIFACT_SOURCES, @@ -184,6 +185,7 @@ export type ArtifactIngestResult = | { readonly kind: 'upload_aborted'; readonly uploadId: string }; export const ARTIFACT_OPERATION_SPECS = { + ...IMAGE_DELIVERY_OPERATION_SPECS, 'artifact.ingest': defineOperation< ArtifactIngestInput, ArtifactIngestResult, diff --git a/packages/runtime-host/src/protocol/image-delivery.ts b/packages/runtime-host/src/protocol/image-delivery.ts new file mode 100644 index 00000000000..b5e44e65c40 --- /dev/null +++ b/packages/runtime-host/src/protocol/image-delivery.ts @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + IMAGE_DELIVERY_FAILURES, + isImageDeliveryRequest, + type ImageDeliveryRequest, + type ImageDeliveryResult, +} from '@maka/core/image-delivery'; +import { isCanonicalArtifactEntityId } from '@maka/core/artifacts'; +import { + requireExactRecord, + requireEntityId, + requireRecord, + requireShapedRecord, +} from './codec.js'; +import { invalidProtocolFrame } from './errors.js'; +import { defineOperation } from './operation-spec.js'; +export interface ResolveImageDeliveryInput extends ImageDeliveryRequest { + readonly sessionId: string; +} +export const IMAGE_DELIVERY_OPERATION_SPECS = { + 'artifact.image.resolve': defineOperation< + ResolveImageDeliveryInput, + ImageDeliveryResult, + | 'host_not_ready' + | 'host_draining' + | 'operation_unavailable' + | 'invalid_request' + | 'not_found' + | 'internal_failure' + >({ + mode: 'command', + availability: 'ready', + errors: [ + 'host_not_ready', + 'host_draining', + 'operation_unavailable', + 'invalid_request', + 'not_found', + 'internal_failure', + ], + decodeInput(value) { + const { sessionId, ...request } = requireShapedRecord( + value, + 'image delivery request', + ['sessionId', 'turnId', 'messageId', 'source'], + ['retry', 'loadRemote'], + ); + if (!isImageDeliveryRequest(request)) + throw invalidProtocolFrame('Invalid image delivery request'); + const { retry, loadRemote, ...identity } = request; + return { + sessionId: requireEntityId(sessionId, 'sessionId'), + ...identity, + ...(retry === true ? { retry: true } : {}), + ...(loadRemote === true ? { loadRemote: true } : {}), + }; + }, + decodeOutput(value) { + const v = requireRecord(value, 'image delivery result'); + if ( + v.status === 'pending' || + v.status === 'unavailable' || + v.status === 'requires_confirmation' + ) { + requireExactRecord(v, 'image delivery status', ['status']); + return { status: v.status }; + } + if (v.status === 'ready' && isCanonicalArtifactEntityId(v.artifactId)) { + requireExactRecord(v, 'image delivery ready', ['status', 'artifactId']); + return { status: 'ready', artifactId: v.artifactId }; + } + if (v.status === 'failed' && IMAGE_DELIVERY_FAILURES.includes(v.reason as never)) { + requireExactRecord(v, 'image delivery failure', ['status', 'reason']); + return { + status: 'failed', + reason: v.reason as Extract['reason'], + }; + } + throw invalidProtocolFrame('Invalid image delivery result'); + }, + }), +} as const; diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index a2b5157ce9f..ac80e7aad17 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -106,7 +106,10 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const; export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const; // Increment when the same protocol version no longer guarantees safe Client-Host // interoperability. Mismatches are rejected before domain commands are admitted. -export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 220 as const; +export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 221 as const; +// 221: Epoch-220 peers do not support `artifact.image.resolve` or its image +// delivery result shapes. Older Hosts reject the unknown operation and close +// the connection, so mixed peers must fail admission. // 220: Client Capability results require a tri-state outcome; session tool // continuity and trace steps carry interrupted results. Older peers cannot // decode the changed strict shapes. diff --git a/packages/runtime-host/src/protocol/operations.ts b/packages/runtime-host/src/protocol/operations.ts index fba841c427c..1f500dfca4d 100644 --- a/packages/runtime-host/src/protocol/operations.ts +++ b/packages/runtime-host/src/protocol/operations.ts @@ -262,6 +262,7 @@ export const REMOTE_OWNER_OPERATION_GRANTS = Object.freeze([ 'agent.graph.stop', 'artifact.delete', 'artifact.ingest', + 'artifact.image.resolve', 'artifact.query', 'client.capability.replace', 'client.capability.unregister', diff --git a/packages/runtime-host/src/server/artifact-coordinator.ts b/packages/runtime-host/src/server/artifact-coordinator.ts index 78bfe7de68c..472b9fa4f8d 100644 --- a/packages/runtime-host/src/server/artifact-coordinator.ts +++ b/packages/runtime-host/src/server/artifact-coordinator.ts @@ -17,6 +17,7 @@ * under the License. */ +import type { ChatImageDeliveryService } from './chat-image-delivery.js'; import { JsonArrayPageBudget } from './json-array-page-budget.js'; import { createHash } from 'node:crypto'; @@ -63,6 +64,13 @@ interface ArtifactUploadMetadata { /** Session-scoped Host projection and deletion authority for Artifacts. */ export class HostArtifactCoordinator { readonly handlers: ArtifactOperationHandlerMap = { + 'artifact.image.resolve': async (input, context) => { + if (context.principalKind === 'session_guest' || !this.imageDelivery) + return { ok: true, result: { status: 'unavailable' } }; + if ((await this.#sessions.probeSessionRemoval(input.sessionId)).kind !== 'present') + return { ok: false, error: { code: 'not_found', message: 'Session was not found' } }; + return { ok: true, result: await this.imageDelivery.resolve(input) }; + }, 'artifact.ingest': (input, context) => this.#sessionAdmission.run(input.sessionId, () => this.#ingest(input, context)), 'artifact.query': (input, context) => @@ -89,6 +97,7 @@ export class HostArtifactCoordinator { now: () => number = Date.now, sessionAccessAuthority?: Pick, onArtifactDeleted?: (sessionId: string, artifactId: string) => void, + private readonly imageDelivery?: ChatImageDeliveryService, ) { this.#store = authenticateInteractiveArtifactStoreWriter(store); this.#requestDrain = requestDrain; diff --git a/packages/runtime-host/src/server/chat-image-artifact.ts b/packages/runtime-host/src/server/chat-image-artifact.ts new file mode 100644 index 00000000000..b2ee0b7a1af --- /dev/null +++ b/packages/runtime-host/src/server/chat-image-artifact.ts @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createHash } from 'node:crypto'; +import { DEFAULT_IMAGE_ARCHIVE_LIMITS, type ImageArchiveLimits } from '@maka/core/image-delivery'; +import type { InteractiveArtifactStoreWriter } from '@maka/storage/artifact-stores'; +import type { ChatImageBytes } from './chat-image-source.js'; + +/** Build the shared ready record; callers own admission, identity and failures. */ +export function readyChatImageArtifact(input: { + id: string; + sessionId: string; + turnId: string; + name: string; + messageId: string; + source: string; + image: ChatImageBytes; + summary?: string; + limits?: ImageArchiveLimits; +}): Parameters[0] { + return { + id: input.id, + sessionId: input.sessionId, + turnId: input.turnId, + name: input.name, + kind: 'image', + content: input.image.bytes, + mimeType: input.image.mimeType, + source: 'tool_result_projection', + ...(input.summary !== undefined ? { summary: input.summary } : {}), + imageDelivery: { + messageId: input.messageId, + source: input.source, + status: 'ready', + contentSha256: createHash('sha256').update(input.image.bytes).digest('hex'), + }, + imageArchiveLimits: input.limits ?? DEFAULT_IMAGE_ARCHIVE_LIMITS, + }; +} diff --git a/packages/runtime-host/src/server/chat-image-delivery.ts b/packages/runtime-host/src/server/chat-image-delivery.ts new file mode 100644 index 00000000000..d8af4f4fb9f --- /dev/null +++ b/packages/runtime-host/src/server/chat-image-delivery.ts @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createHash } from 'node:crypto'; +import { ImageFileReadError } from '@maka/runtime/image-file-reader'; +import type { TextCompleteEvent } from '@maka/core/events'; +import { + IMAGE_MARKDOWN_MAX_LENGTH, + isRemoteImageSource, + type ImageArchiveLimits, + type ImageDeliveryRequest, + type ImageDeliveryResult, + type ImageDeliveryFailure, + type ImageDeliveryIdentity, + type ImageDeliveryAttempt, +} from '@maka/core/image-delivery'; +import { + ImageArchiveQuotaError, + type InteractiveArtifactStoreWriter, +} from '@maka/storage/artifact-stores'; +import type { SessionAdmissionGate } from './session-admission-gate.js'; +import { chatImageSources } from './chat-image-markdown.js'; +import { readyChatImageArtifact } from './chat-image-artifact.js'; +import { abortable } from '../client/wait-for-ready.js'; +import { + downloadChatImage, + ImageSourceError, + localImagePath, + type ChatImageBytes, +} from './chat-image-source.js'; + +type DeliveryIdentity = ImageDeliveryIdentity & ImageDeliveryRequest; +interface DeliveryJob { + readonly done: Promise; + run(): Promise; + cancel(): void; +} +export interface ChatImageDeliveryPorts { + readonly artifacts: Pick< + InteractiveArtifactStoreWriter, + 'create' | 'findImageDelivery' | 'setImageDeliveryAttempt' + >; + readonly admission: SessionAdmissionGate; + isPresent(sessionId: string): Promise; + readMessage(identity: DeliveryIdentity): Promise; + /** Uses the Session read boundary and normalizes failures to ImageFileReadError. */ + readLocalImage(sessionId: string, path: string, signal: AbortSignal): Promise; + canLoadRemote(sessionId: string): Promise; + acquireResidency(): { release(): void }; + persistenceFailed(error: unknown): void; + presentationFailed(error: unknown): void; + readonly sourceReadTimeoutMs?: number; + readonly limits?: ImageArchiveLimits; + readonly download?: typeof downloadChatImage; +} +/** Host owns capture and persistence; clients can only resolve sources already in an assistant message. */ +export class ChatImageDeliveryService { + readonly #jobs = new Map(); + readonly #queue: DeliveryJob[] = []; + readonly #abort = new AbortController(); + #running = 0; + #closed = false; + constructor(private readonly ports: ChatImageDeliveryPorts) {} + + observe(sessionId: string, event: TextCompleteEvent): void { + if (this.#closed || event.text.length > IMAGE_MARKDOWN_MAX_LENGTH) return; + // Reference destinations can grow during streaming. Only settled text + // grants local capture. Remote media loads only when a client displays it. + try { + for (const source of chatImageSources(event.text)) { + if (isRemoteImageSource(source)) continue; + this.#enqueue({ sessionId, turnId: event.turnId, messageId: event.messageId, source }); + } + } catch (error) { + this.ports.presentationFailed(error); + } + } + + async resolve(identity: DeliveryIdentity): Promise { + if (this.#closed) return { status: 'unavailable' }; + const metadata = await this.ports.artifacts.findImageDelivery( + identity.sessionId, + identity.turnId, + identity.messageId, + identity.source, + ); + // A browser decode/read failure is not authority to destroy saved history. + if (metadata?.status === 'ready') return metadata; + if (this.#jobs.has(deliveryKey(identity))) return { status: 'pending' }; + if (metadata?.status === 'failed' && !identity.retry) return metadata; + // A client-provided path is never a read grant. Legacy/restarted deliveries + // must be found in canonical assistant text before any source is opened. + if (!metadata) { + const text = await this.ports.readMessage(identity); + if (text === undefined || !chatImageSources(text).includes(identity.source)) + return { status: 'unavailable' }; + } + if (isRemoteImageSource(identity.source)) { + if (!(await this.ports.canLoadRemote(identity.sessionId))) + return { status: 'failed', reason: 'not_allowed' }; + if (!identity.loadRemote) return { status: 'requires_confirmation' }; + } + return this.#enqueue(identity) + ? { status: 'pending' } + : { status: 'failed', reason: 'queue_full' }; + } + + #enqueue(identity: DeliveryIdentity): boolean { + const key = deliveryKey(identity); + if (this.#jobs.has(key)) return true; + if (this.#closed || this.#jobs.size >= 128) return false; + let settle!: () => void; + const done = new Promise((resolve) => { + settle = resolve; + }); + const residency = this.ports.acquireResidency(); + const finish = () => { + this.#jobs.delete(key); + residency.release(); + settle(); + }; + const job: DeliveryJob = { + done, + cancel: finish, + run: async () => { + try { + await this.#capture(identity); + } catch (error) { + if (!this.#abort.signal.aborted) { + if (error instanceof ImagePersistenceError) this.ports.persistenceFailed(error.cause); + else this.ports.presentationFailed(error); + } + } finally { + finish(); + } + }, + }; + this.#jobs.set(key, job); + this.#queue.push(job); + this.#pump(); + return true; + } + #pump(): void { + while (!this.#closed && this.#running < 2 && this.#queue.length) { + const job = this.#queue.shift()!; + this.#running++; + void job.run().finally(() => { + this.#running--; + this.#pump(); + }); + } + } + async #capture(identity: DeliveryIdentity): Promise { + const { sessionId, turnId, messageId, source } = identity; + const existing = await this.ports.artifacts.findImageDelivery( + sessionId, + turnId, + messageId, + source, + ); + if (existing?.status === 'ready' || (existing?.status === 'failed' && !identity.retry)) return; + const metadata = { messageId, source }; + const resultId = `chat_image_result_${deliveryKey(identity)}`; + const publish = async (write: () => Promise) => + this.ports.admission.runOrJoin(sessionId, async () => { + if (this.#abort.signal.aborted || !(await this.ports.isPresent(sessionId))) return; + try { + await write(); + } catch (error) { + if (error instanceof ImageArchiveQuotaError) throw error; + throw new ImagePersistenceError('Image persistence failed', { cause: error }); + } + }); + const publishAttempt = (attempt: ImageDeliveryAttempt) => + publish(() => this.ports.artifacts.setImageDeliveryAttempt(identity, attempt)); + const publishFailure = (reason: ImageDeliveryFailure) => + publishAttempt({ status: 'failed', reason }); + await publishAttempt({ status: 'pending' }); + if (this.#abort.signal.aborted || !(await this.ports.isPresent(sessionId))) return; + const readAbort = new AbortController(); + // Keep the read deadline alive even when a source has no active handles. + const deadline = setTimeout(() => readAbort.abort(), this.ports.sourceReadTimeoutMs ?? 10_000); + const signal = AbortSignal.any([this.#abort.signal, readAbort.signal]); + let image: ChatImageBytes; + try { + const path = localImagePath(source); + if (path !== undefined) { + image = await abortable(() => this.ports.readLocalImage(sessionId, path, signal), signal); + } else { + if (!identity.loadRemote || !(await this.ports.canLoadRemote(sessionId))) + throw new ImageSourceError('not_allowed'); + image = await (this.ports.download ?? downloadChatImage)(source, signal); + } + } catch (error) { + if (this.#abort.signal.aborted) return; + await publishFailure(failureReason(error, source)); + return; + } finally { + clearTimeout(deadline); + } + if (this.#abort.signal.aborted) return; + if (signal.aborted) { + await publishFailure(failureReason(signal.reason, source)); + return; + } + try { + await publish(() => + this.ports.artifacts.create( + readyChatImageArtifact({ + id: resultId, + sessionId, + turnId, + name: 'chat-image', + ...metadata, + image, + limits: this.ports.limits, + }), + ), + ); + } catch (error) { + if (!(error instanceof ImageArchiveQuotaError)) throw error; + await publishFailure('quota_exceeded'); + } + } + beginDrain(): void { + this.#closed = true; + this.#abort.abort(); + for (const job of this.#queue.splice(0)) job.cancel(); + } + async close(): Promise { + this.beginDrain(); + await this.waitForIdle(); + } + async waitForIdle(): Promise { + await Promise.all([...this.#jobs.values()].map((job) => job.done)); + } +} +class ImagePersistenceError extends Error {} +function deliveryKey(i: DeliveryIdentity): string { + return createHash('sha256') + .update(JSON.stringify([i.sessionId, i.turnId, i.messageId, i.source])) + .digest('hex'); +} +function failureReason(error: unknown, source: string): ImageDeliveryFailure { + if (error instanceof ImageSourceError) return error.reason; + if (error instanceof ImageFileReadError) return error.reason; + return isRemoteImageSource(source) ? 'download_failed' : 'read_failed'; +} diff --git a/packages/runtime-host/src/server/chat-image-markdown.ts b/packages/runtime-host/src/server/chat-image-markdown.ts new file mode 100644 index 00000000000..82ca33b020b --- /dev/null +++ b/packages/runtime-host/src/server/chat-image-markdown.ts @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { isImageDeliverySource } from '@maka/core/image-delivery'; +import { markdownImageSources } from '@maka/core/image-markdown'; + +/** Capture policy excludes explicit attachments and limits jobs per message. */ +export function chatImageSources(text: string): string[] { + return markdownImageSources(text) + .filter((source) => isImageDeliverySource(source) && !source.startsWith('maka:')) + .slice(0, 64); +} diff --git a/packages/runtime-host/src/server/chat-image-source.ts b/packages/runtime-host/src/server/chat-image-source.ts new file mode 100644 index 00000000000..34596f90732 --- /dev/null +++ b/packages/runtime-host/src/server/chat-image-source.ts @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { fileURLToPath } from 'node:url'; +import { ARTIFACT_IMAGE_PREVIEW_MAX_BYTES } from '@maka/core/artifacts'; +import { + ImageFileError, + imageFileFailureReason, + validateImageBytes, +} from '@maka/runtime/image-file'; +import { isRemoteImageSource, type ImageDeliveryFailure } from '@maka/core/image-delivery'; +import { + createProxiedFetchTransport, + PublicNetworkPolicyError, + type ScopedFetch, +} from '@maka/runtime/network/scoped-fetch-transport'; +export class ImageSourceError extends Error { + constructor(readonly reason: ImageDeliveryFailure) { + super(reason); + } +} +export interface ChatImageBytes { + readonly bytes: Uint8Array; + readonly mimeType: string; +} +export function checkedChatImage(bytes: Uint8Array): ChatImageBytes { + try { + return validateImageBytes(bytes, 'chat'); + } catch (error) { + if (!(error instanceof ImageFileError)) throw error; + throw new ImageSourceError(imageFileFailureReason(error)); + } +} +export function localImagePath(source: string): string | undefined { + if (isRemoteImageSource(source)) return undefined; + if (source.startsWith('file:')) { + try { + return fileURLToPath(source); + } catch { + throw new ImageSourceError('not_allowed'); + } + } + if (/^[a-z][a-z0-9+.-]*:/i.test(source) && !/^[a-z]:[\\/]/i.test(source)) + throw new ImageSourceError('not_allowed'); + return source; +} + +/** Network routing and public destination checks belong to the transport. + * Only validated image bytes cross the archive boundary. */ +export async function downloadChatImage( + source: string, + signal: AbortSignal, + options: { readonly fetch?: ScopedFetch } = {}, +): Promise { + const owned = options.fetch ? undefined : createProxiedFetchTransport(null); + const fetch = options.fetch ?? owned!.fetch; + try { + let url = new URL(source); + for (let redirect = 0; redirect <= 3; redirect++) { + signal.throwIfAborted(); + const response = await fetch(url, { + targetPolicy: 'public', + signal, + headers: { accept: 'image/png,image/jpeg,image/webp,image/gif' }, + redirect: 'manual', + credentials: 'omit', + }); + const location = response.headers.get('location'); + if ([301, 302, 303, 307, 308].includes(response.status) && location) { + await response.body?.cancel(); + const next = new URL(location, url); + if (url.protocol === 'https:' && next.protocol !== 'https:') + throw new ImageSourceError('not_allowed'); + url = next; + continue; + } + if (response.status !== 200) { + await response.body?.cancel(); + throw new ImageSourceError('download_failed'); + } + if (Number(response.headers.get('content-length')) > ARTIFACT_IMAGE_PREVIEW_MAX_BYTES) { + await response.body?.cancel(); + throw new ImageSourceError('too_large'); + } + const chunks: Buffer[] = []; + let size = 0; + const reader = response.body?.getReader(); + if (reader) { + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + size += chunk.value.length; + if (size > ARTIFACT_IMAGE_PREVIEW_MAX_BYTES) throw new ImageSourceError('too_large'); + chunks.push(Buffer.from(chunk.value)); + } + } finally { + await reader.cancel().catch(() => {}); + reader.releaseLock(); + } + } + return checkedChatImage(Buffer.concat(chunks, size)); + } + throw new ImageSourceError('download_failed'); + } catch (error) { + if (error instanceof PublicNetworkPolicyError) throw new ImageSourceError('not_allowed'); + throw error; + } finally { + await owned?.close(); + } +} diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index 2771c1605c6..f7df7c43647 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -17,6 +17,10 @@ * under the License. */ +import { createAssistantMessageReader } from './session-transcript-reader.js'; +import { createImageFileReader } from '@maka/runtime/image-file-reader'; +import { ChatImageDeliveryService } from './chat-image-delivery.js'; +import { downloadChatImage, ImageSourceError } from './chat-image-source.js'; import { createWorkHubResultRuntime } from './workhub-result-runtime.js'; import { createWorkHubInspectionTool } from './workhub-inspection-tool.js'; import { copyWorkHubAttachmentsToTarget } from './workhub-message-attachments.js'; @@ -312,6 +316,7 @@ export interface CreateExecutionRuntimeHostCompositionOptions { readonly bootstrapRuntimePolicy?: boolean; readonly skillHomeDirectory?: string; readonly projectDirectoryRoots?: readonly PublishedProjectDirectoryRoot[]; + readonly imageArchiveLimits?: import('@maka/core/image-delivery').ImageArchiveLimits; } export interface ExecutionRuntimeHostCompositionDependencies { @@ -363,6 +368,7 @@ export async function createExecutionRuntimeHostComposition( let sessionEffects: HostSessionEffectCoordinator | undefined; let promptSuggestions: HostPromptSuggestionCoordinator | undefined; let memoryExtraction: HostMemoryExtractionCoordinator | undefined; + let imageDelivery: ChatImageDeliveryService | undefined; let unsubscribeTranscriptChanges: (() => void) | undefined; let unsubscribeRuntimeEventCommits: (() => void) | undefined; let transcriptReader: SessionTranscriptReader | undefined; @@ -571,6 +577,65 @@ export async function createExecutionRuntimeHostComposition( sessionAdmission, sessions: stores.sessionStore, }); + const localImageReader = createImageFileReader({ + ...(filesystemWorker ? { filesystemWorker } : {}), + }); + imageDelivery = new ChatImageDeliveryService({ + artifacts: openedArtifactStore, + ...(options.imageArchiveLimits ? { limits: options.imageArchiveLimits } : {}), + admission: sessionAdmission, + isPresent: async (sessionId) => + (await stores.sessionStore.probeSessionRemoval(sessionId)).kind === 'present', + acquireResidency: () => context.acquireResidency('chat-image-delivery'), + persistenceFailed: (error) => { + console.warn('[runtime-host] Image delivery persistence failed', error); + context.requestDrain(); + }, + presentationFailed: (error) => { + console.warn('[runtime-host] Image presentation failed', error); + }, + // Transcript media belongs to the application's outbound policy, rather + // than the agent's subprocess sandbox. Privacy mode still blocks capture. + // Model-authored URLs can carry encoded data even when WebFetch is disabled; + // automatic media loading is not a Session network or DLP boundary. + canLoadRemote: async () => { + const resolved = await runtimePolicyStores.operations.resolveHostOutboundExecution(); + return resolved.kind === 'ready'; + }, + download: async (source, signal) => { + const resolved = await runtimePolicyStores.operations.resolveHostOutboundExecution(); + if (resolved.kind !== 'ready') throw new ImageSourceError('not_allowed'); + const proxy = toRuntimePolicyProxy( + resolved.networkProxy, + resolved.secretMaterial.networkProxy?.secret, + ); + const transport = createProxiedFetchTransport(proxy); + try { + return await downloadChatImage(source, signal, { fetch: transport.fetch }); + } finally { + await transport.close(); + } + }, + readLocalImage: async (sessionId, path, abortSignal) => { + const [header, boundary] = await Promise.all([ + stores.sessionStore.readHeaderSnapshot(sessionId), + stores.sessionStore.readExecutionBoundary(sessionId), + ]); + const result = await localImageReader({ + path, + cwd: header.cwd, + permissionMode: header.permissionMode, + executionBoundary: boundary, + abortSignal, + }); + return result; + }, + readMessage: createAssistantMessageReader({ + events: stores.runtimeEventStore, + ensureTranscriptLedger: (sessionId) => + requireSessionManager(manager).ensureTranscriptLedgerForRead(sessionId), + }), + }); // Shared with recall's material fetch, so a file brought in from another // Session is answered by the same reader that answers one stored here. const attachmentResources = createArtifactAttachmentResourceReader({ @@ -1778,6 +1843,7 @@ export async function createExecutionRuntimeHostComposition( Date.now, context.sessionAccessAuthority, (sessionId, artifactId) => hostChanges.publishArtifactDeleted(sessionId, artifactId), + imageDelivery, ); rootCoordinator = new RootTurnCoordinator( manager, @@ -1828,6 +1894,7 @@ export async function createExecutionRuntimeHostComposition( // Fresh WorkHub Turns use the coordinating Agent directly. Persisted // routing decisions remain readable for recovery of historical Turns. undefined, + (sessionId, event) => imageDelivery?.observe(sessionId, event), ); const coordinator = rootCoordinator; const pluginModel = createHostPluginModel({ @@ -2787,6 +2854,7 @@ export async function createExecutionRuntimeHostComposition( }); const executionInspect = new HostExecutionInspectCoordinator(stores); const sessionRevisions = new HostSessionRevisionCoordinator({ + ...(options.imageArchiveLimits ? { imageArchiveLimits: options.imageArchiveLimits } : {}), stores, artifacts: openedArtifactStore, sessionTodo: sessionTodoStore, @@ -3144,6 +3212,7 @@ export async function createExecutionRuntimeHostComposition( () => workspaceExecution?.beginDrain(), () => runtimeResources?.beginDrain(), () => messages.beginDrain(), + () => imageDelivery?.beginDrain(), () => interactions.beginDrain(), () => sessionEffects?.beginDrain(), () => promptSuggestions?.beginDrain(), @@ -3156,6 +3225,7 @@ export async function createExecutionRuntimeHostComposition( await rootCloseTask; }, () => workHubResults?.coordinator.close(), + () => imageDelivery?.close(), () => runtimeResources?.close(), () => workspaceExecution?.close(), () => sessionEffects?.close(), @@ -3338,6 +3408,7 @@ export async function createExecutionRuntimeHostComposition( } goalExecutions?.beginDrain(); try { + await imageDelivery?.close(); await workspaceExecution?.close(); } catch (closeError) { errors.push(closeError); diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index 3ad565a90b9..895ea56612e 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -387,6 +387,10 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { private readonly prepareWorkHubRoutingDecision?: ( input: HostWorkHubRoutingDecisionPreparation, ) => Promise, + private readonly observeAssistantText?: ( + sessionId: string, + event: import('@maka/core/events').TextCompleteEvent, + ) => void, ) { this.stores = authenticateExecutionStoresWriter(stores, 'interactive'); this.executionProjection = new HostedExecutionProjectionReader(this.stores); @@ -3227,6 +3231,13 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { // Presentation observers do not participate in execution authority. } } + if (event.type === 'text_complete') { + try { + this.observeAssistantText?.(input.sessionId, event); + } catch { + // Image presentation must never interrupt canonical execution. + } + } if (isRuntimeSessionForwardedEvent(event)) { await this.continuity.acceptRuntimeEvent(input.sessionId, active.runId, event); } else if (isInteractionAnswerAck(event)) { diff --git a/packages/runtime-host/src/server/session-revision-coordinator.ts b/packages/runtime-host/src/server/session-revision-coordinator.ts index e9702c052f7..5dd4db49b51 100644 --- a/packages/runtime-host/src/server/session-revision-coordinator.ts +++ b/packages/runtime-host/src/server/session-revision-coordinator.ts @@ -100,6 +100,7 @@ type ConversationCopyCreateInput = CreateSessionInput & { }; export interface HostSessionRevisionCoordinatorOptions { + readonly imageArchiveLimits?: import('@maka/core/image-delivery').ImageArchiveLimits; readonly stores: ExecutionStoresWriter<'interactive'>; readonly artifacts: InteractiveArtifactStoreWriter; readonly sessionTodo: InteractiveSessionTodoWriter; @@ -534,6 +535,9 @@ export class HostSessionRevisionCoordinator { throw new Error(`Session context references could not be copied: ${contextCopy.reason}`); } const artifactCopy = await this.#artifacts.copyConversationArtifacts({ + ...(this.options.imageArchiveLimits + ? { imageArchiveLimits: this.options.imageArchiveLimits } + : {}), sourceSessionId: input.sourceSessionId, targetSessionId: input.targetSessionId, turnIds: copyTurnIds, diff --git a/packages/runtime-host/src/server/session-transcript-reader.ts b/packages/runtime-host/src/server/session-transcript-reader.ts index 2749bf1372e..abce9aa0959 100644 --- a/packages/runtime-host/src/server/session-transcript-reader.ts +++ b/packages/runtime-host/src/server/session-transcript-reader.ts @@ -650,3 +650,63 @@ function createTranscriptProjection(invocations: readonly RuntimeInvocationRecor }, }; } + +/** Resolves only settled canonical assistant text; presentation cannot grant a file read. */ +export function createAssistantMessageReader(input: { + events: Pick< + ExecutionStoresWriter<'interactive'>['runtimeEventStore'], + 'readTranscriptTurns' | 'readTranscriptRun' + >; + ensureTranscriptLedger(sessionId: string): Promise; +}) { + return async (identity: { + sessionId: string; + turnId: string; + messageId: string; + }): Promise => { + await input.ensureTranscriptLedger(identity.sessionId); + const [turn] = await input.events.readTranscriptTurns(identity.sessionId, { + turnId: identity.turnId, + }); + if (!turn) return undefined; + for ( + let position = turn.firstOrdinal, runs = 0; + position <= turn.lastOrdinal && runs < 16; + runs++ + ) { + const found = await input.events.readTranscriptRun( + identity.sessionId, + { + direction: 'newer', + throughOrdinal: turn.lastOrdinal, + position, + maxEvents: 4096, + maxBytes: 16 * 1024 * 1024, + maxRecordBytes: 8 * 1024 * 1024, + }, + (run, events) => { + let text: string | undefined; + for (const { event } of events) { + if ( + event.partial || + event.turnId !== identity.turnId || + event.role !== 'model' || + event.content?.kind !== 'text' + ) + continue; + if ( + (event.refs?.storedMessageId ?? event.refs?.providerEventId ?? event.id) === + identity.messageId + ) + text = event.content.text; + } + return { text, next: run.lastOrdinal + 1 }; + }, + ); + if (!found) break; + if (found.text !== undefined) return found.text; + position = found.next; + } + return undefined; + }; +} diff --git a/packages/runtime/package.json b/packages/runtime/package.json index 0367cf3a1df..206a249ee6f 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -11,6 +11,8 @@ "./ai-sdk-backend": "./dist/ai-sdk-backend.js", "./mcp-tools": "./dist/mcp-tools.js", "./read-page": "./dist/read-page.js", + "./image-file-reader": "./dist/image-file-reader.js", + "./image-file": "./dist/image-file.js", "./builtin-tools": "./dist/builtin-tools.js", "./shell-tools": "./dist/shell-tools.js", "./shell-run-manager": "./dist/shell-run-manager.js", diff --git a/packages/runtime/src/__tests__/builtin-tools-file-worker.test.ts b/packages/runtime/src/__tests__/builtin-tools-file-worker.test.ts index 23ffb578401..420b06b75b9 100644 --- a/packages/runtime/src/__tests__/builtin-tools-file-worker.test.ts +++ b/packages/runtime/src/__tests__/builtin-tools-file-worker.test.ts @@ -26,6 +26,7 @@ import { createManagedExecutionBoundary } from '@maka/core/sandbox-boundary'; import { createWorkspaceWritePermissionProfile } from '@maka/core/permission-profile'; import { createReadOnlyPermissionProfile } from '@maka/core/permission-profile'; +import { createImageFileReader } from '../image-file-reader.js'; import { buildBuiltinTools } from '../builtin-tools.js'; import { createBoundaryFilesystemExecutor } from '../filesystem-executor.js'; import { createLocalWorkspaceExecutor } from '../workspace-executor.js'; @@ -613,3 +614,32 @@ async function temporaryDirectory(prefix: string): Promise { cleanup.push(path); return await realpath(path); } + +test('chat image reader uses the managed Read boundary and never falls back to host filesystem', async () => { + const cwd = await temporaryDirectory('maka-delivery-reader-'); + const executionBoundary = createManagedExecutionBoundary(createReadOnlyPermissionProfile(), 0); + await assert.rejects( + createImageFileReader()({ path: 'private.png', cwd, executionBoundary }), + /worker|sandbox/i, + ); + let calls = 0; + const reader = createImageFileReader({ + filesystemWorker: { + execute: async (input) => { + calls++; + assert.deepEqual(input.executionBoundary, executionBoundary); + assert.deepEqual(input.operation, { + kind: 'read', + path: 'private.png', + imagePurpose: 'chat', + }); + throw new Error('sandbox permission refused'); + }, + }, + }); + await assert.rejects( + reader({ path: 'private.png', cwd, executionBoundary }), + /permission refused/, + ); + assert.equal(calls, 1); +}); diff --git a/packages/runtime/src/__tests__/filesystem-worker-launch-spec.test.ts b/packages/runtime/src/__tests__/filesystem-worker-launch-spec.test.ts index 2c38163babd..ae91d8e2f03 100644 --- a/packages/runtime/src/__tests__/filesystem-worker-launch-spec.test.ts +++ b/packages/runtime/src/__tests__/filesystem-worker-launch-spec.test.ts @@ -18,12 +18,46 @@ */ import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; import { chmod, copyFile, mkdir, mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { test } from 'node:test'; import { createFilesystemWorkerLaunchSpecProvider } from '../filesystem-worker/launch-spec.js'; +import { + FILESYSTEM_WORKER_BUNDLE_NAME, + resolveFilesystemWorkerBundle, +} from '../filesystem-worker/resource-resolver.js'; +import { FilesystemWorkerResponseSchema } from '../filesystem-worker/protocol.js'; + +for (const packageConfig of ['{invalid JSON', '{"type":"commonjs"}']) { + test(`worker startup does not consult parent package config: ${packageConfig}`, async (t) => { + const root = await mkdtemp(join(tmpdir(), 'maka-worker-module-boundary-')); + t.after(() => rm(root, { recursive: true, force: true })); + const source = await resolveFilesystemWorkerBundle({ kind: 'runtime' }); + assert.ok(source.ok); + const workers = join(root, 'workers'); + await mkdir(workers); + const bundle = join(workers, FILESYSTEM_WORKER_BUNDLE_NAME); + await copyFile(source.path, bundle); + await writeFile(join(root, 'package.json'), packageConfig); + const result = spawnSync(process.execPath, ['--preserve-symlinks-main', bundle], { + cwd: root, + input: '{}', + encoding: 'utf8', + timeout: 5000, + }); + assert.equal( + result.status, + 0, + result.stderr || result.error?.message || 'Worker did not exit successfully', + ); + const response = FilesystemWorkerResponseSchema.parse(JSON.parse(result.stdout)); + assert.equal(response.ok, false); + if (!response.ok) assert.equal(response.error.code, 'invalid_request'); + }); +} test('Linux Electron worker launch does not require a macOS Frameworks directory', async () => { const getLaunchSpec = createFilesystemWorkerLaunchSpecProvider({ diff --git a/packages/runtime/src/__tests__/image-file.test.ts b/packages/runtime/src/__tests__/image-file.test.ts index c8e2636ca32..77010595bbc 100644 --- a/packages/runtime/src/__tests__/image-file.test.ts +++ b/packages/runtime/src/__tests__/image-file.test.ts @@ -18,6 +18,7 @@ */ import { test } from 'node:test'; +import { execFileSync } from 'node:child_process'; import assert from 'node:assert/strict'; import { mkdtemp, realpath, rm, writeFile, truncate } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -26,11 +27,156 @@ import { readWorkspaceFile, validateImageBytes } from '../image-file.js'; import { LocalWorkspaceExecutor } from '../workspace-executor.js'; import { executeFilesystemOperation } from '../filesystem-worker/operations.js'; import { MAX_READ_IMAGE_BYTES } from '@maka/core/attachments'; +import { ARTIFACT_IMAGE_PREVIEW_MAX_BYTES } from '@maka/core/artifacts'; +import { createImageFileReader, ImageFileReadError } from '../image-file-reader.js'; +import { FilesystemWorkerClientError } from '../filesystem-worker/client.js'; +import { executeFilesystemWorkerRequest } from '../filesystem-worker/operations.js'; +import { FILESYSTEM_WORKER_PROTOCOL_VERSION } from '../filesystem-worker/protocol.js'; const ONE_PIXEL_PNG = Buffer.from( 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==', 'base64', ); +// Valid 2560x1 PNG, wider than the model input limit but within chat's byte budget. +const WIDE_PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAACgAAAAABCAYAAAASePczAAAAIUlEQVR4nO3BAQ0AAADCoPdPbQ43oAAAAAAAAAAAAIA7AygBAAEQnI5pAAAAAElFTkSuQmCC', + 'base64', +); +test('chat images cap both edge length and total decoded pixels', () => { + for (const [width, height] of [ + [16_385, 1], + [8192, 8192], + ]) { + const bytes = Buffer.from(ONE_PIXEL_PNG); + bytes.writeUInt32BE(width, 16); + bytes.writeUInt32BE(height, 20); + assert.throws(() => validateImageBytes(bytes, 'chat'), /Image exceeds/); + } + assert.equal(validateImageBytes(WIDE_PNG, 'chat').mimeType, 'image/png'); +}); + +test('chat reader preserves structured Worker failures independently of message wording', async () => { + for (const [reason, expected] of [ + ['not_found', 'not_found'], + ['filesystem_denied', 'not_allowed'], + ['path_denied', 'not_allowed'], + ['sandbox_denied', 'not_allowed'], + ['image_too_large', 'too_large'], + ['invalid_image', 'unsupported_mime'], + ['worker_io_incomplete', 'read_failed'], + ] as const) { + const reader = createImageFileReader({ + filesystemWorker: { + execute: async () => { + throw new FilesystemWorkerClientError({ + reason, + stage: 'operation', + message: '读取失败', + }); + }, + }, + }); + await assert.rejects( + reader({ path: 'image.png', cwd: process.cwd() }), + (error: unknown) => error instanceof ImageFileReadError && error.reason === expected, + ); + } +}); + +test('chat reader preserves local missing paths and workspace boundary denials', async (t) => { + const cwd = await realpath(await mkdtemp(join(tmpdir(), 'maka-chat-read-errors-'))); + t.after(() => rm(cwd, { recursive: true, force: true })); + const reader = createImageFileReader(); + await assert.rejects( + reader({ path: 'missing.png', cwd }), + (error: unknown) => error instanceof ImageFileReadError && error.reason === 'not_found', + ); + await assert.rejects( + reader({ path: join(cwd, '..', 'outside.png'), cwd }), + (error: unknown) => error instanceof ImageFileReadError && error.reason === 'not_allowed', + ); +}); + +test('Worker transports typed image validation failures instead of generic filesystem errors', async (t) => { + const cwd = await realpath(await mkdtemp(join(tmpdir(), 'maka-chat-image-wire-'))); + t.after(() => rm(cwd, { recursive: true, force: true })); + const path = join(cwd, 'image.png'); + for (const [content, expected] of [ + [ONE_PIXEL_PNG.subarray(0, 8), 'invalid_image'], + [Buffer.alloc(ARTIFACT_IMAGE_PREVIEW_MAX_BYTES + 1), 'image_too_large'], + ] as const) { + await writeFile(path, content); + const response = await executeFilesystemWorkerRequest({ + version: FILESYSTEM_WORKER_PROTOCOL_VERSION, + requestId: 'read-image', + operation: { kind: 'read', cwd, path, imagePurpose: 'chat' }, + operationBoundary: { + filesystem: { entries: [{ path: cwd, access: 'read', scope: 'subtree' }] }, + }, + expectedTarget: { + enforcementPath: path, + access: 'read', + scope: 'exact', + targetType: 'file', + identity: 'unchecked', + }, + }); + assert.equal(response.ok, false); + if (!response.ok) assert.equal(response.error.code, expected); + } +}); + +for (const route of ['workspace', 'worker'] as const) { + test(`${route} chat reads admit wide screenshots without relaxing ordinary Read`, async (t) => { + const cwd = await realpath(await mkdtemp(join(tmpdir(), 'maka-chat-image-policy-'))); + t.after(() => rm(cwd, { recursive: true, force: true })); + const path = join(cwd, 'wide.png'); + const read = (imagePurpose?: 'chat') => + route === 'workspace' + ? new LocalWorkspaceExecutor().readFile({ + cwd, + path, + ...(imagePurpose ? { imagePurpose } : {}), + }) + : executeFilesystemOperation( + { kind: 'read', cwd, path, ...(imagePurpose ? { imagePurpose } : {}) }, + { + filesystem: { entries: [{ path: cwd, access: 'read', scope: 'subtree' }] }, + }, + ); + await writeFile(path, WIDE_PNG); + await assert.rejects(read(), /dimensions.*model input limit/i); + const result = await read('chat'); + if ('base64' in result) assert.deepEqual(Buffer.from(result.base64, 'base64'), WIDE_PNG); + else { + assert.ok('bytes' in result && result.bytes instanceof Uint8Array); + assert.deepEqual(Buffer.from(result.bytes), WIDE_PNG); + } + await truncate(path, ARTIFACT_IMAGE_PREVIEW_MAX_BYTES + 1); + await assert.rejects(read('chat'), /2 MiB/); + await writeFile(path, 'not an image'); + await assert.rejects(read('chat'), /not a supported/i); + }); +} + +test('chat reads reject FIFOs without waiting for a writer', { + skip: process.platform === 'win32', + timeout: 1000, +}, async (t) => { + const cwd = await mkdtemp(join(tmpdir(), 'maka-image-fifo-')); + t.after(() => rm(cwd, { recursive: true, force: true })); + const path = join(cwd, 'blocked.png'); + execFileSync('mkfifo', [path]); + await assert.rejects(createImageFileReader()({ path, cwd }), /not a file/i); +}); + +test('chat reads honor cancellation before opening the source', async () => { + const abortSignal = AbortSignal.abort(new Error('capture cancelled')); + await assert.rejects( + readWorkspaceFile('/missing.png', { imagePurpose: 'chat', abortSignal }), + /capture cancelled/, + ); +}); for (const route of ['workspace', 'worker']) { for (const name of ['image.png', 'image', 'image.bin']) { diff --git a/packages/runtime/src/filesystem-executor.ts b/packages/runtime/src/filesystem-executor.ts index 4875f97f2ba..99921db8dc2 100644 --- a/packages/runtime/src/filesystem-executor.ts +++ b/packages/runtime/src/filesystem-executor.ts @@ -394,6 +394,8 @@ function createWorkspaceFilesystemExecutor( const result = await workspace.readFile({ cwd, path, + ...(operation.imagePurpose ? { imagePurpose: operation.imagePurpose } : {}), + ...(abortSignal ? { abortSignal } : {}), }); if ('bytes' in result) { return { kind: 'read_image', bytes: result.bytes, mimeType: result.mimeType }; diff --git a/packages/runtime/src/filesystem-worker/operations.ts b/packages/runtime/src/filesystem-worker/operations.ts index bc2dcfbfe24..75fcdb9f77c 100644 --- a/packages/runtime/src/filesystem-worker/operations.ts +++ b/packages/runtime/src/filesystem-worker/operations.ts @@ -47,7 +47,7 @@ import { StableWriteFailure, writeThroughHandle, } from '../file-stable-write.js'; -import { readWorkspaceFile } from '../image-file.js'; +import { ImageFileError, readWorkspaceFile } from '../image-file.js'; import { FILESYSTEM_WORKER_PROTOCOL_VERSION, operationAccess, @@ -124,9 +124,11 @@ export async function executeFilesystemOperation( 'read', operationBoundary, ); - const file = await readWorkspaceFile(path).catch((error: unknown) => { + const file = await readWorkspaceFile(path, { + ...(operation.imagePurpose ? { imagePurpose: operation.imagePurpose } : {}), + }).catch((error: unknown) => { throw operationError( - 'filesystem_error', + normalizeOperationError(error).code, error instanceof Error ? error.message : 'File could not be read.', ); }); @@ -451,6 +453,11 @@ function operationError( function normalizeOperationError(error: unknown): FilesystemOperationError { if (error instanceof FilesystemOperationError) return error; + if (error instanceof ImageFileError) + return operationError( + error.code === 'ERR_IMAGE_TOO_LARGE' ? 'image_too_large' : 'invalid_image', + error.message, + ); if (error instanceof StableWriteFailure) { return operationError(error.code, error.message); } diff --git a/packages/runtime/src/filesystem-worker/protocol.ts b/packages/runtime/src/filesystem-worker/protocol.ts index efacffe3bc1..bf1512c4fe5 100644 --- a/packages/runtime/src/filesystem-worker/protocol.ts +++ b/packages/runtime/src/filesystem-worker/protocol.ts @@ -22,10 +22,9 @@ import { readContinuationSchema, readPageSchema } from '../read-page.js'; import { validateSandboxBoundaryExpansion } from '@maka/core/sandbox-boundary'; import { GREP_MAX_LINES, GREP_MAX_LINES_PER_FILE, GREP_MAX_MATCH_BYTES } from '../grep-search.js'; -// v10 adds bounded Read pages to v9's exact Grep counts; v11 adds Glob's -// truncation flag to v10's result contract. Older workers cannot satisfy the -// combined result contract and must be rejected at the handshake. -export const FILESYSTEM_WORKER_PROTOCOL_VERSION = 11 as const; +// v12 adds chat image reads with a bounded payload and no model dimension limit. +// Older workers cannot satisfy this read policy and must be rejected at the handshake. +export const FILESYSTEM_WORKER_PROTOCOL_VERSION = 12 as const; /** The single authority on which operation kinds are writes. Shared by the * client (permission/identity decisions) and the worker (operation guards) so @@ -107,6 +106,7 @@ export const FilesystemWorkerOperationSchema = z.union([ kind: z.literal('read'), cwd, path, + imagePurpose: z.literal('chat').optional(), offset: z.number().int().nonnegative().optional(), limit: z.number().int().positive().optional(), continuation: readContinuationSchema.optional(), @@ -251,6 +251,8 @@ export const FilesystemWorkerErrorCodeSchema = z.enum([ 'grep_unavailable', 'sandbox_denied', 'filesystem_denied', + 'image_too_large', + 'invalid_image', 'filesystem_error', // The worker may have applied the mutation before it lost the ability to // report back (e.g. it wrote the file then the post-write identity check diff --git a/packages/runtime/src/image-file-reader.ts b/packages/runtime/src/image-file-reader.ts new file mode 100644 index 00000000000..5be80181fc3 --- /dev/null +++ b/packages/runtime/src/image-file-reader.ts @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { BuildBuiltinToolsOptions } from './builtin-tools.js'; +import { + createBoundaryFilesystemExecutor, + type FilesystemExecuteInput, +} from './filesystem-executor.js'; +import { createLocalWorkspaceExecutor } from './workspace-executor.js'; +import { ImageFileError, imageFileFailureReason } from './image-file.js'; +import { FilesystemWorkerClientError } from './filesystem-worker/client.js'; +import { SandboxCommandError } from './sandbox/errors.js'; + +type ImageFileReadFailure = + | 'not_found' + | 'not_allowed' + | 'too_large' + | 'unsupported_mime' + | 'read_failed'; +/** Normalizes filesystem backends at the image reader boundary. */ +export class ImageFileReadError extends Error { + constructor( + readonly reason: ImageFileReadFailure, + cause?: unknown, + ) { + super(cause instanceof Error ? cause.message : `Image read failed: ${reason}`, { cause }); + this.name = 'ImageFileReadError'; + } +} + +function readFailure(error: unknown): ImageFileReadFailure { + if (error instanceof ImageFileError) return imageFileFailureReason(error); + if (error instanceof SandboxCommandError) return 'not_allowed'; + const code = + error instanceof FilesystemWorkerClientError + ? error.reason + : (error as NodeJS.ErrnoException | undefined)?.code; + switch (code) { + case 'image_too_large': + return 'too_large'; + case 'invalid_image': + return 'unsupported_mime'; + case 'ENOENT': + case 'ENOTDIR': + case 'not_found': + return 'not_found'; + case 'EACCES': + case 'EPERM': + case 'filesystem_denied': + case 'path_denied': + case 'sandbox_denied': + case 'sandbox_required': + case 'sandbox_boundary_required': + return 'not_allowed'; + default: + return 'read_failed'; + } +} +/** The same filesystem boundary as Read, without model snapshots or tool execution. */ +export function createImageFileReader( + options: Pick< + BuildBuiltinToolsOptions, + 'executor' | 'filesystemWorker' | 'permissionProfile' + > = {}, +) { + const filesystem = createBoundaryFilesystemExecutor({ + workspace: options.executor ?? createLocalWorkspaceExecutor(), + ...(options.filesystemWorker ? { worker: options.filesystemWorker } : {}), + ...(options.permissionProfile ? { permissionProfile: options.permissionProfile } : {}), + }); + return async (input: Omit & { path: string }) => { + const { path, ...context } = input; + const result = await filesystem + .execute({ + ...context, + operation: { kind: 'read', path, imagePurpose: 'chat' }, + }) + .catch((error: unknown) => { + throw new ImageFileReadError(readFailure(error), error); + }); + if (result.kind !== 'read_image') throw new ImageFileReadError('unsupported_mime'); + return { bytes: result.bytes, mimeType: result.mimeType }; + }; +} diff --git a/packages/runtime/src/image-file.ts b/packages/runtime/src/image-file.ts index 38f56809d44..c1f0ec5a71f 100644 --- a/packages/runtime/src/image-file.ts +++ b/packages/runtime/src/image-file.ts @@ -17,6 +17,7 @@ * under the License. */ +import { constants } from 'node:fs'; import { open } from 'node:fs/promises'; import { extname } from 'node:path'; import { imageDimensionsFromData } from 'image-dimensions'; @@ -27,10 +28,30 @@ import { READ_IMAGE_TOO_LARGE_MESSAGE, sniffAttachmentMimeType, } from '@maka/core/attachments'; +import { ARTIFACT_IMAGE_PREVIEW_MAX_BYTES } from '@maka/core/artifacts'; + +export interface WorkspaceFileReadOptions { + imagePurpose?: 'chat'; + abortSignal?: AbortSignal; +} const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.gif', '.webp']); export type ImageMimeType = 'image/png' | 'image/jpeg' | 'image/gif' | 'image/webp'; +export class ImageFileError extends Error { + constructor( + readonly code: 'ERR_IMAGE_TOO_LARGE' | 'ERR_INVALID_IMAGE', + message: string, + ) { + super(message); + } +} + +/** Translate validated image failures without changing backend/transport errors. */ +export function imageFileFailureReason(error: ImageFileError): 'too_large' | 'unsupported_mime' { + return error.code === 'ERR_IMAGE_TOO_LARGE' ? 'too_large' : 'unsupported_mime'; +} + export function isSupportedImagePath(path: string): boolean { return IMAGE_EXTENSIONS.has(extname(path).toLowerCase()); } @@ -38,9 +59,22 @@ export function isSupportedImagePath(path: string): boolean { /** Classify a bounded prefix before decoding text or allocating an image body. */ export async function readWorkspaceFile( path: string, + options: WorkspaceFileReadOptions = {}, ): Promise<{ content: string } | { bytes: Uint8Array; mimeType: ImageMimeType }> { - const file = await open(path, 'r'); + options.abortSignal?.throwIfAborted(); + // Chat capture must never block opening a FIFO or read an unbounded text file. + const file = await open( + path, + options.imagePurpose === 'chat' ? constants.O_RDONLY | constants.O_NONBLOCK : 'r', + ); try { + options.abortSignal?.throwIfAborted(); + if (options.imagePurpose === 'chat') { + const size = await file.stat(); + if (!size.isFile()) throw new Error('Image path is not a file.'); + if (size.size > ARTIFACT_IMAGE_PREVIEW_MAX_BYTES) throw imageTooLargeError('chat'); + return validateImageBytes(await file.readFile({ signal: options.abortSignal }), 'chat'); + } const prefix = Buffer.alloc(ATTACHMENT_MIME_SNIFF_BYTES); // A positioned read leaves the descriptor's offset at zero for readFile. const { bytesRead } = await file.read(prefix, 0, prefix.length, 0); @@ -56,13 +90,21 @@ export async function readWorkspaceFile( } } -export function validateImageBytes(bytes: Uint8Array): { +export function validateImageBytes( + bytes: Uint8Array, + purpose: 'model' | 'chat' = 'model', +): { bytes: Uint8Array; mimeType: ImageMimeType; } { - if (bytes.length > MAX_READ_IMAGE_BYTES) throw imageTooLargeError(); + if (bytes.length > (purpose === 'chat' ? ARTIFACT_IMAGE_PREVIEW_MAX_BYTES : MAX_READ_IMAGE_BYTES)) + throw imageTooLargeError(purpose); const mimeType = sniffImageMime(bytes); - if (!mimeType) throw new Error('Image content is not a supported PNG, JPEG, GIF, or WebP file.'); + if (!mimeType) + throw new ImageFileError( + 'ERR_INVALID_IMAGE', + 'Image content is not a supported PNG, JPEG, GIF, or WebP file.', + ); const dimensions = imageDimensionsFromData(bytes); if ( !dimensions || @@ -73,18 +115,35 @@ export function validateImageBytes(bytes: Uint8Array): { dimensions.width <= 0 || dimensions.height <= 0 ) { - throw new Error('Image dimensions could not be read; verify the image file is valid.'); + throw new ImageFileError( + 'ERR_INVALID_IMAGE', + 'Image dimensions could not be read; verify the image file is valid.', + ); } - if (Math.max(dimensions.width, dimensions.height) > MAX_MODEL_IMAGE_EDGE) { + if (purpose === 'model' && Math.max(dimensions.width, dimensions.height) > MAX_MODEL_IMAGE_EDGE) { throw new Error( `Image dimensions ${dimensions.width}x${dimensions.height} exceed the ${MAX_MODEL_IMAGE_EDGE}px model input limit; downscale it and try again.`, ); } + // Compressed byte size does not bound renderer decoding memory. Permit large + // screenshots while rejecting pathological headers before archival/preview. + if ( + purpose === 'chat' && + (Math.max(dimensions.width, dimensions.height) > 16_384 || + dimensions.width * dimensions.height > 32 * 1024 * 1024) + ) { + throw imageTooLargeError(purpose); + } return { bytes, mimeType }; } -function imageTooLargeError(): Error { - return new Error(READ_IMAGE_TOO_LARGE_MESSAGE); +function imageTooLargeError(purpose: 'model' | 'chat' = 'model'): Error { + return new ImageFileError( + 'ERR_IMAGE_TOO_LARGE', + purpose === 'chat' + ? 'Image exceeds a chat preview limit (2 MiB, 16384px per edge, or 32 megapixels); resize it before publishing.' + : READ_IMAGE_TOO_LARGE_MESSAGE, + ); } function sniffImageMime(bytes: Uint8Array): ImageMimeType | undefined { diff --git a/packages/runtime/src/network/__tests__/public-network-policy.test.ts b/packages/runtime/src/network/__tests__/public-network-policy.test.ts new file mode 100644 index 00000000000..1378bd8fd3a --- /dev/null +++ b/packages/runtime/src/network/__tests__/public-network-policy.test.ts @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import dns from 'node:dns/promises'; +import { getEventListeners } from 'node:events'; +import { createServer } from 'node:http'; +import { syncBuiltinESMExports } from 'node:module'; +import net from 'node:net'; +import tls from 'node:tls'; +import { test, type TestContext } from 'node:test'; +import { withTimeout } from '@maka/core/test-only/async-primitives'; +import { + createProxiedFetchTransport, + PublicNetworkPolicyError, +} from '../scoped-fetch-transport.js'; +import { preparePublicNetworkTarget } from '../public-network-policy.js'; + +const policy = { targetPolicy: 'public', redirect: 'manual' } as const; +function restore(t: TestContext) { + t.after(() => { + t.mock.restoreAll(); + syncBuiltinESMExports(); + }); + syncBuiltinESMExports(); +} + +test('public policy is per request; ordinary private requests and automatic redirects still work', async () => { + let hits = 0; + const server = createServer((req, res) => { + hits++; + if (req.url === '/redirect') res.writeHead(302, { location: '/image' }).end(); + else res.end('ok'); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const url = `http://127.0.0.1:${(server.address() as net.AddressInfo).port}`; + const transport = createProxiedFetchTransport(null); + try { + assert.equal(await (await transport.fetch(url + '/redirect')).text(), 'ok'); + await assert.rejects(transport.fetch(url, policy), PublicNetworkPolicyError); + assert.equal(hits, 2); + assert.equal(await (await transport.fetch(url)).text(), 'ok'); + assert.equal(hits, 3); + await assert.rejects( + transport.fetch('http://image.example', { targetPolicy: 'public' }), + /manual redirects/, + ); + } finally { + await transport.close(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); + +for (const address of ['198.18.0.2', '198.19.255.254', '2001:2::6', '2001:2:0:1::2']) { + test(`public policy rejects benchmark DNS and literal ${address} without exceptions`, async (t) => { + const benchmark = { address, family: address.includes(':') ? 6 : 4 }; + let answers = [benchmark]; + const lookup = t.mock.method(dns, 'lookup', async () => answers); + const connect = t.mock.method(net, 'connect', () => + assert.fail('blocked target reached transport'), + ); + restore(t); + const direct = createProxiedFetchTransport(null); + const proxy = createProxiedFetchTransport({ + enabled: true, + type: 'http', + host: '127.0.0.1', + port: 1, + bypassList: [], + }); + try { + for (const resolved of [[benchmark], [{ address: '93.184.216.34', family: 4 }, benchmark]]) { + answers = resolved; + await assert.rejects( + direct.fetch('http://image.example', policy), + PublicNetworkPolicyError, + ); + } + assert.equal(lookup.mock.callCount(), 2); + const literal = `http://${benchmark.family === 6 ? `[${address}]` : address}/`; + for (const transport of [direct, proxy]) { + await assert.rejects(transport.fetch(literal, policy), PublicNetworkPolicyError); + } + assert.equal(lookup.mock.callCount(), 2); + assert.equal(connect.mock.callCount(), 0); + } finally { + await direct.close(); + await proxy.close(); + } + }); +} + +test('public policy rejects local names and private literals before DNS on either route', async (t) => { + const lookup = t.mock.method(dns, 'lookup', async () => assert.fail('forbidden URL reached DNS')); + restore(t); + for (const proxy of [ + null, + { enabled: true, type: 'http' as const, host: '127.0.0.1', port: 1, bypassList: [] }, + ]) { + const transport = createProxiedFetchTransport(proxy); + try { + for (const url of [ + 'http://127.0.0.1', + 'http://0x7f000001', + 'http://10.0.0.1', + 'http://192.168.0.1', + 'http://169.254.169.254', + 'http://100.64.0.1', + 'http://[::1]', + 'http://[::ffff:127.0.0.1]', + 'http://[fd00::1]', + 'http://localhost.', + 'http://router.lan', + 'http://metadata.goog', + 'http://user:password@image.example', + 'file:///tmp/image.png', + ]) { + await assert.rejects(transport.fetch(url, policy), PublicNetworkPolicyError); + } + } finally { + await transport.close(); + } + } + assert.equal(lookup.mock.callCount(), 0); +}); + +test('public direct requests pin admitted DNS, isolate ordinary sockets, and recheck each request', async (t) => { + let hits = 0; + const server = createServer((req, res) => { + assert.equal(req.headers.host, 'image.example'); + hits++; + res.end('ok'); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const port = (server.address() as net.AddressInfo).port; + const realConnect = net.connect; + let answers = [{ address: '93.184.216.34', family: 4 }]; + const lookup = t.mock.method(dns, 'lookup', async () => answers); + let pinnedConnections = 0; + t.mock.method(net, 'connect', (options: net.TcpNetConnectOpts) => { + assert.equal(options.host, 'image.example'); + if (options.lookup) { + pinnedConnections++; + // A fresh lookup would now resolve privately. The connector must use + // the admitted answer, not re-resolve or reuse an ordinary socket. + answers = [{ address: '127.0.0.1', family: 4 }]; + options.lookup(options.host, { all: true }, (error, addresses) => { + assert.equal(error, null); + assert.deepEqual(addresses, [{ address: '93.184.216.34', family: 4 }]); + }); + options.lookup(options.host, {}, (error, address, family) => { + assert.equal(error, null); + assert.equal(address, '93.184.216.34'); + assert.equal(family, 4); + }); + } + return realConnect({ host: '127.0.0.1', port }); + }); + restore(t); + const transport = createProxiedFetchTransport(null); + try { + assert.equal(await (await transport.fetch('http://image.example')).text(), 'ok'); + const mutableUrl = new URL('http://image.example'); + const checked = transport.fetch(mutableUrl, policy); + mutableUrl.hostname = '127.0.0.1'; + assert.equal(await (await checked).text(), 'ok'); + assert.equal(pinnedConnections, 1); + assert.equal(lookup.mock.callCount(), 1); + await assert.rejects(transport.fetch('http://image.example', policy), PublicNetworkPolicyError); + assert.equal(hits, 2); + // The policy on the previous requests did not alter this transport's default. + assert.equal(await (await transport.fetch('http://image.example')).text(), 'ok'); + } finally { + await transport.close(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); + +test('proxy bypass applies public DNS checks only to the direct route', async (t) => { + const lookup = t.mock.method(dns, 'lookup', async () => [{ address: '127.0.0.1', family: 4 }]); + const connect = t.mock.method(net, 'connect', () => assert.fail('blocked bypass reached socket')); + restore(t); + const transport = createProxiedFetchTransport({ + enabled: true, + type: 'http', + host: '127.0.0.1', + port: 1, + bypassList: ['image.example'], + }); + try { + await assert.rejects(transport.fetch('http://image.example', policy), PublicNetworkPolicyError); + assert.equal(lookup.mock.callCount(), 1); + assert.equal(connect.mock.callCount(), 0); + } finally { + await transport.close(); + } +}); + +for (const cancel of ['request', 'transport'] as const) { + test(`public DNS lookup is interruptible by ${cancel} cancellation`, async (t) => { + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + let finish!: (addresses: { address: string; family: number }[]) => void; + t.mock.method(dns, 'lookup', () => { + entered(); + return new Promise((resolve) => { + finish = resolve; + }); + }); + const connect = t.mock.method(net, 'connect', () => assert.fail('canceled DNS reached socket')); + restore(t); + const transport = createProxiedFetchTransport(null); + const controller = new AbortController(); + const pending = assert.rejects( + transport.fetch('http://image.example', { ...policy, signal: controller.signal }), + /stopped|transport closed/, + ); + try { + await started; + if (cancel === 'request') controller.abort(new Error('stopped')); + else await transport.close(); + await withTimeout(pending, 1000, 'DNS cancellation did not settle'); + finish([{ address: '93.184.216.34', family: 4 }]); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(connect.mock.callCount(), 0); + } finally { + await transport.close(); + } + }); +} + +test('public DNS validation cleans abort listeners on success and failure', async (t) => { + let answers = [{ address: '2606:4700:4700::1111', family: 6 }]; + t.mock.method(dns, 'lookup', async () => answers); + restore(t); + const controller = new AbortController(); + assert.deepEqual( + await preparePublicNetworkTarget(new URL('https://image.example'), false, controller.signal), + answers[0], + ); + answers = []; + await assert.rejects( + preparePublicNetworkTarget(new URL('https://image.example'), false, controller.signal), + PublicNetworkPolicyError, + ); + assert.equal(getEventListeners(controller.signal, 'abort').length, 0); +}); + +for (const cancel of ['request', 'transport'] as const) { + test(`public direct TLS handshake preserves SNI and closes on ${cancel} cancellation`, async (t) => { + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + let disconnected!: () => void; + const closed = new Promise((resolve) => { + disconnected = resolve; + }); + const sockets = new Set(); + const server = net.createServer((socket) => { + sockets.add(socket); + socket.on('error', () => {}); + socket.on('close', () => { + sockets.delete(socket); + disconnected(); + }); + socket.once('data', (chunk) => { + assert.equal(chunk[0], 22, 'expected a real TLS ClientHello'); + entered(); + }); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const port = (server.address() as net.AddressInfo).port; + const realConnect = tls.connect; + t.mock.method(dns, 'lookup', async () => [{ address: '93.184.216.34', family: 4 }]); + t.mock.method( + tls, + 'connect', + (options: tls.ConnectionOptions & { lookup?: net.TcpNetConnectOpts['lookup'] }) => { + assert.equal(options.host, 'image.example'); + assert.equal(options.servername, 'image.example'); + assert.notEqual(options.rejectUnauthorized, false); + assert.ok(options.lookup); + options.lookup(options.host!, { all: true }, (error, addresses) => { + assert.equal(error, null); + assert.deepEqual(addresses, [{ address: '93.184.216.34', family: 4 }]); + }); + return realConnect({ ...options, host: '127.0.0.1', port, lookup: undefined }); + }, + ); + restore(t); + const transport = createProxiedFetchTransport(null); + const controller = new AbortController(); + const pending = assert.rejects( + transport.fetch('https://image.example', { ...policy, signal: controller.signal }), + ); + try { + await withTimeout(started, 2000, 'TLS handshake did not start'); + if (cancel === 'request') controller.abort(); + else await transport.close(); + await withTimeout(pending, 1000, 'fetch did not stop'); + await withTimeout(closed, 1000, 'TLS socket survived cancellation'); + } finally { + await transport.close(); + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + } + }); +} diff --git a/packages/runtime/src/network/__tests__/scoped-fetch-transport.test.ts b/packages/runtime/src/network/__tests__/scoped-fetch-transport.test.ts index 9063ced0527..6da5182a0ed 100644 --- a/packages/runtime/src/network/__tests__/scoped-fetch-transport.test.ts +++ b/packages/runtime/src/network/__tests__/scoped-fetch-transport.test.ts @@ -36,7 +36,10 @@ import { } from '../../connection-effect-fetch.js'; import { runConnectionModelDiscoveryEffect } from '../../model-fetcher.js'; import { runConnectionTestEffect, testConnection } from '../../test-connection.js'; -import { createConnectionEffectFetchTransport } from '../scoped-fetch-transport.js'; +import { + createConnectionEffectFetchTransport, + createProxiedFetchTransport, +} from '../scoped-fetch-transport.js'; import { testProxyConnection } from '../proxy-test.js'; import { proxiedFetch } from '../../bots/proxied-fetch.js'; import { setActiveProxy } from '../active-proxy-state.js'; @@ -229,6 +232,37 @@ describe('connection effect network transport', () => { }); } + for (const authenticated of [false, true]) { + test(`public requests preserve SOCKS remote DNS (auth: ${authenticated})`, async () => { + const proxy = await startStalledProxy( + authenticated ? 'socks-auth-http' : 'socks-http', + false, + 'image.example', + ); + const transport = createProxiedFetchTransport({ + ...PROXY_DEFAULTS, + enabled: true, + type: 'socks5', + host: '127.0.0.1', + port: proxy.port, + username: authenticated ? 'proxy-user' : '', + password: authenticated ? 'proxy-password' : '', + bypassList: [], + }); + try { + const response = await transport.fetch('http://image.example/models', { + targetPolicy: 'public', + redirect: 'manual', + signal: AbortSignal.timeout(2000), + }); + assert.equal(await response.text(), 'proxy-ok'); + } finally { + await transport.close(); + await proxy.close(); + } + }); + } + for (const type of ['http', 'https', 'socks5'] as const) { test(`immediate close rejects pending and subsequent requests (${type})`, async () => { const proxy = await startStalledProxy( @@ -904,7 +938,11 @@ async function startSuccessfulTlsConnectProxy(body: string, protocol: 'http/1.1' }; } -async function startStalledProxy(stage: string, rejectTls = false) { +async function startStalledProxy( + stage: string, + rejectTls = false, + targetHost = 'provider.invalid', +) { const sockets = new Set(); let started!: () => void; const reachedStage = new Promise((resolve) => { @@ -965,7 +1003,7 @@ async function startStalledProxy(stage: string, rejectTls = false) { } else if (phase === 'socks-connect' && buffer.length >= 5) { assert.equal(buffer[3], 3, 'the destination hostname must be resolved by the proxy'); if (buffer.length < 7 + buffer[4]!) return; - assert.equal(buffer.subarray(5, 5 + buffer[4]!).toString(), 'provider.invalid'); + assert.equal(buffer.subarray(5, 5 + buffer[4]!).toString(), targetHost); buffer = Buffer.alloc(0); if (stage === 'socks-connect') { phase = 'stalled'; diff --git a/packages/runtime/src/network/public-network-policy.ts b/packages/runtime/src/network/public-network-policy.ts new file mode 100644 index 00000000000..9d1e4e98e83 --- /dev/null +++ b/packages/runtime/src/network/public-network-policy.ts @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { lookup } from 'node:dns/promises'; +import type { LookupAddress } from 'node:dns'; +import { isIP } from 'node:net'; + +export class PublicNetworkPolicyError extends Error { + readonly name = 'PublicNetworkPolicyError'; + constructor() { + super('Network target is not allowed by the public network policy'); + } +} + +/** Validate every URL, then resolve and pin direct destinations. Configured + * proxies resolve named destinations themselves and are trusted to enforce + * their egress boundary. Local DNS cannot establish a proxy's final address. */ +export async function preparePublicNetworkTarget( + url: URL, + useProxy: boolean, + signal: AbortSignal, +): Promise { + signal.throwIfAborted(); + if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) + throw new PublicNetworkPolicyError(); + const host = url.hostname.replace(/^\[|\]$/g, ''); + const family = isIP(host); + if (family) { + if (!publicAddress(host)) throw new PublicNetworkPolicyError(); + return useProxy ? undefined : { address: host, family }; + } + if (!publicHostname(host)) throw new PublicNetworkPolicyError(); + if (useProxy) return undefined; + const addresses = await lookupWithSignal(host, signal); + signal.throwIfAborted(); + if (!addresses.length || addresses.some(({ address }) => !publicAddress(address))) + throw new PublicNetworkPolicyError(); + return addresses[0]; +} + +function lookupWithSignal(host: string, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const abort = () => { + signal.removeEventListener('abort', abort); + reject(signal.reason); + }; + signal.addEventListener('abort', abort, { once: true }); + if (signal.aborted) abort(); + else + void lookup(host, { all: true }) + .then(resolve, reject) + .finally(() => { + signal.removeEventListener('abort', abort); + }); + }); +} + +function publicHostname(host: string): boolean { + const name = host.toLowerCase().replace(/\.$/, ''); + return ( + name.includes('.') && + !/(^|\.)(localhost|local|lan|internal|home|arpa|test|invalid)$/.test(name) && + ![ + 'metadata.google.internal', + 'metadata.goog', + 'metadata.tencentyun.com', + 'instance-data.ec2.internal', + ].includes(name) + ); +} + +function publicAddress(address: string): boolean { + if (isIP(address) === 4) { + const [a = 0, b = 0] = address.split('.').map(Number); + return ( + a > 0 && + a !== 10 && + a !== 127 && + a < 224 && + !(a === 169 && b === 254) && + !(a === 172 && b >= 16 && b <= 31) && + !(a === 192 && b === 168) && + !(a === 100 && b >= 64 && b <= 127) && + !(a === 198 && (b === 18 || b === 19)) + ); + } + if (isIP(address) !== 6) return false; + // Only global-unicast IPv6 is eligible, excluding the benchmarking prefix. + const normalized = new URL(`http://[${address}]`).hostname; + return /^[23][0-9a-f]{3}:/i.test(address) && !/^\[2001:2:(?::|0:)/i.test(normalized); +} diff --git a/packages/runtime/src/network/scoped-fetch-transport.ts b/packages/runtime/src/network/scoped-fetch-transport.ts index ba992830f01..1203021cd05 100644 --- a/packages/runtime/src/network/scoped-fetch-transport.ts +++ b/packages/runtime/src/network/scoped-fetch-transport.ts @@ -23,6 +23,8 @@ import type { ConnectionEffectFetch } from '../connection-effect-fetch.js'; import { matchesBypassList } from './bypass-matcher.js'; import { buildProxyDispatcher } from './proxy-dispatcher.js'; import { buildAbortableConnector } from './abortable-connector.js'; +import { preparePublicNetworkTarget } from './public-network-policy.js'; +export { PublicNetworkPolicyError } from './public-network-policy.js'; export const FETCH_PROXY_SNAPSHOT = Symbol.for('maka.fetch.proxy-snapshot'); @@ -45,8 +47,20 @@ export interface ConnectionEffectFetchTransport { export type ProxiedFetchProxy = ConnectionEffectProxySnapshot; +export interface ScopedFetchInit extends RequestInit { + /** Opt-in public destination policy. Direct DNS is checked and pinned; + * configured proxies own final resolution/egress. Redirects must be manual + * (each next URL is a new checked request) or error. Defaults stay unchanged. */ + readonly targetPolicy?: 'public'; +} + +export type ScopedFetch = ( + input: Parameters[0], + init?: ScopedFetchInit, +) => Promise; + export interface ProxiedFetchTransport { - readonly fetch: typeof globalThis.fetch; + readonly fetch: ScopedFetch; close(): Promise; } @@ -76,27 +90,78 @@ export function createProxiedFetchTransport( // direct and proxy connection establishment too, including TLS handshakes. const connections = new AbortController(); const directDispatcher = new Agent({ connect: buildAbortableConnector(connections.signal) }); + const publicDispatchers = new Set(); let proxyDispatcher: Dispatcher | undefined; let closePromise: Promise | undefined; let closed = false; - const fetch: typeof globalThis.fetch = async (input, init) => { + const fetch: ScopedFetch = async (input, init) => { if (closed) throw new Error('Proxied fetch transport is closed'); const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : input.url; const useProxy = proxySnapshot !== null && !matchesBypassList(new URL(url).hostname, proxySnapshot.bypassList); + const { targetPolicy, ...requestInit } = init ?? {}; + let publicDispatcher: Agent | undefined; + if (targetPolicy !== undefined) { + if (targetPolicy !== 'public') throw new Error('Unknown network target policy'); + const inputRequest = typeof input === 'string' || input instanceof URL ? undefined : input; + const redirect = requestInit.redirect ?? inputRequest?.redirect ?? 'follow'; + if (redirect !== 'manual' && redirect !== 'error') + throw new Error('Public network requests require manual redirects or redirect: error'); + const requestSignal = + requestInit.signal === undefined ? inputRequest?.signal : requestInit.signal; + const signal = requestSignal + ? AbortSignal.any([connections.signal, requestSignal]) + : connections.signal; + const target = await preparePublicNetworkTarget(new URL(url), useProxy, signal); + signal.throwIfAborted(); + requestInit.signal = signal; + if (target) { + // A separate dispatcher prevents reuse of an unchecked connection and + // binds this request to exactly the DNS answer admitted above. + publicDispatcher = new Agent({ + connect: buildAbortableConnector(signal, { + lookup: (_host, options, callback) => + options.all + ? callback(null, [target]) + : callback(null, target.address, target.family), + }), + }); + publicDispatchers.add(publicDispatcher); + } + } if (useProxy) proxyDispatcher ??= buildProxyDispatcher(proxySnapshot, connections.signal) as Dispatcher; - return (await undiciFetch( - input as Parameters[0], - { - ...init, - dispatcher: useProxy ? proxyDispatcher : directDispatcher, - } as Parameters[1], - )) as unknown as Response; + try { + const response = (await undiciFetch( + // Keep a mutable URL object bound to the destination checked before DNS awaited. + (targetPolicy === 'public' && input instanceof URL ? url : input) as Parameters< + typeof undiciFetch + >[0], + { + ...requestInit, + dispatcher: publicDispatcher ?? (useProxy ? proxyDispatcher : directDispatcher), + } as Parameters[1], + )) as unknown as Response; + if (publicDispatcher) { + const dispatcher = publicDispatcher; + // Graceful close waits for the body, without delaying delivery of headers. + void dispatcher + .close() + .catch(() => {}) + .finally(() => publicDispatchers.delete(dispatcher)); + } + return response; + } catch (error) { + if (publicDispatcher) { + await publicDispatcher.destroy().catch(() => {}); + publicDispatchers.delete(publicDispatcher); + } + throw error; + } }; Object.defineProperty(fetch, FETCH_PROXY_SNAPSHOT, { value: proxySnapshot, @@ -107,6 +172,9 @@ export function createProxiedFetchTransport( if (closePromise) return closePromise; closed = true; const destroyed = Promise.all([ + ...[...publicDispatchers].map((dispatcher) => + dispatcher.destroy(new Error('Connection effect fetch transport closed')).catch(() => {}), + ), directDispatcher .destroy(new Error('Connection effect fetch transport closed')) .catch(() => {}), diff --git a/packages/runtime/src/system-prompt/main-session-prompt.ts b/packages/runtime/src/system-prompt/main-session-prompt.ts index d7fc83a0d44..0059d456945 100644 --- a/packages/runtime/src/system-prompt/main-session-prompt.ts +++ b/packages/runtime/src/system-prompt/main-session-prompt.ts @@ -52,6 +52,9 @@ Keep simple answers simple; do not add headings or lists to simple answers. Use short headings and flat lists to organize longer answers. Use fenced code blocks for multiline code and backticks for inline commands, paths, identifiers, and literal values. Prefer descriptive link text for external sources when it is available. +To deliver a generated screenshot or local image, include a Markdown image with its absolute path. The Host automatically saves supported local images after the assistant message completes. Supported remote images in completed assistant messages load automatically near the viewport, subject to the application's outbound/privacy policy and proxy settings. This media loading is independent of session subprocess network restrictions; include remote images only when relevant to the user's request. Reading an image for your own inspection does not deliver it. +Keep source files available until capture finishes. Images above the save limits cannot be previewed automatically. +Including image Markdown does not confirm that capture succeeded or that the client displayed it. Follow a more specific format requested by the user or task.`; } diff --git a/packages/runtime/src/workspace-executor.ts b/packages/runtime/src/workspace-executor.ts index 28a4a07fcee..fc09a8507bc 100644 --- a/packages/runtime/src/workspace-executor.ts +++ b/packages/runtime/src/workspace-executor.ts @@ -44,7 +44,7 @@ import { runProcessWithBoundedTail, runShellWithBoundedTail } from './shell-exec import type { ChildFdInput } from './child-fd-input.js'; import type { ShellPlan } from './shell-detect.js'; import { isSupportedImagePath, readWorkspaceFile } from './image-file.js'; -import type { ImageMimeType } from './image-file.js'; +import type { ImageMimeType, WorkspaceFileReadOptions } from './image-file.js'; import { readTextLineWindow } from './text-line-window.js'; import { searchFiles, type GrepResult } from './grep-search.js'; import { defaultRipgrepCandidates, resolveRipgrepExecutable } from './ripgrep-executable.js'; @@ -85,7 +85,7 @@ export interface WorkspaceExecResult { aborted: boolean; } -export interface WorkspaceReadFileInput { +export interface WorkspaceReadFileInput extends WorkspaceFileReadOptions { cwd: string; path: string; offset?: number; @@ -337,7 +337,7 @@ export class LocalWorkspaceExecutor implements WorkspaceExecutor { } async readFile(input: WorkspaceReadFileInput): Promise { - const file = await readWorkspaceFile(input.path); + const file = await readWorkspaceFile(input.path, input); if ('bytes' in file) return file; return { content: readTextLineWindow(file.content, input.offset, input.limit) }; } @@ -561,9 +561,12 @@ function assertInsideCwd( label: string, ): string { if (!isPathInside(root, candidate)) { - throw new Error( - `${label} path must stay inside session cwd ${JSON.stringify(root)}; ` + - `received ${JSON.stringify(inputPath)}, which resolves to ${JSON.stringify(candidate)}.`, + throw Object.assign( + new Error( + `${label} path must stay inside session cwd ${JSON.stringify(root)}; ` + + `received ${JSON.stringify(inputPath)}, which resolves to ${JSON.stringify(candidate)}.`, + ), + { code: 'EACCES' }, ); } return candidate; diff --git a/packages/storage/src/__tests__/artifact-store.test.ts b/packages/storage/src/__tests__/artifact-store.test.ts index d63d18f73e9..a5553d007e4 100644 --- a/packages/storage/src/__tests__/artifact-store.test.ts +++ b/packages/storage/src/__tests__/artifact-store.test.ts @@ -51,6 +51,7 @@ import { resolveArtifactPath, sanitizeArtifactName, } from '../artifact-store.js'; +import { ImageArchiveQuotaError } from '../artifact-image-storage.js'; import { withArtifactWriterLock } from '../artifact-writer-lock.js'; import { createSqliteArtifactMetadataRepository } from '../sqlite-artifact-metadata.js'; @@ -382,6 +383,91 @@ describe('SQLite Artifact store', () => { }); }); + test('conversation copies enforce the target image budget across root and linked child Sessions', async () => { + await withWorkspace(async (root) => { + const store = createArtifactStore(root); + const limits = { sessionBytes: 3, workspaceBytes: 6 }; + const image = (id: string, sessionId: string, content: string): CreateArtifactInput => ({ + ...artifactInput(id, content, 10), + sessionId, + turnId: 'image-turn', + mimeType: 'image/png', + imageArchiveLimits: limits, + imageDelivery: { + status: 'ready', + messageId: id, + source: '/tmp/image.png', + contentSha256: createHash('sha256').update(content).digest('hex'), + }, + }); + await store.create(image('root-image', 'session-1', 'one')); + await assert.rejects( + store.create(image('extra-image', 'session-1', 'two')), + ImageArchiveQuotaError, + ); + await store.create(image('child-image', 'child-session', 'two')); + await assert.rejects( + store.copyConversationArtifacts({ + sourceSessionId: 'session-1', + targetSessionId: 'session-copy', + turnIds: ['image-turn'], + linkedArtifacts: [{ sessionId: 'child-session', artifactIds: ['child-image'] }], + imageArchiveLimits: limits, + }), + ImageArchiveQuotaError, + ); + const target = await listArtifacts(store, 'session-copy'); + assert.equal(target.length, 1); + assert.equal( + target.reduce((sum, record) => sum + record.sizeBytes, 0), + limits.sessionBytes, + ); + assert.deepEqual(await readArtifactText(store, 'root-image'), { ok: true, text: 'one' }); + assert.deepEqual(await readArtifactText(store, 'child-image', 'child-session'), { + ok: true, + text: 'two', + }); + }); + }); + + test('conversation copies count identical image bytes once and allow verified retries at the limit', async () => { + await withWorkspace(async (root) => { + const store = createArtifactStore(root); + const limits = { sessionBytes: 3, workspaceBytes: 3 }; + for (const [id, sessionId] of [ + ['root-image', 'session-1'], + ['child-image', 'child-session'], + ]) { + await store.create({ + ...artifactInput(id!, 'one', 10), + sessionId: sessionId!, + turnId: 'image-turn', + imageArchiveLimits: limits, + imageDelivery: { + status: 'ready', + messageId: id!, + source: '/tmp/image.png', + contentSha256: createHash('sha256').update('one').digest('hex'), + }, + }); + } + const input = { + sourceSessionId: 'session-1', + targetSessionId: 'session-copy', + turnIds: ['image-turn'], + linkedArtifacts: [{ sessionId: 'child-session', artifactIds: ['child-image'] }], + imageArchiveLimits: limits, + }; + const copied = await store.copyConversationArtifacts(input); + assert.equal(copied.artifactIds.size, 2); + assert.equal((await listArtifacts(store, 'session-copy')).length, 2); + assert.deepEqual( + await store.copyConversationArtifacts({ ...input, existingTarget: 'reuse_verified' }), + copied, + ); + }); + }); + test('includes explicit same-Session Artifacts outside the copied turns', async () => { await withWorkspace(async (root) => { const authority = createArtifactStoreWriteAuthority(root); diff --git a/packages/storage/src/__tests__/image-delivery-storage.test.ts b/packages/storage/src/__tests__/image-delivery-storage.test.ts new file mode 100644 index 00000000000..f3a10a734a9 --- /dev/null +++ b/packages/storage/src/__tests__/image-delivery-storage.test.ts @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { DatabaseSync } from 'node:sqlite'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createHash } from 'node:crypto'; +import { createSqliteArtifactStoreWriteAuthority } from '../artifact-store.js'; +import { assertImageArchiveQuota, ImageArchiveQuotaError } from '../artifact-image-storage.js'; +import { DEFAULT_IMAGE_ARCHIVE_LIMITS } from '@maka/core/image-delivery'; +import { readImageDeliveryAttempt } from '../sqlite-image-delivery.js'; + +const identity = { + sessionId: 'session-1', + turnId: 'turn-1', + messageId: 'message-1', + source: '/tmp/image.png', +}; +const lookup = ( + store: ReturnType['store'], + i = identity, +) => store.findImageDelivery(i.sessionId, i.turnId, i.messageId, i.source); + +test('schema 3 upgrades directly to 4 without rewriting existing artifacts', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-image-schema-')); + let authority = createSqliteArtifactStoreWriteAuthority(root); + try { + const record = await authority.store.create({ + sessionId: identity.sessionId, + turnId: identity.turnId, + id: 'existing-file', + name: 'note.txt', + kind: 'file', + source: 'tool_result', + content: 'retained', + }); + authority.close(); + const db = new DatabaseSync(join(root, 'runtime.sqlite')); + try { + db.exec(`DROP TABLE image_delivery_attempts; DROP INDEX artifact_image_delivery_lookup; + UPDATE operational_schema_migrations SET version = 3 WHERE scope = 'artifact';`); + } finally { + db.close(); + } + authority = createSqliteArtifactStoreWriteAuthority(root); + assert.deepEqual(await authority.store.listTurnArtifacts(identity.sessionId, identity.turnId), [ + record, + ]); + const text = await authority.store.readTextInSession(identity.sessionId, record.id); + assert.ok(text.ok); + if (text.ok) assert.equal(text.text, 'retained'); + const reopened = new DatabaseSync(join(root, 'runtime.sqlite'), { readOnly: true }); + try { + assert.equal( + reopened + .prepare("SELECT version FROM operational_schema_migrations WHERE scope = 'artifact'") + .get()?.version, + 4, + ); + assert.equal( + reopened.prepare('SELECT COUNT(*) AS n FROM image_delivery_attempts').get()?.n, + 0, + ); + } finally { + reopened.close(); + } + } finally { + authority.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +test('attempts survive reopen, stay out of conversation copies, and purge without an artifact', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-image-attempt-')); + let authority = createSqliteArtifactStoreWriteAuthority(root); + try { + await authority.store.setImageDeliveryAttempt(identity, { + status: 'failed', + reason: 'not_found', + }); + assert.equal( + (await authority.store.listPage(identity.sessionId, { offset: 0, limit: 10 })).total, + 0, + ); + assert.deepEqual( + await authority.store.listTurnArtifacts(identity.sessionId, identity.turnId), + [], + ); + authority.close(); + authority = createSqliteArtifactStoreWriteAuthority(root); + assert.deepEqual(await lookup(authority.store), { status: 'failed', reason: 'not_found' }); + await authority.store.copyConversationArtifacts({ + sourceSessionId: identity.sessionId, + targetSessionId: 'session-2', + turnIds: [identity.turnId], + }); + const copied = { ...identity, sessionId: 'session-2' }; + assert.equal(await lookup(authority.store, copied), undefined); + await authority.store.purgeSessionArtifacts(identity.sessionId); + assert.equal(await lookup(authority.store), undefined); + assert.equal(await lookup(authority.store, copied), undefined); + } finally { + authority.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +test('successful publication atomically supersedes attempts and cannot be downgraded by retries', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-image-publish-')); + const authority = createSqliteArtifactStoreWriteAuthority(root); + try { + await authority.store.setImageDeliveryAttempt(identity, { status: 'pending' }); + const content = new Uint8Array([1, 2, 3]); + const record = await authority.store.create({ + ...identity, + id: 'image-result', + name: 'image', + source: 'tool_result_projection', + kind: 'image', + content, + imageDelivery: { + messageId: identity.messageId, + source: identity.source, + status: 'ready', + contentSha256: createHash('sha256').update(content).digest('hex'), + }, + }); + await authority.store.setImageDeliveryAttempt(identity, { + status: 'failed', + reason: 'read_failed', + }); + assert.deepEqual(await lookup(authority.store), { status: 'ready', artifactId: record.id }); + const db = new DatabaseSync(join(root, 'runtime.sqlite'), { readOnly: true }); + try { + assert.equal(readImageDeliveryAttempt(db, identity), undefined); + } finally { + db.close(); + } + await assert.rejects( + authority.store.create({ + ...identity, + id: 'empty-placeholder', + name: 'image', + kind: 'file', + source: 'tool_result_projection', + content: '', + imageDelivery: { + messageId: identity.messageId, + source: identity.source, + status: 'pending', + } as never, + }), + /Invalid image delivery metadata/, + ); + } finally { + authority.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +test('archive policy enforces defaults even when a caller omits its limits', () => { + const existing = { + id: 'existing', + sessionId: identity.sessionId, + turnId: identity.turnId, + name: 'image', + kind: 'image' as const, + source: 'tool_result_projection' as const, + createdAt: 1, + relativePath: 'session-1/existing-image', + sizeBytes: DEFAULT_IMAGE_ARCHIVE_LIMITS.sessionBytes, + imageDelivery: { + messageId: identity.messageId, + source: identity.source, + status: 'ready' as const, + contentSha256: 'a'.repeat(64), + }, + }; + assert.throws( + () => + assertImageArchiveQuota([existing], { + sessionId: identity.sessionId, + sizeBytes: 1, + imageDelivery: { ...existing.imageDelivery, contentSha256: 'b'.repeat(64) }, + }), + ImageArchiveQuotaError, + ); + assert.doesNotThrow(() => + assertImageArchiveQuota([existing], { + sessionId: identity.sessionId, + sizeBytes: existing.sizeBytes, + imageDelivery: existing.imageDelivery, + }), + ); +}); diff --git a/packages/storage/src/__tests__/session-bundle-policy.test.ts b/packages/storage/src/__tests__/session-bundle-policy.test.ts index 6caaad0f43f..ab3035a348d 100644 --- a/packages/storage/src/__tests__/session-bundle-policy.test.ts +++ b/packages/storage/src/__tests__/session-bundle-policy.test.ts @@ -32,6 +32,7 @@ import { importSessionBundleState, listSessionBundleMergeTables, } from '../session-bundle-policy.js'; +import { writeImageDeliveryAttempt, readImageDeliveryAttempt } from '../sqlite-image-delivery.js'; import { createSqliteRuntimeStore } from '../sqlite-runtime-store.js'; test('exports one Session as filtered SQLite', async () => { @@ -51,6 +52,13 @@ test('exports one Session as filtered SQLite', async () => { sourceDatabase .prepare('INSERT INTO usage_pricing_overrides(model_key, record_json) VALUES (?, ?)') .run('private-model', '{}'); + for (const sessionId of [selected.id, excluded.id]) { + writeImageDeliveryAttempt( + sourceDatabase, + { sessionId, turnId: 'turn-1', messageId: 'image-message', source: '/tmp/image.png' }, + { status: 'failed', reason: 'not_found' }, + ); + } sourceDatabase.close(); const plan = await exportSessionBundleState({ @@ -67,6 +75,13 @@ test('exports one Session as filtered SQLite', async () => { .all() .map((row) => (row as { session_id: string }).session_id); assert.deepEqual(ids, [selected.id]); + assert.deepEqual( + database + .prepare('SELECT session_id FROM image_delivery_attempts') + .all() + .map((row) => row.session_id), + [selected.id], + ); assert.equal( ( database.prepare('SELECT COUNT(*) AS count FROM session_messages').get() as { @@ -86,6 +101,22 @@ test('exports one Session as filtered SQLite', async () => { } finally { database.close(); } + const targetRoot = join(base, 'imported'); + await importSessionBundleState({ stateRoot: targetRoot, bundleStateRoot: destinationRoot }); + const imported = new DatabaseSync(join(targetRoot, 'runtime.sqlite'), { readOnly: true }); + try { + assert.deepEqual( + readImageDeliveryAttempt(imported, { + sessionId: selected.id, + turnId: 'turn-1', + messageId: 'image-message', + source: '/tmp/image.png', + }), + { status: 'failed', reason: 'not_found' }, + ); + } finally { + imported.close(); + } } finally { await rm(base, { recursive: true, force: true }); } diff --git a/packages/storage/src/artifact-image-storage.ts b/packages/storage/src/artifact-image-storage.ts new file mode 100644 index 00000000000..16c33498a69 --- /dev/null +++ b/packages/storage/src/artifact-image-storage.ts @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { ArtifactRecord } from '@maka/core/artifacts'; +import { + DEFAULT_IMAGE_ARCHIVE_LIMITS, + type ImageArchiveLimits, + type ImageDeliveryMetadata, +} from '@maka/core/image-delivery'; + +export class ImageArchiveQuotaError extends Error { + readonly name = 'ImageArchiveQuotaError'; + constructor() { + super('Image archive quota exceeded'); + } +} + +/** Pure policy, evaluated by ArtifactStore while holding its writer lock. + * Budgets count unique archived content; the store shares payloads with hard links. */ +export function planImageArchive( + records: readonly ArtifactRecord[], + input: { + sessionId: string; + content: string | Uint8Array; + imageDelivery?: ImageDeliveryMetadata; + imageArchiveLimits?: ImageArchiveLimits; + }, +) { + const digest = + input.imageDelivery?.status === 'ready' ? input.imageDelivery.contentSha256 : undefined; + const size = Buffer.byteLength(input.content); + assertImageArchiveQuota(records, { ...input, sizeBytes: size }); + const sameContent = digest + ? records.find( + (r) => + r.imageDelivery?.status === 'ready' && + r.imageDelivery.contentSha256 === digest && + r.sizeBytes === size, + ) + : undefined; + return { digest, sameContent }; +} + +/** Checked under the Artifact writer lock before publishing captured or copied images. */ +export function assertImageArchiveQuota( + records: readonly ArtifactRecord[], + input: { + sessionId: string; + sizeBytes: number; + imageDelivery?: ImageDeliveryMetadata; + imageArchiveLimits?: ImageArchiveLimits; + }, +): void { + const digest = + input.imageDelivery?.status === 'ready' ? input.imageDelivery.contentSha256 : undefined; + const size = input.sizeBytes; + const limits = input.imageArchiveLimits ?? DEFAULT_IMAGE_ARCHIVE_LIMITS; + if (digest) { + if ( + ![limits.sessionBytes, limits.workspaceBytes].every((v) => Number.isSafeInteger(v) && v >= 0) + ) + throw new Error('Invalid image archive limits'); + const unique = new Map(); + const sessionUnique = new Map(); + for (const record of records) { + const hash = record.imageDelivery?.contentSha256; + if (record.imageDelivery?.status !== 'ready' || !hash) continue; + unique.set(hash, record.sizeBytes); + if (record.sessionId === input.sessionId) sessionUnique.set(hash, record.sizeBytes); + } + const sum = (values: Map) => [...values.values()].reduce((a, b) => a + b, 0); + if ( + sum(unique) + (unique.has(digest) ? 0 : size) > limits.workspaceBytes || + sum(sessionUnique) + (sessionUnique.has(digest) ? 0 : size) > limits.sessionBytes + ) + throw new ImageArchiveQuotaError(); + } +} diff --git a/packages/storage/src/artifact-metadata-codec.ts b/packages/storage/src/artifact-metadata-codec.ts index adabf324a5c..c559baba2e2 100644 --- a/packages/storage/src/artifact-metadata-codec.ts +++ b/packages/storage/src/artifact-metadata-codec.ts @@ -17,6 +17,7 @@ * under the License. */ +import { isImageDeliveryMetadata } from '@maka/core/image-delivery'; import { isAbsolute } from 'node:path'; import { ARTIFACT_KINDS, @@ -42,6 +43,7 @@ const ARTIFACT_RECORD_KEYS = new Set([ 'mimeType', 'source', 'summary', + 'imageDelivery', 'deepResearchRole', ]); @@ -111,6 +113,7 @@ function decodeArtifactRecord(value: unknown, index: number): ArtifactRecord { value.sizeBytes < 0 || !isOptionalNonEmptyString(value.mimeType) || !isOptionalNonEmptyString(value.summary) || + (value.imageDelivery !== undefined && !isImageDeliveryMetadata(value.imageDelivery)) || typeof value.source !== 'string' ) { throw invalidMetadataRecord(index); diff --git a/packages/storage/src/artifact-store.ts b/packages/storage/src/artifact-store.ts index 151a2b7a2a7..745c94db5b6 100644 --- a/packages/storage/src/artifact-store.ts +++ b/packages/storage/src/artifact-store.ts @@ -17,6 +17,16 @@ * under the License. */ +import { assertImageArchiveQuota, planImageArchive } from './artifact-image-storage.js'; +import { + DEFAULT_IMAGE_ARCHIVE_LIMITS, + isImageDeliveryMetadata, + type ImageDeliveryMetadata, + type ImageDeliveryIdentity, + type ImageDeliveryAttempt, + type ImageDeliveryResult, + type ImageArchiveLimits, +} from '@maka/core/image-delivery'; import { createHash, randomUUID } from 'node:crypto'; import { isDeepStrictEqual } from 'node:util'; import { type BigIntStats, constants as fsConstants } from 'node:fs'; @@ -24,6 +34,7 @@ import { access, copyFile, lstat, + link, mkdir, open, readFile, @@ -104,6 +115,8 @@ export interface CreateArtifactInput { summary?: string; now?: number; id?: string; + imageDelivery?: ImageDeliveryMetadata; + imageArchiveLimits?: ImageArchiveLimits; } export type ArtifactListRevision = `sha256:${string}`; @@ -131,6 +144,7 @@ export type ArtifactChunkReadResult = | { readonly ok: false; readonly reason: 'out_of_range' }; export interface ConversationArtifactCopyInput { + readonly imageArchiveLimits?: ImageArchiveLimits; readonly sourceSessionId: string; readonly targetSessionId: string; readonly turnIds: readonly string[]; @@ -189,6 +203,16 @@ export interface ArtifactUpgradeCleanupResult { export interface ArtifactAuthorityStore extends DurableArtifactAttachmentReader { create(input: CreateArtifactInput): Promise; + findImageDelivery( + sessionId: string, + turnId: string, + messageId: string, + source: string, + ): Promise; + setImageDeliveryAttempt( + identity: ImageDeliveryIdentity, + attempt: ImageDeliveryAttempt, + ): Promise; close(): void; copyConversationArtifacts( input: ConversationArtifactCopyInput, @@ -269,11 +293,50 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { this.metadataRepository.close(); } + async findImageDelivery( + sessionId: string, + turnId: string, + messageId: string, + source: string, + ): Promise { + return this.enqueue(async () => + this.metadataRepository.findImageDelivery(sessionId, turnId, messageId, source), + ); + } + + setImageDeliveryAttempt( + identity: ImageDeliveryIdentity, + attempt: ImageDeliveryAttempt, + ): Promise { + assertCanonicalArtifactEntityId(identity.sessionId, 'sessionId'); + assertArtifactTurnKey(identity.turnId); + const acceptedIdentity = Object.freeze({ ...identity }); + const acceptedAttempt = Object.freeze({ ...attempt }); + return this.enqueueMutation(async () => + this.metadataRepository.setImageDeliveryAttempt(acceptedIdentity, acceptedAttempt), + ); + } + async create(input: CreateArtifactInput): Promise { const acceptedInput: CreateArtifactInput = Object.freeze({ ...input, + ...(input.imageDelivery ? { imageDelivery: Object.freeze({ ...input.imageDelivery }) } : {}), + ...(input.imageArchiveLimits + ? { imageArchiveLimits: Object.freeze({ ...input.imageArchiveLimits }) } + : {}), content: typeof input.content === 'string' ? input.content : new Uint8Array(input.content), }); + if ( + acceptedInput.imageDelivery !== undefined && + (!isImageDeliveryMetadata(acceptedInput.imageDelivery) || + acceptedInput.imageDelivery.status !== 'ready') + ) + throw new Error('Invalid image delivery metadata'); + if ( + acceptedInput.imageDelivery?.status === 'ready' && + acceptedInput.imageDelivery.contentSha256 !== sha256(Buffer.from(acceptedInput.content)) + ) + throw new Error('Image delivery digest does not match its bytes'); const id = acceptedInput.id ?? randomUUID(); if (!ARTIFACT_KIND_SET.has(acceptedInput.kind)) throw new Error('Invalid Artifact kind'); if (!ARTIFACT_SOURCE_SET.has(acceptedInput.source)) { @@ -301,6 +364,7 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { relativePath, }); } + const { digest, sameContent } = planImageArchive(this.records, acceptedInput); return this.publishNewArtifactUnlocked( { id, @@ -313,8 +377,20 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { ...(acceptedInput.mimeType ? { mimeType: acceptedInput.mimeType } : {}), source: acceptedInput.source, ...(acceptedInput.summary ? { summary: acceptedInput.summary } : {}), + ...(acceptedInput.imageDelivery ? { imageDelivery: acceptedInput.imageDelivery } : {}), + }, + async (targetPath) => { + if (sameContent) { + const source = await this.prepareRecordRead(sameContent, sameContent.sizeBytes); + if ( + source.ok && + (await hashPreparedArtifact(source)) === digest && + (await tryLinkImagePayload(source.path, targetPath)) + ) + return; + } + await writeFile(targetPath, acceptedInput.content, { flag: 'wx' }); }, - (targetPath) => writeFile(targetPath, acceptedInput.content, { flag: 'wx' }), ); }); } @@ -327,6 +403,9 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { if (input.sourceSessionId === input.targetSessionId) { throw new Error('Artifact conversation copy requires distinct Sessions'); } + const imageArchiveLimits = Object.freeze({ + ...(input.imageArchiveLimits ?? DEFAULT_IMAGE_ARCHIVE_LIMITS), + }); const turnIds = new Set(input.turnIds); const includedArtifactIds = new Set(input.includeArtifactIds ?? []); for (const turnId of turnIds) assertArtifactTurnKey(turnId); @@ -395,10 +474,12 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { input.targetSessionId, targetId, input.existingTarget === 'reuse_verified', + imageArchiveLimits, ); artifactIds.set(record.id, created.id); relativePaths.set(record.relativePath, created.relativePath); } + return { artifactIds, relativePaths }; } @@ -407,6 +488,7 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { targetSessionId: string, targetId: string, reuseVerified: boolean, + imageArchiveLimits: ImageArchiveLimits, ): Promise { const source = prepared.record; const name = sanitizeArtifactName(source.name); @@ -442,9 +524,21 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { } return { ...existing }; } + assertImageArchiveQuota(this.records, { ...expected, imageArchiveLimits }); return this.publishNewArtifactUnlocked( expected, - (targetPath) => copyFile(prepared.path, targetPath, fsConstants.COPYFILE_EXCL), + async (targetPath) => { + if (source.imageDelivery?.status === 'ready') { + const current = await this.prepareRecordRead(source, source.sizeBytes); + if ( + !current.ok || + (await hashPreparedArtifact(current)) !== source.imageDelivery.contentSha256 + ) + throw artifactReplayConflict(source.id); + if (await tryLinkImagePayload(current.path, targetPath)) return; + } + await copyFile(prepared.path, targetPath, fsConstants.COPYFILE_EXCL); + }, source.sizeBytes, ); }); @@ -489,6 +583,7 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { await this.purgeRecordsUnlocked( this.records.filter((record) => record.sessionId === sessionId), ); + this.metadataRepository.purgeImageDeliveryAttempts(sessionId); }); } @@ -600,6 +695,7 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { existing.mimeType !== optionalCanonicalText(input.mimeType) || existing.source !== input.source || existing.summary !== optionalCanonicalText(input.summary) || + !isDeepStrictEqual(existing.imageDelivery, input.imageDelivery) || (input.now !== undefined && existing.createdAt !== input.now) ) { throw artifactReplayConflict(canonical.id); @@ -1331,3 +1427,19 @@ function sniffAllowedBinaryMime(bytes: Uint8Array): string | null { return 'image/svg+xml'; return null; } + +async function tryLinkImagePayload(source: string, target: string): Promise { + try { + await link(source, target); + return true; + } catch (error) { + // Filesystems without hard links retain the regular write/copy fallback. + if ( + !['ENOTSUP', 'EPERM', 'EXDEV', 'EACCES'].includes( + String((error as NodeJS.ErrnoException).code), + ) + ) + throw error; + return false; + } +} diff --git a/packages/storage/src/artifact-stores.ts b/packages/storage/src/artifact-stores.ts index bf950c22e71..7f9246392ec 100644 --- a/packages/storage/src/artifact-stores.ts +++ b/packages/storage/src/artifact-stores.ts @@ -28,6 +28,7 @@ import { type DurableArtifactAttachmentReader, } from './artifact-store.js'; +export { ImageArchiveQuotaError } from './artifact-image-storage.js'; export { sanitizeArtifactName } from './artifact-store.js'; import { assertStorageRootLease, @@ -57,6 +58,8 @@ export interface InteractiveArtifactStoreWriter extends DurableArtifactAttachmen readonly access: 'write'; readonly [writerBrand]: true; create(input: CreateArtifactInput): Promise; + findImageDelivery: ArtifactAuthorityStore['findImageDelivery']; + setImageDeliveryAttempt: ArtifactAuthorityStore['setImageDeliveryAttempt']; /** * Narrow system delete for one Session-owned artifact of a declared source. * @@ -136,6 +139,13 @@ function createWriterFacade( kind: 'interactive', access: 'write', [writerBrand]: true, + findImageDelivery: (sessionId, turnId, messageId, source) => + run(() => store.findImageDelivery(sessionId, turnId, messageId, source)), + setImageDeliveryAttempt: (identity, attempt) => { + const acceptedIdentity = Object.freeze({ ...identity }); + const acceptedAttempt = Object.freeze({ ...attempt }); + return run(() => store.setImageDeliveryAttempt(acceptedIdentity, acceptedAttempt)); + }, listPage: (sessionId, options) => run(() => store.listPage(sessionId, options)), listTurnArtifacts: (sessionId, turnId) => run(() => store.listTurnArtifacts(sessionId, turnId)), getInSession: (sessionId, artifactId) => run(() => store.getInSession(sessionId, artifactId)), @@ -156,6 +166,9 @@ function createWriterFacade( const acceptedInput: ConversationArtifactCopyInput = Object.freeze({ ...input, turnIds: Object.freeze([...input.turnIds]), + ...(input.imageArchiveLimits + ? { imageArchiveLimits: Object.freeze({ ...input.imageArchiveLimits }) } + : {}), ...(input.includeArtifactIds ? { includeArtifactIds: Object.freeze([...input.includeArtifactIds]) } : {}), @@ -189,6 +202,10 @@ function createWriterFacade( function snapshotCreateInput(input: CreateArtifactInput): CreateArtifactInput { return Object.freeze({ ...input, + ...(input.imageDelivery ? { imageDelivery: Object.freeze({ ...input.imageDelivery }) } : {}), + ...(input.imageArchiveLimits + ? { imageArchiveLimits: Object.freeze({ ...input.imageArchiveLimits }) } + : {}), content: typeof input.content === 'string' ? input.content : new Uint8Array(input.content), }); } diff --git a/packages/storage/src/sqlite-artifact-metadata.ts b/packages/storage/src/sqlite-artifact-metadata.ts index df4094ef66e..569f9c551fe 100644 --- a/packages/storage/src/sqlite-artifact-metadata.ts +++ b/packages/storage/src/sqlite-artifact-metadata.ts @@ -18,6 +18,16 @@ */ import { resolve } from 'node:path'; +import type { + ImageDeliveryIdentity, + ImageDeliveryAttempt, + ImageDeliveryResult, +} from '@maka/core/image-delivery'; +import { + readImageDeliveryAttempt, + writeImageDeliveryAttempt, + removeImageDeliveryAttempt, +} from './sqlite-image-delivery.js'; import type { ArtifactRecord } from '@maka/core/artifacts'; import { decodeArtifactRecordJsons } from './artifact-metadata-codec.js'; import { @@ -42,6 +52,57 @@ class SqliteArtifactMetadataRepository { this.#lease = acquireOperationalStateDatabase(resolve(workspaceRoot)); } + findImageDelivery( + sessionId: string, + turnId: string, + messageId: string, + source: string, + ): ImageDeliveryResult | undefined { + this.assertOpen(); + const rows = this.#lease.database + .prepare(` + SELECT record_json FROM artifact_records + WHERE session_id = ? AND json_valid(record_json) + AND json_type(record_json, '$.imageDelivery') = 'object' + AND json_extract(record_json, '$.turnId') = ? + AND json_extract(record_json, '$.imageDelivery.messageId') = ? + AND json_extract(record_json, '$.imageDelivery.source') = ? + AND json_extract(record_json, '$.imageDelivery.status') = 'ready' + ORDER BY created_at DESC, artifact_id DESC LIMIT 1 + `) + .all(sessionId, turnId, messageId, source) as Array<{ record_json: string }>; + const record = decodeRows(rows)[0]; + if (record?.imageDelivery?.status === 'ready') + return { status: 'ready', artifactId: record.id }; + return readImageDeliveryAttempt(this.#lease.database, { sessionId, turnId, messageId, source }); + } + + setImageDeliveryAttempt(identity: ImageDeliveryIdentity, attempt: ImageDeliveryAttempt): void { + this.assertOpen(); + this.#lease.transaction('write', () => { + // Saved history is immutable, including when a retry races publication. + if ( + this.findImageDelivery( + identity.sessionId, + identity.turnId, + identity.messageId, + identity.source, + )?.status === 'ready' + ) + return; + writeImageDeliveryAttempt(this.#lease.database, identity, attempt); + }); + } + + purgeImageDeliveryAttempts(sessionId: string): void { + this.assertOpen(); + this.#lease.transaction('write', () => { + this.#lease.database + .prepare('DELETE FROM image_delivery_attempts WHERE session_id = ?') + .run(sessionId); + }); + } + readAll(): ArtifactRecord[] { this.assertOpen(); const rows = this.#lease.database @@ -81,6 +142,13 @@ class SqliteArtifactMetadataRepository { OR record_json IS NOT excluded.record_json `); for (const record of changes.upserts ?? []) { + if (record.imageDelivery?.status === 'ready') + removeImageDeliveryAttempt(this.#lease.database, { + sessionId: record.sessionId, + turnId: record.turnId, + messageId: record.imageDelivery.messageId, + source: record.imageDelivery.source, + }); upsert.run( record.id, record.sessionId, diff --git a/packages/storage/src/sqlite-artifact-schema.ts b/packages/storage/src/sqlite-artifact-schema.ts index 84a880ed8b0..3ad1819785a 100644 --- a/packages/storage/src/sqlite-artifact-schema.ts +++ b/packages/storage/src/sqlite-artifact-schema.ts @@ -23,7 +23,9 @@ import { isSafeRelativeArtifactPath, } from './artifact-metadata-codec.js'; -export const SQLITE_ARTIFACT_SCHEMA_VERSION = 3; +import { createImageDeliveryAttemptTable } from './sqlite-image-delivery.js'; + +export const SQLITE_ARTIFACT_SCHEMA_VERSION = 4; export function migrateSqliteArtifactDatabase(db: DatabaseSync): void { const columns = db.prepare('PRAGMA table_info(artifact_records)').all() as Array<{ @@ -79,6 +81,11 @@ export function migrateSqliteArtifactDatabase(db: DatabaseSync): void { CREATE UNIQUE INDEX IF NOT EXISTS artifact_records_relative_path ON artifact_records(relative_path); + CREATE INDEX IF NOT EXISTS artifact_image_delivery_lookup + ON artifact_records(session_id, json_extract(record_json, '$.turnId'), + json_extract(record_json, '$.imageDelivery.messageId'), json_extract(record_json, '$.imageDelivery.source')) + WHERE json_valid(record_json) AND json_type(record_json, '$.imageDelivery') = 'object'; + CREATE TABLE IF NOT EXISTS artifact_upgrade_orphan_paths ( relative_path TEXT PRIMARY KEY ); @@ -102,4 +109,5 @@ export function migrateSqliteArtifactDatabase(db: DatabaseSync): void { JSON.stringify(record), ); } + createImageDeliveryAttemptTable(db); } diff --git a/packages/storage/src/sqlite-image-delivery.ts b/packages/storage/src/sqlite-image-delivery.ts new file mode 100644 index 00000000000..678d46926a9 --- /dev/null +++ b/packages/storage/src/sqlite-image-delivery.ts @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { DatabaseSync } from 'node:sqlite'; +import type { ImageDeliveryIdentity, ImageDeliveryAttempt } from '@maka/core/image-delivery'; +import { + isImageDeliveryAttempt, + isImageDeliveryRequest, + IMAGE_DELIVERY_FAILURES, +} from '@maka/core/image-delivery'; + +/** Shares the Artifact writer's transaction and lock; no payload files for attempts. */ +export function createImageDeliveryAttemptTable(db: DatabaseSync): void { + db.exec(`CREATE TABLE IF NOT EXISTS image_delivery_attempts ( + session_id TEXT NOT NULL, turn_id TEXT NOT NULL, message_id TEXT NOT NULL, + source TEXT NOT NULL, status TEXT NOT NULL CHECK (status IN ('pending', 'failed')), + reason TEXT, created_at INTEGER NOT NULL CHECK (created_at >= 0), + CHECK ((status = 'pending' AND reason IS NULL) OR (status = 'failed' AND reason IS NOT NULL)), + PRIMARY KEY (session_id, turn_id, message_id, source) + )`); +} + +export function readImageDeliveryAttempt( + db: DatabaseSync, + identity: ImageDeliveryIdentity, +): ImageDeliveryAttempt | undefined { + const row = db + .prepare(`SELECT status, reason FROM image_delivery_attempts + WHERE session_id = ? AND turn_id = ? AND message_id = ? AND source = ?`) + .get(...identityValues(identity)) as { status: string; reason: unknown } | undefined; + return row ? decodeAttempt(row) : undefined; +} + +function decodeAttempt(row: { status: string; reason: unknown }): ImageDeliveryAttempt { + if (row.status === 'pending' && row.reason === null) return { status: 'pending' }; + if (row.status === 'failed' && IMAGE_DELIVERY_FAILURES.includes(row.reason as never)) + return { + status: 'failed', + reason: row.reason as Extract['reason'], + }; + throw new Error('Invalid image delivery attempt'); +} + +export function writeImageDeliveryAttempt( + db: DatabaseSync, + identity: ImageDeliveryIdentity, + attempt: ImageDeliveryAttempt, + now = Date.now(), +): void { + if ( + !isImageDeliveryRequest({ + turnId: identity.turnId, + messageId: identity.messageId, + source: identity.source, + }) || + !isImageDeliveryAttempt(attempt) + ) + throw new Error('Invalid image delivery attempt'); + db.prepare(`INSERT INTO image_delivery_attempts VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id, turn_id, message_id, source) DO UPDATE SET status = excluded.status, reason = excluded.reason`).run( + ...identityValues(identity), + attempt.status, + attempt.status === 'failed' ? attempt.reason : null, + now, + ); +} + +export function removeImageDeliveryAttempt( + db: DatabaseSync, + identity: ImageDeliveryIdentity, +): void { + db.prepare( + 'DELETE FROM image_delivery_attempts WHERE session_id = ? AND turn_id = ? AND message_id = ? AND source = ?', + ).run(...identityValues(identity)); +} + +function identityValues(identity: ImageDeliveryIdentity): [string, string, string, string] { + return [identity.sessionId, identity.turnId, identity.messageId, identity.source]; +} diff --git a/packages/ui/package.json b/packages/ui/package.json index d94cfb38499..8de5e24cdb8 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -24,7 +24,8 @@ "build": "tsc -p tsconfig.json", "benchmark:redaction": "npm run build && node scripts/benchmark-streaming-redaction.mjs", "typecheck": "tsc -p tsconfig.json --noEmit", - "test:dist": "node --test \"dist/**/*.test.js\"" + "test:dist": "node --test \"dist/**/*.test.js\"", + "test:images:browser": "npm run build && node --test scripts/markdown-images.browser.test.mjs" }, "dependencies": { "@astryxdesign/core": "0.6.3", @@ -38,8 +39,10 @@ "virtua": "0.52.7" }, "devDependencies": { + "@playwright/test": "^1.63.0", "@types/react": "^19.3.0", "@types/react-dom": "^19.3.0", + "esbuild": "^0.28.2", "linkedom": "^0.18.13" } } diff --git a/packages/ui/scripts/markdown-images.browser.test.mjs b/packages/ui/scripts/markdown-images.browser.test.mjs new file mode 100644 index 00000000000..4f0d873537c --- /dev/null +++ b/packages/ui/scripts/markdown-images.browser.test.mjs @@ -0,0 +1,425 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Chromium owns image requests and CSP enforcement; fake DOM tests cannot +// detect an allowed Markdown URL that the desktop policy blocks. +// Run after building @maka/ui: node --test packages/ui/scripts/markdown-images.browser.test.mjs +import assert from 'node:assert/strict'; +import { before, after, test } from 'node:test'; +import { createServer } from 'node:http'; +import { readFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import { build } from 'esbuild'; +import { chromium } from '@playwright/test'; + +const root = fileURLToPath(new URL('../../../', import.meta.url)); +const png = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==', 'base64'); +let appServer, imageServer, browser, appUrl, imageUrl; +let requests = [], failedOnce = false; +before(async () => { + imageServer = createServer((req, res) => { + requests.push({ path: req.url, referer: req.headers.referer }); + if (req.url === '/retry.png' && !failedOnce) { + failedOnce = true; + res.writeHead(404).end(); + } else { + res.setHeader('Content-Type', 'image/png'); + res.end(png); + } + }); + await new Promise(resolve => imageServer.listen(0, '127.0.0.1', resolve)); + imageUrl = `http://127.0.0.1:${imageServer.address().port}`; + const destinations = { + 'links-formats': [ + `![Screenshot](${imageUrl}/image.png)`, + `![Title](${imageUrl}/title.png "Screenshot title")`, + `![Angle](<${imageUrl}/angle.png>)`, + '![Escaped](' + imageUrl + String.raw`/a\(1\).png)`, + '![Reference][picture]', + `[picture]: <${imageUrl}/reference.png>`, + `Build ![Badge](${imageUrl}/badge.png) passing.`, + `![](${imageUrl}/empty.png)`, + '![Local](/tmp/private.png)', + '`![Example](https://example.com/code.png)`', + ].join('\n\n'), + 'links-redacted': `![Signed](${imageUrl}/image.png?token=reasoning-secret)\n\n![Blocked](javascript:alert)`, + 'links-streaming': `![Streaming](${imageUrl}/stream`, + 'angle-saved': '![Screenshot]()', + 'title-saved': `![Screenshot](${imageUrl}/image.png "Screenshot title")`, + 'escaped-saved': String.raw`![Screenshot](/tmp/a\(1\).png)`, + 'reference-saved': '![Screenshot][picture]\n\n[picture]: ', + 'attachment-title': '![Screenshot](maka://runtime/attachments/image-1 "Screenshot title")', + 'streaming-angle-race': '![Screenshot]()', + 'signed-remote': `![Screenshot](${imageUrl}/image.png?token=remote-secret)`, + 'signed-inline': `Build ![Badge](${imageUrl}/image.png?token=inline-secret) passing.`, + 'signed-failed': `![Screenshot](${imageUrl}/image.png?token=failed-secret)`, + 'signed-preview': `![Screenshot](${imageUrl}/image.png?token=preview-secret)`, + }; + const canonicalSources = { + 'angle-saved': ['/tmp/my image.png'], + 'title-saved': [`${imageUrl}/image.png`], + 'escaped-saved': ['/tmp/a(1).png'], + 'reference-saved': ['/tmp/my "image".png'], + 'streaming-angle-race': ['/tmp/my image.png'], + }; + const bundle = await build({ + stdin: { contents: ` + import React from 'react'; + import {createRoot} from 'react-dom/client'; + import {Theme} from '@astryxdesign/core'; + import {makaTheme} from './apps/desktop/src/renderer/astryx-theme/maka.js'; + import {Markdown} from './packages/ui/dist/markdown.js'; + import {LocaleProvider} from './packages/ui/dist/locale-context.js'; + import {ImageDeliveryProvider, ImageMessageProvider} from './packages/ui/dist/image-delivery.js'; + import {SessionAttachmentProvider} from './packages/ui/dist/attachment-image.js'; + import './apps/desktop/src/renderer/styles.css'; + const mode=new URLSearchParams(location.search).get('case') || 'remote'; + const destinations=${JSON.stringify(destinations)}; + const canonicalSources=${JSON.stringify(canonicalSources)}; + let reads=0; const captured=new Map(); window.imageReads=0; window.deliveryQueries=0; window.deliverySources=[]; window.deliveryReady=false; window.deliveryRetries=0; + const race=mode.endsWith('-race'); + let messageId='message'; + let text=destinations[mode] ?? (mode==='attachment' ? '![Screenshot](maka://runtime/attachments/image-1)' : + mode==='local' || mode==='local-saved' || race ? '![Screenshot](/tmp/private.png)' : + '![Screenshot](${imageUrl}/'+(mode==='retry' ? 'retry.png' : 'image.png')+')'); + const root=createRoot(document.getElementById('root')); + const readBytes=async(_session,artifactId)=>{ + if(captured.has(artifactId)) return captured.get(artifactId); + reads++; window.imageReads=reads; return (mode==='attachment' || mode==='read-retry-saved') && reads===1 ? {ok:false,reason:'read_failed'} : + {ok:true,base64:mode.endsWith('corrupt-saved') && reads===1 ? 'iVBORw0KGgo=' : '${png.toString('base64')}',mimeType:'image/png'}; + }; + const preview=mode==='preview' || mode==='signed-preview' || mode==='local'; + const seeded=mode.endsWith('saved') || mode.endsWith('saving') || race; + const resolveDelivery=preview ? undefined : async(_session,request)=>{ + window.deliveryQueries++; + window.deliverySource=request.source; + window.deliverySources.push(request.source); + if(canonicalSources[mode] && !canonicalSources[mode].includes(request.source)) return {status:'unavailable'}; + if(request.retry) window.deliveryRetries++; + if(mode==='privacy') return {status:'failed',reason:'not_allowed'}; + if(mode==='signed-failed') return {status:'failed',reason:'download_failed'}; + if(!seeded) { + if(!request.loadRemote) return {status:'requires_confirmation'}; + const response=await fetch('/capture?source='+encodeURIComponent(request.source)); + const payload=await response.json(); + if(!payload.ok) return {status:'failed',reason:'download_failed'}; + captured.set(request.source,payload); + return {status:'ready',artifactId:request.source}; + } + if(race) return window.deliveryReady ? {status:'ready',artifactId:'saved-image'} : {status:'unavailable'}; + await new Promise(resolve=>setTimeout(resolve,100)); + return mode.endsWith('saving') && window.deliveryQueries===1 ? {status:'pending'} : {status:'ready',artifactId:'saved-image'}; + }; + const markdown=(streaming)=>React.createElement(ImageMessageProvider,{identity:{turnId:'turn',messageId},streaming}, + React.createElement(Markdown,{text,streaming,settledText:race ? text : undefined,density:'compact',imageDisplay:mode.startsWith('links-') ? 'link' : undefined})); + const render=(streaming=race || mode==='links-streaming')=>root.render( + React.createElement(Theme,{theme:makaTheme,mode:'light'}, + React.createElement(LocaleProvider,{locale:'en'}, + React.createElement(SessionAttachmentProvider,{sessionId:'session',readBytes}, + React.createElement(ImageDeliveryProvider,{sessionId:'session',resolve:resolveDelivery}, + React.createElement('div',{style:mode==='offscreen' ? {paddingTop:'2500px'} : {}}, + markdown(streaming))))))); + window.nextMessage=()=>{messageId='next-message'; render(false);}; + window.finishStream=()=>render(false); + window.appendText=value=>{text+=value; render(true);}; + render(); + `, resolveDir: root, loader: 'js' }, + bundle: true, write: false, outdir: '/virtual', format: 'iife', + loader: { '.woff2': 'dataurl', '.woff': 'dataurl', '.ttf': 'dataurl', '.svg': 'dataurl' }, + define: { 'process.env.NODE_ENV': '"production"' }, + }); + const js = bundle.outputFiles.find(file => file.path.endsWith('.js')).text; + const css = bundle.outputFiles.find(file => file.path.endsWith('.css')).text; + const index = await readFile(new URL('../../../apps/desktop/src/renderer/index.html', import.meta.url), 'utf8'); + const csp = index.match(/http-equiv="Content-Security-Policy"\s+content="([^"]+)"/)[1]; + appServer = createServer((req, res) => { + if (req.url.startsWith('/capture?')) { + const source=new URL(req.url, appUrl).searchParams.get('source'); + void fetch(source).then(async response=> { + const payload=response.ok ? {ok:true,base64:Buffer.from(await response.arrayBuffer()).toString('base64'),mimeType:'image/png'} : {ok:false}; + res.setHeader('Content-Type','application/json'); res.end(JSON.stringify(payload)); + }).catch(()=>res.end(JSON.stringify({ok:false}))); return; + } + if (req.url === '/app.js') { res.setHeader('Content-Type', 'text/javascript'); res.end(js); } + else if (req.url === '/app.css') { res.setHeader('Content-Type', 'text/css'); res.end(css); } + else { + res.setHeader('Content-Type', 'text/html'); + res.end(`
`); + } + }); + await new Promise(resolve => appServer.listen(0, '127.0.0.1', resolve)); + appUrl = `http://127.0.0.1:${appServer.address().port}`; + browser = await chromium.launch({ headless: true }); +}); + +after(async () => { + await browser?.close(); + for (const server of [appServer, imageServer]) if (server) await new Promise(resolve => server.close(resolve)); +}); + +async function pageFor(scenario) { + const page = await browser.newPage({ viewport: { width: 720, height: 600 } }); + page.setDefaultTimeout(5000); + await page.goto(`${appUrl}/?case=${scenario}`, { waitUntil: 'domcontentloaded' }); + return page; +} + +async function loaded(page) { + await page.waitForFunction(() => [...document.images].some(image => image.naturalWidth === 1)); +} + +test('image links preserve Markdown destinations without loading image or attachment bytes', async () => { + requests = []; + const page = await pageFor('links-formats'); + try { + await page.getByRole('link', {name:/^Screenshot/}).waitFor(); + assert.equal(await page.getByRole('link', {name:/^Screenshot/}).getAttribute('href'), `${imageUrl}/image.png`); + assert.equal(await page.getByRole('link', {name:/^Title/}).getAttribute('href'), `${imageUrl}/title.png`); + assert.equal(await page.getByRole('link', {name:/^Angle/}).getAttribute('href'), `${imageUrl}/angle.png`); + assert.equal(await page.getByRole('link', {name:/^Escaped/}).getAttribute('href'), `${imageUrl}/a(1).png`); + assert.equal(await page.getByRole('link', {name:/^Reference/}).getAttribute('href'), `${imageUrl}/reference.png`); + assert.equal(await page.getByRole('link', {name:/^Badge/}).getAttribute('href'), `${imageUrl}/badge.png`); + assert.equal(await page.locator('a').filter({hasText:`${imageUrl}/empty.png`}).getAttribute('href'), `${imageUrl}/empty.png`); + assert.equal(await page.locator('a a').count(), 0); + assert.equal(await page.getByRole('link', {name:'Local', exact:true}).count(), 0); + assert.equal(await page.locator('code').innerText(), '![Example](https://example.com/code.png)'); + assert.equal(await page.locator('img, .maka-markdown-image-resource').count(), 0); + assert.deepEqual(await page.evaluate(() => [window.deliveryQueries, window.imageReads]), [0, 0]); + assert.deepEqual(requests, []); + } finally { await page.close(); } +}); + +test('image links do not expose signed destinations or enable unsafe schemes', async () => { + requests = []; + const page = await pageFor('links-redacted'); + try { + await page.locator('[data-maka-contract="markdown"]').waitFor(); + assert.equal(await page.locator('a, img, .maka-markdown-image-resource').count(), 0); + assert.doesNotMatch(await page.locator('#root').innerHTML(), /reasoning-secret/); + assert.deepEqual(await page.evaluate(() => [window.deliveryQueries, window.imageReads]), [0, 0]); + assert.deepEqual(requests, []); + } finally { await page.close(); } +}); + +test('streaming reasoning completes image syntax as a link without fetching it', async () => { + requests = []; + const page = await pageFor('links-streaming'); + try { + await page.waitForFunction(() => typeof window.appendText === 'function'); + await page.evaluate(() => window.appendText('.png)')); + await page.getByRole('link', {name:/^Streaming/}).waitFor(); + await page.evaluate(() => window.finishStream()); + assert.equal(await page.getByRole('link', {name:/^Streaming/}).getAttribute('href'), `${imageUrl}/stream.png`); + assert.equal(await page.locator('img, .maka-markdown-image-resource').count(), 0); + assert.deepEqual(await page.evaluate(() => [window.deliveryQueries, window.imageReads]), [0, 0]); + assert.deepEqual(requests, []); + } finally { await page.close(); } +}); + +test('remote images load automatically as Host-owned bytes with a separate identity per message', async () => { + requests = []; + const page = await pageFor('remote'); + try { + await loaded(page); + assert.equal(await page.getByRole('button', { name: 'Load image', exact: true }).count(), 0); + assert.deepEqual(requests, [{ path: '/image.png', referer: undefined }]); + assert.match(await page.locator('img').getAttribute('src'), /^data:/); + await page.evaluate(() => window.nextMessage()); + await page.waitForFunction(() => window.deliveryQueries >= 2); + await loaded(page); + assert.equal(requests.length, 2); + } finally { await page.close(); } +}); + +for (const scenario of ['preview', 'signed-preview']) { + test(`${scenario}: untrusted Markdown cannot load remote images or signed tracking URLs`, async () => { + requests = []; + const page = await pageFor(scenario); + try { + if(scenario==='signed-preview') { + await page.locator('[data-maka-contract="markdown"]').waitFor(); + assert.equal(await page.getByText('Open in browser', {exact:true}).count(), 0); + } else { + await page.getByText('Remote images cannot be loaded here. You can open this image in your browser.', {exact:true}).waitFor(); + await page.getByText('Open in browser', {exact:true}).waitFor(); + } + assert.deepEqual(requests, []); + assert.equal(await page.locator('img').count(), 0); + assert.equal(await page.getByRole('button', { name: 'Load image', exact: true }).count(), 0); + assert.ok(!(await page.locator('body').innerText()).includes('preview-secret')); + if (scenario === 'preview') { + // Prove the production CSP blocks a bypass of the Markdown component too. + await page.evaluate(url => { + const img=document.createElement('img'); img.src=url; document.body.append(img); + return new Promise(resolve=> { img.onerror=resolve; img.onload=()=>resolve(); }); + }, `${imageUrl}/blocked.png`); + assert.deepEqual(requests, []); + } + } finally { await page.close(); } + }); +} + +for (const scenario of ['signed-remote', 'signed-inline', 'signed-failed']) { + test(`${scenario}: hidden destinations provide no network actions, including failure and inline states`, async () => { + requests = []; + const page = await pageFor(scenario); + try { + await page.locator('[data-maka-contract="markdown"]').waitFor(); + assert.equal(await page.getByRole('button', {name: 'Load image', exact: true}).count(), 0); + assert.equal(await page.getByRole('button', {name: 'Retry', exact: true}).count(), 0); + assert.equal(await page.getByText('Open in browser', {exact: true}).count(), 0); + assert.equal(await page.locator('a[href*="token="]').count(), 0); + assert.equal(await page.locator('img').count(), 0); + assert.doesNotMatch(await page.locator('body').innerText(), /remote-secret|inline-secret|failed-secret/); + assert.deepEqual(requests, []); + } finally { await page.close(); } + }); +} + +test('application privacy policy prevents automatic image loading and can be rechecked', async () => { + requests = []; + const page = await pageFor('privacy'); + try { + await page.getByText('Image not saved: permissions, network settings, or the source address block loading.', {exact: true}).waitFor(); + assert.equal(await page.getByRole('button', {name: 'Load image', exact: true}).count(), 0); + await page.getByRole('button', {name: 'Retry', exact: true}).click(); + await page.getByText('Image not saved: permissions, network settings, or the source address block loading.', {exact: true}).waitFor(); + await page.getByText('Open in browser', {exact: true}).waitFor(); + assert.deepEqual(requests, []); + } finally { await page.close(); } +}); + +test('failed remote image has a working retry control instead of a broken image icon', async () => { + failedOnce = false; + const page = await pageFor('retry'); + try { + await page.getByText('Image not saved: download failed. Try again.').waitFor(); + assert.equal(await page.locator('img').count(), 0); + await page.getByRole('button', { name: 'Retry', exact: true }).click(); + await loaded(page); + } finally { await page.close(); } +}); + +test('attachment read failure recovers in place, and local paths explain how to provide an image', async () => { + const page = await pageFor('attachment'); + try { + await page.getByRole('button', { name: 'Retry', exact: true }).click(); + await loaded(page); + assert.match(await page.locator('img').getAttribute('src'), /^data:image\/png;base64,/); + await page.goto(`${appUrl}/?case=local`); + await page.getByText('This image address cannot be displayed here.').waitFor(); + assert.equal(await page.locator('img').count(), 0); + assert.equal(await page.getByRole('button', { name: 'Load image' }).count(), 0); + } finally { await page.close(); } +}); + +for (const scenario of ['corrupt-saved']) { + test(`${scenario}: a saved image decode failure retries archived bytes without touching its source`, async () => { + requests = []; + const page = await pageFor(scenario); + try { + await page.getByText('Could not load the image. Try again.').waitFor(); + assert.equal(await page.evaluate(() => window.imageReads), 1); + await page.getByRole('button', { name: 'Retry', exact: true }).click(); + await page.waitForFunction(() => [...document.images].some(img => img.src.startsWith('data:image/png;') && img.naturalWidth === 1)); + assert.equal(await page.evaluate(() => window.deliveryRetries), 0); + assert.equal(await page.evaluate(() => window.deliveryQueries), 1); + assert.equal(await page.evaluate(() => window.imageReads), 2); + assert.deepEqual(requests, []); + } finally { await page.close(); } + }); + +} + +test('a transient saved attachment read retries its bytes without invalidating archival or touching origin', async () => { + requests = []; + const page = await pageFor('read-retry-saved'); + try { + await page.getByText('Could not load the image. Try again.').waitFor(); + await page.getByRole('button', { name: 'Retry', exact: true }).click(); + await loaded(page); + assert.equal(await page.evaluate(() => window.deliveryRetries), 0); + assert.equal(await page.evaluate(() => window.deliveryQueries), 1); + assert.equal(await page.evaluate(() => window.imageReads), 2); + assert.deepEqual(requests, []); + } finally { await page.close(); } +}); + +test('offscreen image performs no image request until it approaches the viewport', async () => { + requests = []; + const page = await pageFor('offscreen'); + try { + // Let React effects and the initial IntersectionObserver callback complete. + await page.getByText('Loading image…').waitFor(); + assert.deepEqual(requests, []); + assert.equal(await page.locator('img').count(), 0); + await page.locator('[data-maka-image-state]').scrollIntoViewIfNeeded(); + await loaded(page); + assert.deepEqual(requests, [{ path: '/image.png', referer: undefined }]); + } finally { await page.close(); } +}); + +test('a pending archive stays a placeholder until saved bytes are available', async () => { + requests = []; + const page = await pageFor('saving'); + try { + await page.getByText('Loading image…', { exact: true }).waitFor(); + assert.deepEqual(requests, []); + assert.equal(await page.locator('img').count(), 0); + await page.waitForFunction(() => [...document.images].some(img=>img.src.startsWith('data:image/png;') && img.naturalWidth===1)); + await page.getByText('Loading image…', { exact: true }).waitFor({ state: 'hidden' }); + assert.equal(await page.getByRole('status').filter({ hasText: /\S/ }).count(), 0); + assert.equal(await page.evaluate(() => window.deliveryQueries), 2); + assert.equal(requests.length, 0); + } finally { await page.close(); } +}); + +for (const scenario of ['streaming-race', 'settled-race', 'streaming-angle-race']) { + test(`${scenario}: an unavailable live source recovers in place once Host archival completes`, async () => { + const page = await pageFor(scenario); + try { + await page.waitForFunction(() => window.deliveryQueries === 1); + if (scenario === 'settled-race') { + await page.evaluate(() => window.finishStream()); + await page.waitForFunction(() => window.deliveryQueries >= 2); + } + await page.evaluate(() => { window.deliveryReady = true; }); + await page.waitForFunction(() => [...document.images].some(image => image.src.startsWith('data:') && image.naturalWidth === 1)); + assert.equal(await page.evaluate(() => window.imageReads), 1); + assert.match(await page.locator('img').getAttribute('src'), /^data:image\/png;base64,/); + assert.equal(await page.locator('img').count(), 1); + } finally { await page.close(); } + }); +} + +for (const scenario of ['angle-saved', 'title-saved', 'escaped-saved', 'reference-saved', 'attachment-title']) { + test(`${scenario}: standard Markdown destinations resolve to saved bytes without origin requests`, async () => { + requests = []; + const page = await pageFor(scenario); + try { + await loaded(page); + assert.match(await page.locator('img').getAttribute('src'), /^data:image\/png;base64,/); + assert.deepEqual(requests, []); + assert.equal(await page.evaluate(() => window.imageReads), 1); + if (scenario !== 'attachment-title') { + assert.equal(await page.evaluate(() => window.deliveryQueries), 1); + } + } finally { await page.close(); } + }); +} diff --git a/packages/ui/src/__tests__/attachment-image.test.tsx b/packages/ui/src/__tests__/attachment-image.test.tsx index df3208c456e..1203e69db4b 100644 --- a/packages/ui/src/__tests__/attachment-image.test.tsx +++ b/packages/ui/src/__tests__/attachment-image.test.tsx @@ -72,9 +72,11 @@ async function renderAttachmentMarkdown(text: string, readBytes: ReadAttachmentB const { container, root } = domRoot(); await act(async () => { root.render( - + + - , + + , ); }); return { container, root }; @@ -169,7 +171,7 @@ test('renders a session attachment referenced by assistant Markdown', async () = assert.deepEqual(readRef, { sessionId: 'session-1', artifactId: 'attachment-123' }); }); -test('keeps unreadable assistant attachments as named placeholders', async () => { +test('explains unreadable assistant attachments and offers retry', async () => { const cases: Array<[string, ReadAttachmentBytes]> = [ ['missing', async () => ({ ok: false, reason: 'not_found' })], ['document', async () => ({ ok: true, base64: 'cGRm', mimeType: 'application/pdf' })], @@ -188,7 +190,9 @@ test('keeps unreadable assistant attachments as named placeholders', async () => readBytes, ); assert.equal(container.querySelector('img'), null); - assert.ok(container.textContent.includes(`[${name}]`)); + assert.ok(container.textContent.includes(name)); + assert.ok(container.textContent.includes("Could not load the image")); + assert.equal(container.querySelector("button")?.textContent, "Retry"); } }); @@ -218,14 +222,10 @@ test('retries an attachment image after a transient read failure', async () => { ? { ok: false, reason: 'read_failed' } : { ok: true, base64: 'cmVjb3ZlcmVk', mimeType: 'image/png' }; }; - const { container, root } = await renderAttachmentMarkdown(markdown, readBytes); + const { container } = await renderAttachmentMarkdown(markdown, readBytes); assert.equal(container.querySelector('img'), null); await act(async () => { - root.render( - - - , - ); + container.querySelector('button')!.click(); }); const image = container.querySelector('img[alt="preview"]'); @@ -244,22 +244,26 @@ test('renders an attachment when a streaming Markdown image becomes complete', a }); await act(async () => { root.render( - + + - , + + , ); }); assert.equal(container.querySelector('img'), null); await act(async () => { root.render( - + + - , + + , ); }); diff --git a/packages/ui/src/__tests__/chat-turn-answer-identity.test.tsx b/packages/ui/src/__tests__/chat-turn-answer-identity.test.tsx index 5c9a4ce7cc4..8473762e130 100644 --- a/packages/ui/src/__tests__/chat-turn-answer-identity.test.tsx +++ b/packages/ui/src/__tests__/chat-turn-answer-identity.test.tsx @@ -28,6 +28,8 @@ import { LocalizedChatMessage, TurnView } from '../chat-turn.js'; import { LocaleProvider } from '../locale-context.js'; import type { TurnTimelineItem, TurnViewModel } from '../materialize.js'; import { applyThinkingDelta } from '../thinking-stream.js'; +import { ImageDeliveryProvider, type ResolveImageDelivery } from '../image-delivery.js'; +import { SessionAttachmentProvider } from '../attachment-image.js'; const originalGlobals = { document: globalThis.document, @@ -208,6 +210,59 @@ test('keeps reasoning expanded when its last neighboring tool is projected away' assert.equal(after.getAttribute('aria-expanded'), 'true'); }); +test('process image links never load bytes while the completed final reply still displays an image', async () => { + const { container, root } = domRoot(); + const sources: string[] = []; + let reads = 0; + const resolve: ResolveImageDelivery = async (_session, request) => { + sources.push(request.source); + return { status: 'ready', artifactId: 'final-image' }; + }; + const readBytes = async () => { + reads++; + return { ok: true as const, mimeType: 'image/png', base64: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==' }; + }; + const thinking: TurnTimelineItem = { kind: 'thinking', messageId: 'reasoning', text: '![Reasoning](https://example.com/reasoning.png)' }; + const commentary: TurnTimelineItem = { kind: 'text', messageId: 'commentary', text: '![Intermediate](https://example.com/commentary.png)', live: true, complete: true }; + const final: TurnTimelineItem = { kind: 'text', messageId: 'final', text: '![Final](https://example.com/final.png)' }; + const render = async (timeline: TurnTimelineItem[], completed = false) => { + await act(async () => { + root.render( + + + + + + ); + }); + }; + + // The live last text step is provisional: the next tool can move it into the process. + await render([thinking, commentary]); + assert.ok(container.querySelector('a[href="https://example.com/commentary.png"]')); + assert.equal(container.querySelectorAll('img, .maka-markdown-image-resource').length, 0); + assert.deepEqual(sources, []); + assert.equal(reads, 0); + + await render([thinking, commentary, RUNNING_TOOL]); + assert.ok(container.querySelector('.maka-processing-body a[href="https://example.com/commentary.png"]')); + assert.deepEqual(sources, []); + assert.equal(reads, 0); + + await render([thinking, commentary, { ...RUNNING_TOOL, items: RUNNING_TOOL.items.map(item => ({ ...item, status: 'completed' })) }, final], true); + assert.deepEqual(sources, ['https://example.com/final.png']); + assert.equal(reads, 1); + assert.equal(container.querySelectorAll('img').length, 1); + await act(() => { container.querySelector('.maka-processing-summary')?.dispatchEvent(new window.Event('click', { bubbles: true })); }); + await act(() => { container.querySelector('.maka-deep-thinking [data-slot="activity-card-header"]')?.dispatchEvent(new window.Event('click', { bubbles: true })); }); + assert.ok(container.querySelector('.maka-processing-body a[href="https://example.com/reasoning.png"]')); + assert.ok(container.querySelector('.maka-processing-body a[href="https://example.com/commentary.png"]')); + assert.equal(container.querySelectorAll('.maka-processing-body img, .maka-processing-body .maka-markdown-image-resource').length, 0); + assert.deepEqual(sources, ['https://example.com/final.png']); + assert.equal(reads, 1); +}); + test('redacts secrets before rendering a settled collapsed reasoning preview', async () => { const { container, root } = domRoot(); await renderTurn(root, turnWith([ diff --git a/packages/ui/src/__tests__/markdown-body.test.ts b/packages/ui/src/__tests__/markdown-body.test.ts index eca7a03013f..f9f0752c15d 100644 --- a/packages/ui/src/__tests__/markdown-body.test.ts +++ b/packages/ui/src/__tests__/markdown-body.test.ts @@ -37,6 +37,21 @@ import { MAX_MERMAID_SOURCE_LENGTH, } from '../mermaid-diagram.js'; +it('image links use canonical destinations for titles, angle brackets and escaped parentheses', () => { + for (const [text, destination] of [ + ['![Title](https://example.com/image.png "Screenshot title")', 'https://example.com/image.png'], + ['![Angle]()', 'https://example.com/image.png'], + [String.raw`![Escaped](https://example.com/a\(1\).png)`, 'https://example.com/a(1).png'], + ]) { + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { text, imageDisplay: 'link' }), + })); + assert.ok(markup.includes(`href="${destination}"`), markup); + assert.doesNotMatch(markup, / { const markup = renderToStaticMarkup(createElement(MarkdownBody, { text: '
Clickpayload
', @@ -436,7 +451,7 @@ it('keeps non-allowlisted external schemes inert', () => { }); it('never loads non-allowlisted Markdown image sources', () => { - const markup = renderToStaticMarkup(createElement(MarkdownBody, { + const markup = renderImageMarkdown({ text: [ '![standalone](file:///Users/example/.ssh/id_rsa)', '', @@ -448,7 +463,7 @@ it('never loads non-allowlisted Markdown image sources', () => { '', '[avatar]: file:///Users/example/private.png', ].join('\n'), - })); + }); assert.doesNotMatch(markup, / { const fence = (index: number) => [ '```mermaid', diff --git a/packages/ui/src/__tests__/markdown-image-source.test.ts b/packages/ui/src/__tests__/markdown-image-source.test.ts new file mode 100644 index 00000000000..f6abd597f5e --- /dev/null +++ b/packages/ui/src/__tests__/markdown-image-source.test.ts @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { createMarkdownImageSourceResolver } from '../markdown-image-source.js'; + +test('normalizes only complete image destinations present in canonical Markdown', () => { + const resolve = createMarkdownImageSourceResolver([ + '![space]()', + '![title](https://example.com/image.png "Screenshot title")', + String.raw`![escaped](/tmp/a\(1\).png)`, + '![attachment](maka://runtime/attachments/image-1 "Preview")', + ].join('\n\n')); + assert.equal(resolve(''), '/tmp/my image.png'); + assert.equal(resolve('https://example.com/image.png "Screenshot title"'), 'https://example.com/image.png'); + assert.equal(resolve(String.raw`/tmp/a\(1\).png`), '/tmp/a(1).png'); + assert.equal(resolve('maka://runtime/attachments/image-1 "Preview"'), 'maka://runtime/attachments/image-1'); + assert.equal(resolve('/tmp/plain.png'), '/tmp/plain.png'); +}); + +test('preserves literal title-like filenames resolved from image references', () => { + const source = '/tmp/my "image".png'; + const resolve = createMarkdownImageSourceResolver(`![reference][picture]\n\n[picture]: <${source}>`); + assert.equal(resolve(source), source); +}); + +test('code, raw HTML, incomplete syntax and unrelated destinations do not become image grants', () => { + for (const text of [ + '`![code]()`', + '```md\n![code]()\n```', + '', + '![incomplete](', + '![different]()', + ]) { + assert.equal(createMarkdownImageSourceResolver(text)(''), ''); + } +}); diff --git a/packages/ui/src/attachment-image.tsx b/packages/ui/src/attachment-image.tsx index 19b88e9a64c..dc52a3b3311 100644 --- a/packages/ui/src/attachment-image.tsx +++ b/packages/ui/src/attachment-image.tsx @@ -19,6 +19,7 @@ import { createContext, + useCallback, useContext, useEffect, useMemo, @@ -88,24 +89,47 @@ export function useAttachmentImageSource(ref: { artifactId: string; sessionId?: string; } | undefined): string | undefined { + return useAttachmentImage(ref).src; +} + +export function useAttachmentImage(ref: { + artifactId: string; + sessionId?: string; +} | undefined) { const context = useContext(SessionAttachmentContext); const artifactId = ref?.artifactId; const sessionId = ref?.sessionId ?? context?.sessionId; const loadImage = context?.loadImage; - const [src, setSrc] = useState(undefined); + const [attempt, setAttempt] = useState(0); + const retry = useCallback(() => { + setAttempt((value) => value + 1); + }, []); + const request = useMemo( + () => artifactId && sessionId && loadImage + ? { artifactId, sessionId, loadImage, attempt } + : undefined, + [artifactId, sessionId, loadImage, attempt], + ); + const [result, setResult] = useState<{ + request: typeof request; + status: 'loading' | 'ready' | 'failed'; + src?: string; + }>(); useEffect(() => { - setSrc(undefined); - if (!artifactId || !sessionId || !loadImage) return; + if (!request) return; + setResult({ request, status: 'loading' }); let cancelled = false; - loadImage(sessionId, artifactId) + request.loadImage(request.sessionId, request.artifactId) .then((loaded) => { - if (!cancelled) setSrc(loaded); - }) + if (!cancelled) setResult({ request, status: loaded ? 'ready' : 'failed', src: loaded }); + }); return () => { cancelled = true; }; - }, [artifactId, loadImage, sessionId]); + }, [request]); - return src; + if (!request) return { status: 'unavailable' as const, src: undefined, retry }; + if (result?.request !== request) return { status: 'loading' as const, src: undefined, retry }; + return { status: result.status, src: result.src, retry }; } diff --git a/packages/ui/src/chat-image-resource.ts b/packages/ui/src/chat-image-resource.ts new file mode 100644 index 00000000000..50eb0985632 --- /dev/null +++ b/packages/ui/src/chat-image-resource.ts @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { useState } from 'react'; +import { parseAttachmentResourceRef } from '@maka/core/attachments'; +import { isRemoteImageSource, type ImageDeliveryFailure } from '@maka/core/image-delivery'; +import { useAttachmentImage } from './attachment-image.js'; +import { useImageDelivery } from './image-delivery.js'; + +export type ChatImagePresentation = + | { readonly kind: 'loading' } + | { readonly kind: 'ready'; readonly src: string } + | { readonly kind: 'unavailable'; readonly reason: 'attachment' | 'remote' | 'redacted' | 'unsupported' } + | { readonly kind: 'failed'; readonly reason: 'saved_bytes' | ImageDeliveryFailure }; + +/** Owns delivery, saved-byte and decode transitions. The component only renders this view. */ +export function useChatImageResource(input: { source: string; redacted: boolean; visible: boolean }) { + const explicit = parseAttachmentResourceRef(input.source); + const delivery = useImageDelivery(input.source, input.visible && !explicit, !input.redacted); + const artifactId = explicit?.artifactId ?? (delivery.status === 'ready' ? delivery.artifactId : undefined); + const image = useAttachmentImage(input.visible && artifactId ? { artifactId } : undefined); + const [failedSource, setFailedSource] = useState(); + const [attempt, setAttempt] = useState(0); + const source = artifactId ? image.src : undefined; + const remote = isRemoteImageSource(input.source); + const hiddenRemote = remote && input.redacted; + const remotePlaceholder = input.visible && remote && !artifactId; + let presentation: ChatImagePresentation; + if ((source && failedSource === source) || (artifactId && image.status === 'failed')) + presentation = { kind: 'failed', reason: 'saved_bytes' }; + else if (source) presentation = { kind: 'ready', src: source }; + else if (!input.visible) presentation = { kind: 'loading' }; + else if (explicit && image.status === 'unavailable') presentation = { kind: 'unavailable', reason: 'attachment' }; + else if (remotePlaceholder && hiddenRemote) presentation = { kind: 'unavailable', reason: 'redacted' }; + else if (remotePlaceholder && (!delivery.available || delivery.status === 'requires_confirmation')) + presentation = { kind: 'unavailable', reason: 'remote' }; + else if (delivery.status === 'failed') presentation = { kind: 'failed', reason: delivery.reason }; + else if (delivery.status === 'pending' || artifactId) presentation = { kind: 'loading' }; + else presentation = { kind: 'unavailable', reason: 'unsupported' }; + + const retry = presentation.kind === 'failed' && (!hiddenRemote || artifactId) ? () => { + setFailedSource(undefined); + setAttempt(value => value + 1); + // Once saved, only reread archived bytes; never recapture an expired/deleted source. + if (artifactId) image.retry(); + else delivery.retry(); + } : undefined; + const openSource = remote && !hiddenRemote && (presentation.kind === 'failed' || + presentation.kind === 'unavailable' && presentation.reason === 'remote') ? input.source : undefined; + return { + presentation, attempt, retry, openSource, + onDecodeError: () => { if (source) setFailedSource(source); }, + }; +} diff --git a/packages/ui/src/chat-turn.tsx b/packages/ui/src/chat-turn.tsx index cbbf1cf738b..537326da1f5 100644 --- a/packages/ui/src/chat-turn.tsx +++ b/packages/ui/src/chat-turn.tsx @@ -24,7 +24,8 @@ import { TranscriptDisclosure } from './transcript-disclosure.js'; import { Markdown } from './markdown.js'; import { formatTurnDuration } from './chat-display-helpers.js'; import { formatAbsoluteTimestamp } from '@maka/core/relative-time'; -import { isTimeDrivenMotionEnabled } from './streaming-presentation.js'; +import { isTimeDrivenMotionEnabled, isProgressiveStreamingEnabled } from './streaming-presentation.js'; +import { ImageMessageProvider } from './image-delivery.js'; import { computerRunningLabel } from './tool-activity/computer-action-label.js'; import { Badge, @@ -768,6 +769,7 @@ export const TurnView = memo(function TurnView(props: { key={`processing-${item.id}`} activityObserved={props.activityObserved} entries={item.children} + turnId={turn.turnId} running={!!props.liveStreaming} statusRow={ index === activityProcessIndex && hasLiveStatus @@ -782,7 +784,10 @@ export const TurnView = memo(function TurnView(props: { void; - }; + }); /** * The assistant's answer, in every state it can be in. @@ -1359,22 +1364,25 @@ const AssistantAnswerBubble = memo(function AssistantAnswerBubble(props: Assista : 'maka-chat-message-bubble maka-chat-message-bubble-assistant maka-bubble-streaming' } > - + + + {truncated && ( {/* Colour-name archive, not the semantic one: Astryx paints @@ -1414,8 +1422,10 @@ function timelineEntryKey(item: TurnTimelineItem, index: number): string { /** Render one timeline entry: reasoning disclosure / answer bubble / tool group. */ const TurnTimelineEntry = memo(function TurnTimelineEntry(props: { + turnId?: string; activityObserved?: boolean; item: Exclude; + imageDisplay?: 'image' | 'link'; onStreamingSettled?: (messageId?: string) => void; onOpenLinkedSession?(sessionId: string): void; initialLiveContent?: ReadonlyMap; @@ -1441,9 +1451,11 @@ const TurnTimelineEntry = memo(function TurnTimelineEntry(props: { ); } // Same component either way — a type swap here would remount the answer. - if (item.live !== true) return ; + if (item.live !== true) return ; return ( + + ); } diff --git a/packages/ui/src/image-delivery.tsx b/packages/ui/src/image-delivery.tsx new file mode 100644 index 00000000000..388677b6587 --- /dev/null +++ b/packages/ui/src/image-delivery.tsx @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createContext, useCallback, useContext, useEffect, useMemo, useState, type ReactNode } from 'react'; +import type { ImageDeliveryRequest, ImageDeliveryResult, ResolveImageDelivery } from '@maka/core/image-delivery'; +import { isRemoteImageSource } from '@maka/core/image-delivery'; +export type { ResolveImageDelivery } from '@maka/core/image-delivery'; +const ImageMessageScope = createContext<{ turnId: string; messageId: string; streaming?: boolean } | undefined>(undefined); +/** Installed by assistant messages, outside the generic Markdown renderer. */ +export function ImageMessageProvider(props: { + identity?: { turnId: string; messageId: string }; + streaming?: boolean; + children: ReactNode; +}) { + const value = useMemo(() => props.identity + ? { turnId: props.identity.turnId, messageId: props.identity.messageId, streaming: props.streaming } + : undefined, [props.identity?.turnId, props.identity?.messageId, props.streaming]); + return {props.children}; +} +const DeliveryContext = createContext<{ resolve(request: ImageDeliveryRequest): Promise } | undefined>(undefined); +export function ImageDeliveryProvider(props: { sessionId: string; resolve?: ResolveImageDelivery; children: ReactNode }) { + const value = useMemo(() => { + if (!props.resolve) return undefined; + const resolve = props.resolve; + const pending = new Map>(); + const ready = new Map(); + return { resolve(request: ImageDeliveryRequest) { + const key = JSON.stringify([request.turnId, request.messageId, request.source]); + const cached = ready.get(key); if (cached) return Promise.resolve(cached); + const running = pending.get(key); if (running) return running; + const job = resolve(props.sessionId, request).catch((): ImageDeliveryResult => ({ status: 'failed', reason: 'read_failed' })); + pending.set(key, job); + void job.then(result => { + pending.delete(key); + if (result.status === 'ready') { + ready.set(key, result); if (ready.size > 128) ready.delete(ready.keys().next().value!); + } + }); + return job; + }}; + }, [props.resolve, props.sessionId]); + return {props.children}; +} +export function useImageDelivery(source: string, enabled: boolean, allowRemote: boolean) { + const scope = useContext(ImageMessageScope); + const context = useContext(DeliveryContext); + const [attempt, setAttempt] = useState(0); + // Saved images can resolve even when display redaction forbids a source fetch. + // Redaction protects display/retry UX for recognized secrets, not outbound data: + // an encoded value in a model-authored URL may still pass this presentation check. + const loadRemote = allowRemote && isRemoteImageSource(source); + const identity = useMemo(() => context && scope?.turnId && scope.messageId && enabled + ? { context, turnId: scope.turnId, messageId: scope.messageId, streaming: scope.streaming === true, source, attempt, loadRemote } : undefined, + [context, scope?.turnId, scope?.messageId, scope?.streaming, source, attempt, loadRemote, enabled]); + const [settled, setSettled] = useState<{ identity: typeof identity; result: ImageDeliveryResult }>(); + const retry = useCallback(() => setAttempt(a => a + 1), []); + useEffect(() => { + if (!identity) return; + let cancelled = false; let timer: ReturnType | undefined; let delay = 500; let unavailableRetries = 0; + const query = async (retry: boolean) => { + const result = await identity.context.resolve({ turnId: identity.turnId, messageId: identity.messageId, source: identity.source, ...(retry ? { retry: true } : {}), ...(identity.loadRemote ? { loadRemote: true } : {}) }); + if (cancelled) return; + setSettled({ identity, result }); + // Canonical text can lag the displayed stream. Retry briefly after settlement, + // but do not poll permanently unsupported sources forever. + if (result.status === 'pending' || result.status === 'unavailable' && (identity.streaming || unavailableRetries++ < 3)) { + timer = setTimeout(() => { void query(false); }, delay); delay = Math.min(5000, delay * 2); + } + }; + void query(attempt > 0); + return () => { cancelled = true; clearTimeout(timer); }; + }, [identity, attempt]); + const result: ImageDeliveryResult = !identity ? { status: 'unavailable' } + : settled?.identity === identity ? settled.result : { status: 'pending' }; + return { ...result, retry, available: !!identity }; +} diff --git a/packages/ui/src/markdown-body.tsx b/packages/ui/src/markdown-body.tsx index af17e36ee64..df683852d57 100644 --- a/packages/ui/src/markdown-body.tsx +++ b/packages/ui/src/markdown-body.tsx @@ -29,7 +29,8 @@ * product-specific trust boundaries around that renderer. */ -import { useCallback, useContext, useRef, type ReactNode } from 'react'; +import { MarkdownImage, createMarkdownImagePlugins } from './markdown-image.js'; +import { useCallback, useContext, useMemo, useRef, type ReactNode } from 'react'; import { Markdown as AstryxMarkdown, type MarkdownComponents, @@ -51,8 +52,6 @@ import { MarkdownMath, prepareMarkdownMath, } from './markdown-math.js'; -import { parseAttachmentResourceRef } from '@maka/core/attachments'; -import { useAttachmentImageSource } from './attachment-image.js'; const BASE_MARKDOWN_COMPONENTS = { link: MarkdownLink, @@ -144,6 +143,7 @@ const MARKDOWN_COMPONENTS = { export function MarkdownBody(props: { text: string; + imageDisplay?: 'image' | 'link'; streaming?: boolean; settledText?: string; density?: 'default' | 'compact'; @@ -153,6 +153,7 @@ export function MarkdownBody(props: { (source: string) => prepareMarkdownMath(source, mathCache.current), [], ); + const plugins = useMemo(() => createMarkdownImagePlugins(props.imageDisplay), [props.imageDisplay]); const budgetedText = props.streaming ? props.text : applyMermaidRenderBudget(props.text); const density = props.density ?? 'default'; const components = props.streaming @@ -195,6 +196,7 @@ export function MarkdownBody(props: { // the one combination neither half of the argument asks for. density={density} components={components} + plugins={plugins} isStreaming={props.streaming} settledText={props.settledText} transformSource={transformMathSource} @@ -271,37 +273,6 @@ function MarkdownCode(props: { ); } -function MarkdownImage(props: { src: string; alt: string }) { - const attachment = parseAttachmentResourceRef(props.src); - const attachmentSrc = useAttachmentImageSource( - attachment ? { artifactId: attachment.artifactId } : undefined, - ); - if (attachment) { - if (!attachmentSrc) return [{props.alt}]; - return ( - {props.alt} - ); - } - if (!isSafeMarkdownImageUrl(props.src)) return [{props.alt}]; - // Remote images can be badges or sentence-level icons, so preserve Maka's - // existing inline presentation. Session attachments above are content - // previews and deliberately own a block presentation instead. - return {props.alt}; -} - -function isSafeMarkdownImageUrl(url: string): boolean { - try { - const protocol = new URL(url).protocol; - return protocol === 'http:' || protocol === 'https:'; - } catch { - return false; - } -} - /** * Route internal Markdown navigation through Maka's typed allowlist. Invalid * internal destinations never fall through to the operating system, and diff --git a/packages/ui/src/markdown-image-source.ts b/packages/ui/src/markdown-image-source.ts new file mode 100644 index 00000000000..12cfc8627c6 --- /dev/null +++ b/packages/ui/src/markdown-image-source.ts @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + IMAGE_DELIVERY_SOURCE_MAX_LENGTH, + IMAGE_MARKDOWN_MAX_LENGTH, +} from '@maka/core/image-delivery'; +import { markdownImageSources, parseMarkdownImageDestination } from '@maka/core/image-markdown'; + +/** Astryx can pass destination syntax through its image slot. Resolve it with + * the Host's Markdown parser, without changing the text rendered or stored. + * Parse the document lazily, once, only for destinations needing normalization. + */ +export function createMarkdownImageSourceResolver(text: string): (source: string) => string { + let canonicalSources: Set | undefined; + return (source) => { + if ( + !/[<>"'\\\s&]/.test(source) || + source.length > IMAGE_DELIVERY_SOURCE_MAX_LENGTH || + text.length > IMAGE_MARKDOWN_MAX_LENGTH + ) + return source; + canonicalSources ??= new Set(markdownImageSources(text)); + // Reference destinations already arrive without brackets/title syntax. + // Preserve a canonical path containing literal quotes or whitespace. + if (canonicalSources.has(source)) return source; + const destination = parseMarkdownImageDestination(source); + return destination !== undefined && canonicalSources.has(destination) ? destination : source; + }; +} diff --git a/packages/ui/src/markdown-image.tsx b/packages/ui/src/markdown-image.tsx new file mode 100644 index 00000000000..5b66689ad34 --- /dev/null +++ b/packages/ui/src/markdown-image.tsx @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { useEffect, useRef, useState } from 'react'; +import { Button, IconButton, Spinner, Tooltip } from '@astryxdesign/core'; +import { createMarkdownPlugin, type MarkdownExtensionNode } from '@astryxdesign/core/Markdown/plugins'; +import type { MarkdownAstNode, MarkdownAstRoot } from '@astryxdesign/core/Markdown'; +import { RotateCw, AlertTriangle } from './icons.js'; +import { Link } from '@astryxdesign/core/Link'; +import { useChatImageResource } from './chat-image-resource.js'; +import { getSharedUiCopy } from './shared-ui-copy.js'; +import { useUiLocale } from './locale-context.js'; +import { createMarkdownImageSourceResolver } from './markdown-image-source.js'; +import { redactSecrets } from './redact.js'; + +type PlacedImage = MarkdownExtensionNode<'maka-images', 'image', { src: string; alt: string; inline: boolean; redacted: boolean }>; +// Markdown structure determines inline/block placement; decoded images supply +// their natural dimensions within the presentation's CSS bounds. +export function createMarkdownImagePlugins( + display: 'image' | 'link' = 'image', +) { + return [createMarkdownPlugin<'maka-images', PlacedImage>({ + name: 'maka-images', apiVersion: 1, + transform(document, context) { + if (!context.source.includes('![')) return document; + const canonical = createMarkdownImageSourceResolver(context.source); + const rewrite = (node: MarkdownAstNode, inline = false, phrasing = false, insideLink = false): MarkdownAstNode => { + if (node.type === 'image') { + const source = canonical(node.url); + const redacted = source.includes('') || source !== redactSecrets(source); + if (display === 'link') { + const text = { type: 'text' as const, value: node.alt || redactSecrets(source) }; + const link = insideLink || redacted ? text + : { type: 'link' as const, url: source, children: [text] }; + return phrasing ? link : { type: 'paragraph', children: [link] }; + } + return { + type: 'extension', plugin: 'maka-images', name: 'image', + display: phrasing ? 'inline' : 'block', + data: { src: source, alt: node.alt, inline, redacted }, + } satisfies PlacedImage; + } + if (!('children' in node)) return node; + const childInline = node.type === 'paragraph' ? !isSingleImage(node.children) + : node.type === 'heading' || node.type === 'tableCell' ? true : inline; + const childPhrasing = ['paragraph', 'heading', 'tableCell', 'link', 'strong', 'emphasis', 'delete'].includes(node.type); + return { ...node, children: node.children.map(child => rewrite(child, childInline, childPhrasing, insideLink || node.type === 'link')) } as MarkdownAstNode; + }; + return rewrite(document) as MarkdownAstRoot; + }, + renderers: { image: { + render: ({ node }) => , + toText: node => node.data.alt, + } }, + })]; +} + +function isSingleImage(nodes: readonly MarkdownAstNode[]): boolean { + const meaningful = nodes.filter(node => node.type !== 'text' || node.value.trim()); + if (meaningful.length !== 1) return false; + const node = meaningful[0]!; + return node.type === 'image' || (['link', 'strong', 'emphasis', 'delete'].includes(node.type) + && 'children' in node && isSingleImage(node.children)); +} +/** Presentation only: Host resolves local addresses and archives; UI receives artifact identities. */ +export function MarkdownImage(props: { src: string; alt: string; inline?: boolean; redacted?: boolean }) { + return ; +} +function ImageResource(props: { src: string; redacted: boolean; alt: string; inline?: boolean }) { + const copy = getSharedUiCopy(useUiLocale()).markdown; + const anchor = useRef(null); + const [visible, setVisible] = useState(typeof IntersectionObserver === 'undefined'); + useEffect(() => { + if (visible || !anchor.current) return; + const observer = new IntersectionObserver(entries => { + if (entries.some(entry => entry.isIntersecting)) { setVisible(true); observer.disconnect(); } + }, { rootMargin: '300px' }); + observer.observe(anchor.current); return () => observer.disconnect(); + }, [visible]); + const resource = useChatImageResource({ source: props.src, redacted: props.redacted, visible }); + const state = resource.presentation; + const message = state.kind === 'loading' ? copy.imageLoading + : state.kind === 'failed' ? state.reason === 'saved_bytes' ? copy.imageLoadFailed : copy.imageArchiveFailure(state.reason) + : state.kind === 'unavailable' ? { + attachment: copy.imageUnavailable, + remote: copy.imageRemoteUnavailable, + redacted: copy.imageRemoteRedacted, + unsupported: copy.imageUnsupported, + }[state.reason] : undefined; + const actions = + {resource.retry &&