diff --git a/electron/api/routes/files.ts b/electron/api/routes/files.ts index 7931213..eaf2852 100644 --- a/electron/api/routes/files.ts +++ b/electron/api/routes/files.ts @@ -75,6 +75,38 @@ async function generateImagePreview(filePath: string, mimeType: string): Promise } } +/** + * Resolve a Gateway-emitted outgoing-media URL to the original file on disk. + * Mirror of `electron/main/ipc-handlers.ts::resolveOutgoingMediaUrl` — kept + * in sync so the host-api HTTP path serves the same data as the IPC path. + */ +async function resolveOutgoingMediaUrl( + gatewayUrl: string, +): Promise<{ path: string; mimeType: string } | null> { + try { + const m = gatewayUrl.match(/\/api\/chat\/media\/outgoing\/[^/]+\/([^/]+)\//); + if (!m) return null; + const attachmentId = decodeURIComponent(m[1]); + if (!/^[A-Za-z0-9._-]+$/.test(attachmentId)) return null; + const recordPath = join(homedir(), '.openclaw', 'media', 'outgoing', 'records', `${attachmentId}.json`); + const fsP = await import('node:fs/promises'); + const raw = await fsP.readFile(recordPath, 'utf8'); + const record = JSON.parse(raw) as { + original?: { path?: string; contentType?: string }; + }; + const original = record?.original; + if (!original?.path) return null; + return { + path: original.path, + mimeType: typeof original.contentType === 'string' && original.contentType + ? original.contentType + : 'application/octet-stream', + }; + } catch { + return null; + } +} + export async function handleFileRoutes( req: IncomingMessage, res: ServerResponse, @@ -137,18 +169,39 @@ export async function handleFileRoutes( if (url.pathname === '/api/files/thumbnails' && req.method === 'POST') { try { - const body = await parseJsonBody<{ paths: Array<{ filePath: string; mimeType: string }> }>(req); + const body = await parseJsonBody<{ + paths: Array<{ filePath?: string; gatewayUrl?: string; mimeType: string }>; + }>(req); const fsP = await import('node:fs/promises'); const results: Record = {}; - for (const { filePath, mimeType } of body.paths) { - try { - const s = await fsP.stat(filePath); - const preview = mimeType.startsWith('image/') - ? await generateImagePreview(filePath, mimeType) - : null; - results[filePath] = { preview, fileSize: s.size }; - } catch { - results[filePath] = { preview: null, fileSize: 0 }; + for (const entry of body.paths) { + if (entry.filePath) { + try { + const s = await fsP.stat(entry.filePath); + const preview = entry.mimeType.startsWith('image/') + ? await generateImagePreview(entry.filePath, entry.mimeType) + : null; + results[entry.filePath] = { preview, fileSize: s.size }; + } catch { + results[entry.filePath] = { preview: null, fileSize: 0 }; + } + continue; + } + if (entry.gatewayUrl) { + const resolved = await resolveOutgoingMediaUrl(entry.gatewayUrl); + if (!resolved) { + results[entry.gatewayUrl] = { preview: null, fileSize: 0 }; + continue; + } + try { + const s = await fsP.stat(resolved.path); + const preview = resolved.mimeType.startsWith('image/') + ? await generateImagePreview(resolved.path, resolved.mimeType) + : null; + results[entry.gatewayUrl] = { preview, fileSize: s.size }; + } catch { + results[entry.gatewayUrl] = { preview: null, fileSize: 0 }; + } } } sendJson(res, 200, results); diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 030f80e..4edb4eb 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -2496,25 +2496,93 @@ function registerFileHandlers(): void { } }); - ipcMain.handle('media:getThumbnails', async (_, paths: Array<{ filePath: string; mimeType: string }>) => { + ipcMain.handle('media:getThumbnails', async ( + _, + paths: Array<{ filePath?: string; gatewayUrl?: string; mimeType: string }>, + ) => { const fsP = await import('fs/promises'); const results: Record = {}; - for (const { filePath, mimeType } of paths) { - try { - const s = await fsP.stat(filePath); - let preview: string | null = null; - if (mimeType.startsWith('image/')) { - preview = await generateImagePreview(filePath, mimeType); + for (const entry of paths) { + // Local on-disk file (the original code path). + if (entry.filePath) { + try { + const s = await fsP.stat(entry.filePath); + let preview: string | null = null; + if (entry.mimeType.startsWith('image/')) { + preview = await generateImagePreview(entry.filePath, entry.mimeType); + } + results[entry.filePath] = { preview, fileSize: s.size }; + } catch { + results[entry.filePath] = { preview: null, fileSize: 0 }; + } + continue; + } + // Gateway-injected outgoing media URL. The renderer cannot reach the + // Gateway HTTP server directly (CORS / env drift), so we resolve it + // here against OpenClaw's local outgoing media records and load the + // original file off disk. The URL shape is fixed by OpenClaw: + // /api/chat/media/outgoing///full + if (entry.gatewayUrl) { + const resolved = await resolveOutgoingMediaUrl(entry.gatewayUrl); + if (!resolved) { + results[entry.gatewayUrl] = { preview: null, fileSize: 0 }; + continue; + } + try { + const s = await fsP.stat(resolved.path); + let preview: string | null = null; + if (resolved.mimeType.startsWith('image/')) { + preview = await generateImagePreview(resolved.path, resolved.mimeType); + } + results[entry.gatewayUrl] = { preview, fileSize: s.size }; + } catch { + results[entry.gatewayUrl] = { preview: null, fileSize: 0 }; } - results[filePath] = { preview, fileSize: s.size }; - } catch { - results[filePath] = { preview: null, fileSize: 0 }; } } return results; }); } +/** + * Resolve a Gateway-emitted outgoing-media URL to the original file on disk. + * + * OpenClaw's runtime stages every assistant `MEDIA:/path` artifact under + * `~/.openclaw/media/outgoing/`: + * - `originals/.` — the source bytes copied verbatim + * - `records/.json` — `{ original: { path, contentType, ... }, ... }` + * + * The Gateway then injects an `assistant-media` content block with + * `url:'/api/chat/media/outgoing///full'`. + * We only need the `` segment to look up the record. + */ +async function resolveOutgoingMediaUrl( + gatewayUrl: string, +): Promise<{ path: string; mimeType: string } | null> { + try { + const m = gatewayUrl.match(/\/api\/chat\/media\/outgoing\/[^/]+\/([^/]+)\//); + if (!m) return null; + const attachmentId = decodeURIComponent(m[1]); + if (!/^[A-Za-z0-9._-]+$/.test(attachmentId)) return null; + const recordPath = join(homedir(), '.openclaw', 'media', 'outgoing', 'records', `${attachmentId}.json`); + const fsP = await import('fs/promises'); + const raw = await fsP.readFile(recordPath, 'utf8'); + const record = JSON.parse(raw) as { + original?: { path?: string; contentType?: string }; + }; + const original = record?.original; + if (!original?.path) return null; + return { + path: original.path, + mimeType: typeof original.contentType === 'string' && original.contentType + ? original.contentType + : 'application/octet-stream', + }; + } catch { + return null; + } +} + /** * Session IPC handlers * diff --git a/src/pages/Chat/ChatMessage.tsx b/src/pages/Chat/ChatMessage.tsx index 1e6a083..61445db 100644 --- a/src/pages/Chat/ChatMessage.tsx +++ b/src/pages/Chat/ChatMessage.tsx @@ -275,9 +275,12 @@ export const ChatMessage = memo(function ChatMessage({ }); const filteredProcessAttachments = derivedAttachedFiles.filter((file) => { if (file.source !== 'tool-result' && file.source !== 'message-ref') return true; - // Runtime-produced PDF / spreadsheet artifacts should remain visible - // in the chat even when generic process attachments are folded into - // the execution graph; they are the user-facing output to click. + // Runtime-produced user-facing artifacts (images, PDFs, spreadsheets, + // skill directories, ...) must remain visible in the reply bubble even + // when generic process attachments are folded into the execution graph. + // The graph card itself does not render `_attachedFiles`, so dropping + // images here would leave the user with no way to see them at all. + if (file.mimeType.startsWith('image/')) return true; return isChatPreviewDocument(file) || isDirectoryAttachment(file) || isSkillFileAttachment(file); }); // When a message is attachment-only, keep those attachments visible even if diff --git a/src/pages/Chat/message-utils.ts b/src/pages/Chat/message-utils.ts index e5e955b..36d7245 100644 --- a/src/pages/Chat/message-utils.ts +++ b/src/pages/Chat/message-utils.ts @@ -37,7 +37,12 @@ function cleanUserText(text: string): string { function stripAssistantMediaTags(text: string): string { if (!text) return text; const exts = 'png|jpe?g|gif|webp|bmp|avif|svg|pdf|docx?|xlsx?|pptx?|txt|csv|md|rtf|epub|zip|tar|gz|rar|7z|mp3|wav|ogg|aac|flac|m4a|mp4|mov|avi|mkv|webm|m4v'; - const tagged = new RegExp(`(^|[\\s(\\[{>])(?:MEDIA|media):(?:\\/|~\\/)[^\\s\\n"'()\\[\\],<>]*?\\.(?:${exts})`, 'g'); + // Mirror the relaxed character class in `chat/helpers.ts::extractRawFilePaths` + // so paths with ASCII spaces (e.g. macOS' "截屏 2026-05-06 17.46.51.png") + // are also stripped from the visible bubble. Without this, the bubble + // would still leak the literal `MEDIA:/.../截屏 2026-05-06 17.46.51.png` + // to the user when the underlying path detection succeeds. + const tagged = new RegExp(`(^|[\\s(\\[{>])(?:MEDIA|media):(?:\\/|~\\/)[^\\n"'()\\[\\],<>]*?\\.(?:${exts})(?=$|[\\s\\n"'()\\[\\],<>]|[，。；;,.!?])`, 'g'); return text .replace(tagged, (_, lead: string) => lead) // Collapse the empty lines / orphan whitespace the strip leaves behind. @@ -302,31 +307,50 @@ export function extractMediaRefs(message: RawMessage | unknown): Array<{ filePat } /** - * Extract image attachments from a message. - * Returns array of { mimeType, data } for base64 images. + * Extract inline image attachments from a message. + * + * Returns either: + * - `{ mimeType, data }` — base64 bytes inline (Anthropic / Gateway tool-result) + * - `{ mimeType, url }` — Anthropic source-wrapped remote URL form + * + * Note: the Gateway-injected `assistant-media` shape — flat + * `{ type:'image', url:'/api/chat/media/outgoing/...', mimeType, ... }` — + * is intentionally NOT extracted here because the URL is a Gateway-relative + * path the renderer cannot fetch directly (CORS / env drift). That shape + * is surfaced through `_attachedFiles` instead (see + * `extractImagesAsAttachedFiles` and `enrichWithGatewayMediaBlocks` in + * `src/stores/chat/helpers.ts`), and resolved to a local preview by + * `loadMissingPreviews` -> Main `media:getThumbnails`. */ -export function extractImages(message: RawMessage | unknown): Array<{ mimeType: string; data: string }> { +export function extractImages( + message: RawMessage | unknown, +): Array<{ mimeType: string; data?: string; url?: string }> { if (!message || typeof message !== 'object') return []; const msg = message as Record; const content = msg.content; if (!Array.isArray(content)) return []; - const images: Array<{ mimeType: string; data: string }> = []; + const images: Array<{ mimeType: string; data?: string; url?: string }> = []; for (const block of content as ContentBlock[]) { - if (block.type === 'image') { - // Path 1: Anthropic source-wrapped format - if (block.source) { - const src = block.source; - if (src.type === 'base64' && src.media_type && src.data) { - images.push({ mimeType: src.media_type, data: src.data }); - } - } - // Path 2: Flat format from Gateway tool results {data, mimeType} - else if (block.data) { - images.push({ mimeType: block.mimeType || 'image/jpeg', data: block.data }); + if (block.type !== 'image') continue; + + // Path 1: Anthropic source-wrapped format + if (block.source) { + const src = block.source; + if (src.type === 'base64' && src.media_type && src.data) { + images.push({ mimeType: src.media_type, data: src.data }); + } else if (src.type === 'url' && src.url) { + images.push({ mimeType: src.media_type || 'image/jpeg', url: src.url }); } + continue; } + + // Path 2: Flat format from Gateway tool results {data, mimeType} + if (block.data) { + images.push({ mimeType: block.mimeType || 'image/jpeg', data: block.data }); + } + // Flat `block.url` is intentionally NOT handled here; see comment above. } return images; diff --git a/src/stores/chat.ts b/src/stores/chat.ts index 79f2c18..8de5b01 100644 --- a/src/stores/chat.ts +++ b/src/stores/chat.ts @@ -434,11 +434,43 @@ function trimPathTerminators(filePath: string): string { * Extract raw file paths from message text. * Detects absolute paths (Unix: / or ~/, Windows: C:\ etc.) ending with common file extensions. * Handles both image and non-image files, consistent with channel push message behavior. + * + * Also recognises the `MEDIA:` / `media:` prefix the OpenClaw runtime + * emits for produced artifacts (e.g. + * `MEDIA:/tmp/desktop_screenshot.png`) — without this the leading colon + * trips the URL guard on the unix regex below and the artifact never + * surfaces as an attachment. Mirrors `chat/helpers.ts::extractRawFilePaths`. */ function extractRawFilePaths(text: string): Array<{ filePath: string; mimeType: string }> { const refs: Array<{ filePath: string; mimeType: string }> = []; const seen = new Set(); const exts = 'png|jpe?g|gif|webp|bmp|avif|svg|pdf|docx?|xlsx?|pptx?|txt|csv|md|rtf|epub|zip|tar|gz|rar|7z|mp3|wav|ogg|aac|flac|m4a|mp4|mov|avi|mkv|webm|m4v'; + // Tagged media references (MEDIA:/path, media:~/path, ...). The agent + // runtime uses this prefix as an explicit "this is an artifact" marker, + // so we want them recognised even though the leading colon would + // normally look like a URL scheme. After matching we punch the entire + // `MEDIA:` span out of the working text so the generic unix + // regex below doesn't double-count the bare `/path` suffix. + // The character class deliberately allows ASCII spaces inside the path so + // that macOS' default screenshot filename ("截屏 2026-05-06 17.46.51.png") + // and other space-containing paths the agent emits with the explicit + // `MEDIA:` marker still resolve. Newline and quote characters remain + // path terminators so we don't accidentally swallow trailing prose. + const taggedRegex = new RegExp(`(?:^|[\\s(\\[{>])(?:MEDIA|media):((?:\\/|~\\/)[^\\n"'()\\[\\],<>` + '`' + `]*?\\.(?:${exts}))(?=$|[\\s\\n"'()\\[\\],<>` + '`' + `]|[，。；;,.!?])`, 'g'); + let workingText = text; + let taggedMatch: RegExpExecArray | null; + while ((taggedMatch = taggedRegex.exec(text)) !== null) { + const p = taggedMatch[1]; + if (p && !seen.has(p)) { + seen.add(p); + refs.push({ filePath: p, mimeType: mimeFromExtension(p) }); + } + // Mask the matched span so subsequent regexes can't re-discover the + // same path (e.g. `/two.xlsx` from `MEDIA:~/two.xlsx`). + const start = taggedMatch.index; + const end = start + taggedMatch[0].length; + workingText = workingText.slice(0, start) + ' '.repeat(end - start) + workingText.slice(end); + } // Unix absolute paths (/... or ~/...) — lookbehind rejects mid-token slashes // (e.g. "path/to/file.mp4", "https://example.com/file.mp4") const unixRegex = new RegExp(`(?]*?\\.(?:${exts}))`, 'gi'); @@ -453,7 +485,7 @@ function extractRawFilePaths(text: string): Array<{ filePath: string; mimeType: ); for (const regex of [unixRegex, winRegex, skillDirRegex]) { let match; - while ((match = regex.exec(text)) !== null) { + while ((match = regex.exec(workingText)) !== null) { const p = trimPathTerminators(match[1]); if (p && !seen.has(p)) { seen.add(p); @@ -508,6 +540,22 @@ function extractImagesAsAttachedFiles(content: unknown): AttachedFileMeta[] { preview: `data:${mimeType};base64,${block.data}`, }); } + // Path 3: Flat URL form from Gateway-injected assistant-media messages. + // See `src/stores/chat/helpers.ts` for the canonical implementation. + else if (block.url) { + const mimeType = block.mimeType || 'image/jpeg'; + const fileName = typeof block.alt === 'string' && block.alt + ? block.alt + : 'image'; + files.push({ + fileName, + mimeType, + fileSize: 0, + preview: null, + gatewayUrl: block.url, + source: 'gateway-media', + }); + } } // Recurse into tool_result content blocks if ((block.type === 'tool_result' || block.type === 'toolResult') && block.content) { @@ -627,8 +675,15 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] { // Resolve file path from the matching tool call const matchedPath = msg.toolCallId ? toolCallPaths.get(msg.toolCallId) : undefined; - // 1. Image/file content blocks in the structured content array - const imageFiles = extractImagesAsAttachedFiles(msg.content); + // 1. Image/file content blocks in the structured content array. + // Images embedded inside a tool result are the model's vision data + // (e.g. `read /tmp/foo.png` re-encoded as JPEG so the model can "see" + // the file) — they are NOT user-facing artifacts. The agent surfaces + // user-facing images through `MEDIA:/path` text + the Gateway's + // `assistant-media` injection. Surfacing the vision data here would + // duplicate every screenshot the agent inspects. + const imageFiles = extractImagesAsAttachedFiles(msg.content) + .filter(file => !file.mimeType.startsWith('image/')); if (matchedPath) { for (const f of imageFiles) { if (!f.filePath) { @@ -650,11 +705,17 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] { for (const ref of mediaRefs) { pending.push({ ...makeAttachedFile(ref), source: 'tool-result' }); } - // 3. Raw file paths in tool result text (documents, audio, video, etc.) + // 3. Raw NON-image file paths in tool result text (documents, + // audio, video, ...). Image paths are deliberately ignored: + // `ls -la /tmp/foo.png`, `sips ... && ls -la *.jpg`, etc. + // spam intermediate paths that the user does not want to see + // rendered as separate cards. The canonical user-facing image + // is whatever the agent later emits via `MEDIA:/path` (which + // the Gateway turns into a dedicated assistant-media bubble). for (const ref of extractRawFilePaths(text)) { - if (!mediaRefPaths.has(ref.filePath)) { - pending.push({ ...makeAttachedFile(ref), source: 'tool-result' }); - } + if (mediaRefPaths.has(ref.filePath)) continue; + if (ref.mimeType.startsWith('image/')) continue; + pending.push({ ...makeAttachedFile(ref), source: 'tool-result' }); } } @@ -687,11 +748,30 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] { * Uses local cache for previews when available; missing previews are loaded async. */ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] { + // Pre-compute, per index, whether the *next* assistant message is a + // Gateway-injected `assistant-media` bubble (i.e. has at least one + // `image` content block carrying a flat URL). When that bubble exists, + // the canonical user-facing rendering of the artifact is the bubble + // itself — anything the agent emitted via `MEDIA:/path` in its prior + // text turn would just duplicate the same image, so image-typed raw + // refs on that prior message should be dropped here. + const nextHasGatewayMediaBubble = messages.map((_, idx) => { + const next = messages[idx + 1]; + if (!next || next.role !== 'assistant') return false; + return extractImagesAsAttachedFiles(next.content).some(f => f.gatewayUrl); + }); + return messages.map((msg, idx) => { - // Only process user and assistant messages; skip if already enriched - if ((msg.role !== 'user' && msg.role !== 'assistant') || msg._attachedFiles) return msg; + // Only process user and assistant messages. + if (msg.role !== 'user' && msg.role !== 'assistant') return msg; const text = getMessageText(msg.content); + // Path 0: Gateway-injected outgoing media on this same message + // (an `assistant-media` bubble — image block with flat `url`). + const gatewayMediaFiles: AttachedFileMeta[] = msg.role === 'assistant' + ? extractImagesAsAttachedFiles(msg.content).filter(file => file.gatewayUrl) + : []; + // Path 1: [media attached: path (mime) | path] — guaranteed format from attachment button const mediaRefs = extractMediaRefs(text); const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath)); @@ -699,9 +779,6 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] { // Path 2: Raw file paths. // For assistant messages: scan own text AND the nearest preceding user message text, // but only for non-tool-only assistant messages (i.e. the final answer turn). - // Tool-only messages (thinking + tool calls) should not show file previews — those - // belong to the final answer message that comes after the tool results. - // User messages never get raw-path previews so the image is not shown twice. let rawRefs: Array<{ filePath: string; mimeType: string }> = []; if (msg.role === 'assistant' && !isToolOnlyMessage(msg)) { // Own text @@ -725,16 +802,39 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] { } } - const allRefs = [...mediaRefs, ...rawRefs]; - if (allRefs.length === 0) return msg; + // Dedup vs Gateway-injected bubble: if the very next assistant message + // is a Gateway `assistant-media` bubble, drop image-typed raw refs on + // *this* message — the bubble already covers them. + if (msg.role === 'assistant' && nextHasGatewayMediaBubble[idx]) { + rawRefs = rawRefs.filter(r => !r.mimeType.startsWith('image/')); + } - const files: AttachedFileMeta[] = allRefs.map(ref => { - const cached = _imageCache.get(ref.filePath); - if (cached) return { ...cached, filePath: ref.filePath, source: 'message-ref' as const }; - const fileName = ref.filePath.split(/[\\/]/).pop() || 'file'; - return { fileName, mimeType: ref.mimeType, fileSize: 0, preview: null, filePath: ref.filePath, source: 'message-ref' as const }; - }); - return { ...msg, _attachedFiles: files }; + const allRefs = [...mediaRefs, ...rawRefs]; + if (allRefs.length === 0 && gatewayMediaFiles.length === 0) { + // Preserve any previously-attached `_attachedFiles` (e.g. set by + // `enrichWithToolResultFiles` for non-image artifacts). When nothing + // new applies, returning `msg` unmodified keeps those attachments. + return msg; + } + + const existingFiles = msg._attachedFiles || []; + const existingPaths = new Set(existingFiles.map(file => file.filePath).filter(Boolean)); + const existingGatewayUrls = new Set( + existingFiles.map(file => file.gatewayUrl).filter(Boolean) as string[], + ); + const files: AttachedFileMeta[] = allRefs + .filter(ref => !existingPaths.has(ref.filePath)) + .map(ref => { + const cached = _imageCache.get(ref.filePath); + if (cached) return { ...cached, filePath: ref.filePath, source: 'message-ref' as const }; + const fileName = ref.filePath.split(/[\\/]/).pop() || 'file'; + return { fileName, mimeType: ref.mimeType, fileSize: 0, preview: null, filePath: ref.filePath, source: 'message-ref' as const }; + }); + const dedupedGatewayMedia = gatewayMediaFiles.filter( + file => file.gatewayUrl && !existingGatewayUrls.has(file.gatewayUrl), + ); + if (files.length === 0 && dedupedGatewayMedia.length === 0) return msg; + return { ...msg, _attachedFiles: [...existingFiles, ...files, ...dedupedGatewayMedia] }; }); } @@ -744,24 +844,29 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] { * Handles both [media attached: ...] patterns and raw filePath entries. */ async function loadMissingPreviews(messages: RawMessage[]): Promise { - // Collect all image paths that need previews - const needPreview: Array<{ filePath: string; mimeType: string }> = []; - const seenPaths = new Set(); + // See helpers.ts loadMissingPreviews for the canonical comment block — + // this monolithic copy is kept in sync so legacy chat.ts callers also + // resolve Gateway-injected outgoing media URLs into local previews. + type PreviewRef = { filePath?: string; gatewayUrl?: string; mimeType: string }; + const needPreview: PreviewRef[] = []; + const seenKeys = new Set(); for (const msg of messages) { if (!msg._attachedFiles) continue; - // Path 1: files with explicit filePath field (raw path detection or enriched refs) + // Path 1: files with explicit filePath OR gatewayUrl for (const file of msg._attachedFiles) { - const fp = file.filePath; - if (!fp || seenPaths.has(fp)) continue; - // Images: need preview. Non-images: need file size (for FileCard display). + const key = file.filePath || file.gatewayUrl; + if (!key || seenKeys.has(key)) continue; const needsLoad = file.mimeType.startsWith('image/') ? !file.preview : file.fileSize === 0; - if (needsLoad) { - seenPaths.add(fp); - needPreview.push({ filePath: fp, mimeType: file.mimeType }); + if (!needsLoad) continue; + seenKeys.add(key); + if (file.filePath) { + needPreview.push({ filePath: file.filePath, mimeType: file.mimeType }); + } else if (file.gatewayUrl) { + needPreview.push({ gatewayUrl: file.gatewayUrl, mimeType: file.mimeType }); } } @@ -772,17 +877,19 @@ async function loadMissingPreviews(messages: RawMessage[]): Promise { for (let i = 0; i < refs.length; i++) { const file = msg._attachedFiles[i]; const ref = refs[i]; - if (!file || !ref || seenPaths.has(ref.filePath)) continue; + if (!file || !ref || seenKeys.has(ref.filePath)) continue; const needsLoad = ref.mimeType.startsWith('image/') ? !file.preview : file.fileSize === 0; if (needsLoad) { - seenPaths.add(ref.filePath); - needPreview.push(ref); + seenKeys.add(ref.filePath); + needPreview.push({ filePath: ref.filePath, mimeType: ref.mimeType }); } } } } - if (needPreview.length === 0) return false; + if (needPreview.length === 0) { + return false; + } try { const thumbnails = await hostApiFetch>( @@ -797,15 +904,17 @@ async function loadMissingPreviews(messages: RawMessage[]): Promise { for (const msg of messages) { if (!msg._attachedFiles) continue; - // Update files that have filePath + // Update files that have filePath OR gatewayUrl for (const file of msg._attachedFiles) { - const fp = file.filePath; - if (!fp) continue; - const thumb = thumbnails[fp]; + const key = file.filePath || file.gatewayUrl; + if (!key) continue; + const thumb = thumbnails[key]; if (thumb && (thumb.preview || thumb.fileSize)) { if (thumb.preview) file.preview = thumb.preview; if (thumb.fileSize) file.fileSize = thumb.fileSize; - _imageCache.set(fp, { ...file }); + if (file.filePath) { + _imageCache.set(file.filePath, { ...file }); + } updated = true; } } @@ -1004,11 +1113,35 @@ function isToolResultRole(role: unknown): boolean { } /** True for internal plumbing messages that should never be shown in the UI. */ -function isInternalMessage(msg: { role?: unknown; content?: unknown }): boolean { +function isInternalMessage(msg: { role?: unknown; content?: unknown; idempotencyKey?: unknown; model?: unknown }): boolean { if (msg.role === 'system') return true; const text = getMessageText(msg.content); if (msg.role === 'assistant') { if (/^(HEARTBEAT_OK|NO_REPLY)\s*$/.test(text)) return true; + // OpenClaw's gateway writes a fallback `assistant-media` transcript + // message when its `createManagedOutgoingImageBlocks` pipeline fails + // ("could not be prepared" warning in stderr). The fallback has: + // - model: 'gateway-injected' + // - idempotencyKey: ':assistant-media' + // - content: text-only `MEDIA:` (NOT an image-url block) + // The staging path lives in `~/.openclaw/media/outbound/` which has a + // 120s TTL — the file is gone by the time the user reads the chat. + // The original `MEDIA:/tmp/...` path is already on the previous + // assistant message (the agent's actual reply), so the fallback is + // pure duplicate noise. Hide it in the UI so it neither shows a + // broken card nor competes with the real reply for layout space. + const idempotencyKey = typeof msg.idempotencyKey === 'string' ? msg.idempotencyKey : ''; + const isGatewayInjectedFallback = msg.model === 'gateway-injected' + && idempotencyKey.endsWith(':assistant-media'); + if (isGatewayInjectedFallback) { + const hasImageUrlBlock = Array.isArray(msg.content) + && (msg.content as ContentBlock[]).some( + (block) => block.type === 'image' && typeof block.url === 'string' && !!block.url, + ); + // Real gateway-media bubbles (with an image-url block) ARE the + // canonical render — keep them. Only hide the text-only fallback. + if (!hasImageUrlBlock) return true; + } } // Runtime system injections: these arrive as user or assistant-role messages // but are internal plumbing (exec results, async-command notices, time pings, etc.) @@ -2166,12 +2299,18 @@ export const useChatStore = create((set, get) => ({ const { activeRunId, currentSessionKey } = get(); // Only process events for the current session (when sessionKey is present) - if (eventSessionKey != null && eventSessionKey !== currentSessionKey) return; + if (eventSessionKey != null && eventSessionKey !== currentSessionKey) { + return; + } // Only process events for the active run (or if no active run set) - if (activeRunId && runId && runId !== activeRunId) return; + if (activeRunId && runId && runId !== activeRunId) { + return; + } - if (isDuplicateChatEvent(eventState, event)) return; + if (isDuplicateChatEvent(eventState, event)) { + return; + } _lastChatEventAt = Date.now(); @@ -2285,10 +2424,17 @@ export const useChatStore = create((set, get) => ({ ? getToolCallFilePath(currentStreamForPath, normalizedFinalMessage.toolCallId) : undefined; - // Mirror enrichWithToolResultFiles: collect images + file refs for next assistant msg - const toolFiles: AttachedFileMeta[] = [ - ...extractImagesAsAttachedFiles(normalizedFinalMessage.content), - ]; + // Mirror `enrichWithToolResultFiles`: collect non-image artifacts + // for the next assistant message. Images embedded inside a tool + // result (read tool's vision data) and raw image paths in the + // tool's stdout (sips / ls / file output) are NOT user-facing — + // the canonical render is the Gateway-injected `assistant-media` + // bubble that follows the agent's `MEDIA:` text. Surfacing those + // intermediate images here would duplicate every screenshot the + // agent inspects on its way to the final artifact. + const toolFiles: AttachedFileMeta[] = extractImagesAsAttachedFiles( + normalizedFinalMessage.content, + ).filter(file => !file.mimeType.startsWith('image/')); if (matchedPath) { for (const f of toolFiles) { if (!f.filePath) { @@ -2303,7 +2449,9 @@ export const useChatStore = create((set, get) => ({ const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath)); for (const ref of mediaRefs) toolFiles.push(makeAttachedFile(ref)); for (const ref of extractRawFilePaths(text)) { - if (!mediaRefPaths.has(ref.filePath)) toolFiles.push(makeAttachedFile(ref)); + if (mediaRefPaths.has(ref.filePath)) continue; + if (ref.mimeType.startsWith('image/')) continue; + toolFiles.push(makeAttachedFile(ref)); } } set((s) => { @@ -2334,7 +2482,17 @@ export const useChatStore = create((set, get) => ({ const nextTools = updates.length > 0 ? upsertToolStatuses(s.streamingTools, updates) : s.streamingTools; const streamingTools = hasOutput ? [] : nextTools; - // Attach any images collected from preceding tool results + // Note: it would be tempting to also surface `MEDIA:/path` + // markers from `normalizedFinalMessage.content`'s text here, so + // the agent's reply could attach the original file directly + // (`/tmp/...png`) without waiting for the post-final history + // reload. However, OpenClaw's `splitTrailingDirective` + // (selection-D8_ELZa7.js ~line 904) strips `MEDIA:/...` lines + // out of the streaming text BEFORE it reaches the client, so + // the `final` event we get here never contains the marker. + // Image surfacing is fully handled by the post-final reload + // below + `enrichWithCachedImages` (which dereferences the + // assistant-media bubble's `block.url`). const pendingImgs = s.pendingToolImages; const msgWithImages: RawMessage = pendingImgs.length > 0 ? { @@ -2388,6 +2546,52 @@ export const useChatStore = create((set, get) => ({ if (hasOutput && !toolOnly) { clearHistoryPoll(); void get().loadHistory(true); + + // OpenClaw's gateway processes `MEDIA:/path` markers in the + // assistant reply asynchronously, in the `dispatch.deliver` of + // the `final` payload (see openclaw/dist/chat-DM9hSaNV.js's + // `appendWebchatAgentMediaTranscriptIfNeeded`): + // 1. copy the original file under + // `~/.openclaw/media/outgoing/originals/` + // 2. write the record JSON under + // `~/.openclaw/media/outgoing/records/.json` + // 3. `appendAssistantTranscriptMessage` writes a follow-up + // `assistant-media` message to the session JSONL, with + // `idempotencyKey: ":assistant-media"`. + // That follow-up message is **only persisted** — it is NOT + // re-broadcast as a streaming event. The streaming `final` + // we just consumed only contains the agent's text. The + // assistant-media bubble can only be retrieved via + // `chat.history`, and the persistence runs on the order of + // ~400-500ms after the streaming final. + // + // The immediate `loadHistory(true)` above therefore races the + // gateway's write and almost always misses the bubble. + // + // CRITICAL: we cannot detect from the streaming final alone + // whether the agent emitted a `MEDIA:/path` marker — OpenClaw's + // `splitTrailingDirective` (selection-D8_ELZa7.js line ~904) + // strips `MEDIA:/...` lines from the broadcast text BEFORE it + // reaches the client, so the streaming `final` text is always + // the user-facing prose without the marker. The MEDIA: marker + // only appears in the persisted JSONL transcript (msg N) and + // its companion `assistant-media` bubble (msg N+1). + // + // We therefore unconditionally schedule ONE follow-up quiet + // reload ~1500ms after every assistant `final`. The cost is + // a single extra in-process RPC per assistant turn (cheap); + // when there's no media the second reload returns the same + // history snapshot and is a no-op for the UI. + // `forceNextHistoryLoad` bypasses `HISTORY_LOAD_MIN_INTERVAL_MS` + // so the call is not suppressed by the throttle. + const sessionKeyAtFinal = get().currentSessionKey; + setTimeout(() => { + if (get().currentSessionKey !== sessionKeyAtFinal) { + return; + } + forceNextHistoryLoad(sessionKeyAtFinal); + void get().loadHistory(true); + }, 1500); } } else { // No message in final event - reload history to get complete data diff --git a/src/stores/chat/helpers.ts b/src/stores/chat/helpers.ts index 88dc2d1..45428ee 100644 --- a/src/stores/chat/helpers.ts +++ b/src/stores/chat/helpers.ts @@ -360,7 +360,14 @@ function extractRawFilePaths(text: string): Array<{ filePath: string; mimeType: // normally look like a URL scheme. After matching we punch the entire // `MEDIA:` span out of the working text so the generic unix // regex below doesn't double-count the bare `/path` suffix. - const taggedRegex = new RegExp(`(?:^|[\\s(\\[{>])(?:MEDIA|media):((?:\\/|~\\/)[^\\s\\n"'()\\[\\],<>` + '`' + `]*?\\.(?:${exts}))`, 'g'); + // The character class deliberately allows ASCII spaces inside the path so + // that macOS' default screenshot filename ("截屏 2026-05-06 17.46.51.png") + // and other space-containing paths the agent emits with the explicit + // `MEDIA:` marker still resolve. Newline and quote characters remain + // path terminators so we don't accidentally swallow trailing prose. + // The non-greedy `*?` anchored to `\.` keeps the match minimal so + // multiple `MEDIA:` markers in one paragraph still match independently. + const taggedRegex = new RegExp(`(?:^|[\\s(\\[{>])(?:MEDIA|media):((?:\\/|~\\/)[^\\n"'()\\[\\],<>` + '`' + `]*?\\.(?:${exts}))(?=$|[\\s\\n"'()\\[\\],<>` + '`' + `]|[，。；;,.!?])`, 'g'); let workingText = text; let taggedMatch: RegExpExecArray | null; while ((taggedMatch = taggedRegex.exec(text)) !== null) { @@ -446,6 +453,26 @@ function extractImagesAsAttachedFiles(content: unknown): AttachedFileMeta[] { preview: `data:${mimeType};base64,${block.data}`, }); } + // Path 3: Flat URL form from Gateway-injected assistant-media messages. + // Shape: `{ type:'image', url:'/api/chat/media/outgoing///full', + // mimeType, width, height, alt, openUrl }`. The URL is relative + // to the Gateway HTTP server which the renderer cannot reach directly + // (CORS / env drift). We surface it as an `_attachedFiles` entry whose + // preview is filled in later by `loadMissingPreviews` -> Main proxy. + else if (block.url) { + const mimeType = block.mimeType || 'image/jpeg'; + const fileName = typeof block.alt === 'string' && block.alt + ? block.alt + : 'image'; + files.push({ + fileName, + mimeType, + fileSize: 0, + preview: null, + gatewayUrl: block.url, + source: 'gateway-media', + }); + } } // Recurse into tool_result content blocks if ((block.type === 'tool_result' || block.type === 'toolResult') && block.content) { @@ -568,8 +595,14 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] { // Resolve file path from the matching tool call const matchedPath = msg.toolCallId ? toolCallPaths.get(msg.toolCallId) : undefined; - // 1. Image/file content blocks in the structured content array - const imageFiles = extractImagesAsAttachedFiles(msg.content); + // 1. Image/file content blocks in the structured content array. + // Images embedded inside a tool result are the model's vision data + // (e.g. `read /tmp/foo.png` re-encoded as JPEG so the model can + // "see" the file) — they are NOT user-facing artifacts. The agent + // surfaces user-facing images through `MEDIA:/path` text + the + // Gateway's `assistant-media` injection. + const imageFiles = extractImagesAsAttachedFiles(msg.content) + .filter(file => !file.mimeType.startsWith('image/')); if (matchedPath) { for (const f of imageFiles) { if (!f.filePath) { @@ -588,11 +621,14 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] { for (const ref of mediaRefs) { pending.push(makeAttachedFile(ref, 'tool-result')); } - // 3. Raw file paths in tool result text (documents, audio, video, etc.) + // 3. Raw NON-image file paths in tool result text (documents, + // audio, video, ...). Image paths from intermediate tool stdout + // (`ls -la *.png`, `sips ... && ls`, `file /tmp/x.png`, etc.) + // are deliberately ignored — see comment on Path 1. for (const ref of extractRawFilePaths(text)) { - if (!mediaRefPaths.has(ref.filePath)) { - pending.push(makeAttachedFile(ref, 'tool-result')); - } + if (mediaRefPaths.has(ref.filePath)) continue; + if (ref.mimeType.startsWith('image/')) continue; + pending.push(makeAttachedFile(ref, 'tool-result')); } } @@ -625,6 +661,19 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] { * Uses local cache for previews when available; missing previews are loaded async. */ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] { + // Pre-compute, per index, whether the *next* assistant message is a + // Gateway-injected `assistant-media` bubble (i.e. has at least one + // `image` content block carrying a flat URL). When that bubble exists, + // the canonical user-facing rendering of the artifact is the bubble + // itself — anything the agent emitted via `MEDIA:/path` in its prior + // text turn would just duplicate the same image, so image-typed raw + // refs on that prior message are dropped here. + const nextHasGatewayMediaBubble = messages.map((_, idx) => { + const next = messages[idx + 1]; + if (!next || next.role !== 'assistant') return false; + return extractImagesAsAttachedFiles(next.content).some(f => f.gatewayUrl); + }); + return messages.map((msg, idx) => { // Only process user and assistant messages. Messages may already carry // attachments from tool-result enrichment; still merge in raw paths from @@ -632,6 +681,16 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] { if (msg.role !== 'user' && msg.role !== 'assistant') return msg; const text = getMessageText(msg.content); + // Path 0: Gateway-injected outgoing media — `image` content blocks with + // a flat `url` field (e.g. `/api/chat/media/outgoing///full`). + // The renderer cannot fetch the URL directly, so we surface it as an + // `_attachedFiles` entry whose preview is filled in later by + // `loadMissingPreviews` -> Main `media:getThumbnails` (which dereferences + // the URL to the original file in `~/.openclaw/media/outgoing/`). + const gatewayMediaFiles: AttachedFileMeta[] = msg.role === 'assistant' + ? extractImagesAsAttachedFiles(msg.content).filter(file => file.gatewayUrl) + : []; + // Path 1: [media attached: path (mime) | path] — guaranteed format from attachment button const mediaRefs = extractMediaRefs(text); const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath)); @@ -665,11 +724,21 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] { } } + // Dedup vs Gateway-injected bubble: when the very next assistant + // message is an `assistant-media` bubble, drop image-typed raw refs + // on *this* message — the bubble already covers the artifact. + if (msg.role === 'assistant' && nextHasGatewayMediaBubble[idx]) { + rawRefs = rawRefs.filter(r => !r.mimeType.startsWith('image/')); + } + const allRefs = [...mediaRefs, ...rawRefs]; - if (allRefs.length === 0) return msg; + if (allRefs.length === 0 && gatewayMediaFiles.length === 0) return msg; const existingFiles = msg._attachedFiles || []; const existingPaths = new Set(existingFiles.map(file => file.filePath).filter(Boolean)); + const existingGatewayUrls = new Set( + existingFiles.map(file => file.gatewayUrl).filter(Boolean) as string[], + ); const files: AttachedFileMeta[] = allRefs .filter(ref => !existingPaths.has(ref.filePath)) .map(ref => { @@ -678,8 +747,11 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] { const fileName = ref.filePath.split(/[\\/]/).pop() || 'file'; return { fileName, mimeType: ref.mimeType, fileSize: 0, preview: null, filePath: ref.filePath, source: 'message-ref' }; }); - if (files.length === 0) return msg; - return { ...msg, _attachedFiles: [...existingFiles, ...files] }; + const dedupedGatewayMedia = gatewayMediaFiles.filter( + file => file.gatewayUrl && !existingGatewayUrls.has(file.gatewayUrl), + ); + if (files.length === 0 && dedupedGatewayMedia.length === 0) return msg; + return { ...msg, _attachedFiles: [...existingFiles, ...files, ...dedupedGatewayMedia] }; }); } @@ -689,24 +761,34 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] { * Handles both [media attached: ...] patterns and raw filePath entries. */ async function loadMissingPreviews(messages: RawMessage[]): Promise { - // Collect all image paths that need previews - const needPreview: Array<{ filePath: string; mimeType: string }> = []; - const seenPaths = new Set(); + // Collect all image refs that need previews. The IPC handler accepts: + // - { filePath, mimeType } — local on-disk files + // - { gatewayUrl, mimeType } — Gateway-injected outgoing media; the + // handler resolves the URL to a local file + // via `~/.openclaw/media/outgoing/records/`. + // We use `filePath || gatewayUrl` as the dedupe / lookup key on the way + // back; a file always carries at most one of the two. + type PreviewRef = { filePath?: string; gatewayUrl?: string; mimeType: string }; + const needPreview: PreviewRef[] = []; + const seenKeys = new Set(); for (const msg of messages) { if (!msg._attachedFiles) continue; - // Path 1: files with explicit filePath field (raw path detection or enriched refs) + // Path 1: files with explicit filePath OR gatewayUrl for (const file of msg._attachedFiles) { - const fp = file.filePath; - if (!fp || seenPaths.has(fp)) continue; + const key = file.filePath || file.gatewayUrl; + if (!key || seenKeys.has(key)) continue; // Images: need preview. Non-images: need file size (for FileCard display). const needsLoad = file.mimeType.startsWith('image/') ? !file.preview : file.fileSize === 0; - if (needsLoad) { - seenPaths.add(fp); - needPreview.push({ filePath: fp, mimeType: file.mimeType }); + if (!needsLoad) continue; + seenKeys.add(key); + if (file.filePath) { + needPreview.push({ filePath: file.filePath, mimeType: file.mimeType }); + } else if (file.gatewayUrl) { + needPreview.push({ gatewayUrl: file.gatewayUrl, mimeType: file.mimeType }); } } @@ -717,11 +799,11 @@ async function loadMissingPreviews(messages: RawMessage[]): Promise { for (let i = 0; i < refs.length; i++) { const file = msg._attachedFiles[i]; const ref = refs[i]; - if (!file || !ref || seenPaths.has(ref.filePath)) continue; + if (!file || !ref || seenKeys.has(ref.filePath)) continue; const needsLoad = ref.mimeType.startsWith('image/') ? !file.preview : file.fileSize === 0; if (needsLoad) { - seenPaths.add(ref.filePath); - needPreview.push(ref); + seenKeys.add(ref.filePath); + needPreview.push({ filePath: ref.filePath, mimeType: ref.mimeType }); } } } @@ -739,15 +821,20 @@ async function loadMissingPreviews(messages: RawMessage[]): Promise { for (const msg of messages) { if (!msg._attachedFiles) continue; - // Update files that have filePath + // Update files that have filePath OR gatewayUrl for (const file of msg._attachedFiles) { - const fp = file.filePath; - if (!fp) continue; - const thumb = thumbnails[fp]; + const key = file.filePath || file.gatewayUrl; + if (!key) continue; + const thumb = thumbnails[key]; if (thumb && (thumb.preview || thumb.fileSize)) { if (thumb.preview) file.preview = thumb.preview; if (thumb.fileSize) file.fileSize = thumb.fileSize; - _imageCache.set(fp, { ...file }); + // Only persist local-path entries to the localStorage cache. + // Gateway outgoing URLs are tied to a specific session/attachment + // id and can be stale across runs, so caching is harmful. + if (file.filePath) { + _imageCache.set(file.filePath, { ...file }); + } updated = true; } } diff --git a/src/stores/chat/runtime-event-handlers.ts b/src/stores/chat/runtime-event-handlers.ts index 94e61ba..8e040fc 100644 --- a/src/stores/chat/runtime-event-handlers.ts +++ b/src/stores/chat/runtime-event-handlers.ts @@ -119,8 +119,13 @@ export function handleRuntimeEventState( ? getToolCallFilePath(currentStreamForPath, normalizedFinalMessage.toolCallId) : undefined; - // Mirror enrichWithToolResultFiles: collect images + file refs for next assistant msg + // Mirror `enrichWithToolResultFiles`: collect non-image + // artifacts for the next assistant message. Image content + // blocks (vision data) and image-typed raw paths in the + // tool's stdout are intermediate process noise — see the + // comment in `helpers.ts::enrichWithToolResultFiles`. const toolFiles: AttachedFileMeta[] = extractImagesAsAttachedFiles(normalizedFinalMessage.content) + .filter((file) => !file.mimeType.startsWith('image/')) .map((file) => (file.source ? file : { ...file, source: 'tool-result' })); if (matchedPath) { for (const f of toolFiles) { @@ -136,7 +141,9 @@ export function handleRuntimeEventState( const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath)); for (const ref of mediaRefs) toolFiles.push(makeAttachedFile(ref, 'tool-result')); for (const ref of extractRawFilePaths(text)) { - if (!mediaRefPaths.has(ref.filePath)) toolFiles.push(makeAttachedFile(ref, 'tool-result')); + if (mediaRefPaths.has(ref.filePath)) continue; + if (ref.mimeType.startsWith('image/')) continue; + toolFiles.push(makeAttachedFile(ref, 'tool-result')); } } set((s) => { diff --git a/src/stores/chat/types.ts b/src/stores/chat/types.ts index 9118e66..502b019 100644 --- a/src/stores/chat/types.ts +++ b/src/stores/chat/types.ts @@ -5,7 +5,17 @@ export interface AttachedFileMeta { fileSize: number; preview: string | null; filePath?: string; - source?: 'user-upload' | 'tool-result' | 'message-ref'; + source?: 'user-upload' | 'tool-result' | 'message-ref' | 'gateway-media'; + /** + * For Gateway-injected outgoing media (assistant-media). The Gateway emits + * an `image` content block with a relative URL like + * `/api/chat/media/outgoing///full`. The renderer + * cannot reach Gateway HTTP directly (CORS / env drift), so this URL is + * resolved through the Main-process proxy in `media:getThumbnails`, which + * looks up `~/.openclaw/media/outgoing/records/.json` and + * loads the original file off disk. + */ + gatewayUrl?: string; } /** Raw message from OpenClaw chat.history */ @@ -35,6 +45,21 @@ export interface ContentBlock { /** Flat image format from Gateway tool results (no source wrapper) */ data?: string; mimeType?: string; + /** + * Flat URL on an `image` block. Gateway-injected assistant-media messages + * use this shape: `{ type:'image', url:'/api/chat/media/outgoing/...', mimeType, width, height, alt, openUrl }`. + * Neither nested `source.url` nor flat `data` is set in that case; the + * renderer must read `block.url` directly to surface the artifact. + */ + url?: string; + /** Optional companion of `url` — points at a higher-resolution variant. */ + openUrl?: string; + /** Pixel width of the original image, used for layout hints. */ + width?: number; + /** Pixel height of the original image, used for layout hints. */ + height?: number; + /** Human-readable filename / alt text emitted by the Gateway. */ + alt?: string; id?: string; name?: string; input?: unknown; diff --git a/tests/unit/chat-helpers-enrichment.test.ts b/tests/unit/chat-helpers-enrichment.test.ts new file mode 100644 index 0000000..0e31e32 --- /dev/null +++ b/tests/unit/chat-helpers-enrichment.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + enrichWithToolResultFiles, + enrichWithCachedImages, +} from '@/stores/chat/helpers'; +import type { RawMessage } from '@/stores/chat'; + +vi.mock('@/lib/api-client', () => ({ + invokeIpc: vi.fn(), +})); + +describe('enrichWithToolResultFiles', () => { + it('does not promote image content blocks emitted by `read` tool results', () => { + // The `read` tool re-encodes the file as JPEG so the model can "see" it. + // The resulting image-data block is internal vision data, NOT a + // user-facing artifact — it must NOT spill onto the next assistant + // message as an attachment, otherwise every screenshot the agent + // inspects would render in the chat. + const messages: RawMessage[] = [ + { + role: 'assistant', + id: 'a1', + content: [{ type: 'toolCall', id: 'tc1', name: 'read', input: { path: '/tmp/foo.png' } }], + }, + { + role: 'toolresult', + id: 't1', + toolCallId: 'tc1', + toolName: 'read', + content: [ + { type: 'text', text: 'Read image file [image/jpeg]\n[Image: ...]' }, + { type: 'image', data: 'BASE64_BYTES_HERE', mimeType: 'image/jpeg' }, + ], + }, + { + role: 'assistant', + id: 'a2', + content: [{ type: 'text', text: 'I had a look at the screenshot.' }], + }, + ]; + + const enriched = enrichWithToolResultFiles(messages); + const reply = enriched.find((m) => m.id === 'a2')!; + expect(reply._attachedFiles ?? []).toEqual([]); + }); + + it('does not promote raw image paths from tool result stdout (sips / ls / file)', () => { + // `sips ... && ls -la *.jpg` etc. spam image paths in the tool's + // stdout. Each one used to surface as an `_attachedFiles` entry on + // the next assistant message, causing the canonical artifact to be + // duplicated 3-4 times per send. + const messages: RawMessage[] = [ + { + role: 'assistant', + id: 'a1', + content: [{ type: 'toolCall', id: 'tc1', name: 'exec', input: { command: 'sips ... && ls -la' } }], + }, + { + role: 'toolresult', + id: 't1', + toolCallId: 'tc1', + toolName: 'exec', + content: [{ + type: 'text', + text: '/private/tmp/desktop_screenshot.png\n /private/tmp/desktop_screenshot.jpg\n-rw-r--r--@ 1 me staff 857671 May 6 18:05 /tmp/desktop_screenshot.jpg', + }], + }, + { + role: 'assistant', + id: 'a2', + content: [{ type: 'text', text: 'Compressed to 837KB, sending again.' }], + }, + ]; + + const enriched = enrichWithToolResultFiles(messages); + const reply = enriched.find((m) => m.id === 'a2')!; + expect(reply._attachedFiles ?? []).toEqual([]); + }); + + it('still promotes non-image artifact paths from tool results (PDF / XLSX)', () => { + // Documents emitted by tools (e.g. a Python script that wrote a + // spreadsheet to disk) ARE user-facing — they remain surfaced. + const messages: RawMessage[] = [ + { + role: 'assistant', + id: 'a1', + content: [{ type: 'toolCall', id: 'tc1', name: 'exec', input: { command: '...' } }], + }, + { + role: 'toolresult', + id: 't1', + toolCallId: 'tc1', + toolName: 'exec', + content: [{ type: 'text', text: 'Saved report at /tmp/report.pdf and data at /tmp/sales.xlsx' }], + }, + { + role: 'assistant', + id: 'a2', + content: [{ type: 'text', text: 'Generated.' }], + }, + ]; + + const enriched = enrichWithToolResultFiles(messages); + const reply = enriched.find((m) => m.id === 'a2')!; + const paths = (reply._attachedFiles ?? []).map((f) => f.filePath); + expect(paths).toEqual(expect.arrayContaining(['/tmp/report.pdf', '/tmp/sales.xlsx'])); + expect(paths.find((p) => p?.endsWith('.png'))).toBeUndefined(); + expect(paths.find((p) => p?.endsWith('.jpg'))).toBeUndefined(); + }); + + it('still promotes [media attached: ...] references emitted in tool results', () => { + const messages: RawMessage[] = [ + { + role: 'assistant', + id: 'a1', + content: [{ type: 'toolCall', id: 'tc1', name: 'fetch', input: {} }], + }, + { + role: 'toolresult', + id: 't1', + toolCallId: 'tc1', + toolName: 'fetch', + content: [{ + type: 'text', + text: 'Done [media attached: /tmp/foo.pdf (application/pdf) | /tmp/foo.pdf]', + }], + }, + { + role: 'assistant', + id: 'a2', + content: [{ type: 'text', text: 'Here it is.' }], + }, + ]; + + const enriched = enrichWithToolResultFiles(messages); + const reply = enriched.find((m) => m.id === 'a2')!; + const paths = (reply._attachedFiles ?? []).map((f) => f.filePath); + expect(paths).toContain('/tmp/foo.pdf'); + }); +}); + +describe('enrichWithCachedImages — Gateway media bubble dedup', () => { + it('drops image-typed MEDIA: refs on the reply when the next message is a Gateway assistant-media bubble', () => { + // When the agent emits `MEDIA:/tmp/x.png` the Gateway answers with a + // dedicated `assistant-media` bubble. Surfacing the same image again + // on the prior reply text would render two copies of the screenshot. + const messages: RawMessage[] = [ + { + role: 'assistant', + id: 'reply', + content: [{ type: 'text', text: 'Compressed to 837KB:\n\nMEDIA:/tmp/desktop_screenshot.jpg' }], + }, + { + role: 'assistant', + id: 'gateway-media', + content: [{ + type: 'image', + url: '/api/chat/media/outgoing/agent%3Amain%3As-1/abc-123/full', + mimeType: 'image/jpeg', + alt: 'desktop_screenshot.jpg', + }], + }, + ]; + + const enriched = enrichWithCachedImages(messages); + const reply = enriched.find((m) => m.id === 'reply')!; + const replyPaths = (reply._attachedFiles ?? []).map((f) => f.filePath); + expect(replyPaths).toEqual([]); + + const bubble = enriched.find((m) => m.id === 'gateway-media')!; + const bubbleEntries = bubble._attachedFiles ?? []; + expect(bubbleEntries).toHaveLength(1); + expect(bubbleEntries[0]).toMatchObject({ + gatewayUrl: '/api/chat/media/outgoing/agent%3Amain%3As-1/abc-123/full', + mimeType: 'image/jpeg', + source: 'gateway-media', + }); + }); + + it('keeps non-image MEDIA: refs on the reply even when a Gateway bubble follows', () => { + // Documents do not benefit from the Gateway's image pipeline; they + // should still render as inline cards on the reply text. + const messages: RawMessage[] = [ + { + role: 'assistant', + id: 'reply', + content: [{ + type: 'text', + text: 'Report generated:\n\nMEDIA:/tmp/report.pdf\n\nMEDIA:/tmp/cover.png', + }], + }, + { + role: 'assistant', + id: 'gateway-media', + content: [{ + type: 'image', + url: '/api/chat/media/outgoing/agent%3Amain%3As-1/cover-id/full', + mimeType: 'image/png', + alt: 'cover.png', + }], + }, + ]; + + const enriched = enrichWithCachedImages(messages); + const reply = enriched.find((m) => m.id === 'reply')!; + const replyPaths = (reply._attachedFiles ?? []).map((f) => f.filePath); + expect(replyPaths).toContain('/tmp/report.pdf'); + expect(replyPaths.find((p) => p?.endsWith('.png'))).toBeUndefined(); + }); + + it('attaches a gateway-media entry to the assistant-media bubble itself (text reply + injected bubble)', () => { + // Reproduces the exact shape of session + // ~/.openclaw/agents/main/sessions/5fb6925e-...jsonl + // - msg 12: assistant reply "...MEDIA:/tmp/openclaw/desktop_*.png" + // - msg 13: assistant `image` block with flat `url` + // After enrichment, msg 13 must carry exactly one `_attachedFiles` + // entry sourced from the Gateway URL — otherwise ChatMessage's early + // return swallows the bubble (no text, no tools, no extracted images, + // no attachments → returns null → user sees nothing). + const messages: RawMessage[] = [ + { + role: 'user', + id: 'u1', + content: [{ type: 'text', text: 'Send me a desktop screenshot.' }], + }, + { + role: 'assistant', + id: 'reply', + content: [{ type: 'text', text: 'Done, here it is:\n\nMEDIA:/tmp/openclaw/desktop_20260506_193407.png' }], + }, + { + role: 'assistant', + id: 'gateway-media', + content: [{ + type: 'image', + url: '/api/chat/media/outgoing/agent%3Amain%3As-1/e024afb9-2bc2-4a64-bf43-1c26fc779b6b/full', + openUrl: '/api/chat/media/outgoing/agent%3Amain%3As-1/e024afb9-2bc2-4a64-bf43-1c26fc779b6b/full', + mimeType: 'image/png', + width: 1536, + height: 998, + alt: 'desktop_20260506_193407.png', + }], + }, + ]; + + const enriched = enrichWithCachedImages(messages); + const reply = enriched.find((m) => m.id === 'reply')!; + expect((reply._attachedFiles ?? []).map((f) => f.filePath)).toEqual([]); + + const bubble = enriched.find((m) => m.id === 'gateway-media')!; + const bubbleEntries = bubble._attachedFiles ?? []; + expect(bubbleEntries).toHaveLength(1); + expect(bubbleEntries[0]).toMatchObject({ + gatewayUrl: '/api/chat/media/outgoing/agent%3Amain%3As-1/e024afb9-2bc2-4a64-bf43-1c26fc779b6b/full', + mimeType: 'image/png', + source: 'gateway-media', + }); + }); + + it('keeps image-typed MEDIA: refs when there is no Gateway bubble after the reply', () => { + // If the Gateway is disabled / hasn't injected a bubble, the agent's + // own `MEDIA:` marker is the only signal and must still surface. + const messages: RawMessage[] = [ + { + role: 'assistant', + id: 'reply', + content: [{ type: 'text', text: 'Here is the screenshot:\n\nMEDIA:/tmp/foo.png' }], + }, + ]; + + const enriched = enrichWithCachedImages(messages); + const reply = enriched[0]!; + const replyPaths = (reply._attachedFiles ?? []).map((f) => f.filePath); + expect(replyPaths).toEqual(['/tmp/foo.png']); + }); +}); diff --git a/tests/unit/chat-helpers-extract-paths.test.ts b/tests/unit/chat-helpers-extract-paths.test.ts index e415552..f5b586e 100644 --- a/tests/unit/chat-helpers-extract-paths.test.ts +++ b/tests/unit/chat-helpers-extract-paths.test.ts @@ -40,6 +40,48 @@ describe('extractRawFilePaths', () => { ]); }); + it('captures MEDIA: paths that contain ASCII spaces (macOS screenshot default name)', () => { + // Regression: macOS' default screenshot filename is + // "Screenshot YYYY-MM-DD at HH.MM.SS.png" (en locale) or + // "截屏 YYYY-MM-DD HH.MM.SS.png" (zh locale) + // and the agent typically emits it verbatim via `MEDIA:` after + // `ls ~/Desktop`. The previous regex excluded ASCII spaces from the + // captured path, which made the extractor stop at "Screenshot" and + // never reach the `.png`, so the screenshot silently failed to + // surface as an attachment. + const sample = 'Found it on the desktop, sending now:\n\nMEDIA:/Users/alice/Desktop/Screenshot 2026-05-06 at 17.46.51.png'; + const refs = extractRawFilePaths(sample); + expect(refs).toEqual([ + { + filePath: '/Users/alice/Desktop/Screenshot 2026-05-06 at 17.46.51.png', + mimeType: 'image/png', + }, + ]); + }); + + it('captures MEDIA: paths whose filename is non-ASCII (zh-locale macOS screenshot)', () => { + // Companion to the previous case — make sure paths whose filename + // is full-Unicode (the zh-locale macOS default) are also captured. + const sample = 'Sending the screenshot now:\n\nMEDIA:/Users/alice/Desktop/截屏 2026-05-06 17.46.51.png'; + const refs = extractRawFilePaths(sample); + expect(refs).toEqual([ + { + filePath: '/Users/alice/Desktop/截屏 2026-05-06 17.46.51.png', + mimeType: 'image/png', + }, + ]); + }); + + it('keeps non-MEDIA prose after a space-bearing path out of the captured filename', () => { + // The lookahead must terminate the match at the first non-path character + // (newline, quote, paren, comma, full-stop, ...). Otherwise a long line + // like "MEDIA:/p with spaces.png and then more prose" would gobble the + // trailing narration into the "filename". + const sample = 'MEDIA:/tmp/my shot.png and then more prose'; + const refs = extractRawFilePaths(sample); + expect(refs.map((r) => r.filePath)).toEqual(['/tmp/my shot.png']); + }); + it('detects OpenClaw skill directories without file extensions', () => { const refs = extractRawFilePaths('位置： ~/.openclaw/skills/open-eastmoney'); expect(refs).toEqual([ diff --git a/tests/unit/chat-message.test.tsx b/tests/unit/chat-message.test.tsx index 1da5a90..d20aaf3 100644 --- a/tests/unit/chat-message.test.tsx +++ b/tests/unit/chat-message.test.tsx @@ -46,6 +46,62 @@ describe('ChatMessage attachment dedupe', () => { expect(screen.getByAltText('artifact.png')).toBeInTheDocument(); }); + it('keeps image artifacts visible alongside reply text when process attachments are suppressed', () => { + // Regression for media outgoing being silently dropped when the agent + // accompanies a `MEDIA:/path.png` artifact with any narration text: + // process-attachment filtering used to require PDF/XLSX/dir/skill but + // had no carve-out for images, so the file card never rendered. + const message: RawMessage = { + role: 'assistant', + content: 'Screenshot taken, sending it to you as an attachment.', + _attachedFiles: [ + { + fileName: 'desktop_screenshot.png', + mimeType: 'image/png', + fileSize: 1234, + preview: 'data:image/png;base64,abc', + filePath: '/tmp/desktop_screenshot.png', + source: 'tool-result', + }, + ], + }; + + render( + , + ); + + expect(screen.getByAltText('desktop_screenshot.png')).toBeInTheDocument(); + }); + + it('keeps message-ref image artifacts visible alongside reply text when process attachments are suppressed', () => { + const message: RawMessage = { + role: 'assistant', + content: 'Compressed, sending it to you:', + _attachedFiles: [ + { + fileName: 'desktop_screenshot.jpg', + mimeType: 'image/jpeg', + fileSize: 837_000, + preview: 'data:image/jpeg;base64,xyz', + filePath: '/tmp/desktop_screenshot.jpg', + source: 'message-ref', + }, + ], + }; + + render( + , + ); + + expect(screen.getByAltText('desktop_screenshot.jpg')).toBeInTheDocument(); + }); + it('keeps pdf and spreadsheet artifacts visible when process attachments are suppressed', () => { const message: RawMessage = { role: 'assistant',