d8dfun/clawX
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(chat): optimize image sending stability (#983)

Browse Source
This commit is contained in:
Haze
2026-05-07 10:57:30 +08:00
committed by GitHub
parent 0c8dc8cb72
commit 9ada49afb3
11 changed files with 964 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
electron/api/routes/files.ts
View File

@@ -75,6 +75,38 @@ async function generateImagePreview(filePath: string, mimeType: string): Promise
}
}
/**
* Resolve a Gateway-emitted outgoing-media URL to the original file on disk.
* Mirror of `electron/main/ipc-handlers.ts::resolveOutgoingMediaUrl` — kept
* in sync so the host-api HTTP path serves the same data as the IPC path.
*/
async function resolveOutgoingMediaUrl(
gatewayUrl: string,
): Promise<{ path: string; mimeType: string } | null> {
try {
const m = gatewayUrl.match(/\/api\/chat\/media\/outgoing\/[^/]+\/([^/]+)\//);
if (!m) return null;
const attachmentId = decodeURIComponent(m[1]);
if (!/^[A-Za-z0-9._-]+$/.test(attachmentId)) return null;
const recordPath = join(homedir(), '.openclaw', 'media', 'outgoing', 'records', `${attachmentId}.json`);
const fsP = await import('node:fs/promises');
const raw = await fsP.readFile(recordPath, 'utf8');
const record = JSON.parse(raw) as {
original?: { path?: string; contentType?: string };
};
const original = record?.original;
if (!original?.path) return null;
return {
path: original.path,
mimeType: typeof original.contentType === 'string' && original.contentType
? original.contentType
: 'application/octet-stream',
};
} catch {
return null;
}
}
export async function handleFileRoutes(
req: IncomingMessage,
res: ServerResponse,
@@ -137,18 +169,39 @@ export async function handleFileRoutes(
if (url.pathname === '/api/files/thumbnails' && req.method === 'POST') {
try {
const body = await parseJsonBody<{ paths: Array<{ filePath: string; mimeType: string }> }>(req);
const body = await parseJsonBody<{
paths: Array<{ filePath?: string; gatewayUrl?: string; mimeType: string }>;
}>(req);
const fsP = await import('node:fs/promises');
const results: Record<string, { preview: string | null; fileSize: number }> = {};
for (const { filePath, mimeType } of body.paths) {
try {
const s = await fsP.stat(filePath);
const preview = mimeType.startsWith('image/')
? await generateImagePreview(filePath, mimeType)
: null;
results[filePath] = { preview, fileSize: s.size };
} catch {
results[filePath] = { preview: null, fileSize: 0 };
for (const entry of body.paths) {
if (entry.filePath) {
try {
const s = await fsP.stat(entry.filePath);
const preview = entry.mimeType.startsWith('image/')
? await generateImagePreview(entry.filePath, entry.mimeType)
: null;
results[entry.filePath] = { preview, fileSize: s.size };
} catch {
results[entry.filePath] = { preview: null, fileSize: 0 };
}
continue;
}
if (entry.gatewayUrl) {
const resolved = await resolveOutgoingMediaUrl(entry.gatewayUrl);
if (!resolved) {
results[entry.gatewayUrl] = { preview: null, fileSize: 0 };
continue;
}
try {
const s = await fsP.stat(resolved.path);
const preview = resolved.mimeType.startsWith('image/')
? await generateImagePreview(resolved.path, resolved.mimeType)
: null;
results[entry.gatewayUrl] = { preview, fileSize: s.size };
} catch {
results[entry.gatewayUrl] = { preview: null, fileSize: 0 };
}
}
}
sendJson(res, 200, results);

88
electron/main/ipc-handlers.ts
View File

@@ -2496,25 +2496,93 @@ function registerFileHandlers(): void {
}
});
ipcMain.handle('media:getThumbnails', async (_, paths: Array<{ filePath: string; mimeType: string }>) => {
ipcMain.handle('media:getThumbnails', async (
_,
paths: Array<{ filePath?: string; gatewayUrl?: string; mimeType: string }>,
) => {
const fsP = await import('fs/promises');
const results: Record<string, { preview: string | null; fileSize: number }> = {};
for (const { filePath, mimeType } of paths) {
try {
const s = await fsP.stat(filePath);
let preview: string | null = null;
if (mimeType.startsWith('image/')) {
preview = await generateImagePreview(filePath, mimeType);
for (const entry of paths) {
// Local on-disk file (the original code path).
if (entry.filePath) {
try {
const s = await fsP.stat(entry.filePath);
let preview: string | null = null;
if (entry.mimeType.startsWith('image/')) {
preview = await generateImagePreview(entry.filePath, entry.mimeType);
}
results[entry.filePath] = { preview, fileSize: s.size };
} catch {
results[entry.filePath] = { preview: null, fileSize: 0 };
}
continue;
}
// Gateway-injected outgoing media URL. The renderer cannot reach the
// Gateway HTTP server directly (CORS / env drift), so we resolve it
// here against OpenClaw's local outgoing media records and load the
// original file off disk. The URL shape is fixed by OpenClaw:
// /api/chat/media/outgoing/<urlEncodedSessionKey>/<attachmentId>/full
if (entry.gatewayUrl) {
const resolved = await resolveOutgoingMediaUrl(entry.gatewayUrl);
if (!resolved) {
results[entry.gatewayUrl] = { preview: null, fileSize: 0 };
continue;
}
try {
const s = await fsP.stat(resolved.path);
let preview: string | null = null;
if (resolved.mimeType.startsWith('image/')) {
preview = await generateImagePreview(resolved.path, resolved.mimeType);
}
results[entry.gatewayUrl] = { preview, fileSize: s.size };
} catch {
results[entry.gatewayUrl] = { preview: null, fileSize: 0 };
}
results[filePath] = { preview, fileSize: s.size };
} catch {
results[filePath] = { preview: null, fileSize: 0 };
}
}
return results;
});
}
/**
* Resolve a Gateway-emitted outgoing-media URL to the original file on disk.
*
* OpenClaw's runtime stages every assistant `MEDIA:/path` artifact under
* `~/.openclaw/media/outgoing/`:
* - `originals/<uuid>.<ext>` — the source bytes copied verbatim
* - `records/<attachmentId>.json` — `{ original: { path, contentType, ... }, ... }`
*
* The Gateway then injects an `assistant-media` content block with
* `url:'/api/chat/media/outgoing/<urlEncodedSessionKey>/<attachmentId>/full'`.
* We only need the `<attachmentId>` segment to look up the record.
*/
async function resolveOutgoingMediaUrl(
gatewayUrl: string,
): Promise<{ path: string; mimeType: string } | null> {
try {
const m = gatewayUrl.match(/\/api\/chat\/media\/outgoing\/[^/]+\/([^/]+)\//);
if (!m) return null;
const attachmentId = decodeURIComponent(m[1]);
if (!/^[A-Za-z0-9._-]+$/.test(attachmentId)) return null;
const recordPath = join(homedir(), '.openclaw', 'media', 'outgoing', 'records', `${attachmentId}.json`);
const fsP = await import('fs/promises');
const raw = await fsP.readFile(recordPath, 'utf8');
const record = JSON.parse(raw) as {
original?: { path?: string; contentType?: string };
};
const original = record?.original;
if (!original?.path) return null;
return {
path: original.path,
mimeType: typeof original.contentType === 'string' && original.contentType
? original.contentType
: 'application/octet-stream',
};
} catch {
return null;
}
}
/**
* Session IPC handlers
*

9
src/pages/Chat/ChatMessage.tsx
View File

@@ -275,9 +275,12 @@ export const ChatMessage = memo(function ChatMessage({
});
const filteredProcessAttachments = derivedAttachedFiles.filter((file) => {
if (file.source !== 'tool-result' && file.source !== 'message-ref') return true;
// Runtime-produced PDF / spreadsheet artifacts should remain visible
// in the chat even when generic process attachments are folded into
// the execution graph; they are the user-facing output to click.
// Runtime-produced user-facing artifacts (images, PDFs, spreadsheets,
// skill directories, ...) must remain visible in the reply bubble even
// when generic process attachments are folded into the execution graph.
// The graph card itself does not render `_attachedFiles`, so dropping
// images here would leave the user with no way to see them at all.
if (file.mimeType.startsWith('image/')) return true;
return isChatPreviewDocument(file) || isDirectoryAttachment(file) || isSkillFileAttachment(file);
});
// When a message is attachment-only, keep those attachments visible even if

56
src/pages/Chat/message-utils.ts
View File

@@ -37,7 +37,12 @@ function cleanUserText(text: string): string {
function stripAssistantMediaTags(text: string): string {
if (!text) return text;
const exts = 'png|jpe?g|gif|webp|bmp|avif|svg|pdf|docx?|xlsx?|pptx?|txt|csv|md|rtf|epub|zip|tar|gz|rar|7z|mp3|wav|ogg|aac|flac|m4a|mp4|mov|avi|mkv|webm|m4v';
const tagged = new RegExp(`(^|[\\s(\\[{>])(?:MEDIA|media):(?:\\/|~\\/)[^\\s\\n"'()\\[\\],<>]*?\\.(?:${exts})`, 'g');
// Mirror the relaxed character class in `chat/helpers.ts::extractRawFilePaths`
// so paths with ASCII spaces (e.g. macOS' "截屏 2026-05-06 17.46.51.png")
// are also stripped from the visible bubble. Without this, the bubble
// would still leak the literal `MEDIA:/.../截屏 2026-05-06 17.46.51.png`
// to the user when the underlying path detection succeeds.
const tagged = new RegExp(`(^|[\\s(\\[{>])(?:MEDIA|media):(?:\\/|~\\/)[^\\n"'()\\[\\],<>]*?\\.(?:${exts})(?=$|[\\s\\n"'()\\[\\],<>]|[，。；;,.!?])`, 'g');
return text
.replace(tagged, (_, lead: string) => lead)
// Collapse the empty lines / orphan whitespace the strip leaves behind.
@@ -302,31 +307,50 @@ export function extractMediaRefs(message: RawMessage | unknown): Array<{ filePat
}
/**
* Extract image attachments from a message.
* Returns array of { mimeType, data } for base64 images.
* Extract inline image attachments from a message.
*
* Returns either:
* - `{ mimeType, data }` — base64 bytes inline (Anthropic / Gateway tool-result)
* - `{ mimeType, url }` — Anthropic source-wrapped remote URL form
*
* Note: the Gateway-injected `assistant-media` shape — flat
* `{ type:'image', url:'/api/chat/media/outgoing/...', mimeType, ... }` —
* is intentionally NOT extracted here because the URL is a Gateway-relative
* path the renderer cannot fetch directly (CORS / env drift). That shape
* is surfaced through `_attachedFiles` instead (see
* `extractImagesAsAttachedFiles` and `enrichWithGatewayMediaBlocks` in
* `src/stores/chat/helpers.ts`), and resolved to a local preview by
* `loadMissingPreviews` -> Main `media:getThumbnails`.
*/
export function extractImages(message: RawMessage | unknown): Array<{ mimeType: string; data: string }> {
export function extractImages(
message: RawMessage | unknown,
): Array<{ mimeType: string; data?: string; url?: string }> {
if (!message || typeof message !== 'object') return [];
const msg = message as Record<string, unknown>;
const content = msg.content;
if (!Array.isArray(content)) return [];
const images: Array<{ mimeType: string; data: string }> = [];
const images: Array<{ mimeType: string; data?: string; url?: string }> = [];
for (const block of content as ContentBlock[]) {
if (block.type === 'image') {
// Path 1: Anthropic source-wrapped format
if (block.source) {
const src = block.source;
if (src.type === 'base64' && src.media_type && src.data) {
images.push({ mimeType: src.media_type, data: src.data });
}
}
// Path 2: Flat format from Gateway tool results {data, mimeType}
else if (block.data) {
images.push({ mimeType: block.mimeType || 'image/jpeg', data: block.data });
if (block.type !== 'image') continue;
// Path 1: Anthropic source-wrapped format
if (block.source) {
const src = block.source;
if (src.type === 'base64' && src.media_type && src.data) {
images.push({ mimeType: src.media_type, data: src.data });
} else if (src.type === 'url' && src.url) {
images.push({ mimeType: src.media_type || 'image/jpeg', url: src.url });
}
continue;
}
// Path 2: Flat format from Gateway tool results {data, mimeType}
if (block.data) {
images.push({ mimeType: block.mimeType || 'image/jpeg', data: block.data });
}
// Flat `block.url` is intentionally NOT handled here; see comment above.
}
return images;

304
src/stores/chat.ts
View File

@@ -434,11 +434,43 @@ function trimPathTerminators(filePath: string): string {
* Extract raw file paths from message text.
* Detects absolute paths (Unix: / or ~/, Windows: C:\ etc.) ending with common file extensions.
* Handles both image and non-image files, consistent with channel push message behavior.
*
* Also recognises the `MEDIA:` / `media:` prefix the OpenClaw runtime
* emits for produced artifacts (e.g.
* `MEDIA:/tmp/desktop_screenshot.png`) — without this the leading colon
* trips the URL guard on the unix regex below and the artifact never
* surfaces as an attachment. Mirrors `chat/helpers.ts::extractRawFilePaths`.
*/
function extractRawFilePaths(text: string): Array<{ filePath: string; mimeType: string }> {
const refs: Array<{ filePath: string; mimeType: string }> = [];
const seen = new Set<string>();
const exts = 'png|jpe?g|gif|webp|bmp|avif|svg|pdf|docx?|xlsx?|pptx?|txt|csv|md|rtf|epub|zip|tar|gz|rar|7z|mp3|wav|ogg|aac|flac|m4a|mp4|mov|avi|mkv|webm|m4v';
// Tagged media references (MEDIA:/path, media:~/path, ...). The agent
// runtime uses this prefix as an explicit "this is an artifact" marker,
// so we want them recognised even though the leading colon would
// normally look like a URL scheme. After matching we punch the entire
// `MEDIA:<path>` span out of the working text so the generic unix
// regex below doesn't double-count the bare `/path` suffix.
// The character class deliberately allows ASCII spaces inside the path so
// that macOS' default screenshot filename ("截屏 2026-05-06 17.46.51.png")
// and other space-containing paths the agent emits with the explicit
// `MEDIA:` marker still resolve. Newline and quote characters remain
// path terminators so we don't accidentally swallow trailing prose.
const taggedRegex = new RegExp(`(?:^|[\\s(\\[{>])(?:MEDIA|media):((?:\\/|~\\/)[^\\n"'()\\[\\],<>` + '`' + `]*?\\.(?:${exts}))(?=$|[\\s\\n"'()\\[\\],<>` + '`' + `]|[，。；;,.!?])`, 'g');
let workingText = text;
let taggedMatch: RegExpExecArray | null;
while ((taggedMatch = taggedRegex.exec(text)) !== null) {
const p = taggedMatch[1];
if (p && !seen.has(p)) {
seen.add(p);
refs.push({ filePath: p, mimeType: mimeFromExtension(p) });
}
// Mask the matched span so subsequent regexes can't re-discover the
// same path (e.g. `/two.xlsx` from `MEDIA:~/two.xlsx`).
const start = taggedMatch.index;
const end = start + taggedMatch[0].length;
workingText = workingText.slice(0, start) + ' '.repeat(end - start) + workingText.slice(end);
}
// Unix absolute paths (/... or ~/...) — lookbehind rejects mid-token slashes
// (e.g. "path/to/file.mp4", "https://example.com/file.mp4")
const unixRegex = new RegExp(`(?<![\\w./:])((?:\\/|~\\/)[^\\s\\n"'()\`\\[\\],<>]*?\\.(?:${exts}))`, 'gi');
@@ -453,7 +485,7 @@ function extractRawFilePaths(text: string): Array<{ filePath: string; mimeType:
);
for (const regex of [unixRegex, winRegex, skillDirRegex]) {
let match;
while ((match = regex.exec(text)) !== null) {
while ((match = regex.exec(workingText)) !== null) {
const p = trimPathTerminators(match[1]);
if (p && !seen.has(p)) {
seen.add(p);
@@ -508,6 +540,22 @@ function extractImagesAsAttachedFiles(content: unknown): AttachedFileMeta[] {
preview: `data:${mimeType};base64,${block.data}`,
});
}
// Path 3: Flat URL form from Gateway-injected assistant-media messages.
// See `src/stores/chat/helpers.ts` for the canonical implementation.
else if (block.url) {
const mimeType = block.mimeType || 'image/jpeg';
const fileName = typeof block.alt === 'string' && block.alt
? block.alt
: 'image';
files.push({
fileName,
mimeType,
fileSize: 0,
preview: null,
gatewayUrl: block.url,
source: 'gateway-media',
});
}
}
// Recurse into tool_result content blocks
if ((block.type === 'tool_result' || block.type === 'toolResult') && block.content) {
@@ -627,8 +675,15 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
// Resolve file path from the matching tool call
const matchedPath = msg.toolCallId ? toolCallPaths.get(msg.toolCallId) : undefined;
// 1. Image/file content blocks in the structured content array
const imageFiles = extractImagesAsAttachedFiles(msg.content);
// 1. Image/file content blocks in the structured content array.
// Images embedded inside a tool result are the model's vision data
// (e.g. `read /tmp/foo.png` re-encoded as JPEG so the model can "see"
// the file) — they are NOT user-facing artifacts. The agent surfaces
// user-facing images through `MEDIA:/path` text + the Gateway's
// `assistant-media` injection. Surfacing the vision data here would
// duplicate every screenshot the agent inspects.
const imageFiles = extractImagesAsAttachedFiles(msg.content)
.filter(file => !file.mimeType.startsWith('image/'));
if (matchedPath) {
for (const f of imageFiles) {
if (!f.filePath) {
@@ -650,11 +705,17 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
for (const ref of mediaRefs) {
pending.push({ ...makeAttachedFile(ref), source: 'tool-result' });
}
// 3. Raw file paths in tool result text (documents, audio, video, etc.)
// 3. Raw NON-image file paths in tool result text (documents,
// audio, video, ...). Image paths are deliberately ignored:
// `ls -la /tmp/foo.png`, `sips ... && ls -la *.jpg`, etc.
// spam intermediate paths that the user does not want to see
// rendered as separate cards. The canonical user-facing image
// is whatever the agent later emits via `MEDIA:/path` (which
// the Gateway turns into a dedicated assistant-media bubble).
for (const ref of extractRawFilePaths(text)) {
if (!mediaRefPaths.has(ref.filePath)) {
pending.push({ ...makeAttachedFile(ref), source: 'tool-result' });
}
if (mediaRefPaths.has(ref.filePath)) continue;
if (ref.mimeType.startsWith('image/')) continue;
pending.push({ ...makeAttachedFile(ref), source: 'tool-result' });
}
}
@@ -687,11 +748,30 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
* Uses local cache for previews when available; missing previews are loaded async.
*/
function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
// Pre-compute, per index, whether the *next* assistant message is a
// Gateway-injected `assistant-media` bubble (i.e. has at least one
// `image` content block carrying a flat URL). When that bubble exists,
// the canonical user-facing rendering of the artifact is the bubble
// itself — anything the agent emitted via `MEDIA:/path` in its prior
// text turn would just duplicate the same image, so image-typed raw
// refs on that prior message should be dropped here.
const nextHasGatewayMediaBubble = messages.map((_, idx) => {
const next = messages[idx + 1];
if (!next || next.role !== 'assistant') return false;
return extractImagesAsAttachedFiles(next.content).some(f => f.gatewayUrl);
});
return messages.map((msg, idx) => {
// Only process user and assistant messages; skip if already enriched
if ((msg.role !== 'user' && msg.role !== 'assistant') || msg._attachedFiles) return msg;
// Only process user and assistant messages.
if (msg.role !== 'user' && msg.role !== 'assistant') return msg;
const text = getMessageText(msg.content);
// Path 0: Gateway-injected outgoing media on this same message
// (an `assistant-media` bubble — image block with flat `url`).
const gatewayMediaFiles: AttachedFileMeta[] = msg.role === 'assistant'
? extractImagesAsAttachedFiles(msg.content).filter(file => file.gatewayUrl)
: [];
// Path 1: [media attached: path (mime) | path] — guaranteed format from attachment button
const mediaRefs = extractMediaRefs(text);
const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath));
@@ -699,9 +779,6 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
// Path 2: Raw file paths.
// For assistant messages: scan own text AND the nearest preceding user message text,
// but only for non-tool-only assistant messages (i.e. the final answer turn).
// Tool-only messages (thinking + tool calls) should not show file previews — those
// belong to the final answer message that comes after the tool results.
// User messages never get raw-path previews so the image is not shown twice.
let rawRefs: Array<{ filePath: string; mimeType: string }> = [];
if (msg.role === 'assistant' && !isToolOnlyMessage(msg)) {
// Own text
@@ -725,16 +802,39 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
}
}
const allRefs = [...mediaRefs, ...rawRefs];
if (allRefs.length === 0) return msg;
// Dedup vs Gateway-injected bubble: if the very next assistant message
// is a Gateway `assistant-media` bubble, drop image-typed raw refs on
// *this* message — the bubble already covers them.
if (msg.role === 'assistant' && nextHasGatewayMediaBubble[idx]) {
rawRefs = rawRefs.filter(r => !r.mimeType.startsWith('image/'));
}
const files: AttachedFileMeta[] = allRefs.map(ref => {
const cached = _imageCache.get(ref.filePath);
if (cached) return { ...cached, filePath: ref.filePath, source: 'message-ref' as const };
const fileName = ref.filePath.split(/[\\/]/).pop() || 'file';
return { fileName, mimeType: ref.mimeType, fileSize: 0, preview: null, filePath: ref.filePath, source: 'message-ref' as const };
});
return { ...msg, _attachedFiles: files };
const allRefs = [...mediaRefs, ...rawRefs];
if (allRefs.length === 0 && gatewayMediaFiles.length === 0) {
// Preserve any previously-attached `_attachedFiles` (e.g. set by
// `enrichWithToolResultFiles` for non-image artifacts). When nothing
// new applies, returning `msg` unmodified keeps those attachments.
return msg;
}
const existingFiles = msg._attachedFiles || [];
const existingPaths = new Set(existingFiles.map(file => file.filePath).filter(Boolean));
const existingGatewayUrls = new Set(
existingFiles.map(file => file.gatewayUrl).filter(Boolean) as string[],
);
const files: AttachedFileMeta[] = allRefs
.filter(ref => !existingPaths.has(ref.filePath))
.map(ref => {
const cached = _imageCache.get(ref.filePath);
if (cached) return { ...cached, filePath: ref.filePath, source: 'message-ref' as const };
const fileName = ref.filePath.split(/[\\/]/).pop() || 'file';
return { fileName, mimeType: ref.mimeType, fileSize: 0, preview: null, filePath: ref.filePath, source: 'message-ref' as const };
});
const dedupedGatewayMedia = gatewayMediaFiles.filter(
file => file.gatewayUrl && !existingGatewayUrls.has(file.gatewayUrl),
);
if (files.length === 0 && dedupedGatewayMedia.length === 0) return msg;
return { ...msg, _attachedFiles: [...existingFiles, ...files, ...dedupedGatewayMedia] };
});
}
@@ -744,24 +844,29 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
* Handles both [media attached: ...] patterns and raw filePath entries.
*/
async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
// Collect all image paths that need previews
const needPreview: Array<{ filePath: string; mimeType: string }> = [];
const seenPaths = new Set<string>();
// See helpers.ts loadMissingPreviews for the canonical comment block —
// this monolithic copy is kept in sync so legacy chat.ts callers also
// resolve Gateway-injected outgoing media URLs into local previews.
type PreviewRef = { filePath?: string; gatewayUrl?: string; mimeType: string };
const needPreview: PreviewRef[] = [];
const seenKeys = new Set<string>();
for (const msg of messages) {
if (!msg._attachedFiles) continue;
// Path 1: files with explicit filePath field (raw path detection or enriched refs)
// Path 1: files with explicit filePath OR gatewayUrl
for (const file of msg._attachedFiles) {
const fp = file.filePath;
if (!fp || seenPaths.has(fp)) continue;
// Images: need preview. Non-images: need file size (for FileCard display).
const key = file.filePath || file.gatewayUrl;
if (!key || seenKeys.has(key)) continue;
const needsLoad = file.mimeType.startsWith('image/')
? !file.preview
: file.fileSize === 0;
if (needsLoad) {
seenPaths.add(fp);
needPreview.push({ filePath: fp, mimeType: file.mimeType });
if (!needsLoad) continue;
seenKeys.add(key);
if (file.filePath) {
needPreview.push({ filePath: file.filePath, mimeType: file.mimeType });
} else if (file.gatewayUrl) {
needPreview.push({ gatewayUrl: file.gatewayUrl, mimeType: file.mimeType });
}
}
@@ -772,17 +877,19 @@ async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
for (let i = 0; i < refs.length; i++) {
const file = msg._attachedFiles[i];
const ref = refs[i];
if (!file || !ref || seenPaths.has(ref.filePath)) continue;
if (!file || !ref || seenKeys.has(ref.filePath)) continue;
const needsLoad = ref.mimeType.startsWith('image/') ? !file.preview : file.fileSize === 0;
if (needsLoad) {
seenPaths.add(ref.filePath);
needPreview.push(ref);
seenKeys.add(ref.filePath);
needPreview.push({ filePath: ref.filePath, mimeType: ref.mimeType });
}
}
}
}
if (needPreview.length === 0) return false;
if (needPreview.length === 0) {
return false;
}
try {
const thumbnails = await hostApiFetch<Record<string, { preview: string | null; fileSize: number }>>(
@@ -797,15 +904,17 @@ async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
for (const msg of messages) {
if (!msg._attachedFiles) continue;
// Update files that have filePath
// Update files that have filePath OR gatewayUrl
for (const file of msg._attachedFiles) {
const fp = file.filePath;
if (!fp) continue;
const thumb = thumbnails[fp];
const key = file.filePath || file.gatewayUrl;
if (!key) continue;
const thumb = thumbnails[key];
if (thumb && (thumb.preview || thumb.fileSize)) {
if (thumb.preview) file.preview = thumb.preview;
if (thumb.fileSize) file.fileSize = thumb.fileSize;
_imageCache.set(fp, { ...file });
if (file.filePath) {
_imageCache.set(file.filePath, { ...file });
}
updated = true;
}
}
@@ -1004,11 +1113,35 @@ function isToolResultRole(role: unknown): boolean {
}
/** True for internal plumbing messages that should never be shown in the UI. */
function isInternalMessage(msg: { role?: unknown; content?: unknown }): boolean {
function isInternalMessage(msg: { role?: unknown; content?: unknown; idempotencyKey?: unknown; model?: unknown }): boolean {
if (msg.role === 'system') return true;
const text = getMessageText(msg.content);
if (msg.role === 'assistant') {
if (/^(HEARTBEAT_OK|NO_REPLY)\s*$/.test(text)) return true;
// OpenClaw's gateway writes a fallback `assistant-media` transcript
// message when its `createManagedOutgoingImageBlocks` pipeline fails
// ("could not be prepared" warning in stderr). The fallback has:
// - model: 'gateway-injected'
// - idempotencyKey: '<runId>:assistant-media'
// - content: text-only `MEDIA:<staging path>` (NOT an image-url block)
// The staging path lives in `~/.openclaw/media/outbound/` which has a
// 120s TTL — the file is gone by the time the user reads the chat.
// The original `MEDIA:/tmp/...` path is already on the previous
// assistant message (the agent's actual reply), so the fallback is
// pure duplicate noise. Hide it in the UI so it neither shows a
// broken card nor competes with the real reply for layout space.
const idempotencyKey = typeof msg.idempotencyKey === 'string' ? msg.idempotencyKey : '';
const isGatewayInjectedFallback = msg.model === 'gateway-injected'
&& idempotencyKey.endsWith(':assistant-media');
if (isGatewayInjectedFallback) {
const hasImageUrlBlock = Array.isArray(msg.content)
&& (msg.content as ContentBlock[]).some(
(block) => block.type === 'image' && typeof block.url === 'string' && !!block.url,
);
// Real gateway-media bubbles (with an image-url block) ARE the
// canonical render — keep them. Only hide the text-only fallback.
if (!hasImageUrlBlock) return true;
}
}
// Runtime system injections: these arrive as user or assistant-role messages
// but are internal plumbing (exec results, async-command notices, time pings, etc.)
@@ -2166,12 +2299,18 @@ export const useChatStore = create<ChatState>((set, get) => ({
const { activeRunId, currentSessionKey } = get();
// Only process events for the current session (when sessionKey is present)
if (eventSessionKey != null && eventSessionKey !== currentSessionKey) return;
if (eventSessionKey != null && eventSessionKey !== currentSessionKey) {
return;
}
// Only process events for the active run (or if no active run set)
if (activeRunId && runId && runId !== activeRunId) return;
if (activeRunId && runId && runId !== activeRunId) {
return;
}
if (isDuplicateChatEvent(eventState, event)) return;
if (isDuplicateChatEvent(eventState, event)) {
return;
}
_lastChatEventAt = Date.now();
@@ -2285,10 +2424,17 @@ export const useChatStore = create<ChatState>((set, get) => ({
? getToolCallFilePath(currentStreamForPath, normalizedFinalMessage.toolCallId)
: undefined;
// Mirror enrichWithToolResultFiles: collect images + file refs for next assistant msg
const toolFiles: AttachedFileMeta[] = [
...extractImagesAsAttachedFiles(normalizedFinalMessage.content),
];
// Mirror `enrichWithToolResultFiles`: collect non-image artifacts
// for the next assistant message. Images embedded inside a tool
// result (read tool's vision data) and raw image paths in the
// tool's stdout (sips / ls / file output) are NOT user-facing —
// the canonical render is the Gateway-injected `assistant-media`
// bubble that follows the agent's `MEDIA:` text. Surfacing those
// intermediate images here would duplicate every screenshot the
// agent inspects on its way to the final artifact.
const toolFiles: AttachedFileMeta[] = extractImagesAsAttachedFiles(
normalizedFinalMessage.content,
).filter(file => !file.mimeType.startsWith('image/'));
if (matchedPath) {
for (const f of toolFiles) {
if (!f.filePath) {
@@ -2303,7 +2449,9 @@ export const useChatStore = create<ChatState>((set, get) => ({
const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath));
for (const ref of mediaRefs) toolFiles.push(makeAttachedFile(ref));
for (const ref of extractRawFilePaths(text)) {
if (!mediaRefPaths.has(ref.filePath)) toolFiles.push(makeAttachedFile(ref));
if (mediaRefPaths.has(ref.filePath)) continue;
if (ref.mimeType.startsWith('image/')) continue;
toolFiles.push(makeAttachedFile(ref));
}
}
set((s) => {
@@ -2334,7 +2482,17 @@ export const useChatStore = create<ChatState>((set, get) => ({
const nextTools = updates.length > 0 ? upsertToolStatuses(s.streamingTools, updates) : s.streamingTools;
const streamingTools = hasOutput ? [] : nextTools;
// Attach any images collected from preceding tool results
// Note: it would be tempting to also surface `MEDIA:/path`
// markers from `normalizedFinalMessage.content`'s text here, so
// the agent's reply could attach the original file directly
// (`/tmp/...png`) without waiting for the post-final history
// reload. However, OpenClaw's `splitTrailingDirective`
// (selection-D8_ELZa7.js ~line 904) strips `MEDIA:/...` lines
// out of the streaming text BEFORE it reaches the client, so
// the `final` event we get here never contains the marker.
// Image surfacing is fully handled by the post-final reload
// below + `enrichWithCachedImages` (which dereferences the
// assistant-media bubble's `block.url`).
const pendingImgs = s.pendingToolImages;
const msgWithImages: RawMessage = pendingImgs.length > 0
? {
@@ -2388,6 +2546,52 @@ export const useChatStore = create<ChatState>((set, get) => ({
if (hasOutput && !toolOnly) {
clearHistoryPoll();
void get().loadHistory(true);
// OpenClaw's gateway processes `MEDIA:/path` markers in the
// assistant reply asynchronously, in the `dispatch.deliver` of
// the `final` payload (see openclaw/dist/chat-DM9hSaNV.js's
// `appendWebchatAgentMediaTranscriptIfNeeded`):
// 1. copy the original file under
// `~/.openclaw/media/outgoing/originals/<uuid>`
// 2. write the record JSON under
// `~/.openclaw/media/outgoing/records/<id>.json`
// 3. `appendAssistantTranscriptMessage` writes a follow-up
// `assistant-media` message to the session JSONL, with
// `idempotencyKey: "<runId>:assistant-media"`.
// That follow-up message is **only persisted** — it is NOT
// re-broadcast as a streaming event. The streaming `final`
// we just consumed only contains the agent's text. The
// assistant-media bubble can only be retrieved via
// `chat.history`, and the persistence runs on the order of
// ~400-500ms after the streaming final.
//
// The immediate `loadHistory(true)` above therefore races the
// gateway's write and almost always misses the bubble.
//
// CRITICAL: we cannot detect from the streaming final alone
// whether the agent emitted a `MEDIA:/path` marker — OpenClaw's
// `splitTrailingDirective` (selection-D8_ELZa7.js line ~904)
// strips `MEDIA:/...` lines from the broadcast text BEFORE it
// reaches the client, so the streaming `final` text is always
// the user-facing prose without the marker. The MEDIA: marker
// only appears in the persisted JSONL transcript (msg N) and
// its companion `assistant-media` bubble (msg N+1).
//
// We therefore unconditionally schedule ONE follow-up quiet
// reload ~1500ms after every assistant `final`. The cost is
// a single extra in-process RPC per assistant turn (cheap);
// when there's no media the second reload returns the same
// history snapshot and is a no-op for the UI.
// `forceNextHistoryLoad` bypasses `HISTORY_LOAD_MIN_INTERVAL_MS`
// so the call is not suppressed by the throttle.
const sessionKeyAtFinal = get().currentSessionKey;
setTimeout(() => {
if (get().currentSessionKey !== sessionKeyAtFinal) {
return;
}
forceNextHistoryLoad(sessionKeyAtFinal);
void get().loadHistory(true);
}, 1500);
}
} else {
// No message in final event - reload history to get complete data

141
src/stores/chat/helpers.ts
View File

@@ -360,7 +360,14 @@ function extractRawFilePaths(text: string): Array<{ filePath: string; mimeType:
// normally look like a URL scheme. After matching we punch the entire
// `MEDIA:<path>` span out of the working text so the generic unix
// regex below doesn't double-count the bare `/path` suffix.
const taggedRegex = new RegExp(`(?:^|[\\s(\\[{>])(?:MEDIA|media):((?:\\/|~\\/)[^\\s\\n"'()\\[\\],<>` + '`' + `]*?\\.(?:${exts}))`, 'g');
// The character class deliberately allows ASCII spaces inside the path so
// that macOS' default screenshot filename ("截屏 2026-05-06 17.46.51.png")
// and other space-containing paths the agent emits with the explicit
// `MEDIA:` marker still resolve. Newline and quote characters remain
// path terminators so we don't accidentally swallow trailing prose.
// The non-greedy `*?` anchored to `\.<ext>` keeps the match minimal so
// multiple `MEDIA:` markers in one paragraph still match independently.
const taggedRegex = new RegExp(`(?:^|[\\s(\\[{>])(?:MEDIA|media):((?:\\/|~\\/)[^\\n"'()\\[\\],<>` + '`' + `]*?\\.(?:${exts}))(?=$|[\\s\\n"'()\\[\\],<>` + '`' + `]|[，。；;,.!?])`, 'g');
let workingText = text;
let taggedMatch: RegExpExecArray | null;
while ((taggedMatch = taggedRegex.exec(text)) !== null) {
@@ -446,6 +453,26 @@ function extractImagesAsAttachedFiles(content: unknown): AttachedFileMeta[] {
preview: `data:${mimeType};base64,${block.data}`,
});
}
// Path 3: Flat URL form from Gateway-injected assistant-media messages.
// Shape: `{ type:'image', url:'/api/chat/media/outgoing/<sessionKey>/<id>/full',
// mimeType, width, height, alt, openUrl }`. The URL is relative
// to the Gateway HTTP server which the renderer cannot reach directly
// (CORS / env drift). We surface it as an `_attachedFiles` entry whose
// preview is filled in later by `loadMissingPreviews` -> Main proxy.
else if (block.url) {
const mimeType = block.mimeType || 'image/jpeg';
const fileName = typeof block.alt === 'string' && block.alt
? block.alt
: 'image';
files.push({
fileName,
mimeType,
fileSize: 0,
preview: null,
gatewayUrl: block.url,
source: 'gateway-media',
});
}
}
// Recurse into tool_result content blocks
if ((block.type === 'tool_result' || block.type === 'toolResult') && block.content) {
@@ -568,8 +595,14 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
// Resolve file path from the matching tool call
const matchedPath = msg.toolCallId ? toolCallPaths.get(msg.toolCallId) : undefined;
// 1. Image/file content blocks in the structured content array
const imageFiles = extractImagesAsAttachedFiles(msg.content);
// 1. Image/file content blocks in the structured content array.
// Images embedded inside a tool result are the model's vision data
// (e.g. `read /tmp/foo.png` re-encoded as JPEG so the model can
// "see" the file) — they are NOT user-facing artifacts. The agent
// surfaces user-facing images through `MEDIA:/path` text + the
// Gateway's `assistant-media` injection.
const imageFiles = extractImagesAsAttachedFiles(msg.content)
.filter(file => !file.mimeType.startsWith('image/'));
if (matchedPath) {
for (const f of imageFiles) {
if (!f.filePath) {
@@ -588,11 +621,14 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
for (const ref of mediaRefs) {
pending.push(makeAttachedFile(ref, 'tool-result'));
}
// 3. Raw file paths in tool result text (documents, audio, video, etc.)
// 3. Raw NON-image file paths in tool result text (documents,
// audio, video, ...). Image paths from intermediate tool stdout
// (`ls -la *.png`, `sips ... && ls`, `file /tmp/x.png`, etc.)
// are deliberately ignored — see comment on Path 1.
for (const ref of extractRawFilePaths(text)) {
if (!mediaRefPaths.has(ref.filePath)) {
pending.push(makeAttachedFile(ref, 'tool-result'));
}
if (mediaRefPaths.has(ref.filePath)) continue;
if (ref.mimeType.startsWith('image/')) continue;
pending.push(makeAttachedFile(ref, 'tool-result'));
}
}
@@ -625,6 +661,19 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
* Uses local cache for previews when available; missing previews are loaded async.
*/
function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
// Pre-compute, per index, whether the *next* assistant message is a
// Gateway-injected `assistant-media` bubble (i.e. has at least one
// `image` content block carrying a flat URL). When that bubble exists,
// the canonical user-facing rendering of the artifact is the bubble
// itself — anything the agent emitted via `MEDIA:/path` in its prior
// text turn would just duplicate the same image, so image-typed raw
// refs on that prior message are dropped here.
const nextHasGatewayMediaBubble = messages.map((_, idx) => {
const next = messages[idx + 1];
if (!next || next.role !== 'assistant') return false;
return extractImagesAsAttachedFiles(next.content).some(f => f.gatewayUrl);
});
return messages.map((msg, idx) => {
// Only process user and assistant messages. Messages may already carry
// attachments from tool-result enrichment; still merge in raw paths from
@@ -632,6 +681,16 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
if (msg.role !== 'user' && msg.role !== 'assistant') return msg;
const text = getMessageText(msg.content);
// Path 0: Gateway-injected outgoing media — `image` content blocks with
// a flat `url` field (e.g. `/api/chat/media/outgoing/<sessionKey>/<id>/full`).
// The renderer cannot fetch the URL directly, so we surface it as an
// `_attachedFiles` entry whose preview is filled in later by
// `loadMissingPreviews` -> Main `media:getThumbnails` (which dereferences
// the URL to the original file in `~/.openclaw/media/outgoing/`).
const gatewayMediaFiles: AttachedFileMeta[] = msg.role === 'assistant'
? extractImagesAsAttachedFiles(msg.content).filter(file => file.gatewayUrl)
: [];
// Path 1: [media attached: path (mime) | path] — guaranteed format from attachment button
const mediaRefs = extractMediaRefs(text);
const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath));
@@ -665,11 +724,21 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
}
}
// Dedup vs Gateway-injected bubble: when the very next assistant
// message is an `assistant-media` bubble, drop image-typed raw refs
// on *this* message — the bubble already covers the artifact.
if (msg.role === 'assistant' && nextHasGatewayMediaBubble[idx]) {
rawRefs = rawRefs.filter(r => !r.mimeType.startsWith('image/'));
}
const allRefs = [...mediaRefs, ...rawRefs];
if (allRefs.length === 0) return msg;
if (allRefs.length === 0 && gatewayMediaFiles.length === 0) return msg;
const existingFiles = msg._attachedFiles || [];
const existingPaths = new Set(existingFiles.map(file => file.filePath).filter(Boolean));
const existingGatewayUrls = new Set(
existingFiles.map(file => file.gatewayUrl).filter(Boolean) as string[],
);
const files: AttachedFileMeta[] = allRefs
.filter(ref => !existingPaths.has(ref.filePath))
.map(ref => {
@@ -678,8 +747,11 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
const fileName = ref.filePath.split(/[\\/]/).pop() || 'file';
return { fileName, mimeType: ref.mimeType, fileSize: 0, preview: null, filePath: ref.filePath, source: 'message-ref' };
});
if (files.length === 0) return msg;
return { ...msg, _attachedFiles: [...existingFiles, ...files] };
const dedupedGatewayMedia = gatewayMediaFiles.filter(
file => file.gatewayUrl && !existingGatewayUrls.has(file.gatewayUrl),
);
if (files.length === 0 && dedupedGatewayMedia.length === 0) return msg;
return { ...msg, _attachedFiles: [...existingFiles, ...files, ...dedupedGatewayMedia] };
});
}
@@ -689,24 +761,34 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
* Handles both [media attached: ...] patterns and raw filePath entries.
*/
async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
// Collect all image paths that need previews
const needPreview: Array<{ filePath: string; mimeType: string }> = [];
const seenPaths = new Set<string>();
// Collect all image refs that need previews. The IPC handler accepts:
// - { filePath, mimeType } — local on-disk files
// - { gatewayUrl, mimeType } — Gateway-injected outgoing media; the
// handler resolves the URL to a local file
// via `~/.openclaw/media/outgoing/records/`.
// We use `filePath || gatewayUrl` as the dedupe / lookup key on the way
// back; a file always carries at most one of the two.
type PreviewRef = { filePath?: string; gatewayUrl?: string; mimeType: string };
const needPreview: PreviewRef[] = [];
const seenKeys = new Set<string>();
for (const msg of messages) {
if (!msg._attachedFiles) continue;
// Path 1: files with explicit filePath field (raw path detection or enriched refs)
// Path 1: files with explicit filePath OR gatewayUrl
for (const file of msg._attachedFiles) {
const fp = file.filePath;
if (!fp || seenPaths.has(fp)) continue;
const key = file.filePath || file.gatewayUrl;
if (!key || seenKeys.has(key)) continue;
// Images: need preview. Non-images: need file size (for FileCard display).
const needsLoad = file.mimeType.startsWith('image/')
? !file.preview
: file.fileSize === 0;
if (needsLoad) {
seenPaths.add(fp);
needPreview.push({ filePath: fp, mimeType: file.mimeType });
if (!needsLoad) continue;
seenKeys.add(key);
if (file.filePath) {
needPreview.push({ filePath: file.filePath, mimeType: file.mimeType });
} else if (file.gatewayUrl) {
needPreview.push({ gatewayUrl: file.gatewayUrl, mimeType: file.mimeType });
}
}
@@ -717,11 +799,11 @@ async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
for (let i = 0; i < refs.length; i++) {
const file = msg._attachedFiles[i];
const ref = refs[i];
if (!file || !ref || seenPaths.has(ref.filePath)) continue;
if (!file || !ref || seenKeys.has(ref.filePath)) continue;
const needsLoad = ref.mimeType.startsWith('image/') ? !file.preview : file.fileSize === 0;
if (needsLoad) {
seenPaths.add(ref.filePath);
needPreview.push(ref);
seenKeys.add(ref.filePath);
needPreview.push({ filePath: ref.filePath, mimeType: ref.mimeType });
}
}
}
@@ -739,15 +821,20 @@ async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
for (const msg of messages) {
if (!msg._attachedFiles) continue;
// Update files that have filePath
// Update files that have filePath OR gatewayUrl
for (const file of msg._attachedFiles) {
const fp = file.filePath;
if (!fp) continue;
const thumb = thumbnails[fp];
const key = file.filePath || file.gatewayUrl;
if (!key) continue;
const thumb = thumbnails[key];
if (thumb && (thumb.preview || thumb.fileSize)) {
if (thumb.preview) file.preview = thumb.preview;
if (thumb.fileSize) file.fileSize = thumb.fileSize;
_imageCache.set(fp, { ...file });
// Only persist local-path entries to the localStorage cache.
// Gateway outgoing URLs are tied to a specific session/attachment
// id and can be stale across runs, so caching is harmful.
if (file.filePath) {
_imageCache.set(file.filePath, { ...file });
}
updated = true;
}
}

11
src/stores/chat/runtime-event-handlers.ts
View File

@@ -119,8 +119,13 @@ export function handleRuntimeEventState(
? getToolCallFilePath(currentStreamForPath, normalizedFinalMessage.toolCallId)
: undefined;
// Mirror enrichWithToolResultFiles: collect images + file refs for next assistant msg
// Mirror `enrichWithToolResultFiles`: collect non-image
// artifacts for the next assistant message. Image content
// blocks (vision data) and image-typed raw paths in the
// tool's stdout are intermediate process noise — see the
// comment in `helpers.ts::enrichWithToolResultFiles`.
const toolFiles: AttachedFileMeta[] = extractImagesAsAttachedFiles(normalizedFinalMessage.content)
.filter((file) => !file.mimeType.startsWith('image/'))
.map((file) => (file.source ? file : { ...file, source: 'tool-result' }));
if (matchedPath) {
for (const f of toolFiles) {
@@ -136,7 +141,9 @@ export function handleRuntimeEventState(
const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath));
for (const ref of mediaRefs) toolFiles.push(makeAttachedFile(ref, 'tool-result'));
for (const ref of extractRawFilePaths(text)) {
if (!mediaRefPaths.has(ref.filePath)) toolFiles.push(makeAttachedFile(ref, 'tool-result'));
if (mediaRefPaths.has(ref.filePath)) continue;
if (ref.mimeType.startsWith('image/')) continue;
toolFiles.push(makeAttachedFile(ref, 'tool-result'));
}
}
set((s) => {

27
src/stores/chat/types.ts
View File

@@ -5,7 +5,17 @@ export interface AttachedFileMeta {
fileSize: number;
preview: string | null;
filePath?: string;
source?: 'user-upload' | 'tool-result' | 'message-ref';
source?: 'user-upload' | 'tool-result' | 'message-ref' | 'gateway-media';
/**
* For Gateway-injected outgoing media (assistant-media). The Gateway emits
* an `image` content block with a relative URL like
* `/api/chat/media/outgoing/<sessionKey>/<attachmentId>/full`. The renderer
* cannot reach Gateway HTTP directly (CORS / env drift), so this URL is
* resolved through the Main-process proxy in `media:getThumbnails`, which
* looks up `~/.openclaw/media/outgoing/records/<attachmentId>.json` and
* loads the original file off disk.
*/
gatewayUrl?: string;
}
/** Raw message from OpenClaw chat.history */
@@ -35,6 +45,21 @@ export interface ContentBlock {
/** Flat image format from Gateway tool results (no source wrapper) */
data?: string;
mimeType?: string;
/**
* Flat URL on an `image` block. Gateway-injected assistant-media messages
* use this shape: `{ type:'image', url:'/api/chat/media/outgoing/...', mimeType, width, height, alt, openUrl }`.
* Neither nested `source.url` nor flat `data` is set in that case; the
* renderer must read `block.url` directly to surface the artifact.
*/
url?: string;
/** Optional companion of `url` — points at a higher-resolution variant. */
openUrl?: string;
/** Pixel width of the original image, used for layout hints. */
width?: number;
/** Pixel height of the original image, used for layout hints. */
height?: number;
/** Human-readable filename / alt text emitted by the Gateway. */
alt?: string;
id?: string;
name?: string;
input?: unknown;

276
tests/unit/chat-helpers-enrichment.test.ts Normal file
View File

@@ -0,0 +1,276 @@
import { describe, expect, it, vi } from 'vitest';
import {
enrichWithToolResultFiles,
enrichWithCachedImages,
} from '@/stores/chat/helpers';
import type { RawMessage } from '@/stores/chat';
vi.mock('@/lib/api-client', () => ({
invokeIpc: vi.fn(),
}));
describe('enrichWithToolResultFiles', () => {
it('does not promote image content blocks emitted by `read` tool results', () => {
// The `read` tool re-encodes the file as JPEG so the model can "see" it.
// The resulting image-data block is internal vision data, NOT a
// user-facing artifact — it must NOT spill onto the next assistant
// message as an attachment, otherwise every screenshot the agent
// inspects would render in the chat.
const messages: RawMessage[] = [
{
role: 'assistant',
id: 'a1',
content: [{ type: 'toolCall', id: 'tc1', name: 'read', input: { path: '/tmp/foo.png' } }],
},
{
role: 'toolresult',
id: 't1',
toolCallId: 'tc1',
toolName: 'read',
content: [
{ type: 'text', text: 'Read image file [image/jpeg]\n[Image: ...]' },
{ type: 'image', data: 'BASE64_BYTES_HERE', mimeType: 'image/jpeg' },
],
},
{
role: 'assistant',
id: 'a2',
content: [{ type: 'text', text: 'I had a look at the screenshot.' }],
},
];
const enriched = enrichWithToolResultFiles(messages);
const reply = enriched.find((m) => m.id === 'a2')!;
expect(reply._attachedFiles ?? []).toEqual([]);
});
it('does not promote raw image paths from tool result stdout (sips / ls / file)', () => {
// `sips ... && ls -la *.jpg` etc. spam image paths in the tool's
// stdout. Each one used to surface as an `_attachedFiles` entry on
// the next assistant message, causing the canonical artifact to be
// duplicated 3-4 times per send.
const messages: RawMessage[] = [
{
role: 'assistant',
id: 'a1',
content: [{ type: 'toolCall', id: 'tc1', name: 'exec', input: { command: 'sips ... && ls -la' } }],
},
{
role: 'toolresult',
id: 't1',
toolCallId: 'tc1',
toolName: 'exec',
content: [{
type: 'text',
text: '/private/tmp/desktop_screenshot.png\n /private/tmp/desktop_screenshot.jpg\n-rw-r--r--@ 1 me staff 857671 May 6 18:05 /tmp/desktop_screenshot.jpg',
}],
},
{
role: 'assistant',
id: 'a2',
content: [{ type: 'text', text: 'Compressed to 837KB, sending again.' }],
},
];
const enriched = enrichWithToolResultFiles(messages);
const reply = enriched.find((m) => m.id === 'a2')!;
expect(reply._attachedFiles ?? []).toEqual([]);
});
it('still promotes non-image artifact paths from tool results (PDF / XLSX)', () => {
// Documents emitted by tools (e.g. a Python script that wrote a
// spreadsheet to disk) ARE user-facing — they remain surfaced.
const messages: RawMessage[] = [
{
role: 'assistant',
id: 'a1',
content: [{ type: 'toolCall', id: 'tc1', name: 'exec', input: { command: '...' } }],
},
{
role: 'toolresult',
id: 't1',
toolCallId: 'tc1',
toolName: 'exec',
content: [{ type: 'text', text: 'Saved report at /tmp/report.pdf and data at /tmp/sales.xlsx' }],
},
{
role: 'assistant',
id: 'a2',
content: [{ type: 'text', text: 'Generated.' }],
},
];
const enriched = enrichWithToolResultFiles(messages);
const reply = enriched.find((m) => m.id === 'a2')!;
const paths = (reply._attachedFiles ?? []).map((f) => f.filePath);
expect(paths).toEqual(expect.arrayContaining(['/tmp/report.pdf', '/tmp/sales.xlsx']));
expect(paths.find((p) => p?.endsWith('.png'))).toBeUndefined();
expect(paths.find((p) => p?.endsWith('.jpg'))).toBeUndefined();
});
it('still promotes [media attached: ...] references emitted in tool results', () => {
const messages: RawMessage[] = [
{
role: 'assistant',
id: 'a1',
content: [{ type: 'toolCall', id: 'tc1', name: 'fetch', input: {} }],
},
{
role: 'toolresult',
id: 't1',
toolCallId: 'tc1',
toolName: 'fetch',
content: [{
type: 'text',
text: 'Done [media attached: /tmp/foo.pdf (application/pdf) | /tmp/foo.pdf]',
}],
},
{
role: 'assistant',
id: 'a2',
content: [{ type: 'text', text: 'Here it is.' }],
},
];
const enriched = enrichWithToolResultFiles(messages);
const reply = enriched.find((m) => m.id === 'a2')!;
const paths = (reply._attachedFiles ?? []).map((f) => f.filePath);
expect(paths).toContain('/tmp/foo.pdf');
});
});
describe('enrichWithCachedImages — Gateway media bubble dedup', () => {
it('drops image-typed MEDIA: refs on the reply when the next message is a Gateway assistant-media bubble', () => {
// When the agent emits `MEDIA:/tmp/x.png` the Gateway answers with a
// dedicated `assistant-media` bubble. Surfacing the same image again
// on the prior reply text would render two copies of the screenshot.
const messages: RawMessage[] = [
{
role: 'assistant',
id: 'reply',
content: [{ type: 'text', text: 'Compressed to 837KB:\n\nMEDIA:/tmp/desktop_screenshot.jpg' }],
},
{
role: 'assistant',
id: 'gateway-media',
content: [{
type: 'image',
url: '/api/chat/media/outgoing/agent%3Amain%3As-1/abc-123/full',
mimeType: 'image/jpeg',
alt: 'desktop_screenshot.jpg',
}],
},
];
const enriched = enrichWithCachedImages(messages);
const reply = enriched.find((m) => m.id === 'reply')!;
const replyPaths = (reply._attachedFiles ?? []).map((f) => f.filePath);
expect(replyPaths).toEqual([]);
const bubble = enriched.find((m) => m.id === 'gateway-media')!;
const bubbleEntries = bubble._attachedFiles ?? [];
expect(bubbleEntries).toHaveLength(1);
expect(bubbleEntries[0]).toMatchObject({
gatewayUrl: '/api/chat/media/outgoing/agent%3Amain%3As-1/abc-123/full',
mimeType: 'image/jpeg',
source: 'gateway-media',
});
});
it('keeps non-image MEDIA: refs on the reply even when a Gateway bubble follows', () => {
// Documents do not benefit from the Gateway's image pipeline; they
// should still render as inline cards on the reply text.
const messages: RawMessage[] = [
{
role: 'assistant',
id: 'reply',
content: [{
type: 'text',
text: 'Report generated:\n\nMEDIA:/tmp/report.pdf\n\nMEDIA:/tmp/cover.png',
}],
},
{
role: 'assistant',
id: 'gateway-media',
content: [{
type: 'image',
url: '/api/chat/media/outgoing/agent%3Amain%3As-1/cover-id/full',
mimeType: 'image/png',
alt: 'cover.png',
}],
},
];
const enriched = enrichWithCachedImages(messages);
const reply = enriched.find((m) => m.id === 'reply')!;
const replyPaths = (reply._attachedFiles ?? []).map((f) => f.filePath);
expect(replyPaths).toContain('/tmp/report.pdf');
expect(replyPaths.find((p) => p?.endsWith('.png'))).toBeUndefined();
});
it('attaches a gateway-media entry to the assistant-media bubble itself (text reply + injected bubble)', () => {
// Reproduces the exact shape of session
// ~/.openclaw/agents/main/sessions/5fb6925e-...jsonl
// - msg 12: assistant reply "...MEDIA:/tmp/openclaw/desktop_*.png"
// - msg 13: assistant `image` block with flat `url`
// After enrichment, msg 13 must carry exactly one `_attachedFiles`
// entry sourced from the Gateway URL — otherwise ChatMessage's early
// return swallows the bubble (no text, no tools, no extracted images,
// no attachments → returns null → user sees nothing).
const messages: RawMessage[] = [
{
role: 'user',
id: 'u1',
content: [{ type: 'text', text: 'Send me a desktop screenshot.' }],
},
{
role: 'assistant',
id: 'reply',
content: [{ type: 'text', text: 'Done, here it is:\n\nMEDIA:/tmp/openclaw/desktop_20260506_193407.png' }],
},
{
role: 'assistant',
id: 'gateway-media',
content: [{
type: 'image',
url: '/api/chat/media/outgoing/agent%3Amain%3As-1/e024afb9-2bc2-4a64-bf43-1c26fc779b6b/full',
openUrl: '/api/chat/media/outgoing/agent%3Amain%3As-1/e024afb9-2bc2-4a64-bf43-1c26fc779b6b/full',
mimeType: 'image/png',
width: 1536,
height: 998,
alt: 'desktop_20260506_193407.png',
}],
},
];
const enriched = enrichWithCachedImages(messages);
const reply = enriched.find((m) => m.id === 'reply')!;
expect((reply._attachedFiles ?? []).map((f) => f.filePath)).toEqual([]);
const bubble = enriched.find((m) => m.id === 'gateway-media')!;
const bubbleEntries = bubble._attachedFiles ?? [];
expect(bubbleEntries).toHaveLength(1);
expect(bubbleEntries[0]).toMatchObject({
gatewayUrl: '/api/chat/media/outgoing/agent%3Amain%3As-1/e024afb9-2bc2-4a64-bf43-1c26fc779b6b/full',
mimeType: 'image/png',
source: 'gateway-media',
});
});
it('keeps image-typed MEDIA: refs when there is no Gateway bubble after the reply', () => {
// If the Gateway is disabled / hasn't injected a bubble, the agent's
// own `MEDIA:` marker is the only signal and must still surface.
const messages: RawMessage[] = [
{
role: 'assistant',
id: 'reply',
content: [{ type: 'text', text: 'Here is the screenshot:\n\nMEDIA:/tmp/foo.png' }],
},
];
const enriched = enrichWithCachedImages(messages);
const reply = enriched[0]!;
const replyPaths = (reply._attachedFiles ?? []).map((f) => f.filePath);
expect(replyPaths).toEqual(['/tmp/foo.png']);
});
});

42
tests/unit/chat-helpers-extract-paths.test.ts
View File

@@ -40,6 +40,48 @@ describe('extractRawFilePaths', () => {
]);
});
it('captures MEDIA: paths that contain ASCII spaces (macOS screenshot default name)', () => {
// Regression: macOS' default screenshot filename is
// "Screenshot YYYY-MM-DD at HH.MM.SS.png" (en locale) or
// "截屏 YYYY-MM-DD HH.MM.SS.png" (zh locale)
// and the agent typically emits it verbatim via `MEDIA:` after
// `ls ~/Desktop`. The previous regex excluded ASCII spaces from the
// captured path, which made the extractor stop at "Screenshot" and
// never reach the `.png`, so the screenshot silently failed to
// surface as an attachment.
const sample = 'Found it on the desktop, sending now:\n\nMEDIA:/Users/alice/Desktop/Screenshot 2026-05-06 at 17.46.51.png';
const refs = extractRawFilePaths(sample);
expect(refs).toEqual([
{
filePath: '/Users/alice/Desktop/Screenshot 2026-05-06 at 17.46.51.png',
mimeType: 'image/png',
},
]);
});
it('captures MEDIA: paths whose filename is non-ASCII (zh-locale macOS screenshot)', () => {
// Companion to the previous case — make sure paths whose filename
// is full-Unicode (the zh-locale macOS default) are also captured.
const sample = 'Sending the screenshot now:\n\nMEDIA:/Users/alice/Desktop/截屏 2026-05-06 17.46.51.png';
const refs = extractRawFilePaths(sample);
expect(refs).toEqual([
{
filePath: '/Users/alice/Desktop/截屏 2026-05-06 17.46.51.png',
mimeType: 'image/png',
},
]);
});
it('keeps non-MEDIA prose after a space-bearing path out of the captured filename', () => {
// The lookahead must terminate the match at the first non-path character
// (newline, quote, paren, comma, full-stop, ...). Otherwise a long line
// like "MEDIA:/p with spaces.png and then more prose" would gobble the
// trailing narration into the "filename".
const sample = 'MEDIA:/tmp/my shot.png and then more prose';
const refs = extractRawFilePaths(sample);
expect(refs.map((r) => r.filePath)).toEqual(['/tmp/my shot.png']);
});
it('detects OpenClaw skill directories without file extensions', () => {
const refs = extractRawFilePaths('位置： ~/.openclaw/skills/open-eastmoney');
expect(refs).toEqual([

56
tests/unit/chat-message.test.tsx
View File

@@ -46,6 +46,62 @@ describe('ChatMessage attachment dedupe', () => {
expect(screen.getByAltText('artifact.png')).toBeInTheDocument();
});
it('keeps image artifacts visible alongside reply text when process attachments are suppressed', () => {
// Regression for media outgoing being silently dropped when the agent
// accompanies a `MEDIA:/path.png` artifact with any narration text:
// process-attachment filtering used to require PDF/XLSX/dir/skill but
// had no carve-out for images, so the file card never rendered.
const message: RawMessage = {
role: 'assistant',
content: 'Screenshot taken, sending it to you as an attachment.',
_attachedFiles: [
{
fileName: 'desktop_screenshot.png',
mimeType: 'image/png',
fileSize: 1234,
preview: 'data:image/png;base64,abc',
filePath: '/tmp/desktop_screenshot.png',
source: 'tool-result',
},
],
};
render(
<ChatMessage
message={message}
suppressProcessAttachments
/>,
);
expect(screen.getByAltText('desktop_screenshot.png')).toBeInTheDocument();
});
it('keeps message-ref image artifacts visible alongside reply text when process attachments are suppressed', () => {
const message: RawMessage = {
role: 'assistant',
content: 'Compressed, sending it to you:',
_attachedFiles: [
{
fileName: 'desktop_screenshot.jpg',
mimeType: 'image/jpeg',
fileSize: 837_000,
preview: 'data:image/jpeg;base64,xyz',
filePath: '/tmp/desktop_screenshot.jpg',
source: 'message-ref',
},
],
};
render(
<ChatMessage
message={message}
suppressProcessAttachments
/>,
);
expect(screen.getByAltText('desktop_screenshot.jpg')).toBeInTheDocument();
});
it('keeps pdf and spreadsheet artifacts visible when process attachments are suppressed', () => {
const message: RawMessage = {
role: 'assistant',