d8dfun/clawX
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(chat): optimize image sending stability (#983)

Browse Source
This commit is contained in:
Haze
2026-05-07 10:57:30 +08:00
committed by GitHub
parent 0c8dc8cb72
commit 9ada49afb3
11 changed files with 964 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/pages/Chat/ChatMessage.tsx
View File

@@ -275,9 +275,12 @@ export const ChatMessage = memo(function ChatMessage({
});
const filteredProcessAttachments = derivedAttachedFiles.filter((file) => {
if (file.source !== 'tool-result' && file.source !== 'message-ref') return true;
// Runtime-produced PDF / spreadsheet artifacts should remain visible
// in the chat even when generic process attachments are folded into
// the execution graph; they are the user-facing output to click.
// Runtime-produced user-facing artifacts (images, PDFs, spreadsheets,
// skill directories, ...) must remain visible in the reply bubble even
// when generic process attachments are folded into the execution graph.
// The graph card itself does not render `_attachedFiles`, so dropping
// images here would leave the user with no way to see them at all.
if (file.mimeType.startsWith('image/')) return true;
return isChatPreviewDocument(file) || isDirectoryAttachment(file) || isSkillFileAttachment(file);
});
// When a message is attachment-only, keep those attachments visible even if

56
src/pages/Chat/message-utils.ts
View File

@@ -37,7 +37,12 @@ function cleanUserText(text: string): string {
function stripAssistantMediaTags(text: string): string {
if (!text) return text;
const exts = 'png|jpe?g|gif|webp|bmp|avif|svg|pdf|docx?|xlsx?|pptx?|txt|csv|md|rtf|epub|zip|tar|gz|rar|7z|mp3|wav|ogg|aac|flac|m4a|mp4|mov|avi|mkv|webm|m4v';
const tagged = new RegExp(`(^|[\\s(\\[{>])(?:MEDIA|media):(?:\\/|~\\/)[^\\s\\n"'()\\[\\],<>]*?\\.(?:${exts})`, 'g');
// Mirror the relaxed character class in `chat/helpers.ts::extractRawFilePaths`
// so paths with ASCII spaces (e.g. macOS' "截屏 2026-05-06 17.46.51.png")
// are also stripped from the visible bubble. Without this, the bubble
// would still leak the literal `MEDIA:/.../截屏 2026-05-06 17.46.51.png`
// to the user when the underlying path detection succeeds.
const tagged = new RegExp(`(^|[\\s(\\[{>])(?:MEDIA|media):(?:\\/|~\\/)[^\\n"'()\\[\\],<>]*?\\.(?:${exts})(?=$|[\\s\\n"'()\\[\\],<>]|[，。；;,.!?])`, 'g');
return text
.replace(tagged, (_, lead: string) => lead)
// Collapse the empty lines / orphan whitespace the strip leaves behind.
@@ -302,31 +307,50 @@ export function extractMediaRefs(message: RawMessage | unknown): Array<{ filePat
}
/**
* Extract image attachments from a message.
* Returns array of { mimeType, data } for base64 images.
* Extract inline image attachments from a message.
*
* Returns either:
* - `{ mimeType, data }` — base64 bytes inline (Anthropic / Gateway tool-result)
* - `{ mimeType, url }` — Anthropic source-wrapped remote URL form
*
* Note: the Gateway-injected `assistant-media` shape — flat
* `{ type:'image', url:'/api/chat/media/outgoing/...', mimeType, ... }` —
* is intentionally NOT extracted here because the URL is a Gateway-relative
* path the renderer cannot fetch directly (CORS / env drift). That shape
* is surfaced through `_attachedFiles` instead (see
* `extractImagesAsAttachedFiles` and `enrichWithGatewayMediaBlocks` in
* `src/stores/chat/helpers.ts`), and resolved to a local preview by
* `loadMissingPreviews` -> Main `media:getThumbnails`.
*/
export function extractImages(message: RawMessage | unknown): Array<{ mimeType: string; data: string }> {
export function extractImages(
message: RawMessage | unknown,
): Array<{ mimeType: string; data?: string; url?: string }> {
if (!message || typeof message !== 'object') return [];
const msg = message as Record<string, unknown>;
const content = msg.content;
if (!Array.isArray(content)) return [];
const images: Array<{ mimeType: string; data: string }> = [];
const images: Array<{ mimeType: string; data?: string; url?: string }> = [];
for (const block of content as ContentBlock[]) {
if (block.type === 'image') {
// Path 1: Anthropic source-wrapped format
if (block.source) {
const src = block.source;
if (src.type === 'base64' && src.media_type && src.data) {
images.push({ mimeType: src.media_type, data: src.data });
}
}
// Path 2: Flat format from Gateway tool results {data, mimeType}
else if (block.data) {
images.push({ mimeType: block.mimeType || 'image/jpeg', data: block.data });
if (block.type !== 'image') continue;
// Path 1: Anthropic source-wrapped format
if (block.source) {
const src = block.source;
if (src.type === 'base64' && src.media_type && src.data) {
images.push({ mimeType: src.media_type, data: src.data });
} else if (src.type === 'url' && src.url) {
images.push({ mimeType: src.media_type || 'image/jpeg', url: src.url });
}
continue;
}
// Path 2: Flat format from Gateway tool results {data, mimeType}
if (block.data) {
images.push({ mimeType: block.mimeType || 'image/jpeg', data: block.data });
}
// Flat `block.url` is intentionally NOT handled here; see comment above.
}
return images;

304
src/stores/chat.ts
View File

@@ -434,11 +434,43 @@ function trimPathTerminators(filePath: string): string {
* Extract raw file paths from message text.
* Detects absolute paths (Unix: / or ~/, Windows: C:\ etc.) ending with common file extensions.
* Handles both image and non-image files, consistent with channel push message behavior.
*
* Also recognises the `MEDIA:` / `media:` prefix the OpenClaw runtime
* emits for produced artifacts (e.g.
* `MEDIA:/tmp/desktop_screenshot.png`) — without this the leading colon
* trips the URL guard on the unix regex below and the artifact never
* surfaces as an attachment. Mirrors `chat/helpers.ts::extractRawFilePaths`.
*/
function extractRawFilePaths(text: string): Array<{ filePath: string; mimeType: string }> {
const refs: Array<{ filePath: string; mimeType: string }> = [];
const seen = new Set<string>();
const exts = 'png|jpe?g|gif|webp|bmp|avif|svg|pdf|docx?|xlsx?|pptx?|txt|csv|md|rtf|epub|zip|tar|gz|rar|7z|mp3|wav|ogg|aac|flac|m4a|mp4|mov|avi|mkv|webm|m4v';
// Tagged media references (MEDIA:/path, media:~/path, ...). The agent
// runtime uses this prefix as an explicit "this is an artifact" marker,
// so we want them recognised even though the leading colon would
// normally look like a URL scheme. After matching we punch the entire
// `MEDIA:<path>` span out of the working text so the generic unix
// regex below doesn't double-count the bare `/path` suffix.
// The character class deliberately allows ASCII spaces inside the path so
// that macOS' default screenshot filename ("截屏 2026-05-06 17.46.51.png")
// and other space-containing paths the agent emits with the explicit
// `MEDIA:` marker still resolve. Newline and quote characters remain
// path terminators so we don't accidentally swallow trailing prose.
const taggedRegex = new RegExp(`(?:^|[\\s(\\[{>])(?:MEDIA|media):((?:\\/|~\\/)[^\\n"'()\\[\\],<>` + '`' + `]*?\\.(?:${exts}))(?=$|[\\s\\n"'()\\[\\],<>` + '`' + `]|[，。；;,.!?])`, 'g');
let workingText = text;
let taggedMatch: RegExpExecArray | null;
while ((taggedMatch = taggedRegex.exec(text)) !== null) {
const p = taggedMatch[1];
if (p && !seen.has(p)) {
seen.add(p);
refs.push({ filePath: p, mimeType: mimeFromExtension(p) });
}
// Mask the matched span so subsequent regexes can't re-discover the
// same path (e.g. `/two.xlsx` from `MEDIA:~/two.xlsx`).
const start = taggedMatch.index;
const end = start + taggedMatch[0].length;
workingText = workingText.slice(0, start) + ' '.repeat(end - start) + workingText.slice(end);
}
// Unix absolute paths (/... or ~/...) — lookbehind rejects mid-token slashes
// (e.g. "path/to/file.mp4", "https://example.com/file.mp4")
const unixRegex = new RegExp(`(?<![\\w./:])((?:\\/|~\\/)[^\\s\\n"'()\`\\[\\],<>]*?\\.(?:${exts}))`, 'gi');
@@ -453,7 +485,7 @@ function extractRawFilePaths(text: string): Array<{ filePath: string; mimeType:
);
for (const regex of [unixRegex, winRegex, skillDirRegex]) {
let match;
while ((match = regex.exec(text)) !== null) {
while ((match = regex.exec(workingText)) !== null) {
const p = trimPathTerminators(match[1]);
if (p && !seen.has(p)) {
seen.add(p);
@@ -508,6 +540,22 @@ function extractImagesAsAttachedFiles(content: unknown): AttachedFileMeta[] {
preview: `data:${mimeType};base64,${block.data}`,
});
}
// Path 3: Flat URL form from Gateway-injected assistant-media messages.
// See `src/stores/chat/helpers.ts` for the canonical implementation.
else if (block.url) {
const mimeType = block.mimeType || 'image/jpeg';
const fileName = typeof block.alt === 'string' && block.alt
? block.alt
: 'image';
files.push({
fileName,
mimeType,
fileSize: 0,
preview: null,
gatewayUrl: block.url,
source: 'gateway-media',
});
}
}
// Recurse into tool_result content blocks
if ((block.type === 'tool_result' || block.type === 'toolResult') && block.content) {
@@ -627,8 +675,15 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
// Resolve file path from the matching tool call
const matchedPath = msg.toolCallId ? toolCallPaths.get(msg.toolCallId) : undefined;
// 1. Image/file content blocks in the structured content array
const imageFiles = extractImagesAsAttachedFiles(msg.content);
// 1. Image/file content blocks in the structured content array.
// Images embedded inside a tool result are the model's vision data
// (e.g. `read /tmp/foo.png` re-encoded as JPEG so the model can "see"
// the file) — they are NOT user-facing artifacts. The agent surfaces
// user-facing images through `MEDIA:/path` text + the Gateway's
// `assistant-media` injection. Surfacing the vision data here would
// duplicate every screenshot the agent inspects.
const imageFiles = extractImagesAsAttachedFiles(msg.content)
.filter(file => !file.mimeType.startsWith('image/'));
if (matchedPath) {
for (const f of imageFiles) {
if (!f.filePath) {
@@ -650,11 +705,17 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
for (const ref of mediaRefs) {
pending.push({ ...makeAttachedFile(ref), source: 'tool-result' });
}
// 3. Raw file paths in tool result text (documents, audio, video, etc.)
// 3. Raw NON-image file paths in tool result text (documents,
// audio, video, ...). Image paths are deliberately ignored:
// `ls -la /tmp/foo.png`, `sips ... && ls -la *.jpg`, etc.
// spam intermediate paths that the user does not want to see
// rendered as separate cards. The canonical user-facing image
// is whatever the agent later emits via `MEDIA:/path` (which
// the Gateway turns into a dedicated assistant-media bubble).
for (const ref of extractRawFilePaths(text)) {
if (!mediaRefPaths.has(ref.filePath)) {
pending.push({ ...makeAttachedFile(ref), source: 'tool-result' });
}
if (mediaRefPaths.has(ref.filePath)) continue;
if (ref.mimeType.startsWith('image/')) continue;
pending.push({ ...makeAttachedFile(ref), source: 'tool-result' });
}
}
@@ -687,11 +748,30 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
* Uses local cache for previews when available; missing previews are loaded async.
*/
function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
// Pre-compute, per index, whether the *next* assistant message is a
// Gateway-injected `assistant-media` bubble (i.e. has at least one
// `image` content block carrying a flat URL). When that bubble exists,
// the canonical user-facing rendering of the artifact is the bubble
// itself — anything the agent emitted via `MEDIA:/path` in its prior
// text turn would just duplicate the same image, so image-typed raw
// refs on that prior message should be dropped here.
const nextHasGatewayMediaBubble = messages.map((_, idx) => {
const next = messages[idx + 1];
if (!next || next.role !== 'assistant') return false;
return extractImagesAsAttachedFiles(next.content).some(f => f.gatewayUrl);
});
return messages.map((msg, idx) => {
// Only process user and assistant messages; skip if already enriched
if ((msg.role !== 'user' && msg.role !== 'assistant') || msg._attachedFiles) return msg;
// Only process user and assistant messages.
if (msg.role !== 'user' && msg.role !== 'assistant') return msg;
const text = getMessageText(msg.content);
// Path 0: Gateway-injected outgoing media on this same message
// (an `assistant-media` bubble — image block with flat `url`).
const gatewayMediaFiles: AttachedFileMeta[] = msg.role === 'assistant'
? extractImagesAsAttachedFiles(msg.content).filter(file => file.gatewayUrl)
: [];
// Path 1: [media attached: path (mime) | path] — guaranteed format from attachment button
const mediaRefs = extractMediaRefs(text);
const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath));
@@ -699,9 +779,6 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
// Path 2: Raw file paths.
// For assistant messages: scan own text AND the nearest preceding user message text,
// but only for non-tool-only assistant messages (i.e. the final answer turn).
// Tool-only messages (thinking + tool calls) should not show file previews — those
// belong to the final answer message that comes after the tool results.
// User messages never get raw-path previews so the image is not shown twice.
let rawRefs: Array<{ filePath: string; mimeType: string }> = [];
if (msg.role === 'assistant' && !isToolOnlyMessage(msg)) {
// Own text
@@ -725,16 +802,39 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
}
}
const allRefs = [...mediaRefs, ...rawRefs];
if (allRefs.length === 0) return msg;
// Dedup vs Gateway-injected bubble: if the very next assistant message
// is a Gateway `assistant-media` bubble, drop image-typed raw refs on
// *this* message — the bubble already covers them.
if (msg.role === 'assistant' && nextHasGatewayMediaBubble[idx]) {
rawRefs = rawRefs.filter(r => !r.mimeType.startsWith('image/'));
}
const files: AttachedFileMeta[] = allRefs.map(ref => {
const cached = _imageCache.get(ref.filePath);
if (cached) return { ...cached, filePath: ref.filePath, source: 'message-ref' as const };
const fileName = ref.filePath.split(/[\\/]/).pop() || 'file';
return { fileName, mimeType: ref.mimeType, fileSize: 0, preview: null, filePath: ref.filePath, source: 'message-ref' as const };
});
return { ...msg, _attachedFiles: files };
const allRefs = [...mediaRefs, ...rawRefs];
if (allRefs.length === 0 && gatewayMediaFiles.length === 0) {
// Preserve any previously-attached `_attachedFiles` (e.g. set by
// `enrichWithToolResultFiles` for non-image artifacts). When nothing
// new applies, returning `msg` unmodified keeps those attachments.
return msg;
}
const existingFiles = msg._attachedFiles || [];
const existingPaths = new Set(existingFiles.map(file => file.filePath).filter(Boolean));
const existingGatewayUrls = new Set(
existingFiles.map(file => file.gatewayUrl).filter(Boolean) as string[],
);
const files: AttachedFileMeta[] = allRefs
.filter(ref => !existingPaths.has(ref.filePath))
.map(ref => {
const cached = _imageCache.get(ref.filePath);
if (cached) return { ...cached, filePath: ref.filePath, source: 'message-ref' as const };
const fileName = ref.filePath.split(/[\\/]/).pop() || 'file';
return { fileName, mimeType: ref.mimeType, fileSize: 0, preview: null, filePath: ref.filePath, source: 'message-ref' as const };
});
const dedupedGatewayMedia = gatewayMediaFiles.filter(
file => file.gatewayUrl && !existingGatewayUrls.has(file.gatewayUrl),
);
if (files.length === 0 && dedupedGatewayMedia.length === 0) return msg;
return { ...msg, _attachedFiles: [...existingFiles, ...files, ...dedupedGatewayMedia] };
});
}
@@ -744,24 +844,29 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
* Handles both [media attached: ...] patterns and raw filePath entries.
*/
async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
// Collect all image paths that need previews
const needPreview: Array<{ filePath: string; mimeType: string }> = [];
const seenPaths = new Set<string>();
// See helpers.ts loadMissingPreviews for the canonical comment block —
// this monolithic copy is kept in sync so legacy chat.ts callers also
// resolve Gateway-injected outgoing media URLs into local previews.
type PreviewRef = { filePath?: string; gatewayUrl?: string; mimeType: string };
const needPreview: PreviewRef[] = [];
const seenKeys = new Set<string>();
for (const msg of messages) {
if (!msg._attachedFiles) continue;
// Path 1: files with explicit filePath field (raw path detection or enriched refs)
// Path 1: files with explicit filePath OR gatewayUrl
for (const file of msg._attachedFiles) {
const fp = file.filePath;
if (!fp || seenPaths.has(fp)) continue;
// Images: need preview. Non-images: need file size (for FileCard display).
const key = file.filePath || file.gatewayUrl;
if (!key || seenKeys.has(key)) continue;
const needsLoad = file.mimeType.startsWith('image/')
? !file.preview
: file.fileSize === 0;
if (needsLoad) {
seenPaths.add(fp);
needPreview.push({ filePath: fp, mimeType: file.mimeType });
if (!needsLoad) continue;
seenKeys.add(key);
if (file.filePath) {
needPreview.push({ filePath: file.filePath, mimeType: file.mimeType });
} else if (file.gatewayUrl) {
needPreview.push({ gatewayUrl: file.gatewayUrl, mimeType: file.mimeType });
}
}
@@ -772,17 +877,19 @@ async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
for (let i = 0; i < refs.length; i++) {
const file = msg._attachedFiles[i];
const ref = refs[i];
if (!file || !ref || seenPaths.has(ref.filePath)) continue;
if (!file || !ref || seenKeys.has(ref.filePath)) continue;
const needsLoad = ref.mimeType.startsWith('image/') ? !file.preview : file.fileSize === 0;
if (needsLoad) {
seenPaths.add(ref.filePath);
needPreview.push(ref);
seenKeys.add(ref.filePath);
needPreview.push({ filePath: ref.filePath, mimeType: ref.mimeType });
}
}
}
}
if (needPreview.length === 0) return false;
if (needPreview.length === 0) {
return false;
}
try {
const thumbnails = await hostApiFetch<Record<string, { preview: string | null; fileSize: number }>>(
@@ -797,15 +904,17 @@ async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
for (const msg of messages) {
if (!msg._attachedFiles) continue;
// Update files that have filePath
// Update files that have filePath OR gatewayUrl
for (const file of msg._attachedFiles) {
const fp = file.filePath;
if (!fp) continue;
const thumb = thumbnails[fp];
const key = file.filePath || file.gatewayUrl;
if (!key) continue;
const thumb = thumbnails[key];
if (thumb && (thumb.preview || thumb.fileSize)) {
if (thumb.preview) file.preview = thumb.preview;
if (thumb.fileSize) file.fileSize = thumb.fileSize;
_imageCache.set(fp, { ...file });
if (file.filePath) {
_imageCache.set(file.filePath, { ...file });
}
updated = true;
}
}
@@ -1004,11 +1113,35 @@ function isToolResultRole(role: unknown): boolean {
}
/** True for internal plumbing messages that should never be shown in the UI. */
function isInternalMessage(msg: { role?: unknown; content?: unknown }): boolean {
function isInternalMessage(msg: { role?: unknown; content?: unknown; idempotencyKey?: unknown; model?: unknown }): boolean {
if (msg.role === 'system') return true;
const text = getMessageText(msg.content);
if (msg.role === 'assistant') {
if (/^(HEARTBEAT_OK|NO_REPLY)\s*$/.test(text)) return true;
// OpenClaw's gateway writes a fallback `assistant-media` transcript
// message when its `createManagedOutgoingImageBlocks` pipeline fails
// ("could not be prepared" warning in stderr). The fallback has:
// - model: 'gateway-injected'
// - idempotencyKey: '<runId>:assistant-media'
// - content: text-only `MEDIA:<staging path>` (NOT an image-url block)
// The staging path lives in `~/.openclaw/media/outbound/` which has a
// 120s TTL — the file is gone by the time the user reads the chat.
// The original `MEDIA:/tmp/...` path is already on the previous
// assistant message (the agent's actual reply), so the fallback is
// pure duplicate noise. Hide it in the UI so it neither shows a
// broken card nor competes with the real reply for layout space.
const idempotencyKey = typeof msg.idempotencyKey === 'string' ? msg.idempotencyKey : '';
const isGatewayInjectedFallback = msg.model === 'gateway-injected'
&& idempotencyKey.endsWith(':assistant-media');
if (isGatewayInjectedFallback) {
const hasImageUrlBlock = Array.isArray(msg.content)
&& (msg.content as ContentBlock[]).some(
(block) => block.type === 'image' && typeof block.url === 'string' && !!block.url,
);
// Real gateway-media bubbles (with an image-url block) ARE the
// canonical render — keep them. Only hide the text-only fallback.
if (!hasImageUrlBlock) return true;
}
}
// Runtime system injections: these arrive as user or assistant-role messages
// but are internal plumbing (exec results, async-command notices, time pings, etc.)
@@ -2166,12 +2299,18 @@ export const useChatStore = create<ChatState>((set, get) => ({
const { activeRunId, currentSessionKey } = get();
// Only process events for the current session (when sessionKey is present)
if (eventSessionKey != null && eventSessionKey !== currentSessionKey) return;
if (eventSessionKey != null && eventSessionKey !== currentSessionKey) {
return;
}
// Only process events for the active run (or if no active run set)
if (activeRunId && runId && runId !== activeRunId) return;
if (activeRunId && runId && runId !== activeRunId) {
return;
}
if (isDuplicateChatEvent(eventState, event)) return;
if (isDuplicateChatEvent(eventState, event)) {
return;
}
_lastChatEventAt = Date.now();
@@ -2285,10 +2424,17 @@ export const useChatStore = create<ChatState>((set, get) => ({
? getToolCallFilePath(currentStreamForPath, normalizedFinalMessage.toolCallId)
: undefined;
// Mirror enrichWithToolResultFiles: collect images + file refs for next assistant msg
const toolFiles: AttachedFileMeta[] = [
...extractImagesAsAttachedFiles(normalizedFinalMessage.content),
];
// Mirror `enrichWithToolResultFiles`: collect non-image artifacts
// for the next assistant message. Images embedded inside a tool
// result (read tool's vision data) and raw image paths in the
// tool's stdout (sips / ls / file output) are NOT user-facing —
// the canonical render is the Gateway-injected `assistant-media`
// bubble that follows the agent's `MEDIA:` text. Surfacing those
// intermediate images here would duplicate every screenshot the
// agent inspects on its way to the final artifact.
const toolFiles: AttachedFileMeta[] = extractImagesAsAttachedFiles(
normalizedFinalMessage.content,
).filter(file => !file.mimeType.startsWith('image/'));
if (matchedPath) {
for (const f of toolFiles) {
if (!f.filePath) {
@@ -2303,7 +2449,9 @@ export const useChatStore = create<ChatState>((set, get) => ({
const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath));
for (const ref of mediaRefs) toolFiles.push(makeAttachedFile(ref));
for (const ref of extractRawFilePaths(text)) {
if (!mediaRefPaths.has(ref.filePath)) toolFiles.push(makeAttachedFile(ref));
if (mediaRefPaths.has(ref.filePath)) continue;
if (ref.mimeType.startsWith('image/')) continue;
toolFiles.push(makeAttachedFile(ref));
}
}
set((s) => {
@@ -2334,7 +2482,17 @@ export const useChatStore = create<ChatState>((set, get) => ({
const nextTools = updates.length > 0 ? upsertToolStatuses(s.streamingTools, updates) : s.streamingTools;
const streamingTools = hasOutput ? [] : nextTools;
// Attach any images collected from preceding tool results
// Note: it would be tempting to also surface `MEDIA:/path`
// markers from `normalizedFinalMessage.content`'s text here, so
// the agent's reply could attach the original file directly
// (`/tmp/...png`) without waiting for the post-final history
// reload. However, OpenClaw's `splitTrailingDirective`
// (selection-D8_ELZa7.js ~line 904) strips `MEDIA:/...` lines
// out of the streaming text BEFORE it reaches the client, so
// the `final` event we get here never contains the marker.
// Image surfacing is fully handled by the post-final reload
// below + `enrichWithCachedImages` (which dereferences the
// assistant-media bubble's `block.url`).
const pendingImgs = s.pendingToolImages;
const msgWithImages: RawMessage = pendingImgs.length > 0
? {
@@ -2388,6 +2546,52 @@ export const useChatStore = create<ChatState>((set, get) => ({
if (hasOutput && !toolOnly) {
clearHistoryPoll();
void get().loadHistory(true);
// OpenClaw's gateway processes `MEDIA:/path` markers in the
// assistant reply asynchronously, in the `dispatch.deliver` of
// the `final` payload (see openclaw/dist/chat-DM9hSaNV.js's
// `appendWebchatAgentMediaTranscriptIfNeeded`):
// 1. copy the original file under
// `~/.openclaw/media/outgoing/originals/<uuid>`
// 2. write the record JSON under
// `~/.openclaw/media/outgoing/records/<id>.json`
// 3. `appendAssistantTranscriptMessage` writes a follow-up
// `assistant-media` message to the session JSONL, with
// `idempotencyKey: "<runId>:assistant-media"`.
// That follow-up message is **only persisted** — it is NOT
// re-broadcast as a streaming event. The streaming `final`
// we just consumed only contains the agent's text. The
// assistant-media bubble can only be retrieved via
// `chat.history`, and the persistence runs on the order of
// ~400-500ms after the streaming final.
//
// The immediate `loadHistory(true)` above therefore races the
// gateway's write and almost always misses the bubble.
//
// CRITICAL: we cannot detect from the streaming final alone
// whether the agent emitted a `MEDIA:/path` marker — OpenClaw's
// `splitTrailingDirective` (selection-D8_ELZa7.js line ~904)
// strips `MEDIA:/...` lines from the broadcast text BEFORE it
// reaches the client, so the streaming `final` text is always
// the user-facing prose without the marker. The MEDIA: marker
// only appears in the persisted JSONL transcript (msg N) and
// its companion `assistant-media` bubble (msg N+1).
//
// We therefore unconditionally schedule ONE follow-up quiet
// reload ~1500ms after every assistant `final`. The cost is
// a single extra in-process RPC per assistant turn (cheap);
// when there's no media the second reload returns the same
// history snapshot and is a no-op for the UI.
// `forceNextHistoryLoad` bypasses `HISTORY_LOAD_MIN_INTERVAL_MS`
// so the call is not suppressed by the throttle.
const sessionKeyAtFinal = get().currentSessionKey;
setTimeout(() => {
if (get().currentSessionKey !== sessionKeyAtFinal) {
return;
}
forceNextHistoryLoad(sessionKeyAtFinal);
void get().loadHistory(true);
}, 1500);
}
} else {
// No message in final event - reload history to get complete data

141
src/stores/chat/helpers.ts
View File

@@ -360,7 +360,14 @@ function extractRawFilePaths(text: string): Array<{ filePath: string; mimeType:
// normally look like a URL scheme. After matching we punch the entire
// `MEDIA:<path>` span out of the working text so the generic unix
// regex below doesn't double-count the bare `/path` suffix.
const taggedRegex = new RegExp(`(?:^|[\\s(\\[{>])(?:MEDIA|media):((?:\\/|~\\/)[^\\s\\n"'()\\[\\],<>` + '`' + `]*?\\.(?:${exts}))`, 'g');
// The character class deliberately allows ASCII spaces inside the path so
// that macOS' default screenshot filename ("截屏 2026-05-06 17.46.51.png")
// and other space-containing paths the agent emits with the explicit
// `MEDIA:` marker still resolve. Newline and quote characters remain
// path terminators so we don't accidentally swallow trailing prose.
// The non-greedy `*?` anchored to `\.<ext>` keeps the match minimal so
// multiple `MEDIA:` markers in one paragraph still match independently.
const taggedRegex = new RegExp(`(?:^|[\\s(\\[{>])(?:MEDIA|media):((?:\\/|~\\/)[^\\n"'()\\[\\],<>` + '`' + `]*?\\.(?:${exts}))(?=$|[\\s\\n"'()\\[\\],<>` + '`' + `]|[，。；;,.!?])`, 'g');
let workingText = text;
let taggedMatch: RegExpExecArray | null;
while ((taggedMatch = taggedRegex.exec(text)) !== null) {
@@ -446,6 +453,26 @@ function extractImagesAsAttachedFiles(content: unknown): AttachedFileMeta[] {
preview: `data:${mimeType};base64,${block.data}`,
});
}
// Path 3: Flat URL form from Gateway-injected assistant-media messages.
// Shape: `{ type:'image', url:'/api/chat/media/outgoing/<sessionKey>/<id>/full',
// mimeType, width, height, alt, openUrl }`. The URL is relative
// to the Gateway HTTP server which the renderer cannot reach directly
// (CORS / env drift). We surface it as an `_attachedFiles` entry whose
// preview is filled in later by `loadMissingPreviews` -> Main proxy.
else if (block.url) {
const mimeType = block.mimeType || 'image/jpeg';
const fileName = typeof block.alt === 'string' && block.alt
? block.alt
: 'image';
files.push({
fileName,
mimeType,
fileSize: 0,
preview: null,
gatewayUrl: block.url,
source: 'gateway-media',
});
}
}
// Recurse into tool_result content blocks
if ((block.type === 'tool_result' || block.type === 'toolResult') && block.content) {
@@ -568,8 +595,14 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
// Resolve file path from the matching tool call
const matchedPath = msg.toolCallId ? toolCallPaths.get(msg.toolCallId) : undefined;
// 1. Image/file content blocks in the structured content array
const imageFiles = extractImagesAsAttachedFiles(msg.content);
// 1. Image/file content blocks in the structured content array.
// Images embedded inside a tool result are the model's vision data
// (e.g. `read /tmp/foo.png` re-encoded as JPEG so the model can
// "see" the file) — they are NOT user-facing artifacts. The agent
// surfaces user-facing images through `MEDIA:/path` text + the
// Gateway's `assistant-media` injection.
const imageFiles = extractImagesAsAttachedFiles(msg.content)
.filter(file => !file.mimeType.startsWith('image/'));
if (matchedPath) {
for (const f of imageFiles) {
if (!f.filePath) {
@@ -588,11 +621,14 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
for (const ref of mediaRefs) {
pending.push(makeAttachedFile(ref, 'tool-result'));
}
// 3. Raw file paths in tool result text (documents, audio, video, etc.)
// 3. Raw NON-image file paths in tool result text (documents,
// audio, video, ...). Image paths from intermediate tool stdout
// (`ls -la *.png`, `sips ... && ls`, `file /tmp/x.png`, etc.)
// are deliberately ignored — see comment on Path 1.
for (const ref of extractRawFilePaths(text)) {
if (!mediaRefPaths.has(ref.filePath)) {
pending.push(makeAttachedFile(ref, 'tool-result'));
}
if (mediaRefPaths.has(ref.filePath)) continue;
if (ref.mimeType.startsWith('image/')) continue;
pending.push(makeAttachedFile(ref, 'tool-result'));
}
}
@@ -625,6 +661,19 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
* Uses local cache for previews when available; missing previews are loaded async.
*/
function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
// Pre-compute, per index, whether the *next* assistant message is a
// Gateway-injected `assistant-media` bubble (i.e. has at least one
// `image` content block carrying a flat URL). When that bubble exists,
// the canonical user-facing rendering of the artifact is the bubble
// itself — anything the agent emitted via `MEDIA:/path` in its prior
// text turn would just duplicate the same image, so image-typed raw
// refs on that prior message are dropped here.
const nextHasGatewayMediaBubble = messages.map((_, idx) => {
const next = messages[idx + 1];
if (!next || next.role !== 'assistant') return false;
return extractImagesAsAttachedFiles(next.content).some(f => f.gatewayUrl);
});
return messages.map((msg, idx) => {
// Only process user and assistant messages. Messages may already carry
// attachments from tool-result enrichment; still merge in raw paths from
@@ -632,6 +681,16 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
if (msg.role !== 'user' && msg.role !== 'assistant') return msg;
const text = getMessageText(msg.content);
// Path 0: Gateway-injected outgoing media — `image` content blocks with
// a flat `url` field (e.g. `/api/chat/media/outgoing/<sessionKey>/<id>/full`).
// The renderer cannot fetch the URL directly, so we surface it as an
// `_attachedFiles` entry whose preview is filled in later by
// `loadMissingPreviews` -> Main `media:getThumbnails` (which dereferences
// the URL to the original file in `~/.openclaw/media/outgoing/`).
const gatewayMediaFiles: AttachedFileMeta[] = msg.role === 'assistant'
? extractImagesAsAttachedFiles(msg.content).filter(file => file.gatewayUrl)
: [];
// Path 1: [media attached: path (mime) | path] — guaranteed format from attachment button
const mediaRefs = extractMediaRefs(text);
const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath));
@@ -665,11 +724,21 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
}
}
// Dedup vs Gateway-injected bubble: when the very next assistant
// message is an `assistant-media` bubble, drop image-typed raw refs
// on *this* message — the bubble already covers the artifact.
if (msg.role === 'assistant' && nextHasGatewayMediaBubble[idx]) {
rawRefs = rawRefs.filter(r => !r.mimeType.startsWith('image/'));
}
const allRefs = [...mediaRefs, ...rawRefs];
if (allRefs.length === 0) return msg;
if (allRefs.length === 0 && gatewayMediaFiles.length === 0) return msg;
const existingFiles = msg._attachedFiles || [];
const existingPaths = new Set(existingFiles.map(file => file.filePath).filter(Boolean));
const existingGatewayUrls = new Set(
existingFiles.map(file => file.gatewayUrl).filter(Boolean) as string[],
);
const files: AttachedFileMeta[] = allRefs
.filter(ref => !existingPaths.has(ref.filePath))
.map(ref => {
@@ -678,8 +747,11 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
const fileName = ref.filePath.split(/[\\/]/).pop() || 'file';
return { fileName, mimeType: ref.mimeType, fileSize: 0, preview: null, filePath: ref.filePath, source: 'message-ref' };
});
if (files.length === 0) return msg;
return { ...msg, _attachedFiles: [...existingFiles, ...files] };
const dedupedGatewayMedia = gatewayMediaFiles.filter(
file => file.gatewayUrl && !existingGatewayUrls.has(file.gatewayUrl),
);
if (files.length === 0 && dedupedGatewayMedia.length === 0) return msg;
return { ...msg, _attachedFiles: [...existingFiles, ...files, ...dedupedGatewayMedia] };
});
}
@@ -689,24 +761,34 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
* Handles both [media attached: ...] patterns and raw filePath entries.
*/
async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
// Collect all image paths that need previews
const needPreview: Array<{ filePath: string; mimeType: string }> = [];
const seenPaths = new Set<string>();
// Collect all image refs that need previews. The IPC handler accepts:
// - { filePath, mimeType } — local on-disk files
// - { gatewayUrl, mimeType } — Gateway-injected outgoing media; the
// handler resolves the URL to a local file
// via `~/.openclaw/media/outgoing/records/`.
// We use `filePath || gatewayUrl` as the dedupe / lookup key on the way
// back; a file always carries at most one of the two.
type PreviewRef = { filePath?: string; gatewayUrl?: string; mimeType: string };
const needPreview: PreviewRef[] = [];
const seenKeys = new Set<string>();
for (const msg of messages) {
if (!msg._attachedFiles) continue;
// Path 1: files with explicit filePath field (raw path detection or enriched refs)
// Path 1: files with explicit filePath OR gatewayUrl
for (const file of msg._attachedFiles) {
const fp = file.filePath;
if (!fp || seenPaths.has(fp)) continue;
const key = file.filePath || file.gatewayUrl;
if (!key || seenKeys.has(key)) continue;
// Images: need preview. Non-images: need file size (for FileCard display).
const needsLoad = file.mimeType.startsWith('image/')
? !file.preview
: file.fileSize === 0;
if (needsLoad) {
seenPaths.add(fp);
needPreview.push({ filePath: fp, mimeType: file.mimeType });
if (!needsLoad) continue;
seenKeys.add(key);
if (file.filePath) {
needPreview.push({ filePath: file.filePath, mimeType: file.mimeType });
} else if (file.gatewayUrl) {
needPreview.push({ gatewayUrl: file.gatewayUrl, mimeType: file.mimeType });
}
}
@@ -717,11 +799,11 @@ async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
for (let i = 0; i < refs.length; i++) {
const file = msg._attachedFiles[i];
const ref = refs[i];
if (!file || !ref || seenPaths.has(ref.filePath)) continue;
if (!file || !ref || seenKeys.has(ref.filePath)) continue;
const needsLoad = ref.mimeType.startsWith('image/') ? !file.preview : file.fileSize === 0;
if (needsLoad) {
seenPaths.add(ref.filePath);
needPreview.push(ref);
seenKeys.add(ref.filePath);
needPreview.push({ filePath: ref.filePath, mimeType: ref.mimeType });
}
}
}
@@ -739,15 +821,20 @@ async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
for (const msg of messages) {
if (!msg._attachedFiles) continue;
// Update files that have filePath
// Update files that have filePath OR gatewayUrl
for (const file of msg._attachedFiles) {
const fp = file.filePath;
if (!fp) continue;
const thumb = thumbnails[fp];
const key = file.filePath || file.gatewayUrl;
if (!key) continue;
const thumb = thumbnails[key];
if (thumb && (thumb.preview || thumb.fileSize)) {
if (thumb.preview) file.preview = thumb.preview;
if (thumb.fileSize) file.fileSize = thumb.fileSize;
_imageCache.set(fp, { ...file });
// Only persist local-path entries to the localStorage cache.
// Gateway outgoing URLs are tied to a specific session/attachment
// id and can be stale across runs, so caching is harmful.
if (file.filePath) {
_imageCache.set(file.filePath, { ...file });
}
updated = true;
}
}

11
src/stores/chat/runtime-event-handlers.ts
View File

@@ -119,8 +119,13 @@ export function handleRuntimeEventState(
? getToolCallFilePath(currentStreamForPath, normalizedFinalMessage.toolCallId)
: undefined;
// Mirror enrichWithToolResultFiles: collect images + file refs for next assistant msg
// Mirror `enrichWithToolResultFiles`: collect non-image
// artifacts for the next assistant message. Image content
// blocks (vision data) and image-typed raw paths in the
// tool's stdout are intermediate process noise — see the
// comment in `helpers.ts::enrichWithToolResultFiles`.
const toolFiles: AttachedFileMeta[] = extractImagesAsAttachedFiles(normalizedFinalMessage.content)
.filter((file) => !file.mimeType.startsWith('image/'))
.map((file) => (file.source ? file : { ...file, source: 'tool-result' }));
if (matchedPath) {
for (const f of toolFiles) {
@@ -136,7 +141,9 @@ export function handleRuntimeEventState(
const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath));
for (const ref of mediaRefs) toolFiles.push(makeAttachedFile(ref, 'tool-result'));
for (const ref of extractRawFilePaths(text)) {
if (!mediaRefPaths.has(ref.filePath)) toolFiles.push(makeAttachedFile(ref, 'tool-result'));
if (mediaRefPaths.has(ref.filePath)) continue;
if (ref.mimeType.startsWith('image/')) continue;
toolFiles.push(makeAttachedFile(ref, 'tool-result'));
}
}
set((s) => {

27
src/stores/chat/types.ts
View File

@@ -5,7 +5,17 @@ export interface AttachedFileMeta {
fileSize: number;
preview: string | null;
filePath?: string;
source?: 'user-upload' | 'tool-result' | 'message-ref';
source?: 'user-upload' | 'tool-result' | 'message-ref' | 'gateway-media';
/**
* For Gateway-injected outgoing media (assistant-media). The Gateway emits
* an `image` content block with a relative URL like
* `/api/chat/media/outgoing/<sessionKey>/<attachmentId>/full`. The renderer
* cannot reach Gateway HTTP directly (CORS / env drift), so this URL is
* resolved through the Main-process proxy in `media:getThumbnails`, which
* looks up `~/.openclaw/media/outgoing/records/<attachmentId>.json` and
* loads the original file off disk.
*/
gatewayUrl?: string;
}
/** Raw message from OpenClaw chat.history */
@@ -35,6 +45,21 @@ export interface ContentBlock {
/** Flat image format from Gateway tool results (no source wrapper) */
data?: string;
mimeType?: string;
/**
* Flat URL on an `image` block. Gateway-injected assistant-media messages
* use this shape: `{ type:'image', url:'/api/chat/media/outgoing/...', mimeType, width, height, alt, openUrl }`.
* Neither nested `source.url` nor flat `data` is set in that case; the
* renderer must read `block.url` directly to surface the artifact.
*/
url?: string;
/** Optional companion of `url` — points at a higher-resolution variant. */
openUrl?: string;
/** Pixel width of the original image, used for layout hints. */
width?: number;
/** Pixel height of the original image, used for layout hints. */
height?: number;
/** Human-readable filename / alt text emitted by the Gateway. */
alt?: string;
id?: string;
name?: string;
input?: unknown;