Fix image generate (#1068)

This commit is contained in:
paisley
2026-05-26 18:24:31 +08:00
committed by GitHub
parent 39c8400421
commit be7f8fc768
32 changed files with 1534 additions and 165 deletions

View File

@@ -1773,6 +1773,7 @@ export async function syncOpenAiCompatibleImageRelay(params: {
}
removePluginRegistrations(config, [CLAWX_OPENAI_IMAGE_PROVIDER_KEY]);
await writeOpenClawJson(config);
await removeProviderKeyFromOpenClaw(CLAWX_OPENAI_IMAGE_PROVIDER_KEY);
if (params.apiKey?.trim()) {
await saveProviderKeyToOpenClaw(CLAWX_OPENAI_IMAGE_PROVIDER_KEY, params.apiKey.trim());
}

View File

@@ -47,25 +47,30 @@ type ModelInputModule = {
resolveAgentModelPrimaryValue: (model?: unknown) => string | undefined;
};
const OPENCLAW_IMAGE_GENERATION_RUNTIME = 'openclaw/plugin-sdk/image-generation-runtime';
const OPENCLAW_MEDIA_STORE = 'openclaw/plugin-sdk/media-store';
const OPENCLAW_MEDIA_RUNTIME = 'openclaw/plugin-sdk/media-runtime';
const OPENCLAW_IMAGE_GENERATION_CORE = 'openclaw/plugin-sdk/image-generation-core';
let imageRuntimeModule: ImageGenerationRuntimeModule | null = null;
let mediaStoreModule: MediaStoreModule | null = null;
let imageOpsModule: ImageOpsModule | null = null;
let modelInputModule: ModelInputModule | null = null;
async function importRuntimeModule<T>(relativePath: string): Promise<T> {
const modulePath = resolveOpenClawRuntimeModulePath(relativePath);
async function importOpenClawSdkModule<T>(specifier: string): Promise<T> {
const modulePath = resolveOpenClawRuntimeModulePath(specifier);
return import(pathToFileURL(modulePath).href) as Promise<T>;
}
async function getImageGenerationRuntime(): Promise<ImageGenerationRuntimeModule> {
if (!imageRuntimeModule) {
const mod = await importRuntimeModule<{
t: ImageGenerationRuntimeModule['generateImage'];
n: ImageGenerationRuntimeModule['listRuntimeImageGenerationProviders'];
}>('./dist/runtime-Oyct10ZH.js');
const mod = await importOpenClawSdkModule<{
generateImage: ImageGenerationRuntimeModule['generateImage'];
listRuntimeImageGenerationProviders: ImageGenerationRuntimeModule['listRuntimeImageGenerationProviders'];
}>(OPENCLAW_IMAGE_GENERATION_RUNTIME);
imageRuntimeModule = {
generateImage: mod.t,
listRuntimeImageGenerationProviders: mod.n,
generateImage: mod.generateImage,
listRuntimeImageGenerationProviders: mod.listRuntimeImageGenerationProviders,
};
}
return imageRuntimeModule;
@@ -73,20 +78,20 @@ async function getImageGenerationRuntime(): Promise<ImageGenerationRuntimeModule
async function getMediaStore(): Promise<MediaStoreModule> {
if (!mediaStoreModule) {
const mod = await importRuntimeModule<{ u: MediaStoreModule['saveMediaBuffer'] }>(
'./dist/store-b792nN7l.js',
const mod = await importOpenClawSdkModule<{ saveMediaBuffer: MediaStoreModule['saveMediaBuffer'] }>(
OPENCLAW_MEDIA_STORE,
);
mediaStoreModule = { saveMediaBuffer: mod.u };
mediaStoreModule = { saveMediaBuffer: mod.saveMediaBuffer };
}
return mediaStoreModule;
}
async function getImageOps(): Promise<ImageOpsModule> {
if (!imageOpsModule) {
const mod = await importRuntimeModule<{ a: ImageOpsModule['getImageMetadata'] }>(
'./dist/image-ops-CL9ZQP7k.js',
const mod = await importOpenClawSdkModule<{ getImageMetadata: ImageOpsModule['getImageMetadata'] }>(
OPENCLAW_MEDIA_RUNTIME,
);
imageOpsModule = { getImageMetadata: mod.a };
imageOpsModule = { getImageMetadata: mod.getImageMetadata };
}
return imageOpsModule;
}
@@ -100,10 +105,10 @@ export async function resolveImageGenerationPrimaryFromConfig(
async function getModelInputHelpers(): Promise<ModelInputModule> {
if (!modelInputModule) {
const mod = await importRuntimeModule<{ i: ModelInputModule['resolveAgentModelPrimaryValue'] }>(
'./dist/model-input-B9p-bobB.js',
const mod = await importOpenClawSdkModule<{ resolveAgentModelPrimaryValue: ModelInputModule['resolveAgentModelPrimaryValue'] }>(
OPENCLAW_IMAGE_GENERATION_CORE,
);
modelInputModule = { resolveAgentModelPrimaryValue: mod.i };
modelInputModule = { resolveAgentModelPrimaryValue: mod.resolveAgentModelPrimaryValue };
}
return modelInputModule;
}

View File

@@ -2,14 +2,16 @@
* Global image generation settings (agents.defaults.imageGenerationModel).
*/
import { useCallback, useEffect, useMemo, useState } from 'react';
import { Eye, EyeOff, ImagePlus, Loader2, Play, RefreshCw } from 'lucide-react';
import { Eye, EyeOff, ImagePlus, Loader2, Play, RefreshCw, Trash2 } from 'lucide-react';
import { useTranslation } from 'react-i18next';
import { toast } from 'sonner';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
import { Label } from '@/components/ui/label';
import { Badge } from '@/components/ui/badge';
import { ConfirmDialog } from '@/components/ui/confirm-dialog';
import {
clearImageGenerationSettings,
fetchImageGenerationSettings,
runImageGenerationTest,
saveImageGenerationSettings,
@@ -32,10 +34,12 @@ function extractTestOutputPath(result: unknown): string | null {
}
export function ImageGenerationSettings() {
const { t } = useTranslation(['dashboard', 'settings']);
const { t } = useTranslation(['dashboard', 'settings', 'common']);
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
const [clearing, setClearing] = useState(false);
const [testing, setTesting] = useState(false);
const [clearConfirmOpen, setClearConfirmOpen] = useState(false);
const [snapshot, setSnapshot] = useState<ImageGenerationSettingsSnapshot | null>(null);
const [timeoutMs, setTimeoutMs] = useState('180000');
@@ -79,6 +83,15 @@ export function ImageGenerationSettings() {
);
}, [snapshot, timeoutMs, relayBaseUrl, relayModel, relayApiKey]);
const hasConfiguredRelay = useMemo(() => {
if (!snapshot) return false;
return Boolean(
snapshot.openAiRelay?.enabled
|| snapshot.openAiRelay?.baseUrl?.trim()
|| snapshot.config.primary?.trim(),
);
}, [snapshot]);
const handleSave = async () => {
setSaving(true);
try {
@@ -120,11 +133,32 @@ export function ImageGenerationSettings() {
}
};
const handleClear = async () => {
setClearing(true);
try {
const next = await clearImageGenerationSettings();
setSnapshot(next);
setRelayBaseUrl('');
setRelayModel('gpt-image-2');
setRelayApiKey('');
setShowRelayApiKey(false);
setClearConfirmOpen(false);
toast.success(t('imageGeneration.toast.cleared'));
} catch (error) {
toast.error(error instanceof Error ? error.message : String(error));
} finally {
setClearing(false);
}
};
const handleTest = async () => {
if (dirty) {
toast.message(t('imageGeneration.toast.saveBeforeTest'));
return;
}
if (!hasConfiguredRelay) {
return;
}
setTesting(true);
try {
const result = await runImageGenerationTest({
@@ -368,7 +402,7 @@ export function ImageGenerationSettings() {
variant="outline"
className="rounded-full h-10"
onClick={() => void handleTest()}
disabled={testing || !relayModel.trim()}
disabled={testing || !hasConfiguredRelay || dirty}
data-testid="image-generation-test-button"
>
{testing ? (
@@ -387,9 +421,34 @@ export function ImageGenerationSettings() {
{saving ? <Loader2 className="h-4 w-4 mr-2 animate-spin" /> : null}
{saving ? t('imageGeneration.saving') : t('imageGeneration.save')}
</Button>
<Button
variant="outline"
className="rounded-full h-10 text-destructive hover:text-destructive"
onClick={() => setClearConfirmOpen(true)}
disabled={clearing || !hasConfiguredRelay}
data-testid="image-generation-clear"
>
{clearing ? (
<Loader2 className="h-4 w-4 mr-2 animate-spin" />
) : (
<Trash2 className="h-4 w-4 mr-2" />
)}
{clearing ? t('imageGeneration.clearing') : t('imageGeneration.clear')}
</Button>
</div>
</div>
)}
<ConfirmDialog
open={clearConfirmOpen}
title={t('imageGeneration.clearConfirmTitle')}
message={t('imageGeneration.clearConfirmMessage')}
confirmLabel={t('imageGeneration.clearConfirmAction')}
cancelLabel={t('common:actions.cancel')}
variant="destructive"
onConfirm={handleClear}
onCancel={() => setClearConfirmOpen(false)}
/>
</div>
);
}

View File

@@ -159,10 +159,14 @@
"branchLabel": "branch",
"thinkingLabel": "Thinking",
"errorLabel": "Error",
"imageGenerateLabel": "Image generation",
"agentRun": "{{agent}} execution",
"collapsedSummary": "{{toolCount}} tool calls · {{processCount}} process messages",
"collapseAction": "Collapse execution graph"
},
"imageGeneration": {
"generating": "Generating image, please wait…"
},
"composer": {
"attachFiles": "Attach files",
"pickSkill": "Choose skill",

View File

@@ -78,9 +78,15 @@
"testing": "Generating…",
"save": "Save",
"saving": "Saving…",
"clear": "Clear configuration",
"clearing": "Clearing…",
"clearConfirmTitle": "Clear image generation configuration?",
"clearConfirmMessage": "This removes the custom image endpoint, model selection, and stored API key. Chat providers are not affected.",
"clearConfirmAction": "Clear",
"testPrompt": "A small red circle on a white background, minimal flat illustration.",
"toast": {
"saved": "Image generation settings saved",
"cleared": "Image generation configuration cleared",
"saveBeforeTest": "Save settings before running a test",
"testSuccess": "Test image generated ({{ms}} ms)",
"testSuccessWithPath": "Test image generated ({{ms}} ms): {{path}}",
@@ -90,7 +96,7 @@
"title": "Custom image endpoint",
"description": "Send image generation to a dedicated OpenAI Images-compatible endpoint (/v1/images/generations). This does not change any chat provider.",
"baseUrl": "Image Base URL",
"baseUrlHint": "e.g. https://taolat.com/v1 — /v1 is appended automatically when omitted.",
"baseUrlHint": "e.g. https://api.example.com/v1 — /v1 is appended automatically when omitted.",
"apiKey": "API key",
"apiKeyPlaceholder": "sk-...",
"apiKeyPlaceholderConfigured": "Already configured — leave blank to keep",

View File

@@ -159,10 +159,14 @@
"branchLabel": "branch",
"thinkingLabel": "考え中",
"errorLabel": "エラー",
"imageGenerateLabel": "画像生成",
"agentRun": "{{agent}} の実行",
"collapsedSummary": "ツール呼び出し {{toolCount}} 件 · プロセスメッセージ {{processCount}} 件",
"collapseAction": "実行グラフを折りたたむ"
},
"imageGeneration": {
"generating": "画像を生成しています。少々お待ちください…"
},
"composer": {
"attachFiles": "ファイルを添付",
"pickSkill": "Skill を選択",

View File

@@ -78,9 +78,15 @@
"testing": "生成中…",
"save": "保存",
"saving": "保存中…",
"clear": "設定をクリア",
"clearing": "クリア中…",
"clearConfirmTitle": "画像生成設定をクリアしますか?",
"clearConfirmMessage": "カスタム画像エンドポイント、モデル設定、保存済み API キーを削除します。チャットプロバイダーには影響しません。",
"clearConfirmAction": "クリア",
"testPrompt": "白背景に小さな赤い円、ミニマルなフラットイラスト。",
"toast": {
"saved": "画像生成設定を保存しました",
"cleared": "画像生成設定をクリアしました",
"saveBeforeTest": "テスト前に設定を保存してください",
"testSuccess": "テスト画像を生成しました({{ms}} ms",
"testSuccessWithPath": "テスト画像を生成しました({{ms}} ms{{path}}",
@@ -90,7 +96,7 @@
"title": "カスタム画像エンドポイント",
"description": "画像生成を専用の OpenAI Images 互換エンドポイント(/v1/images/generationsへ送ります。チャットプロバイダーは変更しません。",
"baseUrl": "画像 Base URL",
"baseUrlHint": "例: https://taolat.com/v1 — /v1 が無い場合は自動付与します。",
"baseUrlHint": "例: https://api.example.com/v1 — /v1 が無い場合は自動付与します。",
"apiKey": "API キー",
"apiKeyPlaceholder": "sk-...",
"apiKeyPlaceholderConfigured": "設定済み — 空欄のままなら変更しません",

View File

@@ -159,10 +159,14 @@
"branchLabel": "ветвь",
"thinkingLabel": "Думаю",
"runErrorLabel": "Ошибка модели",
"imageGenerateLabel": "Генерация изображения",
"agentRun": "Выполнение {{agent}}",
"collapsedSummary": "Вызовов инструментов: {{toolCount}} · Промежуточных сообщений: {{processCount}}",
"collapseAction": "Свернуть граф выполнения"
},
"imageGeneration": {
"generating": "Изображение генерируется, подождите…"
},
"composer": {
"attachFiles": "Прикрепить файлы",
"pickSkill": "Выбрать Skill",

View File

@@ -90,7 +90,7 @@
"title": "Пользовательский endpoint изображений",
"description": "Отправлять генерацию изображений на отдельный OpenAI Images-совместимый endpoint (/v1/images/generations), не меняя чат-провайдеры.",
"baseUrl": "Base URL изображений",
"baseUrlHint": "например https://taolat.com/v1 — /v1 добавится автоматически.",
"baseUrlHint": "например https://api.example.com/v1 — /v1 добавится автоматически.",
"apiKey": "API key",
"apiKeyPlaceholder": "sk-...",
"apiKeyPlaceholderConfigured": "Уже настроен — оставьте пустым, чтобы не менять",

View File

@@ -159,10 +159,14 @@
"branchLabel": "分支",
"thinkingLabel": "思考中",
"modelErrorTitle": "模型调用失败",
"imageGenerateLabel": "图片生成",
"agentRun": "{{agent}} 执行",
"collapsedSummary": "{{toolCount}} 个工具调用,{{processCount}} 条过程消息",
"collapseAction": "收起执行关系图"
},
"imageGeneration": {
"generating": "图片生成中,请稍候…"
},
"composer": {
"attachFiles": "添加文件",
"pickSkill": "选择技能",

View File

@@ -78,9 +78,15 @@
"testing": "生成中…",
"save": "保存",
"saving": "保存中…",
"clear": "清除配置",
"clearing": "清除中…",
"clearConfirmTitle": "清除图像生成配置?",
"clearConfirmMessage": "将移除自定义生图端点、模型设置和已保存的 API Key不会影响聊天 Provider。",
"clearConfirmAction": "清除",
"testPrompt": "白色背景上的小红圆,极简扁平插画。",
"toast": {
"saved": "图像生成设置已保存",
"cleared": "图像生成配置已清除",
"saveBeforeTest": "请先保存设置再运行测试",
"testSuccess": "测试图像已生成({{ms}} ms",
"testSuccessWithPath": "测试图像已生成({{ms}} ms{{path}}",
@@ -90,7 +96,7 @@
"title": "自定义生图端点",
"description": "将生图请求发往独立的 OpenAI Images 兼容端点(/v1/images/generations不会改变任何聊天 Provider。",
"baseUrl": "生图 Base URL",
"baseUrlHint": "例如 https://taolat.com/v1未带 /v1 时会自动补全。",
"baseUrlHint": "例如 https://api.example.com/v1未带 /v1 时会自动补全。",
"apiKey": "API Key",
"apiKeyPlaceholder": "sk-...",
"apiKeyPlaceholderConfigured": "已配置,留空则不修改",

View File

@@ -61,6 +61,10 @@ export async function fetchImageGenerationSettings(): Promise<ImageGenerationSet
return response;
}
export async function clearImageGenerationSettings(): Promise<ImageGenerationSettingsSnapshot> {
return saveImageGenerationSettings({ openAiRelayEnabled: false });
}
export async function saveImageGenerationSettings(payload: {
primary?: string | null;
fallbacks?: string[];

View File

@@ -16,7 +16,8 @@ import { Button } from '@/components/ui/button';
import { cn } from '@/lib/utils';
import { invokeIpc, statFile } from '@/lib/api-client';
import type { RawMessage, AttachedFileMeta } from '@/stores/chat';
import { extractText, extractImages, extractToolUse, formatTimestamp } from './message-utils';
import { extractText, extractImages, extractToolUse, formatTimestamp, isUnresolvableImageUrl } from './message-utils';
import { copyImageToClipboard, type ImageCopyTarget } from './copy-image';
interface ChatMessageProps {
message: RawMessage;
@@ -222,7 +223,7 @@ function normalizeLatexDelimiters(input: string): string {
/** Resolve an ExtractedImage to a displayable src string, or null if not possible. */
function imageSrc(img: ExtractedImage): string | null {
if (img.url) return img.url;
if (img.url) return isUnresolvableImageUrl(img.url) ? null : img.url;
if (img.data) return `data:${img.mimeType};base64,${img.data}`;
return null;
}
@@ -248,6 +249,7 @@ export const ChatMessage = memo(function ChatMessage({
const hideAssistantText = suppressAssistantText && !isUser;
const hasText = !hideAssistantText && text.trim().length > 0;
const images = extractImages(message);
const resolvableContentImages = images.filter((img) => imageSrc(img) != null);
const tools = extractToolUse(message);
const visibleTools = suppressToolCards ? [] : tools;
const [validatedPaths, setValidatedPaths] = useState<Record<string, boolean>>({});
@@ -321,16 +323,18 @@ export const ChatMessage = memo(function ChatMessage({
if (!kind || !file.filePath) return true;
return validatedPaths[file.filePath] === true;
});
const attachedFiles = suppressProcessAttachments && (hasText || images.length > 0 || visibleTools.length > 0)
const attachedFiles = suppressProcessAttachments && (hasText || resolvableContentImages.length > 0 || visibleTools.length > 0)
? processVisibleAttachments
: existingDerivedAttachedFiles;
const imageCopyTarget = resolvePrimaryImageCopyTarget(resolvableContentImages, attachedFiles);
const showAssistantHoverBar = !isUser && (hasText || imageCopyTarget != null);
const [lightboxImg, setLightboxImg] = useState<{ src: string; fileName: string; filePath?: string; base64?: string; mimeType?: string } | null>(null);
// Never render tool result messages in chat UI
if (isToolResult) return null;
const hasStreamingToolStatus = isStreaming && streamingTools.length > 0;
if (!hasText && images.length === 0 && visibleTools.length === 0 && attachedFiles.length === 0 && !hasStreamingToolStatus) return null;
if (!hasText && resolvableContentImages.length === 0 && visibleTools.length === 0 && attachedFiles.length === 0 && !hasStreamingToolStatus) return null;
return (
<div
@@ -359,9 +363,9 @@ export const ChatMessage = memo(function ChatMessage({
{/* Images — rendered ABOVE text bubble for user messages */}
{/* Images from content blocks (Gateway session data / channel push photos) */}
{isUser && images.length > 0 && (
{isUser && resolvableContentImages.length > 0 && (
<div className="flex flex-wrap gap-2">
{images.map((img, i) => {
{resolvableContentImages.map((img, i) => {
const src = imageSrc(img);
if (!src) return null;
return (
@@ -384,7 +388,7 @@ export const ChatMessage = memo(function ChatMessage({
{attachedFiles.map((file, i) => {
const isImage = file.mimeType.startsWith('image/');
// Skip image attachments if we already have images from content blocks
if (isImage && images.length > 0) return null;
if (isImage && resolvableContentImages.length > 0) return null;
if (isImage) {
return file.preview ? (
<ImageThumbnail
@@ -420,9 +424,9 @@ export const ChatMessage = memo(function ChatMessage({
)}
{/* Images from content blocks — assistant messages (below text) */}
{!isUser && images.length > 0 && (
{!isUser && resolvableContentImages.length > 0 && (
<div className="flex flex-wrap gap-2">
{images.map((img, i) => {
{resolvableContentImages.map((img, i) => {
const src = imageSrc(img);
if (!src) return null;
return (
@@ -444,7 +448,7 @@ export const ChatMessage = memo(function ChatMessage({
<div className="flex flex-wrap gap-2">
{attachedFiles.map((file, i) => {
const isImage = file.mimeType.startsWith('image/');
if (isImage && images.length > 0) return null;
if (isImage && resolvableContentImages.length > 0) return null;
if (isImage && file.preview) {
return (
<ImagePreviewCard
@@ -476,9 +480,9 @@ export const ChatMessage = memo(function ChatMessage({
</span>
)}
{/* Hover row for assistant messages — only when there is real text content */}
{!isUser && hasText && (
<AssistantHoverBar text={text} timestamp={message.timestamp} />
{/* Hover row for assistant messages */}
{showAssistantHoverBar && (
<AssistantHoverBar text={text} timestamp={message.timestamp} imageCopyTarget={imageCopyTarget} />
)}
</div>
@@ -547,16 +551,60 @@ function ToolStatusBar({
);
}
function resolvePrimaryImageCopyTarget(
images: Array<{ mimeType: string; data?: string; url?: string }>,
attachedFiles: AttachedFileMeta[],
): ImageCopyTarget | null {
for (const file of attachedFiles) {
if (!file.mimeType.startsWith('image/')) continue;
if (file.filePath || file.preview?.startsWith('data:')) {
return {
filePath: file.filePath,
preview: file.preview,
mimeType: file.mimeType,
};
}
}
for (const image of images) {
if (image.data) {
return { base64: image.data, mimeType: image.mimeType };
}
if (image.url?.startsWith('data:')) {
return { preview: image.url, mimeType: image.mimeType };
}
}
return null;
}
// ── Assistant hover bar (timestamp + copy, shown on group hover) ─
function AssistantHoverBar({ text, timestamp }: { text: string; timestamp?: number }) {
function AssistantHoverBar({
text,
timestamp,
imageCopyTarget,
}: {
text: string;
timestamp?: number;
imageCopyTarget?: ImageCopyTarget | null;
}) {
const [copied, setCopied] = useState(false);
const copyContent = useCallback(() => {
navigator.clipboard.writeText(text);
const copyContent = useCallback(async () => {
if (imageCopyTarget) {
const copiedImage = await copyImageToClipboard(imageCopyTarget);
if (copiedImage) {
setCopied(true);
setTimeout(() => setCopied(false), 2000);
return;
}
}
if (!text.trim()) return;
await navigator.clipboard.writeText(text);
setCopied(true);
setTimeout(() => setCopied(false), 2000);
}, [text]);
}, [text, imageCopyTarget]);
return (
<div className="flex items-center justify-between w-full opacity-0 group-hover:opacity-100 transition-opacity duration-200 select-none px-1">
@@ -629,6 +677,12 @@ function AssistantMarkdown({
</a>
);
},
img({ src, alt }) {
if (!src || isUnresolvableImageUrl(String(src))) return null;
return (
<img src={String(src)} alt={typeof alt === 'string' ? alt : 'image'} className="max-w-full rounded-lg" />
);
},
}}
>
{normalizeLatexDelimiters(text)}

View File

@@ -47,6 +47,9 @@ function StepDetailCard({ step }: { step: TaskStep }) {
const isNarration = step.kind === 'message';
const isTool = step.kind === 'tool';
const isThinking = step.kind === 'thinking';
const displayToolLabel = isTool && step.label === 'image_generate'
? t('executionGraph.imageGenerateLabel')
: step.label;
// System steps (subagent branch roots etc.) share the tool row layout:
// bold label + truncated single-line detail preview + click-to-expand,
// i.e. no rounded card / no separate detail line below the title.
@@ -56,7 +59,7 @@ function StepDetailCard({ step }: { step: TaskStep }) {
const hideStatusText = (isTool || isSystem) && step.status === 'completed';
const detailPreview = step.detail?.replace(/\s+/g, ' ').trim();
const canExpand = hasDetail;
const displayLabel = isThinking ? t('executionGraph.thinkingLabel') : step.label;
const displayLabel = isThinking ? t('executionGraph.thinkingLabel') : (isTool ? displayToolLabel : step.label);
return (
<div

View File

@@ -0,0 +1,52 @@
import { readBinaryFile } from '@/lib/api-client';
export interface ImageCopyTarget {
filePath?: string;
preview?: string | null;
mimeType?: string;
base64?: string;
}
function blobFromBytes(data: Uint8Array, mimeType: string): Blob {
const copy = new Uint8Array(data);
return new Blob([copy], { type: mimeType });
}
async function blobFromDataUrl(dataUrl: string): Promise<Blob | null> {
try {
const response = await fetch(dataUrl);
if (!response.ok) return null;
return await response.blob();
} catch {
return null;
}
}
async function writeImageBlob(blob: Blob, fallbackMimeType: string): Promise<boolean> {
if (!navigator.clipboard?.write || typeof ClipboardItem === 'undefined') return false;
const type = blob.type || fallbackMimeType;
await navigator.clipboard.write([new ClipboardItem({ [type]: blob })]);
return true;
}
export async function copyImageToClipboard(target: ImageCopyTarget): Promise<boolean> {
const mimeType = target.mimeType?.startsWith('image/') ? target.mimeType : 'image/png';
if (target.base64) {
const blob = await blobFromDataUrl(`data:${mimeType};base64,${target.base64}`);
return blob ? writeImageBlob(blob, mimeType) : false;
}
if (target.preview?.startsWith('data:')) {
const blob = await blobFromDataUrl(target.preview);
return blob ? writeImageBlob(blob, mimeType) : false;
}
if (target.filePath) {
const result = await readBinaryFile(target.filePath);
if (!result.ok || !result.data) return false;
return writeImageBlob(blobFromBytes(result.data, result.mimeType || mimeType), mimeType);
}
return false;
}

View File

@@ -0,0 +1,162 @@
import type { RawMessage, ToolStatus } from '@/stores/chat';
import {
extractImages,
extractToolUse,
isGeneratingStatusNarration,
isInternalAssistantReplyText,
} from './message-utils';
const IMAGE_GENERATE_TOOL = 'image_generate';
const ASYNC_IMAGE_TASK_START_RE = /Background task started for image generation \(([0-9a-f-]{36})\)/i;
const INTER_SESSION_IMAGE_TASK_RE = /sourceSession=image_generate:([0-9a-f-]{36})/i;
/** Match OpenClaw agents.defaults.imageGenerationModel.timeoutMs default. */
export const IMAGE_GENERATION_TIMEOUT_MS = 180_000;
const IMAGE_GENERATION_TIMEOUT_BUFFER_MS = 15_000;
function toMs(timestamp: number | undefined): number | null {
if (timestamp == null || !Number.isFinite(timestamp)) return null;
return timestamp < 1e12 ? timestamp * 1000 : timestamp;
}
function getMessagePlainText(message: RawMessage): string {
const { content } = message;
if (typeof content === 'string') return content;
if (!Array.isArray(content)) return '';
const parts: string[] = [];
for (const block of content as Array<{ type?: string; text?: string }>) {
if (block.type === 'text' && typeof block.text === 'string') {
parts.push(block.text);
}
}
return parts.join('\n');
}
function messageHasDeliveredImage(message: RawMessage): boolean {
if ((message._attachedFiles ?? []).some((file) => file.mimeType.startsWith('image/'))) {
return true;
}
return extractImages(message).length > 0;
}
function findLastImageGenerateToolCallIndex(segmentMessages: RawMessage[]): number {
for (let index = segmentMessages.length - 1; index >= 0; index -= 1) {
const message = segmentMessages[index];
if (message.role !== 'assistant') continue;
if (extractToolUse(message).some((tool) => tool.name === IMAGE_GENERATE_TOOL)) {
return index;
}
}
return -1;
}
function collectAsyncImageTaskStarts(
segmentMessages: RawMessage[],
): Array<{ taskId: string; startedAtMs: number | null }> {
const starts: Array<{ taskId: string; startedAtMs: number | null }> = [];
for (const message of segmentMessages) {
const role = String(message.role ?? '').toLowerCase();
if (role !== 'toolresult' && role !== 'tool_result') continue;
if (message.toolName !== IMAGE_GENERATE_TOOL) continue;
const match = getMessagePlainText(message).match(ASYNC_IMAGE_TASK_START_RE);
if (!match?.[1]) continue;
starts.push({
taskId: match[1],
startedAtMs: toMs(message.timestamp),
});
}
return starts;
}
function collectCompletedAsyncImageTaskIds(segmentMessages: RawMessage[]): Set<string> {
const completed = new Set<string>();
for (const message of segmentMessages) {
const role = String(message.role ?? '').toLowerCase();
if (role !== 'user') continue;
const text = getMessagePlainText(message).trim();
if (!/^\[Inter-session message\]/i.test(text)) continue;
const match = text.match(INTER_SESSION_IMAGE_TASK_RE);
if (match?.[1]) completed.add(match[1]);
}
return completed;
}
function hasTimedOut(startedAtMs: number | null, now: number): boolean {
if (startedAtMs == null) return false;
return now - startedAtMs > IMAGE_GENERATION_TIMEOUT_MS + IMAGE_GENERATION_TIMEOUT_BUFFER_MS;
}
function hasDeliveredImageAfter(segmentMessages: RawMessage[], fromIndex: number): boolean {
for (let index = fromIndex + 1; index < segmentMessages.length; index += 1) {
const message = segmentMessages[index];
if (message.role !== 'assistant') continue;
if (messageHasDeliveredImage(message)) return true;
}
return false;
}
function hasTerminalReplyAfterToolCall(segmentMessages: RawMessage[], fromIndex: number): boolean {
for (let index = fromIndex + 1; index < segmentMessages.length; index += 1) {
const message = segmentMessages[index];
if (message.role !== 'assistant') continue;
if (extractToolUse(message).length > 0) continue;
const text = getMessagePlainText(message).trim();
if (!text || isInternalAssistantReplyText(text)) continue;
if (isGeneratingStatusNarration(text)) continue;
return true;
}
return false;
}
function hasFreshRunningStreamingTool(streamingTools: ToolStatus[], now: number): boolean {
return streamingTools.some((tool) => {
if (tool.name !== IMAGE_GENERATE_TOOL || tool.status !== 'running') return false;
return now - tool.updatedAt <= IMAGE_GENERATION_TIMEOUT_MS + IMAGE_GENERATION_TIMEOUT_BUFFER_MS;
});
}
/** True while an async `image_generate` task is in flight for this run segment. */
export function isImageGenerationPending(
segmentMessages: RawMessage[],
streamingTools: ToolStatus[] = [],
now = Date.now(),
): boolean {
if (hasFreshRunningStreamingTool(streamingTools, now)) {
return true;
}
const toolCallIndex = findLastImageGenerateToolCallIndex(segmentMessages);
const asyncStarts = collectAsyncImageTaskStarts(segmentMessages);
const completedTaskIds = collectCompletedAsyncImageTaskIds(segmentMessages);
if (toolCallIndex < 0 && asyncStarts.length === 0) {
return false;
}
const toolCallStartedAt = toolCallIndex >= 0
? toMs(segmentMessages[toolCallIndex]?.timestamp)
: null;
const latestAsyncStartedAt = asyncStarts.reduce<number | null>((latest, start) => {
if (start.startedAtMs == null) return latest;
return latest == null ? start.startedAtMs : Math.max(latest, start.startedAtMs);
}, null);
const anchorStartedAt = [toolCallStartedAt, latestAsyncStartedAt]
.filter((value): value is number => value != null)
.reduce<number | null>((latest, value) => (latest == null ? value : Math.max(latest, value)), null);
if (hasTimedOut(anchorStartedAt, now)) {
return false;
}
if (toolCallIndex >= 0) {
if (hasDeliveredImageAfter(segmentMessages, toolCallIndex)) return false;
if (hasTerminalReplyAfterToolCall(segmentMessages, toolCallIndex)) return false;
}
if (asyncStarts.length > 0) {
const hasOpenAsyncTask = asyncStarts.some((start) => !completedTaskIds.has(start.taskId));
if (!hasOpenAsyncTask) return false;
}
return toolCallIndex >= 0 || asyncStarts.some((start) => !completedTaskIds.has(start.taskId));
}

View File

@@ -7,6 +7,7 @@
import { Suspense, lazy, useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { AlertCircle, ArrowDownToLine, Loader2, Sparkles } from 'lucide-react';
import { useChatStore, type RawMessage } from '@/stores/chat';
import { isInternalMessage } from '@/stores/chat/helpers';
import { buildBaselineRunKey, getBaseline } from '@/stores/baseline-cache';
import { useGatewayStore } from '@/stores/gateway';
import { useAgentsStore } from '@/stores/agents';
@@ -18,7 +19,7 @@ import { ChatMessage } from './ChatMessage';
import { ChatInput } from './ChatInput';
import { ExecutionGraphCard } from './ExecutionGraphCard';
import { ChatToolbar } from './ChatToolbar';
import { extractImages, extractText, extractThinking, extractToolUse, normalizeMessageRole, stripProcessMessagePrefix } from './message-utils';
import { extractImages, extractText, extractThinking, extractToolUse, isInternalAssistantReplyText, isInternalProcessNarration, normalizeMessageRole, stripProcessMessagePrefix } from './message-utils';
import {
buildRunSegmentMessageIndices,
deriveTaskSteps,
@@ -30,6 +31,7 @@ import {
segmentHasFinalReply,
type TaskStep,
} from './task-visualization';
import { isImageGenerationPending } from './image-generation-status';
import { useTranslation } from 'react-i18next';
import { cn } from '@/lib/utils';
import { useStickToBottomInstant } from '@/hooks/use-stick-to-bottom-instant';
@@ -95,6 +97,14 @@ function getPrimaryMessageStepTexts(steps: TaskStep[]): string[] {
.map((step) => step.detail!);
}
function sanitizeGraphSteps(steps: TaskStep[]): TaskStep[] {
return steps.filter((step) => {
if (step.kind === 'thinking') return false;
if (step.kind === 'message' && step.detail && isInternalProcessNarration(step.detail)) return false;
return true;
});
}
function buildQuestionDirectoryTitle(message: RawMessage, fallback: string): string {
const normalized = extractText(message).replace(/\s+/g, ' ').trim();
if (!normalized) return fallback;
@@ -103,6 +113,7 @@ function buildQuestionDirectoryTitle(message: RawMessage, fallback: string): str
function isRealUserMessage(msg: RawMessage): boolean {
if (normalizeMessageRole(msg.role) !== 'user') return false;
if (isInternalMessage(msg)) return false;
const content = msg.content;
if (!Array.isArray(content)) return true;
// If every block in the content is a tool_result, this is a Gateway
@@ -111,6 +122,10 @@ function isRealUserMessage(msg: RawMessage): boolean {
return blocks.length === 0 || !blocks.every((b) => b.type === 'tool_result' || b.type === 'toolResult');
}
function hasUserFacingImageAttachments(msg: RawMessage): boolean {
return (msg._attachedFiles ?? []).some((file) => file.mimeType.startsWith('image/'));
}
function generatedFileToTarget(file: GeneratedFile): FilePreviewTarget {
return {
filePath: file.filePath,
@@ -493,15 +508,16 @@ export function Chat() {
&& streamTools.length === 0
&& !hasRunningStreamToolStatus;
let steps = buildSteps(rawStreamingReplyCandidate);
let steps = sanitizeGraphSteps(buildSteps(rawStreamingReplyCandidate));
let streamingReplyText: string | null = null;
if (rawStreamingReplyCandidate) {
const trimmedReplyText = stripProcessMessagePrefix(streamText, getPrimaryMessageStepTexts(steps));
const hasReplyText = trimmedReplyText.trim().length > 0;
const hasReplyText = trimmedReplyText.trim().length > 0
&& !isInternalAssistantReplyText(trimmedReplyText);
if (hasReplyText || hasStreamImages) {
streamingReplyText = trimmedReplyText;
streamingReplyText = hasReplyText ? trimmedReplyText : '';
} else {
steps = buildSteps(false);
steps = sanitizeGraphSteps(buildSteps(false));
}
}
@@ -545,9 +561,9 @@ export function Chat() {
// generated message steps that include accumulated narration + reply
// text. Strip these out — historical message steps (from messages[])
// will be properly recomputed on the next render with fresh data.
const cleanedSteps = cached.steps.filter(
const cleanedSteps = sanitizeGraphSteps(cached.steps.filter(
(s) => !(s.kind === 'message' && s.id.startsWith('stream-message')),
);
));
return [{
triggerIndex: idx,
replyIndex: cached.replyIndex,
@@ -622,6 +638,7 @@ export function Chat() {
}, [messages, subagentCompletionInfos, currentSessionKey, streamingMessage, streamingTools, pendingFinal, sending, hasAnyStreamContent, hasStreamText, hasStreamImages, streamText, streamTools.length, hasRunningStreamToolStatus, childTranscripts, currentAgentId, agents, sessionLabels, graphStepCache, runError, isRunTrigger]);
const hasActiveExecutionGraph = userRunCards.some((card) => card.active);
let latestRunSegmentCompletion = { hasFinalReply: false, hasToolActivity: false };
let pendingImageGeneration = false;
for (let idx = messages.length - 1; idx >= 0; idx -= 1) {
if (!isRealUserMessage(messages[idx]) || subagentCompletionInfos[idx]) continue;
const nextUserIndex = nextUserMessageIndexes[idx];
@@ -632,6 +649,10 @@ export function Chat() {
m.role === 'assistant' && extractToolUse(m).length > 0,
),
};
pendingImageGeneration = isImageGenerationPending(
postTrigger,
nextUserIndex === -1 ? streamingTools : [],
);
break;
}
const runSettledInHistory = latestRunSegmentCompletion.hasFinalReply
@@ -824,7 +845,9 @@ export function Chat() {
</div>
)}
{messages.map((msg, idx) => {
if (foldedNarrationIndices.has(idx)) return null;
if (isInternalMessage(msg) && !hasUserFacingImageAttachments(msg)) return null;
const isFoldedNarration = foldedNarrationIndices.has(idx);
if (isFoldedNarration && !hasUserFacingImageAttachments(msg)) return null;
const suppressToolCards = runSegmentMessageIndices.has(idx);
const isToolOnlyAssistant = normalizeMessageRole(msg.role) === 'assistant'
&& extractToolUse(msg).length > 0
@@ -843,6 +866,7 @@ export function Chat() {
<ChatMessage
message={msg}
textOverride={replyTextOverrides.get(idx)}
suppressAssistantText={isFoldedNarration}
suppressToolCards={suppressToolCards}
suppressProcessAttachments={suppressToolCards}
onOpenFile={handleOpenAttachedFile}
@@ -946,8 +970,12 @@ export function Chat() {
<ActivityIndicator phase="tool_processing" />
)}
{pendingImageGeneration && (
<ImageGeneratingIndicator />
)}
{/* Typing indicator when sending but no stream content yet */}
{inputRunActive && !pendingFinal && !hasAnyStreamContent && !hasActiveExecutionGraph && (
{inputRunActive && !pendingFinal && !hasAnyStreamContent && !hasActiveExecutionGraph && !pendingImageGeneration && (
<TypingIndicator />
)}
</>
@@ -1195,4 +1223,21 @@ function ActivityIndicator({ phase }: { phase: 'tool_processing' }) {
);
}
function ImageGeneratingIndicator() {
const { t } = useTranslation('chat');
return (
<div className="flex gap-3" data-testid="chat-image-generating-indicator">
<div className="flex h-8 w-8 shrink-0 items-center justify-center rounded-full mt-1 bg-black/5 dark:bg-white/5 text-foreground">
<Sparkles className="h-4 w-4" />
</div>
<div className="bg-black/5 dark:bg-white/5 text-foreground rounded-2xl px-4 py-3">
<div className="flex items-center gap-2 text-sm text-muted-foreground">
<Loader2 className="h-3.5 w-3.5 animate-spin text-primary" />
<span>{t('imageGeneration.generating')}</span>
</div>
</div>
</div>
);
}
export default Chat;

View File

@@ -5,13 +5,39 @@
*/
import type { RawMessage, ContentBlock } from '@/stores/chat';
/**
* Strip OpenClaw's inbound-image vision envelope from user message text.
* The runtime expands uploaded images into:
* [Image]
* User text:
* <message or "Process the attached file(s).">
* Description:
* <auto-generated caption>
*/
function stripInboundMediaVisionEnvelope(text: string): string {
if (!/\[Image\]/i.test(text) && !/^User text:/im.test(text) && !/\nDescription:\s*\n/i.test(text)) {
return text;
}
let result = text.replace(/^\s*\[Image\]\s*\n?/i, '');
const userTextBlock = result.match(/^User text:\s*\n([\s\S]*?)(?:\n\s*Description:\s*\n[\s\S]*)?\s*$/i);
if (userTextBlock) {
const userText = userTextBlock[1].trim();
return /^Process the attached file\(s\)\.\s*$/i.test(userText) ? '' : userText;
}
return result.replace(/\n\s*Description:\s*\n[\s\S]*$/i, '').trim();
}
/**
* Clean Gateway metadata from user message text for display.
* Strips: [media attached: ... | ...], [message_id: ...],
* and the timestamp prefix [Day Date Time Timezone].
*/
function cleanUserText(text: string): string {
return text
return stripInboundMediaVisionEnvelope(
text
// Remove [media attached: path (mime) | path] references
.replace(/\s*\[media attached:[^\]]*\]/g, '')
// Remove [message_id: uuid]
@@ -30,7 +56,8 @@ function cleanUserText(text: string): string {
.replace(/^Conversation info\s*\([^)]*\):\s*\{[\s\S]*?\}\s*/i, '')
// Remove Gateway timestamp prefix like [Fri 2026-02-13 22:39 GMT+8]
.replace(/^\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, '')
.trim();
.trim(),
);
}
/**
@@ -51,8 +78,13 @@ function stripAssistantMediaTags(text: string): string {
// would still leak the literal `MEDIA:/.../截屏 2026-05-06 17.46.51.png`
// to the user when the underlying path detection succeeds.
const tagged = new RegExp(`(^|[\\s(\\[{>])(?:MEDIA|media):(?:\\/|~\\/)[^\\n"'()\\[\\],<>]*?\\.(?:${exts})(?=$|[\\s\\n"'()\\[\\],<>]|[,。;;,.!?])`, 'g');
// Bare OpenClaw artifact paths emitted alongside `_attachedFiles` cards.
// Scope to `.openclaw/media/` so normal absolute paths in prose stay visible.
const bareOpenClawMedia = new RegExp(`(^|[\\s(\\[{>])(?:(?:\\/|~\\/)[^\\n"'()\\[\\],<>]*?\\.openclaw\\/media\\/[^\\n"'()\\[\\],<>]*?\\.(?:${exts}))(?=$|[\\s\\n"'()\\[\\],<>]|[,。;;,.!?])`, 'g');
return text
.replace(tagged, (_, lead: string) => lead)
.replace(bareOpenClawMedia, (_, lead: string) => lead)
.replace(/!\[[^\]]*\]\([^)]+\)/g, '')
// Collapse the empty lines / orphan whitespace the strip leaves behind.
.replace(/[ \t]+\n/g, '\n')
.replace(/\n{3,}/g, '\n\n')
@@ -143,6 +175,57 @@ function consumeLeadingSegment(text: string, segment: string): number {
return match ? match[0].length : 0;
}
/** True for OpenClaw internal assistant tokens that must never appear in the chat UI. */
export function isInternalAssistantReplyText(text: string): boolean {
return /^(HEARTBEAT_OK|NO_REPLY)\s*$/i.test(text.trim());
}
/** Interim image-generation status the user should not see as a final answer. */
export function isGeneratingStatusNarration(text: string): boolean {
const trimmed = text.trim();
if (!trimmed) return false;
if (/^(?:图片(?:正在)?生成中|正在生成(?:图片|图像)|生成中)/i.test(trimmed)) return true;
if (/稍等(片刻|一下)?/.test(trimmed) && trimmed.length <= 80 && /生成|generat/i.test(trimmed)) return true;
return false;
}
/** OpenClaw runtime continuation prompt injected between internal async events. */
export function isOpenClawRuntimeEventPrompt(text: string): boolean {
const trimmed = text.trim();
if (!trimmed) return false;
if (/^Continue the OpenClaw runtime event\.?\s*$/i.test(trimmed)) return true;
// Some transcripts store the prompt as the only line in a longer envelope.
return trimmed.split(/\n+/).some((line) => /^Continue the OpenClaw runtime event\.?\s*$/i.test(line.trim()));
}
/** Process narration that should never appear in the execution graph or chat stream. */
export function isInternalProcessNarration(text: string): boolean {
const trimmed = text.trim();
if (!trimmed) return true;
if (isInternalAssistantReplyText(trimmed)) return true;
if (isGeneratingStatusNarration(trimmed)) return true;
if (isOpenClawRuntimeEventPrompt(trimmed)) return true;
if (/^\[Inter-session message\]/i.test(trimmed)) return true;
if (/OpenClaw runtime event/i.test(trimmed)) return true;
if (/Handle the result internally/i.test(trimmed) && /Do not relay it to the user/i.test(trimmed)) {
return true;
}
return false;
}
/** True when a renderer `src` cannot be fetched directly (Gateway proxy / local paths). */
export function isUnresolvableImageUrl(url: string): boolean {
const trimmed = url.trim();
if (!trimmed) return true;
if (trimmed.startsWith('data:')) return false;
if (/^https?:\/\//i.test(trimmed)) return false;
if (trimmed.startsWith('/api/chat/media/')) return true;
if (trimmed.startsWith('file://')) return true;
if (trimmed.includes('.openclaw/media/')) return true;
if (trimmed.startsWith('/') || trimmed.startsWith('~/')) return true;
return false;
}
/**
* Extract displayable text from a message's content field.
* Handles both string content and array-of-blocks content.
@@ -178,6 +261,8 @@ export function extractText(message: RawMessage | unknown): string {
if (isUser && result) {
result = cleanUserText(result);
} else if (!isUser && result) {
if (isInternalAssistantReplyText(result)) return '';
if (isGeneratingStatusNarration(result)) return '';
// Assistant-side cleanup: keep the bubble free of `MEDIA:/path` tags
// that the runtime emits to point at produced artifacts. The same
// path is surfaced as a clickable file card via `_attachedFiles`,
@@ -348,7 +433,7 @@ export function extractImages(
const src = block.source;
if (src.type === 'base64' && src.media_type && src.data) {
images.push({ mimeType: src.media_type, data: src.data });
} else if (src.type === 'url' && src.url) {
} else if (src.type === 'url' && src.url && !isUnresolvableImageUrl(src.url)) {
images.push({ mimeType: src.media_type || 'image/jpeg', url: src.url });
}
continue;

View File

@@ -1,4 +1,12 @@
import { extractText, extractTextSegments, extractThinkingSegments, extractToolUse } from './message-utils';
import {
extractText,
extractTextSegments,
extractToolUse,
isGeneratingStatusNarration,
isInternalAssistantReplyText,
isInternalProcessNarration,
} from './message-utils';
import { isInternalMessage } from '@/stores/chat/helpers';
import type { RawMessage, ToolStatus } from '@/stores/chat';
export type TaskStepStatus = 'running' | 'completed' | 'error';
@@ -37,7 +45,9 @@ export function findReplyMessageIndex(messages: RawMessage[], hasStreamingReply:
for (let idx = messages.length - 1; idx >= 0; idx -= 1) {
const message = messages[idx];
if (!message || message.role !== 'assistant') continue;
if (extractText(message).trim().length === 0) continue;
const replyText = extractText(message).trim();
if (replyText.length === 0 || isInternalAssistantReplyText(replyText)) continue;
if (isGeneratingStatusNarration(replyText)) continue;
return idx;
}
return -1;
@@ -149,7 +159,9 @@ export function segmentHasFinalReply(segmentMessages: RawMessage[]): boolean {
return segmentMessages.some((message, index) => {
if (index <= lastToolUseOffset) return false;
if (message.role !== 'assistant') return false;
if (extractText(message).trim().length === 0) return false;
const replyText = extractText(message).trim();
if (replyText.length === 0 || isInternalAssistantReplyText(replyText)) return false;
if (isGeneratingStatusNarration(replyText)) return false;
const content = message.content;
if (!Array.isArray(content)) return true;
return !(content as Array<{ type?: string }>).some(
@@ -310,7 +322,8 @@ function appendDetailSegments(
): void {
const normalizedSegments = segments
.map((segment) => normalizeText(segment))
.filter((segment): segment is string => !!segment);
.filter((segment): segment is string => !!segment)
.filter((segment) => !isInternalProcessNarration(segment));
normalizedSegments.forEach((detail, index) => {
options.upsertStep({
@@ -361,31 +374,27 @@ export function deriveTaskSteps({
for (const [messageIndex, message] of messages.entries()) {
if (!message || message.role !== 'assistant') continue;
appendDetailSegments(extractThinkingSegments(message), {
idPrefix: `history-thinking-${message.id || messageIndex}`,
label: 'Thinking',
kind: 'thinking',
running: false,
upsertStep,
});
const toolUses = extractToolUse(message);
// Fold any intermediate assistant text into the graph as a narration
// step — including text that lives on a mixed `text + toolCall` message.
// The narration step is emitted BEFORE the tool steps so the graph
// preserves the original ordering (the assistant "thinks out loud" and
// then invokes the tool).
const narrationSegments = extractTextSegments(message);
const graphNarrationSegments = messageIndex === replyIndex
? narrationSegments.slice(0, -1)
: narrationSegments;
appendDetailSegments(graphNarrationSegments, {
idPrefix: `history-message-${message.id || messageIndex}`,
label: 'Message',
kind: 'message',
running: false,
upsertStep,
});
if (!isInternalMessage(message)) {
// Fold any intermediate assistant text into the graph as a narration
// step — including text that lives on a mixed `text + toolCall` message.
// The narration step is emitted BEFORE the tool steps so the graph
// preserves the original ordering (the assistant "thinks out loud" and
// then invokes the tool).
const narrationSegments = extractTextSegments(message);
const graphNarrationSegments = messageIndex === replyIndex
? narrationSegments.slice(0, -1)
: narrationSegments;
appendDetailSegments(graphNarrationSegments, {
idPrefix: `history-message-${message.id || messageIndex}`,
label: 'Message',
kind: 'message',
running: false,
upsertStep,
});
} else if (toolUses.length === 0) {
continue;
}
toolUses.forEach((tool, index) => {
const input = tool.input as Record<string, unknown>;
@@ -403,19 +412,6 @@ export function deriveTaskSteps({
}
if (streamMessage) {
// When the reply is being rendered as a separate bubble
// (omitLastStreamingMessageSegment), thinking that accompanies
// the reply belongs to the bubble — omit it from the graph.
if (!omitLastStreamingMessageSegment) {
appendDetailSegments(extractThinkingSegments(streamMessage), {
idPrefix: 'stream-thinking',
label: 'Thinking',
kind: 'thinking',
running: true,
upsertStep,
});
}
// Stream-time narration should also appear in the execution graph so that
// intermediate process output stays in P1 instead of leaking into the
// assistant reply area.

View File

@@ -45,6 +45,12 @@ import {
type ToolStatus,
} from './chat/types';
import type { ChatGet, ChatSet } from './chat/store-api';
import { enrichWithToolCallAttachments, shouldDropMessageFromHistory } from './chat/helpers';
import {
isGeneratingStatusNarration,
isInternalAssistantReplyText,
isOpenClawRuntimeEventPrompt,
} from '@/pages/Chat/message-utils';
export type {
AttachedFileMeta,
@@ -567,8 +573,25 @@ function normalizeStreamingMessage(message: unknown): unknown {
* `[media attached:` instead — leaving the timestamp in the normalized
* comparison text and breaking optimistic-vs-echo dedupe.
*/
function stripInboundMediaVisionEnvelope(text: string): string {
if (!/\[Image\]/i.test(text) && !/^User text:/im.test(text) && !/\nDescription:\s*\n/i.test(text)) {
return text;
}
let result = text.replace(/^\s*\[Image\]\s*\n?/i, '');
const userTextBlock = result.match(/^User text:\s*\n([\s\S]*?)(?:\n\s*Description:\s*\n[\s\S]*)?\s*$/i);
if (userTextBlock) {
const userText = userTextBlock[1].trim();
return /^Process the attached file\(s\)\.\s*$/i.test(userText) ? '' : userText;
}
return result.replace(/\n\s*Description:\s*\n[\s\S]*$/i, '').trim();
}
function stripGatewayUserMetadata(text: string): string {
return text
return stripInboundMediaVisionEnvelope(
text
.replace(/\s*\[media attached:[^\]]*\]/g, '')
.replace(/\s*\[message_id:\s*[^\]]+\]/g, '')
.replace(/^Sender\s*\([^)]*\)\s*:\s*```[a-z]*\n[\s\S]*?```\s*/i, '')
@@ -579,13 +602,16 @@ function stripGatewayUserMetadata(text: string): string {
.replace(/^Sender\s*:\s*[^\n]*(?:\n\s*)*/i, '')
.replace(/^Conversation info\s*\([^)]*\):\s*```[a-z]*\n[\s\S]*?```\s*/i, '')
.replace(/^Conversation info\s*\([^)]*\):\s*\{[\s\S]*?\}\s*/i, '')
.replace(/^\s*\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, '');
.replace(/^\s*\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, ''),
);
}
function normalizeComparableUserText(content: unknown): string {
return stripGatewayUserMetadata(getMessageText(content))
const text = stripGatewayUserMetadata(getMessageText(content))
.replace(/\s+/g, ' ')
.trim();
if (/^\(file attached\)$/i.test(text)) return '';
return text;
}
function getComparableAttachmentSignature(message: Pick<RawMessage, '_attachedFiles'>): string {
@@ -620,6 +646,13 @@ function matchesOptimisticUserMessage(
if (sameText && sameAttachments) return true;
if (sameText && (!optimisticAttachments || !candidateAttachments) && (timestampMatches || !hasCandidateTimestamp)) return true;
if (sameAttachments && (!optimisticText || !candidateText) && (timestampMatches || !hasCandidateTimestamp)) return true;
const optimisticHadAttachmentsOnly = optimisticAttachments.length > 0 && !optimisticText;
const candidateIsAttachmentEcho = !candidateText
&& /\[(?:media attached:|\s*Image\s*\])/i.test(getMessageText(candidate.content));
if (optimisticHadAttachmentsOnly && candidateIsAttachmentEcho && (timestampMatches || !hasCandidateTimestamp)) {
return true;
}
return false;
}
@@ -768,6 +801,13 @@ function getMessageText(content: unknown): string {
return '';
}
function getMessageTextForFilter(msg: { content?: unknown; text?: unknown }): string {
const fromContent = getMessageText(msg.content);
if (fromContent.trim()) return fromContent;
if (typeof msg.text === 'string') return msg.text;
return '';
}
function getMessageStopReason(message: RawMessage | unknown): string | null {
if (!message || typeof message !== 'object') return null;
const msg = message as Record<string, unknown>;
@@ -853,6 +893,26 @@ function mimeFromExtension(filePath: string): string {
return map[ext] || 'application/octet-stream';
}
function extractFilePathsFromToolArgs(args: Record<string, unknown>): string[] {
const paths: string[] = [];
const direct = args.file_path ?? args.filePath ?? args.path ?? args.file;
if (typeof direct === 'string' && direct.trim()) paths.push(direct.trim());
const attachments = args.attachments;
if (Array.isArray(attachments)) {
for (const item of attachments) {
if (!item || typeof item !== 'object') continue;
const att = item as Record<string, unknown>;
const filePath = att.filePath ?? att.file_path ?? att.path ?? att.file;
if (typeof filePath === 'string' && filePath.trim()) {
paths.push(filePath.trim());
}
}
}
return paths;
}
const DIRECTORY_MIME_TYPE = 'application/x-directory';
function trimPathTerminators(filePath: string): string {
@@ -1018,8 +1078,8 @@ function getToolCallFilePath(msg: RawMessage, toolCallId: string): string | unde
if ((block.type === 'tool_use' || block.type === 'toolCall') && block.id === toolCallId) {
const args = (block.input ?? block.arguments) as Record<string, unknown> | undefined;
if (args) {
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
if (typeof fp === 'string') return fp;
const paths = extractFilePathsFromToolArgs(args);
if (paths[0]) return paths[0];
}
}
}
@@ -1037,8 +1097,8 @@ function getToolCallFilePath(msg: RawMessage, toolCallId: string): string | unde
args = typeof fn.arguments === 'string' ? JSON.parse(fn.arguments) : (fn.arguments ?? fn.input) as Record<string, unknown>;
} catch { /* ignore */ }
if (args) {
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
if (typeof fp === 'string') return fp;
const paths = extractFilePathsFromToolArgs(args);
if (paths[0]) return paths[0];
}
}
}
@@ -1056,8 +1116,8 @@ function collectToolCallPaths(msg: RawMessage, paths: Map<string, string>): void
if ((block.type === 'tool_use' || block.type === 'toolCall') && block.id) {
const args = (block.input ?? block.arguments) as Record<string, unknown> | undefined;
if (args) {
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
if (typeof fp === 'string') paths.set(block.id, fp);
const filePaths = extractFilePathsFromToolArgs(args);
if (filePaths[0]) paths.set(block.id, filePaths[0]);
}
}
}
@@ -1074,8 +1134,8 @@ function collectToolCallPaths(msg: RawMessage, paths: Map<string, string>): void
args = typeof fn.arguments === 'string' ? JSON.parse(fn.arguments) : (fn.arguments ?? fn.input) as Record<string, unknown>;
} catch { /* ignore */ }
if (args) {
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
if (typeof fp === 'string') paths.set(id, fp);
const filePaths = extractFilePathsFromToolArgs(args);
if (filePaths[0]) paths.set(id, filePaths[0]);
}
}
}
@@ -1613,11 +1673,12 @@ function isToolResultRole(role: unknown): boolean {
}
/** True for internal plumbing messages that should never be shown in the UI. */
function isInternalMessage(msg: { role?: unknown; content?: unknown; idempotencyKey?: unknown; model?: unknown }): boolean {
function isInternalMessage(msg: { role?: unknown; content?: unknown; idempotencyKey?: unknown; model?: unknown; text?: unknown }): boolean {
if (msg.role === 'system') return true;
const text = getMessageText(msg.content);
const text = getMessageTextForFilter(msg);
if (msg.role === 'assistant') {
if (/^(HEARTBEAT_OK|NO_REPLY)\s*$/.test(text)) return true;
if (isInternalAssistantReplyText(text)) return true;
if (isGeneratingStatusNarration(text)) return true;
// OpenClaw's gateway writes a fallback `assistant-media` transcript
// message when its `createManagedOutgoingImageBlocks` pipeline fails
// ("could not be prepared" warning in stderr). The fallback has:
@@ -1642,6 +1703,12 @@ function isInternalMessage(msg: { role?: unknown; content?: unknown; idempotency
// canonical render — keep them. Only hide the text-only fallback.
if (!hasImageUrlBlock) return true;
}
if (!text.trim() && Array.isArray(msg.content)) {
const blocks = msg.content as ContentBlock[];
const hasThinking = blocks.some((block) => block.type === 'thinking' && block.thinking?.trim());
const hasVisibleText = blocks.some((block) => block.type === 'text' && block.text?.trim());
if (hasThinking && !hasVisibleText) return true;
}
}
if (msg.role === 'user' && /^\[OpenClaw heartbeat poll\]\s*$/i.test(text.trim())) return true;
// Runtime system injections: these arrive as user or assistant-role messages
@@ -1668,6 +1735,9 @@ function isRuntimeSystemInjection(text: string): boolean {
) {
return true;
}
if (/^\[Inter-session message\]/i.test(normalized)) return true;
if (isOpenClawRuntimeEventPrompt(normalized)) return true;
if (
/^\s*Current time\s*:/i.test(normalized)
&& /^\s*Current time\s*:[^\n]*\/\s*\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+UTC\s*$/i.test(normalized)
@@ -2631,7 +2701,8 @@ export const useChatStore = create<ChatState>((set, get) => ({
// Before filtering: attach images/files from tool_result messages to the next assistant message
const messagesWithToolImages = enrichWithToolResultFiles(rawMessages);
const filteredMessages = messagesWithToolImages.filter((msg) => !isToolResultRole(msg.role) && !isInternalMessage(msg));
const messagesWithToolAttachments = enrichWithToolCallAttachments(messagesWithToolImages);
const filteredMessages = messagesWithToolAttachments.filter((msg) => !shouldDropMessageFromHistory(msg));
// Restore file attachments for user/assistant messages (from cache + text patterns)
const enrichedMessages = enrichWithCachedImages(filteredMessages);
@@ -3024,7 +3095,8 @@ export const useChatStore = create<ChatState>((set, get) => ({
// window with a larger suffix from the transcript. This keeps render
// cost bounded while allowing long conversations to page backwards.
const messagesWithToolImages = enrichWithToolResultFiles(rawMessages);
const filteredMessages = messagesWithToolImages.filter((msg) => !isToolResultRole(msg.role) && !isInternalMessage(msg));
const messagesWithToolAttachments = enrichWithToolCallAttachments(messagesWithToolImages);
const filteredMessages = messagesWithToolAttachments.filter((msg) => !shouldDropMessageFromHistory(msg));
const enrichedMessages = enrichWithCachedImages(filteredMessages);
set({
messages: enrichedMessages,

View File

@@ -1,4 +1,9 @@
import { invokeIpc } from '@/lib/api-client';
import {
isGeneratingStatusNarration,
isInternalAssistantReplyText,
isOpenClawRuntimeEventPrompt,
} from '@/pages/Chat/message-utils';
import type { AttachedFileMeta, ChatSession, ContentBlock, RawMessage, ToolStatus } from './types';
// Module-level timestamp tracking the last chat event received.
@@ -186,8 +191,25 @@ function normalizeStreamingMessage(message: unknown): unknown {
* `[media attached:` instead — leaving the timestamp in the normalized
* comparison text and breaking optimistic-vs-echo dedupe.
*/
function stripInboundMediaVisionEnvelope(text: string): string {
if (!/\[Image\]/i.test(text) && !/^User text:/im.test(text) && !/\nDescription:\s*\n/i.test(text)) {
return text;
}
let result = text.replace(/^\s*\[Image\]\s*\n?/i, '');
const userTextBlock = result.match(/^User text:\s*\n([\s\S]*?)(?:\n\s*Description:\s*\n[\s\S]*)?\s*$/i);
if (userTextBlock) {
const userText = userTextBlock[1].trim();
return /^Process the attached file\(s\)\.\s*$/i.test(userText) ? '' : userText;
}
return result.replace(/\n\s*Description:\s*\n[\s\S]*$/i, '').trim();
}
function stripGatewayUserMetadata(text: string): string {
return text
return stripInboundMediaVisionEnvelope(
text
.replace(/\s*\[media attached:[^\]]*\]/g, '')
.replace(/\s*\[message_id:\s*[^\]]+\]/g, '')
.replace(/^Sender\s*\([^)]*\)\s*:\s*```[a-z]*\n[\s\S]*?```\s*/i, '')
@@ -198,13 +220,16 @@ function stripGatewayUserMetadata(text: string): string {
.replace(/^Sender\s*:\s*[^\n]*(?:\n\s*)*/i, '')
.replace(/^Conversation info\s*\([^)]*\):\s*```[a-z]*\n[\s\S]*?```\s*/i, '')
.replace(/^Conversation info\s*\([^)]*\):\s*\{[\s\S]*?\}\s*/i, '')
.replace(/^\s*\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, '');
.replace(/^\s*\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, ''),
);
}
function normalizeComparableUserText(content: unknown): string {
return stripGatewayUserMetadata(getMessageText(content))
const text = stripGatewayUserMetadata(getMessageText(content))
.replace(/\s+/g, ' ')
.trim();
if (/^\(file attached\)$/i.test(text)) return '';
return text;
}
function getComparableAttachmentSignature(message: Pick<RawMessage, '_attachedFiles'>): string {
@@ -239,6 +264,13 @@ function matchesOptimisticUserMessage(
if (sameText && sameAttachments) return true;
if (sameText && (!optimisticAttachments || !candidateAttachments) && (timestampMatches || !hasCandidateTimestamp)) return true;
if (sameAttachments && (!optimisticText || !candidateText) && (timestampMatches || !hasCandidateTimestamp)) return true;
const optimisticHadAttachmentsOnly = optimisticAttachments.length > 0 && !optimisticText;
const candidateIsAttachmentEcho = !candidateText
&& /\[(?:media attached:|\s*Image\s*\])/i.test(getMessageText(candidate.content));
if (optimisticHadAttachmentsOnly && candidateIsAttachmentEcho && (timestampMatches || !hasCandidateTimestamp)) {
return true;
}
return false;
}
@@ -399,6 +431,13 @@ function getMessageText(content: unknown): string {
return '';
}
function getMessageTextForFilter(msg: { content?: unknown; text?: unknown }): string {
const fromContent = getMessageText(msg.content);
if (fromContent.trim()) return fromContent;
if (typeof msg.text === 'string') return msg.text;
return '';
}
function getMessageStopReason(message: RawMessage | unknown): string | null {
if (!message || typeof message !== 'object') return null;
const msg = message as Record<string, unknown>;
@@ -486,12 +525,131 @@ function mimeFromExtension(filePath: string): string {
return map[ext] || 'application/octet-stream';
}
/** Extract local file paths declared in tool call arguments. */
function extractFilePathsFromToolArgs(args: Record<string, unknown>): string[] {
const paths: string[] = [];
const direct = args.file_path ?? args.filePath ?? args.path ?? args.file;
if (typeof direct === 'string' && direct.trim()) paths.push(direct.trim());
const attachments = args.attachments;
if (Array.isArray(attachments)) {
for (const item of attachments) {
if (!item || typeof item !== 'object') continue;
const att = item as Record<string, unknown>;
const filePath = att.filePath ?? att.file_path ?? att.path ?? att.file;
if (typeof filePath === 'string' && filePath.trim()) {
paths.push(filePath.trim());
}
}
}
return paths;
}
/**
* Surface user-facing attachments declared in assistant tool calls (e.g.
* `message` tool `attachments: [{ filePath }]`) on the calling turn itself.
*/
function enrichWithToolCallAttachments(messages: RawMessage[]): RawMessage[] {
return messages.map((msg) => {
if (msg.role !== 'assistant') return msg;
const attachmentPaths = new Set<string>();
const content = msg.content;
if (Array.isArray(content)) {
for (const block of content as ContentBlock[]) {
if (block.type !== 'tool_use' && block.type !== 'toolCall') continue;
const args = (block.input ?? block.arguments) as Record<string, unknown> | undefined;
if (!args) continue;
for (const filePath of extractFilePathsFromToolArgs(args)) {
attachmentPaths.add(filePath);
}
}
}
const msgAny = msg as unknown as Record<string, unknown>;
const toolCalls = msgAny.tool_calls ?? msgAny.toolCalls;
if (Array.isArray(toolCalls)) {
for (const tc of toolCalls as Array<Record<string, unknown>>) {
const fn = (tc.function ?? tc) as Record<string, unknown>;
let args: Record<string, unknown> | undefined;
try {
args = typeof fn.arguments === 'string'
? JSON.parse(fn.arguments)
: (fn.arguments ?? fn.input) as Record<string, unknown>;
} catch { /* ignore */ }
if (!args) continue;
for (const filePath of extractFilePathsFromToolArgs(args)) {
attachmentPaths.add(filePath);
}
}
}
if (attachmentPaths.size === 0) return msg;
const existingPaths = new Set(
(msg._attachedFiles || []).map((file) => file.filePath).filter(Boolean),
);
const newFiles = [...attachmentPaths]
.filter((filePath) => !existingPaths.has(filePath))
.map((filePath) => makeAttachedFile({ filePath, mimeType: mimeFromExtension(filePath) }, 'tool-result'));
if (newFiles.length === 0) return msg;
return {
...msg,
_attachedFiles: [...(msg._attachedFiles || []), ...newFiles],
};
});
}
const DIRECTORY_MIME_TYPE = 'application/x-directory';
function trimPathTerminators(filePath: string): string {
return filePath.replace(/[;,.!?]+$/u, '');
}
type MarkdownImageRef =
| { filePath: string; mimeType: string; fileName: string }
| { gatewayUrl: string; mimeType: string; fileName: string; source: 'gateway-media' };
/** Extract image targets from markdown `![alt](target)` in assistant text. */
function extractMarkdownImageRefs(text: string): MarkdownImageRef[] {
if (!text) return [];
const refs: MarkdownImageRef[] = [];
const seen = new Set<string>();
const markdownImageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
let match: RegExpExecArray | null;
while ((match = markdownImageRegex.exec(text)) !== null) {
const alt = match[1]?.trim() || 'image';
let target = match[2]?.trim() ?? '';
if (!target) continue;
if (target.startsWith('file://')) {
target = decodeURIComponent(target.replace(/^file:\/\//, ''));
}
if (target.startsWith('/api/chat/media/')) {
if (seen.has(target)) continue;
seen.add(target);
refs.push({
gatewayUrl: target,
mimeType: 'image/png',
fileName: alt,
source: 'gateway-media',
});
continue;
}
const normalizedPath = trimPathTerminators(target);
if (!normalizedPath.startsWith('/') && !normalizedPath.startsWith('~/')) continue;
if (seen.has(normalizedPath)) continue;
seen.add(normalizedPath);
refs.push({
filePath: normalizedPath,
mimeType: mimeFromExtension(normalizedPath),
fileName: alt,
});
}
return refs;
}
/**
* Extract raw file paths from message text.
* Detects absolute paths (Unix: / or ~/, Windows: C:\ etc.) ending with common file extensions.
@@ -661,8 +819,8 @@ function getToolCallFilePath(msg: RawMessage, toolCallId: string): string | unde
if ((block.type === 'tool_use' || block.type === 'toolCall') && block.id === toolCallId) {
const args = (block.input ?? block.arguments) as Record<string, unknown> | undefined;
if (args) {
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
if (typeof fp === 'string') return fp;
const paths = extractFilePathsFromToolArgs(args);
if (paths[0]) return paths[0];
}
}
}
@@ -680,8 +838,8 @@ function getToolCallFilePath(msg: RawMessage, toolCallId: string): string | unde
args = typeof fn.arguments === 'string' ? JSON.parse(fn.arguments) : (fn.arguments ?? fn.input) as Record<string, unknown>;
} catch { /* ignore */ }
if (args) {
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
if (typeof fp === 'string') return fp;
const paths = extractFilePathsFromToolArgs(args);
if (paths[0]) return paths[0];
}
}
}
@@ -699,8 +857,8 @@ function collectToolCallPaths(msg: RawMessage, paths: Map<string, string>): void
if ((block.type === 'tool_use' || block.type === 'toolCall') && block.id) {
const args = (block.input ?? block.arguments) as Record<string, unknown> | undefined;
if (args) {
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
if (typeof fp === 'string') paths.set(block.id, fp);
const filePaths = extractFilePathsFromToolArgs(args);
if (filePaths[0]) paths.set(block.id, filePaths[0]);
}
}
}
@@ -717,8 +875,8 @@ function collectToolCallPaths(msg: RawMessage, paths: Map<string, string>): void
args = typeof fn.arguments === 'string' ? JSON.parse(fn.arguments) : (fn.arguments ?? fn.input) as Record<string, unknown>;
} catch { /* ignore */ }
if (args) {
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
if (typeof fp === 'string') paths.set(id, fp);
const filePaths = extractFilePathsFromToolArgs(args);
if (filePaths[0]) paths.set(id, filePaths[0]);
}
}
}
@@ -744,7 +902,6 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
}
if (isToolResultRole(msg.role)) {
// Resolve file path from the matching tool call
const matchedPath = msg.toolCallId ? toolCallPaths.get(msg.toolCallId) : undefined;
// 1. Image/file content blocks in the structured content array.
@@ -788,6 +945,11 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
}
if (msg.role === 'assistant' && pending.length > 0) {
// Internal-only turns (NO_REPLY, interim narration, ...) must not consume
// pending attachments — the next visible assistant reply should get them.
if (isInternalMessage(msg) && !messageHasToolUse(msg)) {
return msg;
}
const toAttach = pending.splice(0);
// Deduplicate against files already on the assistant message
const existingPaths = new Set(
@@ -857,6 +1019,13 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
if (msg.role === 'assistant' && !isToolOnlyMessage(msg)) {
// Own text
rawRefs = extractRawFilePaths(text).filter(r => !mediaRefPaths.has(r.filePath));
const rawPathSet = new Set(rawRefs.map((ref) => ref.filePath));
for (const ref of extractMarkdownImageRefs(text)) {
if ('filePath' in ref && !mediaRefPaths.has(ref.filePath) && !rawPathSet.has(ref.filePath)) {
rawPathSet.add(ref.filePath);
rawRefs.push({ filePath: ref.filePath, mimeType: ref.mimeType });
}
}
// Nearest preceding user message text (look back up to 5 messages)
const seenPaths = new Set(rawRefs.map(r => r.filePath));
@@ -884,7 +1053,16 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
}
const allRefs = [...mediaRefs, ...rawRefs];
if (allRefs.length === 0 && gatewayMediaFiles.length === 0) return msg;
const markdownImageRefs = msg.role === 'assistant' && !isToolOnlyMessage(msg)
? extractMarkdownImageRefs(text)
: [];
if (
allRefs.length === 0
&& gatewayMediaFiles.length === 0
&& markdownImageRefs.length === 0
) {
return msg;
}
const existingFiles = msg._attachedFiles || [];
const existingPaths = new Set(existingFiles.map(file => file.filePath).filter(Boolean));
@@ -902,8 +1080,19 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
const dedupedGatewayMedia = gatewayMediaFiles.filter(
file => file.gatewayUrl && !existingGatewayUrls.has(file.gatewayUrl),
);
if (files.length === 0 && dedupedGatewayMedia.length === 0) return msg;
return { ...msg, _attachedFiles: [...existingFiles, ...files, ...dedupedGatewayMedia] };
const markdownGatewayMedia: AttachedFileMeta[] = markdownImageRefs
.filter((ref): ref is Extract<MarkdownImageRef, { gatewayUrl: string }> => 'gatewayUrl' in ref)
.filter((ref) => ref.gatewayUrl && !existingGatewayUrls.has(ref.gatewayUrl))
.map((ref) => ({
fileName: ref.fileName,
mimeType: ref.mimeType,
fileSize: 0,
preview: null,
gatewayUrl: ref.gatewayUrl,
source: 'gateway-media' as const,
}));
if (files.length === 0 && dedupedGatewayMedia.length === 0 && markdownGatewayMedia.length === 0) return msg;
return { ...msg, _attachedFiles: [...existingFiles, ...files, ...dedupedGatewayMedia, ...markdownGatewayMedia] };
});
}
@@ -1077,12 +1266,31 @@ function isToolResultRole(role: unknown): boolean {
return normalized === 'toolresult' || normalized === 'tool_result';
}
function messageHasToolUse(msg: { role?: unknown; content?: unknown; tool_calls?: unknown; toolCalls?: unknown }): boolean {
if (msg.role !== 'assistant') return false;
if (Array.isArray(msg.content)) {
const blocks = msg.content as ContentBlock[];
if (blocks.some((block) => block.type === 'tool_use' || block.type === 'toolCall')) {
return true;
}
}
const toolCalls = msg.tool_calls ?? msg.toolCalls;
return Array.isArray(toolCalls) && toolCalls.length > 0;
}
/** True for internal plumbing messages that should never be shown in the UI. */
function isInternalMessage(msg: { role?: unknown; content?: unknown }): boolean {
function isInternalMessage(msg: { role?: unknown; content?: unknown; text?: unknown }): boolean {
if (msg.role === 'system') return true;
const text = getMessageText(msg.content);
const text = getMessageTextForFilter(msg);
if (msg.role === 'assistant') {
if (/^(HEARTBEAT_OK|NO_REPLY)\s*$/.test(text)) return true;
if (isInternalAssistantReplyText(text)) return true;
if (isGeneratingStatusNarration(text)) return true;
if (!text.trim() && Array.isArray(msg.content)) {
const blocks = msg.content as ContentBlock[];
const hasThinking = blocks.some((block) => block.type === 'thinking' && block.thinking?.trim());
const hasVisibleText = blocks.some((block) => block.type === 'text' && block.text?.trim());
if (hasThinking && !hasVisibleText) return true;
}
}
if (msg.role === 'user' && /^\[OpenClaw heartbeat poll\]\s*$/i.test(text.trim())) return true;
// Runtime system injections: these arrive as user or assistant-role messages
@@ -1091,6 +1299,17 @@ function isInternalMessage(msg: { role?: unknown; content?: unknown }): boolean
return false;
}
/**
* History filtering must keep assistant tool-call turns even when their visible
* text is internal narration (e.g. "生成中,稍等" + `image_generate`). Those
* turns power the execution graph and run lifecycle detection.
*/
function shouldDropMessageFromHistory(msg: { role?: unknown; content?: unknown; text?: unknown; tool_calls?: unknown; toolCalls?: unknown }): boolean {
if (isToolResultRole(msg.role)) return true;
if (messageHasToolUse(msg)) return false;
return isInternalMessage(msg);
}
/**
* Detect runtime-injected system messages that should be hidden from the chat UI.
* These are injected by the OpenClaw runtime as user-role messages and include:
@@ -1114,8 +1333,11 @@ function isRuntimeSystemInjection(text: string): boolean {
return true;
}
// Standalone time injection (e.g. "Current time: Wednesday, April 22nd, 2026 - 10:06 (Asia/Shanghai) / 2026-04-22 02:06 UTC")
// Only match when the full message is the time announcement.
if (/^\[Inter-session message\]/i.test(normalized)) return true;
if (isOpenClawRuntimeEventPrompt(normalized)) return true;
// Standalone time injection
if (
/^\s*Current time\s*:/i.test(normalized)
&& /^\s*Current time\s*:[^\n]*\/\s*\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+UTC\s*$/i.test(normalized)
@@ -1439,7 +1661,9 @@ export {
extractRawFilePaths,
makeAttachedFile,
enrichWithToolResultFiles,
enrichWithToolCallAttachments,
isInternalMessage,
shouldDropMessageFromHistory,
isToolResultRole,
enrichWithCachedImages,
loadMissingPreviews,

View File

@@ -3,14 +3,14 @@ import { hostApiFetch } from '@/lib/host-api';
import { useGatewayStore } from '@/stores/gateway';
import {
clearHistoryPoll,
enrichWithToolCallAttachments,
enrichWithCachedImages,
enrichWithToolResultFiles,
getLatestOptimisticUserMessage,
getMessageErrorMessage,
getMessageStopReason,
getMessageText,
isInternalMessage,
isToolResultRole,
shouldDropMessageFromHistory,
loadMissingPreviews,
mergePendingOptimisticUserMessages,
dropRedundantOptimisticUserMessages,
@@ -112,7 +112,8 @@ export function createHistoryActions(
if (!isCurrentSession()) return false;
// Before filtering: attach images/files from tool_result messages to the next assistant message
const messagesWithToolImages = enrichWithToolResultFiles(rawMessages);
const filteredMessages = messagesWithToolImages.filter((msg) => !isToolResultRole(msg.role) && !isInternalMessage(msg));
const messagesWithToolAttachments = enrichWithToolCallAttachments(messagesWithToolImages);
const filteredMessages = messagesWithToolAttachments.filter((msg) => !shouldDropMessageFromHistory(msg));
// Restore file attachments for user/assistant messages (from cache + text patterns)
const enrichedMessages = enrichWithCachedImages(filteredMessages);

View File

@@ -33,6 +33,7 @@ test.describe('Image generation settings page', () => {
await expect(page.getByTestId('image-generation-primary')).toHaveCount(0);
await expect(page.getByTestId('image-generation-fallbacks')).toHaveCount(0);
await expect(page.getByTestId('image-generation-save')).toBeVisible();
await expect(page.getByTestId('image-generation-clear')).toBeDisabled();
});
test('configures an independent OpenAI-compatible image endpoint', async ({ page }) => {
@@ -45,7 +46,7 @@ test.describe('Image generation settings page', () => {
await expect(page.getByTestId('image-generation-settings')).toBeVisible();
await expect(page.getByTestId('image-generation-relay-base-url')).toBeVisible();
await page.getByTestId('image-generation-relay-base-url').fill('https://taolat.com/v1');
await page.getByTestId('image-generation-relay-base-url').fill('https://api.example.com/v1');
await page.getByTestId('image-generation-relay-model').fill('gpt-image-2');
await page.getByTestId('image-generation-relay-api-key').fill('sk-test-image');
@@ -81,7 +82,7 @@ test.describe('Image generation settings page', () => {
],
openAiRelay: {
enabled: true,
baseUrl: 'https://taolat.com/v1',
baseUrl: 'https://api.example.com/v1',
model: 'gpt-image-2',
providerKey: 'clawx-openai-image',
apiKeyConfigured: true,
@@ -103,4 +104,79 @@ test.describe('Image generation settings page', () => {
await expect(page.getByTestId('image-generation-api-key-status')).not.toBeEmpty();
await expect(page.getByTestId('image-generation-relay-api-key')).toHaveAttribute('placeholder', /.+/);
});
test('clears configured image generation settings after confirmation', async ({ electronApp, page }) => {
const configuredResponse = {
success: true,
config: {
primary: 'clawx-openai-image/gpt-image-2',
fallbacks: [],
timeoutMs: 180000,
},
autoProviderFallback: false,
defaultAgentId: 'default',
agents: [
{
id: 'default',
name: 'Default',
isDefault: true,
provider: null,
configured: false,
},
],
openAiRelay: {
enabled: true,
baseUrl: 'https://api.example.com/v1',
model: 'gpt-image-2',
providerKey: 'clawx-openai-image',
apiKeyConfigured: true,
},
};
const clearedResponse = {
...configuredResponse,
config: {
primary: null,
fallbacks: [],
timeoutMs: 180000,
},
openAiRelay: {
enabled: false,
baseUrl: '',
model: 'gpt-image-2',
providerKey: undefined,
apiKeyConfigured: false,
},
};
await installIpcMocks(electronApp, {
hostApi: {
'["/api/media/image-generation","GET"]': {
ok: true,
data: { status: 200, ok: true, json: configuredResponse },
},
'["/api/media/image-generation","PUT"]': {
ok: true,
data: { status: 200, ok: true, json: { success: true, ...clearedResponse } },
},
},
});
await expect(page.getByTestId('setup-page')).toBeVisible();
await page.getByTestId('setup-skip-button').click();
await expect(page.getByTestId('main-layout')).toBeVisible();
await unlockDeveloperMode(page);
await page.getByTestId('sidebar-nav-image-generation').click();
await expect(page.getByTestId('image-generation-relay-base-url')).toHaveValue('https://api.example.com/v1');
await expect(page.getByTestId('image-generation-clear')).toBeEnabled();
await page.getByTestId('image-generation-clear').click();
const confirmDialog = page.getByRole('dialog');
await expect(confirmDialog).toBeVisible();
await confirmDialog.getByRole('button', { name: 'Clear', exact: true }).click();
await expect(page.getByTestId('image-generation-relay-base-url')).toHaveValue('');
await expect(page.getByTestId('image-generation-clear')).toBeDisabled();
});
});

View File

@@ -0,0 +1,25 @@
import { describe, expect, it } from 'vitest';
import { extractText } from '@/pages/Chat/message-utils';
describe('assistant media path display cleanup', () => {
it('strips bare OpenClaw media paths when the image is shown as an attachment card', () => {
const text = [
'宇航员图片生成完成啦 🧑‍🚀✨',
'/Users/zhonghaolu/.openclaw/media/tool-image-generation/clawx-image-1---82d6c7e6-ea44-4850-a24b-9e88e1660683.png',
].join('\n');
expect(extractText({ role: 'assistant', content: text })).toBe('宇航员图片生成完成啦 🧑‍🚀✨');
});
it('still strips MEDIA: tagged OpenClaw artifact paths', () => {
const text = 'Done:\n\nMEDIA:/Users/alice/.openclaw/media/outbound/cat---abc.png';
expect(extractText({ role: 'assistant', content: text })).toBe('Done:');
});
it('strips markdown image syntax that cannot be rendered directly', () => {
const text = '宇航员图片完成啦 🧑‍🚀✨\n\n![Astronaut with Milky Way in helmet visor](/api/chat/media/outgoing/agent%3Amain%3As-1/abc/full)';
expect(extractText({ role: 'assistant', content: text })).toBe('宇航员图片完成啦 🧑‍🚀✨');
});
});

View File

@@ -0,0 +1,56 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { copyImageToClipboard } from '@/pages/Chat/copy-image';
const readBinaryFileMock = vi.fn();
vi.mock('@/lib/api-client', () => ({
readBinaryFile: (...args: unknown[]) => readBinaryFileMock(...args),
}));
describe('copyImageToClipboard', () => {
beforeEach(() => {
readBinaryFileMock.mockReset();
class MockClipboardItem {
constructor(public items: Record<string, Blob>) {}
}
Object.assign(globalThis, { ClipboardItem: MockClipboardItem });
Object.assign(navigator, {
clipboard: {
write: vi.fn(async () => undefined),
writeText: vi.fn(async () => undefined),
},
});
});
it('copies image bytes from a data URL preview', async () => {
const pngBytes = Uint8Array.from([137, 80, 78, 71]);
const base64 = btoa(String.fromCharCode(...pngBytes));
const preview = `data:image/png;base64,${base64}`;
const ok = await copyImageToClipboard({
preview,
mimeType: 'image/png',
});
expect(ok).toBe(true);
expect(navigator.clipboard.write).toHaveBeenCalledTimes(1);
expect(readBinaryFileMock).not.toHaveBeenCalled();
});
it('copies image bytes from a local file path', async () => {
readBinaryFileMock.mockResolvedValueOnce({
ok: true,
data: Uint8Array.from([1, 2, 3]),
mimeType: 'image/png',
});
const ok = await copyImageToClipboard({
filePath: '/tmp/cat.png',
mimeType: 'image/png',
});
expect(ok).toBe(true);
expect(readBinaryFileMock).toHaveBeenCalledWith('/tmp/cat.png');
expect(navigator.clipboard.write).toHaveBeenCalledTimes(1);
});
});

View File

@@ -1,6 +1,7 @@
import { describe, expect, it, vi } from 'vitest';
import {
enrichWithToolResultFiles,
enrichWithToolCallAttachments,
enrichWithCachedImages,
} from '@/stores/chat/helpers';
import type { RawMessage } from '@/stores/chat';
@@ -137,6 +138,38 @@ describe('enrichWithToolResultFiles', () => {
const paths = (reply._attachedFiles ?? []).map((f) => f.filePath);
expect(paths).toContain('/tmp/foo.pdf');
});
it('skips internal NO_REPLY turns when attaching pending tool-result files', () => {
const messages: RawMessage[] = [
{
role: 'assistant',
id: 'a1',
content: [{ type: 'toolCall', id: 'tc1', name: 'exec', input: { command: 'echo report > /tmp/report.pdf' } }],
},
{
role: 'toolresult',
id: 't1',
toolCallId: 'tc1',
toolName: 'exec',
content: [{ type: 'text', text: 'Wrote /tmp/report.pdf' }],
},
{
role: 'assistant',
id: 'no-reply',
content: [{ type: 'text', text: 'NO_REPLY' }],
},
{
role: 'assistant',
id: 'final',
content: [{ type: 'text', text: 'Report is ready.' }],
},
];
const enriched = enrichWithToolResultFiles(messages);
const final = enriched.find((m) => m.id === 'final')!;
expect(final._attachedFiles?.map((file) => file.filePath)).toEqual(['/tmp/report.pdf']);
expect(enriched.find((m) => m.id === 'no-reply')?._attachedFiles ?? []).toEqual([]);
});
});
describe('enrichWithCachedImages — Gateway media bubble dedup', () => {
@@ -273,4 +306,68 @@ describe('enrichWithCachedImages — Gateway media bubble dedup', () => {
const replyPaths = (reply._attachedFiles ?? []).map((f) => f.filePath);
expect(replyPaths).toEqual(['/tmp/foo.png']);
});
it('promotes markdown local image paths to attached files', () => {
const messages: RawMessage[] = [
{
role: 'assistant',
id: 'reply',
content: [{
type: 'text',
text: '宇航员图片完成啦 🧑‍🚀✨\n\n![Astronaut with Milky Way in helmet visor](/Users/me/.openclaw/media/tool-image-generation/cat.png)',
}],
},
];
const enriched = enrichWithCachedImages(messages);
expect(enriched[0]?._attachedFiles?.map((file) => file.filePath)).toEqual([
'/Users/me/.openclaw/media/tool-image-generation/cat.png',
]);
});
it('promotes markdown gateway image URLs to gateway-media attachments', () => {
const messages: RawMessage[] = [
{
role: 'assistant',
id: 'reply',
content: [{
type: 'text',
text: 'Done\n\n![Astronaut with Milky Way in helmet visor](/api/chat/media/outgoing/agent%3Amain%3As-1/abc/full)',
}],
},
];
const enriched = enrichWithCachedImages(messages);
expect(enriched[0]?._attachedFiles?.[0]).toMatchObject({
gatewayUrl: '/api/chat/media/outgoing/agent%3Amain%3As-1/abc/full',
source: 'gateway-media',
fileName: 'Astronaut with Milky Way in helmet visor',
});
});
});
describe('enrichWithToolCallAttachments', () => {
it('attaches image paths from message tool attachments array', () => {
const messages: RawMessage[] = [
{
role: 'assistant',
id: 'send-image',
content: [{
type: 'tool_use',
id: 'tool-1',
name: 'message',
input: {
action: 'send',
attachments: [{ filePath: '/Users/me/.openclaw/media/tool-image-generation/cat.png' }],
},
}],
},
];
const enriched = enrichWithToolCallAttachments(messages);
expect(enriched[0]?._attachedFiles?.map((file) => file.filePath)).toEqual([
'/Users/me/.openclaw/media/tool-image-generation/cat.png',
]);
expect(enriched[0]?._attachedFiles?.[0]?.mimeType).toBe('image/png');
});
});

View File

@@ -7,6 +7,7 @@ const gatewayStoreGetStateMock = vi.fn();
const clearHistoryPoll = vi.fn();
const enrichWithCachedImages = vi.fn((messages) => messages);
const enrichWithToolResultFiles = vi.fn((messages) => messages);
const enrichWithToolCallAttachments = vi.fn((messages) => messages);
const getMessageErrorMessage = vi.fn((message: { errorMessage?: string; error_message?: string } | undefined) => {
if (!message) return null;
return message.errorMessage ?? message.error_message ?? null;
@@ -29,6 +30,23 @@ const isInternalMessage = vi.fn((msg: { role?: unknown; content?: unknown }) =>
}
return false;
});
const messageHasToolUse = (msg: { role?: unknown; content?: unknown; tool_calls?: unknown; toolCalls?: unknown }) => {
if (msg.role !== 'assistant') return false;
if (Array.isArray(msg.content)) {
return (msg.content as Array<{ type?: string }>).some(
(block) => block.type === 'tool_use' || block.type === 'toolCall',
);
}
const toolCalls = (msg as { tool_calls?: unknown; toolCalls?: unknown }).tool_calls
?? (msg as { toolCalls?: unknown }).toolCalls;
return Array.isArray(toolCalls) && toolCalls.length > 0;
};
const shouldDropMessageFromHistory = vi.fn((msg: { role?: unknown; content?: unknown; tool_calls?: unknown; toolCalls?: unknown }) => {
if (isToolResultRole(msg.role)) return true;
if (messageHasToolUse(msg)) return false;
return isInternalMessage(msg);
});
const setLastChatEventAt = vi.fn();
const loadMissingPreviews = vi.fn(async () => false);
const toMs = vi.fn((ts: number) => ts < 1e12 ? ts * 1000 : ts);
@@ -50,6 +68,7 @@ vi.mock('@/stores/chat/helpers', () => ({
clearHistoryPoll: (...args: unknown[]) => clearHistoryPoll(...args),
enrichWithCachedImages: (...args: unknown[]) => enrichWithCachedImages(...args),
enrichWithToolResultFiles: (...args: unknown[]) => enrichWithToolResultFiles(...args),
enrichWithToolCallAttachments: (...args: unknown[]) => enrichWithToolCallAttachments(...args),
getLatestOptimisticUserMessage: (messages: Array<{ role: string; timestamp?: number }>, userTimestampMs: number) =>
[...messages].reverse().find(
(message) => message.role === 'user'
@@ -57,9 +76,13 @@ vi.mock('@/stores/chat/helpers', () => ({
),
getMessageText: (...args: unknown[]) => getMessageText(...args),
hasNonToolAssistantContent: (...args: unknown[]) => hasNonToolAssistantContent(...args),
hasAssistantAfterLastRealUser: (messages: Array<{ role?: string }>) =>
messages.some((message) => message.role === 'assistant'),
isInternalMessage: (...args: unknown[]) => isInternalMessage(...args),
shouldDropMessageFromHistory: (...args: unknown[]) => shouldDropMessageFromHistory(...args),
isToolResultRole: (...args: unknown[]) => isToolResultRole(...args),
loadMissingPreviews: (...args: unknown[]) => loadMissingPreviews(...args),
setLastChatEventAt: (...args: unknown[]) => setLastChatEventAt(...args),
mergePendingOptimisticUserMessages: (_sessionKey: string, messages: unknown[]) => messages,
hasOptimisticServerEcho: (
loadedMessages: Array<{ role: string; timestamp?: number; content?: unknown; _attachedFiles?: Array<{ filePath?: string; fileName?: string; mimeType?: string; fileSize?: number }> }>,

View File

@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest';
import { isInternalMessage } from '@/stores/chat/helpers';
import { isInternalMessage, shouldDropMessageFromHistory } from '@/stores/chat/helpers';
describe('chat internal message filter', () => {
it('filters runtime system injection bundle like async exec completion payload', () => {
@@ -33,4 +33,54 @@ describe('chat internal message filter', () => {
it('filters OpenClaw heartbeat poll user turns', () => {
expect(isInternalMessage({ role: 'user', content: '[OpenClaw heartbeat poll]' })).toBe(true);
});
it('filters inter-session routing payloads', () => {
expect(isInternalMessage({
role: 'user',
content: '[Inter-session message] sourceSession=image_generate:abc sourceTool=image_generate',
})).toBe(true);
});
it('filters OpenClaw runtime continuation prompts', () => {
expect(isInternalMessage({ role: 'user', content: 'Continue the OpenClaw runtime event.' })).toBe(true);
});
it('filters runtime continuation prompts stored on msg.text', () => {
expect(isInternalMessage({
role: 'user',
content: [],
text: 'Continue the OpenClaw runtime event.',
})).toBe(true);
});
it('filters interim image-generation status narration', () => {
expect(isInternalMessage({
role: 'assistant',
content: '生成中,稍等 👨‍🚀',
})).toBe(true);
});
it('keeps assistant tool-call turns in history even when narration is internal', () => {
expect(shouldDropMessageFromHistory({
role: 'assistant',
content: [
{ type: 'text', text: '生成中,稍等 👨‍🚀' },
{ type: 'tool_use', id: 'tool-image', name: 'image_generate', input: { prompt: 'astronaut' } },
],
})).toBe(false);
expect(isInternalMessage({
role: 'assistant',
content: [
{ type: 'text', text: '生成中,稍等 👨‍🚀' },
{ type: 'tool_use', id: 'tool-image', name: 'image_generate', input: { prompt: 'astronaut' } },
],
})).toBe(true);
});
it('filters assistant turns that only contain chain-of-thought', () => {
expect(isInternalMessage({
role: 'assistant',
content: [{ type: 'thinking', thinking: 'The user is asking me to continue with an OpenClaw runtime event.' }],
})).toBe(true);
});
});

View File

@@ -5,6 +5,7 @@ import type { RawMessage } from '@/stores/chat';
vi.mock('@/lib/api-client', () => ({
invokeIpc: vi.fn(),
readBinaryFile: vi.fn(),
statFile: vi.fn(async (path: string) => {
if (path.includes('missing') || path.includes('不存在')) {
return { ok: false, error: 'notFound' };
@@ -405,3 +406,50 @@ describe('ChatMessage reply styling', () => {
expect(bubble).toHaveTextContent('Keep the prompt bubble.');
});
});
describe('ChatMessage image copy', () => {
beforeEach(() => {
class MockClipboardItem {
constructor(public items: Record<string, Blob>) {}
}
Object.assign(globalThis, { ClipboardItem: MockClipboardItem });
Object.assign(navigator, {
clipboard: {
write: vi.fn(async () => undefined),
writeText: vi.fn(async () => undefined),
},
});
});
it('copies image bytes instead of the media URL text when an image attachment is present', async () => {
const { readBinaryFile } = await import('@/lib/api-client');
vi.mocked(readBinaryFile).mockResolvedValueOnce({
ok: true,
data: Uint8Array.from([137, 80, 78, 71]),
mimeType: 'image/png',
});
const message: RawMessage = {
role: 'assistant',
content: 'http://127.0.0.1:18789/api/chat/media/outgoing/agent/main/full',
_attachedFiles: [
{
fileName: 'cat.png',
mimeType: 'image/png',
fileSize: 1234,
preview: null,
filePath: '/tmp/cat.png',
source: 'tool-result',
},
],
};
render(<ChatMessage message={message} />);
fireEvent.click(screen.getByRole('button'));
await vi.waitFor(() => {
expect(navigator.clipboard.write).toHaveBeenCalledTimes(1);
});
expect(navigator.clipboard.writeText).not.toHaveBeenCalled();
});
});

View File

@@ -0,0 +1,58 @@
import { describe, expect, it } from 'vitest';
import { extractText } from '@/pages/Chat/message-utils';
import { matchesOptimisticUserMessage } from '@/stores/chat/helpers';
const gatewayImageEcho = [
'Sender (untrusted metadata):',
'```json',
'{',
' "label": "ClawX (gateway-client)",',
' "id": "gateway-client",',
' "name": "ClawX",',
' "username": "ClawX"',
'}',
'```',
'',
'[media attached: media://inbound/image---abc.png (image/png)]',
'[Image]',
'User text:',
'Process the attached file(s).',
'[media attached: /Users/test/.openclaw/media/outbound/out.png (image/png) | /Users/test/.openclaw/media/outbound/out.png]',
'Description:',
'An astronaut in a white space suit floats in space, reaching a gloved hand toward the viewer.',
].join('\n');
describe('user message display cleanup', () => {
it('hides the inbound-image vision envelope for attachment-only uploads', () => {
expect(extractText({ role: 'user', content: gatewayImageEcho })).toBe('');
});
it('keeps the user caption while stripping auto-generated description', () => {
const content = gatewayImageEcho
.replace('Process the attached file(s).', '改成西装加领带');
expect(extractText({ role: 'user', content })).toBe('改成西装加领带');
});
it('matches optimistic attachment-only bubbles against the gateway vision echo', () => {
const optimistic = {
role: 'user' as const,
content: '(file attached)',
timestamp: 1_700_000_000,
_attachedFiles: [{
fileName: 'out.png',
mimeType: 'image/png',
fileSize: 123,
preview: null,
filePath: '/Users/test/.openclaw/media/outbound/out.png',
}],
};
const candidate = {
role: 'user' as const,
content: gatewayImageEcho,
timestamp: 1_700_000_000,
};
expect(matchesOptimisticUserMessage(candidate, optimistic, 1_700_000_000_000)).toBe(true);
});
});

View File

@@ -0,0 +1,136 @@
import { describe, expect, it } from 'vitest';
import type { RawMessage } from '@/stores/chat';
import {
IMAGE_GENERATION_TIMEOUT_MS,
isImageGenerationPending,
} from '@/pages/Chat/image-generation-status';
const TASK_ID = '27443fdb-6cca-48e6-a3a7-ee34b0491aee';
const NOW = 1_700_000_000_000;
describe('isImageGenerationPending', () => {
it('returns true while waiting after image_generate without a final reply', () => {
const segmentMessages: RawMessage[] = [
{
role: 'assistant',
timestamp: NOW / 1000,
content: [{ type: 'toolCall', id: 'call_1', name: 'image_generate', arguments: { prompt: 'astronaut' } }],
},
{
role: 'assistant',
timestamp: NOW / 1000 + 5,
content: [{ type: 'text', text: '图片生成中,稍等片刻 🧑‍🚀✨' }],
},
];
expect(isImageGenerationPending(segmentMessages, [], NOW)).toBe(true);
});
it('returns false after the inter-session completion event arrives', () => {
const segmentMessages: RawMessage[] = [
{
role: 'toolresult',
toolName: 'image_generate',
timestamp: NOW / 1000,
content: [{
type: 'text',
text: `Background task started for image generation (${TASK_ID}).`,
}],
},
{
role: 'user',
content: [{
type: 'text',
text: `[Inter-session message] sourceSession=image_generate:${TASK_ID} sourceTool=image_generate`,
}],
},
];
expect(isImageGenerationPending(segmentMessages, [], NOW)).toBe(false);
});
it('returns false once a non-generating assistant reply arrives', () => {
const segmentMessages: RawMessage[] = [
{
role: 'assistant',
timestamp: NOW / 1000,
content: [{ type: 'toolCall', id: 'call_1', name: 'image_generate', arguments: { prompt: 'astronaut' } }],
},
{
role: 'assistant',
timestamp: NOW / 1000 + 5,
content: [{ type: 'text', text: '图片生成中,稍等片刻 🧑‍🚀✨' }],
},
{
role: 'assistant',
timestamp: NOW / 1000 + 120,
content: [{ type: 'text', text: '抱歉,图片生成超时了,稍后重试一下吧 🧑‍🚀' }],
},
];
expect(isImageGenerationPending(segmentMessages, [], NOW + 120_000)).toBe(false);
});
it('returns false after the configured timeout even if only generating narration exists', () => {
const segmentMessages: RawMessage[] = [
{
role: 'assistant',
timestamp: NOW / 1000,
content: [{ type: 'toolCall', id: 'call_1', name: 'image_generate', arguments: { prompt: 'astronaut' } }],
},
{
role: 'assistant',
timestamp: NOW / 1000 + 5,
content: [{ type: 'text', text: '图片生成中,稍等片刻 🧑‍🚀✨' }],
},
];
expect(
isImageGenerationPending(
segmentMessages,
[],
NOW + IMAGE_GENERATION_TIMEOUT_MS + 20_000,
),
).toBe(false);
});
it('returns false when a stale streaming tool status is still marked running', () => {
expect(isImageGenerationPending([], [{
name: 'image_generate',
status: 'running',
updatedAt: NOW - IMAGE_GENERATION_TIMEOUT_MS - 60_000,
}], NOW)).toBe(false);
});
it('returns true while image_generate is actively running in streaming tools', () => {
expect(isImageGenerationPending([], [{
name: 'image_generate',
status: 'running',
updatedAt: NOW,
}], NOW)).toBe(true);
});
it('returns false once an image attachment is delivered', () => {
const segmentMessages: RawMessage[] = [
{
role: 'assistant',
timestamp: NOW / 1000,
content: [{ type: 'toolCall', id: 'call_1', name: 'image_generate', arguments: { prompt: 'astronaut' } }],
},
{
role: 'assistant',
timestamp: NOW / 1000 + 90,
content: [{ type: 'text', text: '宇航员图片完成啦 🧑‍🚀✨' }],
_attachedFiles: [{
fileName: 'astronaut.png',
mimeType: 'image/png',
fileSize: 123,
preview: null,
filePath: '/tmp/astronaut.png',
}],
},
];
expect(isImageGenerationPending(segmentMessages, [], NOW + 90_000)).toBe(false);
});
});

View File

@@ -4,7 +4,7 @@ import { stripProcessMessagePrefix } from '@/pages/Chat/message-utils';
import type { RawMessage, ToolStatus } from '@/stores/chat';
describe('deriveTaskSteps', () => {
it('builds running steps from streaming thinking and tool status', () => {
it('builds running steps from streaming tool status without exposing chain-of-thought', () => {
const streamingTools: ToolStatus[] = [
{
name: 'web_search',
@@ -27,12 +27,6 @@ describe('deriveTaskSteps', () => {
});
expect(steps).toEqual([
expect.objectContaining({
id: 'stream-thinking-0',
label: 'Thinking',
status: 'running',
kind: 'thinking',
}),
expect.objectContaining({
label: 'web_search',
status: 'running',
@@ -160,7 +154,7 @@ describe('deriveTaskSteps', () => {
}));
});
it('keeps recent completed steps from assistant history', () => {
it('keeps recent completed tool steps from assistant history', () => {
const messages: RawMessage[] = [
{
role: 'assistant',
@@ -179,12 +173,6 @@ describe('deriveTaskSteps', () => {
});
expect(steps).toEqual([
expect.objectContaining({
id: 'history-thinking-assistant-1-0',
label: 'Thinking',
status: 'completed',
kind: 'thinking',
}),
expect.objectContaining({
id: 'tool-2',
label: 'read_file',
@@ -194,7 +182,7 @@ describe('deriveTaskSteps', () => {
]);
});
it('splits cumulative streaming thinking into separate execution steps', () => {
it('does not expose streaming chain-of-thought in the execution graph', () => {
const steps = deriveTaskSteps({
messages: [],
streamingMessage: {
@@ -208,21 +196,36 @@ describe('deriveTaskSteps', () => {
streamingTools: [],
});
expect(steps).toEqual([]);
});
it('skips internal assistant turns and hides NO_REPLY from the execution graph', () => {
const steps = deriveTaskSteps({
messages: [
{
role: 'assistant',
content: [{ type: 'thinking', thinking: 'Continue the OpenClaw runtime event internally.' }],
},
{
role: 'assistant',
content: [{ type: 'text', text: 'NO_REPLY' }],
},
{
role: 'assistant',
content: [
{ type: 'tool_use', id: 'tool-image', name: 'image_generate', input: { prompt: 'astronaut' } },
],
},
],
streamingMessage: null,
streamingTools: [],
});
expect(steps).toEqual([
expect.objectContaining({
id: 'stream-thinking-0',
detail: 'Reviewing X.',
status: 'completed',
}),
expect.objectContaining({
id: 'stream-thinking-1',
detail: 'Comparing Y.',
status: 'completed',
}),
expect.objectContaining({
id: 'stream-thinking-2',
detail: 'Drafting answer.',
status: 'running',
id: 'tool-image',
label: 'image_generate',
kind: 'tool',
}),
]);
});