diff --git a/electron/utils/openclaw-auth.ts b/electron/utils/openclaw-auth.ts index 620a4fa..1815f27 100644 --- a/electron/utils/openclaw-auth.ts +++ b/electron/utils/openclaw-auth.ts @@ -1773,6 +1773,7 @@ export async function syncOpenAiCompatibleImageRelay(params: { } removePluginRegistrations(config, [CLAWX_OPENAI_IMAGE_PROVIDER_KEY]); await writeOpenClawJson(config); + await removeProviderKeyFromOpenClaw(CLAWX_OPENAI_IMAGE_PROVIDER_KEY); if (params.apiKey?.trim()) { await saveProviderKeyToOpenClaw(CLAWX_OPENAI_IMAGE_PROVIDER_KEY, params.apiKey.trim()); } diff --git a/electron/utils/openclaw-image-generation-runtime.ts b/electron/utils/openclaw-image-generation-runtime.ts index 2fed6af..ef12048 100644 --- a/electron/utils/openclaw-image-generation-runtime.ts +++ b/electron/utils/openclaw-image-generation-runtime.ts @@ -47,25 +47,30 @@ type ModelInputModule = { resolveAgentModelPrimaryValue: (model?: unknown) => string | undefined; }; +const OPENCLAW_IMAGE_GENERATION_RUNTIME = 'openclaw/plugin-sdk/image-generation-runtime'; +const OPENCLAW_MEDIA_STORE = 'openclaw/plugin-sdk/media-store'; +const OPENCLAW_MEDIA_RUNTIME = 'openclaw/plugin-sdk/media-runtime'; +const OPENCLAW_IMAGE_GENERATION_CORE = 'openclaw/plugin-sdk/image-generation-core'; + let imageRuntimeModule: ImageGenerationRuntimeModule | null = null; let mediaStoreModule: MediaStoreModule | null = null; let imageOpsModule: ImageOpsModule | null = null; let modelInputModule: ModelInputModule | null = null; -async function importRuntimeModule(relativePath: string): Promise { - const modulePath = resolveOpenClawRuntimeModulePath(relativePath); +async function importOpenClawSdkModule(specifier: string): Promise { + const modulePath = resolveOpenClawRuntimeModulePath(specifier); return import(pathToFileURL(modulePath).href) as Promise; } async function getImageGenerationRuntime(): Promise { if (!imageRuntimeModule) { - const mod = await importRuntimeModule<{ - t: ImageGenerationRuntimeModule['generateImage']; - n: ImageGenerationRuntimeModule['listRuntimeImageGenerationProviders']; - }>('./dist/runtime-Oyct10ZH.js'); + const mod = await importOpenClawSdkModule<{ + generateImage: ImageGenerationRuntimeModule['generateImage']; + listRuntimeImageGenerationProviders: ImageGenerationRuntimeModule['listRuntimeImageGenerationProviders']; + }>(OPENCLAW_IMAGE_GENERATION_RUNTIME); imageRuntimeModule = { - generateImage: mod.t, - listRuntimeImageGenerationProviders: mod.n, + generateImage: mod.generateImage, + listRuntimeImageGenerationProviders: mod.listRuntimeImageGenerationProviders, }; } return imageRuntimeModule; @@ -73,20 +78,20 @@ async function getImageGenerationRuntime(): Promise { if (!mediaStoreModule) { - const mod = await importRuntimeModule<{ u: MediaStoreModule['saveMediaBuffer'] }>( - './dist/store-b792nN7l.js', + const mod = await importOpenClawSdkModule<{ saveMediaBuffer: MediaStoreModule['saveMediaBuffer'] }>( + OPENCLAW_MEDIA_STORE, ); - mediaStoreModule = { saveMediaBuffer: mod.u }; + mediaStoreModule = { saveMediaBuffer: mod.saveMediaBuffer }; } return mediaStoreModule; } async function getImageOps(): Promise { if (!imageOpsModule) { - const mod = await importRuntimeModule<{ a: ImageOpsModule['getImageMetadata'] }>( - './dist/image-ops-CL9ZQP7k.js', + const mod = await importOpenClawSdkModule<{ getImageMetadata: ImageOpsModule['getImageMetadata'] }>( + OPENCLAW_MEDIA_RUNTIME, ); - imageOpsModule = { getImageMetadata: mod.a }; + imageOpsModule = { getImageMetadata: mod.getImageMetadata }; } return imageOpsModule; } @@ -100,10 +105,10 @@ export async function resolveImageGenerationPrimaryFromConfig( async function getModelInputHelpers(): Promise { if (!modelInputModule) { - const mod = await importRuntimeModule<{ i: ModelInputModule['resolveAgentModelPrimaryValue'] }>( - './dist/model-input-B9p-bobB.js', + const mod = await importOpenClawSdkModule<{ resolveAgentModelPrimaryValue: ModelInputModule['resolveAgentModelPrimaryValue'] }>( + OPENCLAW_IMAGE_GENERATION_CORE, ); - modelInputModule = { resolveAgentModelPrimaryValue: mod.i }; + modelInputModule = { resolveAgentModelPrimaryValue: mod.resolveAgentModelPrimaryValue }; } return modelInputModule; } diff --git a/src/components/settings/ImageGenerationSettings.tsx b/src/components/settings/ImageGenerationSettings.tsx index bbbf072..2d4ccdf 100644 --- a/src/components/settings/ImageGenerationSettings.tsx +++ b/src/components/settings/ImageGenerationSettings.tsx @@ -2,14 +2,16 @@ * Global image generation settings (agents.defaults.imageGenerationModel). */ import { useCallback, useEffect, useMemo, useState } from 'react'; -import { Eye, EyeOff, ImagePlus, Loader2, Play, RefreshCw } from 'lucide-react'; +import { Eye, EyeOff, ImagePlus, Loader2, Play, RefreshCw, Trash2 } from 'lucide-react'; import { useTranslation } from 'react-i18next'; import { toast } from 'sonner'; import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; import { Label } from '@/components/ui/label'; import { Badge } from '@/components/ui/badge'; +import { ConfirmDialog } from '@/components/ui/confirm-dialog'; import { + clearImageGenerationSettings, fetchImageGenerationSettings, runImageGenerationTest, saveImageGenerationSettings, @@ -32,10 +34,12 @@ function extractTestOutputPath(result: unknown): string | null { } export function ImageGenerationSettings() { - const { t } = useTranslation(['dashboard', 'settings']); + const { t } = useTranslation(['dashboard', 'settings', 'common']); const [loading, setLoading] = useState(true); const [saving, setSaving] = useState(false); + const [clearing, setClearing] = useState(false); const [testing, setTesting] = useState(false); + const [clearConfirmOpen, setClearConfirmOpen] = useState(false); const [snapshot, setSnapshot] = useState(null); const [timeoutMs, setTimeoutMs] = useState('180000'); @@ -79,6 +83,15 @@ export function ImageGenerationSettings() { ); }, [snapshot, timeoutMs, relayBaseUrl, relayModel, relayApiKey]); + const hasConfiguredRelay = useMemo(() => { + if (!snapshot) return false; + return Boolean( + snapshot.openAiRelay?.enabled + || snapshot.openAiRelay?.baseUrl?.trim() + || snapshot.config.primary?.trim(), + ); + }, [snapshot]); + const handleSave = async () => { setSaving(true); try { @@ -120,11 +133,32 @@ export function ImageGenerationSettings() { } }; + const handleClear = async () => { + setClearing(true); + try { + const next = await clearImageGenerationSettings(); + setSnapshot(next); + setRelayBaseUrl(''); + setRelayModel('gpt-image-2'); + setRelayApiKey(''); + setShowRelayApiKey(false); + setClearConfirmOpen(false); + toast.success(t('imageGeneration.toast.cleared')); + } catch (error) { + toast.error(error instanceof Error ? error.message : String(error)); + } finally { + setClearing(false); + } + }; + const handleTest = async () => { if (dirty) { toast.message(t('imageGeneration.toast.saveBeforeTest')); return; } + if (!hasConfiguredRelay) { + return; + } setTesting(true); try { const result = await runImageGenerationTest({ @@ -368,7 +402,7 @@ export function ImageGenerationSettings() { variant="outline" className="rounded-full h-10" onClick={() => void handleTest()} - disabled={testing || !relayModel.trim()} + disabled={testing || !hasConfiguredRelay || dirty} data-testid="image-generation-test-button" > {testing ? ( @@ -387,9 +421,34 @@ export function ImageGenerationSettings() { {saving ? : null} {saving ? t('imageGeneration.saving') : t('imageGeneration.save')} + )} + + setClearConfirmOpen(false)} + /> ); } diff --git a/src/i18n/locales/en/chat.json b/src/i18n/locales/en/chat.json index f8024e3..b516e86 100644 --- a/src/i18n/locales/en/chat.json +++ b/src/i18n/locales/en/chat.json @@ -159,10 +159,14 @@ "branchLabel": "branch", "thinkingLabel": "Thinking", "errorLabel": "Error", + "imageGenerateLabel": "Image generation", "agentRun": "{{agent}} execution", "collapsedSummary": "{{toolCount}} tool calls · {{processCount}} process messages", "collapseAction": "Collapse execution graph" }, + "imageGeneration": { + "generating": "Generating image, please wait…" + }, "composer": { "attachFiles": "Attach files", "pickSkill": "Choose skill", diff --git a/src/i18n/locales/en/dashboard.json b/src/i18n/locales/en/dashboard.json index 0c0559c..505dbc6 100644 --- a/src/i18n/locales/en/dashboard.json +++ b/src/i18n/locales/en/dashboard.json @@ -78,9 +78,15 @@ "testing": "Generating…", "save": "Save", "saving": "Saving…", + "clear": "Clear configuration", + "clearing": "Clearing…", + "clearConfirmTitle": "Clear image generation configuration?", + "clearConfirmMessage": "This removes the custom image endpoint, model selection, and stored API key. Chat providers are not affected.", + "clearConfirmAction": "Clear", "testPrompt": "A small red circle on a white background, minimal flat illustration.", "toast": { "saved": "Image generation settings saved", + "cleared": "Image generation configuration cleared", "saveBeforeTest": "Save settings before running a test", "testSuccess": "Test image generated ({{ms}} ms)", "testSuccessWithPath": "Test image generated ({{ms}} ms): {{path}}", @@ -90,7 +96,7 @@ "title": "Custom image endpoint", "description": "Send image generation to a dedicated OpenAI Images-compatible endpoint (/v1/images/generations). This does not change any chat provider.", "baseUrl": "Image Base URL", - "baseUrlHint": "e.g. https://taolat.com/v1 — /v1 is appended automatically when omitted.", + "baseUrlHint": "e.g. https://api.example.com/v1 — /v1 is appended automatically when omitted.", "apiKey": "API key", "apiKeyPlaceholder": "sk-...", "apiKeyPlaceholderConfigured": "Already configured — leave blank to keep", diff --git a/src/i18n/locales/ja/chat.json b/src/i18n/locales/ja/chat.json index 558761a..4d03ef4 100644 --- a/src/i18n/locales/ja/chat.json +++ b/src/i18n/locales/ja/chat.json @@ -159,10 +159,14 @@ "branchLabel": "branch", "thinkingLabel": "考え中", "errorLabel": "エラー", + "imageGenerateLabel": "画像生成", "agentRun": "{{agent}} の実行", "collapsedSummary": "ツール呼び出し {{toolCount}} 件 · プロセスメッセージ {{processCount}} 件", "collapseAction": "実行グラフを折りたたむ" }, + "imageGeneration": { + "generating": "画像を生成しています。少々お待ちください…" + }, "composer": { "attachFiles": "ファイルを添付", "pickSkill": "Skill を選択", diff --git a/src/i18n/locales/ja/dashboard.json b/src/i18n/locales/ja/dashboard.json index 60a4316..9a30cc4 100644 --- a/src/i18n/locales/ja/dashboard.json +++ b/src/i18n/locales/ja/dashboard.json @@ -78,9 +78,15 @@ "testing": "生成中…", "save": "保存", "saving": "保存中…", + "clear": "設定をクリア", + "clearing": "クリア中…", + "clearConfirmTitle": "画像生成設定をクリアしますか？", + "clearConfirmMessage": "カスタム画像エンドポイント、モデル設定、保存済み API キーを削除します。チャットプロバイダーには影響しません。", + "clearConfirmAction": "クリア", "testPrompt": "白背景に小さな赤い円、ミニマルなフラットイラスト。", "toast": { "saved": "画像生成設定を保存しました", + "cleared": "画像生成設定をクリアしました", "saveBeforeTest": "テスト前に設定を保存してください", "testSuccess": "テスト画像を生成しました（{{ms}} ms）", "testSuccessWithPath": "テスト画像を生成しました（{{ms}} ms）：{{path}}", @@ -90,7 +96,7 @@ "title": "カスタム画像エンドポイント", "description": "画像生成を専用の OpenAI Images 互換エンドポイント（/v1/images/generations）へ送ります。チャットプロバイダーは変更しません。", "baseUrl": "画像 Base URL", - "baseUrlHint": "例: https://taolat.com/v1 — /v1 が無い場合は自動付与します。", + "baseUrlHint": "例: https://api.example.com/v1 — /v1 が無い場合は自動付与します。", "apiKey": "API キー", "apiKeyPlaceholder": "sk-...", "apiKeyPlaceholderConfigured": "設定済み — 空欄のままなら変更しません", diff --git a/src/i18n/locales/ru/chat.json b/src/i18n/locales/ru/chat.json index 6f766b4..95cb1e5 100644 --- a/src/i18n/locales/ru/chat.json +++ b/src/i18n/locales/ru/chat.json @@ -159,10 +159,14 @@ "branchLabel": "ветвь", "thinkingLabel": "Думаю", "runErrorLabel": "Ошибка модели", + "imageGenerateLabel": "Генерация изображения", "agentRun": "Выполнение {{agent}}", "collapsedSummary": "Вызовов инструментов: {{toolCount}} · Промежуточных сообщений: {{processCount}}", "collapseAction": "Свернуть граф выполнения" }, + "imageGeneration": { + "generating": "Изображение генерируется, подождите…" + }, "composer": { "attachFiles": "Прикрепить файлы", "pickSkill": "Выбрать Skill", diff --git a/src/i18n/locales/ru/dashboard.json b/src/i18n/locales/ru/dashboard.json index 19a539b..8f751a6 100644 --- a/src/i18n/locales/ru/dashboard.json +++ b/src/i18n/locales/ru/dashboard.json @@ -90,7 +90,7 @@ "title": "Пользовательский endpoint изображений", "description": "Отправлять генерацию изображений на отдельный OpenAI Images-совместимый endpoint (/v1/images/generations), не меняя чат-провайдеры.", "baseUrl": "Base URL изображений", - "baseUrlHint": "например https://taolat.com/v1 — /v1 добавится автоматически.", + "baseUrlHint": "например https://api.example.com/v1 — /v1 добавится автоматически.", "apiKey": "API key", "apiKeyPlaceholder": "sk-...", "apiKeyPlaceholderConfigured": "Уже настроен — оставьте пустым, чтобы не менять", diff --git a/src/i18n/locales/zh/chat.json b/src/i18n/locales/zh/chat.json index 2359dc3..e5c3119 100644 --- a/src/i18n/locales/zh/chat.json +++ b/src/i18n/locales/zh/chat.json @@ -159,10 +159,14 @@ "branchLabel": "分支", "thinkingLabel": "思考中", "modelErrorTitle": "模型调用失败", + "imageGenerateLabel": "图片生成", "agentRun": "{{agent}} 执行", "collapsedSummary": "{{toolCount}} 个工具调用，{{processCount}} 条过程消息", "collapseAction": "收起执行关系图" }, + "imageGeneration": { + "generating": "图片生成中，请稍候…" + }, "composer": { "attachFiles": "添加文件", "pickSkill": "选择技能", diff --git a/src/i18n/locales/zh/dashboard.json b/src/i18n/locales/zh/dashboard.json index 7848989..617d0e9 100644 --- a/src/i18n/locales/zh/dashboard.json +++ b/src/i18n/locales/zh/dashboard.json @@ -78,9 +78,15 @@ "testing": "生成中…", "save": "保存", "saving": "保存中…", + "clear": "清除配置", + "clearing": "清除中…", + "clearConfirmTitle": "清除图像生成配置？", + "clearConfirmMessage": "将移除自定义生图端点、模型设置和已保存的 API Key，不会影响聊天 Provider。", + "clearConfirmAction": "清除", "testPrompt": "白色背景上的小红圆，极简扁平插画。", "toast": { "saved": "图像生成设置已保存", + "cleared": "图像生成配置已清除", "saveBeforeTest": "请先保存设置再运行测试", "testSuccess": "测试图像已生成（{{ms}} ms）", "testSuccessWithPath": "测试图像已生成（{{ms}} ms）：{{path}}", @@ -90,7 +96,7 @@ "title": "自定义生图端点", "description": "将生图请求发往独立的 OpenAI Images 兼容端点（/v1/images/generations），不会改变任何聊天 Provider。", "baseUrl": "生图 Base URL", - "baseUrlHint": "例如 https://taolat.com/v1；未带 /v1 时会自动补全。", + "baseUrlHint": "例如 https://api.example.com/v1；未带 /v1 时会自动补全。", "apiKey": "API Key", "apiKeyPlaceholder": "sk-...", "apiKeyPlaceholderConfigured": "已配置，留空则不修改", diff --git a/src/lib/image-generation.ts b/src/lib/image-generation.ts index 1378719..8505aa7 100644 --- a/src/lib/image-generation.ts +++ b/src/lib/image-generation.ts @@ -61,6 +61,10 @@ export async function fetchImageGenerationSettings(): Promise { + return saveImageGenerationSettings({ openAiRelayEnabled: false }); +} + export async function saveImageGenerationSettings(payload: { primary?: string | null; fallbacks?: string[]; diff --git a/src/pages/Chat/ChatMessage.tsx b/src/pages/Chat/ChatMessage.tsx index b11c634..51d3f06 100644 --- a/src/pages/Chat/ChatMessage.tsx +++ b/src/pages/Chat/ChatMessage.tsx @@ -16,7 +16,8 @@ import { Button } from '@/components/ui/button'; import { cn } from '@/lib/utils'; import { invokeIpc, statFile } from '@/lib/api-client'; import type { RawMessage, AttachedFileMeta } from '@/stores/chat'; -import { extractText, extractImages, extractToolUse, formatTimestamp } from './message-utils'; +import { extractText, extractImages, extractToolUse, formatTimestamp, isUnresolvableImageUrl } from './message-utils'; +import { copyImageToClipboard, type ImageCopyTarget } from './copy-image'; interface ChatMessageProps { message: RawMessage; @@ -222,7 +223,7 @@ function normalizeLatexDelimiters(input: string): string { /** Resolve an ExtractedImage to a displayable src string, or null if not possible. */ function imageSrc(img: ExtractedImage): string | null { - if (img.url) return img.url; + if (img.url) return isUnresolvableImageUrl(img.url) ? null : img.url; if (img.data) return `data:${img.mimeType};base64,${img.data}`; return null; } @@ -248,6 +249,7 @@ export const ChatMessage = memo(function ChatMessage({ const hideAssistantText = suppressAssistantText && !isUser; const hasText = !hideAssistantText && text.trim().length > 0; const images = extractImages(message); + const resolvableContentImages = images.filter((img) => imageSrc(img) != null); const tools = extractToolUse(message); const visibleTools = suppressToolCards ? [] : tools; const [validatedPaths, setValidatedPaths] = useState>({}); @@ -321,16 +323,18 @@ export const ChatMessage = memo(function ChatMessage({ if (!kind || !file.filePath) return true; return validatedPaths[file.filePath] === true; }); - const attachedFiles = suppressProcessAttachments && (hasText || images.length > 0 || visibleTools.length > 0) + const attachedFiles = suppressProcessAttachments && (hasText || resolvableContentImages.length > 0 || visibleTools.length > 0) ? processVisibleAttachments : existingDerivedAttachedFiles; + const imageCopyTarget = resolvePrimaryImageCopyTarget(resolvableContentImages, attachedFiles); + const showAssistantHoverBar = !isUser && (hasText || imageCopyTarget != null); const [lightboxImg, setLightboxImg] = useState<{ src: string; fileName: string; filePath?: string; base64?: string; mimeType?: string } | null>(null); // Never render tool result messages in chat UI if (isToolResult) return null; const hasStreamingToolStatus = isStreaming && streamingTools.length > 0; - if (!hasText && images.length === 0 && visibleTools.length === 0 && attachedFiles.length === 0 && !hasStreamingToolStatus) return null; + if (!hasText && resolvableContentImages.length === 0 && visibleTools.length === 0 && attachedFiles.length === 0 && !hasStreamingToolStatus) return null; return (
0 && ( + {isUser && resolvableContentImages.length > 0 && (
- {images.map((img, i) => { + {resolvableContentImages.map((img, i) => { const src = imageSrc(img); if (!src) return null; return ( @@ -384,7 +388,7 @@ export const ChatMessage = memo(function ChatMessage({ {attachedFiles.map((file, i) => { const isImage = file.mimeType.startsWith('image/'); // Skip image attachments if we already have images from content blocks - if (isImage && images.length > 0) return null; + if (isImage && resolvableContentImages.length > 0) return null; if (isImage) { return file.preview ? ( 0 && ( + {!isUser && resolvableContentImages.length > 0 && (
- {images.map((img, i) => { + {resolvableContentImages.map((img, i) => { const src = imageSrc(img); if (!src) return null; return ( @@ -444,7 +448,7 @@ export const ChatMessage = memo(function ChatMessage({
{attachedFiles.map((file, i) => { const isImage = file.mimeType.startsWith('image/'); - if (isImage && images.length > 0) return null; + if (isImage && resolvableContentImages.length > 0) return null; if (isImage && file.preview) { return ( )} - {/* Hover row for assistant messages — only when there is real text content */} - {!isUser && hasText && ( - + {/* Hover row for assistant messages */} + {showAssistantHoverBar && ( + )}
@@ -547,16 +551,60 @@ function ToolStatusBar({ ); } +function resolvePrimaryImageCopyTarget( + images: Array<{ mimeType: string; data?: string; url?: string }>, + attachedFiles: AttachedFileMeta[], +): ImageCopyTarget | null { + for (const file of attachedFiles) { + if (!file.mimeType.startsWith('image/')) continue; + if (file.filePath || file.preview?.startsWith('data:')) { + return { + filePath: file.filePath, + preview: file.preview, + mimeType: file.mimeType, + }; + } + } + + for (const image of images) { + if (image.data) { + return { base64: image.data, mimeType: image.mimeType }; + } + if (image.url?.startsWith('data:')) { + return { preview: image.url, mimeType: image.mimeType }; + } + } + + return null; +} + // ── Assistant hover bar (timestamp + copy, shown on group hover) ─ -function AssistantHoverBar({ text, timestamp }: { text: string; timestamp?: number }) { +function AssistantHoverBar({ + text, + timestamp, + imageCopyTarget, +}: { + text: string; + timestamp?: number; + imageCopyTarget?: ImageCopyTarget | null; +}) { const [copied, setCopied] = useState(false); - const copyContent = useCallback(() => { - navigator.clipboard.writeText(text); + const copyContent = useCallback(async () => { + if (imageCopyTarget) { + const copiedImage = await copyImageToClipboard(imageCopyTarget); + if (copiedImage) { + setCopied(true); + setTimeout(() => setCopied(false), 2000); + return; + } + } + if (!text.trim()) return; + await navigator.clipboard.writeText(text); setCopied(true); setTimeout(() => setCopied(false), 2000); - }, [text]); + }, [text, imageCopyTarget]); return (
@@ -629,6 +677,12 @@ function AssistantMarkdown({ ); }, + img({ src, alt }) { + if (!src || isUnresolvableImageUrl(String(src))) return null; + return ( + {typeof + ); + }, }} > {normalizeLatexDelimiters(text)} diff --git a/src/pages/Chat/ExecutionGraphCard.tsx b/src/pages/Chat/ExecutionGraphCard.tsx index 1522170..37b39f5 100644 --- a/src/pages/Chat/ExecutionGraphCard.tsx +++ b/src/pages/Chat/ExecutionGraphCard.tsx @@ -47,6 +47,9 @@ function StepDetailCard({ step }: { step: TaskStep }) { const isNarration = step.kind === 'message'; const isTool = step.kind === 'tool'; const isThinking = step.kind === 'thinking'; + const displayToolLabel = isTool && step.label === 'image_generate' + ? t('executionGraph.imageGenerateLabel') + : step.label; // System steps (subagent branch roots etc.) share the tool row layout: // bold label + truncated single-line detail preview + click-to-expand, // i.e. no rounded card / no separate detail line below the title. @@ -56,7 +59,7 @@ function StepDetailCard({ step }: { step: TaskStep }) { const hideStatusText = (isTool || isSystem) && step.status === 'completed'; const detailPreview = step.detail?.replace(/\s+/g, ' ').trim(); const canExpand = hasDetail; - const displayLabel = isThinking ? t('executionGraph.thinkingLabel') : step.label; + const displayLabel = isThinking ? t('executionGraph.thinkingLabel') : (isTool ? displayToolLabel : step.label); return (
{ + try { + const response = await fetch(dataUrl); + if (!response.ok) return null; + return await response.blob(); + } catch { + return null; + } +} + +async function writeImageBlob(blob: Blob, fallbackMimeType: string): Promise { + if (!navigator.clipboard?.write || typeof ClipboardItem === 'undefined') return false; + const type = blob.type || fallbackMimeType; + await navigator.clipboard.write([new ClipboardItem({ [type]: blob })]); + return true; +} + +export async function copyImageToClipboard(target: ImageCopyTarget): Promise { + const mimeType = target.mimeType?.startsWith('image/') ? target.mimeType : 'image/png'; + + if (target.base64) { + const blob = await blobFromDataUrl(`data:${mimeType};base64,${target.base64}`); + return blob ? writeImageBlob(blob, mimeType) : false; + } + + if (target.preview?.startsWith('data:')) { + const blob = await blobFromDataUrl(target.preview); + return blob ? writeImageBlob(blob, mimeType) : false; + } + + if (target.filePath) { + const result = await readBinaryFile(target.filePath); + if (!result.ok || !result.data) return false; + return writeImageBlob(blobFromBytes(result.data, result.mimeType || mimeType), mimeType); + } + + return false; +} diff --git a/src/pages/Chat/image-generation-status.ts b/src/pages/Chat/image-generation-status.ts new file mode 100644 index 0000000..a50311b --- /dev/null +++ b/src/pages/Chat/image-generation-status.ts @@ -0,0 +1,162 @@ +import type { RawMessage, ToolStatus } from '@/stores/chat'; +import { + extractImages, + extractToolUse, + isGeneratingStatusNarration, + isInternalAssistantReplyText, +} from './message-utils'; + +const IMAGE_GENERATE_TOOL = 'image_generate'; +const ASYNC_IMAGE_TASK_START_RE = /Background task started for image generation \(([0-9a-f-]{36})\)/i; +const INTER_SESSION_IMAGE_TASK_RE = /sourceSession=image_generate:([0-9a-f-]{36})/i; +/** Match OpenClaw agents.defaults.imageGenerationModel.timeoutMs default. */ +export const IMAGE_GENERATION_TIMEOUT_MS = 180_000; +const IMAGE_GENERATION_TIMEOUT_BUFFER_MS = 15_000; + +function toMs(timestamp: number | undefined): number | null { + if (timestamp == null || !Number.isFinite(timestamp)) return null; + return timestamp < 1e12 ? timestamp * 1000 : timestamp; +} + +function getMessagePlainText(message: RawMessage): string { + const { content } = message; + if (typeof content === 'string') return content; + if (!Array.isArray(content)) return ''; + const parts: string[] = []; + for (const block of content as Array<{ type?: string; text?: string }>) { + if (block.type === 'text' && typeof block.text === 'string') { + parts.push(block.text); + } + } + return parts.join('\n'); +} + +function messageHasDeliveredImage(message: RawMessage): boolean { + if ((message._attachedFiles ?? []).some((file) => file.mimeType.startsWith('image/'))) { + return true; + } + return extractImages(message).length > 0; +} + +function findLastImageGenerateToolCallIndex(segmentMessages: RawMessage[]): number { + for (let index = segmentMessages.length - 1; index >= 0; index -= 1) { + const message = segmentMessages[index]; + if (message.role !== 'assistant') continue; + if (extractToolUse(message).some((tool) => tool.name === IMAGE_GENERATE_TOOL)) { + return index; + } + } + return -1; +} + +function collectAsyncImageTaskStarts( + segmentMessages: RawMessage[], +): Array<{ taskId: string; startedAtMs: number | null }> { + const starts: Array<{ taskId: string; startedAtMs: number | null }> = []; + for (const message of segmentMessages) { + const role = String(message.role ?? '').toLowerCase(); + if (role !== 'toolresult' && role !== 'tool_result') continue; + if (message.toolName !== IMAGE_GENERATE_TOOL) continue; + const match = getMessagePlainText(message).match(ASYNC_IMAGE_TASK_START_RE); + if (!match?.[1]) continue; + starts.push({ + taskId: match[1], + startedAtMs: toMs(message.timestamp), + }); + } + return starts; +} + +function collectCompletedAsyncImageTaskIds(segmentMessages: RawMessage[]): Set { + const completed = new Set(); + for (const message of segmentMessages) { + const role = String(message.role ?? '').toLowerCase(); + if (role !== 'user') continue; + const text = getMessagePlainText(message).trim(); + if (!/^\[Inter-session message\]/i.test(text)) continue; + const match = text.match(INTER_SESSION_IMAGE_TASK_RE); + if (match?.[1]) completed.add(match[1]); + } + return completed; +} + +function hasTimedOut(startedAtMs: number | null, now: number): boolean { + if (startedAtMs == null) return false; + return now - startedAtMs > IMAGE_GENERATION_TIMEOUT_MS + IMAGE_GENERATION_TIMEOUT_BUFFER_MS; +} + +function hasDeliveredImageAfter(segmentMessages: RawMessage[], fromIndex: number): boolean { + for (let index = fromIndex + 1; index < segmentMessages.length; index += 1) { + const message = segmentMessages[index]; + if (message.role !== 'assistant') continue; + if (messageHasDeliveredImage(message)) return true; + } + return false; +} + +function hasTerminalReplyAfterToolCall(segmentMessages: RawMessage[], fromIndex: number): boolean { + for (let index = fromIndex + 1; index < segmentMessages.length; index += 1) { + const message = segmentMessages[index]; + if (message.role !== 'assistant') continue; + if (extractToolUse(message).length > 0) continue; + + const text = getMessagePlainText(message).trim(); + if (!text || isInternalAssistantReplyText(text)) continue; + if (isGeneratingStatusNarration(text)) continue; + return true; + } + return false; +} + +function hasFreshRunningStreamingTool(streamingTools: ToolStatus[], now: number): boolean { + return streamingTools.some((tool) => { + if (tool.name !== IMAGE_GENERATE_TOOL || tool.status !== 'running') return false; + return now - tool.updatedAt <= IMAGE_GENERATION_TIMEOUT_MS + IMAGE_GENERATION_TIMEOUT_BUFFER_MS; + }); +} + +/** True while an async `image_generate` task is in flight for this run segment. */ +export function isImageGenerationPending( + segmentMessages: RawMessage[], + streamingTools: ToolStatus[] = [], + now = Date.now(), +): boolean { + if (hasFreshRunningStreamingTool(streamingTools, now)) { + return true; + } + + const toolCallIndex = findLastImageGenerateToolCallIndex(segmentMessages); + const asyncStarts = collectAsyncImageTaskStarts(segmentMessages); + const completedTaskIds = collectCompletedAsyncImageTaskIds(segmentMessages); + + if (toolCallIndex < 0 && asyncStarts.length === 0) { + return false; + } + + const toolCallStartedAt = toolCallIndex >= 0 + ? toMs(segmentMessages[toolCallIndex]?.timestamp) + : null; + const latestAsyncStartedAt = asyncStarts.reduce((latest, start) => { + if (start.startedAtMs == null) return latest; + return latest == null ? start.startedAtMs : Math.max(latest, start.startedAtMs); + }, null); + const anchorStartedAt = [toolCallStartedAt, latestAsyncStartedAt] + .filter((value): value is number => value != null) + .reduce((latest, value) => (latest == null ? value : Math.max(latest, value)), null); + + if (hasTimedOut(anchorStartedAt, now)) { + return false; + } + + if (toolCallIndex >= 0) { + if (hasDeliveredImageAfter(segmentMessages, toolCallIndex)) return false; + if (hasTerminalReplyAfterToolCall(segmentMessages, toolCallIndex)) return false; + } + + if (asyncStarts.length > 0) { + const hasOpenAsyncTask = asyncStarts.some((start) => !completedTaskIds.has(start.taskId)); + if (!hasOpenAsyncTask) return false; + } + + return toolCallIndex >= 0 || asyncStarts.some((start) => !completedTaskIds.has(start.taskId)); +} diff --git a/src/pages/Chat/index.tsx b/src/pages/Chat/index.tsx index 96ec103..b1ff4b7 100644 --- a/src/pages/Chat/index.tsx +++ b/src/pages/Chat/index.tsx @@ -7,6 +7,7 @@ import { Suspense, lazy, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { AlertCircle, ArrowDownToLine, Loader2, Sparkles } from 'lucide-react'; import { useChatStore, type RawMessage } from '@/stores/chat'; +import { isInternalMessage } from '@/stores/chat/helpers'; import { buildBaselineRunKey, getBaseline } from '@/stores/baseline-cache'; import { useGatewayStore } from '@/stores/gateway'; import { useAgentsStore } from '@/stores/agents'; @@ -18,7 +19,7 @@ import { ChatMessage } from './ChatMessage'; import { ChatInput } from './ChatInput'; import { ExecutionGraphCard } from './ExecutionGraphCard'; import { ChatToolbar } from './ChatToolbar'; -import { extractImages, extractText, extractThinking, extractToolUse, normalizeMessageRole, stripProcessMessagePrefix } from './message-utils'; +import { extractImages, extractText, extractThinking, extractToolUse, isInternalAssistantReplyText, isInternalProcessNarration, normalizeMessageRole, stripProcessMessagePrefix } from './message-utils'; import { buildRunSegmentMessageIndices, deriveTaskSteps, @@ -30,6 +31,7 @@ import { segmentHasFinalReply, type TaskStep, } from './task-visualization'; +import { isImageGenerationPending } from './image-generation-status'; import { useTranslation } from 'react-i18next'; import { cn } from '@/lib/utils'; import { useStickToBottomInstant } from '@/hooks/use-stick-to-bottom-instant'; @@ -95,6 +97,14 @@ function getPrimaryMessageStepTexts(steps: TaskStep[]): string[] { .map((step) => step.detail!); } +function sanitizeGraphSteps(steps: TaskStep[]): TaskStep[] { + return steps.filter((step) => { + if (step.kind === 'thinking') return false; + if (step.kind === 'message' && step.detail && isInternalProcessNarration(step.detail)) return false; + return true; + }); +} + function buildQuestionDirectoryTitle(message: RawMessage, fallback: string): string { const normalized = extractText(message).replace(/\s+/g, ' ').trim(); if (!normalized) return fallback; @@ -103,6 +113,7 @@ function buildQuestionDirectoryTitle(message: RawMessage, fallback: string): str function isRealUserMessage(msg: RawMessage): boolean { if (normalizeMessageRole(msg.role) !== 'user') return false; + if (isInternalMessage(msg)) return false; const content = msg.content; if (!Array.isArray(content)) return true; // If every block in the content is a tool_result, this is a Gateway @@ -111,6 +122,10 @@ function isRealUserMessage(msg: RawMessage): boolean { return blocks.length === 0 || !blocks.every((b) => b.type === 'tool_result' || b.type === 'toolResult'); } +function hasUserFacingImageAttachments(msg: RawMessage): boolean { + return (msg._attachedFiles ?? []).some((file) => file.mimeType.startsWith('image/')); +} + function generatedFileToTarget(file: GeneratedFile): FilePreviewTarget { return { filePath: file.filePath, @@ -493,15 +508,16 @@ export function Chat() { && streamTools.length === 0 && !hasRunningStreamToolStatus; - let steps = buildSteps(rawStreamingReplyCandidate); + let steps = sanitizeGraphSteps(buildSteps(rawStreamingReplyCandidate)); let streamingReplyText: string | null = null; if (rawStreamingReplyCandidate) { const trimmedReplyText = stripProcessMessagePrefix(streamText, getPrimaryMessageStepTexts(steps)); - const hasReplyText = trimmedReplyText.trim().length > 0; + const hasReplyText = trimmedReplyText.trim().length > 0 + && !isInternalAssistantReplyText(trimmedReplyText); if (hasReplyText || hasStreamImages) { - streamingReplyText = trimmedReplyText; + streamingReplyText = hasReplyText ? trimmedReplyText : ''; } else { - steps = buildSteps(false); + steps = sanitizeGraphSteps(buildSteps(false)); } } @@ -545,9 +561,9 @@ export function Chat() { // generated message steps that include accumulated narration + reply // text. Strip these out — historical message steps (from messages[]) // will be properly recomputed on the next render with fresh data. - const cleanedSteps = cached.steps.filter( + const cleanedSteps = sanitizeGraphSteps(cached.steps.filter( (s) => !(s.kind === 'message' && s.id.startsWith('stream-message')), - ); + )); return [{ triggerIndex: idx, replyIndex: cached.replyIndex, @@ -622,6 +638,7 @@ export function Chat() { }, [messages, subagentCompletionInfos, currentSessionKey, streamingMessage, streamingTools, pendingFinal, sending, hasAnyStreamContent, hasStreamText, hasStreamImages, streamText, streamTools.length, hasRunningStreamToolStatus, childTranscripts, currentAgentId, agents, sessionLabels, graphStepCache, runError, isRunTrigger]); const hasActiveExecutionGraph = userRunCards.some((card) => card.active); let latestRunSegmentCompletion = { hasFinalReply: false, hasToolActivity: false }; + let pendingImageGeneration = false; for (let idx = messages.length - 1; idx >= 0; idx -= 1) { if (!isRealUserMessage(messages[idx]) || subagentCompletionInfos[idx]) continue; const nextUserIndex = nextUserMessageIndexes[idx]; @@ -632,6 +649,10 @@ export function Chat() { m.role === 'assistant' && extractToolUse(m).length > 0, ), }; + pendingImageGeneration = isImageGenerationPending( + postTrigger, + nextUserIndex === -1 ? streamingTools : [], + ); break; } const runSettledInHistory = latestRunSegmentCompletion.hasFinalReply @@ -824,7 +845,9 @@ export function Chat() {
)} {messages.map((msg, idx) => { - if (foldedNarrationIndices.has(idx)) return null; + if (isInternalMessage(msg) && !hasUserFacingImageAttachments(msg)) return null; + const isFoldedNarration = foldedNarrationIndices.has(idx); + if (isFoldedNarration && !hasUserFacingImageAttachments(msg)) return null; const suppressToolCards = runSegmentMessageIndices.has(idx); const isToolOnlyAssistant = normalizeMessageRole(msg.role) === 'assistant' && extractToolUse(msg).length > 0 @@ -843,6 +866,7 @@ export function Chat() { )} + {pendingImageGeneration && ( + + )} + {/* Typing indicator when sending but no stream content yet */} - {inputRunActive && !pendingFinal && !hasAnyStreamContent && !hasActiveExecutionGraph && ( + {inputRunActive && !pendingFinal && !hasAnyStreamContent && !hasActiveExecutionGraph && !pendingImageGeneration && ( )} @@ -1195,4 +1223,21 @@ function ActivityIndicator({ phase }: { phase: 'tool_processing' }) { ); } +function ImageGeneratingIndicator() { + const { t } = useTranslation('chat'); + return ( +
+
+ +
+
+
+ + {t('imageGeneration.generating')} +
+
+
+ ); +} + export default Chat; diff --git a/src/pages/Chat/message-utils.ts b/src/pages/Chat/message-utils.ts index fc3e497..2511f8b 100644 --- a/src/pages/Chat/message-utils.ts +++ b/src/pages/Chat/message-utils.ts @@ -5,13 +5,39 @@ */ import type { RawMessage, ContentBlock } from '@/stores/chat'; +/** + * Strip OpenClaw's inbound-image vision envelope from user message text. + * The runtime expands uploaded images into: + * [Image] + * User text: + * + * Description: + * + */ +function stripInboundMediaVisionEnvelope(text: string): string { + if (!/\[Image\]/i.test(text) && !/^User text:/im.test(text) && !/\nDescription:\s*\n/i.test(text)) { + return text; + } + + let result = text.replace(/^\s*\[Image\]\s*\n?/i, ''); + + const userTextBlock = result.match(/^User text:\s*\n([\s\S]*?)(?:\n\s*Description:\s*\n[\s\S]*)?\s*$/i); + if (userTextBlock) { + const userText = userTextBlock[1].trim(); + return /^Process the attached file\(s\)\.\s*$/i.test(userText) ? '' : userText; + } + + return result.replace(/\n\s*Description:\s*\n[\s\S]*$/i, '').trim(); +} + /** * Clean Gateway metadata from user message text for display. * Strips: [media attached: ... | ...], [message_id: ...], * and the timestamp prefix [Day Date Time Timezone]. */ function cleanUserText(text: string): string { - return text + return stripInboundMediaVisionEnvelope( + text // Remove [media attached: path (mime) | path] references .replace(/\s*\[media attached:[^\]]*\]/g, '') // Remove [message_id: uuid] @@ -30,7 +56,8 @@ function cleanUserText(text: string): string { .replace(/^Conversation info\s*\([^)]*\):\s*\{[\s\S]*?\}\s*/i, '') // Remove Gateway timestamp prefix like [Fri 2026-02-13 22:39 GMT+8] .replace(/^\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, '') - .trim(); + .trim(), + ); } /** @@ -51,8 +78,13 @@ function stripAssistantMediaTags(text: string): string { // would still leak the literal `MEDIA:/.../截屏 2026-05-06 17.46.51.png` // to the user when the underlying path detection succeeds. const tagged = new RegExp(`(^|[\\s(\\[{>])(?:MEDIA|media):(?:\\/|~\\/)[^\\n"'()\\[\\],<>]*?\\.(?:${exts})(?=$|[\\s\\n"'()\\[\\],<>]|[，。；;,.!?])`, 'g'); + // Bare OpenClaw artifact paths emitted alongside `_attachedFiles` cards. + // Scope to `.openclaw/media/` so normal absolute paths in prose stay visible. + const bareOpenClawMedia = new RegExp(`(^|[\\s(\\[{>])(?:(?:\\/|~\\/)[^\\n"'()\\[\\],<>]*?\\.openclaw\\/media\\/[^\\n"'()\\[\\],<>]*?\\.(?:${exts}))(?=$|[\\s\\n"'()\\[\\],<>]|[，。；;,.!?])`, 'g'); return text .replace(tagged, (_, lead: string) => lead) + .replace(bareOpenClawMedia, (_, lead: string) => lead) + .replace(/!\[[^\]]*\]\([^)]+\)/g, '') // Collapse the empty lines / orphan whitespace the strip leaves behind. .replace(/[ \t]+\n/g, '\n') .replace(/\n{3,}/g, '\n\n') @@ -143,6 +175,57 @@ function consumeLeadingSegment(text: string, segment: string): number { return match ? match[0].length : 0; } +/** True for OpenClaw internal assistant tokens that must never appear in the chat UI. */ +export function isInternalAssistantReplyText(text: string): boolean { + return /^(HEARTBEAT_OK|NO_REPLY)\s*$/i.test(text.trim()); +} + +/** Interim image-generation status the user should not see as a final answer. */ +export function isGeneratingStatusNarration(text: string): boolean { + const trimmed = text.trim(); + if (!trimmed) return false; + if (/^(?:图片(?:正在)?生成中|正在生成(?:图片|图像)|生成中)/i.test(trimmed)) return true; + if (/稍等(片刻|一下)?/.test(trimmed) && trimmed.length <= 80 && /生成|generat/i.test(trimmed)) return true; + return false; +} + +/** OpenClaw runtime continuation prompt injected between internal async events. */ +export function isOpenClawRuntimeEventPrompt(text: string): boolean { + const trimmed = text.trim(); + if (!trimmed) return false; + if (/^Continue the OpenClaw runtime event\.?\s*$/i.test(trimmed)) return true; + // Some transcripts store the prompt as the only line in a longer envelope. + return trimmed.split(/\n+/).some((line) => /^Continue the OpenClaw runtime event\.?\s*$/i.test(line.trim())); +} + +/** Process narration that should never appear in the execution graph or chat stream. */ +export function isInternalProcessNarration(text: string): boolean { + const trimmed = text.trim(); + if (!trimmed) return true; + if (isInternalAssistantReplyText(trimmed)) return true; + if (isGeneratingStatusNarration(trimmed)) return true; + if (isOpenClawRuntimeEventPrompt(trimmed)) return true; + if (/^\[Inter-session message\]/i.test(trimmed)) return true; + if (/OpenClaw runtime event/i.test(trimmed)) return true; + if (/Handle the result internally/i.test(trimmed) && /Do not relay it to the user/i.test(trimmed)) { + return true; + } + return false; +} + +/** True when a renderer `src` cannot be fetched directly (Gateway proxy / local paths). */ +export function isUnresolvableImageUrl(url: string): boolean { + const trimmed = url.trim(); + if (!trimmed) return true; + if (trimmed.startsWith('data:')) return false; + if (/^https?:\/\//i.test(trimmed)) return false; + if (trimmed.startsWith('/api/chat/media/')) return true; + if (trimmed.startsWith('file://')) return true; + if (trimmed.includes('.openclaw/media/')) return true; + if (trimmed.startsWith('/') || trimmed.startsWith('~/')) return true; + return false; +} + /** * Extract displayable text from a message's content field. * Handles both string content and array-of-blocks content. @@ -178,6 +261,8 @@ export function extractText(message: RawMessage | unknown): string { if (isUser && result) { result = cleanUserText(result); } else if (!isUser && result) { + if (isInternalAssistantReplyText(result)) return ''; + if (isGeneratingStatusNarration(result)) return ''; // Assistant-side cleanup: keep the bubble free of `MEDIA:/path` tags // that the runtime emits to point at produced artifacts. The same // path is surfaced as a clickable file card via `_attachedFiles`, @@ -348,7 +433,7 @@ export function extractImages( const src = block.source; if (src.type === 'base64' && src.media_type && src.data) { images.push({ mimeType: src.media_type, data: src.data }); - } else if (src.type === 'url' && src.url) { + } else if (src.type === 'url' && src.url && !isUnresolvableImageUrl(src.url)) { images.push({ mimeType: src.media_type || 'image/jpeg', url: src.url }); } continue; diff --git a/src/pages/Chat/task-visualization.ts b/src/pages/Chat/task-visualization.ts index f03e7c5..664e7ea 100644 --- a/src/pages/Chat/task-visualization.ts +++ b/src/pages/Chat/task-visualization.ts @@ -1,4 +1,12 @@ -import { extractText, extractTextSegments, extractThinkingSegments, extractToolUse } from './message-utils'; +import { + extractText, + extractTextSegments, + extractToolUse, + isGeneratingStatusNarration, + isInternalAssistantReplyText, + isInternalProcessNarration, +} from './message-utils'; +import { isInternalMessage } from '@/stores/chat/helpers'; import type { RawMessage, ToolStatus } from '@/stores/chat'; export type TaskStepStatus = 'running' | 'completed' | 'error'; @@ -37,7 +45,9 @@ export function findReplyMessageIndex(messages: RawMessage[], hasStreamingReply: for (let idx = messages.length - 1; idx >= 0; idx -= 1) { const message = messages[idx]; if (!message || message.role !== 'assistant') continue; - if (extractText(message).trim().length === 0) continue; + const replyText = extractText(message).trim(); + if (replyText.length === 0 || isInternalAssistantReplyText(replyText)) continue; + if (isGeneratingStatusNarration(replyText)) continue; return idx; } return -1; @@ -149,7 +159,9 @@ export function segmentHasFinalReply(segmentMessages: RawMessage[]): boolean { return segmentMessages.some((message, index) => { if (index <= lastToolUseOffset) return false; if (message.role !== 'assistant') return false; - if (extractText(message).trim().length === 0) return false; + const replyText = extractText(message).trim(); + if (replyText.length === 0 || isInternalAssistantReplyText(replyText)) return false; + if (isGeneratingStatusNarration(replyText)) return false; const content = message.content; if (!Array.isArray(content)) return true; return !(content as Array<{ type?: string }>).some( @@ -310,7 +322,8 @@ function appendDetailSegments( ): void { const normalizedSegments = segments .map((segment) => normalizeText(segment)) - .filter((segment): segment is string => !!segment); + .filter((segment): segment is string => !!segment) + .filter((segment) => !isInternalProcessNarration(segment)); normalizedSegments.forEach((detail, index) => { options.upsertStep({ @@ -361,31 +374,27 @@ export function deriveTaskSteps({ for (const [messageIndex, message] of messages.entries()) { if (!message || message.role !== 'assistant') continue; - appendDetailSegments(extractThinkingSegments(message), { - idPrefix: `history-thinking-${message.id || messageIndex}`, - label: 'Thinking', - kind: 'thinking', - running: false, - upsertStep, - }); - const toolUses = extractToolUse(message); - // Fold any intermediate assistant text into the graph as a narration - // step — including text that lives on a mixed `text + toolCall` message. - // The narration step is emitted BEFORE the tool steps so the graph - // preserves the original ordering (the assistant "thinks out loud" and - // then invokes the tool). - const narrationSegments = extractTextSegments(message); - const graphNarrationSegments = messageIndex === replyIndex - ? narrationSegments.slice(0, -1) - : narrationSegments; - appendDetailSegments(graphNarrationSegments, { - idPrefix: `history-message-${message.id || messageIndex}`, - label: 'Message', - kind: 'message', - running: false, - upsertStep, - }); + if (!isInternalMessage(message)) { + // Fold any intermediate assistant text into the graph as a narration + // step — including text that lives on a mixed `text + toolCall` message. + // The narration step is emitted BEFORE the tool steps so the graph + // preserves the original ordering (the assistant "thinks out loud" and + // then invokes the tool). + const narrationSegments = extractTextSegments(message); + const graphNarrationSegments = messageIndex === replyIndex + ? narrationSegments.slice(0, -1) + : narrationSegments; + appendDetailSegments(graphNarrationSegments, { + idPrefix: `history-message-${message.id || messageIndex}`, + label: 'Message', + kind: 'message', + running: false, + upsertStep, + }); + } else if (toolUses.length === 0) { + continue; + } toolUses.forEach((tool, index) => { const input = tool.input as Record; @@ -403,19 +412,6 @@ export function deriveTaskSteps({ } if (streamMessage) { - // When the reply is being rendered as a separate bubble - // (omitLastStreamingMessageSegment), thinking that accompanies - // the reply belongs to the bubble — omit it from the graph. - if (!omitLastStreamingMessageSegment) { - appendDetailSegments(extractThinkingSegments(streamMessage), { - idPrefix: 'stream-thinking', - label: 'Thinking', - kind: 'thinking', - running: true, - upsertStep, - }); - } - // Stream-time narration should also appear in the execution graph so that // intermediate process output stays in P1 instead of leaking into the // assistant reply area. diff --git a/src/stores/chat.ts b/src/stores/chat.ts index 7d2e263..0d0812a 100644 --- a/src/stores/chat.ts +++ b/src/stores/chat.ts @@ -45,6 +45,12 @@ import { type ToolStatus, } from './chat/types'; import type { ChatGet, ChatSet } from './chat/store-api'; +import { enrichWithToolCallAttachments, shouldDropMessageFromHistory } from './chat/helpers'; +import { + isGeneratingStatusNarration, + isInternalAssistantReplyText, + isOpenClawRuntimeEventPrompt, +} from '@/pages/Chat/message-utils'; export type { AttachedFileMeta, @@ -567,8 +573,25 @@ function normalizeStreamingMessage(message: unknown): unknown { * `[media attached:` instead — leaving the timestamp in the normalized * comparison text and breaking optimistic-vs-echo dedupe. */ +function stripInboundMediaVisionEnvelope(text: string): string { + if (!/\[Image\]/i.test(text) && !/^User text:/im.test(text) && !/\nDescription:\s*\n/i.test(text)) { + return text; + } + + let result = text.replace(/^\s*\[Image\]\s*\n?/i, ''); + + const userTextBlock = result.match(/^User text:\s*\n([\s\S]*?)(?:\n\s*Description:\s*\n[\s\S]*)?\s*$/i); + if (userTextBlock) { + const userText = userTextBlock[1].trim(); + return /^Process the attached file\(s\)\.\s*$/i.test(userText) ? '' : userText; + } + + return result.replace(/\n\s*Description:\s*\n[\s\S]*$/i, '').trim(); +} + function stripGatewayUserMetadata(text: string): string { - return text + return stripInboundMediaVisionEnvelope( + text .replace(/\s*\[media attached:[^\]]*\]/g, '') .replace(/\s*\[message_id:\s*[^\]]+\]/g, '') .replace(/^Sender\s*\([^)]*\)\s*:\s*```[a-z]*\n[\s\S]*?```\s*/i, '') @@ -579,13 +602,16 @@ function stripGatewayUserMetadata(text: string): string { .replace(/^Sender\s*:\s*[^\n]*(?:\n\s*)*/i, '') .replace(/^Conversation info\s*\([^)]*\):\s*```[a-z]*\n[\s\S]*?```\s*/i, '') .replace(/^Conversation info\s*\([^)]*\):\s*\{[\s\S]*?\}\s*/i, '') - .replace(/^\s*\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, ''); + .replace(/^\s*\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, ''), + ); } function normalizeComparableUserText(content: unknown): string { - return stripGatewayUserMetadata(getMessageText(content)) + const text = stripGatewayUserMetadata(getMessageText(content)) .replace(/\s+/g, ' ') .trim(); + if (/^\(file attached\)$/i.test(text)) return ''; + return text; } function getComparableAttachmentSignature(message: Pick): string { @@ -620,6 +646,13 @@ function matchesOptimisticUserMessage( if (sameText && sameAttachments) return true; if (sameText && (!optimisticAttachments || !candidateAttachments) && (timestampMatches || !hasCandidateTimestamp)) return true; if (sameAttachments && (!optimisticText || !candidateText) && (timestampMatches || !hasCandidateTimestamp)) return true; + + const optimisticHadAttachmentsOnly = optimisticAttachments.length > 0 && !optimisticText; + const candidateIsAttachmentEcho = !candidateText + && /\[(?:media attached:|\s*Image\s*\])/i.test(getMessageText(candidate.content)); + if (optimisticHadAttachmentsOnly && candidateIsAttachmentEcho && (timestampMatches || !hasCandidateTimestamp)) { + return true; + } return false; } @@ -768,6 +801,13 @@ function getMessageText(content: unknown): string { return ''; } +function getMessageTextForFilter(msg: { content?: unknown; text?: unknown }): string { + const fromContent = getMessageText(msg.content); + if (fromContent.trim()) return fromContent; + if (typeof msg.text === 'string') return msg.text; + return ''; +} + function getMessageStopReason(message: RawMessage | unknown): string | null { if (!message || typeof message !== 'object') return null; const msg = message as Record; @@ -853,6 +893,26 @@ function mimeFromExtension(filePath: string): string { return map[ext] || 'application/octet-stream'; } +function extractFilePathsFromToolArgs(args: Record): string[] { + const paths: string[] = []; + const direct = args.file_path ?? args.filePath ?? args.path ?? args.file; + if (typeof direct === 'string' && direct.trim()) paths.push(direct.trim()); + + const attachments = args.attachments; + if (Array.isArray(attachments)) { + for (const item of attachments) { + if (!item || typeof item !== 'object') continue; + const att = item as Record; + const filePath = att.filePath ?? att.file_path ?? att.path ?? att.file; + if (typeof filePath === 'string' && filePath.trim()) { + paths.push(filePath.trim()); + } + } + } + + return paths; +} + const DIRECTORY_MIME_TYPE = 'application/x-directory'; function trimPathTerminators(filePath: string): string { @@ -1018,8 +1078,8 @@ function getToolCallFilePath(msg: RawMessage, toolCallId: string): string | unde if ((block.type === 'tool_use' || block.type === 'toolCall') && block.id === toolCallId) { const args = (block.input ?? block.arguments) as Record | undefined; if (args) { - const fp = args.file_path ?? args.filePath ?? args.path ?? args.file; - if (typeof fp === 'string') return fp; + const paths = extractFilePathsFromToolArgs(args); + if (paths[0]) return paths[0]; } } } @@ -1037,8 +1097,8 @@ function getToolCallFilePath(msg: RawMessage, toolCallId: string): string | unde args = typeof fn.arguments === 'string' ? JSON.parse(fn.arguments) : (fn.arguments ?? fn.input) as Record; } catch { /* ignore */ } if (args) { - const fp = args.file_path ?? args.filePath ?? args.path ?? args.file; - if (typeof fp === 'string') return fp; + const paths = extractFilePathsFromToolArgs(args); + if (paths[0]) return paths[0]; } } } @@ -1056,8 +1116,8 @@ function collectToolCallPaths(msg: RawMessage, paths: Map): void if ((block.type === 'tool_use' || block.type === 'toolCall') && block.id) { const args = (block.input ?? block.arguments) as Record | undefined; if (args) { - const fp = args.file_path ?? args.filePath ?? args.path ?? args.file; - if (typeof fp === 'string') paths.set(block.id, fp); + const filePaths = extractFilePathsFromToolArgs(args); + if (filePaths[0]) paths.set(block.id, filePaths[0]); } } } @@ -1074,8 +1134,8 @@ function collectToolCallPaths(msg: RawMessage, paths: Map): void args = typeof fn.arguments === 'string' ? JSON.parse(fn.arguments) : (fn.arguments ?? fn.input) as Record; } catch { /* ignore */ } if (args) { - const fp = args.file_path ?? args.filePath ?? args.path ?? args.file; - if (typeof fp === 'string') paths.set(id, fp); + const filePaths = extractFilePathsFromToolArgs(args); + if (filePaths[0]) paths.set(id, filePaths[0]); } } } @@ -1613,11 +1673,12 @@ function isToolResultRole(role: unknown): boolean { } /** True for internal plumbing messages that should never be shown in the UI. */ -function isInternalMessage(msg: { role?: unknown; content?: unknown; idempotencyKey?: unknown; model?: unknown }): boolean { +function isInternalMessage(msg: { role?: unknown; content?: unknown; idempotencyKey?: unknown; model?: unknown; text?: unknown }): boolean { if (msg.role === 'system') return true; - const text = getMessageText(msg.content); + const text = getMessageTextForFilter(msg); if (msg.role === 'assistant') { - if (/^(HEARTBEAT_OK|NO_REPLY)\s*$/.test(text)) return true; + if (isInternalAssistantReplyText(text)) return true; + if (isGeneratingStatusNarration(text)) return true; // OpenClaw's gateway writes a fallback `assistant-media` transcript // message when its `createManagedOutgoingImageBlocks` pipeline fails // ("could not be prepared" warning in stderr). The fallback has: @@ -1642,6 +1703,12 @@ function isInternalMessage(msg: { role?: unknown; content?: unknown; idempotency // canonical render — keep them. Only hide the text-only fallback. if (!hasImageUrlBlock) return true; } + if (!text.trim() && Array.isArray(msg.content)) { + const blocks = msg.content as ContentBlock[]; + const hasThinking = blocks.some((block) => block.type === 'thinking' && block.thinking?.trim()); + const hasVisibleText = blocks.some((block) => block.type === 'text' && block.text?.trim()); + if (hasThinking && !hasVisibleText) return true; + } } if (msg.role === 'user' && /^\[OpenClaw heartbeat poll\]\s*$/i.test(text.trim())) return true; // Runtime system injections: these arrive as user or assistant-role messages @@ -1668,6 +1735,9 @@ function isRuntimeSystemInjection(text: string): boolean { ) { return true; } + if (/^\[Inter-session message\]/i.test(normalized)) return true; + if (isOpenClawRuntimeEventPrompt(normalized)) return true; + if ( /^\s*Current time\s*:/i.test(normalized) && /^\s*Current time\s*:[^\n]*\/\s*\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+UTC\s*$/i.test(normalized) @@ -2631,7 +2701,8 @@ export const useChatStore = create((set, get) => ({ // Before filtering: attach images/files from tool_result messages to the next assistant message const messagesWithToolImages = enrichWithToolResultFiles(rawMessages); - const filteredMessages = messagesWithToolImages.filter((msg) => !isToolResultRole(msg.role) && !isInternalMessage(msg)); + const messagesWithToolAttachments = enrichWithToolCallAttachments(messagesWithToolImages); + const filteredMessages = messagesWithToolAttachments.filter((msg) => !shouldDropMessageFromHistory(msg)); // Restore file attachments for user/assistant messages (from cache + text patterns) const enrichedMessages = enrichWithCachedImages(filteredMessages); @@ -3024,7 +3095,8 @@ export const useChatStore = create((set, get) => ({ // window with a larger suffix from the transcript. This keeps render // cost bounded while allowing long conversations to page backwards. const messagesWithToolImages = enrichWithToolResultFiles(rawMessages); - const filteredMessages = messagesWithToolImages.filter((msg) => !isToolResultRole(msg.role) && !isInternalMessage(msg)); + const messagesWithToolAttachments = enrichWithToolCallAttachments(messagesWithToolImages); + const filteredMessages = messagesWithToolAttachments.filter((msg) => !shouldDropMessageFromHistory(msg)); const enrichedMessages = enrichWithCachedImages(filteredMessages); set({ messages: enrichedMessages, diff --git a/src/stores/chat/helpers.ts b/src/stores/chat/helpers.ts index 2c3a0aa..296b4a3 100644 --- a/src/stores/chat/helpers.ts +++ b/src/stores/chat/helpers.ts @@ -1,4 +1,9 @@ import { invokeIpc } from '@/lib/api-client'; +import { + isGeneratingStatusNarration, + isInternalAssistantReplyText, + isOpenClawRuntimeEventPrompt, +} from '@/pages/Chat/message-utils'; import type { AttachedFileMeta, ChatSession, ContentBlock, RawMessage, ToolStatus } from './types'; // Module-level timestamp tracking the last chat event received. @@ -186,8 +191,25 @@ function normalizeStreamingMessage(message: unknown): unknown { * `[media attached:` instead — leaving the timestamp in the normalized * comparison text and breaking optimistic-vs-echo dedupe. */ +function stripInboundMediaVisionEnvelope(text: string): string { + if (!/\[Image\]/i.test(text) && !/^User text:/im.test(text) && !/\nDescription:\s*\n/i.test(text)) { + return text; + } + + let result = text.replace(/^\s*\[Image\]\s*\n?/i, ''); + + const userTextBlock = result.match(/^User text:\s*\n([\s\S]*?)(?:\n\s*Description:\s*\n[\s\S]*)?\s*$/i); + if (userTextBlock) { + const userText = userTextBlock[1].trim(); + return /^Process the attached file\(s\)\.\s*$/i.test(userText) ? '' : userText; + } + + return result.replace(/\n\s*Description:\s*\n[\s\S]*$/i, '').trim(); +} + function stripGatewayUserMetadata(text: string): string { - return text + return stripInboundMediaVisionEnvelope( + text .replace(/\s*\[media attached:[^\]]*\]/g, '') .replace(/\s*\[message_id:\s*[^\]]+\]/g, '') .replace(/^Sender\s*\([^)]*\)\s*:\s*```[a-z]*\n[\s\S]*?```\s*/i, '') @@ -198,13 +220,16 @@ function stripGatewayUserMetadata(text: string): string { .replace(/^Sender\s*:\s*[^\n]*(?:\n\s*)*/i, '') .replace(/^Conversation info\s*\([^)]*\):\s*```[a-z]*\n[\s\S]*?```\s*/i, '') .replace(/^Conversation info\s*\([^)]*\):\s*\{[\s\S]*?\}\s*/i, '') - .replace(/^\s*\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, ''); + .replace(/^\s*\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, ''), + ); } function normalizeComparableUserText(content: unknown): string { - return stripGatewayUserMetadata(getMessageText(content)) + const text = stripGatewayUserMetadata(getMessageText(content)) .replace(/\s+/g, ' ') .trim(); + if (/^\(file attached\)$/i.test(text)) return ''; + return text; } function getComparableAttachmentSignature(message: Pick): string { @@ -239,6 +264,13 @@ function matchesOptimisticUserMessage( if (sameText && sameAttachments) return true; if (sameText && (!optimisticAttachments || !candidateAttachments) && (timestampMatches || !hasCandidateTimestamp)) return true; if (sameAttachments && (!optimisticText || !candidateText) && (timestampMatches || !hasCandidateTimestamp)) return true; + + const optimisticHadAttachmentsOnly = optimisticAttachments.length > 0 && !optimisticText; + const candidateIsAttachmentEcho = !candidateText + && /\[(?:media attached:|\s*Image\s*\])/i.test(getMessageText(candidate.content)); + if (optimisticHadAttachmentsOnly && candidateIsAttachmentEcho && (timestampMatches || !hasCandidateTimestamp)) { + return true; + } return false; } @@ -399,6 +431,13 @@ function getMessageText(content: unknown): string { return ''; } +function getMessageTextForFilter(msg: { content?: unknown; text?: unknown }): string { + const fromContent = getMessageText(msg.content); + if (fromContent.trim()) return fromContent; + if (typeof msg.text === 'string') return msg.text; + return ''; +} + function getMessageStopReason(message: RawMessage | unknown): string | null { if (!message || typeof message !== 'object') return null; const msg = message as Record; @@ -486,12 +525,131 @@ function mimeFromExtension(filePath: string): string { return map[ext] || 'application/octet-stream'; } +/** Extract local file paths declared in tool call arguments. */ +function extractFilePathsFromToolArgs(args: Record): string[] { + const paths: string[] = []; + const direct = args.file_path ?? args.filePath ?? args.path ?? args.file; + if (typeof direct === 'string' && direct.trim()) paths.push(direct.trim()); + + const attachments = args.attachments; + if (Array.isArray(attachments)) { + for (const item of attachments) { + if (!item || typeof item !== 'object') continue; + const att = item as Record; + const filePath = att.filePath ?? att.file_path ?? att.path ?? att.file; + if (typeof filePath === 'string' && filePath.trim()) { + paths.push(filePath.trim()); + } + } + } + + return paths; +} + +/** + * Surface user-facing attachments declared in assistant tool calls (e.g. + * `message` tool `attachments: [{ filePath }]`) on the calling turn itself. + */ +function enrichWithToolCallAttachments(messages: RawMessage[]): RawMessage[] { + return messages.map((msg) => { + if (msg.role !== 'assistant') return msg; + + const attachmentPaths = new Set(); + const content = msg.content; + if (Array.isArray(content)) { + for (const block of content as ContentBlock[]) { + if (block.type !== 'tool_use' && block.type !== 'toolCall') continue; + const args = (block.input ?? block.arguments) as Record | undefined; + if (!args) continue; + for (const filePath of extractFilePathsFromToolArgs(args)) { + attachmentPaths.add(filePath); + } + } + } + + const msgAny = msg as unknown as Record; + const toolCalls = msgAny.tool_calls ?? msgAny.toolCalls; + if (Array.isArray(toolCalls)) { + for (const tc of toolCalls as Array>) { + const fn = (tc.function ?? tc) as Record; + let args: Record | undefined; + try { + args = typeof fn.arguments === 'string' + ? JSON.parse(fn.arguments) + : (fn.arguments ?? fn.input) as Record; + } catch { /* ignore */ } + if (!args) continue; + for (const filePath of extractFilePathsFromToolArgs(args)) { + attachmentPaths.add(filePath); + } + } + } + + if (attachmentPaths.size === 0) return msg; + + const existingPaths = new Set( + (msg._attachedFiles || []).map((file) => file.filePath).filter(Boolean), + ); + const newFiles = [...attachmentPaths] + .filter((filePath) => !existingPaths.has(filePath)) + .map((filePath) => makeAttachedFile({ filePath, mimeType: mimeFromExtension(filePath) }, 'tool-result')); + + if (newFiles.length === 0) return msg; + return { + ...msg, + _attachedFiles: [...(msg._attachedFiles || []), ...newFiles], + }; + }); +} + const DIRECTORY_MIME_TYPE = 'application/x-directory'; function trimPathTerminators(filePath: string): string { return filePath.replace(/[，。；;,.!?]+$/u, ''); } +type MarkdownImageRef = + | { filePath: string; mimeType: string; fileName: string } + | { gatewayUrl: string; mimeType: string; fileName: string; source: 'gateway-media' }; + +/** Extract image targets from markdown `![alt](target)` in assistant text. */ +function extractMarkdownImageRefs(text: string): MarkdownImageRef[] { + if (!text) return []; + const refs: MarkdownImageRef[] = []; + const seen = new Set(); + const markdownImageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g; + let match: RegExpExecArray | null; + while ((match = markdownImageRegex.exec(text)) !== null) { + const alt = match[1]?.trim() || 'image'; + let target = match[2]?.trim() ?? ''; + if (!target) continue; + if (target.startsWith('file://')) { + target = decodeURIComponent(target.replace(/^file:\/\//, '')); + } + if (target.startsWith('/api/chat/media/')) { + if (seen.has(target)) continue; + seen.add(target); + refs.push({ + gatewayUrl: target, + mimeType: 'image/png', + fileName: alt, + source: 'gateway-media', + }); + continue; + } + const normalizedPath = trimPathTerminators(target); + if (!normalizedPath.startsWith('/') && !normalizedPath.startsWith('~/')) continue; + if (seen.has(normalizedPath)) continue; + seen.add(normalizedPath); + refs.push({ + filePath: normalizedPath, + mimeType: mimeFromExtension(normalizedPath), + fileName: alt, + }); + } + return refs; +} + /** * Extract raw file paths from message text. * Detects absolute paths (Unix: / or ~/, Windows: C:\ etc.) ending with common file extensions. @@ -661,8 +819,8 @@ function getToolCallFilePath(msg: RawMessage, toolCallId: string): string | unde if ((block.type === 'tool_use' || block.type === 'toolCall') && block.id === toolCallId) { const args = (block.input ?? block.arguments) as Record | undefined; if (args) { - const fp = args.file_path ?? args.filePath ?? args.path ?? args.file; - if (typeof fp === 'string') return fp; + const paths = extractFilePathsFromToolArgs(args); + if (paths[0]) return paths[0]; } } } @@ -680,8 +838,8 @@ function getToolCallFilePath(msg: RawMessage, toolCallId: string): string | unde args = typeof fn.arguments === 'string' ? JSON.parse(fn.arguments) : (fn.arguments ?? fn.input) as Record; } catch { /* ignore */ } if (args) { - const fp = args.file_path ?? args.filePath ?? args.path ?? args.file; - if (typeof fp === 'string') return fp; + const paths = extractFilePathsFromToolArgs(args); + if (paths[0]) return paths[0]; } } } @@ -699,8 +857,8 @@ function collectToolCallPaths(msg: RawMessage, paths: Map): void if ((block.type === 'tool_use' || block.type === 'toolCall') && block.id) { const args = (block.input ?? block.arguments) as Record | undefined; if (args) { - const fp = args.file_path ?? args.filePath ?? args.path ?? args.file; - if (typeof fp === 'string') paths.set(block.id, fp); + const filePaths = extractFilePathsFromToolArgs(args); + if (filePaths[0]) paths.set(block.id, filePaths[0]); } } } @@ -717,8 +875,8 @@ function collectToolCallPaths(msg: RawMessage, paths: Map): void args = typeof fn.arguments === 'string' ? JSON.parse(fn.arguments) : (fn.arguments ?? fn.input) as Record; } catch { /* ignore */ } if (args) { - const fp = args.file_path ?? args.filePath ?? args.path ?? args.file; - if (typeof fp === 'string') paths.set(id, fp); + const filePaths = extractFilePathsFromToolArgs(args); + if (filePaths[0]) paths.set(id, filePaths[0]); } } } @@ -744,7 +902,6 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] { } if (isToolResultRole(msg.role)) { - // Resolve file path from the matching tool call const matchedPath = msg.toolCallId ? toolCallPaths.get(msg.toolCallId) : undefined; // 1. Image/file content blocks in the structured content array. @@ -788,6 +945,11 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] { } if (msg.role === 'assistant' && pending.length > 0) { + // Internal-only turns (NO_REPLY, interim narration, ...) must not consume + // pending attachments — the next visible assistant reply should get them. + if (isInternalMessage(msg) && !messageHasToolUse(msg)) { + return msg; + } const toAttach = pending.splice(0); // Deduplicate against files already on the assistant message const existingPaths = new Set( @@ -857,6 +1019,13 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] { if (msg.role === 'assistant' && !isToolOnlyMessage(msg)) { // Own text rawRefs = extractRawFilePaths(text).filter(r => !mediaRefPaths.has(r.filePath)); + const rawPathSet = new Set(rawRefs.map((ref) => ref.filePath)); + for (const ref of extractMarkdownImageRefs(text)) { + if ('filePath' in ref && !mediaRefPaths.has(ref.filePath) && !rawPathSet.has(ref.filePath)) { + rawPathSet.add(ref.filePath); + rawRefs.push({ filePath: ref.filePath, mimeType: ref.mimeType }); + } + } // Nearest preceding user message text (look back up to 5 messages) const seenPaths = new Set(rawRefs.map(r => r.filePath)); @@ -884,7 +1053,16 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] { } const allRefs = [...mediaRefs, ...rawRefs]; - if (allRefs.length === 0 && gatewayMediaFiles.length === 0) return msg; + const markdownImageRefs = msg.role === 'assistant' && !isToolOnlyMessage(msg) + ? extractMarkdownImageRefs(text) + : []; + if ( + allRefs.length === 0 + && gatewayMediaFiles.length === 0 + && markdownImageRefs.length === 0 + ) { + return msg; + } const existingFiles = msg._attachedFiles || []; const existingPaths = new Set(existingFiles.map(file => file.filePath).filter(Boolean)); @@ -902,8 +1080,19 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] { const dedupedGatewayMedia = gatewayMediaFiles.filter( file => file.gatewayUrl && !existingGatewayUrls.has(file.gatewayUrl), ); - if (files.length === 0 && dedupedGatewayMedia.length === 0) return msg; - return { ...msg, _attachedFiles: [...existingFiles, ...files, ...dedupedGatewayMedia] }; + const markdownGatewayMedia: AttachedFileMeta[] = markdownImageRefs + .filter((ref): ref is Extract => 'gatewayUrl' in ref) + .filter((ref) => ref.gatewayUrl && !existingGatewayUrls.has(ref.gatewayUrl)) + .map((ref) => ({ + fileName: ref.fileName, + mimeType: ref.mimeType, + fileSize: 0, + preview: null, + gatewayUrl: ref.gatewayUrl, + source: 'gateway-media' as const, + })); + if (files.length === 0 && dedupedGatewayMedia.length === 0 && markdownGatewayMedia.length === 0) return msg; + return { ...msg, _attachedFiles: [...existingFiles, ...files, ...dedupedGatewayMedia, ...markdownGatewayMedia] }; }); } @@ -1077,12 +1266,31 @@ function isToolResultRole(role: unknown): boolean { return normalized === 'toolresult' || normalized === 'tool_result'; } +function messageHasToolUse(msg: { role?: unknown; content?: unknown; tool_calls?: unknown; toolCalls?: unknown }): boolean { + if (msg.role !== 'assistant') return false; + if (Array.isArray(msg.content)) { + const blocks = msg.content as ContentBlock[]; + if (blocks.some((block) => block.type === 'tool_use' || block.type === 'toolCall')) { + return true; + } + } + const toolCalls = msg.tool_calls ?? msg.toolCalls; + return Array.isArray(toolCalls) && toolCalls.length > 0; +} + /** True for internal plumbing messages that should never be shown in the UI. */ -function isInternalMessage(msg: { role?: unknown; content?: unknown }): boolean { +function isInternalMessage(msg: { role?: unknown; content?: unknown; text?: unknown }): boolean { if (msg.role === 'system') return true; - const text = getMessageText(msg.content); + const text = getMessageTextForFilter(msg); if (msg.role === 'assistant') { - if (/^(HEARTBEAT_OK|NO_REPLY)\s*$/.test(text)) return true; + if (isInternalAssistantReplyText(text)) return true; + if (isGeneratingStatusNarration(text)) return true; + if (!text.trim() && Array.isArray(msg.content)) { + const blocks = msg.content as ContentBlock[]; + const hasThinking = blocks.some((block) => block.type === 'thinking' && block.thinking?.trim()); + const hasVisibleText = blocks.some((block) => block.type === 'text' && block.text?.trim()); + if (hasThinking && !hasVisibleText) return true; + } } if (msg.role === 'user' && /^\[OpenClaw heartbeat poll\]\s*$/i.test(text.trim())) return true; // Runtime system injections: these arrive as user or assistant-role messages @@ -1091,6 +1299,17 @@ function isInternalMessage(msg: { role?: unknown; content?: unknown }): boolean return false; } +/** + * History filtering must keep assistant tool-call turns even when their visible + * text is internal narration (e.g. "生成中，稍等" + `image_generate`). Those + * turns power the execution graph and run lifecycle detection. + */ +function shouldDropMessageFromHistory(msg: { role?: unknown; content?: unknown; text?: unknown; tool_calls?: unknown; toolCalls?: unknown }): boolean { + if (isToolResultRole(msg.role)) return true; + if (messageHasToolUse(msg)) return false; + return isInternalMessage(msg); +} + /** * Detect runtime-injected system messages that should be hidden from the chat UI. * These are injected by the OpenClaw runtime as user-role messages and include: @@ -1114,8 +1333,11 @@ function isRuntimeSystemInjection(text: string): boolean { return true; } - // Standalone time injection (e.g. "Current time: Wednesday, April 22nd, 2026 - 10:06 (Asia/Shanghai) / 2026-04-22 02:06 UTC") - // Only match when the full message is the time announcement. + if (/^\[Inter-session message\]/i.test(normalized)) return true; + + if (isOpenClawRuntimeEventPrompt(normalized)) return true; + + // Standalone time injection if ( /^\s*Current time\s*:/i.test(normalized) && /^\s*Current time\s*:[^\n]*\/\s*\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+UTC\s*$/i.test(normalized) @@ -1439,7 +1661,9 @@ export { extractRawFilePaths, makeAttachedFile, enrichWithToolResultFiles, + enrichWithToolCallAttachments, isInternalMessage, + shouldDropMessageFromHistory, isToolResultRole, enrichWithCachedImages, loadMissingPreviews, diff --git a/src/stores/chat/history-actions.ts b/src/stores/chat/history-actions.ts index e725766..e50ac4a 100644 --- a/src/stores/chat/history-actions.ts +++ b/src/stores/chat/history-actions.ts @@ -3,14 +3,14 @@ import { hostApiFetch } from '@/lib/host-api'; import { useGatewayStore } from '@/stores/gateway'; import { clearHistoryPoll, + enrichWithToolCallAttachments, enrichWithCachedImages, enrichWithToolResultFiles, getLatestOptimisticUserMessage, getMessageErrorMessage, getMessageStopReason, getMessageText, - isInternalMessage, - isToolResultRole, + shouldDropMessageFromHistory, loadMissingPreviews, mergePendingOptimisticUserMessages, dropRedundantOptimisticUserMessages, @@ -112,7 +112,8 @@ export function createHistoryActions( if (!isCurrentSession()) return false; // Before filtering: attach images/files from tool_result messages to the next assistant message const messagesWithToolImages = enrichWithToolResultFiles(rawMessages); - const filteredMessages = messagesWithToolImages.filter((msg) => !isToolResultRole(msg.role) && !isInternalMessage(msg)); + const messagesWithToolAttachments = enrichWithToolCallAttachments(messagesWithToolImages); + const filteredMessages = messagesWithToolAttachments.filter((msg) => !shouldDropMessageFromHistory(msg)); // Restore file attachments for user/assistant messages (from cache + text patterns) const enrichedMessages = enrichWithCachedImages(filteredMessages); diff --git a/tests/e2e/image-generation-settings.spec.ts b/tests/e2e/image-generation-settings.spec.ts index 258467a..9895d0e 100644 --- a/tests/e2e/image-generation-settings.spec.ts +++ b/tests/e2e/image-generation-settings.spec.ts @@ -33,6 +33,7 @@ test.describe('Image generation settings page', () => { await expect(page.getByTestId('image-generation-primary')).toHaveCount(0); await expect(page.getByTestId('image-generation-fallbacks')).toHaveCount(0); await expect(page.getByTestId('image-generation-save')).toBeVisible(); + await expect(page.getByTestId('image-generation-clear')).toBeDisabled(); }); test('configures an independent OpenAI-compatible image endpoint', async ({ page }) => { @@ -45,7 +46,7 @@ test.describe('Image generation settings page', () => { await expect(page.getByTestId('image-generation-settings')).toBeVisible(); await expect(page.getByTestId('image-generation-relay-base-url')).toBeVisible(); - await page.getByTestId('image-generation-relay-base-url').fill('https://taolat.com/v1'); + await page.getByTestId('image-generation-relay-base-url').fill('https://api.example.com/v1'); await page.getByTestId('image-generation-relay-model').fill('gpt-image-2'); await page.getByTestId('image-generation-relay-api-key').fill('sk-test-image'); @@ -81,7 +82,7 @@ test.describe('Image generation settings page', () => { ], openAiRelay: { enabled: true, - baseUrl: 'https://taolat.com/v1', + baseUrl: 'https://api.example.com/v1', model: 'gpt-image-2', providerKey: 'clawx-openai-image', apiKeyConfigured: true, @@ -103,4 +104,79 @@ test.describe('Image generation settings page', () => { await expect(page.getByTestId('image-generation-api-key-status')).not.toBeEmpty(); await expect(page.getByTestId('image-generation-relay-api-key')).toHaveAttribute('placeholder', /.+/); }); + + test('clears configured image generation settings after confirmation', async ({ electronApp, page }) => { + const configuredResponse = { + success: true, + config: { + primary: 'clawx-openai-image/gpt-image-2', + fallbacks: [], + timeoutMs: 180000, + }, + autoProviderFallback: false, + defaultAgentId: 'default', + agents: [ + { + id: 'default', + name: 'Default', + isDefault: true, + provider: null, + configured: false, + }, + ], + openAiRelay: { + enabled: true, + baseUrl: 'https://api.example.com/v1', + model: 'gpt-image-2', + providerKey: 'clawx-openai-image', + apiKeyConfigured: true, + }, + }; + const clearedResponse = { + ...configuredResponse, + config: { + primary: null, + fallbacks: [], + timeoutMs: 180000, + }, + openAiRelay: { + enabled: false, + baseUrl: '', + model: 'gpt-image-2', + providerKey: undefined, + apiKeyConfigured: false, + }, + }; + + await installIpcMocks(electronApp, { + hostApi: { + '["/api/media/image-generation","GET"]': { + ok: true, + data: { status: 200, ok: true, json: configuredResponse }, + }, + '["/api/media/image-generation","PUT"]': { + ok: true, + data: { status: 200, ok: true, json: { success: true, ...clearedResponse } }, + }, + }, + }); + + await expect(page.getByTestId('setup-page')).toBeVisible(); + await page.getByTestId('setup-skip-button').click(); + + await expect(page.getByTestId('main-layout')).toBeVisible(); + await unlockDeveloperMode(page); + await page.getByTestId('sidebar-nav-image-generation').click(); + + await expect(page.getByTestId('image-generation-relay-base-url')).toHaveValue('https://api.example.com/v1'); + await expect(page.getByTestId('image-generation-clear')).toBeEnabled(); + + await page.getByTestId('image-generation-clear').click(); + const confirmDialog = page.getByRole('dialog'); + await expect(confirmDialog).toBeVisible(); + await confirmDialog.getByRole('button', { name: 'Clear', exact: true }).click(); + + await expect(page.getByTestId('image-generation-relay-base-url')).toHaveValue(''); + await expect(page.getByTestId('image-generation-clear')).toBeDisabled(); + }); }); diff --git a/tests/unit/chat-assistant-media-display.test.ts b/tests/unit/chat-assistant-media-display.test.ts new file mode 100644 index 0000000..c1c7230 --- /dev/null +++ b/tests/unit/chat-assistant-media-display.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest'; +import { extractText } from '@/pages/Chat/message-utils'; + +describe('assistant media path display cleanup', () => { + it('strips bare OpenClaw media paths when the image is shown as an attachment card', () => { + const text = [ + '宇航员图片生成完成啦 🧑‍🚀✨', + '/Users/zhonghaolu/.openclaw/media/tool-image-generation/clawx-image-1---82d6c7e6-ea44-4850-a24b-9e88e1660683.png', + ].join('\n'); + + expect(extractText({ role: 'assistant', content: text })).toBe('宇航员图片生成完成啦 🧑‍🚀✨'); + }); + + it('still strips MEDIA: tagged OpenClaw artifact paths', () => { + const text = 'Done:\n\nMEDIA:/Users/alice/.openclaw/media/outbound/cat---abc.png'; + + expect(extractText({ role: 'assistant', content: text })).toBe('Done:'); + }); + + it('strips markdown image syntax that cannot be rendered directly', () => { + const text = '宇航员图片完成啦 🧑‍🚀✨\n\n![Astronaut with Milky Way in helmet visor](/api/chat/media/outgoing/agent%3Amain%3As-1/abc/full)'; + + expect(extractText({ role: 'assistant', content: text })).toBe('宇航员图片完成啦 🧑‍🚀✨'); + }); +}); diff --git a/tests/unit/chat-copy-image.test.ts b/tests/unit/chat-copy-image.test.ts new file mode 100644 index 0000000..79ad77f --- /dev/null +++ b/tests/unit/chat-copy-image.test.ts @@ -0,0 +1,56 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { copyImageToClipboard } from '@/pages/Chat/copy-image'; + +const readBinaryFileMock = vi.fn(); + +vi.mock('@/lib/api-client', () => ({ + readBinaryFile: (...args: unknown[]) => readBinaryFileMock(...args), +})); + +describe('copyImageToClipboard', () => { + beforeEach(() => { + readBinaryFileMock.mockReset(); + class MockClipboardItem { + constructor(public items: Record) {} + } + Object.assign(globalThis, { ClipboardItem: MockClipboardItem }); + Object.assign(navigator, { + clipboard: { + write: vi.fn(async () => undefined), + writeText: vi.fn(async () => undefined), + }, + }); + }); + + it('copies image bytes from a data URL preview', async () => { + const pngBytes = Uint8Array.from([137, 80, 78, 71]); + const base64 = btoa(String.fromCharCode(...pngBytes)); + const preview = `data:image/png;base64,${base64}`; + + const ok = await copyImageToClipboard({ + preview, + mimeType: 'image/png', + }); + + expect(ok).toBe(true); + expect(navigator.clipboard.write).toHaveBeenCalledTimes(1); + expect(readBinaryFileMock).not.toHaveBeenCalled(); + }); + + it('copies image bytes from a local file path', async () => { + readBinaryFileMock.mockResolvedValueOnce({ + ok: true, + data: Uint8Array.from([1, 2, 3]), + mimeType: 'image/png', + }); + + const ok = await copyImageToClipboard({ + filePath: '/tmp/cat.png', + mimeType: 'image/png', + }); + + expect(ok).toBe(true); + expect(readBinaryFileMock).toHaveBeenCalledWith('/tmp/cat.png'); + expect(navigator.clipboard.write).toHaveBeenCalledTimes(1); + }); +}); diff --git a/tests/unit/chat-helpers-enrichment.test.ts b/tests/unit/chat-helpers-enrichment.test.ts index 0e31e32..42767ee 100644 --- a/tests/unit/chat-helpers-enrichment.test.ts +++ b/tests/unit/chat-helpers-enrichment.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; import { enrichWithToolResultFiles, + enrichWithToolCallAttachments, enrichWithCachedImages, } from '@/stores/chat/helpers'; import type { RawMessage } from '@/stores/chat'; @@ -137,6 +138,38 @@ describe('enrichWithToolResultFiles', () => { const paths = (reply._attachedFiles ?? []).map((f) => f.filePath); expect(paths).toContain('/tmp/foo.pdf'); }); + + it('skips internal NO_REPLY turns when attaching pending tool-result files', () => { + const messages: RawMessage[] = [ + { + role: 'assistant', + id: 'a1', + content: [{ type: 'toolCall', id: 'tc1', name: 'exec', input: { command: 'echo report > /tmp/report.pdf' } }], + }, + { + role: 'toolresult', + id: 't1', + toolCallId: 'tc1', + toolName: 'exec', + content: [{ type: 'text', text: 'Wrote /tmp/report.pdf' }], + }, + { + role: 'assistant', + id: 'no-reply', + content: [{ type: 'text', text: 'NO_REPLY' }], + }, + { + role: 'assistant', + id: 'final', + content: [{ type: 'text', text: 'Report is ready.' }], + }, + ]; + + const enriched = enrichWithToolResultFiles(messages); + const final = enriched.find((m) => m.id === 'final')!; + expect(final._attachedFiles?.map((file) => file.filePath)).toEqual(['/tmp/report.pdf']); + expect(enriched.find((m) => m.id === 'no-reply')?._attachedFiles ?? []).toEqual([]); + }); }); describe('enrichWithCachedImages — Gateway media bubble dedup', () => { @@ -273,4 +306,68 @@ describe('enrichWithCachedImages — Gateway media bubble dedup', () => { const replyPaths = (reply._attachedFiles ?? []).map((f) => f.filePath); expect(replyPaths).toEqual(['/tmp/foo.png']); }); + + it('promotes markdown local image paths to attached files', () => { + const messages: RawMessage[] = [ + { + role: 'assistant', + id: 'reply', + content: [{ + type: 'text', + text: '宇航员图片完成啦 🧑‍🚀✨\n\n![Astronaut with Milky Way in helmet visor](/Users/me/.openclaw/media/tool-image-generation/cat.png)', + }], + }, + ]; + + const enriched = enrichWithCachedImages(messages); + expect(enriched[0]?._attachedFiles?.map((file) => file.filePath)).toEqual([ + '/Users/me/.openclaw/media/tool-image-generation/cat.png', + ]); + }); + + it('promotes markdown gateway image URLs to gateway-media attachments', () => { + const messages: RawMessage[] = [ + { + role: 'assistant', + id: 'reply', + content: [{ + type: 'text', + text: 'Done\n\n![Astronaut with Milky Way in helmet visor](/api/chat/media/outgoing/agent%3Amain%3As-1/abc/full)', + }], + }, + ]; + + const enriched = enrichWithCachedImages(messages); + expect(enriched[0]?._attachedFiles?.[0]).toMatchObject({ + gatewayUrl: '/api/chat/media/outgoing/agent%3Amain%3As-1/abc/full', + source: 'gateway-media', + fileName: 'Astronaut with Milky Way in helmet visor', + }); + }); +}); + +describe('enrichWithToolCallAttachments', () => { + it('attaches image paths from message tool attachments array', () => { + const messages: RawMessage[] = [ + { + role: 'assistant', + id: 'send-image', + content: [{ + type: 'tool_use', + id: 'tool-1', + name: 'message', + input: { + action: 'send', + attachments: [{ filePath: '/Users/me/.openclaw/media/tool-image-generation/cat.png' }], + }, + }], + }, + ]; + + const enriched = enrichWithToolCallAttachments(messages); + expect(enriched[0]?._attachedFiles?.map((file) => file.filePath)).toEqual([ + '/Users/me/.openclaw/media/tool-image-generation/cat.png', + ]); + expect(enriched[0]?._attachedFiles?.[0]?.mimeType).toBe('image/png'); + }); }); diff --git a/tests/unit/chat-history-actions.test.ts b/tests/unit/chat-history-actions.test.ts index 5e5e739..e60464d 100644 --- a/tests/unit/chat-history-actions.test.ts +++ b/tests/unit/chat-history-actions.test.ts @@ -7,6 +7,7 @@ const gatewayStoreGetStateMock = vi.fn(); const clearHistoryPoll = vi.fn(); const enrichWithCachedImages = vi.fn((messages) => messages); const enrichWithToolResultFiles = vi.fn((messages) => messages); +const enrichWithToolCallAttachments = vi.fn((messages) => messages); const getMessageErrorMessage = vi.fn((message: { errorMessage?: string; error_message?: string } | undefined) => { if (!message) return null; return message.errorMessage ?? message.error_message ?? null; @@ -29,6 +30,23 @@ const isInternalMessage = vi.fn((msg: { role?: unknown; content?: unknown }) => } return false; }); +const messageHasToolUse = (msg: { role?: unknown; content?: unknown; tool_calls?: unknown; toolCalls?: unknown }) => { + if (msg.role !== 'assistant') return false; + if (Array.isArray(msg.content)) { + return (msg.content as Array<{ type?: string }>).some( + (block) => block.type === 'tool_use' || block.type === 'toolCall', + ); + } + const toolCalls = (msg as { tool_calls?: unknown; toolCalls?: unknown }).tool_calls + ?? (msg as { toolCalls?: unknown }).toolCalls; + return Array.isArray(toolCalls) && toolCalls.length > 0; +}; +const shouldDropMessageFromHistory = vi.fn((msg: { role?: unknown; content?: unknown; tool_calls?: unknown; toolCalls?: unknown }) => { + if (isToolResultRole(msg.role)) return true; + if (messageHasToolUse(msg)) return false; + return isInternalMessage(msg); +}); +const setLastChatEventAt = vi.fn(); const loadMissingPreviews = vi.fn(async () => false); const toMs = vi.fn((ts: number) => ts < 1e12 ? ts * 1000 : ts); @@ -50,6 +68,7 @@ vi.mock('@/stores/chat/helpers', () => ({ clearHistoryPoll: (...args: unknown[]) => clearHistoryPoll(...args), enrichWithCachedImages: (...args: unknown[]) => enrichWithCachedImages(...args), enrichWithToolResultFiles: (...args: unknown[]) => enrichWithToolResultFiles(...args), + enrichWithToolCallAttachments: (...args: unknown[]) => enrichWithToolCallAttachments(...args), getLatestOptimisticUserMessage: (messages: Array<{ role: string; timestamp?: number }>, userTimestampMs: number) => [...messages].reverse().find( (message) => message.role === 'user' @@ -57,9 +76,13 @@ vi.mock('@/stores/chat/helpers', () => ({ ), getMessageText: (...args: unknown[]) => getMessageText(...args), hasNonToolAssistantContent: (...args: unknown[]) => hasNonToolAssistantContent(...args), + hasAssistantAfterLastRealUser: (messages: Array<{ role?: string }>) => + messages.some((message) => message.role === 'assistant'), isInternalMessage: (...args: unknown[]) => isInternalMessage(...args), + shouldDropMessageFromHistory: (...args: unknown[]) => shouldDropMessageFromHistory(...args), isToolResultRole: (...args: unknown[]) => isToolResultRole(...args), loadMissingPreviews: (...args: unknown[]) => loadMissingPreviews(...args), + setLastChatEventAt: (...args: unknown[]) => setLastChatEventAt(...args), mergePendingOptimisticUserMessages: (_sessionKey: string, messages: unknown[]) => messages, hasOptimisticServerEcho: ( loadedMessages: Array<{ role: string; timestamp?: number; content?: unknown; _attachedFiles?: Array<{ filePath?: string; fileName?: string; mimeType?: string; fileSize?: number }> }>, diff --git a/tests/unit/chat-internal-message-filter.test.ts b/tests/unit/chat-internal-message-filter.test.ts index 8eeb1d7..cfb14ae 100644 --- a/tests/unit/chat-internal-message-filter.test.ts +++ b/tests/unit/chat-internal-message-filter.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { isInternalMessage } from '@/stores/chat/helpers'; +import { isInternalMessage, shouldDropMessageFromHistory } from '@/stores/chat/helpers'; describe('chat internal message filter', () => { it('filters runtime system injection bundle like async exec completion payload', () => { @@ -33,4 +33,54 @@ describe('chat internal message filter', () => { it('filters OpenClaw heartbeat poll user turns', () => { expect(isInternalMessage({ role: 'user', content: '[OpenClaw heartbeat poll]' })).toBe(true); }); + + it('filters inter-session routing payloads', () => { + expect(isInternalMessage({ + role: 'user', + content: '[Inter-session message] sourceSession=image_generate:abc sourceTool=image_generate', + })).toBe(true); + }); + + it('filters OpenClaw runtime continuation prompts', () => { + expect(isInternalMessage({ role: 'user', content: 'Continue the OpenClaw runtime event.' })).toBe(true); + }); + + it('filters runtime continuation prompts stored on msg.text', () => { + expect(isInternalMessage({ + role: 'user', + content: [], + text: 'Continue the OpenClaw runtime event.', + })).toBe(true); + }); + + it('filters interim image-generation status narration', () => { + expect(isInternalMessage({ + role: 'assistant', + content: '生成中，稍等 👨‍🚀', + })).toBe(true); + }); + + it('keeps assistant tool-call turns in history even when narration is internal', () => { + expect(shouldDropMessageFromHistory({ + role: 'assistant', + content: [ + { type: 'text', text: '生成中，稍等 👨‍🚀' }, + { type: 'tool_use', id: 'tool-image', name: 'image_generate', input: { prompt: 'astronaut' } }, + ], + })).toBe(false); + expect(isInternalMessage({ + role: 'assistant', + content: [ + { type: 'text', text: '生成中，稍等 👨‍🚀' }, + { type: 'tool_use', id: 'tool-image', name: 'image_generate', input: { prompt: 'astronaut' } }, + ], + })).toBe(true); + }); + + it('filters assistant turns that only contain chain-of-thought', () => { + expect(isInternalMessage({ + role: 'assistant', + content: [{ type: 'thinking', thinking: 'The user is asking me to continue with an OpenClaw runtime event.' }], + })).toBe(true); + }); }); diff --git a/tests/unit/chat-message.test.tsx b/tests/unit/chat-message.test.tsx index 2cc1db5..e225b0f 100644 --- a/tests/unit/chat-message.test.tsx +++ b/tests/unit/chat-message.test.tsx @@ -5,6 +5,7 @@ import type { RawMessage } from '@/stores/chat'; vi.mock('@/lib/api-client', () => ({ invokeIpc: vi.fn(), + readBinaryFile: vi.fn(), statFile: vi.fn(async (path: string) => { if (path.includes('missing') || path.includes('不存在')) { return { ok: false, error: 'notFound' }; @@ -405,3 +406,50 @@ describe('ChatMessage reply styling', () => { expect(bubble).toHaveTextContent('Keep the prompt bubble.'); }); }); + +describe('ChatMessage image copy', () => { + beforeEach(() => { + class MockClipboardItem { + constructor(public items: Record) {} + } + Object.assign(globalThis, { ClipboardItem: MockClipboardItem }); + Object.assign(navigator, { + clipboard: { + write: vi.fn(async () => undefined), + writeText: vi.fn(async () => undefined), + }, + }); + }); + + it('copies image bytes instead of the media URL text when an image attachment is present', async () => { + const { readBinaryFile } = await import('@/lib/api-client'); + vi.mocked(readBinaryFile).mockResolvedValueOnce({ + ok: true, + data: Uint8Array.from([137, 80, 78, 71]), + mimeType: 'image/png', + }); + + const message: RawMessage = { + role: 'assistant', + content: 'http://127.0.0.1:18789/api/chat/media/outgoing/agent/main/full', + _attachedFiles: [ + { + fileName: 'cat.png', + mimeType: 'image/png', + fileSize: 1234, + preview: null, + filePath: '/tmp/cat.png', + source: 'tool-result', + }, + ], + }; + + render(); + + fireEvent.click(screen.getByRole('button')); + await vi.waitFor(() => { + expect(navigator.clipboard.write).toHaveBeenCalledTimes(1); + }); + expect(navigator.clipboard.writeText).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/chat-user-message-display.test.ts b/tests/unit/chat-user-message-display.test.ts new file mode 100644 index 0000000..c50ea9d --- /dev/null +++ b/tests/unit/chat-user-message-display.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; +import { extractText } from '@/pages/Chat/message-utils'; +import { matchesOptimisticUserMessage } from '@/stores/chat/helpers'; + +const gatewayImageEcho = [ + 'Sender (untrusted metadata):', + '```json', + '{', + ' "label": "ClawX (gateway-client)",', + ' "id": "gateway-client",', + ' "name": "ClawX",', + ' "username": "ClawX"', + '}', + '```', + '', + '[media attached: media://inbound/image---abc.png (image/png)]', + '[Image]', + 'User text:', + 'Process the attached file(s).', + '[media attached: /Users/test/.openclaw/media/outbound/out.png (image/png) | /Users/test/.openclaw/media/outbound/out.png]', + 'Description:', + 'An astronaut in a white space suit floats in space, reaching a gloved hand toward the viewer.', +].join('\n'); + +describe('user message display cleanup', () => { + it('hides the inbound-image vision envelope for attachment-only uploads', () => { + expect(extractText({ role: 'user', content: gatewayImageEcho })).toBe(''); + }); + + it('keeps the user caption while stripping auto-generated description', () => { + const content = gatewayImageEcho + .replace('Process the attached file(s).', '改成西装加领带'); + + expect(extractText({ role: 'user', content })).toBe('改成西装加领带'); + }); + + it('matches optimistic attachment-only bubbles against the gateway vision echo', () => { + const optimistic = { + role: 'user' as const, + content: '(file attached)', + timestamp: 1_700_000_000, + _attachedFiles: [{ + fileName: 'out.png', + mimeType: 'image/png', + fileSize: 123, + preview: null, + filePath: '/Users/test/.openclaw/media/outbound/out.png', + }], + }; + const candidate = { + role: 'user' as const, + content: gatewayImageEcho, + timestamp: 1_700_000_000, + }; + + expect(matchesOptimisticUserMessage(candidate, optimistic, 1_700_000_000_000)).toBe(true); + }); +}); diff --git a/tests/unit/image-generation-status.test.ts b/tests/unit/image-generation-status.test.ts new file mode 100644 index 0000000..65163e7 --- /dev/null +++ b/tests/unit/image-generation-status.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from 'vitest'; +import type { RawMessage } from '@/stores/chat'; +import { + IMAGE_GENERATION_TIMEOUT_MS, + isImageGenerationPending, +} from '@/pages/Chat/image-generation-status'; + +const TASK_ID = '27443fdb-6cca-48e6-a3a7-ee34b0491aee'; +const NOW = 1_700_000_000_000; + +describe('isImageGenerationPending', () => { + it('returns true while waiting after image_generate without a final reply', () => { + const segmentMessages: RawMessage[] = [ + { + role: 'assistant', + timestamp: NOW / 1000, + content: [{ type: 'toolCall', id: 'call_1', name: 'image_generate', arguments: { prompt: 'astronaut' } }], + }, + { + role: 'assistant', + timestamp: NOW / 1000 + 5, + content: [{ type: 'text', text: '图片生成中，稍等片刻 🧑‍🚀✨' }], + }, + ]; + + expect(isImageGenerationPending(segmentMessages, [], NOW)).toBe(true); + }); + + it('returns false after the inter-session completion event arrives', () => { + const segmentMessages: RawMessage[] = [ + { + role: 'toolresult', + toolName: 'image_generate', + timestamp: NOW / 1000, + content: [{ + type: 'text', + text: `Background task started for image generation (${TASK_ID}).`, + }], + }, + { + role: 'user', + content: [{ + type: 'text', + text: `[Inter-session message] sourceSession=image_generate:${TASK_ID} sourceTool=image_generate`, + }], + }, + ]; + + expect(isImageGenerationPending(segmentMessages, [], NOW)).toBe(false); + }); + + it('returns false once a non-generating assistant reply arrives', () => { + const segmentMessages: RawMessage[] = [ + { + role: 'assistant', + timestamp: NOW / 1000, + content: [{ type: 'toolCall', id: 'call_1', name: 'image_generate', arguments: { prompt: 'astronaut' } }], + }, + { + role: 'assistant', + timestamp: NOW / 1000 + 5, + content: [{ type: 'text', text: '图片生成中，稍等片刻 🧑‍🚀✨' }], + }, + { + role: 'assistant', + timestamp: NOW / 1000 + 120, + content: [{ type: 'text', text: '抱歉，图片生成超时了，稍后重试一下吧 🧑‍🚀' }], + }, + ]; + + expect(isImageGenerationPending(segmentMessages, [], NOW + 120_000)).toBe(false); + }); + + it('returns false after the configured timeout even if only generating narration exists', () => { + const segmentMessages: RawMessage[] = [ + { + role: 'assistant', + timestamp: NOW / 1000, + content: [{ type: 'toolCall', id: 'call_1', name: 'image_generate', arguments: { prompt: 'astronaut' } }], + }, + { + role: 'assistant', + timestamp: NOW / 1000 + 5, + content: [{ type: 'text', text: '图片生成中，稍等片刻 🧑‍🚀✨' }], + }, + ]; + + expect( + isImageGenerationPending( + segmentMessages, + [], + NOW + IMAGE_GENERATION_TIMEOUT_MS + 20_000, + ), + ).toBe(false); + }); + + it('returns false when a stale streaming tool status is still marked running', () => { + expect(isImageGenerationPending([], [{ + name: 'image_generate', + status: 'running', + updatedAt: NOW - IMAGE_GENERATION_TIMEOUT_MS - 60_000, + }], NOW)).toBe(false); + }); + + it('returns true while image_generate is actively running in streaming tools', () => { + expect(isImageGenerationPending([], [{ + name: 'image_generate', + status: 'running', + updatedAt: NOW, + }], NOW)).toBe(true); + }); + + it('returns false once an image attachment is delivered', () => { + const segmentMessages: RawMessage[] = [ + { + role: 'assistant', + timestamp: NOW / 1000, + content: [{ type: 'toolCall', id: 'call_1', name: 'image_generate', arguments: { prompt: 'astronaut' } }], + }, + { + role: 'assistant', + timestamp: NOW / 1000 + 90, + content: [{ type: 'text', text: '宇航员图片完成啦 🧑‍🚀✨' }], + _attachedFiles: [{ + fileName: 'astronaut.png', + mimeType: 'image/png', + fileSize: 123, + preview: null, + filePath: '/tmp/astronaut.png', + }], + }, + ]; + + expect(isImageGenerationPending(segmentMessages, [], NOW + 90_000)).toBe(false); + }); +}); diff --git a/tests/unit/task-visualization.test.ts b/tests/unit/task-visualization.test.ts index 8032dad..3680cc5 100644 --- a/tests/unit/task-visualization.test.ts +++ b/tests/unit/task-visualization.test.ts @@ -4,7 +4,7 @@ import { stripProcessMessagePrefix } from '@/pages/Chat/message-utils'; import type { RawMessage, ToolStatus } from '@/stores/chat'; describe('deriveTaskSteps', () => { - it('builds running steps from streaming thinking and tool status', () => { + it('builds running steps from streaming tool status without exposing chain-of-thought', () => { const streamingTools: ToolStatus[] = [ { name: 'web_search', @@ -27,12 +27,6 @@ describe('deriveTaskSteps', () => { }); expect(steps).toEqual([ - expect.objectContaining({ - id: 'stream-thinking-0', - label: 'Thinking', - status: 'running', - kind: 'thinking', - }), expect.objectContaining({ label: 'web_search', status: 'running', @@ -160,7 +154,7 @@ describe('deriveTaskSteps', () => { })); }); - it('keeps recent completed steps from assistant history', () => { + it('keeps recent completed tool steps from assistant history', () => { const messages: RawMessage[] = [ { role: 'assistant', @@ -179,12 +173,6 @@ describe('deriveTaskSteps', () => { }); expect(steps).toEqual([ - expect.objectContaining({ - id: 'history-thinking-assistant-1-0', - label: 'Thinking', - status: 'completed', - kind: 'thinking', - }), expect.objectContaining({ id: 'tool-2', label: 'read_file', @@ -194,7 +182,7 @@ describe('deriveTaskSteps', () => { ]); }); - it('splits cumulative streaming thinking into separate execution steps', () => { + it('does not expose streaming chain-of-thought in the execution graph', () => { const steps = deriveTaskSteps({ messages: [], streamingMessage: { @@ -208,21 +196,36 @@ describe('deriveTaskSteps', () => { streamingTools: [], }); + expect(steps).toEqual([]); + }); + + it('skips internal assistant turns and hides NO_REPLY from the execution graph', () => { + const steps = deriveTaskSteps({ + messages: [ + { + role: 'assistant', + content: [{ type: 'thinking', thinking: 'Continue the OpenClaw runtime event internally.' }], + }, + { + role: 'assistant', + content: [{ type: 'text', text: 'NO_REPLY' }], + }, + { + role: 'assistant', + content: [ + { type: 'tool_use', id: 'tool-image', name: 'image_generate', input: { prompt: 'astronaut' } }, + ], + }, + ], + streamingMessage: null, + streamingTools: [], + }); + expect(steps).toEqual([ expect.objectContaining({ - id: 'stream-thinking-0', - detail: 'Reviewing X.', - status: 'completed', - }), - expect.objectContaining({ - id: 'stream-thinking-1', - detail: 'Comparing Y.', - status: 'completed', - }), - expect.objectContaining({ - id: 'stream-thinking-2', - detail: 'Drafting answer.', - status: 'running', + id: 'tool-image', + label: 'image_generate', + kind: 'tool', }), ]); });