Fix image generate (#1068)
This commit is contained in:
@@ -1773,6 +1773,7 @@ export async function syncOpenAiCompatibleImageRelay(params: {
|
||||
}
|
||||
removePluginRegistrations(config, [CLAWX_OPENAI_IMAGE_PROVIDER_KEY]);
|
||||
await writeOpenClawJson(config);
|
||||
await removeProviderKeyFromOpenClaw(CLAWX_OPENAI_IMAGE_PROVIDER_KEY);
|
||||
if (params.apiKey?.trim()) {
|
||||
await saveProviderKeyToOpenClaw(CLAWX_OPENAI_IMAGE_PROVIDER_KEY, params.apiKey.trim());
|
||||
}
|
||||
|
||||
@@ -47,25 +47,30 @@ type ModelInputModule = {
|
||||
resolveAgentModelPrimaryValue: (model?: unknown) => string | undefined;
|
||||
};
|
||||
|
||||
const OPENCLAW_IMAGE_GENERATION_RUNTIME = 'openclaw/plugin-sdk/image-generation-runtime';
|
||||
const OPENCLAW_MEDIA_STORE = 'openclaw/plugin-sdk/media-store';
|
||||
const OPENCLAW_MEDIA_RUNTIME = 'openclaw/plugin-sdk/media-runtime';
|
||||
const OPENCLAW_IMAGE_GENERATION_CORE = 'openclaw/plugin-sdk/image-generation-core';
|
||||
|
||||
let imageRuntimeModule: ImageGenerationRuntimeModule | null = null;
|
||||
let mediaStoreModule: MediaStoreModule | null = null;
|
||||
let imageOpsModule: ImageOpsModule | null = null;
|
||||
let modelInputModule: ModelInputModule | null = null;
|
||||
|
||||
async function importRuntimeModule<T>(relativePath: string): Promise<T> {
|
||||
const modulePath = resolveOpenClawRuntimeModulePath(relativePath);
|
||||
async function importOpenClawSdkModule<T>(specifier: string): Promise<T> {
|
||||
const modulePath = resolveOpenClawRuntimeModulePath(specifier);
|
||||
return import(pathToFileURL(modulePath).href) as Promise<T>;
|
||||
}
|
||||
|
||||
async function getImageGenerationRuntime(): Promise<ImageGenerationRuntimeModule> {
|
||||
if (!imageRuntimeModule) {
|
||||
const mod = await importRuntimeModule<{
|
||||
t: ImageGenerationRuntimeModule['generateImage'];
|
||||
n: ImageGenerationRuntimeModule['listRuntimeImageGenerationProviders'];
|
||||
}>('./dist/runtime-Oyct10ZH.js');
|
||||
const mod = await importOpenClawSdkModule<{
|
||||
generateImage: ImageGenerationRuntimeModule['generateImage'];
|
||||
listRuntimeImageGenerationProviders: ImageGenerationRuntimeModule['listRuntimeImageGenerationProviders'];
|
||||
}>(OPENCLAW_IMAGE_GENERATION_RUNTIME);
|
||||
imageRuntimeModule = {
|
||||
generateImage: mod.t,
|
||||
listRuntimeImageGenerationProviders: mod.n,
|
||||
generateImage: mod.generateImage,
|
||||
listRuntimeImageGenerationProviders: mod.listRuntimeImageGenerationProviders,
|
||||
};
|
||||
}
|
||||
return imageRuntimeModule;
|
||||
@@ -73,20 +78,20 @@ async function getImageGenerationRuntime(): Promise<ImageGenerationRuntimeModule
|
||||
|
||||
async function getMediaStore(): Promise<MediaStoreModule> {
|
||||
if (!mediaStoreModule) {
|
||||
const mod = await importRuntimeModule<{ u: MediaStoreModule['saveMediaBuffer'] }>(
|
||||
'./dist/store-b792nN7l.js',
|
||||
const mod = await importOpenClawSdkModule<{ saveMediaBuffer: MediaStoreModule['saveMediaBuffer'] }>(
|
||||
OPENCLAW_MEDIA_STORE,
|
||||
);
|
||||
mediaStoreModule = { saveMediaBuffer: mod.u };
|
||||
mediaStoreModule = { saveMediaBuffer: mod.saveMediaBuffer };
|
||||
}
|
||||
return mediaStoreModule;
|
||||
}
|
||||
|
||||
async function getImageOps(): Promise<ImageOpsModule> {
|
||||
if (!imageOpsModule) {
|
||||
const mod = await importRuntimeModule<{ a: ImageOpsModule['getImageMetadata'] }>(
|
||||
'./dist/image-ops-CL9ZQP7k.js',
|
||||
const mod = await importOpenClawSdkModule<{ getImageMetadata: ImageOpsModule['getImageMetadata'] }>(
|
||||
OPENCLAW_MEDIA_RUNTIME,
|
||||
);
|
||||
imageOpsModule = { getImageMetadata: mod.a };
|
||||
imageOpsModule = { getImageMetadata: mod.getImageMetadata };
|
||||
}
|
||||
return imageOpsModule;
|
||||
}
|
||||
@@ -100,10 +105,10 @@ export async function resolveImageGenerationPrimaryFromConfig(
|
||||
|
||||
async function getModelInputHelpers(): Promise<ModelInputModule> {
|
||||
if (!modelInputModule) {
|
||||
const mod = await importRuntimeModule<{ i: ModelInputModule['resolveAgentModelPrimaryValue'] }>(
|
||||
'./dist/model-input-B9p-bobB.js',
|
||||
const mod = await importOpenClawSdkModule<{ resolveAgentModelPrimaryValue: ModelInputModule['resolveAgentModelPrimaryValue'] }>(
|
||||
OPENCLAW_IMAGE_GENERATION_CORE,
|
||||
);
|
||||
modelInputModule = { resolveAgentModelPrimaryValue: mod.i };
|
||||
modelInputModule = { resolveAgentModelPrimaryValue: mod.resolveAgentModelPrimaryValue };
|
||||
}
|
||||
return modelInputModule;
|
||||
}
|
||||
|
||||
@@ -2,14 +2,16 @@
|
||||
* Global image generation settings (agents.defaults.imageGenerationModel).
|
||||
*/
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import { Eye, EyeOff, ImagePlus, Loader2, Play, RefreshCw } from 'lucide-react';
|
||||
import { Eye, EyeOff, ImagePlus, Loader2, Play, RefreshCw, Trash2 } from 'lucide-react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { toast } from 'sonner';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { Label } from '@/components/ui/label';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { ConfirmDialog } from '@/components/ui/confirm-dialog';
|
||||
import {
|
||||
clearImageGenerationSettings,
|
||||
fetchImageGenerationSettings,
|
||||
runImageGenerationTest,
|
||||
saveImageGenerationSettings,
|
||||
@@ -32,10 +34,12 @@ function extractTestOutputPath(result: unknown): string | null {
|
||||
}
|
||||
|
||||
export function ImageGenerationSettings() {
|
||||
const { t } = useTranslation(['dashboard', 'settings']);
|
||||
const { t } = useTranslation(['dashboard', 'settings', 'common']);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [clearing, setClearing] = useState(false);
|
||||
const [testing, setTesting] = useState(false);
|
||||
const [clearConfirmOpen, setClearConfirmOpen] = useState(false);
|
||||
const [snapshot, setSnapshot] = useState<ImageGenerationSettingsSnapshot | null>(null);
|
||||
|
||||
const [timeoutMs, setTimeoutMs] = useState('180000');
|
||||
@@ -79,6 +83,15 @@ export function ImageGenerationSettings() {
|
||||
);
|
||||
}, [snapshot, timeoutMs, relayBaseUrl, relayModel, relayApiKey]);
|
||||
|
||||
const hasConfiguredRelay = useMemo(() => {
|
||||
if (!snapshot) return false;
|
||||
return Boolean(
|
||||
snapshot.openAiRelay?.enabled
|
||||
|| snapshot.openAiRelay?.baseUrl?.trim()
|
||||
|| snapshot.config.primary?.trim(),
|
||||
);
|
||||
}, [snapshot]);
|
||||
|
||||
const handleSave = async () => {
|
||||
setSaving(true);
|
||||
try {
|
||||
@@ -120,11 +133,32 @@ export function ImageGenerationSettings() {
|
||||
}
|
||||
};
|
||||
|
||||
const handleClear = async () => {
|
||||
setClearing(true);
|
||||
try {
|
||||
const next = await clearImageGenerationSettings();
|
||||
setSnapshot(next);
|
||||
setRelayBaseUrl('');
|
||||
setRelayModel('gpt-image-2');
|
||||
setRelayApiKey('');
|
||||
setShowRelayApiKey(false);
|
||||
setClearConfirmOpen(false);
|
||||
toast.success(t('imageGeneration.toast.cleared'));
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : String(error));
|
||||
} finally {
|
||||
setClearing(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleTest = async () => {
|
||||
if (dirty) {
|
||||
toast.message(t('imageGeneration.toast.saveBeforeTest'));
|
||||
return;
|
||||
}
|
||||
if (!hasConfiguredRelay) {
|
||||
return;
|
||||
}
|
||||
setTesting(true);
|
||||
try {
|
||||
const result = await runImageGenerationTest({
|
||||
@@ -368,7 +402,7 @@ export function ImageGenerationSettings() {
|
||||
variant="outline"
|
||||
className="rounded-full h-10"
|
||||
onClick={() => void handleTest()}
|
||||
disabled={testing || !relayModel.trim()}
|
||||
disabled={testing || !hasConfiguredRelay || dirty}
|
||||
data-testid="image-generation-test-button"
|
||||
>
|
||||
{testing ? (
|
||||
@@ -387,9 +421,34 @@ export function ImageGenerationSettings() {
|
||||
{saving ? <Loader2 className="h-4 w-4 mr-2 animate-spin" /> : null}
|
||||
{saving ? t('imageGeneration.saving') : t('imageGeneration.save')}
|
||||
</Button>
|
||||
<Button
|
||||
variant="outline"
|
||||
className="rounded-full h-10 text-destructive hover:text-destructive"
|
||||
onClick={() => setClearConfirmOpen(true)}
|
||||
disabled={clearing || !hasConfiguredRelay}
|
||||
data-testid="image-generation-clear"
|
||||
>
|
||||
{clearing ? (
|
||||
<Loader2 className="h-4 w-4 mr-2 animate-spin" />
|
||||
) : (
|
||||
<Trash2 className="h-4 w-4 mr-2" />
|
||||
)}
|
||||
{clearing ? t('imageGeneration.clearing') : t('imageGeneration.clear')}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<ConfirmDialog
|
||||
open={clearConfirmOpen}
|
||||
title={t('imageGeneration.clearConfirmTitle')}
|
||||
message={t('imageGeneration.clearConfirmMessage')}
|
||||
confirmLabel={t('imageGeneration.clearConfirmAction')}
|
||||
cancelLabel={t('common:actions.cancel')}
|
||||
variant="destructive"
|
||||
onConfirm={handleClear}
|
||||
onCancel={() => setClearConfirmOpen(false)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -159,10 +159,14 @@
|
||||
"branchLabel": "branch",
|
||||
"thinkingLabel": "Thinking",
|
||||
"errorLabel": "Error",
|
||||
"imageGenerateLabel": "Image generation",
|
||||
"agentRun": "{{agent}} execution",
|
||||
"collapsedSummary": "{{toolCount}} tool calls · {{processCount}} process messages",
|
||||
"collapseAction": "Collapse execution graph"
|
||||
},
|
||||
"imageGeneration": {
|
||||
"generating": "Generating image, please wait…"
|
||||
},
|
||||
"composer": {
|
||||
"attachFiles": "Attach files",
|
||||
"pickSkill": "Choose skill",
|
||||
|
||||
@@ -78,9 +78,15 @@
|
||||
"testing": "Generating…",
|
||||
"save": "Save",
|
||||
"saving": "Saving…",
|
||||
"clear": "Clear configuration",
|
||||
"clearing": "Clearing…",
|
||||
"clearConfirmTitle": "Clear image generation configuration?",
|
||||
"clearConfirmMessage": "This removes the custom image endpoint, model selection, and stored API key. Chat providers are not affected.",
|
||||
"clearConfirmAction": "Clear",
|
||||
"testPrompt": "A small red circle on a white background, minimal flat illustration.",
|
||||
"toast": {
|
||||
"saved": "Image generation settings saved",
|
||||
"cleared": "Image generation configuration cleared",
|
||||
"saveBeforeTest": "Save settings before running a test",
|
||||
"testSuccess": "Test image generated ({{ms}} ms)",
|
||||
"testSuccessWithPath": "Test image generated ({{ms}} ms): {{path}}",
|
||||
@@ -90,7 +96,7 @@
|
||||
"title": "Custom image endpoint",
|
||||
"description": "Send image generation to a dedicated OpenAI Images-compatible endpoint (/v1/images/generations). This does not change any chat provider.",
|
||||
"baseUrl": "Image Base URL",
|
||||
"baseUrlHint": "e.g. https://taolat.com/v1 — /v1 is appended automatically when omitted.",
|
||||
"baseUrlHint": "e.g. https://api.example.com/v1 — /v1 is appended automatically when omitted.",
|
||||
"apiKey": "API key",
|
||||
"apiKeyPlaceholder": "sk-...",
|
||||
"apiKeyPlaceholderConfigured": "Already configured — leave blank to keep",
|
||||
|
||||
@@ -159,10 +159,14 @@
|
||||
"branchLabel": "branch",
|
||||
"thinkingLabel": "考え中",
|
||||
"errorLabel": "エラー",
|
||||
"imageGenerateLabel": "画像生成",
|
||||
"agentRun": "{{agent}} の実行",
|
||||
"collapsedSummary": "ツール呼び出し {{toolCount}} 件 · プロセスメッセージ {{processCount}} 件",
|
||||
"collapseAction": "実行グラフを折りたたむ"
|
||||
},
|
||||
"imageGeneration": {
|
||||
"generating": "画像を生成しています。少々お待ちください…"
|
||||
},
|
||||
"composer": {
|
||||
"attachFiles": "ファイルを添付",
|
||||
"pickSkill": "Skill を選択",
|
||||
|
||||
@@ -78,9 +78,15 @@
|
||||
"testing": "生成中…",
|
||||
"save": "保存",
|
||||
"saving": "保存中…",
|
||||
"clear": "設定をクリア",
|
||||
"clearing": "クリア中…",
|
||||
"clearConfirmTitle": "画像生成設定をクリアしますか?",
|
||||
"clearConfirmMessage": "カスタム画像エンドポイント、モデル設定、保存済み API キーを削除します。チャットプロバイダーには影響しません。",
|
||||
"clearConfirmAction": "クリア",
|
||||
"testPrompt": "白背景に小さな赤い円、ミニマルなフラットイラスト。",
|
||||
"toast": {
|
||||
"saved": "画像生成設定を保存しました",
|
||||
"cleared": "画像生成設定をクリアしました",
|
||||
"saveBeforeTest": "テスト前に設定を保存してください",
|
||||
"testSuccess": "テスト画像を生成しました({{ms}} ms)",
|
||||
"testSuccessWithPath": "テスト画像を生成しました({{ms}} ms):{{path}}",
|
||||
@@ -90,7 +96,7 @@
|
||||
"title": "カスタム画像エンドポイント",
|
||||
"description": "画像生成を専用の OpenAI Images 互換エンドポイント(/v1/images/generations)へ送ります。チャットプロバイダーは変更しません。",
|
||||
"baseUrl": "画像 Base URL",
|
||||
"baseUrlHint": "例: https://taolat.com/v1 — /v1 が無い場合は自動付与します。",
|
||||
"baseUrlHint": "例: https://api.example.com/v1 — /v1 が無い場合は自動付与します。",
|
||||
"apiKey": "API キー",
|
||||
"apiKeyPlaceholder": "sk-...",
|
||||
"apiKeyPlaceholderConfigured": "設定済み — 空欄のままなら変更しません",
|
||||
|
||||
@@ -159,10 +159,14 @@
|
||||
"branchLabel": "ветвь",
|
||||
"thinkingLabel": "Думаю",
|
||||
"runErrorLabel": "Ошибка модели",
|
||||
"imageGenerateLabel": "Генерация изображения",
|
||||
"agentRun": "Выполнение {{agent}}",
|
||||
"collapsedSummary": "Вызовов инструментов: {{toolCount}} · Промежуточных сообщений: {{processCount}}",
|
||||
"collapseAction": "Свернуть граф выполнения"
|
||||
},
|
||||
"imageGeneration": {
|
||||
"generating": "Изображение генерируется, подождите…"
|
||||
},
|
||||
"composer": {
|
||||
"attachFiles": "Прикрепить файлы",
|
||||
"pickSkill": "Выбрать Skill",
|
||||
|
||||
@@ -90,7 +90,7 @@
|
||||
"title": "Пользовательский endpoint изображений",
|
||||
"description": "Отправлять генерацию изображений на отдельный OpenAI Images-совместимый endpoint (/v1/images/generations), не меняя чат-провайдеры.",
|
||||
"baseUrl": "Base URL изображений",
|
||||
"baseUrlHint": "например https://taolat.com/v1 — /v1 добавится автоматически.",
|
||||
"baseUrlHint": "например https://api.example.com/v1 — /v1 добавится автоматически.",
|
||||
"apiKey": "API key",
|
||||
"apiKeyPlaceholder": "sk-...",
|
||||
"apiKeyPlaceholderConfigured": "Уже настроен — оставьте пустым, чтобы не менять",
|
||||
|
||||
@@ -159,10 +159,14 @@
|
||||
"branchLabel": "分支",
|
||||
"thinkingLabel": "思考中",
|
||||
"modelErrorTitle": "模型调用失败",
|
||||
"imageGenerateLabel": "图片生成",
|
||||
"agentRun": "{{agent}} 执行",
|
||||
"collapsedSummary": "{{toolCount}} 个工具调用,{{processCount}} 条过程消息",
|
||||
"collapseAction": "收起执行关系图"
|
||||
},
|
||||
"imageGeneration": {
|
||||
"generating": "图片生成中,请稍候…"
|
||||
},
|
||||
"composer": {
|
||||
"attachFiles": "添加文件",
|
||||
"pickSkill": "选择技能",
|
||||
|
||||
@@ -78,9 +78,15 @@
|
||||
"testing": "生成中…",
|
||||
"save": "保存",
|
||||
"saving": "保存中…",
|
||||
"clear": "清除配置",
|
||||
"clearing": "清除中…",
|
||||
"clearConfirmTitle": "清除图像生成配置?",
|
||||
"clearConfirmMessage": "将移除自定义生图端点、模型设置和已保存的 API Key,不会影响聊天 Provider。",
|
||||
"clearConfirmAction": "清除",
|
||||
"testPrompt": "白色背景上的小红圆,极简扁平插画。",
|
||||
"toast": {
|
||||
"saved": "图像生成设置已保存",
|
||||
"cleared": "图像生成配置已清除",
|
||||
"saveBeforeTest": "请先保存设置再运行测试",
|
||||
"testSuccess": "测试图像已生成({{ms}} ms)",
|
||||
"testSuccessWithPath": "测试图像已生成({{ms}} ms):{{path}}",
|
||||
@@ -90,7 +96,7 @@
|
||||
"title": "自定义生图端点",
|
||||
"description": "将生图请求发往独立的 OpenAI Images 兼容端点(/v1/images/generations),不会改变任何聊天 Provider。",
|
||||
"baseUrl": "生图 Base URL",
|
||||
"baseUrlHint": "例如 https://taolat.com/v1;未带 /v1 时会自动补全。",
|
||||
"baseUrlHint": "例如 https://api.example.com/v1;未带 /v1 时会自动补全。",
|
||||
"apiKey": "API Key",
|
||||
"apiKeyPlaceholder": "sk-...",
|
||||
"apiKeyPlaceholderConfigured": "已配置,留空则不修改",
|
||||
|
||||
@@ -61,6 +61,10 @@ export async function fetchImageGenerationSettings(): Promise<ImageGenerationSet
|
||||
return response;
|
||||
}
|
||||
|
||||
export async function clearImageGenerationSettings(): Promise<ImageGenerationSettingsSnapshot> {
|
||||
return saveImageGenerationSettings({ openAiRelayEnabled: false });
|
||||
}
|
||||
|
||||
export async function saveImageGenerationSettings(payload: {
|
||||
primary?: string | null;
|
||||
fallbacks?: string[];
|
||||
|
||||
@@ -16,7 +16,8 @@ import { Button } from '@/components/ui/button';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { invokeIpc, statFile } from '@/lib/api-client';
|
||||
import type { RawMessage, AttachedFileMeta } from '@/stores/chat';
|
||||
import { extractText, extractImages, extractToolUse, formatTimestamp } from './message-utils';
|
||||
import { extractText, extractImages, extractToolUse, formatTimestamp, isUnresolvableImageUrl } from './message-utils';
|
||||
import { copyImageToClipboard, type ImageCopyTarget } from './copy-image';
|
||||
|
||||
interface ChatMessageProps {
|
||||
message: RawMessage;
|
||||
@@ -222,7 +223,7 @@ function normalizeLatexDelimiters(input: string): string {
|
||||
|
||||
/** Resolve an ExtractedImage to a displayable src string, or null if not possible. */
|
||||
function imageSrc(img: ExtractedImage): string | null {
|
||||
if (img.url) return img.url;
|
||||
if (img.url) return isUnresolvableImageUrl(img.url) ? null : img.url;
|
||||
if (img.data) return `data:${img.mimeType};base64,${img.data}`;
|
||||
return null;
|
||||
}
|
||||
@@ -248,6 +249,7 @@ export const ChatMessage = memo(function ChatMessage({
|
||||
const hideAssistantText = suppressAssistantText && !isUser;
|
||||
const hasText = !hideAssistantText && text.trim().length > 0;
|
||||
const images = extractImages(message);
|
||||
const resolvableContentImages = images.filter((img) => imageSrc(img) != null);
|
||||
const tools = extractToolUse(message);
|
||||
const visibleTools = suppressToolCards ? [] : tools;
|
||||
const [validatedPaths, setValidatedPaths] = useState<Record<string, boolean>>({});
|
||||
@@ -321,16 +323,18 @@ export const ChatMessage = memo(function ChatMessage({
|
||||
if (!kind || !file.filePath) return true;
|
||||
return validatedPaths[file.filePath] === true;
|
||||
});
|
||||
const attachedFiles = suppressProcessAttachments && (hasText || images.length > 0 || visibleTools.length > 0)
|
||||
const attachedFiles = suppressProcessAttachments && (hasText || resolvableContentImages.length > 0 || visibleTools.length > 0)
|
||||
? processVisibleAttachments
|
||||
: existingDerivedAttachedFiles;
|
||||
const imageCopyTarget = resolvePrimaryImageCopyTarget(resolvableContentImages, attachedFiles);
|
||||
const showAssistantHoverBar = !isUser && (hasText || imageCopyTarget != null);
|
||||
const [lightboxImg, setLightboxImg] = useState<{ src: string; fileName: string; filePath?: string; base64?: string; mimeType?: string } | null>(null);
|
||||
|
||||
// Never render tool result messages in chat UI
|
||||
if (isToolResult) return null;
|
||||
|
||||
const hasStreamingToolStatus = isStreaming && streamingTools.length > 0;
|
||||
if (!hasText && images.length === 0 && visibleTools.length === 0 && attachedFiles.length === 0 && !hasStreamingToolStatus) return null;
|
||||
if (!hasText && resolvableContentImages.length === 0 && visibleTools.length === 0 && attachedFiles.length === 0 && !hasStreamingToolStatus) return null;
|
||||
|
||||
return (
|
||||
<div
|
||||
@@ -359,9 +363,9 @@ export const ChatMessage = memo(function ChatMessage({
|
||||
|
||||
{/* Images — rendered ABOVE text bubble for user messages */}
|
||||
{/* Images from content blocks (Gateway session data / channel push photos) */}
|
||||
{isUser && images.length > 0 && (
|
||||
{isUser && resolvableContentImages.length > 0 && (
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{images.map((img, i) => {
|
||||
{resolvableContentImages.map((img, i) => {
|
||||
const src = imageSrc(img);
|
||||
if (!src) return null;
|
||||
return (
|
||||
@@ -384,7 +388,7 @@ export const ChatMessage = memo(function ChatMessage({
|
||||
{attachedFiles.map((file, i) => {
|
||||
const isImage = file.mimeType.startsWith('image/');
|
||||
// Skip image attachments if we already have images from content blocks
|
||||
if (isImage && images.length > 0) return null;
|
||||
if (isImage && resolvableContentImages.length > 0) return null;
|
||||
if (isImage) {
|
||||
return file.preview ? (
|
||||
<ImageThumbnail
|
||||
@@ -420,9 +424,9 @@ export const ChatMessage = memo(function ChatMessage({
|
||||
)}
|
||||
|
||||
{/* Images from content blocks — assistant messages (below text) */}
|
||||
{!isUser && images.length > 0 && (
|
||||
{!isUser && resolvableContentImages.length > 0 && (
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{images.map((img, i) => {
|
||||
{resolvableContentImages.map((img, i) => {
|
||||
const src = imageSrc(img);
|
||||
if (!src) return null;
|
||||
return (
|
||||
@@ -444,7 +448,7 @@ export const ChatMessage = memo(function ChatMessage({
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{attachedFiles.map((file, i) => {
|
||||
const isImage = file.mimeType.startsWith('image/');
|
||||
if (isImage && images.length > 0) return null;
|
||||
if (isImage && resolvableContentImages.length > 0) return null;
|
||||
if (isImage && file.preview) {
|
||||
return (
|
||||
<ImagePreviewCard
|
||||
@@ -476,9 +480,9 @@ export const ChatMessage = memo(function ChatMessage({
|
||||
</span>
|
||||
)}
|
||||
|
||||
{/* Hover row for assistant messages — only when there is real text content */}
|
||||
{!isUser && hasText && (
|
||||
<AssistantHoverBar text={text} timestamp={message.timestamp} />
|
||||
{/* Hover row for assistant messages */}
|
||||
{showAssistantHoverBar && (
|
||||
<AssistantHoverBar text={text} timestamp={message.timestamp} imageCopyTarget={imageCopyTarget} />
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -547,16 +551,60 @@ function ToolStatusBar({
|
||||
);
|
||||
}
|
||||
|
||||
function resolvePrimaryImageCopyTarget(
|
||||
images: Array<{ mimeType: string; data?: string; url?: string }>,
|
||||
attachedFiles: AttachedFileMeta[],
|
||||
): ImageCopyTarget | null {
|
||||
for (const file of attachedFiles) {
|
||||
if (!file.mimeType.startsWith('image/')) continue;
|
||||
if (file.filePath || file.preview?.startsWith('data:')) {
|
||||
return {
|
||||
filePath: file.filePath,
|
||||
preview: file.preview,
|
||||
mimeType: file.mimeType,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
for (const image of images) {
|
||||
if (image.data) {
|
||||
return { base64: image.data, mimeType: image.mimeType };
|
||||
}
|
||||
if (image.url?.startsWith('data:')) {
|
||||
return { preview: image.url, mimeType: image.mimeType };
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── Assistant hover bar (timestamp + copy, shown on group hover) ─
|
||||
|
||||
function AssistantHoverBar({ text, timestamp }: { text: string; timestamp?: number }) {
|
||||
function AssistantHoverBar({
|
||||
text,
|
||||
timestamp,
|
||||
imageCopyTarget,
|
||||
}: {
|
||||
text: string;
|
||||
timestamp?: number;
|
||||
imageCopyTarget?: ImageCopyTarget | null;
|
||||
}) {
|
||||
const [copied, setCopied] = useState(false);
|
||||
|
||||
const copyContent = useCallback(() => {
|
||||
navigator.clipboard.writeText(text);
|
||||
const copyContent = useCallback(async () => {
|
||||
if (imageCopyTarget) {
|
||||
const copiedImage = await copyImageToClipboard(imageCopyTarget);
|
||||
if (copiedImage) {
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 2000);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (!text.trim()) return;
|
||||
await navigator.clipboard.writeText(text);
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 2000);
|
||||
}, [text]);
|
||||
}, [text, imageCopyTarget]);
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between w-full opacity-0 group-hover:opacity-100 transition-opacity duration-200 select-none px-1">
|
||||
@@ -629,6 +677,12 @@ function AssistantMarkdown({
|
||||
</a>
|
||||
);
|
||||
},
|
||||
img({ src, alt }) {
|
||||
if (!src || isUnresolvableImageUrl(String(src))) return null;
|
||||
return (
|
||||
<img src={String(src)} alt={typeof alt === 'string' ? alt : 'image'} className="max-w-full rounded-lg" />
|
||||
);
|
||||
},
|
||||
}}
|
||||
>
|
||||
{normalizeLatexDelimiters(text)}
|
||||
|
||||
@@ -47,6 +47,9 @@ function StepDetailCard({ step }: { step: TaskStep }) {
|
||||
const isNarration = step.kind === 'message';
|
||||
const isTool = step.kind === 'tool';
|
||||
const isThinking = step.kind === 'thinking';
|
||||
const displayToolLabel = isTool && step.label === 'image_generate'
|
||||
? t('executionGraph.imageGenerateLabel')
|
||||
: step.label;
|
||||
// System steps (subagent branch roots etc.) share the tool row layout:
|
||||
// bold label + truncated single-line detail preview + click-to-expand,
|
||||
// i.e. no rounded card / no separate detail line below the title.
|
||||
@@ -56,7 +59,7 @@ function StepDetailCard({ step }: { step: TaskStep }) {
|
||||
const hideStatusText = (isTool || isSystem) && step.status === 'completed';
|
||||
const detailPreview = step.detail?.replace(/\s+/g, ' ').trim();
|
||||
const canExpand = hasDetail;
|
||||
const displayLabel = isThinking ? t('executionGraph.thinkingLabel') : step.label;
|
||||
const displayLabel = isThinking ? t('executionGraph.thinkingLabel') : (isTool ? displayToolLabel : step.label);
|
||||
|
||||
return (
|
||||
<div
|
||||
|
||||
52
src/pages/Chat/copy-image.ts
Normal file
52
src/pages/Chat/copy-image.ts
Normal file
@@ -0,0 +1,52 @@
|
||||
import { readBinaryFile } from '@/lib/api-client';
|
||||
|
||||
export interface ImageCopyTarget {
|
||||
filePath?: string;
|
||||
preview?: string | null;
|
||||
mimeType?: string;
|
||||
base64?: string;
|
||||
}
|
||||
|
||||
function blobFromBytes(data: Uint8Array, mimeType: string): Blob {
|
||||
const copy = new Uint8Array(data);
|
||||
return new Blob([copy], { type: mimeType });
|
||||
}
|
||||
|
||||
async function blobFromDataUrl(dataUrl: string): Promise<Blob | null> {
|
||||
try {
|
||||
const response = await fetch(dataUrl);
|
||||
if (!response.ok) return null;
|
||||
return await response.blob();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function writeImageBlob(blob: Blob, fallbackMimeType: string): Promise<boolean> {
|
||||
if (!navigator.clipboard?.write || typeof ClipboardItem === 'undefined') return false;
|
||||
const type = blob.type || fallbackMimeType;
|
||||
await navigator.clipboard.write([new ClipboardItem({ [type]: blob })]);
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function copyImageToClipboard(target: ImageCopyTarget): Promise<boolean> {
|
||||
const mimeType = target.mimeType?.startsWith('image/') ? target.mimeType : 'image/png';
|
||||
|
||||
if (target.base64) {
|
||||
const blob = await blobFromDataUrl(`data:${mimeType};base64,${target.base64}`);
|
||||
return blob ? writeImageBlob(blob, mimeType) : false;
|
||||
}
|
||||
|
||||
if (target.preview?.startsWith('data:')) {
|
||||
const blob = await blobFromDataUrl(target.preview);
|
||||
return blob ? writeImageBlob(blob, mimeType) : false;
|
||||
}
|
||||
|
||||
if (target.filePath) {
|
||||
const result = await readBinaryFile(target.filePath);
|
||||
if (!result.ok || !result.data) return false;
|
||||
return writeImageBlob(blobFromBytes(result.data, result.mimeType || mimeType), mimeType);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
162
src/pages/Chat/image-generation-status.ts
Normal file
162
src/pages/Chat/image-generation-status.ts
Normal file
@@ -0,0 +1,162 @@
|
||||
import type { RawMessage, ToolStatus } from '@/stores/chat';
|
||||
import {
|
||||
extractImages,
|
||||
extractToolUse,
|
||||
isGeneratingStatusNarration,
|
||||
isInternalAssistantReplyText,
|
||||
} from './message-utils';
|
||||
|
||||
const IMAGE_GENERATE_TOOL = 'image_generate';
|
||||
const ASYNC_IMAGE_TASK_START_RE = /Background task started for image generation \(([0-9a-f-]{36})\)/i;
|
||||
const INTER_SESSION_IMAGE_TASK_RE = /sourceSession=image_generate:([0-9a-f-]{36})/i;
|
||||
/** Match OpenClaw agents.defaults.imageGenerationModel.timeoutMs default. */
|
||||
export const IMAGE_GENERATION_TIMEOUT_MS = 180_000;
|
||||
const IMAGE_GENERATION_TIMEOUT_BUFFER_MS = 15_000;
|
||||
|
||||
function toMs(timestamp: number | undefined): number | null {
|
||||
if (timestamp == null || !Number.isFinite(timestamp)) return null;
|
||||
return timestamp < 1e12 ? timestamp * 1000 : timestamp;
|
||||
}
|
||||
|
||||
function getMessagePlainText(message: RawMessage): string {
|
||||
const { content } = message;
|
||||
if (typeof content === 'string') return content;
|
||||
if (!Array.isArray(content)) return '';
|
||||
const parts: string[] = [];
|
||||
for (const block of content as Array<{ type?: string; text?: string }>) {
|
||||
if (block.type === 'text' && typeof block.text === 'string') {
|
||||
parts.push(block.text);
|
||||
}
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
function messageHasDeliveredImage(message: RawMessage): boolean {
|
||||
if ((message._attachedFiles ?? []).some((file) => file.mimeType.startsWith('image/'))) {
|
||||
return true;
|
||||
}
|
||||
return extractImages(message).length > 0;
|
||||
}
|
||||
|
||||
function findLastImageGenerateToolCallIndex(segmentMessages: RawMessage[]): number {
|
||||
for (let index = segmentMessages.length - 1; index >= 0; index -= 1) {
|
||||
const message = segmentMessages[index];
|
||||
if (message.role !== 'assistant') continue;
|
||||
if (extractToolUse(message).some((tool) => tool.name === IMAGE_GENERATE_TOOL)) {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
function collectAsyncImageTaskStarts(
|
||||
segmentMessages: RawMessage[],
|
||||
): Array<{ taskId: string; startedAtMs: number | null }> {
|
||||
const starts: Array<{ taskId: string; startedAtMs: number | null }> = [];
|
||||
for (const message of segmentMessages) {
|
||||
const role = String(message.role ?? '').toLowerCase();
|
||||
if (role !== 'toolresult' && role !== 'tool_result') continue;
|
||||
if (message.toolName !== IMAGE_GENERATE_TOOL) continue;
|
||||
const match = getMessagePlainText(message).match(ASYNC_IMAGE_TASK_START_RE);
|
||||
if (!match?.[1]) continue;
|
||||
starts.push({
|
||||
taskId: match[1],
|
||||
startedAtMs: toMs(message.timestamp),
|
||||
});
|
||||
}
|
||||
return starts;
|
||||
}
|
||||
|
||||
function collectCompletedAsyncImageTaskIds(segmentMessages: RawMessage[]): Set<string> {
|
||||
const completed = new Set<string>();
|
||||
for (const message of segmentMessages) {
|
||||
const role = String(message.role ?? '').toLowerCase();
|
||||
if (role !== 'user') continue;
|
||||
const text = getMessagePlainText(message).trim();
|
||||
if (!/^\[Inter-session message\]/i.test(text)) continue;
|
||||
const match = text.match(INTER_SESSION_IMAGE_TASK_RE);
|
||||
if (match?.[1]) completed.add(match[1]);
|
||||
}
|
||||
return completed;
|
||||
}
|
||||
|
||||
function hasTimedOut(startedAtMs: number | null, now: number): boolean {
|
||||
if (startedAtMs == null) return false;
|
||||
return now - startedAtMs > IMAGE_GENERATION_TIMEOUT_MS + IMAGE_GENERATION_TIMEOUT_BUFFER_MS;
|
||||
}
|
||||
|
||||
function hasDeliveredImageAfter(segmentMessages: RawMessage[], fromIndex: number): boolean {
|
||||
for (let index = fromIndex + 1; index < segmentMessages.length; index += 1) {
|
||||
const message = segmentMessages[index];
|
||||
if (message.role !== 'assistant') continue;
|
||||
if (messageHasDeliveredImage(message)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function hasTerminalReplyAfterToolCall(segmentMessages: RawMessage[], fromIndex: number): boolean {
|
||||
for (let index = fromIndex + 1; index < segmentMessages.length; index += 1) {
|
||||
const message = segmentMessages[index];
|
||||
if (message.role !== 'assistant') continue;
|
||||
if (extractToolUse(message).length > 0) continue;
|
||||
|
||||
const text = getMessagePlainText(message).trim();
|
||||
if (!text || isInternalAssistantReplyText(text)) continue;
|
||||
if (isGeneratingStatusNarration(text)) continue;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function hasFreshRunningStreamingTool(streamingTools: ToolStatus[], now: number): boolean {
|
||||
return streamingTools.some((tool) => {
|
||||
if (tool.name !== IMAGE_GENERATE_TOOL || tool.status !== 'running') return false;
|
||||
return now - tool.updatedAt <= IMAGE_GENERATION_TIMEOUT_MS + IMAGE_GENERATION_TIMEOUT_BUFFER_MS;
|
||||
});
|
||||
}
|
||||
|
||||
/** True while an async `image_generate` task is in flight for this run segment. */
|
||||
export function isImageGenerationPending(
|
||||
segmentMessages: RawMessage[],
|
||||
streamingTools: ToolStatus[] = [],
|
||||
now = Date.now(),
|
||||
): boolean {
|
||||
if (hasFreshRunningStreamingTool(streamingTools, now)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const toolCallIndex = findLastImageGenerateToolCallIndex(segmentMessages);
|
||||
const asyncStarts = collectAsyncImageTaskStarts(segmentMessages);
|
||||
const completedTaskIds = collectCompletedAsyncImageTaskIds(segmentMessages);
|
||||
|
||||
if (toolCallIndex < 0 && asyncStarts.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const toolCallStartedAt = toolCallIndex >= 0
|
||||
? toMs(segmentMessages[toolCallIndex]?.timestamp)
|
||||
: null;
|
||||
const latestAsyncStartedAt = asyncStarts.reduce<number | null>((latest, start) => {
|
||||
if (start.startedAtMs == null) return latest;
|
||||
return latest == null ? start.startedAtMs : Math.max(latest, start.startedAtMs);
|
||||
}, null);
|
||||
const anchorStartedAt = [toolCallStartedAt, latestAsyncStartedAt]
|
||||
.filter((value): value is number => value != null)
|
||||
.reduce<number | null>((latest, value) => (latest == null ? value : Math.max(latest, value)), null);
|
||||
|
||||
if (hasTimedOut(anchorStartedAt, now)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (toolCallIndex >= 0) {
|
||||
if (hasDeliveredImageAfter(segmentMessages, toolCallIndex)) return false;
|
||||
if (hasTerminalReplyAfterToolCall(segmentMessages, toolCallIndex)) return false;
|
||||
}
|
||||
|
||||
if (asyncStarts.length > 0) {
|
||||
const hasOpenAsyncTask = asyncStarts.some((start) => !completedTaskIds.has(start.taskId));
|
||||
if (!hasOpenAsyncTask) return false;
|
||||
}
|
||||
|
||||
return toolCallIndex >= 0 || asyncStarts.some((start) => !completedTaskIds.has(start.taskId));
|
||||
}
|
||||
@@ -7,6 +7,7 @@
|
||||
import { Suspense, lazy, useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { AlertCircle, ArrowDownToLine, Loader2, Sparkles } from 'lucide-react';
|
||||
import { useChatStore, type RawMessage } from '@/stores/chat';
|
||||
import { isInternalMessage } from '@/stores/chat/helpers';
|
||||
import { buildBaselineRunKey, getBaseline } from '@/stores/baseline-cache';
|
||||
import { useGatewayStore } from '@/stores/gateway';
|
||||
import { useAgentsStore } from '@/stores/agents';
|
||||
@@ -18,7 +19,7 @@ import { ChatMessage } from './ChatMessage';
|
||||
import { ChatInput } from './ChatInput';
|
||||
import { ExecutionGraphCard } from './ExecutionGraphCard';
|
||||
import { ChatToolbar } from './ChatToolbar';
|
||||
import { extractImages, extractText, extractThinking, extractToolUse, normalizeMessageRole, stripProcessMessagePrefix } from './message-utils';
|
||||
import { extractImages, extractText, extractThinking, extractToolUse, isInternalAssistantReplyText, isInternalProcessNarration, normalizeMessageRole, stripProcessMessagePrefix } from './message-utils';
|
||||
import {
|
||||
buildRunSegmentMessageIndices,
|
||||
deriveTaskSteps,
|
||||
@@ -30,6 +31,7 @@ import {
|
||||
segmentHasFinalReply,
|
||||
type TaskStep,
|
||||
} from './task-visualization';
|
||||
import { isImageGenerationPending } from './image-generation-status';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { useStickToBottomInstant } from '@/hooks/use-stick-to-bottom-instant';
|
||||
@@ -95,6 +97,14 @@ function getPrimaryMessageStepTexts(steps: TaskStep[]): string[] {
|
||||
.map((step) => step.detail!);
|
||||
}
|
||||
|
||||
function sanitizeGraphSteps(steps: TaskStep[]): TaskStep[] {
|
||||
return steps.filter((step) => {
|
||||
if (step.kind === 'thinking') return false;
|
||||
if (step.kind === 'message' && step.detail && isInternalProcessNarration(step.detail)) return false;
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function buildQuestionDirectoryTitle(message: RawMessage, fallback: string): string {
|
||||
const normalized = extractText(message).replace(/\s+/g, ' ').trim();
|
||||
if (!normalized) return fallback;
|
||||
@@ -103,6 +113,7 @@ function buildQuestionDirectoryTitle(message: RawMessage, fallback: string): str
|
||||
|
||||
function isRealUserMessage(msg: RawMessage): boolean {
|
||||
if (normalizeMessageRole(msg.role) !== 'user') return false;
|
||||
if (isInternalMessage(msg)) return false;
|
||||
const content = msg.content;
|
||||
if (!Array.isArray(content)) return true;
|
||||
// If every block in the content is a tool_result, this is a Gateway
|
||||
@@ -111,6 +122,10 @@ function isRealUserMessage(msg: RawMessage): boolean {
|
||||
return blocks.length === 0 || !blocks.every((b) => b.type === 'tool_result' || b.type === 'toolResult');
|
||||
}
|
||||
|
||||
function hasUserFacingImageAttachments(msg: RawMessage): boolean {
|
||||
return (msg._attachedFiles ?? []).some((file) => file.mimeType.startsWith('image/'));
|
||||
}
|
||||
|
||||
function generatedFileToTarget(file: GeneratedFile): FilePreviewTarget {
|
||||
return {
|
||||
filePath: file.filePath,
|
||||
@@ -493,15 +508,16 @@ export function Chat() {
|
||||
&& streamTools.length === 0
|
||||
&& !hasRunningStreamToolStatus;
|
||||
|
||||
let steps = buildSteps(rawStreamingReplyCandidate);
|
||||
let steps = sanitizeGraphSteps(buildSteps(rawStreamingReplyCandidate));
|
||||
let streamingReplyText: string | null = null;
|
||||
if (rawStreamingReplyCandidate) {
|
||||
const trimmedReplyText = stripProcessMessagePrefix(streamText, getPrimaryMessageStepTexts(steps));
|
||||
const hasReplyText = trimmedReplyText.trim().length > 0;
|
||||
const hasReplyText = trimmedReplyText.trim().length > 0
|
||||
&& !isInternalAssistantReplyText(trimmedReplyText);
|
||||
if (hasReplyText || hasStreamImages) {
|
||||
streamingReplyText = trimmedReplyText;
|
||||
streamingReplyText = hasReplyText ? trimmedReplyText : '';
|
||||
} else {
|
||||
steps = buildSteps(false);
|
||||
steps = sanitizeGraphSteps(buildSteps(false));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -545,9 +561,9 @@ export function Chat() {
|
||||
// generated message steps that include accumulated narration + reply
|
||||
// text. Strip these out — historical message steps (from messages[])
|
||||
// will be properly recomputed on the next render with fresh data.
|
||||
const cleanedSteps = cached.steps.filter(
|
||||
const cleanedSteps = sanitizeGraphSteps(cached.steps.filter(
|
||||
(s) => !(s.kind === 'message' && s.id.startsWith('stream-message')),
|
||||
);
|
||||
));
|
||||
return [{
|
||||
triggerIndex: idx,
|
||||
replyIndex: cached.replyIndex,
|
||||
@@ -622,6 +638,7 @@ export function Chat() {
|
||||
}, [messages, subagentCompletionInfos, currentSessionKey, streamingMessage, streamingTools, pendingFinal, sending, hasAnyStreamContent, hasStreamText, hasStreamImages, streamText, streamTools.length, hasRunningStreamToolStatus, childTranscripts, currentAgentId, agents, sessionLabels, graphStepCache, runError, isRunTrigger]);
|
||||
const hasActiveExecutionGraph = userRunCards.some((card) => card.active);
|
||||
let latestRunSegmentCompletion = { hasFinalReply: false, hasToolActivity: false };
|
||||
let pendingImageGeneration = false;
|
||||
for (let idx = messages.length - 1; idx >= 0; idx -= 1) {
|
||||
if (!isRealUserMessage(messages[idx]) || subagentCompletionInfos[idx]) continue;
|
||||
const nextUserIndex = nextUserMessageIndexes[idx];
|
||||
@@ -632,6 +649,10 @@ export function Chat() {
|
||||
m.role === 'assistant' && extractToolUse(m).length > 0,
|
||||
),
|
||||
};
|
||||
pendingImageGeneration = isImageGenerationPending(
|
||||
postTrigger,
|
||||
nextUserIndex === -1 ? streamingTools : [],
|
||||
);
|
||||
break;
|
||||
}
|
||||
const runSettledInHistory = latestRunSegmentCompletion.hasFinalReply
|
||||
@@ -824,7 +845,9 @@ export function Chat() {
|
||||
</div>
|
||||
)}
|
||||
{messages.map((msg, idx) => {
|
||||
if (foldedNarrationIndices.has(idx)) return null;
|
||||
if (isInternalMessage(msg) && !hasUserFacingImageAttachments(msg)) return null;
|
||||
const isFoldedNarration = foldedNarrationIndices.has(idx);
|
||||
if (isFoldedNarration && !hasUserFacingImageAttachments(msg)) return null;
|
||||
const suppressToolCards = runSegmentMessageIndices.has(idx);
|
||||
const isToolOnlyAssistant = normalizeMessageRole(msg.role) === 'assistant'
|
||||
&& extractToolUse(msg).length > 0
|
||||
@@ -843,6 +866,7 @@ export function Chat() {
|
||||
<ChatMessage
|
||||
message={msg}
|
||||
textOverride={replyTextOverrides.get(idx)}
|
||||
suppressAssistantText={isFoldedNarration}
|
||||
suppressToolCards={suppressToolCards}
|
||||
suppressProcessAttachments={suppressToolCards}
|
||||
onOpenFile={handleOpenAttachedFile}
|
||||
@@ -946,8 +970,12 @@ export function Chat() {
|
||||
<ActivityIndicator phase="tool_processing" />
|
||||
)}
|
||||
|
||||
{pendingImageGeneration && (
|
||||
<ImageGeneratingIndicator />
|
||||
)}
|
||||
|
||||
{/* Typing indicator when sending but no stream content yet */}
|
||||
{inputRunActive && !pendingFinal && !hasAnyStreamContent && !hasActiveExecutionGraph && (
|
||||
{inputRunActive && !pendingFinal && !hasAnyStreamContent && !hasActiveExecutionGraph && !pendingImageGeneration && (
|
||||
<TypingIndicator />
|
||||
)}
|
||||
</>
|
||||
@@ -1195,4 +1223,21 @@ function ActivityIndicator({ phase }: { phase: 'tool_processing' }) {
|
||||
);
|
||||
}
|
||||
|
||||
function ImageGeneratingIndicator() {
|
||||
const { t } = useTranslation('chat');
|
||||
return (
|
||||
<div className="flex gap-3" data-testid="chat-image-generating-indicator">
|
||||
<div className="flex h-8 w-8 shrink-0 items-center justify-center rounded-full mt-1 bg-black/5 dark:bg-white/5 text-foreground">
|
||||
<Sparkles className="h-4 w-4" />
|
||||
</div>
|
||||
<div className="bg-black/5 dark:bg-white/5 text-foreground rounded-2xl px-4 py-3">
|
||||
<div className="flex items-center gap-2 text-sm text-muted-foreground">
|
||||
<Loader2 className="h-3.5 w-3.5 animate-spin text-primary" />
|
||||
<span>{t('imageGeneration.generating')}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default Chat;
|
||||
|
||||
@@ -5,13 +5,39 @@
|
||||
*/
|
||||
import type { RawMessage, ContentBlock } from '@/stores/chat';
|
||||
|
||||
/**
|
||||
* Strip OpenClaw's inbound-image vision envelope from user message text.
|
||||
* The runtime expands uploaded images into:
|
||||
* [Image]
|
||||
* User text:
|
||||
* <message or "Process the attached file(s).">
|
||||
* Description:
|
||||
* <auto-generated caption>
|
||||
*/
|
||||
function stripInboundMediaVisionEnvelope(text: string): string {
|
||||
if (!/\[Image\]/i.test(text) && !/^User text:/im.test(text) && !/\nDescription:\s*\n/i.test(text)) {
|
||||
return text;
|
||||
}
|
||||
|
||||
let result = text.replace(/^\s*\[Image\]\s*\n?/i, '');
|
||||
|
||||
const userTextBlock = result.match(/^User text:\s*\n([\s\S]*?)(?:\n\s*Description:\s*\n[\s\S]*)?\s*$/i);
|
||||
if (userTextBlock) {
|
||||
const userText = userTextBlock[1].trim();
|
||||
return /^Process the attached file\(s\)\.\s*$/i.test(userText) ? '' : userText;
|
||||
}
|
||||
|
||||
return result.replace(/\n\s*Description:\s*\n[\s\S]*$/i, '').trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean Gateway metadata from user message text for display.
|
||||
* Strips: [media attached: ... | ...], [message_id: ...],
|
||||
* and the timestamp prefix [Day Date Time Timezone].
|
||||
*/
|
||||
function cleanUserText(text: string): string {
|
||||
return text
|
||||
return stripInboundMediaVisionEnvelope(
|
||||
text
|
||||
// Remove [media attached: path (mime) | path] references
|
||||
.replace(/\s*\[media attached:[^\]]*\]/g, '')
|
||||
// Remove [message_id: uuid]
|
||||
@@ -30,7 +56,8 @@ function cleanUserText(text: string): string {
|
||||
.replace(/^Conversation info\s*\([^)]*\):\s*\{[\s\S]*?\}\s*/i, '')
|
||||
// Remove Gateway timestamp prefix like [Fri 2026-02-13 22:39 GMT+8]
|
||||
.replace(/^\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, '')
|
||||
.trim();
|
||||
.trim(),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -51,8 +78,13 @@ function stripAssistantMediaTags(text: string): string {
|
||||
// would still leak the literal `MEDIA:/.../截屏 2026-05-06 17.46.51.png`
|
||||
// to the user when the underlying path detection succeeds.
|
||||
const tagged = new RegExp(`(^|[\\s(\\[{>])(?:MEDIA|media):(?:\\/|~\\/)[^\\n"'()\\[\\],<>]*?\\.(?:${exts})(?=$|[\\s\\n"'()\\[\\],<>]|[,。;;,.!?])`, 'g');
|
||||
// Bare OpenClaw artifact paths emitted alongside `_attachedFiles` cards.
|
||||
// Scope to `.openclaw/media/` so normal absolute paths in prose stay visible.
|
||||
const bareOpenClawMedia = new RegExp(`(^|[\\s(\\[{>])(?:(?:\\/|~\\/)[^\\n"'()\\[\\],<>]*?\\.openclaw\\/media\\/[^\\n"'()\\[\\],<>]*?\\.(?:${exts}))(?=$|[\\s\\n"'()\\[\\],<>]|[,。;;,.!?])`, 'g');
|
||||
return text
|
||||
.replace(tagged, (_, lead: string) => lead)
|
||||
.replace(bareOpenClawMedia, (_, lead: string) => lead)
|
||||
.replace(/!\[[^\]]*\]\([^)]+\)/g, '')
|
||||
// Collapse the empty lines / orphan whitespace the strip leaves behind.
|
||||
.replace(/[ \t]+\n/g, '\n')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
@@ -143,6 +175,57 @@ function consumeLeadingSegment(text: string, segment: string): number {
|
||||
return match ? match[0].length : 0;
|
||||
}
|
||||
|
||||
/** True for OpenClaw internal assistant tokens that must never appear in the chat UI. */
|
||||
export function isInternalAssistantReplyText(text: string): boolean {
|
||||
return /^(HEARTBEAT_OK|NO_REPLY)\s*$/i.test(text.trim());
|
||||
}
|
||||
|
||||
/** Interim image-generation status the user should not see as a final answer. */
|
||||
export function isGeneratingStatusNarration(text: string): boolean {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return false;
|
||||
if (/^(?:图片(?:正在)?生成中|正在生成(?:图片|图像)|生成中)/i.test(trimmed)) return true;
|
||||
if (/稍等(片刻|一下)?/.test(trimmed) && trimmed.length <= 80 && /生成|generat/i.test(trimmed)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/** OpenClaw runtime continuation prompt injected between internal async events. */
|
||||
export function isOpenClawRuntimeEventPrompt(text: string): boolean {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return false;
|
||||
if (/^Continue the OpenClaw runtime event\.?\s*$/i.test(trimmed)) return true;
|
||||
// Some transcripts store the prompt as the only line in a longer envelope.
|
||||
return trimmed.split(/\n+/).some((line) => /^Continue the OpenClaw runtime event\.?\s*$/i.test(line.trim()));
|
||||
}
|
||||
|
||||
/** Process narration that should never appear in the execution graph or chat stream. */
|
||||
export function isInternalProcessNarration(text: string): boolean {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return true;
|
||||
if (isInternalAssistantReplyText(trimmed)) return true;
|
||||
if (isGeneratingStatusNarration(trimmed)) return true;
|
||||
if (isOpenClawRuntimeEventPrompt(trimmed)) return true;
|
||||
if (/^\[Inter-session message\]/i.test(trimmed)) return true;
|
||||
if (/OpenClaw runtime event/i.test(trimmed)) return true;
|
||||
if (/Handle the result internally/i.test(trimmed) && /Do not relay it to the user/i.test(trimmed)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** True when a renderer `src` cannot be fetched directly (Gateway proxy / local paths). */
|
||||
export function isUnresolvableImageUrl(url: string): boolean {
|
||||
const trimmed = url.trim();
|
||||
if (!trimmed) return true;
|
||||
if (trimmed.startsWith('data:')) return false;
|
||||
if (/^https?:\/\//i.test(trimmed)) return false;
|
||||
if (trimmed.startsWith('/api/chat/media/')) return true;
|
||||
if (trimmed.startsWith('file://')) return true;
|
||||
if (trimmed.includes('.openclaw/media/')) return true;
|
||||
if (trimmed.startsWith('/') || trimmed.startsWith('~/')) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract displayable text from a message's content field.
|
||||
* Handles both string content and array-of-blocks content.
|
||||
@@ -178,6 +261,8 @@ export function extractText(message: RawMessage | unknown): string {
|
||||
if (isUser && result) {
|
||||
result = cleanUserText(result);
|
||||
} else if (!isUser && result) {
|
||||
if (isInternalAssistantReplyText(result)) return '';
|
||||
if (isGeneratingStatusNarration(result)) return '';
|
||||
// Assistant-side cleanup: keep the bubble free of `MEDIA:/path` tags
|
||||
// that the runtime emits to point at produced artifacts. The same
|
||||
// path is surfaced as a clickable file card via `_attachedFiles`,
|
||||
@@ -348,7 +433,7 @@ export function extractImages(
|
||||
const src = block.source;
|
||||
if (src.type === 'base64' && src.media_type && src.data) {
|
||||
images.push({ mimeType: src.media_type, data: src.data });
|
||||
} else if (src.type === 'url' && src.url) {
|
||||
} else if (src.type === 'url' && src.url && !isUnresolvableImageUrl(src.url)) {
|
||||
images.push({ mimeType: src.media_type || 'image/jpeg', url: src.url });
|
||||
}
|
||||
continue;
|
||||
|
||||
@@ -1,4 +1,12 @@
|
||||
import { extractText, extractTextSegments, extractThinkingSegments, extractToolUse } from './message-utils';
|
||||
import {
|
||||
extractText,
|
||||
extractTextSegments,
|
||||
extractToolUse,
|
||||
isGeneratingStatusNarration,
|
||||
isInternalAssistantReplyText,
|
||||
isInternalProcessNarration,
|
||||
} from './message-utils';
|
||||
import { isInternalMessage } from '@/stores/chat/helpers';
|
||||
import type { RawMessage, ToolStatus } from '@/stores/chat';
|
||||
|
||||
export type TaskStepStatus = 'running' | 'completed' | 'error';
|
||||
@@ -37,7 +45,9 @@ export function findReplyMessageIndex(messages: RawMessage[], hasStreamingReply:
|
||||
for (let idx = messages.length - 1; idx >= 0; idx -= 1) {
|
||||
const message = messages[idx];
|
||||
if (!message || message.role !== 'assistant') continue;
|
||||
if (extractText(message).trim().length === 0) continue;
|
||||
const replyText = extractText(message).trim();
|
||||
if (replyText.length === 0 || isInternalAssistantReplyText(replyText)) continue;
|
||||
if (isGeneratingStatusNarration(replyText)) continue;
|
||||
return idx;
|
||||
}
|
||||
return -1;
|
||||
@@ -149,7 +159,9 @@ export function segmentHasFinalReply(segmentMessages: RawMessage[]): boolean {
|
||||
return segmentMessages.some((message, index) => {
|
||||
if (index <= lastToolUseOffset) return false;
|
||||
if (message.role !== 'assistant') return false;
|
||||
if (extractText(message).trim().length === 0) return false;
|
||||
const replyText = extractText(message).trim();
|
||||
if (replyText.length === 0 || isInternalAssistantReplyText(replyText)) return false;
|
||||
if (isGeneratingStatusNarration(replyText)) return false;
|
||||
const content = message.content;
|
||||
if (!Array.isArray(content)) return true;
|
||||
return !(content as Array<{ type?: string }>).some(
|
||||
@@ -310,7 +322,8 @@ function appendDetailSegments(
|
||||
): void {
|
||||
const normalizedSegments = segments
|
||||
.map((segment) => normalizeText(segment))
|
||||
.filter((segment): segment is string => !!segment);
|
||||
.filter((segment): segment is string => !!segment)
|
||||
.filter((segment) => !isInternalProcessNarration(segment));
|
||||
|
||||
normalizedSegments.forEach((detail, index) => {
|
||||
options.upsertStep({
|
||||
@@ -361,31 +374,27 @@ export function deriveTaskSteps({
|
||||
for (const [messageIndex, message] of messages.entries()) {
|
||||
if (!message || message.role !== 'assistant') continue;
|
||||
|
||||
appendDetailSegments(extractThinkingSegments(message), {
|
||||
idPrefix: `history-thinking-${message.id || messageIndex}`,
|
||||
label: 'Thinking',
|
||||
kind: 'thinking',
|
||||
running: false,
|
||||
upsertStep,
|
||||
});
|
||||
|
||||
const toolUses = extractToolUse(message);
|
||||
// Fold any intermediate assistant text into the graph as a narration
|
||||
// step — including text that lives on a mixed `text + toolCall` message.
|
||||
// The narration step is emitted BEFORE the tool steps so the graph
|
||||
// preserves the original ordering (the assistant "thinks out loud" and
|
||||
// then invokes the tool).
|
||||
const narrationSegments = extractTextSegments(message);
|
||||
const graphNarrationSegments = messageIndex === replyIndex
|
||||
? narrationSegments.slice(0, -1)
|
||||
: narrationSegments;
|
||||
appendDetailSegments(graphNarrationSegments, {
|
||||
idPrefix: `history-message-${message.id || messageIndex}`,
|
||||
label: 'Message',
|
||||
kind: 'message',
|
||||
running: false,
|
||||
upsertStep,
|
||||
});
|
||||
if (!isInternalMessage(message)) {
|
||||
// Fold any intermediate assistant text into the graph as a narration
|
||||
// step — including text that lives on a mixed `text + toolCall` message.
|
||||
// The narration step is emitted BEFORE the tool steps so the graph
|
||||
// preserves the original ordering (the assistant "thinks out loud" and
|
||||
// then invokes the tool).
|
||||
const narrationSegments = extractTextSegments(message);
|
||||
const graphNarrationSegments = messageIndex === replyIndex
|
||||
? narrationSegments.slice(0, -1)
|
||||
: narrationSegments;
|
||||
appendDetailSegments(graphNarrationSegments, {
|
||||
idPrefix: `history-message-${message.id || messageIndex}`,
|
||||
label: 'Message',
|
||||
kind: 'message',
|
||||
running: false,
|
||||
upsertStep,
|
||||
});
|
||||
} else if (toolUses.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
toolUses.forEach((tool, index) => {
|
||||
const input = tool.input as Record<string, unknown>;
|
||||
@@ -403,19 +412,6 @@ export function deriveTaskSteps({
|
||||
}
|
||||
|
||||
if (streamMessage) {
|
||||
// When the reply is being rendered as a separate bubble
|
||||
// (omitLastStreamingMessageSegment), thinking that accompanies
|
||||
// the reply belongs to the bubble — omit it from the graph.
|
||||
if (!omitLastStreamingMessageSegment) {
|
||||
appendDetailSegments(extractThinkingSegments(streamMessage), {
|
||||
idPrefix: 'stream-thinking',
|
||||
label: 'Thinking',
|
||||
kind: 'thinking',
|
||||
running: true,
|
||||
upsertStep,
|
||||
});
|
||||
}
|
||||
|
||||
// Stream-time narration should also appear in the execution graph so that
|
||||
// intermediate process output stays in P1 instead of leaking into the
|
||||
// assistant reply area.
|
||||
|
||||
@@ -45,6 +45,12 @@ import {
|
||||
type ToolStatus,
|
||||
} from './chat/types';
|
||||
import type { ChatGet, ChatSet } from './chat/store-api';
|
||||
import { enrichWithToolCallAttachments, shouldDropMessageFromHistory } from './chat/helpers';
|
||||
import {
|
||||
isGeneratingStatusNarration,
|
||||
isInternalAssistantReplyText,
|
||||
isOpenClawRuntimeEventPrompt,
|
||||
} from '@/pages/Chat/message-utils';
|
||||
|
||||
export type {
|
||||
AttachedFileMeta,
|
||||
@@ -567,8 +573,25 @@ function normalizeStreamingMessage(message: unknown): unknown {
|
||||
* `[media attached:` instead — leaving the timestamp in the normalized
|
||||
* comparison text and breaking optimistic-vs-echo dedupe.
|
||||
*/
|
||||
function stripInboundMediaVisionEnvelope(text: string): string {
|
||||
if (!/\[Image\]/i.test(text) && !/^User text:/im.test(text) && !/\nDescription:\s*\n/i.test(text)) {
|
||||
return text;
|
||||
}
|
||||
|
||||
let result = text.replace(/^\s*\[Image\]\s*\n?/i, '');
|
||||
|
||||
const userTextBlock = result.match(/^User text:\s*\n([\s\S]*?)(?:\n\s*Description:\s*\n[\s\S]*)?\s*$/i);
|
||||
if (userTextBlock) {
|
||||
const userText = userTextBlock[1].trim();
|
||||
return /^Process the attached file\(s\)\.\s*$/i.test(userText) ? '' : userText;
|
||||
}
|
||||
|
||||
return result.replace(/\n\s*Description:\s*\n[\s\S]*$/i, '').trim();
|
||||
}
|
||||
|
||||
function stripGatewayUserMetadata(text: string): string {
|
||||
return text
|
||||
return stripInboundMediaVisionEnvelope(
|
||||
text
|
||||
.replace(/\s*\[media attached:[^\]]*\]/g, '')
|
||||
.replace(/\s*\[message_id:\s*[^\]]+\]/g, '')
|
||||
.replace(/^Sender\s*\([^)]*\)\s*:\s*```[a-z]*\n[\s\S]*?```\s*/i, '')
|
||||
@@ -579,13 +602,16 @@ function stripGatewayUserMetadata(text: string): string {
|
||||
.replace(/^Sender\s*:\s*[^\n]*(?:\n\s*)*/i, '')
|
||||
.replace(/^Conversation info\s*\([^)]*\):\s*```[a-z]*\n[\s\S]*?```\s*/i, '')
|
||||
.replace(/^Conversation info\s*\([^)]*\):\s*\{[\s\S]*?\}\s*/i, '')
|
||||
.replace(/^\s*\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, '');
|
||||
.replace(/^\s*\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, ''),
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeComparableUserText(content: unknown): string {
|
||||
return stripGatewayUserMetadata(getMessageText(content))
|
||||
const text = stripGatewayUserMetadata(getMessageText(content))
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
if (/^\(file attached\)$/i.test(text)) return '';
|
||||
return text;
|
||||
}
|
||||
|
||||
function getComparableAttachmentSignature(message: Pick<RawMessage, '_attachedFiles'>): string {
|
||||
@@ -620,6 +646,13 @@ function matchesOptimisticUserMessage(
|
||||
if (sameText && sameAttachments) return true;
|
||||
if (sameText && (!optimisticAttachments || !candidateAttachments) && (timestampMatches || !hasCandidateTimestamp)) return true;
|
||||
if (sameAttachments && (!optimisticText || !candidateText) && (timestampMatches || !hasCandidateTimestamp)) return true;
|
||||
|
||||
const optimisticHadAttachmentsOnly = optimisticAttachments.length > 0 && !optimisticText;
|
||||
const candidateIsAttachmentEcho = !candidateText
|
||||
&& /\[(?:media attached:|\s*Image\s*\])/i.test(getMessageText(candidate.content));
|
||||
if (optimisticHadAttachmentsOnly && candidateIsAttachmentEcho && (timestampMatches || !hasCandidateTimestamp)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -768,6 +801,13 @@ function getMessageText(content: unknown): string {
|
||||
return '';
|
||||
}
|
||||
|
||||
function getMessageTextForFilter(msg: { content?: unknown; text?: unknown }): string {
|
||||
const fromContent = getMessageText(msg.content);
|
||||
if (fromContent.trim()) return fromContent;
|
||||
if (typeof msg.text === 'string') return msg.text;
|
||||
return '';
|
||||
}
|
||||
|
||||
function getMessageStopReason(message: RawMessage | unknown): string | null {
|
||||
if (!message || typeof message !== 'object') return null;
|
||||
const msg = message as Record<string, unknown>;
|
||||
@@ -853,6 +893,26 @@ function mimeFromExtension(filePath: string): string {
|
||||
return map[ext] || 'application/octet-stream';
|
||||
}
|
||||
|
||||
function extractFilePathsFromToolArgs(args: Record<string, unknown>): string[] {
|
||||
const paths: string[] = [];
|
||||
const direct = args.file_path ?? args.filePath ?? args.path ?? args.file;
|
||||
if (typeof direct === 'string' && direct.trim()) paths.push(direct.trim());
|
||||
|
||||
const attachments = args.attachments;
|
||||
if (Array.isArray(attachments)) {
|
||||
for (const item of attachments) {
|
||||
if (!item || typeof item !== 'object') continue;
|
||||
const att = item as Record<string, unknown>;
|
||||
const filePath = att.filePath ?? att.file_path ?? att.path ?? att.file;
|
||||
if (typeof filePath === 'string' && filePath.trim()) {
|
||||
paths.push(filePath.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return paths;
|
||||
}
|
||||
|
||||
const DIRECTORY_MIME_TYPE = 'application/x-directory';
|
||||
|
||||
function trimPathTerminators(filePath: string): string {
|
||||
@@ -1018,8 +1078,8 @@ function getToolCallFilePath(msg: RawMessage, toolCallId: string): string | unde
|
||||
if ((block.type === 'tool_use' || block.type === 'toolCall') && block.id === toolCallId) {
|
||||
const args = (block.input ?? block.arguments) as Record<string, unknown> | undefined;
|
||||
if (args) {
|
||||
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
|
||||
if (typeof fp === 'string') return fp;
|
||||
const paths = extractFilePathsFromToolArgs(args);
|
||||
if (paths[0]) return paths[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1037,8 +1097,8 @@ function getToolCallFilePath(msg: RawMessage, toolCallId: string): string | unde
|
||||
args = typeof fn.arguments === 'string' ? JSON.parse(fn.arguments) : (fn.arguments ?? fn.input) as Record<string, unknown>;
|
||||
} catch { /* ignore */ }
|
||||
if (args) {
|
||||
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
|
||||
if (typeof fp === 'string') return fp;
|
||||
const paths = extractFilePathsFromToolArgs(args);
|
||||
if (paths[0]) return paths[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1056,8 +1116,8 @@ function collectToolCallPaths(msg: RawMessage, paths: Map<string, string>): void
|
||||
if ((block.type === 'tool_use' || block.type === 'toolCall') && block.id) {
|
||||
const args = (block.input ?? block.arguments) as Record<string, unknown> | undefined;
|
||||
if (args) {
|
||||
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
|
||||
if (typeof fp === 'string') paths.set(block.id, fp);
|
||||
const filePaths = extractFilePathsFromToolArgs(args);
|
||||
if (filePaths[0]) paths.set(block.id, filePaths[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1074,8 +1134,8 @@ function collectToolCallPaths(msg: RawMessage, paths: Map<string, string>): void
|
||||
args = typeof fn.arguments === 'string' ? JSON.parse(fn.arguments) : (fn.arguments ?? fn.input) as Record<string, unknown>;
|
||||
} catch { /* ignore */ }
|
||||
if (args) {
|
||||
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
|
||||
if (typeof fp === 'string') paths.set(id, fp);
|
||||
const filePaths = extractFilePathsFromToolArgs(args);
|
||||
if (filePaths[0]) paths.set(id, filePaths[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1613,11 +1673,12 @@ function isToolResultRole(role: unknown): boolean {
|
||||
}
|
||||
|
||||
/** True for internal plumbing messages that should never be shown in the UI. */
|
||||
function isInternalMessage(msg: { role?: unknown; content?: unknown; idempotencyKey?: unknown; model?: unknown }): boolean {
|
||||
function isInternalMessage(msg: { role?: unknown; content?: unknown; idempotencyKey?: unknown; model?: unknown; text?: unknown }): boolean {
|
||||
if (msg.role === 'system') return true;
|
||||
const text = getMessageText(msg.content);
|
||||
const text = getMessageTextForFilter(msg);
|
||||
if (msg.role === 'assistant') {
|
||||
if (/^(HEARTBEAT_OK|NO_REPLY)\s*$/.test(text)) return true;
|
||||
if (isInternalAssistantReplyText(text)) return true;
|
||||
if (isGeneratingStatusNarration(text)) return true;
|
||||
// OpenClaw's gateway writes a fallback `assistant-media` transcript
|
||||
// message when its `createManagedOutgoingImageBlocks` pipeline fails
|
||||
// ("could not be prepared" warning in stderr). The fallback has:
|
||||
@@ -1642,6 +1703,12 @@ function isInternalMessage(msg: { role?: unknown; content?: unknown; idempotency
|
||||
// canonical render — keep them. Only hide the text-only fallback.
|
||||
if (!hasImageUrlBlock) return true;
|
||||
}
|
||||
if (!text.trim() && Array.isArray(msg.content)) {
|
||||
const blocks = msg.content as ContentBlock[];
|
||||
const hasThinking = blocks.some((block) => block.type === 'thinking' && block.thinking?.trim());
|
||||
const hasVisibleText = blocks.some((block) => block.type === 'text' && block.text?.trim());
|
||||
if (hasThinking && !hasVisibleText) return true;
|
||||
}
|
||||
}
|
||||
if (msg.role === 'user' && /^\[OpenClaw heartbeat poll\]\s*$/i.test(text.trim())) return true;
|
||||
// Runtime system injections: these arrive as user or assistant-role messages
|
||||
@@ -1668,6 +1735,9 @@ function isRuntimeSystemInjection(text: string): boolean {
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (/^\[Inter-session message\]/i.test(normalized)) return true;
|
||||
if (isOpenClawRuntimeEventPrompt(normalized)) return true;
|
||||
|
||||
if (
|
||||
/^\s*Current time\s*:/i.test(normalized)
|
||||
&& /^\s*Current time\s*:[^\n]*\/\s*\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+UTC\s*$/i.test(normalized)
|
||||
@@ -2631,7 +2701,8 @@ export const useChatStore = create<ChatState>((set, get) => ({
|
||||
|
||||
// Before filtering: attach images/files from tool_result messages to the next assistant message
|
||||
const messagesWithToolImages = enrichWithToolResultFiles(rawMessages);
|
||||
const filteredMessages = messagesWithToolImages.filter((msg) => !isToolResultRole(msg.role) && !isInternalMessage(msg));
|
||||
const messagesWithToolAttachments = enrichWithToolCallAttachments(messagesWithToolImages);
|
||||
const filteredMessages = messagesWithToolAttachments.filter((msg) => !shouldDropMessageFromHistory(msg));
|
||||
// Restore file attachments for user/assistant messages (from cache + text patterns)
|
||||
const enrichedMessages = enrichWithCachedImages(filteredMessages);
|
||||
|
||||
@@ -3024,7 +3095,8 @@ export const useChatStore = create<ChatState>((set, get) => ({
|
||||
// window with a larger suffix from the transcript. This keeps render
|
||||
// cost bounded while allowing long conversations to page backwards.
|
||||
const messagesWithToolImages = enrichWithToolResultFiles(rawMessages);
|
||||
const filteredMessages = messagesWithToolImages.filter((msg) => !isToolResultRole(msg.role) && !isInternalMessage(msg));
|
||||
const messagesWithToolAttachments = enrichWithToolCallAttachments(messagesWithToolImages);
|
||||
const filteredMessages = messagesWithToolAttachments.filter((msg) => !shouldDropMessageFromHistory(msg));
|
||||
const enrichedMessages = enrichWithCachedImages(filteredMessages);
|
||||
set({
|
||||
messages: enrichedMessages,
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
import { invokeIpc } from '@/lib/api-client';
|
||||
import {
|
||||
isGeneratingStatusNarration,
|
||||
isInternalAssistantReplyText,
|
||||
isOpenClawRuntimeEventPrompt,
|
||||
} from '@/pages/Chat/message-utils';
|
||||
import type { AttachedFileMeta, ChatSession, ContentBlock, RawMessage, ToolStatus } from './types';
|
||||
|
||||
// Module-level timestamp tracking the last chat event received.
|
||||
@@ -186,8 +191,25 @@ function normalizeStreamingMessage(message: unknown): unknown {
|
||||
* `[media attached:` instead — leaving the timestamp in the normalized
|
||||
* comparison text and breaking optimistic-vs-echo dedupe.
|
||||
*/
|
||||
function stripInboundMediaVisionEnvelope(text: string): string {
|
||||
if (!/\[Image\]/i.test(text) && !/^User text:/im.test(text) && !/\nDescription:\s*\n/i.test(text)) {
|
||||
return text;
|
||||
}
|
||||
|
||||
let result = text.replace(/^\s*\[Image\]\s*\n?/i, '');
|
||||
|
||||
const userTextBlock = result.match(/^User text:\s*\n([\s\S]*?)(?:\n\s*Description:\s*\n[\s\S]*)?\s*$/i);
|
||||
if (userTextBlock) {
|
||||
const userText = userTextBlock[1].trim();
|
||||
return /^Process the attached file\(s\)\.\s*$/i.test(userText) ? '' : userText;
|
||||
}
|
||||
|
||||
return result.replace(/\n\s*Description:\s*\n[\s\S]*$/i, '').trim();
|
||||
}
|
||||
|
||||
function stripGatewayUserMetadata(text: string): string {
|
||||
return text
|
||||
return stripInboundMediaVisionEnvelope(
|
||||
text
|
||||
.replace(/\s*\[media attached:[^\]]*\]/g, '')
|
||||
.replace(/\s*\[message_id:\s*[^\]]+\]/g, '')
|
||||
.replace(/^Sender\s*\([^)]*\)\s*:\s*```[a-z]*\n[\s\S]*?```\s*/i, '')
|
||||
@@ -198,13 +220,16 @@ function stripGatewayUserMetadata(text: string): string {
|
||||
.replace(/^Sender\s*:\s*[^\n]*(?:\n\s*)*/i, '')
|
||||
.replace(/^Conversation info\s*\([^)]*\):\s*```[a-z]*\n[\s\S]*?```\s*/i, '')
|
||||
.replace(/^Conversation info\s*\([^)]*\):\s*\{[\s\S]*?\}\s*/i, '')
|
||||
.replace(/^\s*\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, '');
|
||||
.replace(/^\s*\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, ''),
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeComparableUserText(content: unknown): string {
|
||||
return stripGatewayUserMetadata(getMessageText(content))
|
||||
const text = stripGatewayUserMetadata(getMessageText(content))
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
if (/^\(file attached\)$/i.test(text)) return '';
|
||||
return text;
|
||||
}
|
||||
|
||||
function getComparableAttachmentSignature(message: Pick<RawMessage, '_attachedFiles'>): string {
|
||||
@@ -239,6 +264,13 @@ function matchesOptimisticUserMessage(
|
||||
if (sameText && sameAttachments) return true;
|
||||
if (sameText && (!optimisticAttachments || !candidateAttachments) && (timestampMatches || !hasCandidateTimestamp)) return true;
|
||||
if (sameAttachments && (!optimisticText || !candidateText) && (timestampMatches || !hasCandidateTimestamp)) return true;
|
||||
|
||||
const optimisticHadAttachmentsOnly = optimisticAttachments.length > 0 && !optimisticText;
|
||||
const candidateIsAttachmentEcho = !candidateText
|
||||
&& /\[(?:media attached:|\s*Image\s*\])/i.test(getMessageText(candidate.content));
|
||||
if (optimisticHadAttachmentsOnly && candidateIsAttachmentEcho && (timestampMatches || !hasCandidateTimestamp)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -399,6 +431,13 @@ function getMessageText(content: unknown): string {
|
||||
return '';
|
||||
}
|
||||
|
||||
function getMessageTextForFilter(msg: { content?: unknown; text?: unknown }): string {
|
||||
const fromContent = getMessageText(msg.content);
|
||||
if (fromContent.trim()) return fromContent;
|
||||
if (typeof msg.text === 'string') return msg.text;
|
||||
return '';
|
||||
}
|
||||
|
||||
function getMessageStopReason(message: RawMessage | unknown): string | null {
|
||||
if (!message || typeof message !== 'object') return null;
|
||||
const msg = message as Record<string, unknown>;
|
||||
@@ -486,12 +525,131 @@ function mimeFromExtension(filePath: string): string {
|
||||
return map[ext] || 'application/octet-stream';
|
||||
}
|
||||
|
||||
/** Extract local file paths declared in tool call arguments. */
|
||||
function extractFilePathsFromToolArgs(args: Record<string, unknown>): string[] {
|
||||
const paths: string[] = [];
|
||||
const direct = args.file_path ?? args.filePath ?? args.path ?? args.file;
|
||||
if (typeof direct === 'string' && direct.trim()) paths.push(direct.trim());
|
||||
|
||||
const attachments = args.attachments;
|
||||
if (Array.isArray(attachments)) {
|
||||
for (const item of attachments) {
|
||||
if (!item || typeof item !== 'object') continue;
|
||||
const att = item as Record<string, unknown>;
|
||||
const filePath = att.filePath ?? att.file_path ?? att.path ?? att.file;
|
||||
if (typeof filePath === 'string' && filePath.trim()) {
|
||||
paths.push(filePath.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return paths;
|
||||
}
|
||||
|
||||
/**
|
||||
* Surface user-facing attachments declared in assistant tool calls (e.g.
|
||||
* `message` tool `attachments: [{ filePath }]`) on the calling turn itself.
|
||||
*/
|
||||
function enrichWithToolCallAttachments(messages: RawMessage[]): RawMessage[] {
|
||||
return messages.map((msg) => {
|
||||
if (msg.role !== 'assistant') return msg;
|
||||
|
||||
const attachmentPaths = new Set<string>();
|
||||
const content = msg.content;
|
||||
if (Array.isArray(content)) {
|
||||
for (const block of content as ContentBlock[]) {
|
||||
if (block.type !== 'tool_use' && block.type !== 'toolCall') continue;
|
||||
const args = (block.input ?? block.arguments) as Record<string, unknown> | undefined;
|
||||
if (!args) continue;
|
||||
for (const filePath of extractFilePathsFromToolArgs(args)) {
|
||||
attachmentPaths.add(filePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const msgAny = msg as unknown as Record<string, unknown>;
|
||||
const toolCalls = msgAny.tool_calls ?? msgAny.toolCalls;
|
||||
if (Array.isArray(toolCalls)) {
|
||||
for (const tc of toolCalls as Array<Record<string, unknown>>) {
|
||||
const fn = (tc.function ?? tc) as Record<string, unknown>;
|
||||
let args: Record<string, unknown> | undefined;
|
||||
try {
|
||||
args = typeof fn.arguments === 'string'
|
||||
? JSON.parse(fn.arguments)
|
||||
: (fn.arguments ?? fn.input) as Record<string, unknown>;
|
||||
} catch { /* ignore */ }
|
||||
if (!args) continue;
|
||||
for (const filePath of extractFilePathsFromToolArgs(args)) {
|
||||
attachmentPaths.add(filePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (attachmentPaths.size === 0) return msg;
|
||||
|
||||
const existingPaths = new Set(
|
||||
(msg._attachedFiles || []).map((file) => file.filePath).filter(Boolean),
|
||||
);
|
||||
const newFiles = [...attachmentPaths]
|
||||
.filter((filePath) => !existingPaths.has(filePath))
|
||||
.map((filePath) => makeAttachedFile({ filePath, mimeType: mimeFromExtension(filePath) }, 'tool-result'));
|
||||
|
||||
if (newFiles.length === 0) return msg;
|
||||
return {
|
||||
...msg,
|
||||
_attachedFiles: [...(msg._attachedFiles || []), ...newFiles],
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
const DIRECTORY_MIME_TYPE = 'application/x-directory';
|
||||
|
||||
function trimPathTerminators(filePath: string): string {
|
||||
return filePath.replace(/[,。;;,.!?]+$/u, '');
|
||||
}
|
||||
|
||||
type MarkdownImageRef =
|
||||
| { filePath: string; mimeType: string; fileName: string }
|
||||
| { gatewayUrl: string; mimeType: string; fileName: string; source: 'gateway-media' };
|
||||
|
||||
/** Extract image targets from markdown `` in assistant text. */
|
||||
function extractMarkdownImageRefs(text: string): MarkdownImageRef[] {
|
||||
if (!text) return [];
|
||||
const refs: MarkdownImageRef[] = [];
|
||||
const seen = new Set<string>();
|
||||
const markdownImageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = markdownImageRegex.exec(text)) !== null) {
|
||||
const alt = match[1]?.trim() || 'image';
|
||||
let target = match[2]?.trim() ?? '';
|
||||
if (!target) continue;
|
||||
if (target.startsWith('file://')) {
|
||||
target = decodeURIComponent(target.replace(/^file:\/\//, ''));
|
||||
}
|
||||
if (target.startsWith('/api/chat/media/')) {
|
||||
if (seen.has(target)) continue;
|
||||
seen.add(target);
|
||||
refs.push({
|
||||
gatewayUrl: target,
|
||||
mimeType: 'image/png',
|
||||
fileName: alt,
|
||||
source: 'gateway-media',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const normalizedPath = trimPathTerminators(target);
|
||||
if (!normalizedPath.startsWith('/') && !normalizedPath.startsWith('~/')) continue;
|
||||
if (seen.has(normalizedPath)) continue;
|
||||
seen.add(normalizedPath);
|
||||
refs.push({
|
||||
filePath: normalizedPath,
|
||||
mimeType: mimeFromExtension(normalizedPath),
|
||||
fileName: alt,
|
||||
});
|
||||
}
|
||||
return refs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract raw file paths from message text.
|
||||
* Detects absolute paths (Unix: / or ~/, Windows: C:\ etc.) ending with common file extensions.
|
||||
@@ -661,8 +819,8 @@ function getToolCallFilePath(msg: RawMessage, toolCallId: string): string | unde
|
||||
if ((block.type === 'tool_use' || block.type === 'toolCall') && block.id === toolCallId) {
|
||||
const args = (block.input ?? block.arguments) as Record<string, unknown> | undefined;
|
||||
if (args) {
|
||||
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
|
||||
if (typeof fp === 'string') return fp;
|
||||
const paths = extractFilePathsFromToolArgs(args);
|
||||
if (paths[0]) return paths[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -680,8 +838,8 @@ function getToolCallFilePath(msg: RawMessage, toolCallId: string): string | unde
|
||||
args = typeof fn.arguments === 'string' ? JSON.parse(fn.arguments) : (fn.arguments ?? fn.input) as Record<string, unknown>;
|
||||
} catch { /* ignore */ }
|
||||
if (args) {
|
||||
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
|
||||
if (typeof fp === 'string') return fp;
|
||||
const paths = extractFilePathsFromToolArgs(args);
|
||||
if (paths[0]) return paths[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -699,8 +857,8 @@ function collectToolCallPaths(msg: RawMessage, paths: Map<string, string>): void
|
||||
if ((block.type === 'tool_use' || block.type === 'toolCall') && block.id) {
|
||||
const args = (block.input ?? block.arguments) as Record<string, unknown> | undefined;
|
||||
if (args) {
|
||||
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
|
||||
if (typeof fp === 'string') paths.set(block.id, fp);
|
||||
const filePaths = extractFilePathsFromToolArgs(args);
|
||||
if (filePaths[0]) paths.set(block.id, filePaths[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -717,8 +875,8 @@ function collectToolCallPaths(msg: RawMessage, paths: Map<string, string>): void
|
||||
args = typeof fn.arguments === 'string' ? JSON.parse(fn.arguments) : (fn.arguments ?? fn.input) as Record<string, unknown>;
|
||||
} catch { /* ignore */ }
|
||||
if (args) {
|
||||
const fp = args.file_path ?? args.filePath ?? args.path ?? args.file;
|
||||
if (typeof fp === 'string') paths.set(id, fp);
|
||||
const filePaths = extractFilePathsFromToolArgs(args);
|
||||
if (filePaths[0]) paths.set(id, filePaths[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -744,7 +902,6 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
|
||||
}
|
||||
|
||||
if (isToolResultRole(msg.role)) {
|
||||
// Resolve file path from the matching tool call
|
||||
const matchedPath = msg.toolCallId ? toolCallPaths.get(msg.toolCallId) : undefined;
|
||||
|
||||
// 1. Image/file content blocks in the structured content array.
|
||||
@@ -788,6 +945,11 @@ function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
|
||||
}
|
||||
|
||||
if (msg.role === 'assistant' && pending.length > 0) {
|
||||
// Internal-only turns (NO_REPLY, interim narration, ...) must not consume
|
||||
// pending attachments — the next visible assistant reply should get them.
|
||||
if (isInternalMessage(msg) && !messageHasToolUse(msg)) {
|
||||
return msg;
|
||||
}
|
||||
const toAttach = pending.splice(0);
|
||||
// Deduplicate against files already on the assistant message
|
||||
const existingPaths = new Set(
|
||||
@@ -857,6 +1019,13 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
|
||||
if (msg.role === 'assistant' && !isToolOnlyMessage(msg)) {
|
||||
// Own text
|
||||
rawRefs = extractRawFilePaths(text).filter(r => !mediaRefPaths.has(r.filePath));
|
||||
const rawPathSet = new Set(rawRefs.map((ref) => ref.filePath));
|
||||
for (const ref of extractMarkdownImageRefs(text)) {
|
||||
if ('filePath' in ref && !mediaRefPaths.has(ref.filePath) && !rawPathSet.has(ref.filePath)) {
|
||||
rawPathSet.add(ref.filePath);
|
||||
rawRefs.push({ filePath: ref.filePath, mimeType: ref.mimeType });
|
||||
}
|
||||
}
|
||||
|
||||
// Nearest preceding user message text (look back up to 5 messages)
|
||||
const seenPaths = new Set(rawRefs.map(r => r.filePath));
|
||||
@@ -884,7 +1053,16 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
|
||||
}
|
||||
|
||||
const allRefs = [...mediaRefs, ...rawRefs];
|
||||
if (allRefs.length === 0 && gatewayMediaFiles.length === 0) return msg;
|
||||
const markdownImageRefs = msg.role === 'assistant' && !isToolOnlyMessage(msg)
|
||||
? extractMarkdownImageRefs(text)
|
||||
: [];
|
||||
if (
|
||||
allRefs.length === 0
|
||||
&& gatewayMediaFiles.length === 0
|
||||
&& markdownImageRefs.length === 0
|
||||
) {
|
||||
return msg;
|
||||
}
|
||||
|
||||
const existingFiles = msg._attachedFiles || [];
|
||||
const existingPaths = new Set(existingFiles.map(file => file.filePath).filter(Boolean));
|
||||
@@ -902,8 +1080,19 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
|
||||
const dedupedGatewayMedia = gatewayMediaFiles.filter(
|
||||
file => file.gatewayUrl && !existingGatewayUrls.has(file.gatewayUrl),
|
||||
);
|
||||
if (files.length === 0 && dedupedGatewayMedia.length === 0) return msg;
|
||||
return { ...msg, _attachedFiles: [...existingFiles, ...files, ...dedupedGatewayMedia] };
|
||||
const markdownGatewayMedia: AttachedFileMeta[] = markdownImageRefs
|
||||
.filter((ref): ref is Extract<MarkdownImageRef, { gatewayUrl: string }> => 'gatewayUrl' in ref)
|
||||
.filter((ref) => ref.gatewayUrl && !existingGatewayUrls.has(ref.gatewayUrl))
|
||||
.map((ref) => ({
|
||||
fileName: ref.fileName,
|
||||
mimeType: ref.mimeType,
|
||||
fileSize: 0,
|
||||
preview: null,
|
||||
gatewayUrl: ref.gatewayUrl,
|
||||
source: 'gateway-media' as const,
|
||||
}));
|
||||
if (files.length === 0 && dedupedGatewayMedia.length === 0 && markdownGatewayMedia.length === 0) return msg;
|
||||
return { ...msg, _attachedFiles: [...existingFiles, ...files, ...dedupedGatewayMedia, ...markdownGatewayMedia] };
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1077,12 +1266,31 @@ function isToolResultRole(role: unknown): boolean {
|
||||
return normalized === 'toolresult' || normalized === 'tool_result';
|
||||
}
|
||||
|
||||
function messageHasToolUse(msg: { role?: unknown; content?: unknown; tool_calls?: unknown; toolCalls?: unknown }): boolean {
|
||||
if (msg.role !== 'assistant') return false;
|
||||
if (Array.isArray(msg.content)) {
|
||||
const blocks = msg.content as ContentBlock[];
|
||||
if (blocks.some((block) => block.type === 'tool_use' || block.type === 'toolCall')) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
const toolCalls = msg.tool_calls ?? msg.toolCalls;
|
||||
return Array.isArray(toolCalls) && toolCalls.length > 0;
|
||||
}
|
||||
|
||||
/** True for internal plumbing messages that should never be shown in the UI. */
|
||||
function isInternalMessage(msg: { role?: unknown; content?: unknown }): boolean {
|
||||
function isInternalMessage(msg: { role?: unknown; content?: unknown; text?: unknown }): boolean {
|
||||
if (msg.role === 'system') return true;
|
||||
const text = getMessageText(msg.content);
|
||||
const text = getMessageTextForFilter(msg);
|
||||
if (msg.role === 'assistant') {
|
||||
if (/^(HEARTBEAT_OK|NO_REPLY)\s*$/.test(text)) return true;
|
||||
if (isInternalAssistantReplyText(text)) return true;
|
||||
if (isGeneratingStatusNarration(text)) return true;
|
||||
if (!text.trim() && Array.isArray(msg.content)) {
|
||||
const blocks = msg.content as ContentBlock[];
|
||||
const hasThinking = blocks.some((block) => block.type === 'thinking' && block.thinking?.trim());
|
||||
const hasVisibleText = blocks.some((block) => block.type === 'text' && block.text?.trim());
|
||||
if (hasThinking && !hasVisibleText) return true;
|
||||
}
|
||||
}
|
||||
if (msg.role === 'user' && /^\[OpenClaw heartbeat poll\]\s*$/i.test(text.trim())) return true;
|
||||
// Runtime system injections: these arrive as user or assistant-role messages
|
||||
@@ -1091,6 +1299,17 @@ function isInternalMessage(msg: { role?: unknown; content?: unknown }): boolean
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* History filtering must keep assistant tool-call turns even when their visible
|
||||
* text is internal narration (e.g. "生成中,稍等" + `image_generate`). Those
|
||||
* turns power the execution graph and run lifecycle detection.
|
||||
*/
|
||||
function shouldDropMessageFromHistory(msg: { role?: unknown; content?: unknown; text?: unknown; tool_calls?: unknown; toolCalls?: unknown }): boolean {
|
||||
if (isToolResultRole(msg.role)) return true;
|
||||
if (messageHasToolUse(msg)) return false;
|
||||
return isInternalMessage(msg);
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect runtime-injected system messages that should be hidden from the chat UI.
|
||||
* These are injected by the OpenClaw runtime as user-role messages and include:
|
||||
@@ -1114,8 +1333,11 @@ function isRuntimeSystemInjection(text: string): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Standalone time injection (e.g. "Current time: Wednesday, April 22nd, 2026 - 10:06 (Asia/Shanghai) / 2026-04-22 02:06 UTC")
|
||||
// Only match when the full message is the time announcement.
|
||||
if (/^\[Inter-session message\]/i.test(normalized)) return true;
|
||||
|
||||
if (isOpenClawRuntimeEventPrompt(normalized)) return true;
|
||||
|
||||
// Standalone time injection
|
||||
if (
|
||||
/^\s*Current time\s*:/i.test(normalized)
|
||||
&& /^\s*Current time\s*:[^\n]*\/\s*\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+UTC\s*$/i.test(normalized)
|
||||
@@ -1439,7 +1661,9 @@ export {
|
||||
extractRawFilePaths,
|
||||
makeAttachedFile,
|
||||
enrichWithToolResultFiles,
|
||||
enrichWithToolCallAttachments,
|
||||
isInternalMessage,
|
||||
shouldDropMessageFromHistory,
|
||||
isToolResultRole,
|
||||
enrichWithCachedImages,
|
||||
loadMissingPreviews,
|
||||
|
||||
@@ -3,14 +3,14 @@ import { hostApiFetch } from '@/lib/host-api';
|
||||
import { useGatewayStore } from '@/stores/gateway';
|
||||
import {
|
||||
clearHistoryPoll,
|
||||
enrichWithToolCallAttachments,
|
||||
enrichWithCachedImages,
|
||||
enrichWithToolResultFiles,
|
||||
getLatestOptimisticUserMessage,
|
||||
getMessageErrorMessage,
|
||||
getMessageStopReason,
|
||||
getMessageText,
|
||||
isInternalMessage,
|
||||
isToolResultRole,
|
||||
shouldDropMessageFromHistory,
|
||||
loadMissingPreviews,
|
||||
mergePendingOptimisticUserMessages,
|
||||
dropRedundantOptimisticUserMessages,
|
||||
@@ -112,7 +112,8 @@ export function createHistoryActions(
|
||||
if (!isCurrentSession()) return false;
|
||||
// Before filtering: attach images/files from tool_result messages to the next assistant message
|
||||
const messagesWithToolImages = enrichWithToolResultFiles(rawMessages);
|
||||
const filteredMessages = messagesWithToolImages.filter((msg) => !isToolResultRole(msg.role) && !isInternalMessage(msg));
|
||||
const messagesWithToolAttachments = enrichWithToolCallAttachments(messagesWithToolImages);
|
||||
const filteredMessages = messagesWithToolAttachments.filter((msg) => !shouldDropMessageFromHistory(msg));
|
||||
// Restore file attachments for user/assistant messages (from cache + text patterns)
|
||||
const enrichedMessages = enrichWithCachedImages(filteredMessages);
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ test.describe('Image generation settings page', () => {
|
||||
await expect(page.getByTestId('image-generation-primary')).toHaveCount(0);
|
||||
await expect(page.getByTestId('image-generation-fallbacks')).toHaveCount(0);
|
||||
await expect(page.getByTestId('image-generation-save')).toBeVisible();
|
||||
await expect(page.getByTestId('image-generation-clear')).toBeDisabled();
|
||||
});
|
||||
|
||||
test('configures an independent OpenAI-compatible image endpoint', async ({ page }) => {
|
||||
@@ -45,7 +46,7 @@ test.describe('Image generation settings page', () => {
|
||||
|
||||
await expect(page.getByTestId('image-generation-settings')).toBeVisible();
|
||||
await expect(page.getByTestId('image-generation-relay-base-url')).toBeVisible();
|
||||
await page.getByTestId('image-generation-relay-base-url').fill('https://taolat.com/v1');
|
||||
await page.getByTestId('image-generation-relay-base-url').fill('https://api.example.com/v1');
|
||||
await page.getByTestId('image-generation-relay-model').fill('gpt-image-2');
|
||||
await page.getByTestId('image-generation-relay-api-key').fill('sk-test-image');
|
||||
|
||||
@@ -81,7 +82,7 @@ test.describe('Image generation settings page', () => {
|
||||
],
|
||||
openAiRelay: {
|
||||
enabled: true,
|
||||
baseUrl: 'https://taolat.com/v1',
|
||||
baseUrl: 'https://api.example.com/v1',
|
||||
model: 'gpt-image-2',
|
||||
providerKey: 'clawx-openai-image',
|
||||
apiKeyConfigured: true,
|
||||
@@ -103,4 +104,79 @@ test.describe('Image generation settings page', () => {
|
||||
await expect(page.getByTestId('image-generation-api-key-status')).not.toBeEmpty();
|
||||
await expect(page.getByTestId('image-generation-relay-api-key')).toHaveAttribute('placeholder', /.+/);
|
||||
});
|
||||
|
||||
test('clears configured image generation settings after confirmation', async ({ electronApp, page }) => {
|
||||
const configuredResponse = {
|
||||
success: true,
|
||||
config: {
|
||||
primary: 'clawx-openai-image/gpt-image-2',
|
||||
fallbacks: [],
|
||||
timeoutMs: 180000,
|
||||
},
|
||||
autoProviderFallback: false,
|
||||
defaultAgentId: 'default',
|
||||
agents: [
|
||||
{
|
||||
id: 'default',
|
||||
name: 'Default',
|
||||
isDefault: true,
|
||||
provider: null,
|
||||
configured: false,
|
||||
},
|
||||
],
|
||||
openAiRelay: {
|
||||
enabled: true,
|
||||
baseUrl: 'https://api.example.com/v1',
|
||||
model: 'gpt-image-2',
|
||||
providerKey: 'clawx-openai-image',
|
||||
apiKeyConfigured: true,
|
||||
},
|
||||
};
|
||||
const clearedResponse = {
|
||||
...configuredResponse,
|
||||
config: {
|
||||
primary: null,
|
||||
fallbacks: [],
|
||||
timeoutMs: 180000,
|
||||
},
|
||||
openAiRelay: {
|
||||
enabled: false,
|
||||
baseUrl: '',
|
||||
model: 'gpt-image-2',
|
||||
providerKey: undefined,
|
||||
apiKeyConfigured: false,
|
||||
},
|
||||
};
|
||||
|
||||
await installIpcMocks(electronApp, {
|
||||
hostApi: {
|
||||
'["/api/media/image-generation","GET"]': {
|
||||
ok: true,
|
||||
data: { status: 200, ok: true, json: configuredResponse },
|
||||
},
|
||||
'["/api/media/image-generation","PUT"]': {
|
||||
ok: true,
|
||||
data: { status: 200, ok: true, json: { success: true, ...clearedResponse } },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await expect(page.getByTestId('setup-page')).toBeVisible();
|
||||
await page.getByTestId('setup-skip-button').click();
|
||||
|
||||
await expect(page.getByTestId('main-layout')).toBeVisible();
|
||||
await unlockDeveloperMode(page);
|
||||
await page.getByTestId('sidebar-nav-image-generation').click();
|
||||
|
||||
await expect(page.getByTestId('image-generation-relay-base-url')).toHaveValue('https://api.example.com/v1');
|
||||
await expect(page.getByTestId('image-generation-clear')).toBeEnabled();
|
||||
|
||||
await page.getByTestId('image-generation-clear').click();
|
||||
const confirmDialog = page.getByRole('dialog');
|
||||
await expect(confirmDialog).toBeVisible();
|
||||
await confirmDialog.getByRole('button', { name: 'Clear', exact: true }).click();
|
||||
|
||||
await expect(page.getByTestId('image-generation-relay-base-url')).toHaveValue('');
|
||||
await expect(page.getByTestId('image-generation-clear')).toBeDisabled();
|
||||
});
|
||||
});
|
||||
|
||||
25
tests/unit/chat-assistant-media-display.test.ts
Normal file
25
tests/unit/chat-assistant-media-display.test.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { extractText } from '@/pages/Chat/message-utils';
|
||||
|
||||
describe('assistant media path display cleanup', () => {
|
||||
it('strips bare OpenClaw media paths when the image is shown as an attachment card', () => {
|
||||
const text = [
|
||||
'宇航员图片生成完成啦 🧑🚀✨',
|
||||
'/Users/zhonghaolu/.openclaw/media/tool-image-generation/clawx-image-1---82d6c7e6-ea44-4850-a24b-9e88e1660683.png',
|
||||
].join('\n');
|
||||
|
||||
expect(extractText({ role: 'assistant', content: text })).toBe('宇航员图片生成完成啦 🧑🚀✨');
|
||||
});
|
||||
|
||||
it('still strips MEDIA: tagged OpenClaw artifact paths', () => {
|
||||
const text = 'Done:\n\nMEDIA:/Users/alice/.openclaw/media/outbound/cat---abc.png';
|
||||
|
||||
expect(extractText({ role: 'assistant', content: text })).toBe('Done:');
|
||||
});
|
||||
|
||||
it('strips markdown image syntax that cannot be rendered directly', () => {
|
||||
const text = '宇航员图片完成啦 🧑🚀✨\n\n';
|
||||
|
||||
expect(extractText({ role: 'assistant', content: text })).toBe('宇航员图片完成啦 🧑🚀✨');
|
||||
});
|
||||
});
|
||||
56
tests/unit/chat-copy-image.test.ts
Normal file
56
tests/unit/chat-copy-image.test.ts
Normal file
@@ -0,0 +1,56 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { copyImageToClipboard } from '@/pages/Chat/copy-image';
|
||||
|
||||
const readBinaryFileMock = vi.fn();
|
||||
|
||||
vi.mock('@/lib/api-client', () => ({
|
||||
readBinaryFile: (...args: unknown[]) => readBinaryFileMock(...args),
|
||||
}));
|
||||
|
||||
describe('copyImageToClipboard', () => {
|
||||
beforeEach(() => {
|
||||
readBinaryFileMock.mockReset();
|
||||
class MockClipboardItem {
|
||||
constructor(public items: Record<string, Blob>) {}
|
||||
}
|
||||
Object.assign(globalThis, { ClipboardItem: MockClipboardItem });
|
||||
Object.assign(navigator, {
|
||||
clipboard: {
|
||||
write: vi.fn(async () => undefined),
|
||||
writeText: vi.fn(async () => undefined),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('copies image bytes from a data URL preview', async () => {
|
||||
const pngBytes = Uint8Array.from([137, 80, 78, 71]);
|
||||
const base64 = btoa(String.fromCharCode(...pngBytes));
|
||||
const preview = `data:image/png;base64,${base64}`;
|
||||
|
||||
const ok = await copyImageToClipboard({
|
||||
preview,
|
||||
mimeType: 'image/png',
|
||||
});
|
||||
|
||||
expect(ok).toBe(true);
|
||||
expect(navigator.clipboard.write).toHaveBeenCalledTimes(1);
|
||||
expect(readBinaryFileMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('copies image bytes from a local file path', async () => {
|
||||
readBinaryFileMock.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
data: Uint8Array.from([1, 2, 3]),
|
||||
mimeType: 'image/png',
|
||||
});
|
||||
|
||||
const ok = await copyImageToClipboard({
|
||||
filePath: '/tmp/cat.png',
|
||||
mimeType: 'image/png',
|
||||
});
|
||||
|
||||
expect(ok).toBe(true);
|
||||
expect(readBinaryFileMock).toHaveBeenCalledWith('/tmp/cat.png');
|
||||
expect(navigator.clipboard.write).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
enrichWithToolResultFiles,
|
||||
enrichWithToolCallAttachments,
|
||||
enrichWithCachedImages,
|
||||
} from '@/stores/chat/helpers';
|
||||
import type { RawMessage } from '@/stores/chat';
|
||||
@@ -137,6 +138,38 @@ describe('enrichWithToolResultFiles', () => {
|
||||
const paths = (reply._attachedFiles ?? []).map((f) => f.filePath);
|
||||
expect(paths).toContain('/tmp/foo.pdf');
|
||||
});
|
||||
|
||||
it('skips internal NO_REPLY turns when attaching pending tool-result files', () => {
|
||||
const messages: RawMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
id: 'a1',
|
||||
content: [{ type: 'toolCall', id: 'tc1', name: 'exec', input: { command: 'echo report > /tmp/report.pdf' } }],
|
||||
},
|
||||
{
|
||||
role: 'toolresult',
|
||||
id: 't1',
|
||||
toolCallId: 'tc1',
|
||||
toolName: 'exec',
|
||||
content: [{ type: 'text', text: 'Wrote /tmp/report.pdf' }],
|
||||
},
|
||||
{
|
||||
role: 'assistant',
|
||||
id: 'no-reply',
|
||||
content: [{ type: 'text', text: 'NO_REPLY' }],
|
||||
},
|
||||
{
|
||||
role: 'assistant',
|
||||
id: 'final',
|
||||
content: [{ type: 'text', text: 'Report is ready.' }],
|
||||
},
|
||||
];
|
||||
|
||||
const enriched = enrichWithToolResultFiles(messages);
|
||||
const final = enriched.find((m) => m.id === 'final')!;
|
||||
expect(final._attachedFiles?.map((file) => file.filePath)).toEqual(['/tmp/report.pdf']);
|
||||
expect(enriched.find((m) => m.id === 'no-reply')?._attachedFiles ?? []).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('enrichWithCachedImages — Gateway media bubble dedup', () => {
|
||||
@@ -273,4 +306,68 @@ describe('enrichWithCachedImages — Gateway media bubble dedup', () => {
|
||||
const replyPaths = (reply._attachedFiles ?? []).map((f) => f.filePath);
|
||||
expect(replyPaths).toEqual(['/tmp/foo.png']);
|
||||
});
|
||||
|
||||
it('promotes markdown local image paths to attached files', () => {
|
||||
const messages: RawMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
id: 'reply',
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: '宇航员图片完成啦 🧑🚀✨\n\n',
|
||||
}],
|
||||
},
|
||||
];
|
||||
|
||||
const enriched = enrichWithCachedImages(messages);
|
||||
expect(enriched[0]?._attachedFiles?.map((file) => file.filePath)).toEqual([
|
||||
'/Users/me/.openclaw/media/tool-image-generation/cat.png',
|
||||
]);
|
||||
});
|
||||
|
||||
it('promotes markdown gateway image URLs to gateway-media attachments', () => {
|
||||
const messages: RawMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
id: 'reply',
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: 'Done\n\n',
|
||||
}],
|
||||
},
|
||||
];
|
||||
|
||||
const enriched = enrichWithCachedImages(messages);
|
||||
expect(enriched[0]?._attachedFiles?.[0]).toMatchObject({
|
||||
gatewayUrl: '/api/chat/media/outgoing/agent%3Amain%3As-1/abc/full',
|
||||
source: 'gateway-media',
|
||||
fileName: 'Astronaut with Milky Way in helmet visor',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('enrichWithToolCallAttachments', () => {
|
||||
it('attaches image paths from message tool attachments array', () => {
|
||||
const messages: RawMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
id: 'send-image',
|
||||
content: [{
|
||||
type: 'tool_use',
|
||||
id: 'tool-1',
|
||||
name: 'message',
|
||||
input: {
|
||||
action: 'send',
|
||||
attachments: [{ filePath: '/Users/me/.openclaw/media/tool-image-generation/cat.png' }],
|
||||
},
|
||||
}],
|
||||
},
|
||||
];
|
||||
|
||||
const enriched = enrichWithToolCallAttachments(messages);
|
||||
expect(enriched[0]?._attachedFiles?.map((file) => file.filePath)).toEqual([
|
||||
'/Users/me/.openclaw/media/tool-image-generation/cat.png',
|
||||
]);
|
||||
expect(enriched[0]?._attachedFiles?.[0]?.mimeType).toBe('image/png');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -7,6 +7,7 @@ const gatewayStoreGetStateMock = vi.fn();
|
||||
const clearHistoryPoll = vi.fn();
|
||||
const enrichWithCachedImages = vi.fn((messages) => messages);
|
||||
const enrichWithToolResultFiles = vi.fn((messages) => messages);
|
||||
const enrichWithToolCallAttachments = vi.fn((messages) => messages);
|
||||
const getMessageErrorMessage = vi.fn((message: { errorMessage?: string; error_message?: string } | undefined) => {
|
||||
if (!message) return null;
|
||||
return message.errorMessage ?? message.error_message ?? null;
|
||||
@@ -29,6 +30,23 @@ const isInternalMessage = vi.fn((msg: { role?: unknown; content?: unknown }) =>
|
||||
}
|
||||
return false;
|
||||
});
|
||||
const messageHasToolUse = (msg: { role?: unknown; content?: unknown; tool_calls?: unknown; toolCalls?: unknown }) => {
|
||||
if (msg.role !== 'assistant') return false;
|
||||
if (Array.isArray(msg.content)) {
|
||||
return (msg.content as Array<{ type?: string }>).some(
|
||||
(block) => block.type === 'tool_use' || block.type === 'toolCall',
|
||||
);
|
||||
}
|
||||
const toolCalls = (msg as { tool_calls?: unknown; toolCalls?: unknown }).tool_calls
|
||||
?? (msg as { toolCalls?: unknown }).toolCalls;
|
||||
return Array.isArray(toolCalls) && toolCalls.length > 0;
|
||||
};
|
||||
const shouldDropMessageFromHistory = vi.fn((msg: { role?: unknown; content?: unknown; tool_calls?: unknown; toolCalls?: unknown }) => {
|
||||
if (isToolResultRole(msg.role)) return true;
|
||||
if (messageHasToolUse(msg)) return false;
|
||||
return isInternalMessage(msg);
|
||||
});
|
||||
const setLastChatEventAt = vi.fn();
|
||||
const loadMissingPreviews = vi.fn(async () => false);
|
||||
const toMs = vi.fn((ts: number) => ts < 1e12 ? ts * 1000 : ts);
|
||||
|
||||
@@ -50,6 +68,7 @@ vi.mock('@/stores/chat/helpers', () => ({
|
||||
clearHistoryPoll: (...args: unknown[]) => clearHistoryPoll(...args),
|
||||
enrichWithCachedImages: (...args: unknown[]) => enrichWithCachedImages(...args),
|
||||
enrichWithToolResultFiles: (...args: unknown[]) => enrichWithToolResultFiles(...args),
|
||||
enrichWithToolCallAttachments: (...args: unknown[]) => enrichWithToolCallAttachments(...args),
|
||||
getLatestOptimisticUserMessage: (messages: Array<{ role: string; timestamp?: number }>, userTimestampMs: number) =>
|
||||
[...messages].reverse().find(
|
||||
(message) => message.role === 'user'
|
||||
@@ -57,9 +76,13 @@ vi.mock('@/stores/chat/helpers', () => ({
|
||||
),
|
||||
getMessageText: (...args: unknown[]) => getMessageText(...args),
|
||||
hasNonToolAssistantContent: (...args: unknown[]) => hasNonToolAssistantContent(...args),
|
||||
hasAssistantAfterLastRealUser: (messages: Array<{ role?: string }>) =>
|
||||
messages.some((message) => message.role === 'assistant'),
|
||||
isInternalMessage: (...args: unknown[]) => isInternalMessage(...args),
|
||||
shouldDropMessageFromHistory: (...args: unknown[]) => shouldDropMessageFromHistory(...args),
|
||||
isToolResultRole: (...args: unknown[]) => isToolResultRole(...args),
|
||||
loadMissingPreviews: (...args: unknown[]) => loadMissingPreviews(...args),
|
||||
setLastChatEventAt: (...args: unknown[]) => setLastChatEventAt(...args),
|
||||
mergePendingOptimisticUserMessages: (_sessionKey: string, messages: unknown[]) => messages,
|
||||
hasOptimisticServerEcho: (
|
||||
loadedMessages: Array<{ role: string; timestamp?: number; content?: unknown; _attachedFiles?: Array<{ filePath?: string; fileName?: string; mimeType?: string; fileSize?: number }> }>,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { isInternalMessage } from '@/stores/chat/helpers';
|
||||
import { isInternalMessage, shouldDropMessageFromHistory } from '@/stores/chat/helpers';
|
||||
|
||||
describe('chat internal message filter', () => {
|
||||
it('filters runtime system injection bundle like async exec completion payload', () => {
|
||||
@@ -33,4 +33,54 @@ describe('chat internal message filter', () => {
|
||||
it('filters OpenClaw heartbeat poll user turns', () => {
|
||||
expect(isInternalMessage({ role: 'user', content: '[OpenClaw heartbeat poll]' })).toBe(true);
|
||||
});
|
||||
|
||||
it('filters inter-session routing payloads', () => {
|
||||
expect(isInternalMessage({
|
||||
role: 'user',
|
||||
content: '[Inter-session message] sourceSession=image_generate:abc sourceTool=image_generate',
|
||||
})).toBe(true);
|
||||
});
|
||||
|
||||
it('filters OpenClaw runtime continuation prompts', () => {
|
||||
expect(isInternalMessage({ role: 'user', content: 'Continue the OpenClaw runtime event.' })).toBe(true);
|
||||
});
|
||||
|
||||
it('filters runtime continuation prompts stored on msg.text', () => {
|
||||
expect(isInternalMessage({
|
||||
role: 'user',
|
||||
content: [],
|
||||
text: 'Continue the OpenClaw runtime event.',
|
||||
})).toBe(true);
|
||||
});
|
||||
|
||||
it('filters interim image-generation status narration', () => {
|
||||
expect(isInternalMessage({
|
||||
role: 'assistant',
|
||||
content: '生成中,稍等 👨🚀',
|
||||
})).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps assistant tool-call turns in history even when narration is internal', () => {
|
||||
expect(shouldDropMessageFromHistory({
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{ type: 'text', text: '生成中,稍等 👨🚀' },
|
||||
{ type: 'tool_use', id: 'tool-image', name: 'image_generate', input: { prompt: 'astronaut' } },
|
||||
],
|
||||
})).toBe(false);
|
||||
expect(isInternalMessage({
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{ type: 'text', text: '生成中,稍等 👨🚀' },
|
||||
{ type: 'tool_use', id: 'tool-image', name: 'image_generate', input: { prompt: 'astronaut' } },
|
||||
],
|
||||
})).toBe(true);
|
||||
});
|
||||
|
||||
it('filters assistant turns that only contain chain-of-thought', () => {
|
||||
expect(isInternalMessage({
|
||||
role: 'assistant',
|
||||
content: [{ type: 'thinking', thinking: 'The user is asking me to continue with an OpenClaw runtime event.' }],
|
||||
})).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { RawMessage } from '@/stores/chat';
|
||||
|
||||
vi.mock('@/lib/api-client', () => ({
|
||||
invokeIpc: vi.fn(),
|
||||
readBinaryFile: vi.fn(),
|
||||
statFile: vi.fn(async (path: string) => {
|
||||
if (path.includes('missing') || path.includes('不存在')) {
|
||||
return { ok: false, error: 'notFound' };
|
||||
@@ -405,3 +406,50 @@ describe('ChatMessage reply styling', () => {
|
||||
expect(bubble).toHaveTextContent('Keep the prompt bubble.');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ChatMessage image copy', () => {
|
||||
beforeEach(() => {
|
||||
class MockClipboardItem {
|
||||
constructor(public items: Record<string, Blob>) {}
|
||||
}
|
||||
Object.assign(globalThis, { ClipboardItem: MockClipboardItem });
|
||||
Object.assign(navigator, {
|
||||
clipboard: {
|
||||
write: vi.fn(async () => undefined),
|
||||
writeText: vi.fn(async () => undefined),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('copies image bytes instead of the media URL text when an image attachment is present', async () => {
|
||||
const { readBinaryFile } = await import('@/lib/api-client');
|
||||
vi.mocked(readBinaryFile).mockResolvedValueOnce({
|
||||
ok: true,
|
||||
data: Uint8Array.from([137, 80, 78, 71]),
|
||||
mimeType: 'image/png',
|
||||
});
|
||||
|
||||
const message: RawMessage = {
|
||||
role: 'assistant',
|
||||
content: 'http://127.0.0.1:18789/api/chat/media/outgoing/agent/main/full',
|
||||
_attachedFiles: [
|
||||
{
|
||||
fileName: 'cat.png',
|
||||
mimeType: 'image/png',
|
||||
fileSize: 1234,
|
||||
preview: null,
|
||||
filePath: '/tmp/cat.png',
|
||||
source: 'tool-result',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
render(<ChatMessage message={message} />);
|
||||
|
||||
fireEvent.click(screen.getByRole('button'));
|
||||
await vi.waitFor(() => {
|
||||
expect(navigator.clipboard.write).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
expect(navigator.clipboard.writeText).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
58
tests/unit/chat-user-message-display.test.ts
Normal file
58
tests/unit/chat-user-message-display.test.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { extractText } from '@/pages/Chat/message-utils';
|
||||
import { matchesOptimisticUserMessage } from '@/stores/chat/helpers';
|
||||
|
||||
const gatewayImageEcho = [
|
||||
'Sender (untrusted metadata):',
|
||||
'```json',
|
||||
'{',
|
||||
' "label": "ClawX (gateway-client)",',
|
||||
' "id": "gateway-client",',
|
||||
' "name": "ClawX",',
|
||||
' "username": "ClawX"',
|
||||
'}',
|
||||
'```',
|
||||
'',
|
||||
'[media attached: media://inbound/image---abc.png (image/png)]',
|
||||
'[Image]',
|
||||
'User text:',
|
||||
'Process the attached file(s).',
|
||||
'[media attached: /Users/test/.openclaw/media/outbound/out.png (image/png) | /Users/test/.openclaw/media/outbound/out.png]',
|
||||
'Description:',
|
||||
'An astronaut in a white space suit floats in space, reaching a gloved hand toward the viewer.',
|
||||
].join('\n');
|
||||
|
||||
describe('user message display cleanup', () => {
|
||||
it('hides the inbound-image vision envelope for attachment-only uploads', () => {
|
||||
expect(extractText({ role: 'user', content: gatewayImageEcho })).toBe('');
|
||||
});
|
||||
|
||||
it('keeps the user caption while stripping auto-generated description', () => {
|
||||
const content = gatewayImageEcho
|
||||
.replace('Process the attached file(s).', '改成西装加领带');
|
||||
|
||||
expect(extractText({ role: 'user', content })).toBe('改成西装加领带');
|
||||
});
|
||||
|
||||
it('matches optimistic attachment-only bubbles against the gateway vision echo', () => {
|
||||
const optimistic = {
|
||||
role: 'user' as const,
|
||||
content: '(file attached)',
|
||||
timestamp: 1_700_000_000,
|
||||
_attachedFiles: [{
|
||||
fileName: 'out.png',
|
||||
mimeType: 'image/png',
|
||||
fileSize: 123,
|
||||
preview: null,
|
||||
filePath: '/Users/test/.openclaw/media/outbound/out.png',
|
||||
}],
|
||||
};
|
||||
const candidate = {
|
||||
role: 'user' as const,
|
||||
content: gatewayImageEcho,
|
||||
timestamp: 1_700_000_000,
|
||||
};
|
||||
|
||||
expect(matchesOptimisticUserMessage(candidate, optimistic, 1_700_000_000_000)).toBe(true);
|
||||
});
|
||||
});
|
||||
136
tests/unit/image-generation-status.test.ts
Normal file
136
tests/unit/image-generation-status.test.ts
Normal file
@@ -0,0 +1,136 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type { RawMessage } from '@/stores/chat';
|
||||
import {
|
||||
IMAGE_GENERATION_TIMEOUT_MS,
|
||||
isImageGenerationPending,
|
||||
} from '@/pages/Chat/image-generation-status';
|
||||
|
||||
const TASK_ID = '27443fdb-6cca-48e6-a3a7-ee34b0491aee';
|
||||
const NOW = 1_700_000_000_000;
|
||||
|
||||
describe('isImageGenerationPending', () => {
|
||||
it('returns true while waiting after image_generate without a final reply', () => {
|
||||
const segmentMessages: RawMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
timestamp: NOW / 1000,
|
||||
content: [{ type: 'toolCall', id: 'call_1', name: 'image_generate', arguments: { prompt: 'astronaut' } }],
|
||||
},
|
||||
{
|
||||
role: 'assistant',
|
||||
timestamp: NOW / 1000 + 5,
|
||||
content: [{ type: 'text', text: '图片生成中,稍等片刻 🧑🚀✨' }],
|
||||
},
|
||||
];
|
||||
|
||||
expect(isImageGenerationPending(segmentMessages, [], NOW)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false after the inter-session completion event arrives', () => {
|
||||
const segmentMessages: RawMessage[] = [
|
||||
{
|
||||
role: 'toolresult',
|
||||
toolName: 'image_generate',
|
||||
timestamp: NOW / 1000,
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: `Background task started for image generation (${TASK_ID}).`,
|
||||
}],
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: `[Inter-session message] sourceSession=image_generate:${TASK_ID} sourceTool=image_generate`,
|
||||
}],
|
||||
},
|
||||
];
|
||||
|
||||
expect(isImageGenerationPending(segmentMessages, [], NOW)).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false once a non-generating assistant reply arrives', () => {
|
||||
const segmentMessages: RawMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
timestamp: NOW / 1000,
|
||||
content: [{ type: 'toolCall', id: 'call_1', name: 'image_generate', arguments: { prompt: 'astronaut' } }],
|
||||
},
|
||||
{
|
||||
role: 'assistant',
|
||||
timestamp: NOW / 1000 + 5,
|
||||
content: [{ type: 'text', text: '图片生成中,稍等片刻 🧑🚀✨' }],
|
||||
},
|
||||
{
|
||||
role: 'assistant',
|
||||
timestamp: NOW / 1000 + 120,
|
||||
content: [{ type: 'text', text: '抱歉,图片生成超时了,稍后重试一下吧 🧑🚀' }],
|
||||
},
|
||||
];
|
||||
|
||||
expect(isImageGenerationPending(segmentMessages, [], NOW + 120_000)).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false after the configured timeout even if only generating narration exists', () => {
|
||||
const segmentMessages: RawMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
timestamp: NOW / 1000,
|
||||
content: [{ type: 'toolCall', id: 'call_1', name: 'image_generate', arguments: { prompt: 'astronaut' } }],
|
||||
},
|
||||
{
|
||||
role: 'assistant',
|
||||
timestamp: NOW / 1000 + 5,
|
||||
content: [{ type: 'text', text: '图片生成中,稍等片刻 🧑🚀✨' }],
|
||||
},
|
||||
];
|
||||
|
||||
expect(
|
||||
isImageGenerationPending(
|
||||
segmentMessages,
|
||||
[],
|
||||
NOW + IMAGE_GENERATION_TIMEOUT_MS + 20_000,
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false when a stale streaming tool status is still marked running', () => {
|
||||
expect(isImageGenerationPending([], [{
|
||||
name: 'image_generate',
|
||||
status: 'running',
|
||||
updatedAt: NOW - IMAGE_GENERATION_TIMEOUT_MS - 60_000,
|
||||
}], NOW)).toBe(false);
|
||||
});
|
||||
|
||||
it('returns true while image_generate is actively running in streaming tools', () => {
|
||||
expect(isImageGenerationPending([], [{
|
||||
name: 'image_generate',
|
||||
status: 'running',
|
||||
updatedAt: NOW,
|
||||
}], NOW)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false once an image attachment is delivered', () => {
|
||||
const segmentMessages: RawMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
timestamp: NOW / 1000,
|
||||
content: [{ type: 'toolCall', id: 'call_1', name: 'image_generate', arguments: { prompt: 'astronaut' } }],
|
||||
},
|
||||
{
|
||||
role: 'assistant',
|
||||
timestamp: NOW / 1000 + 90,
|
||||
content: [{ type: 'text', text: '宇航员图片完成啦 🧑🚀✨' }],
|
||||
_attachedFiles: [{
|
||||
fileName: 'astronaut.png',
|
||||
mimeType: 'image/png',
|
||||
fileSize: 123,
|
||||
preview: null,
|
||||
filePath: '/tmp/astronaut.png',
|
||||
}],
|
||||
},
|
||||
];
|
||||
|
||||
expect(isImageGenerationPending(segmentMessages, [], NOW + 90_000)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -4,7 +4,7 @@ import { stripProcessMessagePrefix } from '@/pages/Chat/message-utils';
|
||||
import type { RawMessage, ToolStatus } from '@/stores/chat';
|
||||
|
||||
describe('deriveTaskSteps', () => {
|
||||
it('builds running steps from streaming thinking and tool status', () => {
|
||||
it('builds running steps from streaming tool status without exposing chain-of-thought', () => {
|
||||
const streamingTools: ToolStatus[] = [
|
||||
{
|
||||
name: 'web_search',
|
||||
@@ -27,12 +27,6 @@ describe('deriveTaskSteps', () => {
|
||||
});
|
||||
|
||||
expect(steps).toEqual([
|
||||
expect.objectContaining({
|
||||
id: 'stream-thinking-0',
|
||||
label: 'Thinking',
|
||||
status: 'running',
|
||||
kind: 'thinking',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
label: 'web_search',
|
||||
status: 'running',
|
||||
@@ -160,7 +154,7 @@ describe('deriveTaskSteps', () => {
|
||||
}));
|
||||
});
|
||||
|
||||
it('keeps recent completed steps from assistant history', () => {
|
||||
it('keeps recent completed tool steps from assistant history', () => {
|
||||
const messages: RawMessage[] = [
|
||||
{
|
||||
role: 'assistant',
|
||||
@@ -179,12 +173,6 @@ describe('deriveTaskSteps', () => {
|
||||
});
|
||||
|
||||
expect(steps).toEqual([
|
||||
expect.objectContaining({
|
||||
id: 'history-thinking-assistant-1-0',
|
||||
label: 'Thinking',
|
||||
status: 'completed',
|
||||
kind: 'thinking',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
id: 'tool-2',
|
||||
label: 'read_file',
|
||||
@@ -194,7 +182,7 @@ describe('deriveTaskSteps', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('splits cumulative streaming thinking into separate execution steps', () => {
|
||||
it('does not expose streaming chain-of-thought in the execution graph', () => {
|
||||
const steps = deriveTaskSteps({
|
||||
messages: [],
|
||||
streamingMessage: {
|
||||
@@ -208,21 +196,36 @@ describe('deriveTaskSteps', () => {
|
||||
streamingTools: [],
|
||||
});
|
||||
|
||||
expect(steps).toEqual([]);
|
||||
});
|
||||
|
||||
it('skips internal assistant turns and hides NO_REPLY from the execution graph', () => {
|
||||
const steps = deriveTaskSteps({
|
||||
messages: [
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [{ type: 'thinking', thinking: 'Continue the OpenClaw runtime event internally.' }],
|
||||
},
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [{ type: 'text', text: 'NO_REPLY' }],
|
||||
},
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{ type: 'tool_use', id: 'tool-image', name: 'image_generate', input: { prompt: 'astronaut' } },
|
||||
],
|
||||
},
|
||||
],
|
||||
streamingMessage: null,
|
||||
streamingTools: [],
|
||||
});
|
||||
|
||||
expect(steps).toEqual([
|
||||
expect.objectContaining({
|
||||
id: 'stream-thinking-0',
|
||||
detail: 'Reviewing X.',
|
||||
status: 'completed',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
id: 'stream-thinking-1',
|
||||
detail: 'Comparing Y.',
|
||||
status: 'completed',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
id: 'stream-thinking-2',
|
||||
detail: 'Drafting answer.',
|
||||
status: 'running',
|
||||
id: 'tool-image',
|
||||
label: 'image_generate',
|
||||
kind: 'tool',
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user