mirror of
https://github.com/siteboon/claudecodeui.git
synced 2026-07-07 06:02:43 +08:00
feat: support attached images for all providers
This commit is contained in:
93
server/modules/assets/assets.routes.ts
Normal file
93
server/modules/assets/assets.routes.ts
Normal file
@@ -0,0 +1,93 @@
|
||||
import fsSync, { promises as fs } from 'node:fs';
|
||||
|
||||
import express from 'express';
|
||||
import mime from 'mime-types';
|
||||
import multer from 'multer';
|
||||
|
||||
import {
|
||||
buildStoredImageRecords,
|
||||
ensureImageAssetsDir,
|
||||
isAllowedImageMimeType,
|
||||
resolveImageAssetFile,
|
||||
} from '@/modules/assets/services/image-assets.service.js';
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// Multer writes uploads straight into the global assets folder; the service
|
||||
// owns the folder location and the response record shape.
|
||||
const storage = multer.diskStorage({
|
||||
destination: (req, file, cb) => {
|
||||
ensureImageAssetsDir()
|
||||
.then((assetsDir) => cb(null, assetsDir))
|
||||
.catch((error) => cb(error as Error, ''));
|
||||
},
|
||||
filename: (req, file, cb) => {
|
||||
const uniqueSuffix = `${Date.now()}-${Math.round(Math.random() * 1e9)}`;
|
||||
const sanitizedName = file.originalname.replace(/[^a-zA-Z0-9.-]/g, '_');
|
||||
cb(null, `${uniqueSuffix}-${sanitizedName}`);
|
||||
},
|
||||
});
|
||||
|
||||
const upload = multer({
|
||||
storage,
|
||||
fileFilter: (req, file, cb) => {
|
||||
if (isAllowedImageMimeType(file.mimetype)) {
|
||||
cb(null, true);
|
||||
} else {
|
||||
cb(new Error('Invalid file type. Only JPEG, PNG, GIF, WebP, and SVG are allowed.'));
|
||||
}
|
||||
},
|
||||
limits: {
|
||||
fileSize: 5 * 1024 * 1024, // 5MB
|
||||
files: 5,
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Stores chat image attachments in the global `~/.cloudcli/assets` folder and
|
||||
* returns their absolute paths for use in provider prompts and chat history.
|
||||
*/
|
||||
router.post('/images', (req, res) => {
|
||||
upload.array('images', 5)(req, res, (err: unknown) => {
|
||||
if (err) {
|
||||
const message = err instanceof Error ? err.message : 'Upload failed';
|
||||
return res.status(400).json({ error: message });
|
||||
}
|
||||
|
||||
const files = Array.isArray(req.files) ? req.files : [];
|
||||
if (files.length === 0) {
|
||||
return res.status(400).json({ error: 'No image files provided' });
|
||||
}
|
||||
|
||||
res.json({ images: buildStoredImageRecords(files) });
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Serves one stored image asset by filename. Only files directly inside the
|
||||
* global assets folder are reachable; traversal attempts resolve to null.
|
||||
*/
|
||||
router.get('/images/:filename', async (req, res) => {
|
||||
const resolved = resolveImageAssetFile(req.params.filename);
|
||||
if (!resolved) {
|
||||
return res.status(400).json({ error: 'Invalid asset filename' });
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.access(resolved);
|
||||
} catch {
|
||||
return res.status(404).json({ error: 'Asset not found' });
|
||||
}
|
||||
|
||||
res.setHeader('Content-Type', mime.lookup(resolved) || 'application/octet-stream');
|
||||
const fileStream = fsSync.createReadStream(resolved);
|
||||
fileStream.pipe(res);
|
||||
fileStream.on('error', (error) => {
|
||||
console.error('Error streaming image asset:', error);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).json({ error: 'Error reading asset' });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
export default router;
|
||||
3
server/modules/assets/index.ts
Normal file
3
server/modules/assets/index.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
// Express router mounted at /api/assets by server/index.js (upload + serving
|
||||
// of chat image attachments stored in the global ~/.cloudcli/assets folder).
|
||||
export { default as assetsRoutes } from './assets.routes.js';
|
||||
82
server/modules/assets/services/image-assets.service.ts
Normal file
82
server/modules/assets/services/image-assets.service.ts
Normal file
@@ -0,0 +1,82 @@
|
||||
import { promises as fs } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import { getGlobalImageAssetsDir, toPosixPath } from '@/shared/image-attachments.js';
|
||||
|
||||
/**
|
||||
* Image mime types accepted for chat attachment uploads. SVG is allowed for
|
||||
* storage/preview even though some providers (Claude API) skip it at send time.
|
||||
*/
|
||||
const ALLOWED_IMAGE_MIME_TYPES = new Set([
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'image/gif',
|
||||
'image/webp',
|
||||
'image/svg+xml',
|
||||
]);
|
||||
|
||||
// Used only by this service and the assets routes via the barrel file.
|
||||
type StoredImageAsset = {
|
||||
/** Original upload filename, for display. */
|
||||
name: string;
|
||||
/** Absolute posix-normalized path inside the global assets folder. */
|
||||
path: string;
|
||||
size: number;
|
||||
mimeType: string;
|
||||
};
|
||||
|
||||
// Shape of one multer-stored file; kept local because only this module reads it.
|
||||
type UploadedImageFile = {
|
||||
originalname: string;
|
||||
filename: string;
|
||||
size: number;
|
||||
mimetype: string;
|
||||
};
|
||||
|
||||
/** Returns whether one uploaded mime type may be stored as a chat image asset. */
|
||||
export function isAllowedImageMimeType(mimeType: string): boolean {
|
||||
return ALLOWED_IMAGE_MIME_TYPES.has(mimeType);
|
||||
}
|
||||
|
||||
/** Creates the global `~/.cloudcli/assets` folder if needed and returns it. */
|
||||
export async function ensureImageAssetsDir(): Promise<string> {
|
||||
const assetsDir = getGlobalImageAssetsDir();
|
||||
await fs.mkdir(assetsDir, { recursive: true });
|
||||
return assetsDir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps multer-stored upload files to the attachment records returned to the
|
||||
* chat composer. The absolute path is what providers receive and what session
|
||||
* history carries back to the UI.
|
||||
*/
|
||||
export function buildStoredImageRecords(files: UploadedImageFile[]): StoredImageAsset[] {
|
||||
const assetsDir = getGlobalImageAssetsDir();
|
||||
return files.map((file) => ({
|
||||
name: file.originalname,
|
||||
path: toPosixPath(path.join(assetsDir, file.filename)),
|
||||
size: file.size,
|
||||
mimeType: file.mimetype,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves one asset filename to its absolute path inside the global assets
|
||||
* folder, or null when the name is empty, contains path separators/traversal,
|
||||
* or would escape the folder. This is the only lookup the serving route uses,
|
||||
* so nothing outside `~/.cloudcli/assets` can ever be read through it.
|
||||
*/
|
||||
export function resolveImageAssetFile(filename: string): string | null {
|
||||
const trimmed = typeof filename === 'string' ? filename.trim() : '';
|
||||
if (!trimmed || trimmed.includes('/') || trimmed.includes('\\') || trimmed.includes('..')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const assetsDir = path.resolve(getGlobalImageAssetsDir());
|
||||
const resolved = path.resolve(assetsDir, trimmed);
|
||||
if (!resolved.startsWith(assetsDir + path.sep)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return resolved;
|
||||
}
|
||||
46
server/modules/assets/tests/image-assets.service.test.ts
Normal file
46
server/modules/assets/tests/image-assets.service.test.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
|
||||
import {
|
||||
buildStoredImageRecords,
|
||||
isAllowedImageMimeType,
|
||||
resolveImageAssetFile,
|
||||
} from '@/modules/assets/services/image-assets.service.js';
|
||||
|
||||
const ASSETS_DIR = path.join(os.homedir(), '.cloudcli', 'assets');
|
||||
|
||||
test('isAllowedImageMimeType accepts image formats and rejects the rest', () => {
|
||||
assert.equal(isAllowedImageMimeType('image/png'), true);
|
||||
assert.equal(isAllowedImageMimeType('image/svg+xml'), true);
|
||||
assert.equal(isAllowedImageMimeType('application/pdf'), false);
|
||||
assert.equal(isAllowedImageMimeType('text/html'), false);
|
||||
});
|
||||
|
||||
test('buildStoredImageRecords returns absolute posix paths in the assets dir', () => {
|
||||
const records = buildStoredImageRecords([
|
||||
{ originalname: 'shot.png', filename: '123-456-shot.png', size: 42, mimetype: 'image/png' },
|
||||
]);
|
||||
|
||||
assert.equal(records.length, 1);
|
||||
assert.equal(records[0].name, 'shot.png');
|
||||
assert.equal(records[0].size, 42);
|
||||
assert.equal(records[0].mimeType, 'image/png');
|
||||
assert.equal(records[0].path, `${ASSETS_DIR.replace(/\\/g, '/')}/123-456-shot.png`);
|
||||
});
|
||||
|
||||
test('resolveImageAssetFile resolves plain filenames inside the assets dir', () => {
|
||||
const resolved = resolveImageAssetFile('123-shot.png');
|
||||
assert.equal(resolved, path.join(path.resolve(ASSETS_DIR), '123-shot.png'));
|
||||
});
|
||||
|
||||
test('resolveImageAssetFile rejects traversal and separator attempts', () => {
|
||||
assert.equal(resolveImageAssetFile(''), null);
|
||||
assert.equal(resolveImageAssetFile(' '), null);
|
||||
assert.equal(resolveImageAssetFile('../auth.db'), null);
|
||||
assert.equal(resolveImageAssetFile('..'), null);
|
||||
assert.equal(resolveImageAssetFile('sub/dir.png'), null);
|
||||
assert.equal(resolveImageAssetFile('sub\\dir.png'), null);
|
||||
assert.equal(resolveImageAssetFile('a..b/../c.png'), null);
|
||||
});
|
||||
@@ -313,6 +313,18 @@ export class ClaudeSessionsProvider implements IProviderSessions {
|
||||
|
||||
if (raw.message?.role === 'user' && raw.message?.content && raw.isMeta !== true) {
|
||||
if (Array.isArray(raw.message.content)) {
|
||||
// Image attachments sent through the SDK are persisted as base64
|
||||
// `image` blocks next to the prompt text. Collect them so the UI can
|
||||
// render them on the user bubble.
|
||||
const imageAttachments: Array<{ data: string }> = [];
|
||||
for (const part of raw.message.content) {
|
||||
if (part?.type === 'image' && part.source?.type === 'base64' && typeof part.source.data === 'string') {
|
||||
const mediaType = typeof part.source.media_type === 'string' ? part.source.media_type : 'image/png';
|
||||
imageAttachments.push({ data: `data:${mediaType};base64,${part.source.data}` });
|
||||
}
|
||||
}
|
||||
let imagesAttached = false;
|
||||
|
||||
for (let partIndex = 0; partIndex < raw.message.content.length; partIndex++) {
|
||||
const part = raw.message.content[partIndex];
|
||||
if (part.type === 'tool_result') {
|
||||
@@ -339,7 +351,9 @@ export class ClaudeSessionsProvider implements IProviderSessions {
|
||||
kind: 'text',
|
||||
role: 'user',
|
||||
content: text,
|
||||
images: !imagesAttached && imageAttachments.length > 0 ? imageAttachments : undefined,
|
||||
}));
|
||||
imagesAttached = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -359,9 +373,25 @@ export class ClaudeSessionsProvider implements IProviderSessions {
|
||||
kind: 'text',
|
||||
role: 'user',
|
||||
content: textParts,
|
||||
images: imageAttachments.length > 0 ? imageAttachments : undefined,
|
||||
}));
|
||||
imagesAttached = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Image-only turns still deserve a user bubble even without text.
|
||||
if (!imagesAttached && imageAttachments.length > 0) {
|
||||
messages.push(createNormalizedMessage({
|
||||
id: `${baseId}_images`,
|
||||
sessionId,
|
||||
timestamp: ts,
|
||||
provider: PROVIDER,
|
||||
kind: 'text',
|
||||
role: 'user',
|
||||
content: '',
|
||||
images: imageAttachments,
|
||||
}));
|
||||
}
|
||||
} else if (typeof raw.message.content === 'string') {
|
||||
const text = raw.message.content;
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import fsSync from 'node:fs';
|
||||
import readline from 'node:readline';
|
||||
|
||||
import { sessionsDb } from '@/modules/database/index.js';
|
||||
import { toImageAttachments } from '@/shared/image-attachments.js';
|
||||
import type { IProviderSessions } from '@/shared/interfaces.js';
|
||||
import type { AnyRecord, FetchHistoryOptions, FetchHistoryResult, NormalizedMessage } from '@/shared/types.js';
|
||||
import { createNormalizedMessage, generateMessageId, readObjectRecord, sliceTailPage } from '@/shared/utils.js';
|
||||
@@ -31,6 +32,42 @@ function isVisibleCodexUserMessage(payload: AnyRecord | null | undefined): boole
|
||||
return typeof payload.message === 'string' && payload.message.trim().length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the image attachments Codex records on `user_message` events.
|
||||
* Turns sent with `local_image` input items land in `local_images` as file
|
||||
* paths (verified against real rollout JSONL); the `images` array can carry
|
||||
* base64 data URLs, which are passed through as inline `data` attachments so
|
||||
* the UI can preview them without a file lookup.
|
||||
*
|
||||
* Exported for tests.
|
||||
*/
|
||||
export function extractCodexUserImages(
|
||||
payload: AnyRecord | null | undefined,
|
||||
): Array<{ path?: string; data?: string }> | undefined {
|
||||
if (!payload) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const candidates = [
|
||||
...(Array.isArray(payload.local_images) ? payload.local_images : []),
|
||||
...(Array.isArray(payload.images) ? payload.images : []),
|
||||
];
|
||||
|
||||
const attachments: Array<{ path?: string; data?: string }> = [];
|
||||
for (const entry of candidates) {
|
||||
if (typeof entry !== 'string' || !entry.trim()) {
|
||||
continue;
|
||||
}
|
||||
if (entry.startsWith('data:')) {
|
||||
attachments.push({ data: entry });
|
||||
} else {
|
||||
attachments.push(...toImageAttachments([entry]));
|
||||
}
|
||||
}
|
||||
|
||||
return attachments.length > 0 ? attachments : undefined;
|
||||
}
|
||||
|
||||
function extractCodexTextContent(content: unknown): string {
|
||||
if (!Array.isArray(content)) {
|
||||
return typeof content === 'string' ? content : '';
|
||||
@@ -104,6 +141,7 @@ async function getCodexSessionMessages(
|
||||
role: 'user',
|
||||
content: entry.payload.message,
|
||||
},
|
||||
images: extractCodexUserImages(entry.payload as AnyRecord),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -296,7 +334,8 @@ export class CodexSessionsProvider implements IProviderSessions {
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
: String(raw.message.content || '');
|
||||
if (!content.trim()) {
|
||||
const rawImages = Array.isArray(raw.images) && raw.images.length > 0 ? raw.images : undefined;
|
||||
if (!content.trim() && !rawImages) {
|
||||
return [];
|
||||
}
|
||||
return [createNormalizedMessage({
|
||||
@@ -307,6 +346,7 @@ export class CodexSessionsProvider implements IProviderSessions {
|
||||
kind: 'text',
|
||||
role: 'user',
|
||||
content,
|
||||
images: rawImages,
|
||||
})];
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import crypto from 'node:crypto';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
import { parseImagesInputTag } from '@/shared/image-attachments.js';
|
||||
import type { IProviderSessions } from '@/shared/interfaces.js';
|
||||
import type { AnyRecord, FetchHistoryOptions, FetchHistoryResult, NormalizedMessage } from '@/shared/types.js';
|
||||
import {
|
||||
@@ -24,7 +25,7 @@ type CursorJsonBlob = CursorDbBlob & {
|
||||
parsed: AnyRecord;
|
||||
};
|
||||
|
||||
type CursorMessageBlob = {
|
||||
export type CursorMessageBlob = {
|
||||
id: string;
|
||||
sequence: number;
|
||||
rowid: number;
|
||||
@@ -76,6 +77,27 @@ function unwrapUserQueryText(value: string, role: 'user' | 'assistant'): string
|
||||
return value.replace(/^\s*<timestamp>[\s\S]*?<\/timestamp>\s*/, '').trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Unwraps one user-authored text payload and splits off the `<images_input>`
|
||||
* attachment block appended by the chat composer. Assistant text passes
|
||||
* through untouched.
|
||||
*/
|
||||
function extractUserTextAndImages(
|
||||
value: string,
|
||||
role: 'user' | 'assistant',
|
||||
): { text: string; images?: Array<{ path: string; name?: string }> } {
|
||||
const unwrapped = unwrapUserQueryText(value, role);
|
||||
if (role !== 'user') {
|
||||
return { text: unwrapped };
|
||||
}
|
||||
|
||||
const { text, attachments } = parseImagesInputTag(unwrapped);
|
||||
return {
|
||||
text,
|
||||
images: attachments.length > 0 ? attachments : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeToolId(value: unknown): string | null {
|
||||
if (typeof value !== 'string') {
|
||||
return null;
|
||||
@@ -405,8 +427,11 @@ export class CursorSessionsProvider implements IProviderSessions {
|
||||
/**
|
||||
* Converts Cursor SQLite message blobs into normalized messages and attaches
|
||||
* matching tool results to their tool_use entries.
|
||||
*
|
||||
* Public so tests can drive history normalization with synthetic blobs
|
||||
* without needing a real Cursor store.db.
|
||||
*/
|
||||
private normalizeCursorBlobs(blobs: CursorMessageBlob[], sessionId: string | null): NormalizedMessage[] {
|
||||
normalizeCursorBlobs(blobs: CursorMessageBlob[], sessionId: string | null): NormalizedMessage[] {
|
||||
const messages: NormalizedMessage[] = [];
|
||||
const toolUseMap = new Map<string, NormalizedMessage>();
|
||||
const baseTime = Date.now();
|
||||
@@ -446,7 +471,16 @@ export class CursorSessionsProvider implements IProviderSessions {
|
||||
text = unwrapUserQueryText(content.message.content, role);
|
||||
}
|
||||
}
|
||||
if (text?.trim()) {
|
||||
const { text: cleanText, images } = role === 'user'
|
||||
? (() => {
|
||||
const parsed = parseImagesInputTag(text);
|
||||
return {
|
||||
text: parsed.text,
|
||||
images: parsed.attachments.length > 0 ? parsed.attachments : undefined,
|
||||
};
|
||||
})()
|
||||
: { text, images: undefined };
|
||||
if (cleanText?.trim() || images) {
|
||||
messages.push(createNormalizedMessage({
|
||||
id: baseId,
|
||||
sessionId,
|
||||
@@ -454,7 +488,8 @@ export class CursorSessionsProvider implements IProviderSessions {
|
||||
provider: PROVIDER,
|
||||
kind: 'text',
|
||||
role,
|
||||
content: text,
|
||||
content: cleanText,
|
||||
images,
|
||||
sequence: blob.sequence,
|
||||
rowid: blob.rowid,
|
||||
}));
|
||||
@@ -506,8 +541,8 @@ export class CursorSessionsProvider implements IProviderSessions {
|
||||
}
|
||||
|
||||
if (part?.type === 'text' && part?.text) {
|
||||
const normalizedPartText = unwrapUserQueryText(part.text, role);
|
||||
if (!normalizedPartText) {
|
||||
const { text: normalizedPartText, images } = extractUserTextAndImages(part.text, role);
|
||||
if (!normalizedPartText && !images) {
|
||||
continue;
|
||||
}
|
||||
messages.push(createNormalizedMessage({
|
||||
@@ -518,6 +553,7 @@ export class CursorSessionsProvider implements IProviderSessions {
|
||||
kind: 'text',
|
||||
role,
|
||||
content: normalizedPartText,
|
||||
images,
|
||||
sequence: blob.sequence,
|
||||
rowid: blob.rowid,
|
||||
}));
|
||||
@@ -557,8 +593,8 @@ export class CursorSessionsProvider implements IProviderSessions {
|
||||
&& content.content.trim()
|
||||
&& !isInternalCursorText(content.content)
|
||||
) {
|
||||
const normalizedText = unwrapUserQueryText(content.content, role);
|
||||
if (!normalizedText) {
|
||||
const { text: normalizedText, images } = extractUserTextAndImages(content.content, role);
|
||||
if (!normalizedText && !images) {
|
||||
continue;
|
||||
}
|
||||
messages.push(createNormalizedMessage({
|
||||
@@ -569,6 +605,7 @@ export class CursorSessionsProvider implements IProviderSessions {
|
||||
kind: 'text',
|
||||
role,
|
||||
content: normalizedText,
|
||||
images,
|
||||
sequence: blob.sequence,
|
||||
rowid: blob.rowid,
|
||||
}));
|
||||
|
||||
@@ -3,6 +3,7 @@ import fs from 'node:fs/promises';
|
||||
import readline from 'node:readline';
|
||||
|
||||
import { sessionsDb } from '@/modules/database/index.js';
|
||||
import { parseImagesInputTag } from '@/shared/image-attachments.js';
|
||||
import type { IProviderSessions } from '@/shared/interfaces.js';
|
||||
import type { AnyRecord, FetchHistoryOptions, FetchHistoryResult, NormalizedMessage } from '@/shared/types.js';
|
||||
import { createNormalizedMessage, generateMessageId, readObjectRecord, sliceTailPage } from '@/shared/utils.js';
|
||||
@@ -128,7 +129,8 @@ async function getGeminiLegacySessionMessages(sessionFilePath: string): Promise<
|
||||
}
|
||||
}
|
||||
|
||||
async function getGeminiJsonlSessionMessages(sessionFilePath: string): Promise<GeminiHistoryResult> {
|
||||
// Exported for tests: parses one Gemini CLI JSONL transcript from disk.
|
||||
export async function getGeminiJsonlSessionMessages(sessionFilePath: string): Promise<GeminiHistoryResult> {
|
||||
const messages: AnyRecord[] = [];
|
||||
let tokenUsage: AnyRecord | undefined;
|
||||
|
||||
@@ -159,12 +161,18 @@ async function getGeminiJsonlSessionMessages(sessionFilePath: string): Promise<G
|
||||
|
||||
const role = mapGeminiRole(entry.type);
|
||||
if (role) {
|
||||
const textContent = extractGeminiTextContent(entry.content);
|
||||
if (textContent.trim()) {
|
||||
const rawTextContent = extractGeminiTextContent(entry.content);
|
||||
// User prompts sent with attachments carry an <images_input> path
|
||||
// list; strip it for display and surface the paths as images.
|
||||
const { text: textContent, attachments } = role === 'user'
|
||||
? parseImagesInputTag(rawTextContent)
|
||||
: { text: rawTextContent, attachments: [] };
|
||||
if (textContent.trim() || attachments.length > 0) {
|
||||
messages.push({
|
||||
type: 'message',
|
||||
uuid: typeof entry.id === 'string' ? entry.id : undefined,
|
||||
message: { role, content: textContent },
|
||||
images: attachments.length > 0 ? attachments : undefined,
|
||||
timestamp: entry.timestamp || null,
|
||||
});
|
||||
}
|
||||
@@ -413,8 +421,9 @@ export class GeminiSessionsProvider implements IProviderSessions {
|
||||
}
|
||||
|
||||
const role = raw.message?.role || raw.role;
|
||||
const content = raw.message?.content || raw.content;
|
||||
if (!role || !content) {
|
||||
const content = raw.message?.content ?? raw.content ?? '';
|
||||
const rawImages = Array.isArray(raw.images) && raw.images.length > 0 ? raw.images : undefined;
|
||||
if (!role || (!content && !rawImages)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -475,7 +484,12 @@ export class GeminiSessionsProvider implements IProviderSessions {
|
||||
}));
|
||||
}
|
||||
}
|
||||
} else if (typeof content === 'string' && content.trim()) {
|
||||
} else if (typeof content === 'string' && (content.trim() || rawImages)) {
|
||||
// Legacy (non-JSONL) session files reach this branch with the raw
|
||||
// prompt text, so strip any <images_input> block here as well.
|
||||
const { text: cleanContent, attachments } = normalizedRole === 'user'
|
||||
? parseImagesInputTag(content)
|
||||
: { text: content, attachments: [] };
|
||||
normalized.push(createNormalizedMessage({
|
||||
id: baseId,
|
||||
sessionId,
|
||||
@@ -483,7 +497,8 @@ export class GeminiSessionsProvider implements IProviderSessions {
|
||||
provider: PROVIDER,
|
||||
kind: 'text',
|
||||
role: normalizedRole,
|
||||
content,
|
||||
content: cleanContent,
|
||||
images: rawImages ?? (attachments.length > 0 ? attachments : undefined),
|
||||
}));
|
||||
} else {
|
||||
const textContent = extractGeminiTextContent(content);
|
||||
|
||||
@@ -2,6 +2,7 @@ import fsSync from 'node:fs';
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
|
||||
import { parseImagesInputTag } from '@/shared/image-attachments.js';
|
||||
import type { IProviderSessions } from '@/shared/interfaces.js';
|
||||
import type { AnyRecord, FetchHistoryOptions, FetchHistoryResult, NormalizedMessage } from '@/shared/types.js';
|
||||
import {
|
||||
@@ -418,8 +419,13 @@ export class OpenCodeSessionsProvider implements IProviderSessions {
|
||||
}
|
||||
|
||||
if (partType === 'text') {
|
||||
const content = extractText(partData);
|
||||
if (content.trim()) {
|
||||
const rawContent = extractText(partData);
|
||||
// User prompts sent with attachments carry an <images_input> path
|
||||
// list; strip it for display and surface the paths as images.
|
||||
const { text: content, attachments } = messageRole === 'user'
|
||||
? parseImagesInputTag(rawContent)
|
||||
: { text: rawContent, attachments: [] };
|
||||
if (content.trim() || attachments.length > 0) {
|
||||
normalized.push(createNormalizedMessage({
|
||||
id: baseId,
|
||||
sessionId,
|
||||
@@ -428,6 +434,7 @@ export class OpenCodeSessionsProvider implements IProviderSessions {
|
||||
kind: 'text',
|
||||
role: messageRole === 'user' ? 'user' : 'assistant',
|
||||
content,
|
||||
images: attachments.length > 0 ? attachments : undefined,
|
||||
}));
|
||||
}
|
||||
continue;
|
||||
|
||||
@@ -46,7 +46,7 @@ const PROVIDER_CAPABILITIES: Record<LLMProvider, ProviderCapabilities> = {
|
||||
provider: 'cursor',
|
||||
permissionModes: ['default', 'acceptEdits', 'bypassPermissions', 'plan'],
|
||||
defaultPermissionMode: 'default',
|
||||
supportsImages: false,
|
||||
supportsImages: true,
|
||||
supportsAbort: true,
|
||||
supportsPermissionRequests: false,
|
||||
supportsTokenUsage: false,
|
||||
@@ -56,7 +56,7 @@ const PROVIDER_CAPABILITIES: Record<LLMProvider, ProviderCapabilities> = {
|
||||
provider: 'codex',
|
||||
permissionModes: ['default', 'acceptEdits', 'bypassPermissions'],
|
||||
defaultPermissionMode: 'default',
|
||||
supportsImages: false,
|
||||
supportsImages: true,
|
||||
supportsAbort: true,
|
||||
supportsPermissionRequests: false,
|
||||
supportsTokenUsage: true,
|
||||
@@ -66,7 +66,7 @@ const PROVIDER_CAPABILITIES: Record<LLMProvider, ProviderCapabilities> = {
|
||||
provider: 'gemini',
|
||||
permissionModes: ['default', 'acceptEdits', 'bypassPermissions', 'plan'],
|
||||
defaultPermissionMode: 'default',
|
||||
supportsImages: false,
|
||||
supportsImages: true,
|
||||
supportsAbort: true,
|
||||
supportsPermissionRequests: false,
|
||||
supportsTokenUsage: true,
|
||||
@@ -76,7 +76,7 @@ const PROVIDER_CAPABILITIES: Record<LLMProvider, ProviderCapabilities> = {
|
||||
provider: 'opencode',
|
||||
permissionModes: ['default'],
|
||||
defaultPermissionMode: 'default',
|
||||
supportsImages: false,
|
||||
supportsImages: true,
|
||||
supportsAbort: true,
|
||||
supportsPermissionRequests: false,
|
||||
supportsTokenUsage: true,
|
||||
|
||||
@@ -9,6 +9,7 @@ import Database from 'better-sqlite3';
|
||||
import { closeConnection, initializeDatabase, sessionsDb } from '@/modules/database/index.js';
|
||||
import { OpenCodeSessionSynchronizer } from '@/modules/providers/list/opencode/opencode-session-synchronizer.provider.js';
|
||||
import { OpenCodeSessionsProvider } from '@/modules/providers/list/opencode/opencode-sessions.provider.js';
|
||||
import { appendImagesInputTag } from '@/shared/image-attachments.js';
|
||||
|
||||
const patchHomeDir = (nextHomeDir: string) => {
|
||||
const original = os.homedir;
|
||||
@@ -321,6 +322,41 @@ test('OpenCode session synchronizer adopts the pending app session before watche
|
||||
}
|
||||
});
|
||||
|
||||
test('OpenCode sessions provider strips <images_input> from user turns and exposes attachments', { concurrency: false }, async () => {
|
||||
const tempRoot = await mkdtemp(path.join(os.tmpdir(), 'opencode-session-images-'));
|
||||
const workspacePath = path.join(tempRoot, 'workspace');
|
||||
await mkdir(workspacePath, { recursive: true });
|
||||
const restoreHomeDir = patchHomeDir(tempRoot);
|
||||
|
||||
try {
|
||||
await createOpenCodeDatabase(tempRoot, workspacePath);
|
||||
|
||||
// Rewrite the user text part with the tagged prompt the runtime sends.
|
||||
const taggedPrompt = appendImagesInputTag('Look at this screenshot.', [
|
||||
{ path: 'C:/Users/x/.cloudcli/assets/shot.png' },
|
||||
]);
|
||||
const db = new Database(path.join(tempRoot, '.local', 'share', 'opencode', 'opencode.db'));
|
||||
try {
|
||||
db.prepare('UPDATE part SET data = ? WHERE id = ?').run(
|
||||
JSON.stringify({ type: 'text', text: taggedPrompt }),
|
||||
'part-user-text',
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
|
||||
const provider = new OpenCodeSessionsProvider();
|
||||
const history = await provider.fetchHistory('open-session-1');
|
||||
const userMessage = history.messages.find((message) => message.kind === 'text' && message.role === 'user');
|
||||
|
||||
assert.equal(userMessage?.content, 'Look at this screenshot.');
|
||||
assert.deepEqual(userMessage?.images, [{ path: 'C:/Users/x/.cloudcli/assets/shot.png' }]);
|
||||
} finally {
|
||||
restoreHomeDir();
|
||||
await rm(tempRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('OpenCode sessions provider normalizes quoted live text and skips user echoes', () => {
|
||||
const provider = new OpenCodeSessionsProvider();
|
||||
const normalized = provider.normalizeMessage({
|
||||
|
||||
236
server/modules/providers/tests/provider-image-history.test.ts
Normal file
236
server/modules/providers/tests/provider-image-history.test.ts
Normal file
@@ -0,0 +1,236 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
|
||||
import { ClaudeSessionsProvider } from '@/modules/providers/list/claude/claude-sessions.provider.js';
|
||||
import { CodexSessionsProvider, extractCodexUserImages } from '@/modules/providers/list/codex/codex-sessions.provider.js';
|
||||
import { CursorSessionsProvider } from '@/modules/providers/list/cursor/cursor-sessions.provider.js';
|
||||
import { getGeminiJsonlSessionMessages } from '@/modules/providers/list/gemini/gemini-sessions.provider.js';
|
||||
import { appendImagesInputTag } from '@/shared/image-attachments.js';
|
||||
|
||||
const SESSION_ID = 'session-1';
|
||||
|
||||
// ---------------------------------------------------------------- Claude
|
||||
|
||||
test('claude history: base64 image blocks surface as user message images', () => {
|
||||
const provider = new ClaudeSessionsProvider();
|
||||
const entry = {
|
||||
uuid: 'u1',
|
||||
timestamp: '2026-07-03T10:00:00.000Z',
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'What is in this screenshot?' },
|
||||
{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'QUJD' } },
|
||||
{ type: 'image', source: { type: 'base64', media_type: 'image/jpeg', data: 'REVG' } },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const messages = provider.normalizeMessage(entry, SESSION_ID);
|
||||
|
||||
assert.equal(messages.length, 1);
|
||||
assert.equal(messages[0].kind, 'text');
|
||||
assert.equal(messages[0].role, 'user');
|
||||
assert.equal(messages[0].content, 'What is in this screenshot?');
|
||||
assert.deepEqual(messages[0].images, [
|
||||
{ data: 'data:image/png;base64,QUJD' },
|
||||
{ data: 'data:image/jpeg;base64,REVG' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('claude history: image-only user turns still produce a bubble', () => {
|
||||
const provider = new ClaudeSessionsProvider();
|
||||
const entry = {
|
||||
uuid: 'u2',
|
||||
timestamp: '2026-07-03T10:00:00.000Z',
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'QUJD' } },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const messages = provider.normalizeMessage(entry, SESSION_ID);
|
||||
|
||||
assert.equal(messages.length, 1);
|
||||
assert.equal(messages[0].role, 'user');
|
||||
assert.equal(messages[0].content, '');
|
||||
assert.deepEqual(messages[0].images, [{ data: 'data:image/png;base64,QUJD' }]);
|
||||
});
|
||||
|
||||
test('claude history: plain text user turns carry no images field', () => {
|
||||
const provider = new ClaudeSessionsProvider();
|
||||
const entry = {
|
||||
uuid: 'u3',
|
||||
timestamp: '2026-07-03T10:00:00.000Z',
|
||||
message: { role: 'user', content: [{ type: 'text', text: 'hello' }] },
|
||||
};
|
||||
|
||||
const messages = provider.normalizeMessage(entry, SESSION_ID);
|
||||
assert.equal(messages.length, 1);
|
||||
assert.equal(messages[0].images, undefined);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- Codex
|
||||
|
||||
test('codex history: user_message payload images become path attachments', () => {
|
||||
// Real rollout shape: local_image input items land in `local_images`,
|
||||
// while `images` stays an empty array.
|
||||
assert.deepEqual(
|
||||
extractCodexUserImages({
|
||||
type: 'user_message',
|
||||
message: 'can u see attached image?',
|
||||
images: [],
|
||||
local_images: ['C:\\proj\\.cloudcli\\assets\\a.png'],
|
||||
}),
|
||||
[{ path: 'C:/proj/.cloudcli/assets/a.png' }],
|
||||
);
|
||||
assert.deepEqual(
|
||||
extractCodexUserImages({ type: 'user_message', message: 'hi', images: ['/proj/b.jpg'] }),
|
||||
[{ path: '/proj/b.jpg' }],
|
||||
);
|
||||
assert.equal(extractCodexUserImages({ type: 'user_message', message: 'hi' }), undefined);
|
||||
assert.equal(extractCodexUserImages({ type: 'user_message', message: 'hi', images: [], local_images: [] }), undefined);
|
||||
});
|
||||
|
||||
test('codex history: base64 data URLs pass through as inline data attachments', () => {
|
||||
const dataUrl = 'data:image/png;base64,QUJD';
|
||||
assert.deepEqual(
|
||||
extractCodexUserImages({
|
||||
type: 'user_message',
|
||||
message: 'look',
|
||||
images: [dataUrl],
|
||||
local_images: ['C:\\proj\\a.png'],
|
||||
}),
|
||||
[{ path: 'C:/proj/a.png' }, { data: dataUrl }],
|
||||
);
|
||||
});
|
||||
|
||||
test('codex history: normalized user entries keep their images', () => {
|
||||
const provider = new CodexSessionsProvider();
|
||||
const messages = provider.normalizeMessage(
|
||||
{
|
||||
timestamp: '2026-07-03T10:00:00.000Z',
|
||||
message: { role: 'user', content: 'Look at this' },
|
||||
images: [{ path: '.cloudcli/assets/a.png' }],
|
||||
},
|
||||
SESSION_ID,
|
||||
);
|
||||
|
||||
assert.equal(messages.length, 1);
|
||||
assert.equal(messages[0].role, 'user');
|
||||
assert.equal(messages[0].content, 'Look at this');
|
||||
assert.deepEqual(messages[0].images, [{ path: '.cloudcli/assets/a.png' }]);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- Gemini
|
||||
|
||||
test('gemini history: <images_input> tag is stripped and paths attached', async () => {
|
||||
const tempDir = await mkdtemp(path.join(os.tmpdir(), 'gemini-image-history-'));
|
||||
const jsonlPath = path.join(tempDir, 'session.jsonl');
|
||||
try {
|
||||
const taggedPrompt = appendImagesInputTag('Compare these designs', [
|
||||
{ path: '.cloudcli/assets/1-a.png' },
|
||||
{ path: '.cloudcli/assets/2-b.png' },
|
||||
]);
|
||||
const lines = [
|
||||
JSON.stringify({ type: 'user', id: 'm1', content: taggedPrompt, timestamp: '2026-07-03T10:00:00.000Z' }),
|
||||
JSON.stringify({ type: 'gemini', id: 'm2', content: 'They differ in spacing.', timestamp: '2026-07-03T10:00:05.000Z' }),
|
||||
];
|
||||
await writeFile(jsonlPath, `${lines.join('\n')}\n`, 'utf8');
|
||||
|
||||
const { messages } = await getGeminiJsonlSessionMessages(jsonlPath);
|
||||
|
||||
assert.equal(messages.length, 2);
|
||||
assert.equal(messages[0].message.role, 'user');
|
||||
assert.equal(messages[0].message.content, 'Compare these designs');
|
||||
assert.deepEqual(messages[0].images, [
|
||||
{ path: '.cloudcli/assets/1-a.png' },
|
||||
{ path: '.cloudcli/assets/2-b.png' },
|
||||
]);
|
||||
// Assistant text is left untouched.
|
||||
assert.equal(messages[1].message.role, 'assistant');
|
||||
assert.equal(messages[1].images, undefined);
|
||||
} finally {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('gemini history: prompts without a tag are unchanged', async () => {
|
||||
const tempDir = await mkdtemp(path.join(os.tmpdir(), 'gemini-image-history-'));
|
||||
const jsonlPath = path.join(tempDir, 'session.jsonl');
|
||||
try {
|
||||
await writeFile(
|
||||
jsonlPath,
|
||||
`${JSON.stringify({ type: 'user', id: 'm1', content: 'plain prompt', timestamp: '2026-07-03T10:00:00.000Z' })}\n`,
|
||||
'utf8',
|
||||
);
|
||||
|
||||
const { messages } = await getGeminiJsonlSessionMessages(jsonlPath);
|
||||
assert.equal(messages.length, 1);
|
||||
assert.equal(messages[0].message.content, 'plain prompt');
|
||||
assert.equal(messages[0].images, undefined);
|
||||
} finally {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------- Cursor
|
||||
|
||||
test('cursor history: <images_input> inside user_query is stripped and attached', () => {
|
||||
const provider = new CursorSessionsProvider();
|
||||
const taggedPrompt = appendImagesInputTag('Fix the layout bug', [{ path: '.cloudcli/assets/shot.png' }]);
|
||||
const blobs = [
|
||||
{
|
||||
id: 'blob1',
|
||||
sequence: 1,
|
||||
rowid: 1,
|
||||
content: {
|
||||
role: 'user',
|
||||
content: `<timestamp>2026-07-03</timestamp>\n<user_query>${taggedPrompt}</user_query>`,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'blob2',
|
||||
sequence: 2,
|
||||
rowid: 2,
|
||||
content: {
|
||||
role: 'assistant',
|
||||
content: [{ type: 'text', text: 'Done — the flex container was wrong.' }],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const messages = provider.normalizeCursorBlobs(blobs, SESSION_ID);
|
||||
|
||||
assert.equal(messages.length, 2);
|
||||
assert.equal(messages[0].role, 'user');
|
||||
assert.equal(messages[0].content, 'Fix the layout bug');
|
||||
assert.deepEqual(messages[0].images, [{ path: '.cloudcli/assets/shot.png' }]);
|
||||
assert.equal(messages[1].role, 'assistant');
|
||||
assert.equal(messages[1].images, undefined);
|
||||
});
|
||||
|
||||
test('cursor history: user text without a tag keeps existing behavior', () => {
|
||||
const provider = new CursorSessionsProvider();
|
||||
const blobs = [
|
||||
{
|
||||
id: 'blob1',
|
||||
sequence: 1,
|
||||
rowid: 1,
|
||||
content: {
|
||||
role: 'user',
|
||||
content: '<timestamp>2026-07-03</timestamp>\n<user_query>plain question</user_query>',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const messages = provider.normalizeCursorBlobs(blobs, SESSION_ID);
|
||||
assert.equal(messages.length, 1);
|
||||
assert.equal(messages[0].content, 'plain question');
|
||||
assert.equal(messages[0].images, undefined);
|
||||
});
|
||||
Reference in New Issue
Block a user