refactor: add gemini jsonl session support

This commit is contained in:
Haileyesus
2026-04-29 19:21:57 +03:00
parent c7d4fa915e
commit 0595ba8fed
3 changed files with 589 additions and 65 deletions

View File

@@ -1,14 +1,17 @@
import crypto from 'node:crypto';
import os from 'node:os';
import path from 'node:path';
import { readFile } from 'node:fs/promises';
import { sessionsDb } from '@/modules/database/index.js';
import { projectsDb, sessionsDb } from '@/modules/database/index.js';
import {
findFilesRecursivelyCreatedAfter,
normalizeProjectPath,
normalizeSessionName,
readFileTimestamps,
} from '@/shared/utils.js';
import type { IProviderSessionSynchronizer } from '@/shared/interfaces.js';
import type { AnyRecord } from '@/shared/types.js';
type ParsedSession = {
sessionId: string;
@@ -16,6 +19,13 @@ type ParsedSession = {
sessionName?: string;
};
type GeminiJsonlMetadata = {
sessionId: string;
projectPath?: string;
projectHash?: string;
firstUserMessage?: string;
};
/**
* Session indexer for Gemini transcript artifacts.
*/
@@ -24,31 +34,50 @@ export class GeminiSessionSynchronizer implements IProviderSessionSynchronizer {
private readonly geminiHome = path.join(os.homedir(), '.gemini');
/**
* Scans Gemini session JSON files and upserts discovered sessions into DB.
* Scans Gemini legacy JSON and new JSONL artifacts and upserts sessions into DB.
*/
async synchronize(since?: Date): Promise<number> {
const projectHashLookup = this.buildProjectHashLookup();
const legacySessionFiles = await findFilesRecursivelyCreatedAfter(
path.join(this.geminiHome, 'sessions'),
'.json',
since ?? null
);
const tempFiles = await findFilesRecursivelyCreatedAfter(
const legacyTempFiles = await findFilesRecursivelyCreatedAfter(
path.join(this.geminiHome, 'tmp'),
'.json',
since ?? null
);
const files = [...legacySessionFiles, ...tempFiles];
const jsonlSessionFiles = await findFilesRecursivelyCreatedAfter(
path.join(this.geminiHome, 'sessions'),
'.jsonl',
since ?? null
);
const jsonlTempFiles = await findFilesRecursivelyCreatedAfter(
path.join(this.geminiHome, 'tmp'),
'.jsonl',
since ?? null
);
// Process legacy JSON first, then JSONL. If both exist for a session id,
// the JSONL artifact becomes the canonical jsonl_path via upsert.
const files = [
...legacySessionFiles,
...legacyTempFiles,
...jsonlSessionFiles,
...jsonlTempFiles,
];
let processed = 0;
for (const filePath of files) {
if (
filePath.startsWith(path.join(this.geminiHome, 'tmp'))
&& !filePath.includes(`${path.sep}chats${path.sep}`)
) {
if (this.shouldSkipTempArtifact(filePath)) {
continue;
}
const parsed = await this.processSessionFile(filePath);
const parsed = filePath.endsWith('.jsonl')
? await this.processJsonlSessionFile(filePath, projectHashLookup)
: await this.processLegacySessionFile(filePath);
if (!parsed) {
continue;
}
@@ -70,21 +99,20 @@ export class GeminiSessionSynchronizer implements IProviderSessionSynchronizer {
}
/**
* Parses and upserts one Gemini session JSON artifact.
* Parses and upserts one Gemini legacy JSON or JSONL artifact.
*/
async synchronizeFile(filePath: string): Promise<string | null> {
if (!filePath.endsWith('.json')) {
if (!filePath.endsWith('.json') && !filePath.endsWith('.jsonl')) {
return null;
}
if (
filePath.startsWith(path.join(this.geminiHome, 'tmp'))
&& !filePath.includes(`${path.sep}chats${path.sep}`)
) {
if (this.shouldSkipTempArtifact(filePath)) {
return null;
}
const parsed = await this.processSessionFile(filePath);
const parsed = filePath.endsWith('.jsonl')
? await this.processJsonlSessionFile(filePath, this.buildProjectHashLookup())
: await this.processLegacySessionFile(filePath);
if (!parsed) {
return null;
}
@@ -102,12 +130,12 @@ export class GeminiSessionSynchronizer implements IProviderSessionSynchronizer {
}
/**
* Extracts session metadata from one Gemini JSON artifact.
* Extracts session metadata from one Gemini legacy JSON artifact.
*/
private async processSessionFile(filePath: string): Promise<ParsedSession | null> {
private async processLegacySessionFile(filePath: string): Promise<ParsedSession | null> {
try {
const content = await readFile(filePath, 'utf8');
const data = JSON.parse(content) as Record<string, any>;
const data = JSON.parse(content) as AnyRecord;
const sessionId =
typeof data.sessionId === 'string'
@@ -119,27 +147,16 @@ export class GeminiSessionSynchronizer implements IProviderSessionSynchronizer {
return null;
}
let projectPath = typeof data.projectPath === 'string' ? data.projectPath : '';
if (!projectPath && filePath.includes(`${path.sep}chats${path.sep}`)) {
const chatsDir = path.dirname(filePath);
const workspaceDir = path.dirname(chatsDir);
const projectRootPath = path.join(workspaceDir, '.project_root');
try {
const rootContent = await readFile(projectRootPath, 'utf8');
projectPath = rootContent.trim();
} catch {
// Some Gemini artifacts do not ship a .project_root marker.
}
}
const workspaceProjectPath = await this.resolveProjectPathFromChatWorkspace(filePath);
const projectPath = typeof data.projectPath === 'string' && data.projectPath.trim().length > 0
? data.projectPath
: workspaceProjectPath;
if (!projectPath) {
return null;
}
const messages = Array.isArray(data.messages) ? data.messages : [];
const firstMessage = messages[0] as Record<string, any> | undefined;
const firstMessage = messages[0] as AnyRecord | undefined;
let rawName: string | undefined;
if (Array.isArray(firstMessage?.content) && typeof firstMessage.content[0]?.text === 'string') {
@@ -157,4 +174,228 @@ export class GeminiSessionSynchronizer implements IProviderSessionSynchronizer {
return null;
}
}
/**
* Extracts session metadata from one Gemini JSONL artifact.
*/
private async processJsonlSessionFile(
filePath: string,
projectHashLookup: Map<string, string>
): Promise<ParsedSession | null> {
const metadata = await this.extractJsonlMetadata(filePath);
if (!metadata) {
return null;
}
let projectPath = typeof metadata.projectPath === 'string' ? metadata.projectPath.trim() : '';
if (!projectPath) {
const workspaceProjectPath = await this.resolveProjectPathFromChatWorkspace(filePath);
if (workspaceProjectPath) {
projectPath = workspaceProjectPath;
}
}
if (!projectPath && typeof metadata.projectHash === 'string') {
projectPath = projectHashLookup.get(metadata.projectHash.trim().toLowerCase()) ?? '';
}
if (!projectPath) {
return null;
}
// Once we resolve a project hash/path pair, keep it in-memory for this sync run.
if (typeof metadata.projectHash === 'string' && metadata.projectHash.trim()) {
projectHashLookup.set(metadata.projectHash.trim().toLowerCase(), projectPath);
}
return {
sessionId: metadata.sessionId,
projectPath,
sessionName: normalizeSessionName(metadata.firstUserMessage, 'New Gemini Chat'),
};
}
/**
* Reads first useful metadata from Gemini JSONL files.
*/
private async extractJsonlMetadata(filePath: string): Promise<GeminiJsonlMetadata | null> {
try {
const content = await readFile(filePath, 'utf8');
const lines = content.split('\n');
let sessionId: string | undefined;
let projectPath: string | undefined;
let projectHash: string | undefined;
let firstUserMessage: string | undefined;
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed) {
continue;
}
let parsed: AnyRecord;
try {
parsed = JSON.parse(trimmed) as AnyRecord;
} catch {
continue;
}
if (!sessionId && typeof parsed.sessionId === 'string') {
sessionId = parsed.sessionId;
}
if (!projectPath && typeof parsed.projectPath === 'string') {
projectPath = parsed.projectPath;
}
if (!projectHash && typeof parsed.projectHash === 'string') {
projectHash = parsed.projectHash;
}
if (!firstUserMessage && parsed.type === 'user') {
firstUserMessage = this.extractGeminiTextContent(parsed.content);
}
if (sessionId && (projectPath || projectHash) && firstUserMessage) {
break;
}
}
if (!sessionId) {
return null;
}
return {
sessionId,
projectPath,
projectHash,
firstUserMessage,
};
} catch {
return null;
}
}
/**
* Tries to resolve project root from Gemini tmp chat workspaces.
*/
private async resolveProjectPathFromChatWorkspace(filePath: string): Promise<string> {
if (!filePath.includes(`${path.sep}chats${path.sep}`)) {
return '';
}
const chatsDir = path.dirname(filePath);
const workspaceDir = path.dirname(chatsDir);
const projectRootPath = path.join(workspaceDir, '.project_root');
try {
const rootContent = await readFile(projectRootPath, 'utf8');
return rootContent.trim();
} catch {
return '';
}
}
/**
* Builds a hash->path lookup for Gemini JSONL metadata that stores projectHash.
*/
private buildProjectHashLookup(): Map<string, string> {
const lookup = new Map<string, string>();
const knownPaths = new Set<string>();
for (const project of projectsDb.getProjectPaths()) {
if (typeof project.project_path === 'string' && project.project_path.trim()) {
knownPaths.add(project.project_path.trim());
}
}
for (const session of sessionsDb.getAllSessions()) {
if (session.provider === this.provider && typeof session.project_path === 'string' && session.project_path.trim()) {
knownPaths.add(session.project_path.trim());
}
}
for (const knownPath of knownPaths) {
this.addProjectHashCandidates(lookup, knownPath);
}
return lookup;
}
/**
* Adds likely Gemini hash variants for one project path.
*/
private addProjectHashCandidates(lookup: Map<string, string>, projectPath: string): void {
const trimmed = projectPath.trim();
if (!trimmed) {
return;
}
const normalized = normalizeProjectPath(trimmed);
const resolved = path.resolve(trimmed);
const resolvedNormalized = normalizeProjectPath(resolved);
const candidates = new Set<string>([
trimmed,
normalized,
resolved,
resolvedNormalized,
]);
if (process.platform === 'win32') {
for (const candidate of [...candidates]) {
candidates.add(candidate.toLowerCase());
}
}
for (const candidate of candidates) {
if (!candidate) {
continue;
}
const hash = this.sha256(candidate);
if (!lookup.has(hash)) {
lookup.set(hash, trimmed);
}
}
}
/**
* Returns first user text from Gemini content payload shapes.
*/
private extractGeminiTextContent(content: unknown): string | undefined {
if (typeof content === 'string' && content.trim().length > 0) {
return content;
}
if (!Array.isArray(content)) {
return undefined;
}
for (const part of content) {
if (typeof part === 'string' && part.trim().length > 0) {
return part;
}
if (part && typeof part === 'object' && typeof (part as AnyRecord).text === 'string') {
const text = (part as AnyRecord).text;
if (text.trim().length > 0) {
return text;
}
}
}
return undefined;
}
/**
* Keeps tmp scanning scoped to chat artifacts only.
*/
private shouldSkipTempArtifact(filePath: string): boolean {
return (
filePath.startsWith(path.join(this.geminiHome, 'tmp'))
&& !filePath.includes(`${path.sep}chats${path.sep}`)
);
}
private sha256(value: string): string {
return crypto.createHash('sha256').update(value).digest('hex');
}
}

View File

@@ -1,4 +1,6 @@
import fsSync from 'node:fs';
import fs from 'node:fs/promises';
import readline from 'node:readline';
import { sessionsDb } from '@/modules/database/index.js';
import type { IProviderSessions } from '@/shared/interfaces.js';
@@ -7,45 +9,241 @@ import { createNormalizedMessage, generateMessageId, readObjectRecord } from '@/
const PROVIDER = 'gemini';
async function getGeminiCliSessionMessages(sessionId: string): Promise<AnyRecord[]> {
const sessionFilePath = sessionsDb.getSessionById(sessionId)?.jsonl_path;
if (!sessionFilePath) {
return [];
type GeminiHistoryResult = {
messages: AnyRecord[];
tokenUsage?: unknown;
};
function mapGeminiRole(value: unknown): 'user' | 'assistant' | null {
if (value === 'user') {
return 'user';
}
if (value === 'gemini' || value === 'assistant') {
return 'assistant';
}
return null;
}
function extractGeminiTextContent(content: unknown): string {
if (typeof content === 'string') {
return content;
}
if (!Array.isArray(content)) {
return '';
}
return content
.map((part) => {
if (typeof part === 'string') {
return part;
}
if (!part || typeof part !== 'object') {
return '';
}
const record = part as AnyRecord;
if (typeof record.text === 'string') {
return record.text;
}
return '';
})
.filter(Boolean)
.join('\n');
}
function extractGeminiThoughts(thoughts: unknown): string {
if (!Array.isArray(thoughts)) {
return '';
}
return thoughts
.map((item) => {
if (!item || typeof item !== 'object') {
return '';
}
const record = item as AnyRecord;
const subject = typeof record.subject === 'string' ? record.subject.trim() : '';
const description = typeof record.description === 'string' ? record.description.trim() : '';
if (subject && description) {
return `${subject}: ${description}`;
}
return description || subject;
})
.filter(Boolean)
.join('\n');
}
function buildGeminiTokenUsage(tokens: unknown): AnyRecord | undefined {
if (!tokens || typeof tokens !== 'object') {
return undefined;
}
const record = tokens as AnyRecord;
const input = Number(record.input || 0);
const output = Number(record.output || 0);
const cached = Number(record.cached || 0);
const thoughts = Number(record.thoughts || 0);
const tool = Number(record.tool || 0);
const totalFromFields = input + output + cached + thoughts + tool;
const total = Number(record.total || totalFromFields || 0);
return {
used: total,
total: total,
breakdown: {
input,
output,
cached,
thoughts,
tool,
},
};
}
async function getGeminiLegacySessionMessages(sessionFilePath: string): Promise<GeminiHistoryResult> {
try {
const data = await fs.readFile(sessionFilePath, 'utf8');
const session = JSON.parse(data) as AnyRecord;
const sourceMessages = Array.isArray(session.messages) ? session.messages : [];
return sourceMessages.map((msg: AnyRecord) => {
const role = msg.type === 'user'
? 'user'
: (msg.type === 'gemini' || msg.type === 'assistant')
? 'assistant'
: msg.type;
let content = '';
if (typeof msg.content === 'string') {
content = msg.content;
} else if (Array.isArray(msg.content)) {
content = msg.content
.filter((part: AnyRecord) => part?.text)
.map((part: AnyRecord) => part.text)
.join('\n');
const messages: AnyRecord[] = [];
for (const msg of sourceMessages) {
const role = mapGeminiRole(msg.type ?? msg.role);
if (!role) {
continue;
}
return {
messages.push({
type: 'message',
message: { role, content },
uuid: typeof msg.id === 'string' ? msg.id : undefined,
message: { role, content: msg.content },
timestamp: msg.timestamp || null,
};
});
});
}
return { messages };
} catch {
return [];
return { messages: [] };
}
}
async function getGeminiJsonlSessionMessages(sessionFilePath: string): Promise<GeminiHistoryResult> {
const messages: AnyRecord[] = [];
let tokenUsage: AnyRecord | undefined;
try {
const fileStream = fsSync.createReadStream(sessionFilePath);
const lineReader = readline.createInterface({
input: fileStream,
crlfDelay: Infinity,
});
for await (const line of lineReader) {
const trimmed = line.trim();
if (!trimmed) {
continue;
}
let entry: AnyRecord;
try {
entry = JSON.parse(trimmed) as AnyRecord;
} catch {
continue;
}
// Metadata/update lines (e.g. {$set:{lastUpdated:...}}) do not represent chat messages.
if (entry.$set) {
continue;
}
const role = mapGeminiRole(entry.type);
if (role) {
const textContent = extractGeminiTextContent(entry.content);
if (textContent.trim()) {
messages.push({
type: 'message',
uuid: typeof entry.id === 'string' ? entry.id : undefined,
message: { role, content: textContent },
timestamp: entry.timestamp || null,
});
}
const thinkingContent = extractGeminiThoughts(entry.thoughts);
if (thinkingContent.trim()) {
messages.push({
type: 'thinking',
uuid: typeof entry.id === 'string' ? `${entry.id}_thinking` : undefined,
message: { role: 'assistant', content: thinkingContent },
timestamp: entry.timestamp || null,
isReasoning: true,
});
}
if (role === 'assistant') {
const usage = buildGeminiTokenUsage(entry.tokens);
if (usage) {
tokenUsage = usage;
}
}
continue;
}
if (entry.type === 'tool_use') {
messages.push({
type: 'tool_use',
uuid: typeof entry.id === 'string' ? entry.id : undefined,
timestamp: entry.timestamp || null,
toolName: entry.tool_name || entry.name || 'Tool',
toolInput: entry.parameters ?? entry.input ?? entry.arguments ?? '',
toolCallId: entry.tool_id || entry.toolCallId || entry.id,
});
continue;
}
if (entry.type === 'tool_result') {
messages.push({
type: 'tool_result',
uuid: typeof entry.id === 'string' ? entry.id : undefined,
timestamp: entry.timestamp || null,
toolCallId: entry.tool_id || entry.toolCallId || entry.id || '',
output: entry.output ?? entry.result ?? '',
isError: Boolean(entry.error) || entry.status === 'error',
});
}
}
} catch {
return { messages: [] };
}
messages.sort(
(a, b) => new Date(a.timestamp || 0).getTime() - new Date(b.timestamp || 0).getTime(),
);
return { messages, tokenUsage };
}
async function getGeminiCliSessionMessages(sessionId: string): Promise<GeminiHistoryResult> {
const sessionFilePath = sessionsDb.getSessionById(sessionId)?.jsonl_path;
if (!sessionFilePath) {
return { messages: [] };
}
if (sessionFilePath.endsWith('.jsonl')) {
return getGeminiJsonlSessionMessages(sessionFilePath);
}
return getGeminiLegacySessionMessages(sessionFilePath);
}
export class GeminiSessionsProvider implements IProviderSessions {
/**
* Normalizes live Gemini stream-json events into the shared message shape.
@@ -156,24 +354,73 @@ export class GeminiSessionsProvider implements IProviderSessions {
): Promise<FetchHistoryResult> {
const { limit = null, offset = 0 } = options;
let rawMessages: AnyRecord[];
let result: GeminiHistoryResult;
try {
rawMessages = await getGeminiCliSessionMessages(sessionId);
result = await getGeminiCliSessionMessages(sessionId);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
console.warn(`[GeminiProvider] Failed to load session ${sessionId}:`, message);
return { messages: [], total: 0, hasMore: false, offset: 0, limit: null };
}
const rawMessages = result.messages;
const normalized: NormalizedMessage[] = [];
for (let i = 0; i < rawMessages.length; i++) {
const raw = rawMessages[i];
const ts = raw.timestamp || new Date().toISOString();
const baseId = raw.uuid || generateMessageId('gemini');
if (raw.type === 'thinking' || raw.isReasoning) {
const thinkingContent = typeof raw.message?.content === 'string'
? raw.message.content
: typeof raw.content === 'string'
? raw.content
: '';
if (thinkingContent.trim()) {
normalized.push(createNormalizedMessage({
id: baseId,
sessionId,
timestamp: ts,
provider: PROVIDER,
kind: 'thinking',
content: thinkingContent,
}));
}
continue;
}
if (raw.type === 'tool_use' || raw.toolName) {
normalized.push(createNormalizedMessage({
id: baseId,
sessionId,
timestamp: ts,
provider: PROVIDER,
kind: 'tool_use',
toolName: raw.toolName || 'Tool',
toolInput: raw.toolInput,
toolId: raw.toolCallId || baseId,
}));
continue;
}
if (raw.type === 'tool_result') {
normalized.push(createNormalizedMessage({
id: baseId,
sessionId,
timestamp: ts,
provider: PROVIDER,
kind: 'tool_result',
toolId: raw.toolCallId || '',
content: raw.output === undefined ? '' : String(raw.output),
isError: Boolean(raw.isError),
}));
continue;
}
const role = raw.message?.role || raw.role;
const content = raw.message?.content || raw.content;
if (!role || !content) {
continue;
}
@@ -182,8 +429,26 @@ export class GeminiSessionsProvider implements IProviderSessions {
if (Array.isArray(content)) {
for (let partIdx = 0; partIdx < content.length; partIdx++) {
const part = content[partIdx];
if (part.type === 'text' && part.text) {
const part = content[partIdx] as AnyRecord | string;
if (typeof part === 'string' && part.trim()) {
normalized.push(createNormalizedMessage({
id: `${baseId}_${partIdx}`,
sessionId,
timestamp: ts,
provider: PROVIDER,
kind: 'text',
role: normalizedRole,
content: part,
}));
continue;
}
if (!part || typeof part !== 'object') {
continue;
}
if ((part.type === 'text' || !part.type) && typeof part.text === 'string' && part.text.trim()) {
normalized.push(createNormalizedMessage({
id: `${baseId}_${partIdx}`,
sessionId,
@@ -227,6 +492,19 @@ export class GeminiSessionsProvider implements IProviderSessions {
role: normalizedRole,
content,
}));
} else {
const textContent = extractGeminiTextContent(content);
if (textContent.trim()) {
normalized.push(createNormalizedMessage({
id: baseId,
sessionId,
timestamp: ts,
provider: PROVIDER,
kind: 'text',
role: normalizedRole,
content: textContent,
}));
}
}
}
@@ -257,6 +535,7 @@ export class GeminiSessionsProvider implements IProviderSessions {
hasMore: pageLimit === null ? false : start + pageLimit < normalized.length,
offset: start,
limit: pageLimit,
tokenUsage: result.tokenUsage,
};
}
}

View File

@@ -66,7 +66,7 @@ let watcherRescheduleAfterRefresh = false;
*/
function isWatcherTargetFile(provider: LLMProvider, filePath: string): boolean {
if (provider === 'gemini') {
return filePath.endsWith('.json');
return filePath.endsWith('.json') || filePath.endsWith('.jsonl');
}
return filePath.endsWith('.jsonl');
@@ -194,6 +194,10 @@ async function onUpdate(
return;
}
console.log(`Session synchronization triggered by ${eventType} event for provider "${provider}"`, {
filePath,
sessionId: result.sessionId,
});
queuePendingWatcherUpdate(eventType, provider, result.sessionId);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);