refactor: move search to module

This commit is contained in:
Haileyesus
2026-04-27 21:39:48 +03:00
parent 9a8fb116ef
commit 50ee3c7548
6 changed files with 1017 additions and 778 deletions

View File

@@ -17,11 +17,8 @@
* features that still need on-disk data:
* - Session message reads for each provider (Claude/Codex/Gemini) for
* `GET /api/providers/sessions/:sessionId/messages`.
* - Conversation search (`searchConversations`) which scans JSONL history.
* - (Project row removal / JSONL cleanup is handled in
* `modules/projects/services/project-delete.service.ts`.)
* - Manual project registration (`addProjectManually`) which syncs to
* ~/.claude/project-config.json for backwards compatibility.
*/
import fsSync, { promises as fs } from 'fs';
@@ -29,9 +26,6 @@ import path from 'path';
import readline from 'readline';
import os from 'os';
import { generateDisplayName } from '@/modules/projects';
import sessionManager from './sessionManager.js';
import { projectsDb } from './modules/database/index.js';
/**
@@ -68,160 +62,6 @@ function claudeFolderNameFromPath(projectPath) {
return projectPath.replace(/[^a-zA-Z0-9-]/g, '-');
}
// Cache for extracted project directories
const projectDirectoryCache = new Map();
// Load project configuration file
async function loadProjectConfig() {
const configPath = path.join(os.homedir(), '.claude', 'project-config.json');
try {
const configData = await fs.readFile(configPath, 'utf8');
return JSON.parse(configData);
} catch (error) {
// Return empty config if file doesn't exist
return {};
}
}
// Save project configuration file
async function saveProjectConfig(config) {
const claudeDir = path.join(os.homedir(), '.claude');
const configPath = path.join(claudeDir, 'project-config.json');
// Ensure the .claude directory exists
try {
await fs.mkdir(claudeDir, { recursive: true });
} catch (error) {
if (error.code !== 'EEXIST') {
throw error;
}
}
await fs.writeFile(configPath, JSON.stringify(config, null, 2), 'utf8');
}
// Resolve a Claude-encoded folder name back to an absolute project directory
// by inspecting cached metadata and JSONL `cwd` fields. Used only by the
// legacy name-based helpers below (`getSessions`, `deleteProject`, etc.) and
// by the conversation search; id-based routes use `getProjectPathById`.
async function extractProjectDirectory(projectName) {
// Check cache first
if (projectDirectoryCache.has(projectName)) {
return projectDirectoryCache.get(projectName);
}
// Check project config for originalPath (manually added projects via UI or platform)
// This handles projects with dashes in their directory names correctly
const config = await loadProjectConfig();
if (config[projectName]?.originalPath) {
const originalPath = config[projectName].originalPath;
projectDirectoryCache.set(projectName, originalPath);
return originalPath;
}
const projectDir = path.join(os.homedir(), '.claude', 'projects', projectName);
const cwdCounts = new Map();
let latestTimestamp = 0;
let latestCwd = null;
let extractedPath;
try {
// Check if the project directory exists
await fs.access(projectDir);
const files = await fs.readdir(projectDir);
const jsonlFiles = files.filter(file => file.endsWith('.jsonl'));
if (jsonlFiles.length === 0) {
// Fall back to decoded project name if no sessions
extractedPath = projectName.replace(/-/g, '/');
} else {
// Process all JSONL files to collect cwd values
for (const file of jsonlFiles) {
const jsonlFile = path.join(projectDir, file);
const fileStream = fsSync.createReadStream(jsonlFile);
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
for await (const line of rl) {
if (line.trim()) {
try {
const entry = JSON.parse(line);
if (entry.cwd) {
// Count occurrences of each cwd
cwdCounts.set(entry.cwd, (cwdCounts.get(entry.cwd) || 0) + 1);
// Track the most recent cwd
const timestamp = new Date(entry.timestamp || 0).getTime();
if (timestamp > latestTimestamp) {
latestTimestamp = timestamp;
latestCwd = entry.cwd;
}
}
} catch (parseError) {
// Skip malformed lines
}
}
}
}
// Determine the best cwd to use
if (cwdCounts.size === 0) {
// No cwd found, fall back to decoded project name
extractedPath = projectName.replace(/-/g, '/');
} else if (cwdCounts.size === 1) {
// Only one cwd, use it
extractedPath = Array.from(cwdCounts.keys())[0];
} else {
// Multiple cwd values - prefer the most recent one if it has reasonable usage
const mostRecentCount = cwdCounts.get(latestCwd) || 0;
const maxCount = Math.max(...cwdCounts.values());
// Use most recent if it has at least 25% of the max count
if (mostRecentCount >= maxCount * 0.25) {
extractedPath = latestCwd;
} else {
// Otherwise use the most frequently used cwd
for (const [cwd, count] of cwdCounts.entries()) {
if (count === maxCount) {
extractedPath = cwd;
break;
}
}
}
// Fallback (shouldn't reach here)
if (!extractedPath) {
extractedPath = latestCwd || projectName.replace(/-/g, '/');
}
}
}
// Cache the result
projectDirectoryCache.set(projectName, extractedPath);
return extractedPath;
} catch (error) {
// If the directory doesn't exist, just use the decoded project name
if (error.code === 'ENOENT') {
extractedPath = projectName.replace(/-/g, '/');
} else {
console.error(`Error extracting project directory for ${projectName}:`, error);
// Fall back to decoded project name for other errors
extractedPath = projectName.replace(/-/g, '/');
}
// Cache the fallback result too
projectDirectoryCache.set(projectName, extractedPath);
return extractedPath;
}
}
async function getSessions(projectName, limit = 5, offset = 0) {
const projectDir = path.join(os.homedir(), '.claude', 'projects', projectName);
@@ -811,561 +651,6 @@ async function deleteCodexSession(sessionId) {
}
}
async function searchConversations(query, limit = 50, onProjectResult = null, signal = null) {
const safeQuery = typeof query === 'string' ? query.trim() : '';
const safeLimit = Math.max(1, Math.min(Number.isFinite(limit) ? limit : 50, 200));
const claudeDir = path.join(os.homedir(), '.claude', 'projects');
const config = await loadProjectConfig();
const results = [];
let totalMatches = 0;
const words = safeQuery.toLowerCase().split(/\s+/).filter(w => w.length > 0);
if (words.length === 0) return { results: [], totalMatches: 0, query: safeQuery };
const isAborted = () => signal?.aborted === true;
const isSystemMessage = (textContent) => {
return typeof textContent === 'string' && (
textContent.startsWith('<command-name>') ||
textContent.startsWith('<command-message>') ||
textContent.startsWith('<command-args>') ||
textContent.startsWith('<local-command-stdout>') ||
textContent.startsWith('<system-reminder>') ||
textContent.startsWith('Caveat:') ||
textContent.startsWith('This session is being continued from a previous') ||
textContent.startsWith('Invalid API key') ||
textContent.includes('{"subtasks":') ||
textContent.includes('CRITICAL: You MUST respond with ONLY a JSON') ||
textContent === 'Warmup'
);
};
const extractText = (content) => {
if (typeof content === 'string') return content;
if (Array.isArray(content)) {
return content
.filter(part => part.type === 'text' && part.text)
.map(part => part.text)
.join(' ');
}
return '';
};
const escapeRegex = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const wordPatterns = words.map(w => new RegExp(`(?<!\\p{L})${escapeRegex(w)}(?!\\p{L})`, 'u'));
const allWordsMatch = (textLower) => {
return wordPatterns.every(p => p.test(textLower));
};
const buildSnippet = (text, textLower, snippetLen = 150) => {
let firstIndex = -1;
let firstWordLen = 0;
for (const w of words) {
const re = new RegExp(`(?<!\\p{L})${escapeRegex(w)}(?!\\p{L})`, 'u');
const m = re.exec(textLower);
if (m && (firstIndex === -1 || m.index < firstIndex)) {
firstIndex = m.index;
firstWordLen = w.length;
}
}
if (firstIndex === -1) firstIndex = 0;
const halfLen = Math.floor(snippetLen / 2);
let start = Math.max(0, firstIndex - halfLen);
let end = Math.min(text.length, firstIndex + halfLen + firstWordLen);
let snippet = text.slice(start, end).replace(/\n/g, ' ');
const prefix = start > 0 ? '...' : '';
const suffix = end < text.length ? '...' : '';
snippet = prefix + snippet + suffix;
const snippetLower = snippet.toLowerCase();
const highlights = [];
for (const word of words) {
const re = new RegExp(`(?<!\\p{L})${escapeRegex(word)}(?!\\p{L})`, 'gu');
let match;
while ((match = re.exec(snippetLower)) !== null) {
highlights.push({ start: match.index, end: match.index + word.length });
}
}
highlights.sort((a, b) => a.start - b.start);
const merged = [];
for (const h of highlights) {
const last = merged[merged.length - 1];
if (last && h.start <= last.end) {
last.end = Math.max(last.end, h.end);
} else {
merged.push({ ...h });
}
}
return { snippet, highlights: merged };
};
try {
await fs.access(claudeDir);
const entries = await fs.readdir(claudeDir, { withFileTypes: true });
const projectDirs = entries.filter(e => e.isDirectory());
let scannedProjects = 0;
const totalProjects = projectDirs.length;
for (const projectEntry of projectDirs) {
if (totalMatches >= safeLimit || isAborted()) break;
const projectName = projectEntry.name;
const projectDir = path.join(claudeDir, projectName);
const displayName = config[projectName]?.displayName
|| await generateDisplayName(projectName);
let files;
try {
files = await fs.readdir(projectDir);
} catch {
continue;
}
const jsonlFiles = files.filter(
file => file.endsWith('.jsonl') && !file.startsWith('agent-')
);
// Also include the DB `projectId` so the frontend (which now identifies
// projects by `projectId`) can match search results to the
// currently-loaded project list without a second round-trip.
let searchProjectId = null;
try {
const resolvedPath = await extractProjectDirectory(projectName);
const dbRow = projectsDb.getProjectPath(resolvedPath);
if (dbRow?.project_id) {
searchProjectId = dbRow.project_id;
}
} catch {
// Best-effort: if we cannot resolve the projectId, the result is still
// usable on the backend but the frontend will skip the auto-select.
}
const projectResult = {
projectId: searchProjectId,
projectName,
projectDisplayName: displayName,
sessions: []
};
for (const file of jsonlFiles) {
if (totalMatches >= safeLimit || isAborted()) break;
const filePath = path.join(projectDir, file);
const sessionMatches = new Map();
const sessionSummaries = new Map();
const pendingSummaries = new Map();
const sessionLastMessages = new Map();
let currentSessionId = null;
try {
const fileStream = fsSync.createReadStream(filePath);
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
for await (const line of rl) {
if (totalMatches >= safeLimit || isAborted()) break;
if (!line.trim()) continue;
let entry;
try {
entry = JSON.parse(line);
} catch {
continue;
}
if (entry.sessionId) {
currentSessionId = entry.sessionId;
}
if (entry.type === 'summary' && entry.summary) {
const sid = entry.sessionId || currentSessionId;
if (sid) {
sessionSummaries.set(sid, entry.summary);
} else if (entry.leafUuid) {
pendingSummaries.set(entry.leafUuid, entry.summary);
}
}
// Apply pending summary via parentUuid
if (entry.parentUuid && currentSessionId && !sessionSummaries.has(currentSessionId)) {
const pending = pendingSummaries.get(entry.parentUuid);
if (pending) sessionSummaries.set(currentSessionId, pending);
}
// Track last user/assistant message for fallback title
if (entry.message?.content && currentSessionId && !entry.isApiErrorMessage) {
const role = entry.message.role;
if (role === 'user' || role === 'assistant') {
const text = extractText(entry.message.content);
if (text && !isSystemMessage(text)) {
if (!sessionLastMessages.has(currentSessionId)) {
sessionLastMessages.set(currentSessionId, {});
}
const msgs = sessionLastMessages.get(currentSessionId);
if (role === 'user') msgs.user = text;
else msgs.assistant = text;
}
}
}
if (!entry.message?.content) continue;
if (entry.message.role !== 'user' && entry.message.role !== 'assistant') continue;
if (entry.isApiErrorMessage) continue;
const text = extractText(entry.message.content);
if (!text || isSystemMessage(text)) continue;
const textLower = text.toLowerCase();
if (!allWordsMatch(textLower)) continue;
const sessionId = entry.sessionId || currentSessionId || file.replace('.jsonl', '');
if (!sessionMatches.has(sessionId)) {
sessionMatches.set(sessionId, []);
}
const matches = sessionMatches.get(sessionId);
if (matches.length < 2) {
const { snippet, highlights } = buildSnippet(text, textLower);
matches.push({
role: entry.message.role,
snippet,
highlights,
timestamp: entry.timestamp || null,
provider: 'claude',
messageUuid: entry.uuid || null
});
totalMatches++;
}
}
} catch {
continue;
}
for (const [sessionId, matches] of sessionMatches) {
projectResult.sessions.push({
sessionId,
provider: 'claude',
sessionSummary: sessionSummaries.get(sessionId) || (() => {
const msgs = sessionLastMessages.get(sessionId);
const lastMsg = msgs?.user || msgs?.assistant;
return lastMsg ? (lastMsg.length > 50 ? lastMsg.substring(0, 50) + '...' : lastMsg) : 'New Session';
})(),
matches
});
}
}
// Search Codex sessions for this project
try {
const actualProjectDir = await extractProjectDirectory(projectName);
if (actualProjectDir && !isAborted() && totalMatches < safeLimit) {
await searchCodexSessionsForProject(
actualProjectDir, projectResult, words, allWordsMatch, extractText, isSystemMessage,
buildSnippet, safeLimit, () => totalMatches, (n) => { totalMatches += n; }, isAborted
);
}
} catch {
// Skip codex search errors
}
// Search Gemini sessions for this project
try {
const actualProjectDir = await extractProjectDirectory(projectName);
if (actualProjectDir && !isAborted() && totalMatches < safeLimit) {
await searchGeminiSessionsForProject(
actualProjectDir, projectResult, words, allWordsMatch,
buildSnippet, safeLimit, () => totalMatches, (n) => { totalMatches += n; }
);
}
} catch {
// Skip gemini search errors
}
scannedProjects++;
if (projectResult.sessions.length > 0) {
results.push(projectResult);
if (onProjectResult) {
onProjectResult({ projectResult, totalMatches, scannedProjects, totalProjects });
}
} else if (onProjectResult && scannedProjects % 10 === 0) {
onProjectResult({ projectResult: null, totalMatches, scannedProjects, totalProjects });
}
}
} catch {
// claudeDir doesn't exist
}
return { results, totalMatches, query: safeQuery };
}
async function searchCodexSessionsForProject(
projectPath, projectResult, words, allWordsMatch, extractText, isSystemMessage,
buildSnippet, limit, getTotalMatches, addMatches, isAborted
) {
const normalizedProjectPath = normalizeComparablePath(projectPath);
if (!normalizedProjectPath) return;
const codexSessionsDir = path.join(os.homedir(), '.codex', 'sessions');
try {
await fs.access(codexSessionsDir);
} catch {
return;
}
const jsonlFiles = await findCodexJsonlFiles(codexSessionsDir);
for (const filePath of jsonlFiles) {
if (getTotalMatches() >= limit || isAborted()) break;
try {
const fileStream = fsSync.createReadStream(filePath);
const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
// First pass: read session_meta to check project path match
let sessionMeta = null;
for await (const line of rl) {
if (!line.trim()) continue;
try {
const entry = JSON.parse(line);
if (entry.type === 'session_meta' && entry.payload) {
sessionMeta = entry.payload;
break;
}
} catch { continue; }
}
// Skip sessions that don't belong to this project
if (!sessionMeta) continue;
const sessionProjectPath = normalizeComparablePath(sessionMeta.cwd);
if (sessionProjectPath !== normalizedProjectPath) continue;
// Second pass: re-read file to find matching messages
const fileStream2 = fsSync.createReadStream(filePath);
const rl2 = readline.createInterface({ input: fileStream2, crlfDelay: Infinity });
let latestUserMessageText = null;
const matches = [];
for await (const line of rl2) {
if (getTotalMatches() >= limit || isAborted()) break;
if (!line.trim()) continue;
let entry;
try { entry = JSON.parse(line); } catch { continue; }
let text = null;
let role = null;
if (entry.type === 'event_msg' && entry.payload?.type === 'user_message' && entry.payload.message) {
text = entry.payload.message;
role = 'user';
latestUserMessageText = text;
} else if (entry.type === 'response_item' && entry.payload?.type === 'message') {
const contentParts = entry.payload.content || [];
if (entry.payload.role === 'user') {
text = contentParts
.filter(p => p.type === 'input_text' && p.text)
.map(p => p.text)
.join(' ');
role = 'user';
if (text) latestUserMessageText = text;
} else if (entry.payload.role === 'assistant') {
text = contentParts
.filter(p => p.type === 'output_text' && p.text)
.map(p => p.text)
.join(' ');
role = 'assistant';
}
}
if (!text || !role) continue;
const textLower = text.toLowerCase();
if (!allWordsMatch(textLower)) continue;
if (matches.length < 2) {
const { snippet, highlights } = buildSnippet(text, textLower);
matches.push({ role, snippet, highlights, timestamp: entry.timestamp || null, provider: 'codex' });
addMatches(1);
}
}
if (matches.length > 0) {
projectResult.sessions.push({
sessionId: sessionMeta.id,
provider: 'codex',
sessionSummary: latestUserMessageText
? (latestUserMessageText.length > 50 ? latestUserMessageText.substring(0, 50) + '...' : latestUserMessageText)
: 'Codex Session',
matches
});
}
} catch {
continue;
}
}
}
async function searchGeminiSessionsForProject(
projectPath, projectResult, words, allWordsMatch,
buildSnippet, limit, getTotalMatches, addMatches
) {
// 1) Search in-memory sessions (created via UI)
for (const [sessionId, session] of sessionManager.sessions) {
if (getTotalMatches() >= limit) break;
if (session.projectPath !== projectPath) continue;
const matches = [];
for (const msg of session.messages) {
if (getTotalMatches() >= limit) break;
if (msg.role !== 'user' && msg.role !== 'assistant') continue;
const text = typeof msg.content === 'string' ? msg.content
: Array.isArray(msg.content) ? msg.content.filter(p => p.type === 'text').map(p => p.text).join(' ')
: '';
if (!text) continue;
const textLower = text.toLowerCase();
if (!allWordsMatch(textLower)) continue;
if (matches.length < 2) {
const { snippet, highlights } = buildSnippet(text, textLower);
matches.push({
role: msg.role, snippet, highlights,
timestamp: msg.timestamp ? msg.timestamp.toISOString() : null,
provider: 'gemini'
});
addMatches(1);
}
}
if (matches.length > 0) {
const firstUserMsg = session.messages.find(m => m.role === 'user');
const summary = firstUserMsg?.content
? (typeof firstUserMsg.content === 'string'
? (firstUserMsg.content.length > 50 ? firstUserMsg.content.substring(0, 50) + '...' : firstUserMsg.content)
: 'Gemini Session')
: 'Gemini Session';
projectResult.sessions.push({
sessionId,
provider: 'gemini',
sessionSummary: summary,
matches
});
}
}
// 2) Search Gemini CLI sessions on disk (~/.gemini/tmp/<project>/chats/*.json)
const normalizedProjectPath = normalizeComparablePath(projectPath);
if (!normalizedProjectPath) return;
const geminiTmpDir = path.join(os.homedir(), '.gemini', 'tmp');
try {
await fs.access(geminiTmpDir);
} catch {
return;
}
const trackedSessionIds = new Set();
for (const [sid] of sessionManager.sessions) {
trackedSessionIds.add(sid);
}
let projectDirs;
try {
projectDirs = await fs.readdir(geminiTmpDir);
} catch {
return;
}
for (const projectDir of projectDirs) {
if (getTotalMatches() >= limit) break;
const projectRootFile = path.join(geminiTmpDir, projectDir, '.project_root');
let projectRoot;
try {
projectRoot = (await fs.readFile(projectRootFile, 'utf8')).trim();
} catch {
continue;
}
if (normalizeComparablePath(projectRoot) !== normalizedProjectPath) continue;
const chatsDir = path.join(geminiTmpDir, projectDir, 'chats');
let chatFiles;
try {
chatFiles = await fs.readdir(chatsDir);
} catch {
continue;
}
for (const chatFile of chatFiles) {
if (getTotalMatches() >= limit) break;
if (!chatFile.endsWith('.json')) continue;
try {
const filePath = path.join(chatsDir, chatFile);
const data = await fs.readFile(filePath, 'utf8');
const session = JSON.parse(data);
if (!session.messages || !Array.isArray(session.messages)) continue;
const cliSessionId = session.sessionId || chatFile.replace('.json', '');
if (trackedSessionIds.has(cliSessionId)) continue;
const matches = [];
let firstUserText = null;
for (const msg of session.messages) {
if (getTotalMatches() >= limit) break;
const role = msg.type === 'user' ? 'user'
: (msg.type === 'gemini' || msg.type === 'assistant') ? 'assistant'
: null;
if (!role) continue;
let text = '';
if (typeof msg.content === 'string') {
text = msg.content;
} else if (Array.isArray(msg.content)) {
text = msg.content
.filter(p => p.text)
.map(p => p.text)
.join(' ');
}
if (!text) continue;
if (role === 'user' && !firstUserText) firstUserText = text;
const textLower = text.toLowerCase();
if (!allWordsMatch(textLower)) continue;
if (matches.length < 2) {
const { snippet, highlights } = buildSnippet(text, textLower);
matches.push({
role, snippet, highlights,
timestamp: msg.timestamp || null,
provider: 'gemini'
});
addMatches(1);
}
}
if (matches.length > 0) {
const summary = firstUserText
? (firstUserText.length > 50 ? firstUserText.substring(0, 50) + '...' : firstUserText)
: 'Gemini CLI Session';
projectResult.sessions.push({
sessionId: cliSessionId,
provider: 'gemini',
sessionSummary: summary,
matches
});
}
} catch {
continue;
}
}
}
}
// Only functions with consumers outside this module are exported. Folder-name
// based helpers (`getSessions`, `deleteSession`, etc.) are kept as internal
// implementation details of the id-based wrappers below.
@@ -1373,6 +658,5 @@ export {
deleteSessionById,
getProjectPathById,
claudeFolderNameFromPath,
deleteCodexSession,
searchConversations
deleteCodexSession
};