From 3e268e201a35e72932f0d463ed4b329dd5b3a8d9 Mon Sep 17 00:00:00 2001 From: Haileyesus Date: Wed, 25 Mar 2026 10:46:08 +0300 Subject: [PATCH] feat: implement basic file watcher and session updater --- docs/backend/architecture.md | 910 +++++++++++++----- server/src/modules/watcher/file-watcher.ts | 165 ++++ .../get-workspaces/get-workspaces.ts | 324 +++++++ server/src/realtime/chat.gateway.ts | 0 server/src/realtime/events.ts | 0 server/src/realtime/index.ts | 0 server/src/runner.ts | 35 +- server/src/shared/database/init-db.ts | 2 +- server/src/shared/database/migrations.ts | 12 +- .../database/repositories/scan-state.db.ts | 38 + .../database/repositories/session-names.ts | 109 --- .../database/repositories/sessions.db.ts | 87 ++ .../workspace-original-paths.db.ts | 12 + server/src/shared/database/schema.ts | 40 +- server/src/shared/database/types.ts | 22 +- server/src/shared/platform/path.test.ts | 19 +- server/src/shared/platform/path.ts | 47 +- server/src/shared/types/app.ts | 3 + 18 files changed, 1448 insertions(+), 377 deletions(-) create mode 100644 server/src/modules/watcher/file-watcher.ts create mode 100644 server/src/modules/workspace/get-workspaces/get-workspaces.ts create mode 100644 server/src/realtime/chat.gateway.ts create mode 100644 server/src/realtime/events.ts create mode 100644 server/src/realtime/index.ts create mode 100644 server/src/shared/database/repositories/scan-state.db.ts delete mode 100644 server/src/shared/database/repositories/session-names.ts create mode 100644 server/src/shared/database/repositories/sessions.db.ts create mode 100644 server/src/shared/database/repositories/workspace-original-paths.db.ts diff --git a/docs/backend/architecture.md b/docs/backend/architecture.md index 4a311949..3d92878d 100644 --- a/docs/backend/architecture.md +++ b/docs/backend/architecture.md @@ -2,326 +2,770 @@ ## Goal -This structure keeps the Day 1 runtime stable while giving the backend a clear home for shared HTTP concerns, shared types, OpenAPI work, and feature modules. The current runtime still lives in `server/index.js`, but everything new should be shaped around the layout below. +This document defines the target backend shape for the TypeScript refactor in `server/src`. -## Structure +The main constraints are: + +- Keep the current API paths stable while the internals move out of `server/index.js` and `server/routes/*.js`. +- Migrate one route family at a time instead of doing a big-bang rewrite. +- Keep controllers thin and move business logic into services and repositories. +- Keep transport concerns, provider adapters, and domain logic separate. + +## Current State + +As of March 18, 2026: + +- `server/src/bootstrap.ts` starts the TypeScript backend entrypoint. +- `server/src/app.ts` still bridges into `server/src/runner.ts`. +- `server/src/shared/*` already contains reusable TypeScript building blocks such as: + - `shared/http/api-response.ts` + - `shared/http/async-handler.ts` + - `shared/http/error-handler.ts` + - `shared/http/request-context.ts` + - `shared/utils/app-error.ts` + - `shared/database/repositories/*` +- `server/src/modules/*` mostly exist as placeholder folders. +- The real runtime behavior still mostly lives in: + - `server/index.js` + - `server/routes/*.js` +- `docs/backend/endpoint-inventory.md` is the migration checklist for the existing HTTP surface. + +## Recommended Target Structure ```text server/ - index.js - start.js + index.js # legacy runtime bridge during migration + start.js # production entrypoint for compiled output src/ - app.ts - bootstrap.ts + bootstrap.ts # starts the backend process + app.ts # creates the app/server and registers modules + runner.ts # transitional runtime while legacy code still exists config/ + load-env-vars.ts runtime.ts + realtime/ + index.ts # attaches websocket handlers to the HTTP server + chat.gateway.ts # chat websocket behavior + shell.gateway.ts # shell websocket behavior + events.ts # shared broadcast helpers shared/ + auth/ + authenticate-token.ts # future TS auth middleware + validate-api-key.ts # future TS API-key middleware + database/ + connection.ts + init-db.ts + migrations.ts + schema.ts + types.ts + repositories/ + api-keys.ts + app-config.ts + credentials.ts + session-names.ts + users.ts + docs/ + openapi.ts http/ api-response.ts async-handler.ts error-handler.ts not-found-handler.ts request-context.ts + input/ + parse-boolean.ts # future shared query/body parsers + parse-pagination.ts platform/ - runtime-platform.ts - text.ts - stream.ts - shell.ts + index.ts path.ts + runtime-platform.ts + shell.ts + stream.ts + text.ts + types.ts types/ app.ts http.ts - docs/ - openapi.ts utils/ app-error.ts logger.ts modules/ auth/ + index.ts + auth.routes.ts + auth.controller.ts + auth.service.ts + auth.schemas.ts + auth.types.ts cli-auth/ + index.ts + cli-auth.routes.ts + cli-auth.controller.ts + cli-auth.service.ts user/ + index.ts + user.routes.ts + user.controller.ts + user.service.ts + user.schemas.ts + user.types.ts settings/ + index.ts + settings.routes.ts + settings.controller.ts + settings.service.ts + settings.schemas.ts + commands/ + index.ts + commands.routes.ts + commands.controller.ts + commands.service.ts projects/ + index.ts + projects.routes.ts + projects.controller.ts + projects.service.ts + projects.schemas.ts + workspace-path.policy.ts files/ + index.ts + files.routes.ts + files.controller.ts + files.service.ts + files.schemas.ts sessions/ + index.ts + sessions.routes.ts + sessions.controller.ts + sessions.service.ts + sessions.schemas.ts git/ + index.ts + git.routes.ts + git.controller.ts + git.service.ts taskmaster/ + index.ts + taskmaster.routes.ts + taskmaster.controller.ts + taskmaster.service.ts agent/ + index.ts + agent.routes.ts + agent.controller.ts + agent.service.ts + system/ + index.ts + system.routes.ts + system.controller.ts + system.service.ts + spa-fallback.ts providers/ claude/ + index.ts + claude.service.ts + claude-session.service.ts codex/ + index.ts + codex.routes.ts + codex.controller.ts + codex.service.ts cursor/ + index.ts + cursor.routes.ts + cursor.controller.ts + cursor.service.ts gemini/ + index.ts + gemini.routes.ts + gemini.controller.ts + gemini.service.ts mcp/ + index.ts + mcp.routes.ts + mcp.controller.ts + mcp.service.ts plugins/ + index.ts + plugins.routes.ts + plugins.controller.ts + plugins.service.ts + testing/ + fixtures/ ``` -## File And Folder Roles +## Why This Shape Works For This Repo -- `server/index.js` - Temporary compatibility boundary for the old monolith. Day 1 keeps behavior here so the new TypeScript layout can grow around a stable runtime. - Example: the existing websocket handlers, inline routes, and provider startup still live here until they are migrated module by module. +- You already have shared TypeScript HTTP/database utilities in `server/src/shared`. +- The legacy route surface is already grouped by domain in `server/routes/*.js`. +- The biggest remaining problem is not "what framework should I use", it is "where should each existing endpoint live and how do I migrate it without breaking paths". +- A module-per-domain structure solves that without forcing a rewrite of all helpers at once. -- `server/start.js` - Thin production entrypoint for the compiled backend output. - Example: `server:start` uses this file to verify `server/dist/bootstrap.js` exists and then loads it. +## Route Ownership -- `src/bootstrap.ts` - Executable backend entrypoint used by `npm run server` and `npm run server:dev`. - Example: `bootstrap.ts` should stay thin and do nothing except start the app, so later it remains safe to call in dev, prod, tests, or worker modes. +Keep the public paths stable. Only move the code behind them. -- `src/app.ts` - Composition root for the backend application. - Example: today it bridges into `index.js`; later it will create the Express app, apply shared middleware, register modules, attach websocket setup, and return the running application shape. +| Module | Mount path(s) | Current source(s) | Notes | +| --- | --- | --- | --- | +| `auth` | `/api/auth` | `server/routes/auth.js` | Login, register, logout, auth status, current user | +| `cli-auth` | `/api/cli` | `server/routes/cli-auth.js` | Provider CLI auth status endpoints | +| `user` | `/api/user` | `server/routes/user.js` | Git identity and onboarding state | +| `settings` | `/api/settings` | `server/routes/settings.js` | API keys, credentials, notification preferences, push setup | +| `commands` | `/api/commands` | `server/routes/commands.js` | Slash command list, load, execute | +| `projects` | `/api/projects`, `/api/browse-filesystem`, `/api/create-folder` | `server/routes/projects.js`, `server/index.js` | Workspace discovery and workspace creation helpers | +| `files` | `/api/projects/:projectName/file`, `/api/projects/:projectName/files` | `server/index.js` | File tree, read/write, upload, rename, delete | +| `sessions` | `/api/projects/:projectName/sessions`, `/api/sessions/:sessionId/rename`, `/api/search/conversations` | `server/index.js`, provider route files | Project/provider session history | +| `git` | `/api/git` | `server/routes/git.js` | Repo operations and git automation | +| `taskmaster` | `/api/taskmaster` | `server/routes/taskmaster.js` | TaskMaster setup, PRDs, tasks, parsing | +| `agent` | `/api/agent` | `server/routes/agent.js` | External agent orchestration | +| `system` | `/health`, `/api/system`, `/api/transcribe`, SPA fallback | `server/index.js` | Health, app update, transcription, non-API fallback. Register the SPA fallback last. | +| `providers/codex` | `/api/codex` | `server/routes/codex.js` | Config, MCP-adjacent behavior, Codex sessions | +| `providers/cursor` | `/api/cursor` | `server/routes/cursor.js` | Config, MCP, Cursor sessions | +| `providers/gemini` | `/api/gemini` | `server/routes/gemini.js` | Gemini session endpoints | +| `providers/mcp` | `/api/mcp`, `/api/mcp-utils` | `server/routes/mcp.js`, `server/routes/mcp-utils.js` | Shared MCP HTTP endpoints | +| `providers/plugins` | `/api/plugins` | `server/routes/plugins.js` | Plugin install, enable/disable, assets | +| `providers/claude` | no standalone HTTP prefix yet | `server/index.js`, `server/claude-sdk.js` | Service/adapters used by chat and agent flows | +| `realtime` | `/ws`, `/shell` | `server/index.js` | Websocket transport, not HTTP routes | -- `src/config/` - Runtime configuration helpers and environment-aware path logic. - Example: `config/runtime.ts` resolves the project root, server root, legacy runtime path, and built bootstrap path without scattering path math across the app. +## Module Convention -- `src/shared/http/` - Shared HTTP-level behavior that every module can reuse. - Example: `api-response.ts` is where standard API response builders live. - Example: `error-handler.ts` is where thrown `AppError` instances get translated into JSON payloads. - Example: `request-context.ts` is where request IDs, timestamps, and per-request metadata are attached. - Example: `async-handler.ts` removes repeated `try/catch(next)` wrappers in controllers. - Example: `not-found-handler.ts` is the generic fallback for unknown API routes. +Every feature module should follow the same internal shape: -- `src/shared/platform/` - Shared OS-adapter helpers for shell spawning, line ending normalization, streaming stdout/stderr parsing, and path normalization. - Example: `platform/stream.ts` is where process output gets split into complete lines without leaking CRLF edge cases into feature code. - Example: `platform/shell.ts` is where PowerShell-vs-bash command construction lives so provider modules do not branch on `process.platform`. +```text +modules/user/ + index.ts # registers the module on the app + user.routes.ts # express.Router + middleware order + user.controller.ts # request/response mapping only + user.service.ts # business rules and orchestration + user.schemas.ts # input parsing and validation + user.types.ts # module-local types only +``` -- `src/shared/types/` - Global type aliases that are safe to share across modules. This layer uses `type`, not `interface`. - Example: `types/http.ts` defines `ApiMeta`, `ApiErrorShape`, `RequestContext`, `AuthenticatedRequest`, and `EndpointInventoryRecord`. - Example: `types/app.ts` defines `RuntimePaths`, `AppLocals`, and `ServerApplication`. +### Responsibilities -- `src/shared/docs/` - Shared documentation helpers and future OpenAPI registry code. - Example: `docs/openapi.ts` is the future home for global tags like `Auth`, `Projects`, `Files`, `Git`, and `Providers`, plus reusable schema registration. -- `src/shared/utils/` - Shared non-HTTP utilities that stay generic and reusable. - Example: `utils/app-error.ts` defines `AppError`, which feature modules can throw without knowing how HTTP serialization works. - Example: `utils/logger.ts` is the centralized logger surface so modules do not ad-hoc `console.log` everywhere. +- `index.ts` + Owns the mount path. + `app.ts` should not know route internals. -- `src/modules/` - Feature boundaries. Every business area gets its own folder so request schemas, controllers, services, serializers, and docs stay close to the feature they belong to. +- `*.routes.ts` + Declares routes, middleware order, and child-router mounting. + It should not contain SQL, filesystem logic, or long validation blocks. -- `src/modules/auth/` - Local authentication flows. - Example: login, register, logout, and auth-status endpoints belong here. +- `*.controller.ts` + Reads `req`, calls schema parsers and services, then writes the response. + It should stay thin. -- `src/modules/cli-auth/` - CLI/provider authentication status flows for Claude, Cursor, Codex, and Gemini CLIs. - Example: `/api/cli/claude/status` belongs here because it checks local CLI auth rather than app-user auth. +- `*.service.ts` + Owns business logic. + It can call repositories, filesystem helpers, platform helpers, provider adapters, and child processes. -- `src/modules/user/` - User-specific settings and onboarding state. - Example: git identity setup and onboarding completion endpoints belong here. +- `*.schemas.ts` + Validates/parses request data close to the module. + Throw `AppError` with `400` status for bad input. -- `src/modules/settings/` - App-level stored secrets and toggles. - Example: API keys and credential storage endpoints belong here because they configure backend access rather than user identity. +- `*.types.ts` + Holds module-local types that do not belong in `shared/types`. -- `src/modules/projects/` - Workspace and project registration concerns. - Example: project listing, project creation, workspace creation, and project rename/delete flows belong here. +## Application Setup -- `src/modules/files/` - File tree and workspace file operations only. - Example: read file, save file, upload file, create file, rename file, delete file, and image upload endpoints belong here. - Example boundary rule: this module should not decide how projects are discovered; it only operates inside an already resolved project/workspace. +The current `server/src/app.ts` still starts a transitional runner. +Once the HTTP runtime moves into `server/src`, the shape should look like this: -- `src/modules/sessions/` - Conversation and provider session history concerns. - Example: list sessions, fetch session messages, rename sessions, delete sessions, token-usage lookups, and conversation search belong here. +```ts +import cors from 'cors'; +import express from 'express'; +import http from 'http'; -- `src/modules/git/` - Repository operations and git intelligence. - Example: status, diff, branch listing, checkout, commit, push, publish, discard, and AI commit-message generation endpoints belong here. +import { registerAgentModule } from '@/modules/agent/index.js'; +import { registerAuthModule } from '@/modules/auth/index.js'; +import { registerCliAuthModule } from '@/modules/cli-auth/index.js'; +import { registerCommandsModule } from '@/modules/commands/index.js'; +import { registerFilesModule } from '@/modules/files/index.js'; +import { registerGitModule } from '@/modules/git/index.js'; +import { registerProjectsModule } from '@/modules/projects/index.js'; +import { registerSessionsModule } from '@/modules/sessions/index.js'; +import { registerSettingsModule } from '@/modules/settings/index.js'; +import { registerSystemModule } from '@/modules/system/index.js'; +import { registerTaskmasterModule } from '@/modules/taskmaster/index.js'; +import { registerUserModule } from '@/modules/user/index.js'; +import { registerCodexProviderModule } from '@/modules/providers/codex/index.js'; +import { registerCursorProviderModule } from '@/modules/providers/cursor/index.js'; +import { registerGeminiProviderModule } from '@/modules/providers/gemini/index.js'; +import { registerMcpProviderModule } from '@/modules/providers/mcp/index.js'; +import { registerPluginsProviderModule } from '@/modules/providers/plugins/index.js'; +import { registerSpaFallback } from '@/modules/system/spa-fallback.js'; +import { errorHandler } from '@/shared/http/error-handler.js'; +import { notFoundHandler } from '@/shared/http/not-found-handler.js'; +import { requestContextMiddleware } from '@/shared/http/request-context.js'; +import { attachRealtimeHandlers } from '@/realtime/index.js'; -- `src/modules/taskmaster/` - TaskMaster-specific project workflows. - Example: detect installation, initialize TaskMaster, manage PRDs, add/update tasks, parse PRDs, and apply templates belong here. +export function createHttpRuntime() { + const app = express(); + const server = http.createServer(app); -- `src/modules/agent/` - External agent execution API. - Example: `/api/agent` belongs here because it orchestrates provider selection, cloning, project reuse, branch creation, streaming, and optional PR creation. + // Global middleware should be registered once, before feature modules. + app.use(cors()); + app.use(express.json({ limit: '10mb' })); + app.use(express.urlencoded({ extended: true })); + app.use(requestContextMiddleware); -- `src/modules/providers/` - Provider-specific integrations that are narrower than the general `agent` API. - Example: provider session readers or provider-specific config endpoints should live here so Claude, Codex, Cursor, and Gemini logic do not bleed into unrelated modules. + // Keep health/system API routes early so diagnostics still work + // even if a later feature module throws during registration. + registerSystemModule(app); -- `src/modules/providers/claude/` - Claude-specific runtime concerns. - Example: if Claude gets module-specific schemas or adapters later, they belong here rather than inside generic session code. + // Public/auth routes first. + registerAuthModule(app); -- `src/modules/providers/codex/` - Codex-specific config, session, and MCP-adjacent logic. - Example: Codex MCP CLI endpoints and session history parsing can move here over time. + // Authenticated feature modules after auth. + registerCliAuthModule(app); + registerUserModule(app); + registerSettingsModule(app); + registerCommandsModule(app); + registerProjectsModule(app); + registerFilesModule(app); + registerSessionsModule(app); + registerGitModule(app); + registerTaskmasterModule(app); + registerAgentModule(app); + registerCodexProviderModule(app); + registerCursorProviderModule(app); + registerGeminiProviderModule(app); + registerMcpProviderModule(app); + registerPluginsProviderModule(app); -- `src/modules/providers/cursor/` - Cursor-specific config, MCP, and stored session behavior. - Example: Cursor config reads, MCP server mutation, and SQLite-backed session history belong here. + // Websocket setup should live outside HTTP controllers. + attachRealtimeHandlers(server, app); -- `src/modules/providers/gemini/` - Gemini-specific config and session behavior. - Example: Gemini session message history and provider CLI lifecycle hooks belong here. + // The React fallback must be last or it will swallow real API routes. + registerSpaFallback(app); -- `src/modules/providers/mcp/` - MCP surfaces that are shared across providers or not owned by a single provider module. - Example: generic Claude MCP CLI/config endpoints and helper endpoints belong here. + app.use(notFoundHandler); + app.use(errorHandler); -- `src/modules/providers/plugins/` - Plugin runtime and plugin asset delivery. - Example: plugin listing, installation, update, enable/disable, and asset serving can move here even though plugins are not an LLM provider; this keeps third-party integration surfaces grouped together. + return { app, server }; +} +``` + +## Example Module: `user` + +This is a good first migration candidate because the endpoints are small, the domain is clear, and you already have a typed user repository in `server/src/shared/database/repositories/users.ts`. + +### Folder + +```text +server/src/modules/user/ + index.ts + user.routes.ts + user.controller.ts + user.service.ts + user.schemas.ts + user.types.ts +``` + +### `index.ts` + +```ts +import type { Express } from 'express'; + +import { createUserRouter } from './user.routes.js'; + +export function registerUserModule(app: Express): void { + // Keep the mount path in one place. + // This makes it easy to see who owns `/api/user`. + app.use('/api/user', createUserRouter()); +} +``` + +### `user.routes.ts` + +```ts +import { Router } from 'express'; + +import { asyncHandler } from '@/shared/http/async-handler.js'; +// Day 1: reuse the existing JS middleware while auth is still being migrated. +import { authenticateToken } from '../../../middleware/auth.js'; + +import { + completeOnboardingController, + getGitConfigController, + getOnboardingStatusController, + updateGitConfigController, +} from './user.controller.js'; + +export function createUserRouter(): Router { + const router = Router(); + + // Routes only describe the HTTP surface and middleware order. + router.get('/git-config', authenticateToken, asyncHandler(getGitConfigController)); + router.post('/git-config', authenticateToken, asyncHandler(updateGitConfigController)); + router.get('/onboarding-status', authenticateToken, asyncHandler(getOnboardingStatusController)); + router.post('/complete-onboarding', authenticateToken, asyncHandler(completeOnboardingController)); + + return router; +} +``` + +### `user.schemas.ts` + +```ts +import { AppError } from '@/shared/utils/app-error.js'; + +export type UpdateGitConfigInput = { + gitName: string; + gitEmail: string; +}; + +export function parseUpdateGitConfigInput(body: unknown): UpdateGitConfigInput { + const value = body as Partial | null; + + // Keep input validation close to the module so controllers stay thin. + if (!value?.gitName || !value?.gitEmail) { + throw new AppError('Git name and email are required', { + code: 'VALIDATION_ERROR', + statusCode: 400, + }); + } + + const gitName = value.gitName.trim(); + const gitEmail = value.gitEmail.trim(); + + if (!gitName) { + throw new AppError('Git name is required', { + code: 'VALIDATION_ERROR', + statusCode: 400, + }); + } + + if (!/^[^\\s@]+@[^\\s@]+\\.[^\\s@]+$/.test(gitEmail)) { + throw new AppError('Invalid email format', { + code: 'VALIDATION_ERROR', + statusCode: 400, + }); + } + + return { + gitName, + gitEmail, + }; +} +``` + +### `user.service.ts` + +```ts +import { userDb } from '@/shared/database/repositories/users.js'; +import { AppError } from '@/shared/utils/app-error.js'; + +import type { UpdateGitConfigInput } from './user.schemas.js'; + +export const userService = { + getGitConfig(userId: number) { + const gitConfig = userDb.getGitConfig(userId); + + return { + gitName: gitConfig?.git_name ?? null, + gitEmail: gitConfig?.git_email ?? null, + }; + }, + + updateGitConfig(userId: number, input: UpdateGitConfigInput) { + if (!userDb.getUserById(userId)) { + throw new AppError('User not found', { + code: 'USER_NOT_FOUND', + statusCode: 404, + }); + } + + // Services own business rules and side effects. + // If you later re-add `git config --global`, it belongs here. + userDb.updateGitConfig(userId, input.gitName, input.gitEmail); + + return { + gitName: input.gitName, + gitEmail: input.gitEmail, + }; + }, + + getOnboardingStatus(userId: number) { + return { + hasCompletedOnboarding: userDb.hasCompletedOnboarding(userId), + }; + }, + + completeOnboarding(userId: number) { + userDb.completeOnboarding(userId); + + return { + message: 'Onboarding completed successfully', + }; + }, +}; +``` + +### `user.controller.ts` + +```ts +import type { Response } from 'express'; + +import { + createApiMeta, + createApiSuccessResponse, +} from '@/shared/http/api-response.js'; +import { getRequestContext } from '@/shared/http/request-context.js'; +import type { AuthenticatedRequest } from '@/shared/types/http.js'; +import { AppError } from '@/shared/utils/app-error.js'; + +import { parseUpdateGitConfigInput } from './user.schemas.js'; +import { userService } from './user.service.js'; + +function getUserId(req: AuthenticatedRequest): number { + const userId = Number(req.user?.id); + + if (!Number.isFinite(userId)) { + throw new AppError('Authenticated user is missing', { + code: 'UNAUTHENTICATED', + statusCode: 401, + }); + } + + return userId; +} + +function getMeta(req: AuthenticatedRequest) { + const context = getRequestContext(req); + return createApiMeta(context?.requestId, context?.startedAt); +} + +export async function getGitConfigController( + req: AuthenticatedRequest, + res: Response +): Promise { + const data = userService.getGitConfig(getUserId(req)); + res.json(createApiSuccessResponse(data, getMeta(req))); +} + +export async function updateGitConfigController( + req: AuthenticatedRequest, + res: Response +): Promise { + const input = parseUpdateGitConfigInput(req.body); + const data = userService.updateGitConfig(getUserId(req), input); + res.json(createApiSuccessResponse(data, getMeta(req))); +} + +export async function getOnboardingStatusController( + req: AuthenticatedRequest, + res: Response +): Promise { + const data = userService.getOnboardingStatus(getUserId(req)); + res.json(createApiSuccessResponse(data, getMeta(req))); +} + +export async function completeOnboardingController( + req: AuthenticatedRequest, + res: Response +): Promise { + const data = userService.completeOnboarding(getUserId(req)); + res.json(createApiSuccessResponse(data, getMeta(req))); +} +``` + +## Nested Route Example: `files` + +`files` is the best example of why route ownership and mount paths matter. +The files module should stay separate from `projects`, but its routes still need access to `:projectName`. + +### `files/index.ts` + +```ts +import type { Express } from 'express'; + +import { createFilesRouter } from './files.routes.js'; + +export function registerFilesModule(app: Express): void { + // The files module owns project-scoped file operations. + app.use('/api/projects/:projectName', createFilesRouter()); +} +``` + +### `files.routes.ts` + +```ts +import { Router } from 'express'; + +import { asyncHandler } from '@/shared/http/async-handler.js'; +import { authenticateToken } from '../../../middleware/auth.js'; + +import { + deleteFilesController, + getFileController, + getFilesController, + putFileController, +} from './files.controller.js'; + +export function createFilesRouter(): Router { + // `mergeParams: true` lets this router read `req.params.projectName` + // from the mount path declared in `index.ts`. + const router = Router({ mergeParams: true }); + + router.get('/file', authenticateToken, asyncHandler(getFileController)); + router.put('/file', authenticateToken, asyncHandler(putFileController)); + router.get('/files', authenticateToken, asyncHandler(getFilesController)); + router.delete('/files', authenticateToken, asyncHandler(deleteFilesController)); + + return router; +} +``` + +### Why This Separation Is Useful + +- `projects` decides what a project/workspace is. +- `files` only operates inside a resolved project. +- `sessions` only deals with chat/session history. +- This prevents one giant `projects.service.ts` from becoming the new monolith. + +## Provider Module Pattern + +Provider modules should follow the same structure, but they usually own adapter logic as well as HTTP routes. + +Example shape for `providers/codex`: + +```text +modules/providers/codex/ + index.ts + codex.routes.ts + codex.controller.ts + codex.service.ts + codex.types.ts + codex-mcp.service.ts + codex-sessions.service.ts +``` + +Use this rule: + +- If the code is specific to one provider, keep it inside that provider folder. +- If the code is shared across multiple providers, move it into `shared/` or `providers/mcp/`. +- If the code is an external-facing orchestration endpoint that can call different providers, keep it in `modules/agent/`. + +## Realtime Structure + +Do not leave websocket logic inside `app.ts` or feature controllers. + +Use a dedicated `server/src/realtime/` area: + +```text +server/src/realtime/ + index.ts + chat.gateway.ts + shell.gateway.ts + events.ts +``` + +Suggested responsibilities: + +- `index.ts` + Attaches websocket handlers to the HTTP server. + +- `chat.gateway.ts` + Owns `/ws` message handling, provider session streaming, reconnect behavior, and approval flows. + +- `shell.gateway.ts` + Owns `/shell` PTY session lifecycle, resize events, and output streaming. + +- `events.ts` + Owns broadcast helpers used by HTTP modules such as TaskMaster or project updates. ## Boundary Rules -- `app.ts` wires modules together; it should not contain feature logic. -- `config/` resolves environment and filesystem context; it should not know HTTP payload details. -- `shared/http/` owns transport concerns; it should not know feature rules like how to rename a project. -- `shared/types/` only contains reusable type aliases; avoid feature-specific types here unless multiple modules truly share them. -- `modules//` owns its own future `routes`, `controllers`, `services`, `schemas`, and `docs`. -- `projects`, `files`, and `sessions` stay separate on purpose: - `projects` decides what a workspace/project is. - `files` operates inside a resolved workspace. - `sessions` manages chat/session history and search. -- `agent` stays separate from `providers`: - `agent` is orchestration for external callers. - `providers/*` are provider-specific adapters and APIs. +- `app.ts` wires modules together. It should not contain feature logic. +- `bootstrap.ts` starts the process. It should not know route details. +- `routes` files should not contain SQL, filesystem logic, provider process spawning, or long validation blocks. +- `controllers` should not import `better-sqlite3`, `fs`, `node-pty`, or raw provider SDKs. +- `services` own business rules, orchestration, and side effects. +- `repositories` own SQL only. +- `shared/*` is only for code used by two or more modules. +- `providers/*` is for provider-specific behavior. +- `agent` is orchestration above provider modules. +- `realtime/*` owns websocket transport, not HTTP modules. +- `system` owns health/update/fallback endpoints, not project/file/session behavior. -## Day 1 Notes +## Migration Order -- The runtime still executes through `server/index.js` for safety. -- The new `src/` structure is now the required home for all new backend code. -- The generated inventory in `docs/backend/endpoint-inventory.*` is the source of truth for what must be migrated into these folders next. +This order keeps risk low: -## Package Scripts +1. `auth`, `cli-auth`, `user` + Small route files, easy to validate, minimal path behavior. -These scripts live in `package.json`. The key distinction is: +2. `settings`, `commands` + Still mostly isolated and already grouped in dedicated route files. -- `server` and `server:dev` run the backend directly from TypeScript. -- `server:start` runs the compiled backend through `server/start.js`. -- `build` only builds the frontend. -- `server:build` only builds the backend. -- `start` runs the full production-style flow. +3. `projects`, `files`, `sessions` + These are more coupled and require clear boundaries around `projectName`. -### Development Scripts +4. `git`, `taskmaster` + Heavier side effects and more external process behavior. -- `npm run dev` - Starts the frontend and backend together. - Use this for normal full-stack development when you want Vite and the API server running at the same time. - Example: you are editing a React screen that calls `/api/projects` and also changing the backend route behavior. +5. `providers/*` + Each provider can move independently once shared platform helpers are stable. + +6. `agent` + Migrate last because it orchestrates several other pieces. + +7. `realtime/*` + Move websocket logic after the supporting services are already modular. + +## Day 1 Migration Rules + +- Keep path compatibility first. +- Move one route family at a time. +- It is acceptable to import legacy JS middleware or helpers from a new TS module temporarily. +- When a route family is migrated: + - register the new TS module in `app.ts` + - remove the old route registration from `server/index.js` + - keep the response shape unchanged unless you intentionally version it +- Prefer moving business logic into services first, then move transport code. + +## Practical Setup Checklist + +When you create a new module, use this checklist: + +1. Add the folder under `server/src/modules//`. +2. Create `index.ts`, `*.routes.ts`, `*.controller.ts`, `*.service.ts`, and `*.schemas.ts`. +3. Keep the mount path inside `index.ts`. +4. Reuse `asyncHandler`, `AppError`, `createApiSuccessResponse`, and `requestContextMiddleware`. +5. Reuse typed repositories from `server/src/shared/database/repositories/*` whenever possible. +6. Register the module in `server/src/app.ts`. +7. Move one legacy route at a time and verify the response shape still matches the frontend. + +## Package Script Notes + +These scripts already match the refactor workflow: - `npm run server:dev` - Starts the backend in watch mode with `tsx watch --tsconfig server/tsconfig.json server/src/bootstrap.ts`. - Use this for backend-only development. - Example: you are refactoring request handling, logging, module structure, or shared HTTP utilities and want automatic restarts. - -- `npm run server` - Starts the backend once from TypeScript with `tsx --tsconfig server/tsconfig.json server/src/bootstrap.ts`. - Use this when you want a stable one-shot backend process. - Example: you want to reproduce a startup bug, inspect logs without reload noise, or test one backend flow manually. - -- `npm run client` - Starts only the Vite frontend dev server. - Use this for frontend-only work when a backend is already running elsewhere. - Example: you are polishing UI layout or fixing a component state bug and do not need to restart the API server. - -### Build And Runtime Scripts - -- `npm run build` - Builds the frontend into `dist/`. - Use this to verify production frontend bundling. - Example: you changed React routing, code-splitting, or CSS and want to confirm the frontend still builds. - -- `npm run server:build` - Compiles the backend TypeScript using `server/tsconfig.json` into `server/dist/`. - Use this to verify backend build correctness. - Example: you changed `server/src/app.ts`, shared types, or future module imports and want to confirm compiled output is valid. - -- `npm run server:start` - Starts the built backend through `server/start.js`. - Use this after `npm run server:build` when you want to run compiled backend output only. - Example: dev mode works, but you want to make sure the production entrypoint and compiled files also work correctly. - -- `npm run start` - Runs `npm run build`, then `npm run server:build`, then `npm run server:start`. - Use this as the closest local equivalent to a production run. - Example: before shipping, you want to confirm the built frontend and built backend work together, not just the watch-mode setup. - -- `npm run preview` - Serves the built frontend bundle with Vite preview. - Use this when you want to inspect the built frontend output specifically. - Example: you want to check whether a client-side issue only appears in production assets. - Note: this does not replace the backend server. API routes still require the backend to be running separately. - -### Validation Scripts - -- `npm run typecheck:client` - Runs TypeScript checking for the frontend only. - Use this after frontend code changes. - Example: you changed hook types, component props, or frontend shared models. + Use for backend-only refactor work in `server/src`. - `npm run typecheck:server` - Runs TypeScript checking for the backend only. - Use this after backend code changes. - Example: you changed shared HTTP helpers, backend imports, or module boundaries. + Run after every backend module migration. -- `npm run typecheck` - Runs both frontend and backend typechecks. - Use this as the default correctness check before commit or PR. - Example: you changed `src/` and `server/src/` in the same branch and want one command to validate both. +- `npm run test:server` + Use for shared platform tests and expand it as new TS modules gain tests. -- `npm run lint` - Runs ESLint on `src/` only. - Use this for frontend lint validation. - Example: you changed React files and want to catch unused imports, hook issues, or style violations. +- `npm run server:build` + Confirms the refactored backend compiles into `server/dist`. -- `npm run lint:fix` - Runs ESLint on `src/` and auto-fixes what it can. - Use this after frontend edits when you want quick cleanup. - Example: you renamed components and want ESLint to remove stale imports and apply automatic fixes. +- `npm run verify:server` + Best validation command before merging backend refactor work. -### Release And Lifecycle Scripts +## Summary -- `npm run release` - Runs `release.sh`, which loads `GITHUB_TOKEN` from `.env` and then executes `release-it`. - Use this only when intentionally creating a release. - Example: you are cutting a new tagged version and want versioning/changelog automation. +The simplest stable rule set for this refactor is: -- `prepublishOnly` - Runs automatically before `npm publish`. - It builds both frontend and backend first. - Example: this prevents publishing a broken package that was never built. +- keep API paths the same +- move code into domain modules +- keep routes thin +- keep business logic in services +- keep SQL in repositories +- keep provider code inside provider folders +- keep websocket code outside HTTP modules -- `postinstall` - Runs automatically after `npm install`. - It executes `scripts/fix-node-pty.js`. - Example: this helps keep native terminal integration working after dependency installation. - -- `prepare` - Runs automatically during install in development contexts. - It sets up Husky hooks. - Example: this ensures local git hooks are installed without requiring a separate setup command. - -### Recommended Workflows - -- Full-stack local development: - `npm install` then `npm run dev` - -- Backend-only refactor work: - `npm run server:dev` then `npm run typecheck:server` - -- Frontend-only work: - `npm run client`, `npm run typecheck:client`, and `npm run lint` - -- Pre-PR validation: - `npm run typecheck`, `npm run lint`, `npm run build`, and `npm run server:build` - -- Production-style local verification: - `npm run start` - -- Release preparation: - `npm run typecheck`, `npm run build`, `npm run server:build`, and `npm run release` +If you follow that consistently, `server/src` will become the real backend instead of a second monolith with TypeScript syntax. diff --git a/server/src/modules/watcher/file-watcher.ts b/server/src/modules/watcher/file-watcher.ts new file mode 100644 index 00000000..79c07bfd --- /dev/null +++ b/server/src/modules/watcher/file-watcher.ts @@ -0,0 +1,165 @@ +import chokidar from "chokidar"; +import path from "path"; +import os from "os"; +import { promises as fsPromises } from "fs"; +import { logger } from "@/shared/utils/logger.js"; +import { + processClaudeSessionFile, + processCodexSessionFile, + processGeminiSessionFile, + processCursorSessionFile, + getSessions +} from "@/modules/workspace/get-workspaces/get-workspaces.js"; +import { sessionsDb } from "@/shared/database/repositories/sessions.db.js"; +import { LLMProvider } from "@/shared/types/app.js"; + +let projectsWatchers = []; + +// File system watchers for provider project/session folders +const PROVIDER_WATCH_PATHS: { provider: LLMProvider; rootPath: string }[] = [ + { + provider: "claude", + rootPath: path.join(os.homedir(), ".claude", "projects"), + }, + { + provider: "cursor", + rootPath: path.join(os.homedir(), ".cursor", "chats") + }, + { + provider: "codex", + rootPath: path.join(os.homedir(), ".codex", "sessions"), + }, + { + provider: "gemini", + rootPath: path.join(os.homedir(), ".gemini", "sessions"), + }, +]; + +const WATCHER_IGNORED_PATTERNS = [ + "**/node_modules/**", + "**/.git/**", + "**/dist/**", + "**/build/**", + "**/*.tmp", + "**/*.swp", + "**/.DS_Store", +]; + +type EventType = "add" | "change" | "unlink" | "addDir" | "unlinkDir"; + + +const onUpdate = async ( + eventType: EventType, + filePath: string, + provider: LLMProvider, +) => { + try { + console.log("[eventType] detected: ", eventType, " filePath: ", filePath, " provider: ", provider); + + switch (eventType) { + case "add": + case "change": { + let sessionId: string | null = null; + let workspacePath: string | null = null; + let sessionName = `Untitled ${provider} Session`; + + switch (provider) { + case "claude": { + const result = await processClaudeSessionFile(filePath); + if (result) { + sessionId = result.sessionId; + workspacePath = result.workspacePath; + sessionName = result.sessionName || sessionName; + } + break; + } + case "codex": { + const result = await processCodexSessionFile(filePath); + if (result) { + sessionId = result.sessionId; + workspacePath = result.workspacePath; + sessionName = result.sessionName || sessionName; + } + break; + } + case "gemini": { + const result = await processGeminiSessionFile(filePath); + if (result) { + sessionId = result.sessionId; + workspacePath = result.workspacePath; + sessionName = result.sessionName || sessionName; + } + break; + } + case "cursor": { + const result = await processCursorSessionFile(filePath); + if (result) { + sessionId = result.sessionId; + workspacePath = result.workspacePath; + sessionName = result.sessionName || sessionName; + } + break; + } + } + + if (sessionId && workspacePath) { + sessionsDb.createSession(sessionId, provider, workspacePath, sessionName); + } + break; + } + } + } catch (error: any) { + logger.error( + `[ERROR] Failed to handle ${provider} file change for ${filePath}:`, + error, + ); + } +}; + +// Setup file system watchers for Claude, Cursor, and Codex project/session folders +export async function initializeWatcher() { + logger.info("Setting up project watchers for providers..."); + + await getSessions(); + + for (const { provider, rootPath } of PROVIDER_WATCH_PATHS) { + try { + // chokidar v4 emits ENOENT via the "error" event for missing roots and will not auto-recover. + // Ensure provider folders exist before creating the watcher so watching stays active. + await fsPromises.mkdir(rootPath, { recursive: true }); + + logger.info(`Setting up watcher for ${provider} at: ${rootPath}`); + + const watcher = chokidar.watch(rootPath, { + ignored: WATCHER_IGNORED_PATTERNS, + persistent: true, + ignoreInitial: true, // Don't fire events for existing files on startup + followSymlinks: false, + depth: 6, // Reasonable depth limit + usePolling: true, // Use polling to fix Windows fs.watch buffering/batching issues. It now stops relying on the OS's native file-system events and instead manually checks the files for changes at a set interval. + interval: 2000, // Poll every 2000ms + binaryInterval: 6000, // We set a high amount because checking large binary files for changes using polling is much more CPU-intensive than checking small text files. + // Removed awaitWriteFinish to prevent delays when LLM streams to the file + + }); + + // Set up event listeners + watcher + .on("add", (filePath) => onUpdate("add", filePath, provider)) + .on("change", (filePath) => + onUpdate("change", filePath, provider), + ) + .on("error", (error: any) => { + logger.error(`[ERROR] ${provider} watcher error: ${error.message}`); + }) + .on("ready", () => { }); + + projectsWatchers.push(watcher); + } catch (error: any) { + logger.error( + `[ERROR] Failed to setup ${provider} watcher for ${rootPath}:`, + error, + ); + } + } +} diff --git a/server/src/modules/workspace/get-workspaces/get-workspaces.ts b/server/src/modules/workspace/get-workspaces/get-workspaces.ts new file mode 100644 index 00000000..b050837a --- /dev/null +++ b/server/src/modules/workspace/get-workspaces/get-workspaces.ts @@ -0,0 +1,324 @@ +import os from 'os'; +import path from 'path'; +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import readline from 'readline'; +import { sessionsDb } from '@/shared/database/repositories/sessions.db.js'; +import crypto from 'node:crypto'; +import { scanStateDb } from '@/shared/database/repositories/scan-state.db.js'; + +// ============================================================================ +// 1. SHARED TYPES & UTILITIES +// ============================================================================ +// By extracting file traversal and JSONL parsing, we remove 80% of the duplication. + +type SessionData = { + sessionId: string; + workspacePath: string; + sessionName?: string; +} + +/** + * Reads a JSONL file and builds a Map of Key -> Value. + * Useful for index files like history.jsonl or session_index.jsonl. + */ +export async function buildLookupMap(filePath: string, keyField: string, valueField: string): Promise> { + const lookup = new Map(); + try { + const fileStream = fs.createReadStream(filePath); + const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity }); + + for await (const line of rl) { + if (!line.trim()) continue; + const data = JSON.parse(line); + // We use the first occurrence. In history files, this is usually the start of the thread. + if (data[keyField] && data[valueField] && !lookup.has(data[keyField])) { + lookup.set(data[keyField], data[valueField]); + } + } + } catch (e) { /* File might not exist yet */ } + return lookup; +} + +/** + * Recursively walks a directory tree and returns a flat array of all files + * matching a specific extension (e.g., '.jsonl' or '.json'). + * It will only find the files created after + */ +async function findFilesRecursivelyCreatedAfterLastScan( + dirPath: string, + extension: string, + fileList: string[] = [] +): Promise { + try { + const entries = await fsp.readdir(dirPath, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dirPath, entry.name); + + if (entry.isDirectory()) { + await findFilesRecursivelyCreatedAfterLastScan(fullPath, extension, fileList); + } else if (entry.isFile() && entry.name.endsWith(extension)) { + const lastScanDate = scanStateDb.getLastScannedAt(); + + if (lastScanDate) { + // Check file CREATION time (birthtime) against our last scan time + const stats = await fsp.stat(fullPath); + if (stats.birthtime > lastScanDate) { + fileList.push(fullPath); + console.log("=====> full path is: ", fullPath) + } + } else { + fileList.push(fullPath); + } + } + } + } catch (e) { + // Fail silently for directories that don't exist or lack read permissions + } + return fileList; +} + +/** + * Reads a file line-by-line, parsing each line as JSON. + * It passes the parsed JSON to a custom `extractorFn`. As soon as the extractor + * successfully finds both a sessionId and workspacePath, it closes the file and returns. + */ +export async function extractFirstValidJsonlData( + filePath: string, + extractorFn: (parsedJson: any) => Partial | null | undefined +): Promise { + try { + const fileStream = fs.createReadStream(filePath); + const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity }); + + for await (const line of rl) { + if (!line.trim()) continue; + const parsedData = JSON.parse(line); + const extracted = extractorFn(parsedData); + + // If our custom extractor found what we need, return early + if (extracted?.sessionId && extracted?.workspacePath) { + rl.close(); + fileStream.close(); + return extracted as SessionData; + } + } + } catch (e) { + // Ignored errors + } + return null; +} +// ============================================================================ +// 2. JSONL-BASED PROVIDERS (Claude & Codex) +// ============================================================================ +// Now, these functions only need to define WHERE to look, and HOW to map the JSON. + +// ----- Claude ----- +export async function processClaudeSessionFile(file: string, nameMap?: Map): Promise { + if (!nameMap) { + const base = path.join(os.homedir(), '.claude'); + nameMap = await buildLookupMap(path.join(base, 'history.jsonl'), 'sessionId', 'display'); + } + + // Claude puts cwd and sessionId directly on the root object + return extractFirstValidJsonlData(file, (data) => ({ + workspacePath: data?.cwd, + sessionId: data?.sessionId, + sessionName: nameMap!.get(data?.sessionId) || 'Untitled Claude Session' + })); +} + +async function getClaudeSessions() { + const base = path.join(os.homedir(), '.claude'); + // Pre-load names from history index + const nameMap = await buildLookupMap(path.join(base, 'history.jsonl'), 'sessionId', 'display'); + + const files = await findFilesRecursivelyCreatedAfterLastScan(path.join(base, 'projects'), '.jsonl'); + for (const file of files) { + const result = await processClaudeSessionFile(file, nameMap); + + if (result) { + sessionsDb.createSession(result.sessionId, 'claude', result.workspacePath, result.sessionName); + } + } +} + +// ----- Codex ----- +export async function processCodexSessionFile(file: string, nameMap?: Map): Promise { + if (!nameMap) { + const base = path.join(os.homedir(), '.codex'); + nameMap = await buildLookupMap(path.join(base, 'session_index.jsonl'), 'id', 'thread_name'); + } + + // Codex nests the required data inside a `payload` object + return extractFirstValidJsonlData(file, (data) => ({ + workspacePath: data?.payload?.cwd, + sessionId: data?.payload?.id, + sessionName: nameMap!.get(data?.payload?.id) || 'Untitled Codex Session' + })); +} + +async function getCodexSessions() { + const base = path.join(os.homedir(), '.codex'); + // Use the thread_name attribute as requested + const nameMap = await buildLookupMap(path.join(base, 'session_index.jsonl'), 'id', 'thread_name'); + + const files = await findFilesRecursivelyCreatedAfterLastScan(path.join(base, 'sessions'), '.jsonl'); + + for (const file of files) { + const result = await processCodexSessionFile(file, nameMap); + + if (result) { + sessionsDb.createSession(result.sessionId, 'codex', result.workspacePath, result.sessionName); + } + } +} +// ============================================================================ +// 3. STANDARD JSON PROVIDERS (Gemini) +// ============================================================================ + +// ----- Gemini ----- +export async function processGeminiSessionFile(file: string): Promise { + try { + // Gemini uses standard JSON (not JSONL), so we read the whole file at once + + const fileContent = await fsp.readFile(file, 'utf8'); + const data = JSON.parse(fileContent); + if (data?.id && data?.projectPath) { + return { + sessionId: data.id, + workspacePath: data.projectPath, + sessionName: data.messages?.[0]?.content || 'New Gemini Chat' + }; + } + } catch (e) { + // Ignore parsing error for gemini + } + return null; +} + +async function getGeminiSessions() { + const geminiPath = path.join(os.homedir(), '.gemini', 'sessions'); + const files = await findFilesRecursivelyCreatedAfterLastScan(geminiPath, '.json'); + + for (const file of files) { + const result = await processGeminiSessionFile(file); + if (result) { + sessionsDb.createSession(result.sessionId, 'gemini', result.workspacePath, result.sessionName); + } + } +} + +// ============================================================================ +// 4. COMPLEX CUSTOM PROVIDERS (Cursor) +// ============================================================================ + +// ----- Cursor ----- +function md5(input: string): string { + return crypto.createHash('md5').update(input).digest('hex'); +} + +export async function extractWorkspacePathFromWorkerLog(filePath: string): Promise { + try { + const fileStream = fs.createReadStream(filePath, { encoding: 'utf8' }); + + const rl = readline.createInterface({ + input: fileStream, + crlfDelay: Infinity + }); + + for await (const line of rl) { + const match = line.match(/workspacePath=(.*)$/); + const firstMatch = match?.[1]; + + if (firstMatch) { + rl.close(); + fileStream.close(); + return firstMatch; + } + } + } catch { + // ignore errors + } + + return null; +} + +export async function processCursorSessionFile(file: string): Promise { + const sessionId = path.basename(file, '.jsonl'); + const grandparentDir = path.dirname(path.dirname(file)); + const workerLogPath = path.join(grandparentDir, 'worker.log'); + const workspacePath = await extractWorkspacePathFromWorkerLog(workerLogPath); + + if (!workspacePath) return null; + + return extractFirstValidJsonlData(file, (lineJson) => { + if (lineJson.role === 'user') { + const rawText = lineJson.message?.content?.[0]?.text || ''; + // Strip tags and trim + const cleanName = rawText.replace(/<\/?user_query>/g, '').trim().split('\n'); + return { sessionId: sessionId as string, workspacePath, sessionName: cleanName[0] || "Untitled Cursor Session" }; + } + return null; + }); +} + +async function getCursorSessions() { + try { + const cursorBase = path.join(os.homedir(), '.cursor'); + const projectsDir = path.join(cursorBase, 'projects'); + const projectDirs = await fsp.readdir(projectsDir); + const seenWorkspacePaths = new Set(); + + for (const projectDir of projectDirs) { + const workerLogPath = path.join(projectsDir, projectDir, 'worker.log'); + const workspacePath = await extractWorkspacePathFromWorkerLog(workerLogPath); + + if (!workspacePath || seenWorkspacePaths.has(workspacePath)) continue; + + seenWorkspacePaths.add(workspacePath); + const workspaceHash = md5(workspacePath); + const chatsDir = path.join(cursorBase, 'chats', workspaceHash); + + const sessionFiles = await findFilesRecursivelyCreatedAfterLastScan(chatsDir, '.jsonl'); + + for (const file of sessionFiles) { + const result = await processCursorSessionFile(file); + + if (result) { + sessionsDb.createSession(result.sessionId, 'cursor', result.workspacePath, result.sessionName); + } + } + } + } catch (e) { + // Base cursor directory or projects directory likely doesn't exist + } +} + + +export async function getSessions() { + + // 1. Start the timer with a unique label + console.time("🚀 Workspace sync total time"); + + console.log("Starting workspace sync..."); + try { + // Wrapping in Promise.all allows these to process concurrently, speeding up the boot time + await Promise.allSettled([ + getClaudeSessions(), + getCodexSessions(), + getGeminiSessions(), + getCursorSessions() + ]); + + scanStateDb.updateLastScannedAt(); + } catch (error) { + console.error("An error occurred during sync:", error); + } finally { + console.log("----------------------------------"); + // 2. Stop the timer using the exact same label + // This will print: 🚀 Workspace sync total time: 123.456ms + console.timeEnd("🚀 Workspace sync total time"); + console.log("Workspace synchronization complete."); + } +} diff --git a/server/src/realtime/chat.gateway.ts b/server/src/realtime/chat.gateway.ts new file mode 100644 index 00000000..e69de29b diff --git a/server/src/realtime/events.ts b/server/src/realtime/events.ts new file mode 100644 index 00000000..e69de29b diff --git a/server/src/realtime/index.ts b/server/src/realtime/index.ts new file mode 100644 index 00000000..e69de29b diff --git a/server/src/runner.ts b/server/src/runner.ts index ae934b0b..0e44a2aa 100644 --- a/server/src/runner.ts +++ b/server/src/runner.ts @@ -1,6 +1,37 @@ +import express from 'express'; +import http from 'http'; + import { userDb } from "@/shared/database/repositories/users.js"; +import { initializeDatabase } from '@/shared/database/init-db.js'; +import { initializeWatcher } from '@/modules/watcher/file-watcher.js'; console.log("----------------Hello there, Refactored Runner!-------------------"); -console.log("User db initialized"); -console.log("First user:", userDb.getFirstUser()); +const app = express(); +const server = http.createServer(app); + +const serverPortEnv = process.env.SERVER_PORT; +const SERVER_PORT = serverPortEnv ? Number.parseInt(serverPortEnv) : 3001; + +if (Number.isNaN(SERVER_PORT)) { + throw new Error(`Invalid SERVER_PORT value: ${serverPortEnv}`); +} +const HOST = process.env.HOST || '0.0.0.0'; + +async function main() { + try { + await initializeDatabase(); + + server.listen(SERVER_PORT, HOST, async () => { + console.log(`Server is running on http://${HOST}:${SERVER_PORT}`); + + await initializeWatcher(); + }); + + } catch (error) { + console.error("Failed to initialize database:", error); + process.exit(1); + } +} + +await main(); \ No newline at end of file diff --git a/server/src/shared/database/init-db.ts b/server/src/shared/database/init-db.ts index 6c9f2b45..0a02e79f 100644 --- a/server/src/shared/database/init-db.ts +++ b/server/src/shared/database/init-db.ts @@ -9,7 +9,7 @@ export const initializeDatabase = async () => { const db = getConnection(); db.exec(INIT_SCHEMA_SQL); logger.info('Database schema applied'); - runMigrations(db); + runMigrations(db); // ? If we rename the database to something new, would a migration be still necessary? } catch (err) { const message = err instanceof Error ? err.message : String(err); logger.error('Database initialization failed', { error: message }); diff --git a/server/src/shared/database/migrations.ts b/server/src/shared/database/migrations.ts index 40c35096..80a35a26 100644 --- a/server/src/shared/database/migrations.ts +++ b/server/src/shared/database/migrations.ts @@ -1,5 +1,5 @@ import { Database } from "better-sqlite3"; -import { APP_CONFIG_TABLE_SCHEMA_SQL, SESSION_NAMES_TABLE_SCHEMA_SQL } from "@/shared/database/schema.js"; +import { APP_CONFIG_TABLE_SCHEMA_SQL, LAST_SCANNED_AT_SQL, SESSIONS_TABLE_SCHEMA_SQL, WORK_SPACE_PATH_SQL } from "@/shared/database/schema.js"; import { logger } from "@/shared/utils/logger.js"; const addColumnToUsersTableIfNotExists = ( @@ -29,12 +29,16 @@ export const runMigrations = (db: Database) => { // Create app_config table if it doesn't exist (for existing installations) db.exec(APP_CONFIG_TABLE_SCHEMA_SQL); - // Create session_names table if it doesn't exist (for existing installations) - db.exec(SESSION_NAMES_TABLE_SCHEMA_SQL); + // Create sessions table if it doesn't exist (for existing installations) + db.exec(SESSIONS_TABLE_SCHEMA_SQL); db.exec( - "CREATE INDEX IF NOT EXISTS idx_session_names_lookup ON session_names(session_id, provider)", + "CREATE INDEX IF NOT EXISTS idx_session_ids_lookup ON sessions(session_id)" ); + db.exec(WORK_SPACE_PATH_SQL); + + db.exec(LAST_SCANNED_AT_SQL); + logger.info("Database migrations completed successfully"); } catch (error: any) { diff --git a/server/src/shared/database/repositories/scan-state.db.ts b/server/src/shared/database/repositories/scan-state.db.ts new file mode 100644 index 00000000..5099d5b4 --- /dev/null +++ b/server/src/shared/database/repositories/scan-state.db.ts @@ -0,0 +1,38 @@ +import { getConnection } from "@/shared/database/connection.js"; +import { ScanStateRow } from "@/shared/database/types.js"; + +export const scanStateDb = { + getLastScannedAt() { + const db = getConnection(); + + const row = db + .prepare(`SELECT last_scanned_at FROM scan_state WHERE id = 1`) + .get() as ScanStateRow; + + if (!row) { + return null; // Before any scan, the row is undefined. + } + + let lastScannedDate: Date | null = null; + const lastScannedStr = row.last_scanned_at; + + if (lastScannedStr) { + // SQLite CURRENT_TIMESTAMP returns UTC in "YYYY-MM-DD HH:MM:SS" format. + // Replace space with 'T' and append 'Z' to parse reliably in JS across all platforms. + lastScannedDate = new Date(lastScannedStr.replace(' ', 'T') + 'Z'); + } + + return lastScannedDate; + }, + + updateLastScannedAt() { + const db = getConnection(); + + db.prepare(` + INSERT INTO scan_state (id, last_scanned_at) + VALUES (1, CURRENT_TIMESTAMP) + ON CONFLICT (id) + DO UPDATE SET last_scanned_at = CURRENT_TIMESTAMP + `).run(); + } +}; \ No newline at end of file diff --git a/server/src/shared/database/repositories/session-names.ts b/server/src/shared/database/repositories/session-names.ts deleted file mode 100644 index 597b41b0..00000000 --- a/server/src/shared/database/repositories/session-names.ts +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Session names repository. - * - * Manages custom display names for provider sessions. When a user - * renames a chat session in the UI, the override is stored here - * and applied on top of the CLI-generated summary. - */ - -import { getConnection } from '@/shared/database/connection.js'; -import type { - SessionNameLookupRow, - SessionWithSummary, -} from '@/shared/database/types.js'; -import { logger } from '@/shared/utils/logger.js'; - -// --------------------------------------------------------------------------- -// Queries -// --------------------------------------------------------------------------- - -export const sessionNamesDb = { - /** Inserts or updates a custom session name (upsert on session_id + provider). */ - setName(sessionId: string, provider: string, customName: string): void { - const db = getConnection(); - db.prepare( - `INSERT INTO session_names (session_id, provider, custom_name) - VALUES (?, ?, ?) - ON CONFLICT(session_id, provider) - DO UPDATE SET custom_name = excluded.custom_name, - updated_at = CURRENT_TIMESTAMP` - ).run(sessionId, provider, customName); - }, - - /** Returns the custom name for a single session, or null if unset. */ - getName(sessionId: string, provider: string): string | null { - const db = getConnection(); - const row = db - .prepare( - 'SELECT custom_name FROM session_names WHERE session_id = ? AND provider = ?' - ) - .get(sessionId, provider) as { custom_name: string } | undefined; - return row?.custom_name ?? null; - }, - - /** - * Batch lookup for multiple session IDs. - * Returns a Map for efficient overlay onto session lists. - */ - getNames(sessionIds: string[], provider: string): Map { - if (sessionIds.length === 0) return new Map(); - - const db = getConnection(); - const placeholders = sessionIds.map(() => '?').join(','); - const rows = db - .prepare( - `SELECT session_id, custom_name FROM session_names - WHERE session_id IN (${placeholders}) AND provider = ?` - ) - .all(...sessionIds, provider) as SessionNameLookupRow[]; - - return new Map(rows.map((r) => [r.session_id, r.custom_name])); - }, - - /** Removes a custom session name. Returns true if a row was deleted. */ - deleteName(sessionId: string, provider: string): boolean { - const db = getConnection(); - return ( - db - .prepare( - 'DELETE FROM session_names WHERE session_id = ? AND provider = ?' - ) - .run(sessionId, provider).changes > 0 - ); - }, -}; - -// --------------------------------------------------------------------------- -// Session overlay helper -// --------------------------------------------------------------------------- - -/** - * Overlays custom session names from the database onto a list of sessions. - * Mutates each session's `summary` field in-place when a custom name exists. - * - * This is the typed equivalent of the legacy `applyCustomSessionNames` function. - * Non-fatal: logs a warning on failure instead of throwing. - */ -export function applyCustomSessionNames( - sessions: SessionWithSummary[] | undefined | null, - provider: string -): void { - if (!sessions?.length) return; - - try { - const ids = sessions.map((s) => s.id); - const customNames = sessionNamesDb.getNames(ids, provider); - - for (const session of sessions) { - const custom = customNames.get(session.id); - if (custom) { - session.summary = custom; - } - } - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - logger.warn(`Failed to apply custom session names for ${provider}`, { - error: message, - }); - } -} diff --git a/server/src/shared/database/repositories/sessions.db.ts b/server/src/shared/database/repositories/sessions.db.ts new file mode 100644 index 00000000..ff82db39 --- /dev/null +++ b/server/src/shared/database/repositories/sessions.db.ts @@ -0,0 +1,87 @@ +import { workspaceOriginalPathsDb } from '@/shared/database/repositories/workspace-original-paths.db.js'; +import { getConnection } from '@/shared/database/connection.js'; +import type { + SessionNameLookupRow, + SessionWithSummary, +} from '@/shared/database/types.js'; +import { logger } from '@/shared/utils/logger.js'; + +// --------------------------------------------------------------------------- +// Queries +// --------------------------------------------------------------------------- + +export const sessionsDb = { + + createSession(session_id: string, provider: string, workspacePath: string, customName?: string): void { + const db = getConnection(); + + // First, ensure the workspace path is recorded in the workspace_original_paths table + // since it's a foreign key in the sessions table. + workspaceOriginalPathsDb.createWorkspacePath(workspacePath); + + db.prepare( + 'INSERT OR IGNORE INTO sessions (session_id, provider, custom_name, workspace_path) VALUES (?, ?, ?, ?)' + ).run(session_id, provider, customName, workspacePath); + }, + + deleteSession(session_id: string): void { + const db = getConnection(); + db.prepare('DELETE FROM sessions WHERE session_id = ?').run(session_id); + } + + + // /** Inserts or updates a custom session name (upsert on session_id + provider). */ + // setName(sessionId: string, provider: string, customName: string): void { + // const db = getConnection(); + // db.prepare( + // `INSERT INTO session_names (session_id, provider, custom_name) + // VALUES (?, ?, ?) + // ON CONFLICT(session_id, provider) + // DO UPDATE SET custom_name = excluded.custom_name, + // updated_at = CURRENT_TIMESTAMP` + // ).run(sessionId, provider, customName); + // }, + + /** Returns the custom name for a single session, or null if unset. */ + // getName(sessionId: string, provider: string): string | null { + // const db = getConnection(); + // const row = db + // .prepare( + // 'SELECT custom_name FROM session_names WHERE session_id = ? AND provider = ?' + // ) + // .get(sessionId, provider) as { custom_name: string } | undefined; + // return row?.custom_name ?? null; + // }, + + /** + * Batch lookup for multiple session IDs. + * Returns a Map for efficient overlay onto session lists. + */ + // getNames(sessionIds: string[], provider: string): Map { + // if (sessionIds.length === 0) return new Map(); + + // const db = getConnection(); + // const placeholders = sessionIds.map(() => '?').join(','); + // const rows = db + // .prepare( + // `SELECT session_id, custom_name FROM session_names + // WHERE session_id IN (${placeholders}) AND provider = ?` + // ) + // .all(...sessionIds, provider) as SessionNameLookupRow[]; + + // return new Map(rows.map((r) => [r.session_id, r.custom_name])); + // }, + + /** Removes a custom session name. Returns true if a row was deleted. */ + // deleteName(sessionId: string, provider: string): boolean { + // const db = getConnection(); + // return ( + // db + // .prepare( + // 'DELETE FROM session_names WHERE session_id = ? AND provider = ?' + // ) + // .run(sessionId, provider).changes > 0 + // ); + // }, +}; + diff --git a/server/src/shared/database/repositories/workspace-original-paths.db.ts b/server/src/shared/database/repositories/workspace-original-paths.db.ts new file mode 100644 index 00000000..7a25f090 --- /dev/null +++ b/server/src/shared/database/repositories/workspace-original-paths.db.ts @@ -0,0 +1,12 @@ +import { getConnection } from '@/shared/database/connection.js'; + +export const workspaceOriginalPathsDb = { + createWorkspacePath(workspacePath: string): void { + const db = getConnection(); + db.prepare(` + INSERT INTO workspace_original_paths (workspace_path) + VALUES (?) + ON CONFLICT(workspace_path) DO NOTHING + `).run(workspacePath); + }, +} \ No newline at end of file diff --git a/server/src/shared/database/schema.ts b/server/src/shared/database/schema.ts index 76c8c5ec..9e9975c2 100644 --- a/server/src/shared/database/schema.ts +++ b/server/src/shared/database/schema.ts @@ -39,18 +39,31 @@ CREATE TABLE IF NOT EXISTS user_credentials ( ); `; -export const SESSION_NAMES_TABLE_SCHEMA_SQL = ` -CREATE TABLE IF NOT EXISTS session_names ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - session_id TEXT NOT NULL, - provider TEXT NOT NULL DEFAULT 'claude', - custom_name TEXT NOT NULL, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, - UNIQUE(session_id, provider) +export const SESSIONS_TABLE_SCHEMA_SQL = ` +CREATE TABLE IF NOT EXISTS sessions ( + session_id TEXT PRIMARY KEY NOT NULL, + provider TEXT NOT NULL, + custom_name TEXT, + workspace_path TEXT NOT NULL, + FOREIGN KEY (workspace_path) REFERENCES workspace_original_paths(workspace_path) + ON DELETE CASCADE + ON UPDATE CASCADE ); `; +export const WORK_SPACE_PATH_SQL = ` +CREATE TABLE IF NOT EXISTS workspace_original_paths ( + workspace_path TEXT PRIMARY KEY NOT NULL +); +` + +export const LAST_SCANNED_AT_SQL = ` +CREATE TABLE IF NOT EXISTS scan_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + last_scanned_at TIMESTAMP NULL +); +` + export const APP_CONFIG_TABLE_SCHEMA_SQL = ` CREATE TABLE IF NOT EXISTS app_config ( key TEXT PRIMARY KEY, @@ -70,20 +83,21 @@ CREATE INDEX IF NOT EXISTS idx_users_username ON users(username); CREATE INDEX IF NOT EXISTS idx_users_active ON users(is_active); ${API_KEYS_TABLE_SCHEMA_SQL} - CREATE INDEX IF NOT EXISTS idx_api_keys_key ON api_keys(api_key); CREATE INDEX IF NOT EXISTS idx_api_keys_user_id ON api_keys(user_id); CREATE INDEX IF NOT EXISTS idx_api_keys_active ON api_keys(is_active); ${USER_CREDENTIALS_TABLE_SCHEMA_SQL} - CREATE INDEX IF NOT EXISTS idx_user_credentials_user_id ON user_credentials(user_id); CREATE INDEX IF NOT EXISTS idx_user_credentials_type ON user_credentials(credential_type); CREATE INDEX IF NOT EXISTS idx_user_credentials_active ON user_credentials(is_active); -${SESSION_NAMES_TABLE_SCHEMA_SQL} +${SESSIONS_TABLE_SCHEMA_SQL} +CREATE INDEX IF NOT EXISTS idx_session_ids_lookup ON sessions(session_id); -CREATE INDEX IF NOT EXISTS idx_session_names_lookup ON session_names(session_id, provider); +${WORK_SPACE_PATH_SQL} + +${LAST_SCANNED_AT_SQL} ${APP_CONFIG_TABLE_SCHEMA_SQL} `; diff --git a/server/src/shared/database/types.ts b/server/src/shared/database/types.ts index 442fff3d..1881ffc2 100644 --- a/server/src/shared/database/types.ts +++ b/server/src/shared/database/types.ts @@ -6,6 +6,8 @@ * from SELECT queries; input types represent what goes into INSERT/UPDATE. */ +import { LLMProvider } from "@/shared/types/app.js"; + // --------------------------------------------------------------------------- // Users // --------------------------------------------------------------------------- @@ -94,17 +96,15 @@ export type CreateCredentialResult = { // Session Names // --------------------------------------------------------------------------- -export type SessionNameRow = { - id: number; +export type SessionsRow = { session_id: string; - provider: string; + provider: LLMProvider; + workspacePath: string; custom_name: string; - created_at: string; - updated_at: string; }; /** Minimal shape used in batch lookups. */ -export type SessionNameLookupRow = Pick; +export type SessionNameLookupRow = Pick; /** * Any object that has an `id` and `summary` field. @@ -116,6 +116,16 @@ export type SessionWithSummary = { [key: string]: unknown; }; + + +// --------------------------------------------------------------------------- +// Scan State +// --------------------------------------------------------------------------- +export type ScanStateRow = { + last_scanned_at: string; +} + + // --------------------------------------------------------------------------- // App Config // --------------------------------------------------------------------------- diff --git a/server/src/shared/platform/path.test.ts b/server/src/shared/platform/path.test.ts index ce563924..adfd4ee5 100644 --- a/server/src/shared/platform/path.test.ts +++ b/server/src/shared/platform/path.test.ts @@ -1,7 +1,12 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { arePathsEquivalent, normalizePathForPlatform, toPortablePath } from './path.js'; +import { + arePathsEquivalent, + normalizeComparablePath, + normalizePathForPlatform, + toPortablePath, +} from './path.js'; // This test verifies path strings can be normalized for logs and platform-specific execution. test('path helpers normalize separators in both directions', () => { @@ -27,3 +32,15 @@ test('arePathsEquivalent follows the case rules of the target platform', () => { false, ); }); + +// This test verifies path comparison keys stay stable across long-path prefixes and dot segments. +test('normalizeComparablePath resolves paths using the target platform rules', () => { + assert.equal( + normalizeComparablePath('\\\\?\\C:\\Repo\\..\\Repo\\File.txt', 'windows'), + 'c:\\repo\\file.txt', + ); + assert.equal( + normalizeComparablePath('/repo/../repo/File.txt', 'linux'), + '/repo/File.txt', + ); +}); diff --git a/server/src/shared/platform/path.ts b/server/src/shared/platform/path.ts index 432687af..5eaa7eac 100644 --- a/server/src/shared/platform/path.ts +++ b/server/src/shared/platform/path.ts @@ -17,18 +17,49 @@ export function normalizePathForPlatform( return value.replace(/[\\/]+/g, separator); } +/** + * Normalizes a path into a stable comparison key for the target platform. + * + * This helper intentionally does more than separator normalization: + * it trims incidental whitespace, removes the Windows long-path prefix when + * present, resolves `.` and `..`, and applies the platform's case rules. + * + * The return value is meant for equality checks, map keys, and de-duplication. + * It should not be used as a display string because Windows casing is lowered + * on purpose to preserve case-insensitive comparisons. + */ +export function normalizeComparablePath( + value: string, + platform: RuntimePlatform = resolveRuntimePlatform(), +): string { + if (typeof value !== 'string') { + return ''; + } + + const trimmedValue = value.trim(); + if (trimmedValue.length === 0) { + return ''; + } + + const withoutLongPathPrefix = trimmedValue.startsWith('\\\\?\\') + ? trimmedValue.slice(4) + : trimmedValue; + + // This branch resolves paths using the target platform instead of the host OS. + const pathModule = isWindowsPlatform(platform) ? path.win32 : path.posix; + const normalizedInput = normalizePathForPlatform(withoutLongPathPrefix, platform); + const resolvedPath = pathModule.resolve(pathModule.normalize(normalizedInput)); + + return isWindowsPlatform(platform) + ? resolvedPath.toLowerCase() + : resolvedPath; +} + // This helper compares paths using the case-sensitivity rules of the target platform. export function arePathsEquivalent( left: string, right: string, platform: RuntimePlatform = resolveRuntimePlatform(), ): boolean { - // This branch uses the target platform's path semantics instead of the host machine's semantics. - const pathModule = isWindowsPlatform(platform) ? path.win32 : path.posix; - const normalizedLeft = pathModule.normalize(normalizePathForPlatform(left, platform)); - const normalizedRight = pathModule.normalize(normalizePathForPlatform(right, platform)); - - return isWindowsPlatform(platform) - ? normalizedLeft.toLowerCase() === normalizedRight.toLowerCase() - : normalizedLeft === normalizedRight; + return normalizeComparablePath(left, platform) === normalizeComparablePath(right, platform); } diff --git a/server/src/shared/types/app.ts b/server/src/shared/types/app.ts index 039c2dbf..e0ae83ae 100644 --- a/server/src/shared/types/app.ts +++ b/server/src/shared/types/app.ts @@ -18,3 +18,6 @@ export type ServerApplication = { runtimePaths: RuntimePaths; start: () => Promise; }; + +// --------------------------------------------------------------------------- +export type LLMProvider = 'claude' | 'codex' | 'cursor' | 'gemini'; \ No newline at end of file