diff --git a/server/browser-use-mcp.ts b/server/browser-use-mcp.ts index 238bff35..d1ece36a 100644 --- a/server/browser-use-mcp.ts +++ b/server/browser-use-mcp.ts @@ -69,7 +69,7 @@ const sessionIdSchema = { const tools: ToolDefinition[] = [ { name: 'browser_create_session', - description: 'Create a temporary Browser session that the agent can control. Optionally provide a background profileName to reuse cookies and storage.', + description: 'Create a Browser session that the agent can control. Uses the configured persistent profile by default; optionally provide profileName to override it.', inputSchema: { type: 'object', properties: { diff --git a/server/index.js b/server/index.js index d957ef58..df1a4224 100755 --- a/server/index.js +++ b/server/index.js @@ -63,7 +63,7 @@ import pluginsRoutes from './routes/plugins.js'; import providerRoutes from './modules/providers/provider.routes.js'; import browserUseRoutes from './modules/browser-use/browser-use.routes.js'; import browserUseMcpRoutes from './modules/browser-use/browser-use-mcp.routes.js'; -import { browserUseService } from './modules/browser-use/browser-use.service.js'; +import { browserUseService, VIEWER_COOKIE_NAME } from './modules/browser-use/index.js'; import { startEnabledPluginServers, stopAllPlugins, getPluginPort } from './utils/plugin-process-manager.js'; import { initializeDatabase, projectsDb, sessionsDb } from './modules/database/index.js'; import { configureWebPush } from './services/vapid-keys.js'; @@ -145,6 +145,8 @@ const wss = createWebSocketServer(server, { shouldAutoOpenUrlFromOutput, }, getPluginPort, + browserUseViewer: (ws, pathname) => browserUseService.handleViewerWebSocket(ws, pathname), + authenticateBrowserUseViewer: authenticateBrowserUseViewerPath, }); // Make WebSocket server available to routes @@ -210,11 +212,41 @@ app.use('/api/gemini', authenticateToken, geminiRoutes); // Plugins API Routes (protected) app.use('/api/plugins', authenticateToken, pluginsRoutes); +function readCookieValue(header, name) { + if (!header) return null; + const prefix = `${name}=`; + const cookie = String(header).split(';').map((part) => part.trim()).find((part) => part.startsWith(prefix)); + return cookie ? decodeURIComponent(cookie.slice(prefix.length)) : null; +} + +function authenticateBrowserUseViewerPath(pathname, token) { + const parts = String(pathname || '').split('/'); + const sessionId = parts[4]; + if (parts[1] !== 'api' || parts[2] !== 'browser-use' || parts[3] !== 'sessions' || parts[5] !== 'viewer' || parts[6] !== 'websockify') { + return false; + } + return browserUseService.validateViewerToken(decodeURIComponent(sessionId), token); +} + +function authenticateBrowserUse(req, res, next) { + const match = /^\/sessions\/([^/]+)\/viewer(?:\/|$)/.exec(req.path || ''); + if (match) { + const sessionId = decodeURIComponent(match[1]); + const token = typeof req.query.viewerToken === 'string' + ? req.query.viewerToken + : readCookieValue(req.headers.cookie, VIEWER_COOKIE_NAME); + if (browserUseService.validateViewerToken(sessionId, token)) { + return next(); + } + } + return authenticateToken(req, res, next); +} + // Browser MCP bridge API (local token protected) app.use('/api/browser-use-mcp', browserUseMcpRoutes); // Browser API Routes (protected) -app.use('/api/browser-use', authenticateToken, browserUseRoutes); +app.use('/api/browser-use', authenticateBrowserUse, browserUseRoutes); // Unified provider MCP routes (protected) app.use('/api/providers', authenticateToken, providerRoutes); diff --git a/server/modules/browser-use/browser-use.routes.ts b/server/modules/browser-use/browser-use.routes.ts index ad428079..507a830f 100644 --- a/server/modules/browser-use/browser-use.routes.ts +++ b/server/modules/browser-use/browser-use.routes.ts @@ -1,6 +1,7 @@ import express from 'express'; import { browserUseService } from '@/modules/browser-use/browser-use.service.js'; +import { VIEWER_COOKIE_NAME, VIEWER_TOKEN_TTL_MS } from '@/modules/browser-use/browser-use.viewer.js'; const router = express.Router(); @@ -8,6 +9,45 @@ function readParam(value: string | string[] | undefined): string { return Array.isArray(value) ? value[0] || '' : value || ''; } +const SAFE_VIEWER_ROOT_FILES = new Set(['vnc.html', 'favicon.ico', 'manifest.json']); +const SAFE_VIEWER_ROOT_DIRS = new Set(['app', 'core', 'vendor', 'assets', 'images', 'utils']); + +function isSafeViewerPath(viewerPath: string): boolean { + if (!viewerPath || viewerPath.startsWith('/') || viewerPath.includes('..') || viewerPath.includes('\\')) { + return false; + } + + if (!/^[A-Za-z0-9][A-Za-z0-9._~/-]*$/.test(viewerPath)) { + return false; + } + + if (SAFE_VIEWER_ROOT_FILES.has(viewerPath)) { + return true; + } + + const [rootDir] = viewerPath.split('/'); + return Boolean(rootDir && SAFE_VIEWER_ROOT_DIRS.has(rootDir)); +} + +function isSecureRequest(req: express.Request): boolean { + const forwardedProto = String(req.headers['x-forwarded-proto'] || '') + .split(',')[0] + .trim() + .toLowerCase(); + return req.secure || forwardedProto === 'https'; +} + +function readQueryString(originalUrl: string): string { + const queryIndex = originalUrl.indexOf('?'); + if (queryIndex < 0) { + return ''; + } + const params = new URLSearchParams(originalUrl.slice(queryIndex + 1)); + params.delete('viewerToken'); + const nextQuery = params.toString(); + return nextQuery ? `?${nextQuery}` : ''; +} + router.get('/status', async (_req, res) => { try { res.json({ success: true, data: await browserUseService.getStatus() }); @@ -62,13 +102,60 @@ router.get('/sessions', async (_req, res) => { try { res.json({ success: true, data: { sessions: await browserUseService.listSessions() } }); } catch (error) { - res.status(401).json({ + res.status(500).json({ success: false, error: error instanceof Error ? error.message : 'Failed to list browser sessions.', }); } }); +router.get('/sessions/:sessionId/viewer/*', async (req, res) => { + try { + const sessionId = readParam(req.params.sessionId); + const originalPath = req.originalUrl.split('?')[0] || ''; + const viewerMarker = `/sessions/${sessionId}/viewer/`; + const markerIndex = originalPath.indexOf(viewerMarker); + const rawViewerPath = markerIndex >= 0 ? originalPath.slice(markerIndex + viewerMarker.length) : 'vnc.html'; + const viewerPath = decodeURIComponent(rawViewerPath).replace(/^\/+/, '') || 'vnc.html'; + if (!isSafeViewerPath(viewerPath)) { + res.status(400).json({ success: false, error: 'Invalid Browser viewer path.' }); + return; + } + + const viewerToken = readParam(req.query.viewerToken as string | string[] | undefined); + if (viewerPath === 'vnc.html' && browserUseService.validateViewerToken(sessionId, viewerToken)) { + res.cookie(VIEWER_COOKIE_NAME, viewerToken, { + httpOnly: true, + sameSite: 'lax', + secure: isSecureRequest(req), + maxAge: VIEWER_TOKEN_TTL_MS, + path: '/api/browser-use/sessions/' + encodeURIComponent(sessionId) + '/viewer', + }); + } + const target = browserUseService.getViewerProxyTarget(sessionId); + const query = readQueryString(req.originalUrl); + const upstream = await fetch(`http://127.0.0.1:${target.websockifyPort}/${viewerPath}${query}`, { + headers: { + accept: String(req.headers.accept || '*/*'), + }, + }); + const contentType = upstream.headers.get('content-type'); + if (contentType) { + res.setHeader('content-type', contentType); + } + const cacheControl = viewerPath === 'vnc.html' ? 'no-store' : 'public, max-age=3600'; + res.setHeader('cache-control', cacheControl); + res.status(upstream.status); + const body = Buffer.from(await upstream.arrayBuffer()); + res.send(body); + } catch (error) { + res.status(404).json({ + success: false, + error: error instanceof Error ? error.message : 'Browser viewer is not available.', + }); + } +}); + router.post('/sessions/:sessionId/stop', async (req, res) => { try { const result = await browserUseService.stopSession(readParam(req.params.sessionId)); diff --git a/server/modules/browser-use/browser-use.service.ts b/server/modules/browser-use/browser-use.service.ts index 280ff730..efcf8da3 100644 --- a/server/modules/browser-use/browser-use.service.ts +++ b/server/modules/browser-use/browser-use.service.ts @@ -1,128 +1,86 @@ import { createRequire } from 'node:module'; -import { randomBytes, randomUUID } from 'node:crypto'; -import { spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { execFileSync, spawn } from 'node:child_process'; import fs from 'node:fs'; +import net from 'node:net'; import os from 'node:os'; import path from 'node:path'; -import { appConfigDb } from '@/modules/database/index.js'; +import { WebSocket } from 'ws'; + import { providerMcpService } from '@/modules/providers/index.js'; import { getModuleDir } from '@/utils/runtime-paths.js'; +import { + getOrCreateMcpToken, + getProfilePath, + normalizeBrowserBackend, + PROFILE_ROOT, + readSettings, + resolveSessionProfileName, + useVisibleCamoufoxBackend, + writeSettings, +} from './browser-use.settings.js'; +import type { + BrowserUseSession, + BrowserUseSettings, + PublicBrowserUseSession, + RuntimeHandle, + RuntimeProbe, + RuntimeReadiness, +} from './browser-use.types.js'; +import { getViewerUrl, handleViewerWebSocket, VIEWER_TOKEN_TTL_MS } from './browser-use.viewer.js'; + const require = createRequire(import.meta.url); const __dirname = getModuleDir(import.meta.url); const IS_PLATFORM = process.env.VITE_IS_PLATFORM === 'true'; const MAX_SESSIONS_PER_OWNER = Number.parseInt(process.env.CLOUDCLI_BROWSER_USE_MAX_SESSIONS_PER_OWNER || '3', 10); const SESSION_TTL_MS = Number.parseInt(process.env.CLOUDCLI_BROWSER_USE_SESSION_TTL_MS || String(30 * 60 * 1000), 10); -const BROWSER_USE_SETTINGS_KEY = 'browser_use_settings'; -const BROWSER_USE_MCP_TOKEN_KEY = 'browser_use_mcp_token'; - -type BrowserUseRuntime = 'cloud' | 'local'; -type BrowserUseSessionStatus = 'ready' | 'stopped' | 'unavailable'; - -type BrowserUseSession = { - id: string; - ownerId: string; - createdBy: 'agent'; - runtime: BrowserUseRuntime; - status: BrowserUseSessionStatus; - url: string | null; - title: string | null; - screenshotDataUrl: string | null; - createdAt: string; - updatedAt: string; - lastAction: string | null; - message: string | null; - profileName: string | null; - viewport: { - width: number; - height: number; - } | null; - cursor: { - x: number; - y: number; - actor: 'agent'; - } | null; -}; - -type PublicBrowserUseSession = Omit; - -type RuntimeHandle = { - browser?: any; - context?: any; - page?: any; -}; - -type BrowserUseSettings = { - enabled: boolean; -}; - -type RuntimeReadiness = { - playwright: any | null; - playwrightInstalled: boolean; - chromiumInstalled: boolean; - chromiumExecutablePath: string | null; - installInProgress: boolean; - installMessage: string | null; -}; - -type RuntimeProbe = Omit; const sessions = new Map(); const handles = new Map(); +const reservedDisplays = new Set(); +const viewerTokens = new Map(); let installPromise: Promise<{ success: boolean; message: string }> | null = null; let lastInstallMessage: string | null = null; let runtimeProbeCache: { value: RuntimeProbe; updatedAt: number } | null = null; -const DEFAULT_SETTINGS: BrowserUseSettings = { - enabled: false, -}; const AGENT_OWNER_ID = 'agent'; -const PROFILE_ROOT = path.join(os.homedir(), '.cloudcli', 'browser-use', 'profiles'); const MCP_SERVER_NAME = 'cloudcli-browser'; const LEGACY_MCP_SERVER_NAMES = ['cloudcli-browser-use']; const RUNTIME_READINESS_CACHE_TTL_MS = 30_000; +const VISIBLE_BROWSER_ENABLED = process.env.CLOUDCLI_BROWSER_USE_VISIBLE !== 'false'; +const RUNTIME_ROOT = process.env.CLOUDCLI_BROWSER_USE_RUNTIME_ROOT || '/opt/claudecodeui/.runtime-browser'; +const NOVNC_ROOT = process.env.CLOUDCLI_BROWSER_USE_NOVNC_ROOT || path.join(RUNTIME_ROOT, 'novnc'); +const X11VNC_BIN = process.env.CLOUDCLI_BROWSER_USE_X11VNC_BIN || path.join(RUNTIME_ROOT, 'rootfs/usr/bin/x11vnc'); +const X11VNC_LIB_DIR = process.env.CLOUDCLI_BROWSER_USE_X11VNC_LIB_DIR || path.join(RUNTIME_ROOT, 'rootfs/usr/lib/x86_64-linux-gnu'); +const X11VNC_EXTRA_LIB_DIR = process.env.CLOUDCLI_BROWSER_USE_X11VNC_EXTRA_LIB_DIR || path.join(RUNTIME_ROOT, 'rootfs/lib/x86_64-linux-gnu'); +const LOG_RUNTIME_PROCESS_OUTPUT = process.env.CLOUDCLI_BROWSER_USE_RUNTIME_LOGS === 'true'; -function getRuntime(): BrowserUseRuntime { +function getRuntime(): 'cloud' | 'local' { return IS_PLATFORM ? 'cloud' : 'local'; } -function readSettings(): BrowserUseSettings { +function getCamoufoxExecutablePath(): string | null { + const configured = process.env.CLOUDCLI_BROWSER_USE_CAMOUFOX_EXECUTABLE; + if (configured && fs.existsSync(configured)) { + return configured; + } + try { - const raw = appConfigDb.get(BROWSER_USE_SETTINGS_KEY); - if (!raw) { - return DEFAULT_SETTINGS; - } - - const parsed = JSON.parse(raw) as Partial; - return { - enabled: parsed.enabled === true, - }; - } catch (error: any) { - console.warn('[Browser] Failed to read settings:', error?.message || error); - return DEFAULT_SETTINGS; + const output = execFileSync(path.join(os.homedir(), '.local/bin/camoufox'), ['path'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + const executablePath = fs.statSync(output).isDirectory() + ? path.join(output, 'camoufox') + : output; + return fs.existsSync(executablePath) ? executablePath : null; + } catch { + return null; } } -function writeSettings(settings: BrowserUseSettings): BrowserUseSettings { - const normalized = { - enabled: settings.enabled === true, - }; - - appConfigDb.set(BROWSER_USE_SETTINGS_KEY, JSON.stringify(normalized)); - return normalized; -} - -function getOrCreateMcpToken(): string { - const existing = appConfigDb.get(BROWSER_USE_MCP_TOKEN_KEY); - if (existing) { - return existing; - } - const token = randomBytes(32).toString('hex'); - appConfigDb.set(BROWSER_USE_MCP_TOKEN_KEY, token); - return token; -} - function getSetupMessage(settings: BrowserUseSettings, readiness: RuntimeReadiness): string { if (!settings.enabled) { return 'Browser is disabled in settings.'; @@ -132,6 +90,26 @@ function getSetupMessage(settings: BrowserUseSettings, readiness: RuntimeReadine return 'Install Playwright and Chromium to use browser sessions.'; } + if (settings.browserBackend === 'camoufox-vnc' && !getCamoufoxExecutablePath()) { + return 'Camoufox is selected, but Camoufox is not installed.'; + } + + if (useVisibleCamoufoxBackend(settings)) { + if (!VISIBLE_BROWSER_ENABLED) { + return 'Camoufox is selected, but visible browser sessions are disabled.'; + } + if (!getCamoufoxExecutablePath()) { + return 'Camoufox is selected, but Camoufox is not installed.'; + } + if (!fs.existsSync(X11VNC_BIN)) { + return 'Camoufox is selected, but x11vnc is missing.'; + } + if (!fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html'))) { + return 'Camoufox is selected, but noVNC is missing.'; + } + return readiness.installMessage || 'Camoufox runtime is not ready.'; + } + if (!readiness.chromiumInstalled) { return 'Playwright is installed, but Chromium is missing. Install the Chromium runtime to continue.'; } @@ -176,24 +154,6 @@ async function removeMcpServerFromAllProviders(name: string) { return results.map((result) => ({ ...result, name })); } -function normalizeProfileName(profileName?: string | null): string | null { - const normalized = String(profileName || '').trim(); - if (!normalized) { - return null; - } - - return normalized.slice(0, 80); -} - -function getProfilePath(profileName: string): string { - const safeName = profileName - .toLowerCase() - .replace(/[^a-z0-9._-]+/g, '-') - .replace(/^-+|-+$/g, '') - .slice(0, 80) || 'default'; - return path.join(PROFILE_ROOT, safeName); -} - function probeRuntime(): RuntimeProbe { const playwright = getPlaywright(); const readiness: RuntimeProbe = { @@ -238,6 +198,126 @@ function getRuntimeReadiness(options: { force?: boolean } = {}): RuntimeReadines }; } +function findAvailablePort(): Promise { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.on('error', reject); + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + server.close(() => { + if (typeof address === 'object' && address?.port) { + resolve(address.port); + } else { + reject(new Error('Failed to reserve a browser runtime port.')); + } + }); + }); + }); +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function killRuntimeProcesses(processes?: Array>) { + processes?.forEach((child) => child.kill('SIGTERM')); +} + +function reserveDisplay(): string { + for (let index = 90; index < 140; index += 1) { + const display = `:${index}`; + if (!reservedDisplays.has(display)) { + reservedDisplays.add(display); + return display; + } + } + + throw new Error('No browser display slots are available.'); +} + +function spawnRuntimeProcess(command: string, args: string[], options: { env?: NodeJS.ProcessEnv } = {}) { + const child = spawn(command, args, { + env: { ...process.env, ...options.env }, + stdio: ['ignore', 'ignore', 'pipe'], + }); + child.stderr?.on('data', (chunk) => { + if (!LOG_RUNTIME_PROCESS_OUTPUT) { + return; + } + const text = String(chunk).trim(); + if (text) { + console.warn(`[Browser runtime] ${path.basename(command)}: ${text}`); + } + }); + child.on('error', (error) => { + console.warn(`[Browser runtime] ${path.basename(command)} failed:`, error.message); + }); + return child; +} + +async function startVisibleRuntime(): Promise & { processes: Array> }> { + const display = reserveDisplay(); + const vncPort = await findAvailablePort(); + const websockifyPort = await findAvailablePort(); + const processes: Array> = []; + + try { + processes.push(spawnRuntimeProcess('Xvfb', [ + display, + '-screen', + '0', + '1440x900x24', + '-ac', + '-nolisten', + 'tcp', + ])); + await delay(700); + + if (!fs.existsSync(X11VNC_BIN)) { + throw new Error(`x11vnc is missing at ${X11VNC_BIN}.`); + } + processes.push(spawnRuntimeProcess(X11VNC_BIN, [ + '-display', + display, + '-localhost', + '-forever', + '-shared', + '-rfbport', + String(vncPort), + '-nopw', + '-quiet', + ], { + env: { + LD_LIBRARY_PATH: `${X11VNC_LIB_DIR}:${X11VNC_EXTRA_LIB_DIR}:${process.env.LD_LIBRARY_PATH || ''}`, + }, + })); + await delay(500); + + if (!fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html'))) { + throw new Error(`noVNC is missing at ${NOVNC_ROOT}.`); + } + processes.push(spawnRuntimeProcess(path.join(os.homedir(), '.local/bin/websockify'), [ + '--web', + NOVNC_ROOT, + `127.0.0.1:${websockifyPort}`, + `127.0.0.1:${vncPort}`, + ])); + await delay(500); + + return { + display, + vncPort, + websockifyPort, + noVncRoot: NOVNC_ROOT, + processes, + }; + } catch (error) { + killRuntimeProcesses(processes); + reservedDisplays.delete(display); + throw error; + } +} + const INSTALL_COMMAND_TIMEOUT_MS = Number.parseInt( process.env.CLOUDCLI_BROWSER_USE_INSTALL_TIMEOUT_MS || String(10 * 60 * 1000), 10, @@ -350,6 +430,42 @@ function publicSession(session: BrowserUseSession): PublicBrowserUseSession { return publicFields; } +function getSessionViewer(sessionId: string): RuntimeHandle['viewer'] | null { + const session = sessions.get(sessionId); + if (!session || session.ownerId !== AGENT_OWNER_ID || session.status !== 'ready') { + return null; + } + return handles.get(sessionId)?.viewer || null; +} + +function createViewerToken(sessionId: string): string { + const token = randomUUID(); + viewerTokens.set(sessionId, { + token, + expiresAt: Date.now() + VIEWER_TOKEN_TTL_MS, + }); + return token; +} + +function deleteViewerToken(sessionId: string) { + viewerTokens.delete(sessionId); +} + +function validateViewerTokenForSession(sessionId: string, token: string | null | undefined): boolean { + if (!token) { + return false; + } + const viewer = getSessionViewer(sessionId); + const stored = viewerTokens.get(sessionId); + if (!viewer || !stored || stored.token !== token || stored.expiresAt < Date.now()) { + if (stored?.expiresAt && stored.expiresAt < Date.now()) { + viewerTokens.delete(sessionId); + } + return false; + } + return true; +} + function ownerSessions(ownerId: string): BrowserUseSession[] { return [...sessions.values()].filter((session) => session.ownerId === ownerId); } @@ -357,8 +473,13 @@ function ownerSessions(ownerId: string): BrowserUseSession[] { async function closeHandle(sessionId: string): Promise { const handle = handles.get(sessionId); handles.delete(sessionId); + deleteViewerToken(sessionId); await handle?.context?.close?.().catch(() => undefined); await handle?.browser?.close().catch(() => undefined); + killRuntimeProcesses(handle?.processes); + if (handle?.viewer?.display) { + reservedDisplays.delete(handle.viewer.display); + } } async function expireStaleSessions(now = Date.now()): Promise { @@ -424,6 +545,11 @@ export const browserUseService = { const current = readSettings(); const nextSettings = { enabled: typeof settings.enabled === 'boolean' ? settings.enabled : current.enabled, + persistSessions: typeof settings.persistSessions === 'boolean' ? settings.persistSessions : current.persistSessions, + defaultProfileName: typeof settings.defaultProfileName === 'string' + ? settings.defaultProfileName + : current.defaultProfileName, + browserBackend: settings.browserBackend ? normalizeBrowserBackend(settings.browserBackend) : current.browserBackend, }; const next = writeSettings(nextSettings); @@ -439,14 +565,28 @@ export const browserUseService = { async getStatus() { const settings = readSettings(); const readiness = getRuntimeReadiness(); - const available = settings.enabled && readiness.playwrightInstalled && readiness.chromiumInstalled; + const useVisibleBackend = useVisibleCamoufoxBackend(settings); + const visibleCamoufoxReady = useVisibleBackend + && VISIBLE_BROWSER_ENABLED + && readiness.playwrightInstalled + && Boolean(getCamoufoxExecutablePath()) + && fs.existsSync(X11VNC_BIN) + && fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html')); + const available = settings.enabled + && readiness.playwrightInstalled + && (useVisibleBackend ? visibleCamoufoxReady : readiness.chromiumInstalled); return { enabled: settings.enabled, runtime: getRuntime(), + backend: useVisibleBackend ? 'camoufox-vnc' : 'playwright', + browserBackend: settings.browserBackend, available, playwrightInstalled: readiness.playwrightInstalled, chromiumInstalled: readiness.chromiumInstalled, + camoufoxInstalled: Boolean(getCamoufoxExecutablePath()), + noVncInstalled: fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html')), + x11vncInstalled: fs.existsSync(X11VNC_BIN), installInProgress: readiness.installInProgress, sessionCount: sessions.size, message: available @@ -505,7 +645,7 @@ export const browserUseService = { } await expireStaleSessions(); - const profileName = normalizeProfileName(options?.profileName); + const profileName = resolveSessionProfileName(settings, options?.profileName); const now = new Date().toISOString(); const session: BrowserUseSession = { @@ -521,6 +661,9 @@ export const browserUseService = { updatedAt: now, lastAction: 'create', message: null, + backend: useVisibleCamoufoxBackend(settings) ? 'camoufox-vnc' : 'playwright', + viewerUrl: null, + viewerEmbedUrl: null, profileName, viewport: { width: 1440, height: 900 }, cursor: null, @@ -532,7 +675,13 @@ export const browserUseService = { } const readiness = getRuntimeReadiness(); - if (!settings.enabled || !readiness.playwrightInstalled || !readiness.chromiumInstalled || !readiness.playwright) { + const useVisibleBackend = useVisibleCamoufoxBackend(settings); + const visibleCamoufoxReady = useVisibleBackend + && VISIBLE_BROWSER_ENABLED + && Boolean(getCamoufoxExecutablePath()) + && fs.existsSync(X11VNC_BIN) + && fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html')); + if (!settings.enabled || !readiness.playwrightInstalled || !readiness.playwright || (useVisibleBackend ? !visibleCamoufoxReady : !readiness.chromiumInstalled)) { session.message = getSetupMessage(settings, readiness); sessions.set(session.id, session); return publicSession(session); @@ -541,31 +690,73 @@ export const browserUseService = { let browser: any | undefined; let context: any | undefined; let page: any; - const launchOptions = { - headless: true, + let viewer: RuntimeHandle['viewer']; + let processes: RuntimeHandle['processes']; + const launchOptions: Record = { + headless: !useVisibleBackend, args: ['--disable-dev-shm-usage'], }; - const contextOptions = { - viewport: { width: 1440, height: 900 }, - serviceWorkers: 'block', - }; + const contextOptions = useVisibleBackend + ? { viewport: null } + : { + viewport: { width: 1440, height: 900 }, + serviceWorkers: 'block', + }; - if (profileName) { - fs.mkdirSync(PROFILE_ROOT, { recursive: true }); - context = await readiness.playwright.chromium.launchPersistentContext(getProfilePath(profileName), { - ...launchOptions, - ...contextOptions, - }); - page = context.pages()[0] || await context.newPage(); - } else { - browser = await readiness.playwright.chromium.launch(launchOptions); - context = await browser.newContext(contextOptions); - page = await context.newPage(); + try { + if (useVisibleBackend) { + const camoufoxExecutable = getCamoufoxExecutablePath(); + if (!camoufoxExecutable) { + throw new Error('Camoufox is not installed.'); + } + const runtime = await startVisibleRuntime(); + viewer = { + display: runtime.display, + vncPort: runtime.vncPort, + websockifyPort: runtime.websockifyPort, + noVncRoot: runtime.noVncRoot, + }; + processes = runtime.processes; + launchOptions.executablePath = camoufoxExecutable; + launchOptions.env = { + ...process.env, + DISPLAY: runtime.display, + LD_LIBRARY_PATH: `${X11VNC_LIB_DIR}:${X11VNC_EXTRA_LIB_DIR}:${process.env.LD_LIBRARY_PATH || ''}`, + }; + launchOptions.args = []; + session.backend = 'camoufox-vnc'; + const viewerToken = createViewerToken(session.id); + session.viewerUrl = getViewerUrl(session.id, viewerToken); + session.viewerEmbedUrl = session.viewerUrl; + } + + if (profileName) { + fs.mkdirSync(PROFILE_ROOT, { recursive: true }); + const browserType = useVisibleBackend ? readiness.playwright.firefox : readiness.playwright.chromium; + context = await browserType.launchPersistentContext(getProfilePath(profileName), { + ...launchOptions, + ...contextOptions, + }); + page = context.pages()[0] || await context.newPage(); + } else { + const browserType = useVisibleBackend ? readiness.playwright.firefox : readiness.playwright.chromium; + browser = await browserType.launch(launchOptions); + context = await browser.newContext(contextOptions); + page = await context.newPage(); + } + } catch (error) { + await context?.close?.().catch(() => undefined); + await browser?.close?.().catch(() => undefined); + killRuntimeProcesses(processes); + if (viewer?.display) { + reservedDisplays.delete(viewer.display); + } + throw error; } session.status = 'ready'; session.message = 'Browser session is ready.'; sessions.set(session.id, session); - handles.set(session.id, { browser, context, page }); + handles.set(session.id, { browser, context, page, processes, viewer }); await captureSession(session, page); return publicSession(session); }, @@ -812,6 +1003,25 @@ export const browserUseService = { return { deleted: true, sessionId }; }, + getViewerProxyTarget(sessionId: string) { + const viewer = getSessionViewer(sessionId); + if (!viewer) { + throw new Error('Browser viewer is not available for this session.'); + } + return { + websockifyPort: viewer.websockifyPort, + noVncRoot: viewer.noVncRoot, + }; + }, + + validateViewerToken(sessionId: string, token: string | null | undefined) { + return validateViewerTokenForSession(sessionId, token); + }, + + handleViewerWebSocket(clientWs: WebSocket, pathname: string) { + handleViewerWebSocket(clientWs, pathname, getSessionViewer); + }, + async agentStopSession(sessionId: string) { await this.getAgentSession(sessionId); return this.stopSession(sessionId); diff --git a/server/modules/browser-use/browser-use.settings.ts b/server/modules/browser-use/browser-use.settings.ts new file mode 100644 index 00000000..314ade15 --- /dev/null +++ b/server/modules/browser-use/browser-use.settings.ts @@ -0,0 +1,102 @@ +import { randomBytes } from 'node:crypto'; +import os from 'node:os'; +import path from 'node:path'; + +import { appConfigDb } from '@/modules/database/index.js'; + +import type { BrowserUseBackend, BrowserUseSettings } from './browser-use.types.js'; + +const IS_PLATFORM = process.env.VITE_IS_PLATFORM === 'true'; +const BROWSER_USE_SETTINGS_KEY = 'browser_use_settings'; +const BROWSER_USE_MCP_TOKEN_KEY = 'browser_use_mcp_token'; + +export const DEFAULT_BROWSER_USE_SETTINGS: BrowserUseSettings = { + enabled: false, + persistSessions: false, + defaultProfileName: 'default', + browserBackend: IS_PLATFORM ? 'camoufox-vnc' : 'playwright', +}; + +export const PROFILE_ROOT = path.join(os.homedir(), '.cloudcli', 'browser-use', 'profiles'); + +export function normalizeBrowserBackend(value: unknown): BrowserUseBackend { + return value === 'playwright' || value === 'camoufox-vnc' + ? value + : DEFAULT_BROWSER_USE_SETTINGS.browserBackend; +} + +export function normalizeProfileName(profileName?: string | null): string | null { + const normalized = String(profileName || '').trim(); + if (!normalized) { + return null; + } + + return normalized.slice(0, 80); +} + +export function normalizeDefaultProfileName(profileName?: string | null): string { + return normalizeProfileName(profileName) || DEFAULT_BROWSER_USE_SETTINGS.defaultProfileName; +} + +export function resolveSessionProfileName(settings: BrowserUseSettings, profileName?: string | null): string | null { + const requestedProfileName = normalizeProfileName(profileName); + if (requestedProfileName) { + return requestedProfileName; + } + return settings.persistSessions ? normalizeDefaultProfileName(settings.defaultProfileName) : null; +} + +export function getProfilePath(profileName: string): string { + const safeName = profileName + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 80) || 'default'; + return path.join(PROFILE_ROOT, safeName); +} + +export function useVisibleCamoufoxBackend(settings: BrowserUseSettings): boolean { + return settings.browserBackend === 'camoufox-vnc'; +} + +export function readSettings(): BrowserUseSettings { + try { + const raw = appConfigDb.get(BROWSER_USE_SETTINGS_KEY); + if (!raw) { + return DEFAULT_BROWSER_USE_SETTINGS; + } + + const parsed = JSON.parse(raw) as Partial; + return { + enabled: parsed.enabled === true, + persistSessions: parsed.persistSessions === true, + defaultProfileName: normalizeDefaultProfileName(parsed.defaultProfileName), + browserBackend: normalizeBrowserBackend(parsed.browserBackend), + }; + } catch (error: any) { + console.warn('[Browser] Failed to read settings:', error?.message || error); + return DEFAULT_BROWSER_USE_SETTINGS; + } +} + +export function writeSettings(settings: BrowserUseSettings): BrowserUseSettings { + const normalized = { + enabled: settings.enabled === true, + persistSessions: settings.persistSessions === true, + defaultProfileName: normalizeDefaultProfileName(settings.defaultProfileName), + browserBackend: normalizeBrowserBackend(settings.browserBackend), + }; + + appConfigDb.set(BROWSER_USE_SETTINGS_KEY, JSON.stringify(normalized)); + return normalized; +} + +export function getOrCreateMcpToken(): string { + const existing = appConfigDb.get(BROWSER_USE_MCP_TOKEN_KEY); + if (existing) { + return existing; + } + const token = randomBytes(32).toString('hex'); + appConfigDb.set(BROWSER_USE_MCP_TOKEN_KEY, token); + return token; +} diff --git a/server/modules/browser-use/browser-use.types.ts b/server/modules/browser-use/browser-use.types.ts new file mode 100644 index 00000000..f542d67c --- /dev/null +++ b/server/modules/browser-use/browser-use.types.ts @@ -0,0 +1,66 @@ +import type { spawn } from 'node:child_process'; + +export type BrowserUseRuntime = 'cloud' | 'local'; +export type BrowserUseBackend = 'playwright' | 'camoufox-vnc'; +export type BrowserUseSessionStatus = 'ready' | 'stopped' | 'unavailable'; + +export type BrowserUseSession = { + id: string; + ownerId: string; + createdBy: 'agent'; + runtime: BrowserUseRuntime; + status: BrowserUseSessionStatus; + url: string | null; + title: string | null; + screenshotDataUrl: string | null; + createdAt: string; + updatedAt: string; + lastAction: string | null; + message: string | null; + backend: BrowserUseBackend; + viewerUrl: string | null; + viewerEmbedUrl: string | null; + profileName: string | null; + viewport: { + width: number; + height: number; + } | null; + cursor: { + x: number; + y: number; + actor: 'agent'; + } | null; +}; + +export type PublicBrowserUseSession = Omit; + +export type RuntimeHandle = { + browser?: any; + context?: any; + page?: any; + processes?: Array>; + viewer?: { + display: string; + vncPort: number; + websockifyPort: number; + noVncRoot: string; + }; +}; + +export type BrowserUseSettings = { + enabled: boolean; + persistSessions: boolean; + defaultProfileName: string; + browserBackend: BrowserUseBackend; +}; + +export type RuntimeReadiness = { + playwright: any | null; + playwrightInstalled: boolean; + chromiumInstalled: boolean; + chromiumExecutablePath: string | null; + installInProgress: boolean; + installMessage: string | null; +}; + +export type RuntimeProbe = Omit; diff --git a/server/modules/browser-use/browser-use.viewer.ts b/server/modules/browser-use/browser-use.viewer.ts new file mode 100644 index 00000000..5ddb7ec2 --- /dev/null +++ b/server/modules/browser-use/browser-use.viewer.ts @@ -0,0 +1,71 @@ +import { WebSocket } from 'ws'; + +import type { RuntimeHandle } from './browser-use.types.js'; + +type BrowserUseViewer = NonNullable; + +export const VIEWER_COOKIE_NAME = 'browser_use_viewer_token'; +export const VIEWER_TOKEN_TTL_MS = Number.parseInt( + process.env.CLOUDCLI_BROWSER_USE_VIEWER_TOKEN_TTL_MS || String(30 * 60 * 1000), + 10, +); + +export function getViewerUrl(sessionId: string, viewerToken?: string): string { + const basePath = `/api/browser-use/sessions/${encodeURIComponent(sessionId)}/viewer`; + const websockifyPath = viewerToken + ? `${basePath}/websockify?viewerToken=${encodeURIComponent(viewerToken)}` + : `${basePath}/websockify`; + const params = new URLSearchParams({ + autoconnect: '1', + resize: 'scale', + reconnect: '1', + path: websockifyPath, + }); + if (viewerToken) { + params.set('viewerToken', viewerToken); + } + return `${basePath}/vnc.html?${params.toString()}`; +} + +export function handleViewerWebSocket( + clientWs: WebSocket, + pathname: string, + getSessionViewer: (sessionId: string) => BrowserUseViewer | null | undefined, +) { + const match = /^\/api\/browser-use\/sessions\/([^/]+)\/viewer\/websockify\/?$/.exec(pathname); + const sessionId = match ? decodeURIComponent(match[1]) : ''; + const viewer = sessionId ? getSessionViewer(sessionId) : null; + if (!viewer) { + clientWs.close(4404, 'Browser viewer not found'); + return; + } + + const upstream = new WebSocket(`ws://127.0.0.1:${viewer.websockifyPort}`); + upstream.on('open', () => { + clientWs.on('message', (data) => { + if (upstream.readyState === WebSocket.OPEN) { + upstream.send(data); + } + }); + upstream.on('message', (data) => { + if (clientWs.readyState === WebSocket.OPEN) { + clientWs.send(data); + } + }); + }); + upstream.on('close', (code, reason) => { + if (clientWs.readyState === WebSocket.OPEN) { + clientWs.close(code, reason); + } + }); + upstream.on('error', () => { + if (clientWs.readyState === WebSocket.OPEN) { + clientWs.close(4502, 'Browser viewer upstream error'); + } + }); + clientWs.on('close', () => { + if (upstream.readyState === WebSocket.OPEN) { + upstream.close(); + } + }); +} diff --git a/server/modules/browser-use/index.ts b/server/modules/browser-use/index.ts new file mode 100644 index 00000000..e7d0783d --- /dev/null +++ b/server/modules/browser-use/index.ts @@ -0,0 +1,2 @@ +export { browserUseService } from './browser-use.service.js'; +export { VIEWER_COOKIE_NAME } from './browser-use.viewer.js'; diff --git a/server/modules/websocket/services/websocket-server.service.ts b/server/modules/websocket/services/websocket-server.service.ts index 2ba2ec6e..b6dbbff1 100644 --- a/server/modules/websocket/services/websocket-server.service.ts +++ b/server/modules/websocket/services/websocket-server.service.ts @@ -1,8 +1,9 @@ import type { Server as HttpServer } from 'node:http'; -import { WebSocketServer, type VerifyClientCallbackSync } from 'ws'; +import { WebSocket, WebSocketServer, type VerifyClientCallbackSync } from 'ws'; import { handleChatConnection } from '@/modules/websocket/services/chat-websocket.service.js'; +import { VIEWER_COOKIE_NAME } from '@/modules/browser-use/index.js'; import { verifyWebSocketClient } from '@/modules/websocket/services/websocket-auth.service.js'; import { handlePluginWsProxy } from '@/modules/websocket/services/plugin-websocket-proxy.service.js'; import { handleShellConnection } from '@/modules/websocket/services/shell-websocket.service.js'; @@ -13,8 +14,21 @@ type WebSocketServerDependencies = { chat: Parameters[2]; shell: Parameters[1]; getPluginPort: Parameters[2]; + browserUseViewer?: (ws: WebSocket, pathname: string) => void; + authenticateBrowserUseViewer?: (pathname: string, token: string | null) => boolean; }; +function readCookieValue(header: unknown, name: string): string | null { + if (!header) return null; + const prefix = `${name}=`; + const cookie = String(header).split(';').map((part) => part.trim()).find((part) => part.startsWith(prefix)); + return cookie ? decodeURIComponent(cookie.slice(prefix.length)) : null; +} + +function getBrowserUseViewerToken(url: URL, headers: Record): string | null { + return url.searchParams.get('viewerToken') || readCookieValue(headers.cookie, VIEWER_COOKIE_NAME); +} + /** * Creates and wires the server-wide websocket gateway used for chat, shell, and * plugin proxy routes. @@ -27,7 +41,19 @@ export function createWebSocketServer( server, verifyClient: (( info: Parameters>[0] - ) => verifyWebSocketClient(info, dependencies.verifyClient)), + ) => { + const requestUrl = new URL(info.req.url ?? '/', 'http://localhost'); + if ( + requestUrl.pathname.startsWith('/api/browser-use/sessions/') + && requestUrl.pathname.endsWith('/viewer/websockify') + ) { + const token = getBrowserUseViewerToken(requestUrl, info.req.headers as Record); + if (dependencies.authenticateBrowserUseViewer?.(requestUrl.pathname, token)) { + return true; + } + } + return verifyWebSocketClient(info, dependencies.verifyClient); + }), }); wss.on('connection', (ws, request) => { @@ -68,6 +94,11 @@ export function createWebSocketServer( return; } + if (pathname.startsWith('/api/browser-use/sessions/') && pathname.endsWith('/viewer/websockify')) { + dependencies.browserUseViewer?.(ws, pathname); + return; + } + console.log('[WARN] Unknown WebSocket path:', pathname); ws.close(); }); diff --git a/src/components/browser-use/view/BrowserUsePanel.tsx b/src/components/browser-use/view/BrowserUsePanel.tsx index b5b5f2a9..e887f804 100644 --- a/src/components/browser-use/view/BrowserUsePanel.tsx +++ b/src/components/browser-use/view/BrowserUsePanel.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { Bot, Clock3, @@ -7,6 +7,7 @@ import { ExternalLink, Loader2, MonitorPlay, + MousePointer2, RefreshCw, Settings, Square, @@ -19,9 +20,14 @@ import { Badge, Button } from '../../../shared/view/ui'; import { authenticatedFetch } from '../../../utils/api'; import type { SettingsMainTab } from '../../settings/types/types'; +const BROWSER_USE_GUIDE_URL = 'https://cloudcli.ai/docs/browser-use'; +const BROWSER_USE_CACHE_TTL_MS = 30_000; + type BrowserUseStatus = { enabled: boolean; available: boolean; + backend: 'playwright' | 'camoufox-vnc'; + browserBackend: 'playwright' | 'camoufox-vnc'; playwrightInstalled: boolean; chromiumInstalled: boolean; installInProgress: boolean; @@ -39,6 +45,9 @@ type BrowserUseSession = { updatedAt: string; lastAction: string | null; message: string | null; + backend?: 'playwright' | 'camoufox-vnc'; + viewerUrl?: string | null; + viewerEmbedUrl?: string | null; createdBy: 'agent'; profileName: string | null; viewport: { @@ -54,17 +63,48 @@ type BrowserUseSession = { type BrowserUsePanelProps = { isVisible: boolean; + projectId?: string | null; onShowSettings?: (tab?: SettingsMainTab) => void; }; +type BrowserUsePanelCacheEntry = { + status: BrowserUseStatus | null; + sessions: BrowserUseSession[]; + selectedSessionId: string | null; + updatedAt: number; +}; + +const browserUsePanelCache = new Map(); + async function readJson(response: Response): Promise { - const data = await response.json(); + const text = await response.text(); + let data: any = {}; + if (text) { + try { + data = JSON.parse(text); + } catch { + throw new Error(response.ok ? 'Received an invalid Browser response.' : `Browser request failed (${response.status}).`); + } + } if (!response.ok || data.success === false) { throw new Error(data.error || data.details || `Request failed (${response.status})`); } return data as T; } +async function fetchBrowserPanelData() { + const [statusResponse, sessionsResponse] = await Promise.all([ + authenticatedFetch('/api/browser-use/status'), + authenticatedFetch('/api/browser-use/sessions'), + ]); + const statusData = await readJson<{ data: BrowserUseStatus }>(statusResponse); + const sessionsData = await readJson<{ data: { sessions: BrowserUseSession[] } }>(sessionsResponse); + return { + status: statusData.data, + sessions: [...sessionsData.data.sessions].sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt)), + }; +} + function formatRelativeTime(value: string | null): string { if (!value) return 'Never'; @@ -119,20 +159,42 @@ function getStatusDot(status: BrowserUseSession['status']): string { return 'bg-border'; } +function getEngineLabel(backend?: BrowserUseStatus['backend'] | BrowserUseSession['backend']): string { + return backend === 'camoufox-vnc' ? 'Visible browser' : 'Playwright'; +} + const PROMPTS = [ 'Use Browser to inspect the checkout flow and report any broken UI states.', 'Open with Browser, interact with the page, and summarize what changed after each step.', ]; -export default function BrowserUsePanel({ isVisible, onShowSettings }: BrowserUsePanelProps) { - const [status, setStatus] = useState(null); - const [sessions, setSessions] = useState([]); - const [selectedSessionId, setSelectedSessionId] = useState(null); +function getBrowserUseCacheKey(projectId?: string | null): string { + return projectId ? `browser-use:project:${projectId}` : 'browser-use:global'; +} + +function getFreshCacheEntry(cacheKey: string): BrowserUsePanelCacheEntry | null { + const entry = browserUsePanelCache.get(cacheKey); + if (!entry || Date.now() - entry.updatedAt > BROWSER_USE_CACHE_TTL_MS) { + return null; + } + return entry; +} + +export default function BrowserUsePanel({ isVisible, projectId, onShowSettings }: BrowserUsePanelProps) { + const cacheKey = getBrowserUseCacheKey(projectId); + const initialCacheEntry = getFreshCacheEntry(cacheKey); + const [status, setStatus] = useState(() => initialCacheEntry?.status ?? null); + const [sessions, setSessions] = useState(() => initialCacheEntry?.sessions ?? []); + const [selectedSessionId, setSelectedSessionId] = useState(() => ( + initialCacheEntry?.selectedSessionId || initialCacheEntry?.sessions[0]?.id || null + )); + const [hasLoadedOnce, setHasLoadedOnce] = useState(Boolean(initialCacheEntry)); const [isRefreshing, setIsRefreshing] = useState(false); const [isBusy, setIsBusy] = useState(false); const [isInstalling, setIsInstalling] = useState(false); const [isFullscreen, setIsFullscreen] = useState(false); const [error, setError] = useState(null); + const activeLoadIdRef = useRef(0); const selectedSession = useMemo( () => sessions.find((session) => session.id === selectedSessionId) || sessions[0] || null, @@ -140,8 +202,12 @@ export default function BrowserUsePanel({ isVisible, onShowSettings }: BrowserUs ); const activeSessions = sessions.filter((session) => session.status === 'ready'); - const needsBrowserBinaries = Boolean(status?.enabled && (!status.playwrightInstalled || !status.chromiumInstalled)); - const runtimeLabel = !status?.enabled + const isInitialLoading = isRefreshing && !hasLoadedOnce && sessions.length === 0; + const isBackgroundRefreshing = isRefreshing && !isInitialLoading; + const needsBrowserBinaries = Boolean(status?.enabled && !status.available); + const runtimeLabel = isInitialLoading + ? 'Loading' + : !status?.enabled ? 'Disabled' : status.available ? 'Ready' @@ -157,29 +223,73 @@ export default function BrowserUsePanel({ isVisible, onShowSettings }: BrowserUs : null; const refresh = useCallback(async () => { + const loadId = activeLoadIdRef.current + 1; + activeLoadIdRef.current = loadId; setIsRefreshing(true); try { - const [statusResponse, sessionsResponse] = await Promise.all([ - authenticatedFetch('/api/browser-use/status'), - authenticatedFetch('/api/browser-use/sessions'), - ]); - const statusData = await readJson<{ data: BrowserUseStatus }>(statusResponse); - const sessionsData = await readJson<{ data: { sessions: BrowserUseSession[] } }>(sessionsResponse); - const nextSessions = sessionsData.data.sessions; - setStatus(statusData.data); + let nextData: Awaited>; + try { + nextData = await fetchBrowserPanelData(); + } catch (error) { + if (loadId !== activeLoadIdRef.current) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 350)); + nextData = await fetchBrowserPanelData(); + } + if (activeLoadIdRef.current !== loadId) { + return; + } + const nextSessions = nextData.sessions; + setStatus(nextData.status); setSessions(nextSessions); - setSelectedSessionId((current) => ( - current && nextSessions.some((session) => session.id === current) + setHasLoadedOnce(true); + let nextSelectedSessionId: string | null = null; + setSelectedSessionId((current) => { + nextSelectedSessionId = current && nextSessions.some((session) => session.id === current) ? current - : nextSessions[0]?.id || null - )); + : nextSessions[0]?.id || null; + return nextSelectedSessionId; + }); + browserUsePanelCache.set(cacheKey, { + status: nextData.status, + sessions: nextSessions, + selectedSessionId: nextSelectedSessionId, + updatedAt: Date.now(), + }); setError(null); } catch (err) { + if (activeLoadIdRef.current !== loadId) { + return; + } + setHasLoadedOnce(true); setError(err instanceof Error ? err.message : 'Failed to load Browser'); } finally { - setIsRefreshing(false); + if (activeLoadIdRef.current === loadId) { + setIsRefreshing(false); + } } - }, []); + }, [cacheKey]); + + useEffect(() => { + const cachedEntry = browserUsePanelCache.get(cacheKey); + if (!cachedEntry) return; + browserUsePanelCache.set(cacheKey, { + ...cachedEntry, + selectedSessionId, + updatedAt: Date.now(), + }); + }, [cacheKey, selectedSessionId]); + + useEffect(() => { + const cachedEntry = getFreshCacheEntry(cacheKey); + setStatus(cachedEntry?.status ?? null); + setSessions(cachedEntry?.sessions ?? []); + setSelectedSessionId(cachedEntry?.selectedSessionId || cachedEntry?.sessions[0]?.id || null); + setHasLoadedOnce(Boolean(cachedEntry)); + setError(null); + activeLoadIdRef.current += 1; + }, [cacheKey]); useEffect(() => { if (!isVisible) return; @@ -253,6 +363,10 @@ export default function BrowserUsePanel({ isVisible, onShowSettings }: BrowserUs {formatRelativeTime(session.updatedAt)} - {formatAction(session.lastAction)} +
+ {getEngineLabel(session.backend)} + {session.profileName || 'Temporary'} +
); }; @@ -270,9 +384,18 @@ export default function BrowserUsePanel({ isVisible, onShowSettings }: BrowserUs

{status?.enabled - ? 'Agent browser sessions appear here while an AI task is using Browser.' - : 'Enable Browser in settings to let agents open monitored browser sessions.'} + ? 'When an agent opens a browser, you can watch the latest screenshot, take control in a new tab, or end the running session.' + : 'Enable Browser to let agents open websites, test flows, capture screenshots, and debug UI from a real page.'}

+ + Read the Browser guide + + @@ -312,10 +435,19 @@ export default function BrowserUsePanel({ isVisible, onShowSettings }: BrowserUs ); + const renderLoadingState = () => ( +
+
+ + Loading browser sessions... +
+
+ ); + const renderBrowserSurface = (fullscreen = false) => (
{selectedSession?.screenshotDataUrl ? ( -
+
Browser session screenshot
)} + {selectedSession?.viewerEmbedUrl && selectedSession.status === 'ready' && ( + + )}
) : (
@@ -350,10 +494,29 @@ export default function BrowserUsePanel({ isVisible, onShowSettings }: BrowserUs {runtimeLabel} + + {getEngineLabel(status?.backend)} +
-

Monitor browser sessions opened by AI agents.

+

Watch and manage browser sessions agents use to test real websites.

+ {isBackgroundRefreshing && ( +
+ + Refreshing sessions... +
+ )}
+ {onShowSettings && ( + )} -
) : showChatNewSession ? (
diff --git a/src/components/settings/view/tabs/browser-use-settings/BrowserUseSettingsTab.tsx b/src/components/settings/view/tabs/browser-use-settings/BrowserUseSettingsTab.tsx index da94ffe2..90c93581 100644 --- a/src/components/settings/view/tabs/browser-use-settings/BrowserUseSettingsTab.tsx +++ b/src/components/settings/view/tabs/browser-use-settings/BrowserUseSettingsTab.tsx @@ -1,22 +1,32 @@ import { useCallback, useEffect, useState } from 'react'; -import { Download, Loader2 } from 'lucide-react'; +import { Download, ExternalLink, Eye, Loader2, Zap } from 'lucide-react'; -import { Button } from '../../../../../shared/view/ui'; +import { Button, Input } from '../../../../../shared/view/ui'; import { authenticatedFetch } from '../../../../../utils/api'; import SettingsCard from '../../SettingsCard'; import SettingsRow from '../../SettingsRow'; import SettingsSection from '../../SettingsSection'; import SettingsToggle from '../../SettingsToggle'; +const BROWSER_USE_GUIDE_URL = 'https://cloudcli.ai/docs/browser-use'; + type BrowserUseSettings = { enabled: boolean; + persistSessions: boolean; + defaultProfileName: string; + browserBackend: 'playwright' | 'camoufox-vnc'; }; type BrowserUseStatus = { enabled: boolean; available: boolean; + backend: 'playwright' | 'camoufox-vnc'; + browserBackend: 'playwright' | 'camoufox-vnc'; playwrightInstalled: boolean; chromiumInstalled: boolean; + camoufoxInstalled: boolean; + noVncInstalled: boolean; + x11vncInstalled: boolean; installInProgress: boolean; message: string; }; @@ -37,11 +47,13 @@ export default function BrowserUseSettingsTab() { const [isSaving, setIsSaving] = useState(false); const [isInstalling, setIsInstalling] = useState(false); const [error, setError] = useState(null); + const [profileNameDraft, setProfileNameDraft] = useState('default'); const loadSettings = useCallback(async () => { const settingsResponse = await authenticatedFetch('/api/browser-use/settings'); const settingsData = await readJson<{ data: { settings: BrowserUseSettings } }>(settingsResponse); setSettings(settingsData.data.settings); + setProfileNameDraft(settingsData.data.settings.defaultProfileName || 'default'); }, []); const loadStatus = useCallback(async () => { @@ -101,8 +113,20 @@ export default function BrowserUseSettingsTab() { } }; + const saveProfileName = async () => { + const nextName = profileNameDraft.trim() || 'default'; + setProfileNameDraft(nextName); + if (nextName === settings?.defaultProfileName) { + return; + } + await updateSettings({ defaultProfileName: nextName }); + }; + const browserEnabled = settings?.enabled === true; - const needsBrowserBinaries = Boolean(browserEnabled && status && (!status.playwrightInstalled || !status.chromiumInstalled)); + const persistSessions = settings?.persistSessions === true; + const selectedBackend = settings?.browserBackend || 'playwright'; + const effectiveBackend = status?.backend || 'playwright'; + const needsBrowserBinaries = Boolean(browserEnabled && status && !status.available); const runtimeLabel = (installed?: boolean) => { if (isStatusLoading && !status) { return 'checking...'; @@ -114,12 +138,12 @@ export default function BrowserUseSettingsTab() {
{isSettingsLoading && !settings ? ( @@ -127,20 +151,142 @@ export default function BrowserUseSettingsTab() { void updateSettings({ enabled: value })} - ariaLabel="Enable Browser" + ariaLabel="Give Agents Browser Access" disabled={isSaving} /> )} + {!browserEnabled && ( + + )} + + {browserEnabled && ( + <> +
+
+
Browser Engine
+
+ Pick the kind of browser experience agents should use for new sessions. +
+
+
+ {([ + { + value: 'playwright' as const, + label: 'Playwright', + description: 'Best for quick checks, screenshots, and automated page interaction when no manual login is needed.', + icon: Zap, + }, + { + value: 'camoufox-vnc' as const, + label: 'Camoufox + noVNC', + description: 'Best when a person may need to log in, approve a step, or watch the browser session live.', + icon: Eye, + }, + ]).map((option) => { + const Icon = option.icon; + const selected = selectedBackend === option.value; + return ( + + ); + })} +
+
+ + + {isSettingsLoading && !settings ? ( + + ) : ( + void updateSettings({ persistSessions: value })} + ariaLabel="Remember Browser Logins" + disabled={isSaving} + /> + )} + + + {persistSessions && ( + + setProfileNameDraft(event.target.value)} + onBlur={() => void saveProfileName()} + onKeyDown={(event) => { + if (event.key === 'Enter') { + event.currentTarget.blur(); + } + }} + disabled={isSaving || isSettingsLoading} + className="w-40" + aria-label="Default Browser Profile" + /> + + )} + + )} + + {browserEnabled && (
+ + Backend: {effectiveBackend === 'camoufox-vnc' ? 'Camoufox + noVNC' : 'Playwright'} + Playwright: {runtimeLabel(status?.playwrightInstalled)} Chromium: {runtimeLabel(status?.chromiumInstalled)} + + Camoufox: {runtimeLabel(status?.camoufoxInstalled)} + + + noVNC: {runtimeLabel(status?.noVncInstalled)} + Status: {isStatusLoading && !status ? 'checking...' : status?.available ? 'ready' : browserEnabled ? 'setup required' : 'disabled'} @@ -172,12 +318,23 @@ export default function BrowserUseSettingsTab() {
)} + + Read the Browser guide + + + {error && (
{error}
)}
+ )}