diff --git a/server/browser-use-mcp.ts b/server/browser-use-mcp.ts new file mode 100644 index 00000000..22a4c3e4 --- /dev/null +++ b/server/browser-use-mcp.ts @@ -0,0 +1,390 @@ +#!/usr/bin/env node +import './load-env.js'; + +type JsonRpcRequest = { + jsonrpc: '2.0'; + id?: string | number | null; + method: string; + params?: Record; +}; + +type ToolDefinition = { + name: string; + description: string; + inputSchema: Record; +}; + +const textResponse = (text: string) => ({ + content: [{ type: 'text', text }], +}); + +const jsonResponse = (value: unknown) => textResponse(JSON.stringify(value, null, 2)); + +const readString = (value: unknown, name: string): string => { + if (typeof value !== 'string' || value.trim() === '') { + throw new Error(`${name} is required.`); + } + return value.trim(); +}; + +const readOptionalString = (value: unknown): string | undefined => + typeof value === 'string' && value.trim() ? value.trim() : undefined; + +const readNumber = (value: unknown): number | undefined => + typeof value === 'number' && Number.isFinite(value) ? value : undefined; + +const apiUrl = (process.env.CLOUDCLI_BROWSER_USE_API_URL || 'http://127.0.0.1:3001/api/browser-use-mcp').replace(/\/$/, ''); +const apiToken = process.env.CLOUDCLI_BROWSER_USE_MCP_TOKEN || ''; + +async function callBrowserUseApi(toolName: string, input: Record) { + if (!apiToken) { + throw new Error('CLOUDCLI_BROWSER_USE_MCP_TOKEN is not configured.'); + } + + const response = await fetch(`${apiUrl}/tools/${encodeURIComponent(toolName)}`, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(input), + }); + const data = await response.json() as { success?: boolean; data?: unknown; error?: string }; + if (!response.ok || data.success === false) { + throw new Error(data.error || `Browser Use API request failed (${response.status})`); + } + return data.data; +} + +const sessionIdSchema = { + type: 'object', + properties: { + sessionId: { type: 'string', description: 'Browser Use session id.' }, + }, + required: ['sessionId'], +}; + +const tools: ToolDefinition[] = [ + { + name: 'browser_create_session', + description: 'Create a temporary Browser Use session that the agent can control. Optionally provide a background profileName to reuse cookies and storage.', + inputSchema: { + type: 'object', + properties: { + profileName: { type: 'string', description: 'Optional background profile name for persistent browser storage.' }, + }, + }, + }, + { + name: 'browser_list_sessions', + description: 'List Browser Use sessions currently available to agents.', + inputSchema: { type: 'object', properties: {} }, + }, + { + name: 'browser_snapshot', + description: 'Capture current page metadata, screenshot data URL, and visible body text for a Browser Use session.', + inputSchema: sessionIdSchema, + }, + { + name: 'browser_take_screenshot', + description: 'Capture the latest screenshot for a Browser Use session.', + inputSchema: sessionIdSchema, + }, + { + name: 'browser_navigate', + description: 'Navigate a Browser Use session to an HTTP or HTTPS URL.', + inputSchema: { + type: 'object', + properties: { + sessionId: { type: 'string' }, + url: { type: 'string' }, + }, + required: ['sessionId', 'url'], + }, + }, + { + name: 'browser_click', + description: 'Click an element by CSS selector, visible text, or x/y coordinates.', + inputSchema: { + type: 'object', + properties: { + sessionId: { type: 'string' }, + selector: { type: 'string' }, + text: { type: 'string' }, + x: { type: 'number' }, + y: { type: 'number' }, + }, + required: ['sessionId'], + }, + }, + { + name: 'browser_type', + description: 'Type text into the focused page or fill a CSS selector. Set submit to press Enter after typing.', + inputSchema: { + type: 'object', + properties: { + sessionId: { type: 'string' }, + selector: { type: 'string' }, + text: { type: 'string' }, + submit: { type: 'boolean' }, + }, + required: ['sessionId', 'text'], + }, + }, + { + name: 'browser_fill_form', + description: 'Fill multiple form fields using CSS selectors.', + inputSchema: { + type: 'object', + properties: { + sessionId: { type: 'string' }, + fields: { + type: 'array', + items: { + type: 'object', + properties: { + selector: { type: 'string' }, + value: { type: 'string' }, + }, + required: ['selector', 'value'], + }, + }, + }, + required: ['sessionId', 'fields'], + }, + }, + { + name: 'browser_press_key', + description: 'Press a keyboard key, for example Enter, Escape, Tab, or Control+A.', + inputSchema: { + type: 'object', + properties: { + sessionId: { type: 'string' }, + key: { type: 'string' }, + }, + required: ['sessionId', 'key'], + }, + }, + { + name: 'browser_select_option', + description: 'Select option values in a select element found by CSS selector.', + inputSchema: { + type: 'object', + properties: { + sessionId: { type: 'string' }, + selector: { type: 'string' }, + values: { type: 'array', items: { type: 'string' } }, + }, + required: ['sessionId', 'selector', 'values'], + }, + }, + { + name: 'browser_wait_for', + description: 'Wait for visible text, a URL pattern, or a short timeout.', + inputSchema: { + type: 'object', + properties: { + sessionId: { type: 'string' }, + text: { type: 'string' }, + url: { type: 'string' }, + timeoutMs: { type: 'number' }, + }, + required: ['sessionId'], + }, + }, + { + name: 'browser_tabs', + description: 'List, open, select, or close tabs in a Browser Use session.', + inputSchema: { + type: 'object', + properties: { + sessionId: { type: 'string' }, + action: { type: 'string', enum: ['list', 'new', 'select', 'close'] }, + index: { type: 'number' }, + url: { type: 'string' }, + }, + required: ['sessionId'], + }, + }, + { + name: 'browser_close_session', + description: 'Stop a Browser Use session controlled by agents.', + inputSchema: sessionIdSchema, + }, +]; + +async function callTool(name: string, args: Record) { + switch (name) { + case 'browser_create_session': + return jsonResponse(await callBrowserUseApi(name, { + profileName: readOptionalString(args.profileName), + })); + case 'browser_list_sessions': + return jsonResponse(await callBrowserUseApi(name, {})); + case 'browser_snapshot': + return jsonResponse(await callBrowserUseApi(name, { sessionId: readString(args.sessionId, 'sessionId') })); + case 'browser_take_screenshot': { + return jsonResponse(await callBrowserUseApi(name, { sessionId: readString(args.sessionId, 'sessionId') })); + } + case 'browser_navigate': + return jsonResponse(await callBrowserUseApi(name, { + sessionId: readString(args.sessionId, 'sessionId'), + url: readString(args.url, 'url'), + })); + case 'browser_click': + return jsonResponse(await callBrowserUseApi(name, { + sessionId: readString(args.sessionId, 'sessionId'), + selector: readOptionalString(args.selector), + text: readOptionalString(args.text), + x: readNumber(args.x), + y: readNumber(args.y), + })); + case 'browser_type': + return jsonResponse(await callBrowserUseApi(name, { + sessionId: readString(args.sessionId, 'sessionId'), + selector: readOptionalString(args.selector), + text: readString(args.text, 'text'), + submit: args.submit === true, + })); + case 'browser_fill_form': { + const fields = Array.isArray(args.fields) + ? args.fields.map((field) => { + const record = field as Record; + return { + selector: readString(record.selector, 'field.selector'), + value: readString(record.value, 'field.value'), + }; + }) + : []; + return jsonResponse(await callBrowserUseApi(name, { + sessionId: readString(args.sessionId, 'sessionId'), + fields, + })); + } + case 'browser_press_key': + return jsonResponse(await callBrowserUseApi(name, { + sessionId: readString(args.sessionId, 'sessionId'), + key: readString(args.key, 'key'), + })); + case 'browser_select_option': + return jsonResponse(await callBrowserUseApi(name, { + sessionId: readString(args.sessionId, 'sessionId'), + selector: readString(args.selector, 'selector'), + values: Array.isArray(args.values) ? args.values.filter((value): value is string => typeof value === 'string') : [], + })); + case 'browser_wait_for': + return jsonResponse(await callBrowserUseApi(name, { + sessionId: readString(args.sessionId, 'sessionId'), + text: readOptionalString(args.text), + url: readOptionalString(args.url), + timeoutMs: readNumber(args.timeoutMs), + })); + case 'browser_tabs': + return jsonResponse(await callBrowserUseApi(name, { + sessionId: readString(args.sessionId, 'sessionId'), + action: args.action === 'new' || args.action === 'select' || args.action === 'close' || args.action === 'list' + ? args.action + : undefined, + index: readNumber(args.index), + url: readOptionalString(args.url), + })); + case 'browser_close_session': + return jsonResponse(await callBrowserUseApi(name, { sessionId: readString(args.sessionId, 'sessionId') })); + default: + throw new Error(`Unknown tool: ${name}`); + } +} + +async function handleMessage(message: JsonRpcRequest) { + if (message.method === 'initialize') { + return { + protocolVersion: '2024-11-05', + capabilities: { tools: {} }, + serverInfo: { name: 'cloudcli-browser-use', version: '1.0.0' }, + }; + } + + if (message.method === 'tools/list') { + return { tools }; + } + + if (message.method === 'tools/call') { + const params = message.params || {}; + const name = readString(params.name, 'name'); + const args = (params.arguments && typeof params.arguments === 'object' + ? params.arguments + : {}) as Record; + return callTool(name, args); + } + + if (message.method.startsWith('notifications/')) { + return undefined; + } + + throw new Error(`Unsupported method: ${message.method}`); +} + +function writeMessage(message: Record) { + const payload = JSON.stringify(message); + process.stdout.write(`Content-Length: ${Buffer.byteLength(payload, 'utf8')}\r\n\r\n${payload}`); +} + +function sendResult(id: string | number | null | undefined, result: unknown) { + if (id === undefined) { + return; + } + writeMessage({ jsonrpc: '2.0', id, result }); +} + +function sendError(id: string | number | null | undefined, error: unknown) { + if (id === undefined) { + return; + } + writeMessage({ + jsonrpc: '2.0', + id, + error: { + code: -32000, + message: error instanceof Error ? error.message : String(error), + }, + }); +} + +let buffer = Buffer.alloc(0); + +process.stdin.on('data', (chunk) => { + buffer = Buffer.concat([buffer, chunk]); + while (true) { + const headerEnd = buffer.indexOf('\r\n\r\n'); + if (headerEnd === -1) { + return; + } + + const header = buffer.slice(0, headerEnd).toString('utf8'); + const lengthMatch = /Content-Length:\s*(\d+)/i.exec(header); + if (!lengthMatch) { + buffer = buffer.slice(headerEnd + 4); + continue; + } + + const length = Number.parseInt(lengthMatch[1], 10); + const messageStart = headerEnd + 4; + const messageEnd = messageStart + length; + if (buffer.length < messageEnd) { + return; + } + + const rawMessage = buffer.slice(messageStart, messageEnd).toString('utf8'); + buffer = buffer.slice(messageEnd); + + void (async () => { + const request = JSON.parse(rawMessage) as JsonRpcRequest; + try { + const result = await handleMessage(request); + sendResult(request.id, result); + } catch (error) { + sendError(request.id, error); + } + })(); + } +}); diff --git a/server/cli.js b/server/cli.js index 9fa99ae3..e6daacc4 100755 --- a/server/cli.js +++ b/server/cli.js @@ -8,6 +8,7 @@ * (no args) - Start the server (default) * start - Start the server * sandbox - Manage Docker sandbox environments + * browser-use-mcp - Run Browser Use MCP stdio server * status - Show configuration and data locations * help - Show help information * version - Show version information @@ -605,6 +606,10 @@ async function startServer() { await import('./index.js'); } +async function startBrowserUseMcp() { + await import('./browser-use-mcp.js'); +} + // Parse CLI arguments function parseArgs(args) { const parsed = { command: 'start', options: {} }; @@ -658,6 +663,9 @@ async function main() { case 'sandbox': await sandboxCommand(remainingArgs || []); break; + case 'browser-use-mcp': + await startBrowserUseMcp(); + break; case 'status': case 'info': showStatus(); diff --git a/server/index.js b/server/index.js index e0254720..eef1c45d 100755 --- a/server/index.js +++ b/server/index.js @@ -62,6 +62,7 @@ import geminiRoutes from './routes/gemini.js'; import pluginsRoutes from './routes/plugins.js'; import providerRoutes from './modules/providers/provider.routes.js'; import browserUseRoutes from './modules/browser-use/browser-use.routes.js'; +import browserUseMcpRoutes from './modules/browser-use/browser-use-mcp.routes.js'; import { browserUseService } from './modules/browser-use/browser-use.service.js'; import computerUseRoutes from './modules/computer-use/computer-use.routes.js'; import { startEnabledPluginServers, stopAllPlugins, getPluginPort } from './utils/plugin-process-manager.js'; @@ -196,6 +197,9 @@ app.use('/api/gemini', authenticateToken, geminiRoutes); // Plugins API Routes (protected) app.use('/api/plugins', authenticateToken, pluginsRoutes); +// Browser Use MCP bridge API (local token protected) +app.use('/api/browser-use-mcp', browserUseMcpRoutes); + // Browser Use API Routes (protected) app.use('/api/browser-use', authenticateToken, browserUseRoutes); diff --git a/server/modules/browser-use/browser-use-mcp.routes.ts b/server/modules/browser-use/browser-use-mcp.routes.ts new file mode 100644 index 00000000..335ffa18 --- /dev/null +++ b/server/modules/browser-use/browser-use-mcp.routes.ts @@ -0,0 +1,120 @@ +import express from 'express'; + +import { browserUseService } from '@/modules/browser-use/browser-use.service.js'; + +const router = express.Router(); + +function readBearerToken(header: unknown): string | null { + if (typeof header !== 'string') { + return null; + } + const match = /^Bearer\s+(.+)$/i.exec(header.trim()); + return match?.[1] || null; +} + +router.use((req, res, next) => { + const expected = browserUseService.getMcpToken(); + const token = readBearerToken(req.headers.authorization) || String(req.headers['x-browser-use-mcp-token'] || ''); + if (!token || token !== expected) { + res.status(401).json({ success: false, error: 'Invalid Browser Use MCP token.' }); + return; + } + next(); +}); + +router.post('/tools/:toolName', async (req, res) => { + try { + const input = (req.body && typeof req.body === 'object' ? req.body : {}) as Record; + const sessionId = typeof input.sessionId === 'string' ? input.sessionId : ''; + const toolName = req.params.toolName; + let result: unknown; + + switch (toolName) { + case 'browser_create_session': + result = await browserUseService.createAgentSession({ + profileName: typeof input.profileName === 'string' ? input.profileName : null, + }); + break; + case 'browser_list_sessions': + result = await browserUseService.listAgentSessions(); + break; + case 'browser_snapshot': + case 'browser_take_screenshot': + result = await browserUseService.agentSnapshot(sessionId); + break; + case 'browser_navigate': + result = await browserUseService.agentNavigate(sessionId, String(input.url || '')); + break; + case 'browser_click': + result = await browserUseService.agentClick(sessionId, { + selector: typeof input.selector === 'string' ? input.selector : undefined, + text: typeof input.text === 'string' ? input.text : undefined, + x: typeof input.x === 'number' ? input.x : undefined, + y: typeof input.y === 'number' ? input.y : undefined, + }); + break; + case 'browser_type': + result = await browserUseService.agentType(sessionId, { + selector: typeof input.selector === 'string' ? input.selector : undefined, + text: String(input.text || ''), + submit: input.submit === true, + }); + break; + case 'browser_fill_form': + result = await browserUseService.agentFillForm( + sessionId, + Array.isArray(input.fields) + ? input.fields.map((field) => { + const record = field as Record; + return { + selector: String(record.selector || ''), + value: String(record.value || ''), + }; + }) + : [], + ); + break; + case 'browser_press_key': + result = await browserUseService.agentPressKey(sessionId, String(input.key || '')); + break; + case 'browser_select_option': + result = await browserUseService.agentSelectOption( + sessionId, + String(input.selector || ''), + Array.isArray(input.values) ? input.values.filter((value): value is string => typeof value === 'string') : [], + ); + break; + case 'browser_wait_for': + result = await browserUseService.agentWaitFor(sessionId, { + text: typeof input.text === 'string' ? input.text : undefined, + url: typeof input.url === 'string' ? input.url : undefined, + timeoutMs: typeof input.timeoutMs === 'number' ? input.timeoutMs : undefined, + }); + break; + case 'browser_tabs': + result = await browserUseService.agentTabs(sessionId, { + action: input.action === 'new' || input.action === 'select' || input.action === 'close' || input.action === 'list' + ? input.action + : undefined, + index: typeof input.index === 'number' ? input.index : undefined, + url: typeof input.url === 'string' ? input.url : undefined, + }); + break; + case 'browser_close_session': + result = await browserUseService.agentStopSession(sessionId); + break; + default: + res.status(404).json({ success: false, error: `Unknown Browser Use MCP tool "${toolName}".` }); + return; + } + + res.json({ success: true, data: result }); + } catch (error) { + res.status(400).json({ + success: false, + error: error instanceof Error ? error.message : 'Browser Use MCP tool failed.', + }); + } +}); + +export default router; diff --git a/server/modules/browser-use/browser-use.routes.ts b/server/modules/browser-use/browser-use.routes.ts index f5dc563b..16f65d7e 100644 --- a/server/modules/browser-use/browser-use.routes.ts +++ b/server/modules/browser-use/browser-use.routes.ts @@ -22,8 +22,66 @@ function readParam(value: string | string[] | undefined): string { return Array.isArray(value) ? value[0] || '' : value || ''; } -router.get('/status', (_req, res) => { - res.json({ success: true, data: browserUseService.getStatus() }); +router.get('/status', async (_req, res) => { + try { + res.json({ success: true, data: await browserUseService.getStatus() }); + } catch (error) { + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to load Browser Use status.', + }); + } +}); + +router.get('/settings', async (_req, res) => { + try { + res.json({ success: true, data: { settings: await browserUseService.getSettings() } }); + } catch (error) { + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to load Browser Use settings.', + }); + } +}); + +router.put('/settings', async (req, res) => { + try { + const settings = await browserUseService.updateSettings(req.body || {}); + res.json({ success: true, data: { settings } }); + } catch (error) { + res.status(400).json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to save Browser Use settings.', + }); + } +}); + +router.post('/agent-tools/register', async (_req, res) => { + try { + const result = await browserUseService.registerAgentMcp(); + res.status(201).json({ success: true, data: result }); + } catch (error) { + res.status(400).json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to register Browser Use MCP.', + }); + } +}); + +router.post('/runtime/install', async (_req, res) => { + try { + const result = await browserUseService.installRuntime(); + res.status(result.success ? 200 : 500).json({ + success: result.success, + data: result, + error: result.success ? undefined : result.message, + }); + } catch (error) { + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to install Browser Use runtime.', + }); + } }); router.get('/sessions', async (req: AuthenticatedRequest, res) => { @@ -61,6 +119,57 @@ router.post('/sessions/:sessionId/navigate', async (req: AuthenticatedRequest, r } }); +router.post('/sessions/:sessionId/click', async (req: AuthenticatedRequest, res) => { + try { + const session = await browserUseService.userClick(requireUser(req), readParam(req.params.sessionId), { + x: Number(req.body?.x), + y: Number(req.body?.y), + }); + res.json({ success: true, data: { session } }); + } catch (error) { + res.status(400).json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to click browser session.', + }); + } +}); + +router.post('/sessions/:sessionId/press-key', async (req: AuthenticatedRequest, res) => { + try { + const session = await browserUseService.userPressKey(requireUser(req), readParam(req.params.sessionId), String(req.body?.key || '')); + res.json({ success: true, data: { session } }); + } catch (error) { + res.status(400).json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to send browser key input.', + }); + } +}); + +router.post('/sessions/:sessionId/agent-access/grant', async (req: AuthenticatedRequest, res) => { + try { + const session = await browserUseService.grantAgentAccess(requireUser(req), readParam(req.params.sessionId)); + res.json({ success: true, data: { session } }); + } catch (error) { + res.status(400).json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to grant agent access.', + }); + } +}); + +router.post('/sessions/:sessionId/agent-access/revoke', async (req: AuthenticatedRequest, res) => { + try { + const session = await browserUseService.revokeAgentAccess(requireUser(req), readParam(req.params.sessionId)); + res.json({ success: true, data: { session } }); + } catch (error) { + res.status(400).json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to revoke agent access.', + }); + } +}); + router.post('/sessions/:sessionId/stop', async (req: AuthenticatedRequest, res) => { try { const result = await browserUseService.stopSession(requireUser(req), readParam(req.params.sessionId)); @@ -73,4 +182,16 @@ router.post('/sessions/:sessionId/stop', async (req: AuthenticatedRequest, res) } }); +router.delete('/sessions/:sessionId', async (req: AuthenticatedRequest, res) => { + try { + const result = await browserUseService.deleteSession(requireUser(req), readParam(req.params.sessionId)); + res.json({ success: true, data: result }); + } catch (error) { + res.status(400).json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to delete browser session.', + }); + } +}); + export default router; diff --git a/server/modules/browser-use/browser-use.service.ts b/server/modules/browser-use/browser-use.service.ts index 4ca96695..06fe255b 100644 --- a/server/modules/browser-use/browser-use.service.ts +++ b/server/modules/browser-use/browser-use.service.ts @@ -1,13 +1,24 @@ import { createRequire } from 'node:module'; -import { randomUUID } from 'node:crypto'; +import { randomBytes, randomUUID } from 'node:crypto'; +import { spawn } from 'node:child_process'; import dns from 'node:dns/promises'; +import fs from 'node:fs'; +import os from 'node:os'; import net from 'node:net'; +import path from 'node:path'; + +import { appConfigDb } from '@/modules/database/repositories/app-config.js'; +import { providerMcpService } from '@/modules/providers/services/mcp.service.js'; +import { getModuleDir } from '@/utils/runtime-paths.js'; const require = createRequire(import.meta.url); +const __dirname = getModuleDir(import.meta.url); const IS_PLATFORM = process.env.VITE_IS_PLATFORM === 'true'; const MAX_SESSIONS_PER_OWNER = Number.parseInt(process.env.CLOUDCLI_BROWSER_USE_MAX_SESSIONS_PER_OWNER || '3', 10); const SESSION_TTL_MS = Number.parseInt(process.env.CLOUDCLI_BROWSER_USE_SESSION_TTL_MS || String(30 * 60 * 1000), 10); const ALLOW_PRIVATE_NETWORKS = process.env.CLOUDCLI_BROWSER_USE_ALLOW_PRIVATE_NETWORKS === '1'; +const BROWSER_USE_SETTINGS_KEY = 'browser_use_settings'; +const BROWSER_USE_MCP_TOKEN_KEY = 'browser_use_mcp_token'; type BrowserUseRuntime = 'cloud' | 'local'; type BrowserUseSessionStatus = 'ready' | 'stopped' | 'unavailable'; @@ -15,6 +26,7 @@ type BrowserUseSessionStatus = 'ready' | 'stopped' | 'unavailable'; type BrowserUseSession = { id: string; ownerId: string; + createdBy: 'user' | 'agent'; runtime: BrowserUseRuntime; status: BrowserUseSessionStatus; url: string | null; @@ -24,12 +36,24 @@ type BrowserUseSession = { updatedAt: string; lastAction: string | null; message: string | null; + agentAccessEnabled: boolean; + profileName: string | null; + viewport: { + width: number; + height: number; + } | null; + cursor: { + x: number; + y: number; + actor: 'agent' | 'user'; + } | null; }; type PublicBrowserUseSession = Omit; type RuntimeHandle = { browser?: any; + context?: any; page?: any; }; @@ -37,23 +61,90 @@ type BrowserUseOwner = { id: string | number; }; +type BrowserUseSettings = { + enabled: boolean; + agentToolsEnabled: boolean; +}; + +type RuntimeReadiness = { + playwright: any | null; + playwrightInstalled: boolean; + chromiumInstalled: boolean; + chromiumExecutablePath: string | null; + installInProgress: boolean; + installMessage: string | null; +}; + const sessions = new Map(); const handles = new Map(); +let installPromise: Promise<{ success: boolean; message: string }> | null = null; +let lastInstallMessage: string | null = null; + +const DEFAULT_SETTINGS: BrowserUseSettings = { + enabled: false, + agentToolsEnabled: false, +}; +const AGENT_OWNER_ID = 'agent'; +const PROFILE_ROOT = path.join(os.homedir(), '.cloudcli', 'browser-use', 'profiles'); +const MCP_SERVER_NAME = 'cloudcli-browser-use'; +const MCP_PROVIDERS = ['claude', 'codex', 'cursor', 'gemini', 'opencode']; function getRuntime(): BrowserUseRuntime { return IS_PLATFORM ? 'cloud' : 'local'; } -function isBrowserUseEnabled(): boolean { - return process.env.CLOUDCLI_BROWSER_USE_ENABLED === '1'; +function readSettings(): BrowserUseSettings { + try { + const raw = appConfigDb.get(BROWSER_USE_SETTINGS_KEY); + if (!raw) { + return DEFAULT_SETTINGS; + } + + const parsed = JSON.parse(raw) as Partial; + return { + enabled: parsed.enabled === true, + agentToolsEnabled: parsed.agentToolsEnabled === true, + }; + } catch (error: any) { + console.warn('[Browser Use] Failed to read settings:', error?.message || error); + return DEFAULT_SETTINGS; + } } -function getSetupMessage(): string { - if (!isBrowserUseEnabled()) { - return 'Browser Use is disabled. Set CLOUDCLI_BROWSER_USE_ENABLED=1 after provisioning a Playwright/Chromium runtime.'; +function writeSettings(settings: BrowserUseSettings): BrowserUseSettings { + const normalized = { + enabled: settings.enabled === true, + agentToolsEnabled: settings.agentToolsEnabled === true, + }; + + appConfigDb.set(BROWSER_USE_SETTINGS_KEY, JSON.stringify(normalized)); + return normalized; +} + +function getOrCreateMcpToken(): string { + const existing = appConfigDb.get(BROWSER_USE_MCP_TOKEN_KEY); + if (existing) { + return existing; + } + const token = randomBytes(32).toString('hex'); + appConfigDb.set(BROWSER_USE_MCP_TOKEN_KEY, token); + return token; +} + +function getSetupMessage(settings: BrowserUseSettings, readiness: RuntimeReadiness): string { + if (!settings.enabled) { + return 'Browser Use is disabled in settings.'; } - return 'Playwright is not available in this runtime. Install/provision Playwright or point CloudCLI at a managed browser worker.'; + if (!readiness.playwrightInstalled) { + return 'Install Playwright and Chromium to use browser sessions.'; + } + + if (!readiness.chromiumInstalled) { + return 'Playwright is installed, but Chromium is missing. Install the Chromium runtime to continue.'; + } + + return readiness.installMessage || 'Browser Use runtime is not ready.'; } function getPlaywright(): any | null { @@ -64,6 +155,137 @@ function getPlaywright(): any | null { } } +function getMcpCommand(): { command: string; args: string[] } { + const serverDir = path.resolve(__dirname, '..', '..'); + const mcpScriptPath = path.join(serverDir, 'browser-use-mcp.js'); + if (fs.existsSync(mcpScriptPath)) { + return { + command: process.execPath, + args: [mcpScriptPath], + }; + } + + return { + command: 'cloudcli', + args: ['browser-use-mcp'], + }; +} + +function getMcpApiUrl(): string { + const port = process.env.SERVER_PORT || process.env.PORT || '3001'; + return `http://127.0.0.1:${port}/api/browser-use-mcp`; +} + +function normalizeProfileName(profileName?: string | null): string | null { + const normalized = String(profileName || '').trim(); + if (!normalized) { + return null; + } + + return normalized.slice(0, 80); +} + +function getProfilePath(profileName: string): string { + const safeName = profileName + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 80) || 'default'; + return path.join(PROFILE_ROOT, safeName); +} + +function getRuntimeReadiness(): RuntimeReadiness { + const playwright = getPlaywright(); + const readiness: RuntimeReadiness = { + playwright, + playwrightInstalled: Boolean(playwright), + chromiumInstalled: false, + chromiumExecutablePath: null, + installInProgress: Boolean(installPromise), + installMessage: lastInstallMessage, + }; + + if (!playwright) { + return readiness; + } + + try { + const executablePath = playwright.chromium.executablePath(); + readiness.chromiumExecutablePath = executablePath; + readiness.chromiumInstalled = Boolean(executablePath && fs.existsSync(executablePath)); + } catch { + readiness.chromiumInstalled = false; + } + + return readiness; +} + +function runCommand(command: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: process.cwd(), + env: process.env, + shell: false, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const output: string[] = []; + + child.stdout.on('data', (chunk) => output.push(String(chunk))); + child.stderr.on('data', (chunk) => output.push(String(chunk))); + child.on('error', reject); + child.on('close', (code) => { + if (code === 0) { + resolve(); + return; + } + + reject(new Error(output.join('').trim() || `${command} ${args.join(' ')} exited with code ${code}`)); + }); + }); +} + +function formatInstallError(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + if (message.includes('sudo') && message.includes('password')) { + return 'Installing Chromium system dependencies requires administrator privileges. Run `npx playwright install-deps chromium` on the machine where CloudCLI runs, then try again.'; + } + return message || 'Failed to install Browser Use runtime.'; +} + +async function installRuntime(): Promise<{ success: boolean; message: string }> { + if (installPromise) { + return installPromise; + } + + const npmCommand = process.platform === 'win32' ? 'npm.cmd' : 'npm'; + installPromise = (async () => { + try { + lastInstallMessage = 'Installing Playwright package...'; + await runCommand(npmCommand, ['install', '--no-save', '--no-package-lock', 'playwright']); + + if (process.platform === 'linux') { + lastInstallMessage = 'Installing Chromium system dependencies...'; + await runCommand(npmCommand, ['exec', '--', 'playwright', 'install-deps', 'chromium']); + } + + lastInstallMessage = 'Installing Chromium runtime...'; + await runCommand(npmCommand, ['exec', '--', 'playwright', 'install', 'chromium']); + + lastInstallMessage = 'Browser Use runtime installed.'; + return { success: true, message: lastInstallMessage }; + } catch (error) { + lastInstallMessage = formatInstallError(error); + return { success: false, message: lastInstallMessage }; + } + })(); + + try { + return await installPromise; + } finally { + installPromise = null; + } +} + function getOwnerId(owner: BrowserUseOwner): string { if (owner.id === undefined || owner.id === null || String(owner.id).trim() === '') { throw new Error('Authenticated user is required.'); @@ -184,9 +406,14 @@ function ownerSessions(ownerId: string): BrowserUseSession[] { return [...sessions.values()].filter((session) => session.ownerId === ownerId); } +function canAccessSession(ownerId: string, session: BrowserUseSession): boolean { + return session.ownerId === ownerId || session.ownerId === AGENT_OWNER_ID || session.agentAccessEnabled; +} + async function closeHandle(sessionId: string): Promise { const handle = handles.get(sessionId); handles.delete(sessionId); + await handle?.context?.close?.().catch(() => undefined); await handle?.browser?.close().catch(() => undefined); } @@ -214,40 +441,150 @@ async function captureSession(session: BrowserUseSession, page: any): Promise null); session.url = page.url() || session.url; + session.viewport = page.viewportSize?.() || session.viewport; session.updatedAt = new Date().toISOString(); } +async function getActionPoint(page: any, input: { selector?: string; text?: string; x?: number; y?: number }) { + if (typeof input.x === 'number' && typeof input.y === 'number') { + return { x: input.x, y: input.y }; + } + + const locator = input.selector + ? page.locator(input.selector).first() + : input.text + ? page.getByText(input.text, { exact: false }).first() + : null; + + if (!locator) { + return null; + } + + const box = await locator.boundingBox().catch(() => null); + if (!box) { + return null; + } + + return { + x: Math.round(box.x + box.width / 2), + y: Math.round(box.y + box.height / 2), + }; +} + export const browserUseService = { - getStatus() { - const playwright = getPlaywright(); - const enabled = isBrowserUseEnabled() && Boolean(playwright); + async getSettings() { + return readSettings(); + }, + + async updateSettings(settings: Partial) { + const current = readSettings(); + const nextSettings = { + ...current, + enabled: typeof settings.enabled === 'boolean' ? settings.enabled : current.enabled, + agentToolsEnabled: typeof settings.agentToolsEnabled === 'boolean' + ? settings.agentToolsEnabled + : current.agentToolsEnabled, + }; + if (!nextSettings.enabled) { + nextSettings.agentToolsEnabled = false; + } + + const next = writeSettings(nextSettings); + if (next.agentToolsEnabled) { + await this.registerAgentMcp(); + } else if (current.agentToolsEnabled) { + await this.unregisterAgentMcp(); + } + return next; + }, + + async getStatus() { + const settings = readSettings(); + const readiness = getRuntimeReadiness(); + const available = settings.enabled && readiness.playwrightInstalled && readiness.chromiumInstalled; return { - enabled, + enabled: settings.enabled, runtime: getRuntime(), - available: enabled, + available, + playwrightInstalled: readiness.playwrightInstalled, + chromiumInstalled: readiness.chromiumInstalled, + installInProgress: readiness.installInProgress, sessionCount: sessions.size, - mcpRecommended: true, - message: enabled + agentToolsEnabled: settings.agentToolsEnabled, + mcpRecommended: !settings.agentToolsEnabled, + message: available ? 'Browser Use runtime is available.' - : getSetupMessage(), + : getSetupMessage(settings, readiness), + }; + }, + + async registerAgentMcp() { + const { command, args } = getMcpCommand(); + const results = await providerMcpService.addMcpServerToAllProviders({ + name: MCP_SERVER_NAME, + scope: 'user', + transport: 'stdio', + command, + args, + env: { + CLOUDCLI_BROWSER_USE_MCP_TOKEN: getOrCreateMcpToken(), + CLOUDCLI_BROWSER_USE_API_URL: getMcpApiUrl(), + }, + }); + return { name: MCP_SERVER_NAME, command, args, results }; + }, + + getMcpToken() { + return getOrCreateMcpToken(); + }, + + async unregisterAgentMcp() { + const results = await Promise.all(MCP_PROVIDERS.map(async (provider) => { + try { + const result = await providerMcpService.removeProviderMcpServer(provider, { + name: MCP_SERVER_NAME, + scope: 'user', + }); + return { provider, removed: result.removed }; + } catch (error) { + return { + provider, + removed: false, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } + })); + return { name: MCP_SERVER_NAME, results }; + }, + + async installRuntime() { + const result = await installRuntime(); + return { + ...result, + status: await this.getStatus(), }; }, async listSessions(owner: BrowserUseOwner) { const ownerId = getOwnerId(owner); await expireStaleSessions(); - return ownerSessions(ownerId).map(publicSession); + return [...sessions.values()] + .filter((session) => canAccessSession(ownerId, session)) + .map(publicSession); }, - async createSession(owner: BrowserUseOwner) { + async createSession(owner: BrowserUseOwner, options?: { createdBy?: 'user' | 'agent'; profileName?: string | null; agentAccessEnabled?: boolean }) { const ownerId = getOwnerId(owner); await expireStaleSessions(); + const createdBy = options?.createdBy ?? 'user'; + const profileName = normalizeProfileName(options?.profileName); const now = new Date().toISOString(); const session: BrowserUseSession = { id: randomUUID(), ownerId, + createdBy, runtime: getRuntime(), status: 'unavailable', url: null, @@ -257,6 +594,10 @@ export const browserUseService = { updatedAt: now, lastAction: 'create', message: null, + agentAccessEnabled: options?.agentAccessEnabled ?? createdBy === 'agent', + profileName, + viewport: { width: 1440, height: 900 }, + cursor: null, }; const activeOwnerSessions = ownerSessions(ownerId).filter((item) => item.status === 'ready'); @@ -264,33 +605,111 @@ export const browserUseService = { throw new Error(`Browser Use is limited to ${MAX_SESSIONS_PER_OWNER} active sessions per user.`); } - const playwright = getPlaywright(); - if (!isBrowserUseEnabled() || !playwright) { - session.message = getSetupMessage(); + const settings = readSettings(); + const readiness = getRuntimeReadiness(); + if (!settings.enabled || !readiness.playwrightInstalled || !readiness.chromiumInstalled || !readiness.playwright) { + session.message = getSetupMessage(settings, readiness); sessions.set(session.id, session); return publicSession(session); } - const browser = await playwright.chromium.launch({ + let browser: any | undefined; + let context: any | undefined; + let page: any; + const launchOptions = { headless: true, args: ['--disable-dev-shm-usage'], - }); - const page = await browser.newPage({ viewport: { width: 1440, height: 900 } }); + }; + const contextOptions = { + viewport: { width: 1440, height: 900 }, + serviceWorkers: 'block', + }; + + if (profileName) { + fs.mkdirSync(PROFILE_ROOT, { recursive: true }); + context = await readiness.playwright.chromium.launchPersistentContext(getProfilePath(profileName), { + ...launchOptions, + ...contextOptions, + }); + page = context.pages()[0] || await context.newPage(); + } else { + browser = await readiness.playwright.chromium.launch(launchOptions); + context = await browser.newContext(contextOptions); + page = await context.newPage(); + } await attachRequestGuard(page); session.status = 'ready'; session.message = 'Browser session is ready.'; sessions.set(session.id, session); - handles.set(session.id, { browser, page }); + handles.set(session.id, { browser, context, page }); await captureSession(session, page); return publicSession(session); }, + async grantAgentAccess(owner: BrowserUseOwner, sessionId: string) { + const ownerId = getOwnerId(owner); + const session = sessions.get(sessionId); + if (!session || (session.ownerId !== ownerId && session.ownerId !== AGENT_OWNER_ID)) { + throw new Error('Browser session not found.'); + } + session.agentAccessEnabled = true; + session.updatedAt = new Date().toISOString(); + session.lastAction = 'agent_access:grant'; + return publicSession(session); + }, + + async revokeAgentAccess(owner: BrowserUseOwner, sessionId: string) { + const ownerId = getOwnerId(owner); + const session = sessions.get(sessionId); + if (!session || (session.ownerId !== ownerId && session.ownerId !== AGENT_OWNER_ID)) { + throw new Error('Browser session not found.'); + } + session.agentAccessEnabled = false; + session.updatedAt = new Date().toISOString(); + session.lastAction = 'agent_access:revoke'; + return publicSession(session); + }, + + async listAgentSessions() { + const settings = readSettings(); + if (!settings.enabled || !settings.agentToolsEnabled) { + return []; + } + await expireStaleSessions(); + return [...sessions.values()] + .filter((session) => session.agentAccessEnabled || session.ownerId === AGENT_OWNER_ID) + .map(publicSession); + }, + + async createAgentSession(options?: { profileName?: string | null }) { + const settings = readSettings(); + if (!settings.enabled || !settings.agentToolsEnabled) { + throw new Error('Browser Use agent tools are disabled.'); + } + return this.createSession( + { id: AGENT_OWNER_ID }, + { createdBy: 'agent', profileName: options?.profileName, agentAccessEnabled: true }, + ); + }, + + async getAgentSession(sessionId: string) { + const settings = readSettings(); + if (!settings.enabled || !settings.agentToolsEnabled) { + throw new Error('Browser Use agent tools are disabled.'); + } + const session = sessions.get(sessionId); + if (!session || (!session.agentAccessEnabled && session.ownerId !== AGENT_OWNER_ID)) { + throw new Error('Browser session is not shared with agents.'); + } + return session; + }, + async navigate(owner: BrowserUseOwner, sessionId: string, rawUrl: string) { const ownerId = getOwnerId(owner); await expireStaleSessions(); const session = sessions.get(sessionId); - if (!session || session.ownerId !== ownerId) { + if (!session || !canAccessSession(ownerId, session)) { throw new Error('Browser session not found.'); } @@ -306,14 +725,202 @@ export const browserUseService = { const url = await normalizeUrl(rawUrl); await handle.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }); session.lastAction = `navigate:${url}`; + session.cursor = null; await captureSession(session, handle.page); return publicSession(session); }, + async agentNavigate(sessionId: string, rawUrl: string) { + await this.getAgentSession(sessionId); + return this.navigate({ id: AGENT_OWNER_ID }, sessionId, rawUrl).catch(async (error) => { + const session = await this.getAgentSession(sessionId); + if (session.ownerId !== AGENT_OWNER_ID) { + const url = await normalizeUrl(rawUrl); + const handle = handles.get(sessionId); + if (!handle?.page) { + throw new Error('Browser runtime handle is not available.'); + } + await handle.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }); + session.lastAction = `navigate:${url}`; + await captureSession(session, handle.page); + return publicSession(session); + } + throw error; + }); + }, + + async agentSnapshot(sessionId: string) { + const session = await this.getAgentSession(sessionId); + const handle = handles.get(sessionId); + if (!handle?.page) { + throw new Error('Browser runtime handle is not available.'); + } + await captureSession(session, handle.page); + const text = await handle.page.locator('body').innerText({ timeout: 5_000 }).catch(() => ''); + return { + session: publicSession(session), + text: text.slice(0, 30_000), + }; + }, + + async agentClick(sessionId: string, input: { selector?: string; text?: string; x?: number; y?: number }) { + const session = await this.getAgentSession(sessionId); + const handle = handles.get(sessionId); + if (!handle?.page) { + throw new Error('Browser runtime handle is not available.'); + } + const point = await getActionPoint(handle.page, input); + + if (input.selector) { + await handle.page.locator(input.selector).first().click({ timeout: 10_000 }); + } else if (input.text) { + await handle.page.getByText(input.text, { exact: false }).first().click({ timeout: 10_000 }); + } else if (typeof input.x === 'number' && typeof input.y === 'number') { + await handle.page.mouse.click(input.x, input.y); + } else { + throw new Error('Provide selector, text, or x/y coordinates.'); + } + + session.lastAction = 'click'; + session.cursor = point ? { ...point, actor: 'agent' } : null; + await captureSession(session, handle.page); + return publicSession(session); + }, + + async agentType(sessionId: string, input: { selector?: string; text: string; submit?: boolean }) { + const session = await this.getAgentSession(sessionId); + const handle = handles.get(sessionId); + if (!handle?.page) { + throw new Error('Browser runtime handle is not available.'); + } + + if (input.selector) { + await handle.page.locator(input.selector).first().fill(input.text, { timeout: 10_000 }); + session.cursor = await getActionPoint(handle.page, input).then((point) => ( + point ? { ...point, actor: 'agent' as const } : null + )); + } else { + await handle.page.keyboard.type(input.text); + } + if (input.submit) { + await handle.page.keyboard.press('Enter'); + } + + session.lastAction = 'type'; + await captureSession(session, handle.page); + return publicSession(session); + }, + + async agentFillForm(sessionId: string, fields: Array<{ selector: string; value: string }>) { + const session = await this.getAgentSession(sessionId); + const handle = handles.get(sessionId); + if (!handle?.page) { + throw new Error('Browser runtime handle is not available.'); + } + for (const field of fields) { + await handle.page.locator(field.selector).first().fill(field.value, { timeout: 10_000 }); + } + session.lastAction = 'fill_form'; + if (fields[0]) { + session.cursor = await getActionPoint(handle.page, { selector: fields[0].selector }).then((point) => ( + point ? { ...point, actor: 'agent' as const } : null + )); + } + await captureSession(session, handle.page); + return publicSession(session); + }, + + async agentPressKey(sessionId: string, key: string) { + const session = await this.getAgentSession(sessionId); + const handle = handles.get(sessionId); + if (!handle?.page) { + throw new Error('Browser runtime handle is not available.'); + } + await handle.page.keyboard.press(key); + session.lastAction = `press_key:${key}`; + await captureSession(session, handle.page); + return publicSession(session); + }, + + async agentSelectOption(sessionId: string, selector: string, values: string[]) { + const session = await this.getAgentSession(sessionId); + const handle = handles.get(sessionId); + if (!handle?.page) { + throw new Error('Browser runtime handle is not available.'); + } + await handle.page.locator(selector).first().selectOption(values, { timeout: 10_000 }); + session.lastAction = 'select_option'; + session.cursor = await getActionPoint(handle.page, { selector }).then((point) => ( + point ? { ...point, actor: 'agent' as const } : null + )); + await captureSession(session, handle.page); + return publicSession(session); + }, + + async agentWaitFor(sessionId: string, input: { text?: string; url?: string; timeoutMs?: number }) { + const session = await this.getAgentSession(sessionId); + const handle = handles.get(sessionId); + if (!handle?.page) { + throw new Error('Browser runtime handle is not available.'); + } + const timeout = Math.max(250, Math.min(input.timeoutMs || 5_000, 30_000)); + if (input.text) { + await handle.page.getByText(input.text, { exact: false }).first().waitFor({ timeout }); + } else if (input.url) { + await handle.page.waitForURL(input.url, { timeout }); + } else { + await handle.page.waitForTimeout(timeout); + } + session.lastAction = 'wait_for'; + await captureSession(session, handle.page); + return publicSession(session); + }, + + async agentTabs(sessionId: string, input: { action?: 'list' | 'new' | 'select' | 'close'; index?: number; url?: string }) { + const session = await this.getAgentSession(sessionId); + const handle = handles.get(sessionId); + if (!handle?.context || !handle?.page) { + throw new Error('Browser runtime handle is not available.'); + } + const action = input.action || 'list'; + if (action === 'new') { + const page = await handle.context.newPage(); + handles.set(sessionId, { ...handle, page }); + await attachRequestGuard(page); + if (input.url) { + await this.agentNavigate(sessionId, input.url); + } + } else if (action === 'select') { + const page = handle.context.pages()[input.index || 0]; + if (!page) { + throw new Error('Tab not found.'); + } + handles.set(sessionId, { ...handle, page }); + } else if (action === 'close') { + const pages = handle.context.pages(); + const page = pages[input.index ?? pages.indexOf(handle.page)]; + if (!page) { + throw new Error('Tab not found.'); + } + await page.close(); + handles.set(sessionId, { ...handle, page: handle.context.pages()[0] || await handle.context.newPage() }); + } + const updatedHandle = handles.get(sessionId); + await captureSession(session, updatedHandle?.page || handle.page); + return { + session: publicSession(session), + tabs: handle.context.pages().map((page: any, index: number) => ({ + index, + url: page.url(), + active: page === (updatedHandle?.page || handle.page), + })), + }; + }, + async stopSession(owner: BrowserUseOwner, sessionId: string) { const ownerId = getOwnerId(owner); const session = sessions.get(sessionId); - if (!session || session.ownerId !== ownerId) { + if (!session || !canAccessSession(ownerId, session)) { return { stopped: false }; } @@ -322,10 +929,70 @@ export const browserUseService = { session.status = 'stopped'; session.updatedAt = new Date().toISOString(); session.lastAction = 'stop'; - session.message = 'Browser session stopped.'; + session.message = 'Browser session stopped. Create a new session to continue browsing.'; return { stopped: true, session: publicSession(session) }; }, + async deleteSession(owner: BrowserUseOwner, sessionId: string) { + const ownerId = getOwnerId(owner); + const session = sessions.get(sessionId); + if (!session || !canAccessSession(ownerId, session)) { + return { deleted: false }; + } + + await closeHandle(sessionId); + sessions.delete(sessionId); + return { deleted: true, sessionId }; + }, + + async userClick(owner: BrowserUseOwner, sessionId: string, input: { x: number; y: number }) { + const ownerId = getOwnerId(owner); + const session = sessions.get(sessionId); + if (!session || !canAccessSession(ownerId, session)) { + throw new Error('Browser session not found.'); + } + if (session.status !== 'ready') { + throw new Error(session.message || 'Browser session is not available.'); + } + + const handle = handles.get(sessionId); + if (!handle?.page) { + throw new Error('Browser runtime handle is not available.'); + } + + await handle.page.mouse.click(input.x, input.y); + session.lastAction = 'click'; + session.cursor = { x: input.x, y: input.y, actor: 'user' }; + await captureSession(session, handle.page); + return publicSession(session); + }, + + async userPressKey(owner: BrowserUseOwner, sessionId: string, key: string) { + const ownerId = getOwnerId(owner); + const session = sessions.get(sessionId); + if (!session || !canAccessSession(ownerId, session)) { + throw new Error('Browser session not found.'); + } + if (session.status !== 'ready') { + throw new Error(session.message || 'Browser session is not available.'); + } + + const handle = handles.get(sessionId); + if (!handle?.page) { + throw new Error('Browser runtime handle is not available.'); + } + + await handle.page.keyboard.press(key); + session.lastAction = `press_key:${key}`; + await captureSession(session, handle.page); + return publicSession(session); + }, + + async agentStopSession(sessionId: string) { + await this.getAgentSession(sessionId); + return this.stopSession({ id: AGENT_OWNER_ID }, sessionId); + }, + async stopAllSessions() { await Promise.all([...sessions.keys()].map(async (sessionId) => { await closeHandle(sessionId); diff --git a/server/modules/browser-use/tests/browser-use.service.test.ts b/server/modules/browser-use/tests/browser-use.service.test.ts index 3a291682..162e9439 100644 --- a/server/modules/browser-use/tests/browser-use.service.test.ts +++ b/server/modules/browser-use/tests/browser-use.service.test.ts @@ -15,27 +15,16 @@ test('browser use blocks private and local network addresses by default', () => }); test('browser use sessions are listed only for their owner', async () => { - const originalEnabled = process.env.CLOUDCLI_BROWSER_USE_ENABLED; - process.env.CLOUDCLI_BROWSER_USE_ENABLED = '0'; - const ownerA = { id: `owner-a-${Date.now()}-${Math.random()}` }; const ownerB = { id: `owner-b-${Date.now()}-${Math.random()}` }; - try { - const ownerASession = await browserUseService.createSession(ownerA); - await browserUseService.createSession(ownerB); + const ownerASession = await browserUseService.createSession(ownerA); + await browserUseService.createSession(ownerB); - const ownerASessions = await browserUseService.listSessions(ownerA); - const ownerBSessions = await browserUseService.listSessions(ownerB); + const ownerASessions = await browserUseService.listSessions(ownerA); + const ownerBSessions = await browserUseService.listSessions(ownerB); - assert.equal(ownerASessions.some((session) => session.id === ownerASession.id), true); - assert.equal(ownerBSessions.some((session) => session.id === ownerASession.id), false); - assert.equal(Object.hasOwn(ownerASession, 'ownerId'), false); - } finally { - if (originalEnabled === undefined) { - delete process.env.CLOUDCLI_BROWSER_USE_ENABLED; - } else { - process.env.CLOUDCLI_BROWSER_USE_ENABLED = originalEnabled; - } - } + assert.equal(ownerASessions.some((session) => session.id === ownerASession.id), true); + assert.equal(ownerBSessions.some((session) => session.id === ownerASession.id), false); + assert.equal(Object.hasOwn(ownerASession, 'ownerId'), false); }); diff --git a/src/components/browser-use/view/BrowserUsePanel.tsx b/src/components/browser-use/view/BrowserUsePanel.tsx index d2494a2a..e0e6311c 100644 --- a/src/components/browser-use/view/BrowserUsePanel.tsx +++ b/src/components/browser-use/view/BrowserUsePanel.tsx @@ -1,5 +1,5 @@ -import { useCallback, useEffect, useMemo, useState } from 'react'; -import { ExternalLink, Globe, MonitorPlay, Navigation, Pause, RefreshCw, Square } from 'lucide-react'; +import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent, type MouseEvent } from 'react'; +import { Bot, Download, Expand, ExternalLink, Globe, Loader2, MonitorPlay, Navigation, RefreshCw, Share2, Square, Trash2, X } from 'lucide-react'; import { Badge, Button } from '../../../shared/view/ui'; import { authenticatedFetch } from '../../../utils/api'; @@ -7,15 +7,17 @@ import { authenticatedFetch } from '../../../utils/api'; type BrowserUseStatus = { enabled: boolean; available: boolean; - runtime: 'cloud' | 'local'; + playwrightInstalled: boolean; + chromiumInstalled: boolean; + installInProgress: boolean; sessionCount: number; + agentToolsEnabled: boolean; mcpRecommended: boolean; message: string; }; type BrowserUseSession = { id: string; - runtime: 'cloud' | 'local'; status: 'ready' | 'stopped' | 'unavailable'; url: string | null; title: string | null; @@ -24,6 +26,18 @@ type BrowserUseSession = { updatedAt: string; lastAction: string | null; message: string | null; + agentAccessEnabled: boolean; + createdBy: 'user' | 'agent'; + profileName: string | null; + viewport: { + width: number; + height: number; + } | null; + cursor: { + x: number; + y: number; + actor: 'agent' | 'user'; + } | null; }; type BrowserUsePanelProps = { @@ -44,7 +58,10 @@ export default function BrowserUsePanel({ isVisible }: BrowserUsePanelProps) { const [selectedSessionId, setSelectedSessionId] = useState(null); const [targetUrl, setTargetUrl] = useState('https://example.com'); const [isBusy, setIsBusy] = useState(false); + const [isInstalling, setIsInstalling] = useState(false); + const [isFullscreen, setIsFullscreen] = useState(false); const [error, setError] = useState(null); + const viewerRef = useRef(null); const selectedSession = useMemo( () => sessions.find((session) => session.id === selectedSessionId) || sessions[0] || null, @@ -72,6 +89,11 @@ export default function BrowserUsePanel({ isVisible }: BrowserUsePanelProps) { void refresh().catch((err) => setError(err instanceof Error ? err.message : 'Failed to load Browser Use')); }, [isVisible, refresh]); + useEffect(() => { + if (!selectedSession?.url) return; + setTargetUrl(selectedSession.url); + }, [selectedSession?.id, selectedSession?.url]); + const runAction = useCallback(async (action: () => Promise) => { setIsBusy(true); setError(null); @@ -108,6 +130,129 @@ export default function BrowserUsePanel({ isVisible }: BrowserUsePanelProps) { await readJson(response); }); + const deleteSession = () => runAction(async () => { + if (!selectedSession) return; + const response = await authenticatedFetch(`/api/browser-use/sessions/${selectedSession.id}`, { method: 'DELETE' }); + await readJson(response); + setIsFullscreen(false); + }); + + const grantAgentAccess = () => runAction(async () => { + if (!selectedSession) return; + const response = await authenticatedFetch(`/api/browser-use/sessions/${selectedSession.id}/agent-access/grant`, { method: 'POST' }); + await readJson(response); + }); + + const revokeAgentAccess = () => runAction(async () => { + if (!selectedSession) return; + const response = await authenticatedFetch(`/api/browser-use/sessions/${selectedSession.id}/agent-access/revoke`, { method: 'POST' }); + await readJson(response); + }); + + const installBrowserBinaries = () => runAction(async () => { + setIsInstalling(true); + try { + const response = await authenticatedFetch('/api/browser-use/runtime/install', { method: 'POST' }); + await readJson(response); + } finally { + setIsInstalling(false); + } + }); + + const clickViewer = useCallback((event: MouseEvent) => { + if (!selectedSession || selectedSession.status !== 'ready' || !selectedSession.viewport) { + return; + } + viewerRef.current?.focus(); + + const bounds = event.currentTarget.getBoundingClientRect(); + const scaleX = selectedSession.viewport.width / bounds.width; + const scaleY = selectedSession.viewport.height / bounds.height; + const x = Math.round((event.clientX - bounds.left) * scaleX); + const y = Math.round((event.clientY - bounds.top) * scaleY); + + void runAction(async () => { + const response = await authenticatedFetch(`/api/browser-use/sessions/${selectedSession.id}/click`, { + method: 'POST', + body: JSON.stringify({ x, y }), + }); + await readJson(response); + }); + }, [runAction, selectedSession]); + + const keyForEvent = useCallback((event: KeyboardEvent) => { + if (event.key === ' ') return 'Space'; + return event.key; + }, []); + + const pressViewerKey = useCallback((event: KeyboardEvent) => { + if (!selectedSession || selectedSession.status !== 'ready') { + return; + } + + const ignoredKeys = new Set(['Shift', 'Control', 'Alt', 'Meta', 'CapsLock']); + if (ignoredKeys.has(event.key)) { + return; + } + + event.preventDefault(); + const key = keyForEvent(event); + void runAction(async () => { + const response = await authenticatedFetch(`/api/browser-use/sessions/${selectedSession.id}/press-key`, { + method: 'POST', + body: JSON.stringify({ key }), + }); + await readJson(response); + }); + }, [keyForEvent, runAction, selectedSession]); + + const needsBrowserBinaries = Boolean(status?.enabled && (!status.playwrightInstalled || !status.chromiumInstalled)); + + const cursorStyle = selectedSession?.cursor && selectedSession.viewport + ? { + left: `${(selectedSession.cursor.x / selectedSession.viewport.width) * 100}%`, + top: `${(selectedSession.cursor.y / selectedSession.viewport.height) * 100}%`, + } + : null; + + const renderBrowserSurface = (fullscreen = false) => ( +
+ {selectedSession?.screenshotDataUrl ? ( +
+ Browser session screenshot + {cursorStyle && ( +
+
+
+ )} +
+ ) : ( +
+ +
+ {selectedSession?.message || 'Create a browser session to start.'} +
+

+ Install browser binaries from this panel or enable Browser Use from Settings. +

+
+ )} +
+ ); + return (
@@ -115,22 +260,26 @@ export default function BrowserUsePanel({ isVisible }: BrowserUsePanelProps) {

Browser Use

- {status && ( - - {status.runtime} - - )}

- Managed Playwright browser sessions with owner-scoped screenshots and navigation. + Create browser sessions, watch agent activity, and decide which sessions agents may control.

+ + Guide + + - @@ -139,13 +288,43 @@ export default function BrowserUsePanel({ isVisible }: BrowserUsePanelProps) {
{error && ( @@ -204,30 +405,32 @@ export default function BrowserUsePanel({ isVisible }: BrowserUsePanelProps) {
{selectedSession?.url || 'No page loaded'} -
-
- {selectedSession?.screenshotDataUrl ? ( - Browser session screenshot - ) : ( -
- -
- {selectedSession?.message || 'Create a browser session to start.'} -
-

- This panel shows captured browser screenshots. Interactive agent control should use the guarded Browser Use API. -

-
+ {selectedSession?.agentAccessEnabled && ( + + + Agent access active + )}
+ {renderBrowserSurface()}
+ {isFullscreen && selectedSession && ( +
+
+
+
{selectedSession.title || selectedSession.url || 'Browser session'}
+ +
+ {renderBrowserSurface(true)} +
+
+ )} ); } diff --git a/src/components/main-content/types/types.ts b/src/components/main-content/types/types.ts index a7398795..6ae16ec8 100644 --- a/src/components/main-content/types/types.ts +++ b/src/components/main-content/types/types.ts @@ -64,6 +64,7 @@ export type MainContentHeaderProps = { selectedProject: Project; selectedSession: ProjectSession | null; shouldShowTasksTab: boolean; + shouldShowBrowserTab: boolean; isMobile: boolean; onMenuClick: () => void; }; diff --git a/src/components/main-content/view/MainContent.tsx b/src/components/main-content/view/MainContent.tsx index 5c14cd6d..b8930e30 100644 --- a/src/components/main-content/view/MainContent.tsx +++ b/src/components/main-content/view/MainContent.tsx @@ -1,4 +1,4 @@ -import React, { useEffect } from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; import ChatInterface from '../../chat/view/ChatInterface'; import FileTree from '../../file-tree/view/FileTree'; @@ -12,6 +12,7 @@ import { useTaskMaster } from '../../../contexts/TaskMasterContext'; import { usePaletteOpsRegister } from '../../../contexts/PaletteOpsContext'; import { useTasksSettings } from '../../../contexts/TasksSettingsContext'; import { useUiPreferences } from '../../../hooks/useUiPreferences'; +import { authenticatedFetch } from '../../../utils/api'; import { useEditorSidebar } from '../../code-editor/hooks/useEditorSidebar'; import EditorSidebar from '../../code-editor/view/EditorSidebar'; import type { Project } from '../../../types/app'; @@ -57,8 +58,10 @@ function MainContent({ const { currentProject, setCurrentProject } = useTaskMaster() as TaskMasterContextValue; const { tasksEnabled, isTaskMasterInstalled } = useTasksSettings() as TasksSettingsContextValue; + const [browserUseEnabled, setBrowserUseEnabled] = useState(false); const shouldShowTasksTab = Boolean(tasksEnabled && isTaskMasterInstalled); + const shouldShowBrowserTab = browserUseEnabled; const { editingFile, @@ -92,6 +95,28 @@ function MainContent({ } }, [shouldShowTasksTab, activeTab, setActiveTab]); + const loadBrowserUseSettings = useCallback(async () => { + try { + const response = await authenticatedFetch('/api/browser-use/settings'); + const data = await response.json(); + setBrowserUseEnabled(Boolean(response.ok && data?.success !== false && data?.data?.settings?.enabled)); + } catch { + setBrowserUseEnabled(false); + } + }, []); + + useEffect(() => { + void loadBrowserUseSettings(); + window.addEventListener('browserUseSettingsChanged', loadBrowserUseSettings); + return () => window.removeEventListener('browserUseSettingsChanged', loadBrowserUseSettings); + }, [loadBrowserUseSettings]); + + useEffect(() => { + if (!shouldShowBrowserTab && activeTab === 'browser') { + setActiveTab('chat'); + } + }, [shouldShowBrowserTab, activeTab, setActiveTab]); + usePaletteOpsRegister({ openFile: (filePath: string) => { setActiveTab('files'); @@ -115,6 +140,7 @@ function MainContent({ selectedProject={selectedProject} selectedSession={selectedSession} shouldShowTasksTab={shouldShowTasksTab} + shouldShowBrowserTab={shouldShowBrowserTab} isMobile={isMobile} onMenuClick={onMenuClick} /> @@ -173,7 +199,7 @@ function MainContent({ {shouldShowTasksTab && } - {activeTab === 'browser' && ( + {shouldShowBrowserTab && activeTab === 'browser' && (
diff --git a/src/components/main-content/view/subcomponents/MainContentHeader.tsx b/src/components/main-content/view/subcomponents/MainContentHeader.tsx index a9025c2b..f75013ce 100644 --- a/src/components/main-content/view/subcomponents/MainContentHeader.tsx +++ b/src/components/main-content/view/subcomponents/MainContentHeader.tsx @@ -10,6 +10,7 @@ export default function MainContentHeader({ selectedProject, selectedSession, shouldShowTasksTab, + shouldShowBrowserTab, isMobile, onMenuClick, }: MainContentHeaderProps) { @@ -59,6 +60,7 @@ export default function MainContentHeader({ activeTab={activeTab} setActiveTab={setActiveTab} shouldShowTasksTab={shouldShowTasksTab} + shouldShowBrowserTab={shouldShowBrowserTab} /> {canScrollRight && ( diff --git a/src/components/main-content/view/subcomponents/MainContentTabSwitcher.tsx b/src/components/main-content/view/subcomponents/MainContentTabSwitcher.tsx index 37b79d41..d8d23940 100644 --- a/src/components/main-content/view/subcomponents/MainContentTabSwitcher.tsx +++ b/src/components/main-content/view/subcomponents/MainContentTabSwitcher.tsx @@ -11,6 +11,7 @@ type MainContentTabSwitcherProps = { activeTab: AppTab; setActiveTab: Dispatch>; shouldShowTasksTab: boolean; + shouldShowBrowserTab: boolean; }; type BuiltInTab = { @@ -35,10 +36,16 @@ const BASE_TABS: BuiltInTab[] = [ { kind: 'builtin', id: 'shell', labelKey: 'tabs.shell', icon: Terminal }, { kind: 'builtin', id: 'files', labelKey: 'tabs.files', icon: Folder }, { kind: 'builtin', id: 'git', labelKey: 'tabs.git', icon: GitBranch }, - { kind: 'builtin', id: 'browser', labelKey: 'tabs.browser', icon: MonitorPlay }, { kind: 'builtin', id: 'computer', labelKey: 'tabs.computer', icon: MonitorCog }, ]; +const BROWSER_TAB: BuiltInTab = { + kind: 'builtin', + id: 'browser', + labelKey: 'tabs.browser', + icon: MonitorPlay, +}; + const TASKS_TAB: BuiltInTab = { kind: 'builtin', id: 'tasks', @@ -50,11 +57,16 @@ export default function MainContentTabSwitcher({ activeTab, setActiveTab, shouldShowTasksTab, + shouldShowBrowserTab, }: MainContentTabSwitcherProps) { const { t } = useTranslation(); const { plugins } = usePlugins(); - const builtInTabs: BuiltInTab[] = shouldShowTasksTab ? [...BASE_TABS, TASKS_TAB] : BASE_TABS; + const builtInTabs: BuiltInTab[] = [ + ...BASE_TABS, + ...(shouldShowBrowserTab ? [BROWSER_TAB] : []), + ...(shouldShowTasksTab ? [TASKS_TAB] : []), + ]; const pluginTabs: PluginTab[] = plugins .filter((p) => p.enabled) diff --git a/src/components/settings/constants/constants.ts b/src/components/settings/constants/constants.ts index 7d4c353d..37fa9df3 100644 --- a/src/components/settings/constants/constants.ts +++ b/src/components/settings/constants/constants.ts @@ -6,6 +6,7 @@ import { Info, KeyRound, ListChecks, + MonitorPlay, Palette, Plug, } from 'lucide-react'; @@ -32,6 +33,7 @@ export const SETTINGS_MAIN_TABS: SettingsMainTabMeta[] = [ { id: 'git', label: 'Git', keywords: 'git github commits', icon: GitBranch }, { id: 'api', label: 'API Tokens', keywords: 'api tokens auth keys', icon: KeyRound }, { id: 'tasks', label: 'Tasks', keywords: 'tasks taskmaster', icon: ListChecks }, + { id: 'browser', label: 'Browser Use', keywords: 'browser use playwright chromium automation', icon: MonitorPlay }, { id: 'notifications', label: 'Notifications', keywords: 'notifications alerts push', icon: Bell }, { id: 'plugins', label: 'Plugins', keywords: 'plugins extensions integrations', icon: Plug }, { id: 'about', label: 'About', keywords: 'about version info', icon: Info }, diff --git a/src/components/settings/hooks/useSettingsController.ts b/src/components/settings/hooks/useSettingsController.ts index 70e9ed1c..a172b831 100644 --- a/src/components/settings/hooks/useSettingsController.ts +++ b/src/components/settings/hooks/useSettingsController.ts @@ -54,7 +54,7 @@ type NotificationPreferencesResponse = { type ActiveLoginProvider = AgentProvider | ''; -const KNOWN_MAIN_TABS: SettingsMainTab[] = ['agents', 'appearance', 'git', 'api', 'tasks', 'notifications', 'plugins']; +const KNOWN_MAIN_TABS: SettingsMainTab[] = ['agents', 'appearance', 'git', 'api', 'tasks', 'browser', 'notifications', 'plugins', 'about']; const normalizeMainTab = (tab: string): SettingsMainTab => { // Keep backwards compatibility with older callers that still pass "tools". diff --git a/src/components/settings/types/types.ts b/src/components/settings/types/types.ts index f68cacc4..672be1ee 100644 --- a/src/components/settings/types/types.ts +++ b/src/components/settings/types/types.ts @@ -3,7 +3,7 @@ import type { Dispatch, SetStateAction } from 'react'; import type { LLMProvider } from '../../../types/app'; import type { ProviderAuthStatus } from '../../provider-auth/types'; -export type SettingsMainTab = 'agents' | 'appearance' | 'git' | 'api' | 'tasks' | 'notifications' | 'plugins' | 'about'; +export type SettingsMainTab = 'agents' | 'appearance' | 'git' | 'api' | 'tasks' | 'browser' | 'notifications' | 'plugins' | 'about'; export type AgentProvider = LLMProvider; export type AgentCategory = 'account' | 'permissions' | 'mcp'; export type ProjectSortOrder = 'name' | 'date'; diff --git a/src/components/settings/view/Settings.tsx b/src/components/settings/view/Settings.tsx index 8340a547..800440e0 100644 --- a/src/components/settings/view/Settings.tsx +++ b/src/components/settings/view/Settings.tsx @@ -7,6 +7,7 @@ import AgentsSettingsTab from '../view/tabs/agents-settings/AgentsSettingsTab'; import AppearanceSettingsTab from '../view/tabs/AppearanceSettingsTab'; import CredentialsSettingsTab from '../view/tabs/api-settings/CredentialsSettingsTab'; import GitSettingsTab from '../view/tabs/git-settings/GitSettingsTab'; +import BrowserUseSettingsTab from '../view/tabs/browser-use-settings/BrowserUseSettingsTab'; import NotificationsSettingsTab from '../view/tabs/NotificationsSettingsTab'; import TasksSettingsTab from '../view/tabs/tasks-settings/TasksSettingsTab'; import PluginSettingsTab from '../../plugins/view/PluginSettingsTab'; @@ -139,17 +140,19 @@ function Settings({ isOpen, onClose, projects = [], initialTab = 'agents' }: Set {activeTab === 'tasks' && } - {activeTab === 'notifications' && ( - - )} + {activeTab === 'browser' && } + + {activeTab === 'notifications' && ( + + )} {activeTab === 'api' && } diff --git a/src/components/settings/view/SettingsSidebar.tsx b/src/components/settings/view/SettingsSidebar.tsx index 149c1492..dde32a9e 100644 --- a/src/components/settings/view/SettingsSidebar.tsx +++ b/src/components/settings/view/SettingsSidebar.tsx @@ -1,4 +1,4 @@ -import { Bell, Bot, GitBranch, Info, Key, ListChecks, Palette, Puzzle } from 'lucide-react'; +import { Bell, Bot, GitBranch, Info, Key, ListChecks, MonitorPlay, Palette, Puzzle } from 'lucide-react'; import { useTranslation } from 'react-i18next'; import { cn } from '../../../lib/utils'; import { PillBar, Pill } from '../../../shared/view/ui'; @@ -21,6 +21,7 @@ const NAV_ITEMS: NavItem[] = [ { id: 'git', labelKey: 'mainTabs.git', icon: GitBranch }, { id: 'api', labelKey: 'mainTabs.apiTokens', icon: Key }, { id: 'tasks', labelKey: 'mainTabs.tasks', icon: ListChecks }, + { id: 'browser', labelKey: 'mainTabs.browser', icon: MonitorPlay }, { id: 'plugins', labelKey: 'mainTabs.plugins', icon: Puzzle }, { id: 'notifications', labelKey: 'mainTabs.notifications', icon: Bell }, { id: 'about', labelKey: 'mainTabs.about', icon: Info }, diff --git a/src/components/settings/view/tabs/browser-use-settings/BrowserUseSettingsTab.tsx b/src/components/settings/view/tabs/browser-use-settings/BrowserUseSettingsTab.tsx new file mode 100644 index 00000000..c109918f --- /dev/null +++ b/src/components/settings/view/tabs/browser-use-settings/BrowserUseSettingsTab.tsx @@ -0,0 +1,192 @@ +import { useCallback, useEffect, useState } from 'react'; +import { Download, ExternalLink, Loader2 } from 'lucide-react'; + +import { Button } from '../../../../../shared/view/ui'; +import { authenticatedFetch } from '../../../../../utils/api'; +import SettingsCard from '../../SettingsCard'; +import SettingsRow from '../../SettingsRow'; +import SettingsSection from '../../SettingsSection'; +import SettingsToggle from '../../SettingsToggle'; + +type BrowserUseSettings = { + enabled: boolean; + agentToolsEnabled: boolean; +}; + +type BrowserUseStatus = { + enabled: boolean; + available: boolean; + playwrightInstalled: boolean; + chromiumInstalled: boolean; + installInProgress: boolean; + agentToolsEnabled: boolean; + message: string; +}; + +async function readJson(response: Response): Promise { + const data = await response.json(); + if (!response.ok || data.success === false) { + throw new Error(data.error || data.details || `Request failed (${response.status})`); + } + return data as T; +} + +export default function BrowserUseSettingsTab() { + const [settings, setSettings] = useState({ enabled: false, agentToolsEnabled: false }); + const [status, setStatus] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const [isSaving, setIsSaving] = useState(false); + const [isInstalling, setIsInstalling] = useState(false); + const [error, setError] = useState(null); + + const loadState = useCallback(async () => { + setError(null); + const [settingsResponse, statusResponse] = await Promise.all([ + authenticatedFetch('/api/browser-use/settings'), + authenticatedFetch('/api/browser-use/status'), + ]); + const settingsData = await readJson<{ data: { settings: BrowserUseSettings } }>(settingsResponse); + const statusData = await readJson<{ data: BrowserUseStatus }>(statusResponse); + setSettings(settingsData.data.settings); + setStatus(statusData.data); + }, []); + + useEffect(() => { + setIsLoading(true); + void loadState() + .catch((err) => setError(err instanceof Error ? err.message : 'Failed to load Browser Use settings')) + .finally(() => setIsLoading(false)); + }, [loadState]); + + const updateSettings = async (nextSettings: Partial) => { + setIsSaving(true); + setError(null); + try { + const response = await authenticatedFetch('/api/browser-use/settings', { + method: 'PUT', + body: JSON.stringify(nextSettings), + }); + const data = await readJson<{ data: { settings: BrowserUseSettings } }>(response); + setSettings(data.data.settings); + window.dispatchEvent(new Event('browserUseSettingsChanged')); + await loadState(); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to save Browser Use settings'); + } finally { + setIsSaving(false); + } + }; + + const installBrowserBinaries = async () => { + setIsInstalling(true); + setError(null); + try { + const response = await authenticatedFetch('/api/browser-use/runtime/install', { method: 'POST' }); + await readJson(response); + await loadState(); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to install browser binaries'); + } finally { + setIsInstalling(false); + } + }; + + const needsBrowserBinaries = Boolean(settings.enabled && status && (!status.playwrightInstalled || !status.chromiumInstalled)); + + return ( +
+ + +
+
+
How Browser Use Works
+

+ Learn what agents can do with browser sessions, when to share access, and what the current limitations are. +

+
+ + Open Guide + + +
+ + + void updateSettings({ enabled: value })} + ariaLabel="Enable Browser Use" + disabled={isLoading || isSaving} + /> + + + + void updateSettings({ agentToolsEnabled: value })} + ariaLabel="Enable Browser Tools for Agents" + disabled={isLoading || isSaving || !settings.enabled} + /> + + + {(needsBrowserBinaries || error) && ( +
+ {needsBrowserBinaries && ( +
+
+
Browser binaries required
+

+ {status?.message || 'Install the browser binaries needed to create Browser Use sessions.'} +

+
+ + Playwright: {status?.playwrightInstalled ? 'installed' : 'missing'} + + + Chromium: {status?.chromiumInstalled ? 'installed' : 'missing'} + +
+
+ + +
+ )} + + {error && ( +
+ {error} +
+ )} +
+ )} + + +
+ ); +} diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index d5bc7900..bae8db89 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -94,6 +94,7 @@ "git": "Git", "apiTokens": "API & Tokens", "tasks": "Tasks", + "browser": "Browser Use", "notifications": "Notifications", "plugins": "Plugins", "about": "About"