diff --git a/server/modules/browser-use/browser-use.routes.ts b/server/modules/browser-use/browser-use.routes.ts index 167912cd..6eff6af6 100644 --- a/server/modules/browser-use/browser-use.routes.ts +++ b/server/modules/browser-use/browser-use.routes.ts @@ -4,20 +4,6 @@ import { browserUseService } from '@/modules/browser-use/browser-use.service.js' const router = express.Router(); -type AuthenticatedRequest = express.Request & { - user?: { - id?: string | number; - }; -}; - -function requireUser(req: AuthenticatedRequest): { id: string | number } { - const userId = req.user?.id; - if (userId === undefined || userId === null) { - throw new Error('Authenticated user is required.'); - } - return { id: userId }; -} - function readParam(value: string | string[] | undefined): string { return Array.isArray(value) ? value[0] || '' : value || ''; } @@ -56,18 +42,6 @@ router.put('/settings', async (req, res) => { } }); -router.post('/agent-tools/register', async (_req, res) => { - try { - const result = await browserUseService.registerAgentMcp(); - res.status(201).json({ success: true, data: result }); - } catch (error) { - res.status(400).json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to register Browser Use MCP.', - }); - } -}); - router.post('/runtime/install', async (_req, res) => { try { const result = await browserUseService.installRuntime(); @@ -84,9 +58,9 @@ router.post('/runtime/install', async (_req, res) => { } }); -router.get('/sessions', async (req: AuthenticatedRequest, res) => { +router.get('/sessions', async (_req, res) => { try { - res.json({ success: true, data: { sessions: await browserUseService.listSessions(requireUser(req)) } }); + res.json({ success: true, data: { sessions: await browserUseService.listSessions() } }); } catch (error) { res.status(401).json({ success: false, @@ -95,90 +69,9 @@ router.get('/sessions', async (req: AuthenticatedRequest, res) => { } }); -router.post('/sessions', async (req: AuthenticatedRequest, res) => { +router.post('/sessions/:sessionId/stop', async (req, res) => { try { - const session = await browserUseService.createSession(requireUser(req)); - res.status(session.status === 'unavailable' ? 202 : 201).json({ success: true, data: { session } }); - } catch (error) { - res.status(400).json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to create browser session.', - }); - } -}); - -router.post('/sessions/:sessionId/navigate', async (req: AuthenticatedRequest, res) => { - try { - const session = await browserUseService.navigate(requireUser(req), readParam(req.params.sessionId), String(req.body?.url || '')); - res.json({ success: true, data: { session } }); - } catch (error) { - res.status(400).json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to navigate browser session.', - }); - } -}); - -router.post('/sessions/:sessionId/click', async (req: AuthenticatedRequest, res) => { - try { - const x = Number(req.body?.x); - const y = Number(req.body?.y); - if (!Number.isFinite(x) || !Number.isFinite(y)) { - throw new Error('Click requires numeric x and y coordinates.'); - } - const session = await browserUseService.userClick(requireUser(req), readParam(req.params.sessionId), { x, y }); - res.json({ success: true, data: { session } }); - } catch (error) { - res.status(400).json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to click browser session.', - }); - } -}); - -router.post('/sessions/:sessionId/press-key', async (req: AuthenticatedRequest, res) => { - try { - const key = String(req.body?.key || '').trim(); - if (!key) { - throw new Error('A key is required.'); - } - const session = await browserUseService.userPressKey(requireUser(req), readParam(req.params.sessionId), key); - res.json({ success: true, data: { session } }); - } catch (error) { - res.status(400).json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to send browser key input.', - }); - } -}); - -router.post('/sessions/:sessionId/agent-access/grant', async (req: AuthenticatedRequest, res) => { - try { - const session = await browserUseService.grantAgentAccess(requireUser(req), readParam(req.params.sessionId)); - res.json({ success: true, data: { session } }); - } catch (error) { - res.status(400).json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to grant agent access.', - }); - } -}); - -router.post('/sessions/:sessionId/agent-access/revoke', async (req: AuthenticatedRequest, res) => { - try { - const session = await browserUseService.revokeAgentAccess(requireUser(req), readParam(req.params.sessionId)); - res.json({ success: true, data: { session } }); - } catch (error) { - res.status(400).json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to revoke agent access.', - }); - } -}); - -router.post('/sessions/:sessionId/stop', async (req: AuthenticatedRequest, res) => { - try { - const result = await browserUseService.stopSession(requireUser(req), readParam(req.params.sessionId)); + const result = await browserUseService.stopSession(readParam(req.params.sessionId)); res.json({ success: true, data: result }); } catch (error) { res.status(400).json({ @@ -188,9 +81,9 @@ router.post('/sessions/:sessionId/stop', async (req: AuthenticatedRequest, res) } }); -router.delete('/sessions/:sessionId', async (req: AuthenticatedRequest, res) => { +router.delete('/sessions/:sessionId', async (req, res) => { try { - const result = await browserUseService.deleteSession(requireUser(req), readParam(req.params.sessionId)); + const result = await browserUseService.deleteSession(readParam(req.params.sessionId)); res.json({ success: true, data: result }); } catch (error) { res.status(400).json({ diff --git a/server/modules/browser-use/browser-use.service.ts b/server/modules/browser-use/browser-use.service.ts index 7906b49e..5d14f17c 100644 --- a/server/modules/browser-use/browser-use.service.ts +++ b/server/modules/browser-use/browser-use.service.ts @@ -8,7 +8,7 @@ import net from 'node:net'; import path from 'node:path'; import { appConfigDb } from '@/modules/database/index.js'; -import { providerMcpService } from '@/modules/providers/services/mcp.service.js'; +import { providerMcpService } from '@/modules/providers/index.js'; import { getModuleDir } from '@/utils/runtime-paths.js'; const require = createRequire(import.meta.url); @@ -26,7 +26,7 @@ type BrowserUseSessionStatus = 'ready' | 'stopped' | 'unavailable'; type BrowserUseSession = { id: string; ownerId: string; - createdBy: 'user' | 'agent'; + createdBy: 'agent'; runtime: BrowserUseRuntime; status: BrowserUseSessionStatus; url: string | null; @@ -36,7 +36,6 @@ type BrowserUseSession = { updatedAt: string; lastAction: string | null; message: string | null; - agentAccessEnabled: boolean; profileName: string | null; viewport: { width: number; @@ -45,7 +44,7 @@ type BrowserUseSession = { cursor: { x: number; y: number; - actor: 'agent' | 'user'; + actor: 'agent'; } | null; }; @@ -57,13 +56,8 @@ type RuntimeHandle = { page?: any; }; -type BrowserUseOwner = { - id: string | number; -}; - type BrowserUseSettings = { enabled: boolean; - agentToolsEnabled: boolean; }; type RuntimeReadiness = { @@ -82,7 +76,6 @@ let lastInstallMessage: string | null = null; const DEFAULT_SETTINGS: BrowserUseSettings = { enabled: false, - agentToolsEnabled: false, }; const AGENT_OWNER_ID = 'agent'; const PROFILE_ROOT = path.join(os.homedir(), '.cloudcli', 'browser-use', 'profiles'); @@ -103,7 +96,6 @@ function readSettings(): BrowserUseSettings { const parsed = JSON.parse(raw) as Partial; return { enabled: parsed.enabled === true, - agentToolsEnabled: parsed.agentToolsEnabled === true, }; } catch (error: any) { console.warn('[Browser Use] Failed to read settings:', error?.message || error); @@ -114,7 +106,6 @@ function readSettings(): BrowserUseSettings { function writeSettings(settings: BrowserUseSettings): BrowserUseSettings { const normalized = { enabled: settings.enabled === true, - agentToolsEnabled: settings.agentToolsEnabled === true, }; appConfigDb.set(BROWSER_USE_SETTINGS_KEY, JSON.stringify(normalized)); @@ -244,7 +235,6 @@ function runCommand(command: string, args: string[]): Promise { fn(); }; - // Guard against a stuck npm/playwright process hanging the install request forever. const timer = setTimeout(() => { child.kill('SIGKILL'); finish(() => reject(new Error( @@ -309,14 +299,6 @@ async function installRuntime(): Promise<{ success: boolean; message: string }> } } -function getOwnerId(owner: BrowserUseOwner): string { - if (owner.id === undefined || owner.id === null || String(owner.id).trim() === '') { - throw new Error('Authenticated user is required.'); - } - - return String(owner.id); -} - function isPrivateIpv4(address: string): boolean { const parts = address.split('.').map((part) => Number.parseInt(part, 10)); if (parts.length !== 4 || parts.some((part) => Number.isNaN(part) || part < 0 || part > 255)) { @@ -410,8 +392,6 @@ async function assertAllowedBrowserRequest(rawUrl: string): Promise { } async function attachRequestGuard(context: any): Promise { - // Attach at the context level so the guard also covers popups, window.open targets, - // and any replacement pages created during the session lifecycle. await context.route('**/*', async (route: any) => { try { await assertAllowedBrowserRequest(route.request().url()); @@ -431,10 +411,6 @@ function ownerSessions(ownerId: string): BrowserUseSession[] { return [...sessions.values()].filter((session) => session.ownerId === ownerId); } -function canAccessSession(ownerId: string, session: BrowserUseSession): boolean { - return session.ownerId === ownerId || session.ownerId === AGENT_OWNER_ID || session.agentAccessEnabled; -} - async function closeHandle(sessionId: string): Promise { const handle = handles.get(sessionId); handles.delete(sessionId); @@ -504,21 +480,15 @@ export const browserUseService = { async updateSettings(settings: Partial) { const current = readSettings(); const nextSettings = { - ...current, enabled: typeof settings.enabled === 'boolean' ? settings.enabled : current.enabled, - agentToolsEnabled: typeof settings.agentToolsEnabled === 'boolean' - ? settings.agentToolsEnabled - : current.agentToolsEnabled, }; - if (!nextSettings.enabled) { - nextSettings.agentToolsEnabled = false; - } const next = writeSettings(nextSettings); - if (next.agentToolsEnabled) { + if (next.enabled) { await this.registerAgentMcp(); - } else if (current.agentToolsEnabled) { + } else if (current.enabled) { await this.unregisterAgentMcp(); + await this.stopAllSessions(); } return next; }, @@ -536,8 +506,6 @@ export const browserUseService = { chromiumInstalled: readiness.chromiumInstalled, installInProgress: readiness.installInProgress, sessionCount: sessions.size, - agentToolsEnabled: settings.agentToolsEnabled, - mcpRecommended: !settings.agentToolsEnabled, message: available ? 'Browser Use runtime is available.' : getSetupMessage(settings, readiness), @@ -591,25 +559,27 @@ export const browserUseService = { }; }, - async listSessions(owner: BrowserUseOwner) { - const ownerId = getOwnerId(owner); + async listSessions() { await expireStaleSessions(); return [...sessions.values()] - .filter((session) => canAccessSession(ownerId, session)) + .filter((session) => session.ownerId === AGENT_OWNER_ID) .map(publicSession); }, - async createSession(owner: BrowserUseOwner, options?: { createdBy?: 'user' | 'agent'; profileName?: string | null; agentAccessEnabled?: boolean }) { - const ownerId = getOwnerId(owner); + async createAgentSession(options?: { profileName?: string | null }) { + const settings = readSettings(); + if (!settings.enabled) { + throw new Error('Browser Use agent tools are disabled.'); + } + await expireStaleSessions(); - const createdBy = options?.createdBy ?? 'user'; const profileName = normalizeProfileName(options?.profileName); const now = new Date().toISOString(); const session: BrowserUseSession = { id: randomUUID(), - ownerId, - createdBy, + ownerId: AGENT_OWNER_ID, + createdBy: 'agent', runtime: getRuntime(), status: 'unavailable', url: null, @@ -619,18 +589,16 @@ export const browserUseService = { updatedAt: now, lastAction: 'create', message: null, - agentAccessEnabled: options?.agentAccessEnabled ?? createdBy === 'agent', profileName, viewport: { width: 1440, height: 900 }, cursor: null, }; - const activeOwnerSessions = ownerSessions(ownerId).filter((item) => item.status === 'ready'); + const activeOwnerSessions = ownerSessions(AGENT_OWNER_ID).filter((item) => item.status === 'ready'); if (activeOwnerSessions.length >= MAX_SESSIONS_PER_OWNER) { - throw new Error(`Browser Use is limited to ${MAX_SESSIONS_PER_OWNER} active sessions per user.`); + throw new Error(`Browser Use is limited to ${MAX_SESSIONS_PER_OWNER} active agent sessions.`); } - const settings = readSettings(); const readiness = getRuntimeReadiness(); if (!settings.enabled || !readiness.playwrightInstalled || !readiness.chromiumInstalled || !readiness.playwright) { session.message = getSetupMessage(settings, readiness); @@ -671,70 +639,35 @@ export const browserUseService = { return publicSession(session); }, - async grantAgentAccess(owner: BrowserUseOwner, sessionId: string) { - const ownerId = getOwnerId(owner); - const session = sessions.get(sessionId); - if (!session || (session.ownerId !== ownerId && session.ownerId !== AGENT_OWNER_ID)) { - throw new Error('Browser session not found.'); - } - session.agentAccessEnabled = true; - session.updatedAt = new Date().toISOString(); - session.lastAction = 'agent_access:grant'; - return publicSession(session); - }, - - async revokeAgentAccess(owner: BrowserUseOwner, sessionId: string) { - const ownerId = getOwnerId(owner); - const session = sessions.get(sessionId); - if (!session || (session.ownerId !== ownerId && session.ownerId !== AGENT_OWNER_ID)) { - throw new Error('Browser session not found.'); - } - session.agentAccessEnabled = false; - session.updatedAt = new Date().toISOString(); - session.lastAction = 'agent_access:revoke'; - return publicSession(session); - }, - async listAgentSessions() { const settings = readSettings(); - if (!settings.enabled || !settings.agentToolsEnabled) { + if (!settings.enabled) { return []; } await expireStaleSessions(); return [...sessions.values()] - .filter((session) => session.agentAccessEnabled || session.ownerId === AGENT_OWNER_ID) + .filter((session) => session.ownerId === AGENT_OWNER_ID) .map(publicSession); }, - async createAgentSession(options?: { profileName?: string | null }) { - const settings = readSettings(); - if (!settings.enabled || !settings.agentToolsEnabled) { - throw new Error('Browser Use agent tools are disabled.'); - } - return this.createSession( - { id: AGENT_OWNER_ID }, - { createdBy: 'agent', profileName: options?.profileName, agentAccessEnabled: true }, - ); - }, - async getAgentSession(sessionId: string) { const settings = readSettings(); - if (!settings.enabled || !settings.agentToolsEnabled) { + if (!settings.enabled) { throw new Error('Browser Use agent tools are disabled.'); } const session = sessions.get(sessionId); - if (!session || (!session.agentAccessEnabled && session.ownerId !== AGENT_OWNER_ID)) { - throw new Error('Browser session is not shared with agents.'); + if (!session || session.ownerId !== AGENT_OWNER_ID) { + throw new Error('Browser session not found.'); } return session; }, - async navigate(owner: BrowserUseOwner, sessionId: string, rawUrl: string) { - const ownerId = getOwnerId(owner); + async agentNavigate(sessionId: string, rawUrl: string) { + await this.getAgentSession(sessionId); await expireStaleSessions(); const session = sessions.get(sessionId); - if (!session || !canAccessSession(ownerId, session)) { + if (!session || session.ownerId !== AGENT_OWNER_ID) { throw new Error('Browser session not found.'); } @@ -755,25 +688,6 @@ export const browserUseService = { return publicSession(session); }, - async agentNavigate(sessionId: string, rawUrl: string) { - await this.getAgentSession(sessionId); - return this.navigate({ id: AGENT_OWNER_ID }, sessionId, rawUrl).catch(async (error) => { - const session = await this.getAgentSession(sessionId); - if (session.ownerId !== AGENT_OWNER_ID) { - const url = await normalizeUrl(rawUrl); - const handle = handles.get(sessionId); - if (!handle?.page) { - throw new Error('Browser runtime handle is not available.'); - } - await handle.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }); - session.lastAction = `navigate:${url}`; - await captureSession(session, handle.page); - return publicSession(session); - } - throw error; - }); - }, - async agentSnapshot(sessionId: string) { const session = await this.getAgentSession(sessionId); const handle = handles.get(sessionId); @@ -911,7 +825,6 @@ export const browserUseService = { if (action === 'new') { const page = await handle.context.newPage(); handles.set(sessionId, { ...handle, page }); - // Request guard is attached at the context level, so new pages are already covered. if (input.url) { await this.agentNavigate(sessionId, input.url); } @@ -942,10 +855,9 @@ export const browserUseService = { }; }, - async stopSession(owner: BrowserUseOwner, sessionId: string) { - const ownerId = getOwnerId(owner); + async stopSession(sessionId: string) { const session = sessions.get(sessionId); - if (!session || !canAccessSession(ownerId, session)) { + if (!session || session.ownerId !== AGENT_OWNER_ID) { return { stopped: false }; } @@ -958,10 +870,9 @@ export const browserUseService = { return { stopped: true, session: publicSession(session) }; }, - async deleteSession(owner: BrowserUseOwner, sessionId: string) { - const ownerId = getOwnerId(owner); + async deleteSession(sessionId: string) { const session = sessions.get(sessionId); - if (!session || !canAccessSession(ownerId, session)) { + if (!session || session.ownerId !== AGENT_OWNER_ID) { return { deleted: false }; } @@ -970,52 +881,9 @@ export const browserUseService = { return { deleted: true, sessionId }; }, - async userClick(owner: BrowserUseOwner, sessionId: string, input: { x: number; y: number }) { - const ownerId = getOwnerId(owner); - const session = sessions.get(sessionId); - if (!session || !canAccessSession(ownerId, session)) { - throw new Error('Browser session not found.'); - } - if (session.status !== 'ready') { - throw new Error(session.message || 'Browser session is not available.'); - } - - const handle = handles.get(sessionId); - if (!handle?.page) { - throw new Error('Browser runtime handle is not available.'); - } - - await handle.page.mouse.click(input.x, input.y); - session.lastAction = 'click'; - session.cursor = { x: input.x, y: input.y, actor: 'user' }; - await captureSession(session, handle.page); - return publicSession(session); - }, - - async userPressKey(owner: BrowserUseOwner, sessionId: string, key: string) { - const ownerId = getOwnerId(owner); - const session = sessions.get(sessionId); - if (!session || !canAccessSession(ownerId, session)) { - throw new Error('Browser session not found.'); - } - if (session.status !== 'ready') { - throw new Error(session.message || 'Browser session is not available.'); - } - - const handle = handles.get(sessionId); - if (!handle?.page) { - throw new Error('Browser runtime handle is not available.'); - } - - await handle.page.keyboard.press(key); - session.lastAction = `press_key:${key}`; - await captureSession(session, handle.page); - return publicSession(session); - }, - async agentStopSession(sessionId: string) { await this.getAgentSession(sessionId); - return this.stopSession({ id: AGENT_OWNER_ID }, sessionId); + return this.stopSession(sessionId); }, async stopAllSessions() { diff --git a/server/modules/browser-use/tests/browser-use.service.test.ts b/server/modules/browser-use/tests/browser-use.service.test.ts index 162e9439..3aefcd2d 100644 --- a/server/modules/browser-use/tests/browser-use.service.test.ts +++ b/server/modules/browser-use/tests/browser-use.service.test.ts @@ -14,17 +14,8 @@ test('browser use blocks private and local network addresses by default', () => assert.equal(isBlockedBrowserUseAddress('2001:4860:4860::8888'), false); }); -test('browser use sessions are listed only for their owner', async () => { - const ownerA = { id: `owner-a-${Date.now()}-${Math.random()}` }; - const ownerB = { id: `owner-b-${Date.now()}-${Math.random()}` }; +test('browser use monitor list starts empty without agent sessions', async () => { + const sessions = await browserUseService.listSessions(); - const ownerASession = await browserUseService.createSession(ownerA); - await browserUseService.createSession(ownerB); - - const ownerASessions = await browserUseService.listSessions(ownerA); - const ownerBSessions = await browserUseService.listSessions(ownerB); - - assert.equal(ownerASessions.some((session) => session.id === ownerASession.id), true); - assert.equal(ownerBSessions.some((session) => session.id === ownerASession.id), false); - assert.equal(Object.hasOwn(ownerASession, 'ownerId'), false); + assert.deepEqual(sessions, []); }); diff --git a/server/modules/providers/index.ts b/server/modules/providers/index.ts index 0d8d8edd..e057e200 100644 --- a/server/modules/providers/index.ts +++ b/server/modules/providers/index.ts @@ -1,5 +1,6 @@ export { sessionSynchronizerService } from './services/session-synchronizer.service.js'; export { providerSkillsService } from './services/skills.service.js'; +export { providerMcpService } from './services/mcp.service.js'; export { initializeSessionsWatcher } from './services/sessions-watcher.service.js'; export { closeSessionsWatcher } from './services/sessions-watcher.service.js'; diff --git a/src/components/browser-use/view/BrowserUsePanel.tsx b/src/components/browser-use/view/BrowserUsePanel.tsx index e0e6311c..5a7825ba 100644 --- a/src/components/browser-use/view/BrowserUsePanel.tsx +++ b/src/components/browser-use/view/BrowserUsePanel.tsx @@ -1,5 +1,5 @@ -import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent, type MouseEvent } from 'react'; -import { Bot, Download, Expand, ExternalLink, Globe, Loader2, MonitorPlay, Navigation, RefreshCw, Share2, Square, Trash2, X } from 'lucide-react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { Bot, Clock3, Download, Expand, ExternalLink, Loader2, MonitorPlay, RefreshCw, Settings, Square, Trash2, X } from 'lucide-react'; import { Badge, Button } from '../../../shared/view/ui'; import { authenticatedFetch } from '../../../utils/api'; @@ -11,8 +11,6 @@ type BrowserUseStatus = { chromiumInstalled: boolean; installInProgress: boolean; sessionCount: number; - agentToolsEnabled: boolean; - mcpRecommended: boolean; message: string; }; @@ -26,8 +24,7 @@ type BrowserUseSession = { updatedAt: string; lastAction: string | null; message: string | null; - agentAccessEnabled: boolean; - createdBy: 'user' | 'agent'; + createdBy: 'agent'; profileName: string | null; viewport: { width: number; @@ -36,12 +33,13 @@ type BrowserUseSession = { cursor: { x: number; y: number; - actor: 'agent' | 'user'; + actor: 'agent'; } | null; }; type BrowserUsePanelProps = { isVisible: boolean; + onShowSettings?: () => void; }; async function readJson(response: Response): Promise { @@ -52,16 +50,69 @@ async function readJson(response: Response): Promise { return data as T; } -export default function BrowserUsePanel({ isVisible }: BrowserUsePanelProps) { +function formatRelativeTime(value: string | null): string { + if (!value) { + return 'Never'; + } + + const timestamp = Date.parse(value); + if (!Number.isFinite(timestamp)) { + return 'Unknown'; + } + + const elapsedSeconds = Math.max(0, Math.round((Date.now() - timestamp) / 1000)); + if (elapsedSeconds < 10) return 'Just now'; + if (elapsedSeconds < 60) return `${elapsedSeconds}s ago`; + const elapsedMinutes = Math.round(elapsedSeconds / 60); + if (elapsedMinutes < 60) return `${elapsedMinutes}m ago`; + const elapsedHours = Math.round(elapsedMinutes / 60); + if (elapsedHours < 24) return `${elapsedHours}h ago`; + return `${Math.round(elapsedHours / 24)}d ago`; +} + +function getDomain(url: string | null): string { + if (!url) { + return 'No page loaded'; + } + + try { + return new URL(url).hostname; + } catch { + return url; + } +} + +function formatAction(action: string | null): string { + if (!action) { + return 'Waiting'; + } + return action.replace(/_/g, ' ').replace(/:/g, ': '); +} + +function getStatusTone(status: BrowserUseSession['status']): string { + if (status === 'ready') { + return 'border-emerald-500/30 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300'; + } + if (status === 'stopped') { + return 'border-border bg-muted text-muted-foreground'; + } + return 'border-amber-500/30 bg-amber-500/10 text-amber-700 dark:text-amber-300'; +} + +const PROMPTS = [ + 'Use Browser Use to open the staging checkout flow, try the main path, and summarize anything that looks broken.', + 'Use Browser Use to inspect the page at , capture what changed after each click, and report UI issues with screenshots.', +]; + +export default function BrowserUsePanel({ isVisible, onShowSettings }: BrowserUsePanelProps) { const [status, setStatus] = useState(null); const [sessions, setSessions] = useState([]); const [selectedSessionId, setSelectedSessionId] = useState(null); - const [targetUrl, setTargetUrl] = useState('https://example.com'); + const [isRefreshing, setIsRefreshing] = useState(false); const [isBusy, setIsBusy] = useState(false); const [isInstalling, setIsInstalling] = useState(false); const [isFullscreen, setIsFullscreen] = useState(false); const [error, setError] = useState(null); - const viewerRef = useRef(null); const selectedSession = useMemo( () => sessions.find((session) => session.id === selectedSessionId) || sessions[0] || null, @@ -69,31 +120,35 @@ export default function BrowserUsePanel({ isVisible }: BrowserUsePanelProps) { ); const refresh = useCallback(async () => { - const [statusResponse, sessionsResponse] = await Promise.all([ - authenticatedFetch('/api/browser-use/status'), - authenticatedFetch('/api/browser-use/sessions'), - ]); - const statusData = await readJson<{ data: BrowserUseStatus }>(statusResponse); - const sessionsData = await readJson<{ data: { sessions: BrowserUseSession[] } }>(sessionsResponse); - setStatus(statusData.data); - setSessions(sessionsData.data.sessions); - setSelectedSessionId((current) => ( - current && sessionsData.data.sessions.some((session) => session.id === current) - ? current - : sessionsData.data.sessions[0]?.id || null - )); + setIsRefreshing(true); + try { + const [statusResponse, sessionsResponse] = await Promise.all([ + authenticatedFetch('/api/browser-use/status'), + authenticatedFetch('/api/browser-use/sessions'), + ]); + const statusData = await readJson<{ data: BrowserUseStatus }>(statusResponse); + const sessionsData = await readJson<{ data: { sessions: BrowserUseSession[] } }>(sessionsResponse); + const nextSessions = sessionsData.data.sessions; + setStatus(statusData.data); + setSessions(nextSessions); + setSelectedSessionId((current) => ( + current && nextSessions.some((session) => session.id === current) + ? current + : nextSessions[0]?.id || null + )); + setError(null); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load Browser Use'); + } finally { + setIsRefreshing(false); + } }, []); useEffect(() => { if (!isVisible) return; - void refresh().catch((err) => setError(err instanceof Error ? err.message : 'Failed to load Browser Use')); + void refresh(); }, [isVisible, refresh]); - useEffect(() => { - if (!selectedSession?.url) return; - setTargetUrl(selectedSession.url); - }, [selectedSession?.id, selectedSession?.url]); - const runAction = useCallback(async (action: () => Promise) => { setIsBusy(true); setError(null); @@ -107,23 +162,6 @@ export default function BrowserUsePanel({ isVisible }: BrowserUsePanelProps) { } }, [refresh]); - const createSession = () => runAction(async () => { - const response = await authenticatedFetch('/api/browser-use/sessions', { method: 'POST' }); - const data = await readJson<{ data: { session: BrowserUseSession } }>(response); - setSelectedSessionId(data.data.session.id); - }); - - const navigate = () => runAction(async () => { - if (!selectedSession) { - throw new Error('Create a browser session first.'); - } - const response = await authenticatedFetch(`/api/browser-use/sessions/${selectedSession.id}/navigate`, { - method: 'POST', - body: JSON.stringify({ url: targetUrl }), - }); - await readJson(response); - }); - const stopSession = () => runAction(async () => { if (!selectedSession) return; const response = await authenticatedFetch(`/api/browser-use/sessions/${selectedSession.id}/stop`, { method: 'POST' }); @@ -137,18 +175,6 @@ export default function BrowserUsePanel({ isVisible }: BrowserUsePanelProps) { setIsFullscreen(false); }); - const grantAgentAccess = () => runAction(async () => { - if (!selectedSession) return; - const response = await authenticatedFetch(`/api/browser-use/sessions/${selectedSession.id}/agent-access/grant`, { method: 'POST' }); - await readJson(response); - }); - - const revokeAgentAccess = () => runAction(async () => { - if (!selectedSession) return; - const response = await authenticatedFetch(`/api/browser-use/sessions/${selectedSession.id}/agent-access/revoke`, { method: 'POST' }); - await readJson(response); - }); - const installBrowserBinaries = () => runAction(async () => { setIsInstalling(true); try { @@ -159,54 +185,16 @@ export default function BrowserUsePanel({ isVisible }: BrowserUsePanelProps) { } }); - const clickViewer = useCallback((event: MouseEvent) => { - if (!selectedSession || selectedSession.status !== 'ready' || !selectedSession.viewport) { - return; - } - viewerRef.current?.focus(); - - const bounds = event.currentTarget.getBoundingClientRect(); - const scaleX = selectedSession.viewport.width / bounds.width; - const scaleY = selectedSession.viewport.height / bounds.height; - const x = Math.round((event.clientX - bounds.left) * scaleX); - const y = Math.round((event.clientY - bounds.top) * scaleY); - - void runAction(async () => { - const response = await authenticatedFetch(`/api/browser-use/sessions/${selectedSession.id}/click`, { - method: 'POST', - body: JSON.stringify({ x, y }), - }); - await readJson(response); - }); - }, [runAction, selectedSession]); - - const keyForEvent = useCallback((event: KeyboardEvent) => { - if (event.key === ' ') return 'Space'; - return event.key; - }, []); - - const pressViewerKey = useCallback((event: KeyboardEvent) => { - if (!selectedSession || selectedSession.status !== 'ready') { - return; - } - - const ignoredKeys = new Set(['Shift', 'Control', 'Alt', 'Meta', 'CapsLock']); - if (ignoredKeys.has(event.key)) { - return; - } - - event.preventDefault(); - const key = keyForEvent(event); - void runAction(async () => { - const response = await authenticatedFetch(`/api/browser-use/sessions/${selectedSession.id}/press-key`, { - method: 'POST', - body: JSON.stringify({ key }), - }); - await readJson(response); - }); - }, [keyForEvent, runAction, selectedSession]); - const needsBrowserBinaries = Boolean(status?.enabled && (!status.playwrightInstalled || !status.chromiumInstalled)); + const activeSessions = sessions.filter((session) => session.status === 'ready'); + const inactiveSessions = sessions.filter((session) => session.status !== 'ready'); + const statusLabel = !status?.enabled + ? 'Disabled' + : status.available + ? 'Ready' + : status.installInProgress || isInstalling + ? 'Installing' + : 'Setup required'; const cursorStyle = selectedSession?.cursor && selectedSession.viewport ? { @@ -215,20 +203,37 @@ export default function BrowserUsePanel({ isVisible }: BrowserUsePanelProps) { } : null; - const renderBrowserSurface = (fullscreen = false) => ( -
( + + ); + + const renderBrowserSurface = (fullscreen = false) => ( +
{selectedSession?.screenshotDataUrl ? (
Browser session screenshot {cursorStyle && (
- {selectedSession?.message || 'Create a browser session to start.'} + {selectedSession?.message || 'No browser screenshot yet.'}

- Install browser binaries from this panel or enable Browser Use from Settings. + Agent-created browser sessions appear here after the agent starts using Browser Use.

)} @@ -260,48 +265,47 @@ export default function BrowserUsePanel({ isVisible }: BrowserUsePanelProps) {

Browser Use

+ {statusLabel}

- Create browser sessions, watch agent activity, and decide which sessions agents may control. + Watch browser sessions created by AI agents and stop them when needed.

-
-
-