import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent, type MouseEvent } from 'react'; import { Bot, Camera, Download, Expand, Loader2, MonitorCog, RefreshCw, ShieldCheck, Square, Trash2, X } from 'lucide-react'; import { Badge, Button } from '../../../shared/view/ui'; import { authenticatedFetch } from '../../../utils/api'; type ComputerUseStatus = { enabled: boolean; runtime: 'cloud' | 'local'; available: boolean; requiresDesktopBridge: boolean; nutInstalled: boolean; screenshotInstalled: boolean; installInProgress: boolean; sessionCount: number; agentToolsEnabled: boolean; mcpRecommended: boolean; message: string; }; type ComputerUseSession = { id: string; status: 'ready' | 'stopped' | 'unavailable'; screenshotDataUrl: string | null; createdAt: string; updatedAt: string; lastAction: string | null; message: string | null; agentAccessEnabled: boolean; createdBy: 'user' | 'agent'; displaySize: { width: number; height: number; } | null; cursor: { x: number; y: number; actor: 'agent' | 'user'; } | null; }; type ComputerUsePanelProps = { isVisible: boolean; }; async function readJson(response: Response): Promise { const data = await response.json(); if (!response.ok || data.success === false) { throw new Error(data.error || data.details || `Request failed (${response.status})`); } return data as T; } export default function ComputerUsePanel({ isVisible }: ComputerUsePanelProps) { const [status, setStatus] = useState(null); const [sessions, setSessions] = useState([]); const [selectedSessionId, setSelectedSessionId] = useState(null); const [isBusy, setIsBusy] = useState(false); const [isInstalling, setIsInstalling] = useState(false); const [isFullscreen, setIsFullscreen] = useState(false); const [error, setError] = useState(null); const viewerRef = useRef(null); const selectedSession = useMemo( () => sessions.find((session) => session.id === selectedSessionId) || sessions[0] || null, [selectedSessionId, sessions], ); const refresh = useCallback(async () => { const [statusResponse, sessionsResponse] = await Promise.all([ authenticatedFetch('/api/computer-use/status'), authenticatedFetch('/api/computer-use/sessions'), ]); const statusData = await readJson<{ data: ComputerUseStatus }>(statusResponse); const sessionsData = await readJson<{ data: { sessions: ComputerUseSession[] } }>(sessionsResponse); setStatus(statusData.data); setSessions(sessionsData.data.sessions); setSelectedSessionId((current) => ( current && sessionsData.data.sessions.some((session) => session.id === current) ? current : sessionsData.data.sessions[0]?.id || null )); }, []); useEffect(() => { if (!isVisible) return; void refresh().catch((err) => setError(err instanceof Error ? err.message : 'Failed to load Computer Use')); }, [isVisible, refresh]); // Poll while an active session exists so agent-driven changes show up live. useEffect(() => { if (!isVisible || !selectedSession || selectedSession.status !== 'ready') return; const timer = window.setInterval(() => { void refresh().catch(() => undefined); }, 1500); return () => window.clearInterval(timer); }, [isVisible, selectedSession, refresh]); const runAction = useCallback(async (action: () => Promise) => { setIsBusy(true); setError(null); try { await action(); await refresh(); } catch (err) { setError(err instanceof Error ? err.message : 'Computer Use action failed'); } finally { setIsBusy(false); } }, [refresh]); const createSession = () => runAction(async () => { const response = await authenticatedFetch('/api/computer-use/sessions', { method: 'POST' }); const data = await readJson<{ data: { session: ComputerUseSession } }>(response); setSelectedSessionId(data.data.session.id); }); const captureScreenshot = () => runAction(async () => { if (!selectedSession) return; const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}/screenshot`, { method: 'POST' }); await readJson(response); }); const stopSession = () => runAction(async () => { if (!selectedSession) return; const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}/stop`, { method: 'POST' }); await readJson(response); }); const deleteSession = () => runAction(async () => { if (!selectedSession) return; const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}`, { method: 'DELETE' }); await readJson(response); setIsFullscreen(false); }); const grantControl = () => runAction(async () => { if (!selectedSession) return; const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}/consent/grant`, { method: 'POST' }); await readJson(response); }); const revokeControl = () => runAction(async () => { if (!selectedSession) return; const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}/consent/revoke`, { method: 'POST' }); await readJson(response); }); const installRuntime = () => runAction(async () => { setIsInstalling(true); try { const response = await authenticatedFetch('/api/computer-use/runtime/install', { method: 'POST' }); await readJson(response); } finally { setIsInstalling(false); } }); const clickViewer = useCallback((event: MouseEvent) => { if (!selectedSession || selectedSession.status !== 'ready' || !selectedSession.displaySize) { return; } viewerRef.current?.focus(); const bounds = event.currentTarget.getBoundingClientRect(); const scaleX = selectedSession.displaySize.width / bounds.width; const scaleY = selectedSession.displaySize.height / bounds.height; const x = Math.round((event.clientX - bounds.left) * scaleX); const y = Math.round((event.clientY - bounds.top) * scaleY); void runAction(async () => { const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}/click`, { method: 'POST', body: JSON.stringify({ x, y, double: event.detail === 2 }), }); await readJson(response); }); }, [runAction, selectedSession]); const keyForEvent = useCallback((event: KeyboardEvent) => { if (event.key === ' ') return 'Space'; const parts: string[] = []; if (event.ctrlKey) parts.push('ctrl'); if (event.altKey) parts.push('alt'); if (event.shiftKey && event.key.length > 1) parts.push('shift'); if (event.metaKey) parts.push('meta'); parts.push(event.key); return parts.join('+'); }, []); const pressViewerKey = useCallback((event: KeyboardEvent) => { if (!selectedSession || selectedSession.status !== 'ready') { return; } const ignoredKeys = new Set(['Shift', 'Control', 'Alt', 'Meta', 'CapsLock']); if (ignoredKeys.has(event.key)) { return; } event.preventDefault(); const key = keyForEvent(event); void runAction(async () => { const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}/press-key`, { method: 'POST', body: JSON.stringify({ key }), }); await readJson(response); }); }, [keyForEvent, runAction, selectedSession]); const needsRuntime = Boolean(status?.enabled && status.runtime === 'local' && (!status.nutInstalled || !status.screenshotInstalled)); const isCloud = status?.runtime === 'cloud'; const cursorStyle = selectedSession?.cursor && selectedSession.displaySize ? { left: `${(selectedSession.cursor.x / selectedSession.displaySize.width) * 100}%`, top: `${(selectedSession.cursor.y / selectedSession.displaySize.height) * 100}%`, } : null; const renderSurface = (fullscreen = false) => (
{selectedSession?.screenshotDataUrl ? (
Desktop screenshot {cursorStyle && (
)}
) : (
{selectedSession?.message || 'Start a Computer Use session to capture your desktop.'}

{isCloud ? 'Cloud Computer Use requires a linked local CloudCLI Desktop Agent.' : 'Install the desktop control runtime from this panel or enable Computer Use from Settings.'}

)}
); return (

Computer Use

{status && {status.runtime}}

Capture your desktop and let agents drive the mouse and keyboard — only while you grant control.

{selectedSession?.agentAccessEnabled ? ( ) : ( )}
{error && (
{error}
)}
{selectedSession?.displaySize ? `${selectedSession.displaySize.width}×${selectedSession.displaySize.height}` : 'No screen captured'} {selectedSession?.agentAccessEnabled && ( Agent control active )}
{renderSurface()}

Click the screenshot to click the real desktop. Focus the view and type to send keystrokes.

{isFullscreen && selectedSession && (
Desktop session
{renderSurface(true)}
)}
); }