import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent, type MouseEvent } from 'react'; import { Bot, Camera, Download, Expand, Loader2, MonitorCog, RefreshCw, Settings, ShieldCheck, Square, Trash2, X } from 'lucide-react'; import { cn } from '../../../lib/utils'; import { Badge, Button } from '../../../shared/view/ui'; import { authenticatedFetch } from '../../../utils/api'; import type { SettingsMainTab } from '../../settings/types/types'; type ComputerUseStatus = { enabled: boolean; runtime: 'cloud' | 'local'; available: boolean; desktopAgentConnected?: boolean; desktopAgentCount?: number; nutInstalled: boolean; screenshotInstalled: boolean; installInProgress: boolean; sessionCount: number; message: string; }; type ComputerUseSession = { id: string; status: 'ready' | 'stopped' | 'unavailable'; screenshotDataUrl: string | null; createdAt: string; updatedAt: string; lastAction: string | null; message: string | null; agentAccessEnabled: boolean; createdBy: 'user' | 'agent'; displaySize: { width: number; height: number; } | null; cursor: { x: number; y: number; actor: 'agent' | 'user'; } | null; }; type ComputerUsePanelProps = { isVisible: boolean; onShowSettings?: (tab?: SettingsMainTab) => void; }; async function readJson(response: Response): Promise { const data = await response.json(); if (!response.ok || data.success === false) { throw new Error(data.error || data.details || `Request failed (${response.status})`); } return data as T; } function getRuntimeTone(status: ComputerUseStatus | null, installing: boolean): string { if (!status?.enabled) return 'border-border bg-muted text-muted-foreground'; if (status.runtime === 'cloud') { return status.desktopAgentConnected ? 'border-primary/30 bg-primary/5 text-foreground' : 'border-amber-500/30 bg-amber-500/10 text-amber-700 dark:text-amber-300'; } if (status.available) return 'border-primary/30 bg-primary/5 text-foreground'; if (status.installInProgress || installing) return 'border-primary/30 bg-primary/5 text-foreground'; return 'border-border bg-background text-muted-foreground'; } function getRuntimeLabel(status: ComputerUseStatus | null, installing: boolean): string { if (!status?.enabled) return 'Disabled'; if (status.runtime === 'cloud') { const count = status.desktopAgentCount ?? (status.desktopAgentConnected ? 1 : 0); if (count > 1) return `${count} desktops linked`; if (count === 1) return 'Desktop linked'; return 'Desktop not linked'; } if (status.available) return 'Ready'; if (status.installInProgress || installing) return 'Installing'; return 'Setup required'; } export default function ComputerUsePanel({ isVisible, onShowSettings }: ComputerUsePanelProps) { const [status, setStatus] = useState(null); const [sessions, setSessions] = useState([]); const [selectedSessionId, setSelectedSessionId] = useState(null); const [isRefreshing, setIsRefreshing] = useState(false); const [isBusy, setIsBusy] = useState(false); const [isInstalling, setIsInstalling] = useState(false); const [isFullscreen, setIsFullscreen] = useState(false); const [error, setError] = useState(null); const viewerRef = useRef(null); const selectedSession = useMemo( () => sessions.find((session) => session.id === selectedSessionId) || sessions[0] || null, [selectedSessionId, sessions], ); const refresh = useCallback(async () => { setIsRefreshing(true); try { const [statusResponse, sessionsResponse] = await Promise.all([ authenticatedFetch('/api/computer-use/status'), authenticatedFetch('/api/computer-use/sessions'), ]); const statusData = await readJson<{ data: ComputerUseStatus }>(statusResponse); const sessionsData = await readJson<{ data: { sessions: ComputerUseSession[] } }>(sessionsResponse); setStatus(statusData.data); setSessions(sessionsData.data.sessions); setSelectedSessionId((current) => ( current && sessionsData.data.sessions.some((session) => session.id === current) ? current : sessionsData.data.sessions[0]?.id || null )); setError(null); } finally { setIsRefreshing(false); } }, []); useEffect(() => { if (!isVisible) return; void refresh().catch((err) => setError(err instanceof Error ? err.message : 'Failed to load Computer Use')); }, [isVisible, refresh]); const handleRefresh = useCallback(() => { void refresh().catch((err) => setError(err instanceof Error ? err.message : 'Failed to refresh Computer Use')); }, [refresh]); // Poll while an active session exists so agent-driven changes show up live. useEffect(() => { if (!isVisible || !selectedSession || selectedSession.status !== 'ready') return; const timer = window.setInterval(() => { void refresh().catch(() => undefined); }, 1500); return () => window.clearInterval(timer); }, [isVisible, selectedSession, refresh]); const runAction = useCallback(async (action: () => Promise) => { setIsBusy(true); setError(null); try { await action(); await refresh(); } catch (err) { setError(err instanceof Error ? err.message : 'Computer Use action failed'); } finally { setIsBusy(false); } }, [refresh]); const captureScreenshot = () => runAction(async () => { if (!selectedSession) return; const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}/screenshot`, { method: 'POST' }); await readJson(response); }); const stopSession = () => runAction(async () => { if (!selectedSession) return; const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}/stop`, { method: 'POST' }); await readJson(response); }); const deleteSession = () => runAction(async () => { if (!selectedSession) return; const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}`, { method: 'DELETE' }); await readJson(response); setIsFullscreen(false); }); const grantControl = () => runAction(async () => { if (!selectedSession) return; const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}/consent/grant`, { method: 'POST' }); await readJson(response); }); const revokeControl = () => runAction(async () => { if (!selectedSession) return; const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}/consent/revoke`, { method: 'POST' }); await readJson(response); }); const installRuntime = () => runAction(async () => { setIsInstalling(true); try { const response = await authenticatedFetch('/api/computer-use/runtime/install', { method: 'POST' }); await readJson(response); } finally { setIsInstalling(false); } }); const clickViewer = useCallback((event: MouseEvent) => { if (!selectedSession || selectedSession.status !== 'ready' || !selectedSession.displaySize) { return; } viewerRef.current?.focus(); const bounds = event.currentTarget.getBoundingClientRect(); const scaleX = selectedSession.displaySize.width / bounds.width; const scaleY = selectedSession.displaySize.height / bounds.height; const x = Math.round((event.clientX - bounds.left) * scaleX); const y = Math.round((event.clientY - bounds.top) * scaleY); void runAction(async () => { const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}/click`, { method: 'POST', body: JSON.stringify({ x, y, double: event.detail === 2 }), }); await readJson(response); }); }, [runAction, selectedSession]); const keyForEvent = useCallback((event: KeyboardEvent) => { if (event.key === ' ') return 'Space'; const parts: string[] = []; if (event.ctrlKey) parts.push('ctrl'); if (event.altKey) parts.push('alt'); if (event.shiftKey && event.key.length > 1) parts.push('shift'); if (event.metaKey) parts.push('meta'); parts.push(event.key); return parts.join('+'); }, []); const pressViewerKey = useCallback((event: KeyboardEvent) => { if (!selectedSession || selectedSession.status !== 'ready') { return; } const ignoredKeys = new Set(['Shift', 'Control', 'Alt', 'Meta', 'CapsLock']); if (ignoredKeys.has(event.key)) { return; } event.preventDefault(); const key = keyForEvent(event); void runAction(async () => { const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}/press-key`, { method: 'POST', body: JSON.stringify({ key }), }); await readJson(response); }); }, [keyForEvent, runAction, selectedSession]); const needsRuntime = Boolean(status?.enabled && status.runtime === 'local' && (!status.nutInstalled || !status.screenshotInstalled)); const isCloud = status?.runtime === 'cloud'; const desktopAgentCount = status?.desktopAgentCount ?? (status?.desktopAgentConnected ? 1 : 0); const runtimeLabel = getRuntimeLabel(status, isInstalling); const cursorStyle = selectedSession?.cursor && selectedSession.displaySize ? { left: `${(selectedSession.cursor.x / selectedSession.displaySize.width) * 100}%`, top: `${(selectedSession.cursor.y / selectedSession.displaySize.height) * 100}%`, } : null; const renderSurface = (fullscreen = false) => (
{selectedSession?.screenshotDataUrl ? (
Desktop screenshot {cursorStyle && (
)}
) : (
{selectedSession?.message || 'No active Computer Use session.'}

{isCloud ? 'Agents create sessions automatically. Keep the CloudCLI desktop app connected to approve control requests.' : 'Agents create sessions automatically. Enable Computer Use and install the local runtime if needed.'}

)}
); return (

Computer Use

{runtimeLabel}

{isCloud ? 'Monitor cloud agent desktop sessions and linked desktops.' : 'Monitor local desktop sessions and grant control only when an agent needs it.'}

{onShowSettings && ( )}
{!isCloud && selectedSession?.agentAccessEnabled ? ( ) : !isCloud ? ( ) : null}
{error && (
{error}
)}
{selectedSession?.displaySize ? `${selectedSession.displaySize.width}Ă—${selectedSession.displaySize.height}` : 'No screen captured'} {selectedSession?.agentAccessEnabled && ( {isCloud ? 'Desktop-approved session' : 'Agent control active'} )}
{renderSurface()}

{selectedSession ? 'Click the screenshot to click the real desktop. Focus the view and type to send keystrokes.' : 'Computer Use sessions appear here after an agent requests desktop access.'}

{isFullscreen && selectedSession && (
Desktop session
{renderSurface(true)}
)}
); }