feat: improve Computer Use linking status

This commit is contained in:
Simos Mikelatos
2026-06-19 13:47:16 +00:00
parent 218e8e2e38
commit 4d70a2588c
14 changed files with 474 additions and 231 deletions

View File

@@ -1,13 +1,17 @@
import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent, type MouseEvent } from 'react';
import { Bot, Camera, Download, Expand, Loader2, MonitorCog, RefreshCw, ShieldCheck, Square, Trash2, X } from 'lucide-react';
import { Bot, Camera, Download, Expand, Loader2, MonitorCog, RefreshCw, Settings, ShieldCheck, Square, Trash2, X } from 'lucide-react';
import { cn } from '../../../lib/utils';
import { Badge, Button } from '../../../shared/view/ui';
import { authenticatedFetch } from '../../../utils/api';
import type { SettingsMainTab } from '../../settings/types/types';
type ComputerUseStatus = {
enabled: boolean;
runtime: 'cloud' | 'local';
available: boolean;
desktopAgentConnected?: boolean;
desktopAgentCount?: number;
nutInstalled: boolean;
screenshotInstalled: boolean;
installInProgress: boolean;
@@ -38,6 +42,7 @@ type ComputerUseSession = {
type ComputerUsePanelProps = {
isVisible: boolean;
onShowSettings?: (tab?: SettingsMainTab) => void;
};
async function readJson<T>(response: Response): Promise<T> {
@@ -48,10 +53,36 @@ async function readJson<T>(response: Response): Promise<T> {
return data as T;
}
export default function ComputerUsePanel({ isVisible }: ComputerUsePanelProps) {
function getRuntimeTone(status: ComputerUseStatus | null, installing: boolean): string {
if (!status?.enabled) return 'border-border bg-muted text-muted-foreground';
if (status.runtime === 'cloud') {
return status.desktopAgentConnected
? 'border-primary/30 bg-primary/5 text-foreground'
: 'border-amber-500/30 bg-amber-500/10 text-amber-700 dark:text-amber-300';
}
if (status.available) return 'border-primary/30 bg-primary/5 text-foreground';
if (status.installInProgress || installing) return 'border-primary/30 bg-primary/5 text-foreground';
return 'border-border bg-background text-muted-foreground';
}
function getRuntimeLabel(status: ComputerUseStatus | null, installing: boolean): string {
if (!status?.enabled) return 'Disabled';
if (status.runtime === 'cloud') {
const count = status.desktopAgentCount ?? (status.desktopAgentConnected ? 1 : 0);
if (count > 1) return `${count} desktops linked`;
if (count === 1) return 'Desktop linked';
return 'Desktop not linked';
}
if (status.available) return 'Ready';
if (status.installInProgress || installing) return 'Installing';
return 'Setup required';
}
export default function ComputerUsePanel({ isVisible, onShowSettings }: ComputerUsePanelProps) {
const [status, setStatus] = useState<ComputerUseStatus | null>(null);
const [sessions, setSessions] = useState<ComputerUseSession[]>([]);
const [selectedSessionId, setSelectedSessionId] = useState<string | null>(null);
const [isRefreshing, setIsRefreshing] = useState(false);
const [isBusy, setIsBusy] = useState(false);
const [isInstalling, setIsInstalling] = useState(false);
const [isFullscreen, setIsFullscreen] = useState(false);
@@ -64,20 +95,25 @@ export default function ComputerUsePanel({ isVisible }: ComputerUsePanelProps) {
);
const refresh = useCallback(async () => {
setError(null);
const [statusResponse, sessionsResponse] = await Promise.all([
authenticatedFetch('/api/computer-use/status'),
authenticatedFetch('/api/computer-use/sessions'),
]);
const statusData = await readJson<{ data: ComputerUseStatus }>(statusResponse);
const sessionsData = await readJson<{ data: { sessions: ComputerUseSession[] } }>(sessionsResponse);
setStatus(statusData.data);
setSessions(sessionsData.data.sessions);
setSelectedSessionId((current) => (
current && sessionsData.data.sessions.some((session) => session.id === current)
? current
: sessionsData.data.sessions[0]?.id || null
));
setIsRefreshing(true);
try {
const [statusResponse, sessionsResponse] = await Promise.all([
authenticatedFetch('/api/computer-use/status'),
authenticatedFetch('/api/computer-use/sessions'),
]);
const statusData = await readJson<{ data: ComputerUseStatus }>(statusResponse);
const sessionsData = await readJson<{ data: { sessions: ComputerUseSession[] } }>(sessionsResponse);
setStatus(statusData.data);
setSessions(sessionsData.data.sessions);
setSelectedSessionId((current) => (
current && sessionsData.data.sessions.some((session) => session.id === current)
? current
: sessionsData.data.sessions[0]?.id || null
));
setError(null);
} finally {
setIsRefreshing(false);
}
}, []);
useEffect(() => {
@@ -207,6 +243,8 @@ export default function ComputerUsePanel({ isVisible }: ComputerUsePanelProps) {
const needsRuntime = Boolean(status?.enabled && status.runtime === 'local' && (!status.nutInstalled || !status.screenshotInstalled));
const isCloud = status?.runtime === 'cloud';
const desktopAgentCount = status?.desktopAgentCount ?? (status?.desktopAgentConnected ? 1 : 0);
const runtimeLabel = getRuntimeLabel(status, isInstalling);
const cursorStyle = selectedSession?.cursor && selectedSession.displaySize
? {
@@ -262,31 +300,68 @@ export default function ComputerUsePanel({ isVisible }: ComputerUsePanelProps) {
<div className="flex items-center gap-2">
<MonitorCog className="h-4 w-4 text-primary" />
<h3 className="text-sm font-semibold text-foreground">Computer Use</h3>
{status && <Badge variant="outline" className="text-[11px]">{status.runtime}</Badge>}
<Badge variant="outline" className={cn('text-[10px]', getRuntimeTone(status, isInstalling))}>
{runtimeLabel}
</Badge>
</div>
<p className="mt-0.5 text-xs text-muted-foreground">
{isCloud
? 'Monitor cloud agent desktop sessions and stop access when needed.'
? 'Monitor cloud agent desktop sessions and linked desktops.'
: 'Monitor local desktop sessions and grant control only when an agent needs it.'}
</p>
</div>
<div className="flex flex-wrap items-center gap-2">
<div className="flex items-center gap-1.5">
{onShowSettings && (
<Button
variant="ghost"
size="sm"
className="h-7 w-7 p-0"
onClick={() => onShowSettings('computer')}
title="Open Computer Use settings"
aria-label="Open Computer Use settings"
>
<Settings className="h-3.5 w-3.5" />
</Button>
)}
<Button
variant="outline"
variant="ghost"
size="sm"
className="h-7 w-7 p-0"
onClick={handleRefresh}
disabled={isBusy}
disabled={isRefreshing || isBusy}
title="Refresh Computer Use"
aria-label="Refresh Computer Use"
>
<RefreshCw className="h-4 w-4" />
Refresh
<RefreshCw className={cn('h-3.5 w-3.5', isRefreshing && 'animate-spin')} />
</Button>
</div>
</div>
<div className="grid min-h-0 flex-1 grid-cols-1 lg:grid-cols-[300px_minmax(0,1fr)]">
<aside className="border-b border-border/60 p-3 lg:border-b-0 lg:border-r">
{needsRuntime && (
{isCloud && (
<div className="rounded-lg border border-border/70 bg-card/40 p-3">
<div className="flex items-start justify-between gap-3">
<div className="min-w-0">
<div className="text-xs font-medium uppercase tracking-wide text-muted-foreground">Cloud desktop access</div>
<div className="mt-1 text-sm font-medium text-foreground">{runtimeLabel}</div>
</div>
<Badge variant="outline" className={cn('shrink-0 text-[10px]', getRuntimeTone(status, isInstalling))}>
{desktopAgentCount > 0 ? `${desktopAgentCount} linked` : 'Not linked'}
</Badge>
</div>
<p className="mt-2 text-xs leading-relaxed text-muted-foreground">
{desktopAgentCount > 1
? 'More than one CloudCLI Desktop app is linked. Agents will use one available desktop.'
: desktopAgentCount === 1
? 'CloudCLI Desktop is connected. Approval prompts appear on that computer.'
: 'Open CloudCLI Desktop on the computer you want agents to use, connect the same account, and enable Computer Use.'}
</p>
</div>
)}
{needsRuntime && (
<div className={cn('rounded-lg border border-border/70 bg-card/40 p-3', isCloud && 'mt-3')}>
<div className="text-xs font-medium uppercase tracking-wide text-muted-foreground">Desktop runtime required</div>
<p className="mt-2 text-xs leading-relaxed text-muted-foreground">
{status?.message || 'Install the desktop control runtime to enable Computer Use.'}

View File

@@ -264,7 +264,7 @@ function MainContent({
{shouldShowComputerTab && activeTab === 'computer' && (
<div className="h-full overflow-hidden">
<ComputerUsePanel isVisible={activeTab === 'computer'} />
<ComputerUsePanel isVisible={activeTab === 'computer'} onShowSettings={onShowSettings} />
</div>
)}

View File

@@ -1,5 +1,5 @@
import { useCallback, useEffect, useState } from 'react';
import { Download, Loader2 } from 'lucide-react';
import { Download, Loader2, RefreshCw } from 'lucide-react';
import { Button } from '../../../../../shared/view/ui';
import { authenticatedFetch } from '../../../../../utils/api';
@@ -17,6 +17,7 @@ type ComputerUseStatus = {
runtime: 'cloud' | 'local';
available: boolean;
desktopAgentConnected?: boolean;
desktopAgentCount?: number;
nutInstalled: boolean;
screenshotInstalled: boolean;
installInProgress: boolean;
@@ -51,13 +52,21 @@ export default function ComputerUseSettingsTab() {
setStatus(statusData.data);
}, []);
useEffect(() => {
const refreshState = useCallback(async () => {
setIsLoading(true);
void loadState()
.catch((err) => setError(err instanceof Error ? err.message : 'Failed to load Computer Use settings'))
.finally(() => setIsLoading(false));
try {
await loadState();
} catch (err) {
setError(err instanceof Error ? err.message : 'Failed to load Computer Use settings');
} finally {
setIsLoading(false);
}
}, [loadState]);
useEffect(() => {
void refreshState();
}, [refreshState]);
const updateSettings = async (nextSettings: Partial<ComputerUseSettings>) => {
setIsSaving(true);
setError(null);
@@ -94,9 +103,10 @@ export default function ComputerUseSettingsTab() {
const isCloud = status?.runtime === 'cloud';
const effectiveEnabled = isCloud ? status?.enabled === true : settings.enabled;
const needsRuntime = Boolean(effectiveEnabled && !isCloud && status && (!status.nutInstalled || !status.screenshotInstalled));
const desktopAgentCount = status?.desktopAgentCount ?? (status?.desktopAgentConnected ? 1 : 0);
const modeDescription = isCloud
? 'Cloud Computer Use connects a hosted agent to the CloudCLI desktop app on your machine. Agents create sessions automatically through MCP, and approval happens in the desktop app.'
: 'Local Computer Use runs on this machine. Agents create sessions automatically through MCP, but input actions require you to grant control from the Computer tab.';
? 'Let cloud agents request access to your own computer through CloudCLI Desktop.'
: 'Let local agents request access to this computer.';
return (
<div className="space-y-8">
@@ -108,14 +118,14 @@ export default function ComputerUseSettingsTab() {
<div className="flex flex-col gap-3 px-4 py-4">
<div className="rounded-md border border-amber-300/50 bg-amber-50 px-3 py-2 text-sm text-amber-800 dark:border-amber-900/50 dark:bg-amber-950/30 dark:text-amber-200">
{isCloud
? 'Computer Use can control your entire desktop after you approve the request in the CloudCLI desktop app. Use Stop in the Computer tab to end the active session.'
: 'Computer Use can control your entire desktop. Agents act only while you grant control from the Computer tab, and any action stops the moment you press Stop.'}
? 'A cloud agent can use your desktop only after you approve the request in CloudCLI Desktop. Stop ends access immediately.'
: 'Agents can use your desktop only while you grant control from the Computer tab. Stop ends access immediately.'}
</div>
{effectiveEnabled && (
<div className="rounded-md border border-border bg-muted/40 px-3 py-2 text-sm text-muted-foreground">
{isCloud
? 'When a cloud agent needs desktop access, it will create a session automatically. Keep CloudCLI Desktop running and connected to this environment to receive approval prompts.'
: 'When a local agent needs desktop access, it will create a session automatically. Open the Computer tab to review the session, grant control, or stop it. On macOS, grant Accessibility and Screen Recording to CloudCLI Desktop if prompted.'}
? 'Keep CloudCLI Desktop open on the computer you want agents to use.'
: 'Open the Computer tab to review requests, grant control, or stop a session.'}
</div>
)}
</div>
@@ -123,15 +133,31 @@ export default function ComputerUseSettingsTab() {
{isCloud ? (
<SettingsRow
label="Cloud desktop access"
description="Managed by the CloudCLI desktop app. Agents can use computer tools when a desktop agent is linked to this cloud environment."
description={status?.desktopAgentConnected
? `${desktopAgentCount} ${desktopAgentCount === 1 ? 'desktop app is' : 'desktop apps are'} connected to this environment.`
: 'Not connected yet. Link happens from CloudCLI Desktop on your computer.'}
>
<div className={`rounded-md border px-2.5 py-1 text-xs font-medium ${
status?.desktopAgentConnected
? 'border-emerald-500/30 text-emerald-600 dark:text-emerald-300'
: 'border-amber-500/30 text-amber-600 dark:text-amber-300'
}`}
>
{status?.desktopAgentConnected ? 'Desktop linked' : 'Desktop not linked'}
<div className="flex items-center gap-2">
<Button
type="button"
size="sm"
onClick={() => void refreshState()}
disabled={isLoading}
className="h-8"
>
<RefreshCw className={`h-4 w-4 ${isLoading ? 'animate-spin' : ''}`} />
Refresh
</Button>
<div className={`rounded-md border px-2.5 py-1 text-xs font-medium ${
status?.desktopAgentConnected
? 'border-emerald-500/30 text-emerald-600 dark:text-emerald-300'
: 'border-amber-500/30 text-amber-600 dark:text-amber-300'
}`}
>
{status?.desktopAgentConnected
? `${desktopAgentCount} linked`
: 'Not linked'}
</div>
</div>
</SettingsRow>
) : (
@@ -150,9 +176,23 @@ export default function ComputerUseSettingsTab() {
{(needsRuntime || isCloud || error) && (
<div className="space-y-4 px-4 py-4">
{isCloud && (
{isCloud && !status?.desktopAgentConnected && (
<div className="rounded-md border border-border bg-muted/40 px-3 py-3 text-sm text-muted-foreground">
<div className="font-medium text-foreground">To link this computer</div>
<ol className="mt-2 list-decimal space-y-1 pl-5">
<li>Open CloudCLI Desktop on the computer you want agents to use.</li>
<li>Connect the same CloudCLI account used for this cloud environment.</li>
<li>Open Desktop Settings and turn on Computer Use.</li>
<li>Keep the desktop app running. This status changes to Desktop linked automatically.</li>
</ol>
</div>
)}
{isCloud && status?.desktopAgentConnected && (
<div className="rounded-md border border-border bg-muted/40 px-3 py-2 text-sm text-muted-foreground">
{status?.message || 'Cloud Computer Use requires a linked CloudCLI Desktop Agent on the user machine.'}
{desktopAgentCount > 1
? `${desktopAgentCount} desktops are linked. Agents will use one available desktop; stop Computer Use on any desktop you do not want agents to control.`
: 'CloudCLI Desktop is linked. Approval prompts will appear there when an agent requests desktop access.'}
</div>
)}