d8dfun/claudecodeui
1
0
Fork 0
mirror of https://github.com/siteboon/claudecodeui.git synced 2026-06-21 00:22:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add CloudCLI computer use semantics, desktop helper packaging, and permission onboarding

Browse Source
This commit is contained in:
Simos Mikelatos
2026-06-19 12:09:55 +00:00
parent a35200f340
commit 1726705459
37 changed files with 3036 additions and 426 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
src/components/computer-use/view/ComputerUsePanel.tsx
View File

@@ -8,13 +8,10 @@ type ComputerUseStatus = {
enabled: boolean;
runtime: 'cloud' | 'local';
available: boolean;
requiresDesktopBridge: boolean;
nutInstalled: boolean;
screenshotInstalled: boolean;
installInProgress: boolean;
sessionCount: number;
agentToolsEnabled: boolean;
mcpRecommended: boolean;
message: string;
};
@@ -114,12 +111,6 @@ export default function ComputerUsePanel({ isVisible }: ComputerUsePanelProps) {
}
}, [refresh]);
const createSession = () => runAction(async () => {
const response = await authenticatedFetch('/api/computer-use/sessions', { method: 'POST' });
const data = await readJson<{ data: { session: ComputerUseSession } }>(response);
setSelectedSessionId(data.data.session.id);
});
const captureScreenshot = () => runAction(async () => {
if (!selectedSession) return;
const response = await authenticatedFetch(`/api/computer-use/sessions/${selectedSession.id}/screenshot`, { method: 'POST' });
@@ -252,12 +243,12 @@ export default function ComputerUsePanel({ isVisible }: ComputerUsePanelProps) {
<div className="max-w-md px-6 text-center">
<MonitorCog className="mx-auto h-10 w-10 text-neutral-500" />
<div className="mt-3 text-sm font-medium text-neutral-100">
{selectedSession?.message || 'Start a Computer Use session to capture your desktop.'}
{selectedSession?.message || 'No active Computer Use session.'}
</div>
<p className="mt-2 text-xs leading-relaxed text-neutral-400">
{isCloud
? 'Cloud Computer Use requires a linked local CloudCLI Desktop Agent.'
: 'Install the desktop control runtime from this panel or enable Computer Use from Settings.'}
? 'Agents create sessions automatically. Keep the CloudCLI desktop app connected to approve control requests.'
: 'Agents create sessions automatically. Enable Computer Use and install the local runtime if needed.'}
</p>
</div>
)}
@@ -274,7 +265,9 @@ export default function ComputerUsePanel({ isVisible }: ComputerUsePanelProps) {
{status && <Badge variant="outline" className="text-[11px]">{status.runtime}</Badge>}
</div>
<p className="mt-0.5 text-xs text-muted-foreground">
Capture your desktop and let agents drive the mouse and keyboard — only while you grant control.
{isCloud
? 'Monitor cloud agent desktop sessions and stop access when needed.'
: 'Monitor local desktop sessions and grant control only when an agent needs it.'}
</p>
</div>
<div className="flex flex-wrap items-center gap-2">
@@ -287,10 +280,6 @@ export default function ComputerUsePanel({ isVisible }: ComputerUsePanelProps) {
<RefreshCw className="h-4 w-4" />
Refresh
</Button>
<Button size="sm" onClick={createSession} disabled={isBusy || !status?.available}>
<MonitorCog className="h-4 w-4" />
New Session
</Button>
</div>
</div>
@@ -333,13 +322,20 @@ export default function ComputerUsePanel({ isVisible }: ComputerUsePanelProps) {
<ShieldCheck className="h-3.5 w-3.5" />
Safety
</div>
<p className="mt-1.5">
Agents can act on a session only while you have granted control. Use
<span className="font-medium text-foreground"> Grant Control </span>
to allow agent actions, and
<span className="font-medium text-foreground"> Stop </span>
to revoke instantly.
</p>
{isCloud ? (
<p className="mt-1.5">
Agents create sessions automatically through MCP. The CloudCLI desktop app asks for approval on this
computer, and <span className="font-medium text-foreground">Stop</span> ends the session and clears access.
</p>
) : (
<p className="mt-1.5">
Agents create sessions automatically through MCP but cannot act until you grant control here. Use
<span className="font-medium text-foreground"> Grant Control </span>
to allow agent actions, and
<span className="font-medium text-foreground"> Stop </span>
to revoke instantly.
</p>
)}
</div>
{sessions.map((session) => (
<button
@@ -373,7 +369,7 @@ export default function ComputerUsePanel({ isVisible }: ComputerUsePanelProps) {
))}
{sessions.length === 0 && (
<div className="rounded-lg border border-dashed border-border/70 px-3 py-8 text-center text-xs text-muted-foreground">
No Computer Use sessions yet.
Agents will create sessions automatically when they need desktop access.
</div>
)}
</div>
@@ -385,22 +381,22 @@ export default function ComputerUsePanel({ isVisible }: ComputerUsePanelProps) {
<Camera className="h-4 w-4" />
Screenshot
</Button>
{selectedSession?.agentAccessEnabled ? (
{!isCloud && selectedSession?.agentAccessEnabled ? (
<Button variant="outline" size="sm" onClick={revokeControl} disabled={isBusy || !selectedSession}>
<X className="h-4 w-4" />
Revoke Control
</Button>
) : (
) : !isCloud ? (
<Button
variant="outline"
size="sm"
onClick={grantControl}
disabled={isBusy || !selectedSession || selectedSession.status !== 'ready' || !status?.agentToolsEnabled}
disabled={isBusy || !selectedSession || selectedSession.status !== 'ready' || !status?.enabled}
>
<Bot className="h-4 w-4" />
Grant Control
</Button>
)}
) : null}
<Button variant="outline" size="sm" onClick={() => setIsFullscreen(true)} disabled={!selectedSession?.screenshotDataUrl}>
<Expand className="h-4 w-4" />
Full Screen
@@ -433,14 +429,16 @@ export default function ComputerUsePanel({ isVisible }: ComputerUsePanelProps) {
{selectedSession?.agentAccessEnabled && (
<span className="ml-auto inline-flex items-center gap-1 rounded border border-emerald-500/30 px-2 py-0.5 text-emerald-600 dark:text-emerald-300">
<Bot className="h-3.5 w-3.5" />
Agent control active
{isCloud ? 'Desktop-approved session' : 'Agent control active'}
</span>
)}
</div>
{renderSurface()}
</div>
<p className="mx-auto mt-2 max-w-6xl text-center text-xs text-muted-foreground">
Click the screenshot to click the real desktop. Focus the view and type to send keystrokes.
{selectedSession
? 'Click the screenshot to click the real desktop. Focus the view and type to send keystrokes.'
: 'Computer Use sessions appear here after an agent requests desktop access.'}
</p>
</div>
</main>

22
src/components/main-content/view/MainContent.tsx
View File

@@ -121,9 +121,25 @@ function MainContent({
const loadComputerUseSettings = useCallback(async () => {
try {
const response = await authenticatedFetch('/api/computer-use/settings');
const data = await response.json();
setComputerUseEnabled(Boolean(response.ok && data?.success !== false && data?.data?.settings?.enabled));
const [settingsResponse, statusResponse] = await Promise.all([
authenticatedFetch('/api/computer-use/settings'),
authenticatedFetch('/api/computer-use/status'),
]);
const settingsData = await settingsResponse.json();
const statusData = await statusResponse.json();
const runtime = statusData?.data?.runtime;
const settingsEnabled = Boolean(
settingsResponse.ok &&
settingsData?.success !== false &&
settingsData?.data?.settings?.enabled
);
const cloudEnabled = Boolean(
statusResponse.ok &&
statusData?.success !== false &&
runtime === 'cloud' &&
statusData?.data?.enabled
);
setComputerUseEnabled(runtime === 'cloud' ? cloudEnabled : settingsEnabled);
} catch {
setComputerUseEnabled(false);
}

63
src/components/settings/view/tabs/computer-use-settings/ComputerUseSettingsTab.tsx
View File

@@ -10,17 +10,16 @@ import SettingsToggle from '../../SettingsToggle';
type ComputerUseSettings = {
enabled: boolean;
agentToolsEnabled: boolean;
};
type ComputerUseStatus = {
enabled: boolean;
runtime: 'cloud' | 'local';
available: boolean;
desktopAgentConnected?: boolean;
nutInstalled: boolean;
screenshotInstalled: boolean;
installInProgress: boolean;
agentToolsEnabled: boolean;
message: string;
};
@@ -33,7 +32,7 @@ async function readJson<T>(response: Response): Promise<T> {
}
export default function ComputerUseSettingsTab() {
const [settings, setSettings] = useState<ComputerUseSettings>({ enabled: false, agentToolsEnabled: false });
const [settings, setSettings] = useState<ComputerUseSettings>({ enabled: false });
const [status, setStatus] = useState<ComputerUseStatus | null>(null);
const [isLoading, setIsLoading] = useState(true);
const [isSaving, setIsSaving] = useState(false);
@@ -93,33 +92,61 @@ export default function ComputerUseSettingsTab() {
};
const isCloud = status?.runtime === 'cloud';
const needsRuntime = Boolean(settings.enabled && !isCloud && status && (!status.nutInstalled || !status.screenshotInstalled));
const effectiveEnabled = isCloud ? status?.enabled === true : settings.enabled;
const needsRuntime = Boolean(effectiveEnabled && !isCloud && status && (!status.nutInstalled || !status.screenshotInstalled));
const modeDescription = isCloud
? 'Cloud Computer Use connects a hosted agent to the CloudCLI desktop app on your machine. Agents create sessions automatically through MCP, and approval happens in the desktop app.'
: 'Local Computer Use runs on this machine. Agents create sessions automatically through MCP, but input actions require you to grant control from the Computer tab.';
return (
<div className="space-y-8">
<SettingsSection
title="Computer Use"
description="Let agents see your desktop and drive the mouse and keyboard through a guarded, consent-gated control loop."
description={modeDescription}
>
<SettingsCard divided>
<div className="flex flex-col gap-3 px-4 py-4">
<div className="rounded-md border border-amber-300/50 bg-amber-50 px-3 py-2 text-sm text-amber-800 dark:border-amber-900/50 dark:bg-amber-950/30 dark:text-amber-200">
Computer Use can control your entire desktop. Agents act only while you grant control from the
Computer panel, and any action stops the moment you press Stop.
{isCloud
? 'Computer Use can control your entire desktop after you approve the request in the CloudCLI desktop app. Use Stop in the Computer tab to end the active session.'
: 'Computer Use can control your entire desktop. Agents act only while you grant control from the Computer tab, and any action stops the moment you press Stop.'}
</div>
{effectiveEnabled && (
<div className="rounded-md border border-border bg-muted/40 px-3 py-2 text-sm text-muted-foreground">
{isCloud
? 'When a cloud agent needs desktop access, it will create a session automatically. Keep CloudCLI Desktop running and connected to this environment to receive approval prompts.'
: 'When a local agent needs desktop access, it will create a session automatically. Open the Computer tab to review the session, grant control, or stop it. On macOS, grant Accessibility and Screen Recording to CloudCLI Desktop if prompted.'}
</div>
)}
</div>
<SettingsRow
label="Enable Computer Use"
description="Registers Computer Use for supported agents and allows CloudCLI to create guarded desktop control sessions on this machine."
>
<SettingsToggle
checked={settings.enabled}
onChange={(value) => void updateSettings({ enabled: value, agentToolsEnabled: value })}
ariaLabel="Enable Computer Use"
disabled={isLoading || isSaving}
/>
</SettingsRow>
{isCloud ? (
<SettingsRow
label="Cloud desktop access"
description="Managed by the CloudCLI desktop app. Agents can use computer tools when a desktop agent is linked to this cloud environment."
>
<div className={`rounded-md border px-2.5 py-1 text-xs font-medium ${
status?.desktopAgentConnected
? 'border-emerald-500/30 text-emerald-600 dark:text-emerald-300'
: 'border-amber-500/30 text-amber-600 dark:text-amber-300'
}`}
>
{status?.desktopAgentConnected ? 'Desktop linked' : 'Desktop not linked'}
</div>
</SettingsRow>
) : (
<SettingsRow
label="Enable Computer Use"
description="Registers Computer Use for supported agents and allows CloudCLI to create guarded desktop control sessions on this machine."
>
<SettingsToggle
checked={settings.enabled}
onChange={(value) => void updateSettings({ enabled: value })}
ariaLabel="Enable Computer Use"
disabled={isLoading || isSaving}
/>
</SettingsRow>
)}
{(needsRuntime || isCloud || error) && (
<div className="space-y-4 px-4 py-4">