feat: add CloudCLI computer use semantics, desktop helper packaging, and permission onboarding

This commit is contained in:
Simos Mikelatos
2026-06-19 12:09:55 +00:00
parent a35200f340
commit 1726705459
37 changed files with 3036 additions and 426 deletions

View File

@@ -421,6 +421,50 @@ svg {
gap: 12px;
}
.cc-permissions {
display: grid;
gap: 10px;
padding: 12px;
border: 1px solid var(--b-subtle);
border-radius: 12px;
background: var(--s1);
}
.cc-note {
color: var(--tx2);
font-size: 12px;
line-height: 1.45;
}
.cc-permission-row {
display: grid;
grid-template-columns: minmax(0, 1fr) auto;
gap: 12px;
align-items: center;
padding-top: 10px;
border-top: 1px solid var(--b-subtle);
}
.cc-permission-title {
color: var(--tx);
font-size: 13px;
font-weight: 600;
}
.cc-permission-detail {
margin-top: 2px;
color: var(--tx2);
font-size: 12px;
line-height: 1.4;
}
.cc-permission-actions {
display: flex;
flex-wrap: wrap;
justify-content: flex-end;
gap: 8px;
}
.cc-kv {
display: flex;
align-items: center;

View File

@@ -9,6 +9,13 @@ window.__MOCK_STATE__ = {
localServerRunning: false,
localStartupLogs: [],
computerUse: { enabled: false, consentMode: 'ask', running: false, connectedCount: 0, targetCount: 0 },
computerUsePermissions: {
platform: 'darwin',
supported: true,
accessibility: 'not_granted',
screenRecording: 'not_determined',
message: 'macOS requires Accessibility and Screen Recording for Computer Use.',
},
environments: [
{ id: 'env-api', name: 'api-gateway', subdomain: 'api-gateway', access_url: 'https://api-gateway.cloudcli.ai', status: 'running', region: 'fra1', agent: 'Claude Code' },
{ id: 'env-web', name: 'web-frontend', subdomain: 'web-frontend', access_url: 'https://web-frontend.cloudcli.ai', status: 'stopped', region: 'sfo1', agent: 'Codex' },
@@ -73,6 +80,16 @@ window.__MOCK_STATE__ = {
mockState.computerUse.running = mockState.computerUse.enabled;
return Promise.resolve(clone(mockState));
},
requestComputerUsePermission: function (permission) {
mockState.computerUsePermissions = mockState.computerUsePermissions || {};
if (permission === 'accessibility') mockState.computerUsePermissions.accessibility = 'granted';
if (permission === 'screen') mockState.computerUsePermissions.screenRecording = 'granted';
if (permission === 'all') {
mockState.computerUsePermissions.accessibility = 'granted';
mockState.computerUsePermissions.screenRecording = 'granted';
}
return Promise.resolve(clone(mockState));
},
openEnvironment: function (id) {
var env = (mockState.environments || []).filter(function (item) { return item.id === id; })[0];
if (env) {
@@ -333,6 +350,10 @@ window.__MOCK_STATE__ = {
consentMode: current.consentMode === 'auto' ? 'auto' : 'ask',
});
});
case 'computer-permission':
return CC.run('Opening permission settings...', function () {
return bridge.requestComputerUsePermission(node.getAttribute('data-cc-computer-permission'));
});
case 'settings-toggle':
return CC.run('Opening desktop settings...', function () { return bridge.showDesktopSettings(); });
case 'desktop-settings-toggle':
@@ -480,12 +501,47 @@ window.__MOCK_STATE__ = {
);
};
function permissionLabel(value) {
if (value === 'granted') return 'Granted';
if (value === 'denied' || value === 'restricted') return 'Needs attention';
if (value === 'not_applicable') return 'Not required';
return 'Not granted';
}
function permissionTone(value) {
if (value === 'granted' || value === 'not_applicable') return 'ok';
if (value === 'denied' || value === 'restricted') return 'warn';
return 'idle';
}
function renderComputerPermissionRow(key, label, detail, status) {
return '<div class="cc-permission-row">' +
'<div><div class="cc-permission-title">' + CC.esc(label) + '</div><div class="cc-permission-detail">' + CC.esc(detail) + '</div></div>' +
'<div class="cc-permission-actions"><span class="badge ' + permissionTone(status) + '">' + CC.esc(permissionLabel(status)) + '</span>' +
(status === 'granted' || status === 'not_applicable'
? ''
: '<button class="btn sm" data-cc-action="computer-permission" data-cc-computer-permission="' + CC.esc(key) + '">Open settings</button>') +
'</div>' +
'</div>';
}
function renderComputerPermissions(state) {
var permissions = state.computerUsePermissions || {};
if (!permissions.supported) {
return '<div class="cc-note">' + CC.esc(permissions.message || 'No additional OS permission setup is required from CloudCLI on this platform.') + '</div>';
}
return '<div class="cc-note">' + CC.esc(permissions.message || 'Grant the required OS permissions before approving agent control.') + '</div>' +
renderComputerPermissionRow('accessibility', 'Accessibility', 'Allows CloudCLI to click, type, and use accessibility actions.', permissions.accessibility) +
renderComputerPermissionRow('screen', 'Screen Recording', 'Allows CloudCLI to capture screenshots for agent observation.', permissions.screenRecording);
}
CC.buildComputerUseSection = function (state) {
var computerUse = state.computerUse || {};
var body =
'<div class="cc-surface">' +
'<label class="cc-toggle"><input type="checkbox" data-cc-computer-enabled="true"' + (computerUse.enabled ? ' checked' : '') + '><span><b>Enable Computer Use</b><br>Let CloudCLI use the computer. Agents cannot act until you approve a session.</span></label>';
if (computerUse.enabled) {
body += '<div class="cc-permissions">' + renderComputerPermissions(state) + '</div>';
body += '<div class="cc-choice-group">' +
CC.renderRadioOption('computer-access-mode', 'ask', computerUse.consentMode !== 'auto', 'Ask before each session', 'Agents can request control, but you approve every session.') +
CC.renderRadioOption('computer-access-mode', 'auto', computerUse.consentMode === 'auto', 'Unattended access', 'Trusted agents can use this computer without a local approval prompt.') +

View File

@@ -1,4 +1,4 @@
import { app, BrowserWindow, clipboard, dialog, ipcMain, shell } from 'electron';
import { app, BrowserWindow, clipboard, dialog, ipcMain, shell, systemPreferences } from 'electron';
import { spawn } from 'node:child_process';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -141,9 +141,64 @@ function getDesktopState() {
activeTabId: tabs.activeTabId,
environments: cloud.getEnvironments().map(serializeEnvironment),
computerUse: computerAgent?.getState() || { enabled: false, consentMode: 'ask', running: false, connectedCount: 0, targetCount: 0 },
computerUsePermissions: getComputerUsePermissions(),
};
}
function getComputerUsePermissions() {
if (process.platform !== 'darwin') {
return {
platform: process.platform,
supported: false,
accessibility: 'not_applicable',
screenRecording: 'not_applicable',
message: 'No OS permission onboarding is required from CloudCLI on this platform.',
};
}
let accessibility = 'unknown';
let screenRecording = 'unknown';
try {
accessibility = systemPreferences.isTrustedAccessibilityClient(false) ? 'granted' : 'not_granted';
} catch {
accessibility = 'unknown';
}
try {
screenRecording = systemPreferences.getMediaAccessStatus('screen');
} catch {
screenRecording = 'unknown';
}
return {
platform: 'darwin',
supported: true,
accessibility,
screenRecording,
message: accessibility === 'granted' && screenRecording === 'granted'
? 'macOS permissions are granted.'
: 'macOS requires Accessibility and Screen Recording for Computer Use.',
};
}
async function requestComputerUsePermission(permission) {
if (process.platform !== 'darwin') {
return getDesktopState();
}
if (permission === 'accessibility') {
systemPreferences.isTrustedAccessibilityClient(true);
} else if (permission === 'screen') {
await shell.openExternal('x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture');
} else if (permission === 'all') {
systemPreferences.isTrustedAccessibilityClient(true);
await shell.openExternal('x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture');
} else {
throw new Error(`Unknown Computer Use permission: ${permission}`);
}
return getDesktopState();
}
async function openExternalUrl(url) {
if (String(url).startsWith(`${CALLBACK_PROTOCOL}://`)) {
await handleDeepLink(url);
@@ -678,6 +733,7 @@ function registerIpcHandlers() {
return getDesktopState();
});
ipcMain.handle('cloudcli-desktop:update-computer-use', async (_event, settings) => updateComputerUse(settings));
ipcMain.handle('cloudcli-desktop:request-computer-use-permission', async (_event, permission) => requestComputerUsePermission(permission));
ipcMain.handle('cloudcli-desktop:show-desktop-settings', async () => desktopWindow.showDesktopSettings());
ipcMain.handle('cloudcli-desktop:show-local-settings', async () => desktopWindow.showLocalSettings());
ipcMain.handle('cloudcli-desktop:close-settings-window', async () => {

View File

@@ -17,6 +17,7 @@ if (window.location.protocol === 'file:') {
showComputerAccess: () => ipcRenderer.invoke('cloudcli-desktop:show-computer-access'),
showLocalSettings: () => ipcRenderer.invoke('cloudcli-desktop:show-local-settings'),
updateComputerUse: (settings) => ipcRenderer.invoke('cloudcli-desktop:update-computer-use', settings),
requestComputerUsePermission: (permission) => ipcRenderer.invoke('cloudcli-desktop:request-computer-use-permission', permission),
showDesktopSettings: () => ipcRenderer.invoke('cloudcli-desktop:show-desktop-settings'),
closeSettingsWindow: () => ipcRenderer.invoke('cloudcli-desktop:close-settings-window'),
showActiveEnvironmentActionsMenu: () => ipcRenderer.invoke('cloudcli-desktop:show-active-environment-actions-menu'),

View File

@@ -59,7 +59,7 @@ export class ServerInstaller {
}
getBundleName() {
return `cloudcli-server-${this.version}-${this.platform}-${this.arch}.tar.gz`;
return `cloudcli-local-server-${this.version}-${this.platform}-${this.arch}.tar.gz`;
}
getBundleUrl() {