Harden desktop workflows and computer use handling

This commit is contained in:
Simos Mikelatos
2026-06-19 06:21:13 +00:00
parent 531833bc87
commit 2af3d38afe
13 changed files with 73 additions and 26 deletions

View File

@@ -141,14 +141,23 @@ async function runAction(type: string, params: Record<string, unknown>): Promise
return { ...(await snapshot(target)), position, cursor: position };
}
case 'mouse_move':
await executor.moveTo(target, point as Point);
if (!point) {
throw new Error('mouse_move requires a valid point.');
}
await executor.moveTo(target, point);
return { ...(await snapshot(target)), cursor: point };
case 'click':
await executor.click(target, (params.button as ClickButton) || 'left', point, params.double === true);
return { ...(await snapshot(target)), cursor: point ?? null };
case 'drag':
await executor.drag(target, asPoint(params.from) as Point, asPoint(params.to) as Point, (params.button as ClickButton) || 'left');
return { ...(await snapshot(target)), cursor: asPoint(params.to) ?? null };
case 'drag': {
const from = asPoint(params.from);
const to = asPoint(params.to);
if (!from || !to) {
throw new Error('drag requires valid from and to points.');
}
await executor.drag(target, from, to, (params.button as ClickButton) || 'left');
return { ...(await snapshot(target)), cursor: to };
}
case 'type':
await executor.type(String(params.text ?? ''));
return snapshot(target);

View File

@@ -376,12 +376,13 @@ process.stdin.on('data', (chunk) => {
buffer = buffer.slice(messageEnd);
void (async () => {
const request = JSON.parse(rawMessage) as JsonRpcRequest;
let request: JsonRpcRequest | null = null;
try {
request = JSON.parse(rawMessage) as JsonRpcRequest;
const result = await handleMessage(request);
sendResult(request.id, result);
} catch (error) {
sendError(request.id, error);
sendError(request?.id ?? null, error);
}
})();
}

View File

@@ -125,9 +125,19 @@ router.post('/sessions/:sessionId/screenshot', async (req: AuthenticatedRequest,
router.post('/sessions/:sessionId/click', async (req: AuthenticatedRequest, res) => {
try {
const x = Number(req.body?.x);
const y = Number(req.body?.y);
if (!Number.isFinite(x) || !Number.isFinite(y)) {
res.status(400).json({
success: false,
error: 'Valid numeric coordinates are required.',
});
return;
}
const session = await computerUseService.userClick(requireUser(req), readParam(req.params.sessionId), {
x: Number(req.body?.x),
y: Number(req.body?.y),
x,
y,
button: toButton(req.body?.button),
double: req.body?.double === true,
});

View File

@@ -1,4 +1,3 @@
import { createRequire } from 'node:module';
import { randomBytes, randomUUID } from 'node:crypto';
import { spawn } from 'node:child_process';
import fs from 'node:fs';