fix(browser-use): support visible Camoufox on desktop platforms

This commit is contained in:
Simos Mikelatos
2026-07-09 20:14:49 +00:00
parent f7171be383
commit 1dfca21558
3 changed files with 108 additions and 49 deletions

View File

@@ -49,6 +49,9 @@ const MCP_SERVER_NAME = 'cloudcli-browser';
const LEGACY_MCP_SERVER_NAMES = ['cloudcli-browser-use']; const LEGACY_MCP_SERVER_NAMES = ['cloudcli-browser-use'];
const RUNTIME_READINESS_CACHE_TTL_MS = 30_000; const RUNTIME_READINESS_CACHE_TTL_MS = 30_000;
const VISIBLE_BROWSER_ENABLED = process.env.CLOUDCLI_BROWSER_USE_VISIBLE !== 'false'; const VISIBLE_BROWSER_ENABLED = process.env.CLOUDCLI_BROWSER_USE_VISIBLE !== 'false';
// The noVNC viewer pipeline (Xvfb + x11vnc + websockify) only exists on Linux.
// On other platforms visible Camoufox sessions open a normal window on the host desktop.
const VNC_VIEWER_SUPPORTED = process.platform === 'linux';
const RUNTIME_ROOT = process.env.CLOUDCLI_BROWSER_USE_RUNTIME_ROOT || '/opt/claudecodeui/.runtime-browser'; const RUNTIME_ROOT = process.env.CLOUDCLI_BROWSER_USE_RUNTIME_ROOT || '/opt/claudecodeui/.runtime-browser';
const NOVNC_ROOT = process.env.CLOUDCLI_BROWSER_USE_NOVNC_ROOT || path.join(RUNTIME_ROOT, 'novnc'); const NOVNC_ROOT = process.env.CLOUDCLI_BROWSER_USE_NOVNC_ROOT || path.join(RUNTIME_ROOT, 'novnc');
const X11VNC_BIN = process.env.CLOUDCLI_BROWSER_USE_X11VNC_BIN || path.join(RUNTIME_ROOT, 'rootfs/usr/bin/x11vnc'); const X11VNC_BIN = process.env.CLOUDCLI_BROWSER_USE_X11VNC_BIN || path.join(RUNTIME_ROOT, 'rootfs/usr/bin/x11vnc');
@@ -61,24 +64,51 @@ function getRuntime(): 'cloud' | 'local' {
return IS_PLATFORM ? 'cloud' : 'local'; return IS_PLATFORM ? 'cloud' : 'local';
} }
const CAMOUFOX_CLI_CANDIDATES = [
path.join(os.homedir(), '.local/bin/camoufox'),
// PATH lookup covers pipx on Linux/macOS and the pip Scripts dir on Windows.
'camoufox',
];
const CAMOUFOX_BINARY_CANDIDATES = process.platform === 'win32'
? ['camoufox.exe', 'camoufox']
: process.platform === 'darwin'
? ['Camoufox.app/Contents/MacOS/camoufox', 'camoufox']
: ['camoufox'];
function findCamoufoxBinary(installPath: string): string | null {
if (!fs.statSync(installPath).isDirectory()) {
return fs.existsSync(installPath) ? installPath : null;
}
for (const candidate of CAMOUFOX_BINARY_CANDIDATES) {
const executablePath = path.join(installPath, candidate);
if (fs.existsSync(executablePath)) {
return executablePath;
}
}
return null;
}
function getCamoufoxExecutablePath(): string | null { function getCamoufoxExecutablePath(): string | null {
const configured = process.env.CLOUDCLI_BROWSER_USE_CAMOUFOX_EXECUTABLE; const configured = process.env.CLOUDCLI_BROWSER_USE_CAMOUFOX_EXECUTABLE;
if (configured && fs.existsSync(configured)) { if (configured && fs.existsSync(configured)) {
return configured; return findCamoufoxBinary(configured);
} }
try { for (const cli of CAMOUFOX_CLI_CANDIDATES) {
const output = execFileSync(path.join(os.homedir(), '.local/bin/camoufox'), ['path'], { try {
encoding: 'utf8', const output = execFileSync(cli, ['path'], {
stdio: ['ignore', 'pipe', 'ignore'], encoding: 'utf8',
}).trim(); stdio: ['ignore', 'pipe', 'ignore'],
const executablePath = fs.statSync(output).isDirectory() }).trim();
? path.join(output, 'camoufox') const executablePath = findCamoufoxBinary(output);
: output; if (executablePath) {
return fs.existsSync(executablePath) ? executablePath : null; return executablePath;
} catch { }
return null; } catch {
// Try the next candidate.
}
} }
return null;
} }
function getSetupMessage(settings: BrowserUseSettings, readiness: RuntimeReadiness): string { function getSetupMessage(settings: BrowserUseSettings, readiness: RuntimeReadiness): string {
@@ -101,11 +131,13 @@ function getSetupMessage(settings: BrowserUseSettings, readiness: RuntimeReadine
if (!getCamoufoxExecutablePath()) { if (!getCamoufoxExecutablePath()) {
return 'Camoufox is selected, but Camoufox is not installed.'; return 'Camoufox is selected, but Camoufox is not installed.';
} }
if (!fs.existsSync(X11VNC_BIN)) { if (VNC_VIEWER_SUPPORTED) {
return 'Camoufox is selected, but x11vnc is missing.'; if (!fs.existsSync(X11VNC_BIN)) {
} return 'Camoufox is selected, but x11vnc is missing.';
if (!fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html'))) { }
return 'Camoufox is selected, but noVNC is missing.'; if (!fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html'))) {
return 'Camoufox is selected, but noVNC is missing.';
}
} }
return readiness.installMessage || 'Camoufox runtime is not ready.'; return readiness.installMessage || 'Camoufox runtime is not ready.';
} }
@@ -117,6 +149,16 @@ function getSetupMessage(settings: BrowserUseSettings, readiness: RuntimeReadine
return readiness.installMessage || 'Browser runtime is not ready.'; return readiness.installMessage || 'Browser runtime is not ready.';
} }
function isVncRuntimeInstalled(): boolean {
return fs.existsSync(X11VNC_BIN) && fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html'));
}
function isVisibleCamoufoxReady(): boolean {
return VISIBLE_BROWSER_ENABLED
&& Boolean(getCamoufoxExecutablePath())
&& (!VNC_VIEWER_SUPPORTED || isVncRuntimeInstalled());
}
function getPlaywright(): any | null { function getPlaywright(): any | null {
try { try {
return require('playwright'); return require('playwright');
@@ -712,11 +754,8 @@ export const browserUseService = {
const readiness = getRuntimeReadiness(); const readiness = getRuntimeReadiness();
const useVisibleBackend = useVisibleCamoufoxBackend(settings); const useVisibleBackend = useVisibleCamoufoxBackend(settings);
const visibleCamoufoxReady = useVisibleBackend const visibleCamoufoxReady = useVisibleBackend
&& VISIBLE_BROWSER_ENABLED
&& readiness.playwrightInstalled && readiness.playwrightInstalled
&& Boolean(getCamoufoxExecutablePath()) && isVisibleCamoufoxReady();
&& fs.existsSync(X11VNC_BIN)
&& fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html'));
const available = settings.enabled const available = settings.enabled
&& readiness.playwrightInstalled && readiness.playwrightInstalled
&& (useVisibleBackend ? visibleCamoufoxReady : readiness.chromiumInstalled); && (useVisibleBackend ? visibleCamoufoxReady : readiness.chromiumInstalled);
@@ -726,6 +765,7 @@ export const browserUseService = {
runtime: getRuntime(), runtime: getRuntime(),
backend: useVisibleBackend ? 'camoufox-vnc' : 'playwright', backend: useVisibleBackend ? 'camoufox-vnc' : 'playwright',
browserBackend: settings.browserBackend, browserBackend: settings.browserBackend,
viewerMode: VNC_VIEWER_SUPPORTED ? 'novnc' : 'window',
available, available,
playwrightInstalled: readiness.playwrightInstalled, playwrightInstalled: readiness.playwrightInstalled,
chromiumInstalled: readiness.chromiumInstalled, chromiumInstalled: readiness.chromiumInstalled,
@@ -821,11 +861,7 @@ export const browserUseService = {
const readiness = getRuntimeReadiness(); const readiness = getRuntimeReadiness();
const useVisibleBackend = useVisibleCamoufoxBackend(settings); const useVisibleBackend = useVisibleCamoufoxBackend(settings);
const visibleCamoufoxReady = useVisibleBackend const visibleCamoufoxReady = useVisibleBackend && isVisibleCamoufoxReady();
&& VISIBLE_BROWSER_ENABLED
&& Boolean(getCamoufoxExecutablePath())
&& fs.existsSync(X11VNC_BIN)
&& fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html'));
if (!settings.enabled || !readiness.playwrightInstalled || !readiness.playwright || (useVisibleBackend ? !visibleCamoufoxReady : !readiness.chromiumInstalled)) { if (!settings.enabled || !readiness.playwrightInstalled || !readiness.playwright || (useVisibleBackend ? !visibleCamoufoxReady : !readiness.chromiumInstalled)) {
session.message = getSetupMessage(settings, readiness); session.message = getSetupMessage(settings, readiness);
sessions.set(session.id, session); sessions.set(session.id, session);
@@ -854,25 +890,31 @@ export const browserUseService = {
if (!camoufoxExecutable) { if (!camoufoxExecutable) {
throw new Error('Camoufox is not installed.'); throw new Error('Camoufox is not installed.');
} }
const runtime = await startVisibleRuntime();
viewer = {
display: runtime.display,
vncPort: runtime.vncPort,
websockifyPort: runtime.websockifyPort,
noVncRoot: runtime.noVncRoot,
};
processes = runtime.processes;
launchOptions.executablePath = camoufoxExecutable; launchOptions.executablePath = camoufoxExecutable;
launchOptions.env = {
...process.env,
DISPLAY: runtime.display,
LD_LIBRARY_PATH: `${X11VNC_LIB_DIR}:${X11VNC_EXTRA_LIB_DIR}:${process.env.LD_LIBRARY_PATH || ''}`,
};
launchOptions.args = []; launchOptions.args = [];
session.backend = 'camoufox-vnc'; session.backend = 'camoufox-vnc';
const viewerToken = createViewerToken(session.id);
session.viewerUrl = getViewerUrl(session.id, viewerToken); if (VNC_VIEWER_SUPPORTED) {
session.viewerEmbedUrl = session.viewerUrl; const runtime = await startVisibleRuntime();
viewer = {
display: runtime.display,
vncPort: runtime.vncPort,
websockifyPort: runtime.websockifyPort,
noVncRoot: runtime.noVncRoot,
};
processes = runtime.processes;
launchOptions.env = {
...process.env,
DISPLAY: runtime.display,
LD_LIBRARY_PATH: `${X11VNC_LIB_DIR}:${X11VNC_EXTRA_LIB_DIR}:${process.env.LD_LIBRARY_PATH || ''}`,
};
const viewerToken = createViewerToken(session.id);
session.viewerUrl = getViewerUrl(session.id, viewerToken);
session.viewerEmbedUrl = session.viewerUrl;
} else {
// Without a VNC pipeline the browser opens as a normal window on the host desktop.
session.message = 'Browser window is open on the machine running CloudCLI.';
}
} }
if (profileName) { if (profileName) {
@@ -896,7 +938,7 @@ export const browserUseService = {
throw error; throw error;
} }
session.status = 'ready'; session.status = 'ready';
session.message = 'Browser session is ready.'; session.message = session.message || 'Browser session is ready.';
sessions.set(session.id, session); sessions.set(session.id, session);
handles.set(session.id, { browser, context, page, processes, viewer }); handles.set(session.id, { browser, context, page, processes, viewer });
await captureSession(session, page); await captureSession(session, page);

View File

@@ -28,6 +28,7 @@ type BrowserUseStatus = {
available: boolean; available: boolean;
backend: 'playwright' | 'camoufox-vnc'; backend: 'playwright' | 'camoufox-vnc';
browserBackend: 'playwright' | 'camoufox-vnc'; browserBackend: 'playwright' | 'camoufox-vnc';
viewerMode?: 'novnc' | 'window';
playwrightInstalled: boolean; playwrightInstalled: boolean;
chromiumInstalled: boolean; chromiumInstalled: boolean;
installInProgress: boolean; installInProgress: boolean;
@@ -205,6 +206,7 @@ export default function BrowserUsePanel({ isVisible, projectId, onShowSettings }
const isInitialLoading = isRefreshing && !hasLoadedOnce && sessions.length === 0; const isInitialLoading = isRefreshing && !hasLoadedOnce && sessions.length === 0;
const isBackgroundRefreshing = isRefreshing && !isInitialLoading; const isBackgroundRefreshing = isRefreshing && !isInitialLoading;
const needsBrowserBinaries = Boolean(status?.enabled && !status.available); const needsBrowserBinaries = Boolean(status?.enabled && !status.available);
const usesLocalWindowViewer = status?.viewerMode === 'window';
const runtimeLabel = isInitialLoading const runtimeLabel = isInitialLoading
? 'Loading' ? 'Loading'
: !status?.enabled : !status?.enabled
@@ -625,6 +627,14 @@ export default function BrowserUsePanel({ isVisible, projectId, onShowSettings }
Take control Take control
</Button> </Button>
)} )}
{usesLocalWindowViewer && selectedSession?.backend === 'camoufox-vnc' && !selectedSession.viewerUrl && selectedSession.status === 'ready' && (
<span
className="hidden rounded border border-border/70 bg-muted/30 px-2 py-1 text-[10px] text-muted-foreground md:inline"
title="This visible session runs as a browser window on the machine running CloudCLI"
>
Window on host
</span>
)}
<Button variant="ghost" size="sm" className="h-8 w-8 p-0" onClick={() => setIsFullscreen(true)} disabled={!selectedSession?.screenshotDataUrl} title="Full screen" aria-label="Full screen"> <Button variant="ghost" size="sm" className="h-8 w-8 p-0" onClick={() => setIsFullscreen(true)} disabled={!selectedSession?.screenshotDataUrl} title="Full screen" aria-label="Full screen">
<Expand className="h-4 w-4" /> <Expand className="h-4 w-4" />
</Button> </Button>

View File

@@ -22,6 +22,7 @@ type BrowserUseStatus = {
available: boolean; available: boolean;
backend: 'playwright' | 'camoufox-vnc'; backend: 'playwright' | 'camoufox-vnc';
browserBackend: 'playwright' | 'camoufox-vnc'; browserBackend: 'playwright' | 'camoufox-vnc';
viewerMode?: 'novnc' | 'window';
playwrightInstalled: boolean; playwrightInstalled: boolean;
chromiumInstalled: boolean; chromiumInstalled: boolean;
camoufoxInstalled: boolean; camoufoxInstalled: boolean;
@@ -131,6 +132,8 @@ export default function BrowserUseSettingsTab() {
const persistSessions = settings?.persistSessions === true; const persistSessions = settings?.persistSessions === true;
const selectedBackend = settings?.browserBackend || 'playwright'; const selectedBackend = settings?.browserBackend || 'playwright';
const effectiveBackend = status?.backend || 'playwright'; const effectiveBackend = status?.backend || 'playwright';
const usesLocalWindowViewer = status?.viewerMode === 'window';
const camoufoxLabel = usesLocalWindowViewer ? 'Camoufox (visible window)' : 'Camoufox + noVNC';
const needsBrowserBinaries = Boolean(browserEnabled && status && !status.available); const needsBrowserBinaries = Boolean(browserEnabled && status && !status.available);
const runtimeLabel = (installed?: boolean) => { const runtimeLabel = (installed?: boolean) => {
if (isStatusLoading && !status) { if (isStatusLoading && !status) {
@@ -205,8 +208,10 @@ export default function BrowserUseSettingsTab() {
}, },
{ {
value: 'camoufox-vnc' as const, value: 'camoufox-vnc' as const,
label: 'Camoufox + noVNC', label: camoufoxLabel,
description: 'Best when a person may need to log in, approve a step, or watch the browser session live.', description: usesLocalWindowViewer
? 'Best when a person may need to log in or approve a step. The browser opens as a window on the machine running CloudCLI.'
: 'Best when a person may need to log in, approve a step, or watch the browser session live.',
icon: Eye, icon: Eye,
}, },
]).map((option) => { ]).map((option) => {
@@ -288,7 +293,7 @@ export default function BrowserUseSettingsTab() {
<div className="space-y-4 px-4 py-4"> <div className="space-y-4 px-4 py-4">
<div className="flex flex-wrap gap-2 text-xs text-muted-foreground"> <div className="flex flex-wrap gap-2 text-xs text-muted-foreground">
<span className="rounded-md border border-border px-2 py-1"> <span className="rounded-md border border-border px-2 py-1">
Backend: {effectiveBackend === 'camoufox-vnc' ? 'Camoufox + noVNC' : 'Playwright'} Backend: {effectiveBackend === 'camoufox-vnc' ? camoufoxLabel : 'Playwright'}
</span> </span>
<span className="rounded-md border border-border px-2 py-1"> <span className="rounded-md border border-border px-2 py-1">
Playwright: {runtimeLabel(status?.playwrightInstalled)} Playwright: {runtimeLabel(status?.playwrightInstalled)}
@@ -299,9 +304,11 @@ export default function BrowserUseSettingsTab() {
<span className="rounded-md border border-border px-2 py-1"> <span className="rounded-md border border-border px-2 py-1">
Camoufox: {runtimeLabel(status?.camoufoxInstalled)} Camoufox: {runtimeLabel(status?.camoufoxInstalled)}
</span> </span>
<span className="rounded-md border border-border px-2 py-1"> {!usesLocalWindowViewer && (
noVNC: {runtimeLabel(status?.noVncInstalled)} <span className="rounded-md border border-border px-2 py-1">
</span> noVNC: {runtimeLabel(status?.noVncInstalled)}
</span>
)}
<span className="rounded-md border border-border px-2 py-1"> <span className="rounded-md border border-border px-2 py-1">
Status: {isStatusLoading && !status ? 'checking...' : status?.available ? 'ready' : browserEnabled ? 'setup required' : 'disabled'} Status: {isStatusLoading && !status ? 'checking...' : status?.available ? 'ready' : browserEnabled ? 'setup required' : 'disabled'}
</span> </span>