fix(browser-use): support visible Camoufox on desktop platforms

2026-07-11 00:45:42 +08:00 · 2026-07-09 20:14:49 +00:00
parent f7171be383
commit 1dfca21558
3 changed files with 108 additions and 49 deletions
--- a/server/modules/browser-use/browser-use.service.ts
+++ b/server/modules/browser-use/browser-use.service.ts
@@ -49,6 +49,9 @@ const MCP_SERVER_NAME = 'cloudcli-browser';
 const LEGACY_MCP_SERVER_NAMES = ['cloudcli-browser-use'];
 const RUNTIME_READINESS_CACHE_TTL_MS = 30_000;
 const VISIBLE_BROWSER_ENABLED = process.env.CLOUDCLI_BROWSER_USE_VISIBLE !== 'false';
+// The noVNC viewer pipeline (Xvfb + x11vnc + websockify) only exists on Linux.
+// On other platforms visible Camoufox sessions open a normal window on the host desktop.
+const VNC_VIEWER_SUPPORTED = process.platform === 'linux';
 const RUNTIME_ROOT = process.env.CLOUDCLI_BROWSER_USE_RUNTIME_ROOT || '/opt/claudecodeui/.runtime-browser';
 const NOVNC_ROOT = process.env.CLOUDCLI_BROWSER_USE_NOVNC_ROOT || path.join(RUNTIME_ROOT, 'novnc');
 const X11VNC_BIN = process.env.CLOUDCLI_BROWSER_USE_X11VNC_BIN || path.join(RUNTIME_ROOT, 'rootfs/usr/bin/x11vnc');
@@ -61,24 +64,51 @@ function getRuntime(): 'cloud' | 'local' {
  return IS_PLATFORM ? 'cloud' : 'local';
 }

+const CAMOUFOX_CLI_CANDIDATES = [
+  path.join(os.homedir(), '.local/bin/camoufox'),
+  // PATH lookup covers pipx on Linux/macOS and the pip Scripts dir on Windows.
+  'camoufox',
+];
+const CAMOUFOX_BINARY_CANDIDATES = process.platform === 'win32'
+  ? ['camoufox.exe', 'camoufox']
+  : process.platform === 'darwin'
+    ? ['Camoufox.app/Contents/MacOS/camoufox', 'camoufox']
+    : ['camoufox'];
+
+function findCamoufoxBinary(installPath: string): string | null {
+  if (!fs.statSync(installPath).isDirectory()) {
+    return fs.existsSync(installPath) ? installPath : null;
+  }
+  for (const candidate of CAMOUFOX_BINARY_CANDIDATES) {
+    const executablePath = path.join(installPath, candidate);
+    if (fs.existsSync(executablePath)) {
+      return executablePath;
+    }
+  }
+  return null;
+}
+
 function getCamoufoxExecutablePath(): string | null {
  const configured = process.env.CLOUDCLI_BROWSER_USE_CAMOUFOX_EXECUTABLE;
  if (configured && fs.existsSync(configured)) {
-    return configured;
+    return findCamoufoxBinary(configured);
  }

-  try {
-    const output = execFileSync(path.join(os.homedir(), '.local/bin/camoufox'), ['path'], {
-      encoding: 'utf8',
-      stdio: ['ignore', 'pipe', 'ignore'],
-    }).trim();
-    const executablePath = fs.statSync(output).isDirectory()
-      ? path.join(output, 'camoufox')
-      : output;
-    return fs.existsSync(executablePath) ? executablePath : null;
-  } catch {
-    return null;
+  for (const cli of CAMOUFOX_CLI_CANDIDATES) {
+    try {
+      const output = execFileSync(cli, ['path'], {
+        encoding: 'utf8',
+        stdio: ['ignore', 'pipe', 'ignore'],
+      }).trim();
+      const executablePath = findCamoufoxBinary(output);
+      if (executablePath) {
+        return executablePath;
+      }
+    } catch {
+      // Try the next candidate.
+    }
  }
+  return null;
 }

 function getSetupMessage(settings: BrowserUseSettings, readiness: RuntimeReadiness): string {
@@ -101,11 +131,13 @@ function getSetupMessage(settings: BrowserUseSettings, readiness: RuntimeReadine
    if (!getCamoufoxExecutablePath()) {
      return 'Camoufox is selected, but Camoufox is not installed.';
    }
-    if (!fs.existsSync(X11VNC_BIN)) {
-      return 'Camoufox is selected, but x11vnc is missing.';
-    }
-    if (!fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html'))) {
-      return 'Camoufox is selected, but noVNC is missing.';
+    if (VNC_VIEWER_SUPPORTED) {
+      if (!fs.existsSync(X11VNC_BIN)) {
+        return 'Camoufox is selected, but x11vnc is missing.';
+      }
+      if (!fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html'))) {
+        return 'Camoufox is selected, but noVNC is missing.';
+      }
    }
    return readiness.installMessage || 'Camoufox runtime is not ready.';
  }
@@ -117,6 +149,16 @@ function getSetupMessage(settings: BrowserUseSettings, readiness: RuntimeReadine
  return readiness.installMessage || 'Browser runtime is not ready.';
 }

+function isVncRuntimeInstalled(): boolean {
+  return fs.existsSync(X11VNC_BIN) && fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html'));
+}
+
+function isVisibleCamoufoxReady(): boolean {
+  return VISIBLE_BROWSER_ENABLED
+    && Boolean(getCamoufoxExecutablePath())
+    && (!VNC_VIEWER_SUPPORTED || isVncRuntimeInstalled());
+}
+
 function getPlaywright(): any | null {
  try {
    return require('playwright');
@@ -712,11 +754,8 @@ export const browserUseService = {
    const readiness = getRuntimeReadiness();
    const useVisibleBackend = useVisibleCamoufoxBackend(settings);
    const visibleCamoufoxReady = useVisibleBackend
-      && VISIBLE_BROWSER_ENABLED
      && readiness.playwrightInstalled
-      && Boolean(getCamoufoxExecutablePath())
-      && fs.existsSync(X11VNC_BIN)
-      && fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html'));
+      && isVisibleCamoufoxReady();
    const available = settings.enabled
      && readiness.playwrightInstalled
      && (useVisibleBackend ? visibleCamoufoxReady : readiness.chromiumInstalled);
@@ -726,6 +765,7 @@ export const browserUseService = {
      runtime: getRuntime(),
      backend: useVisibleBackend ? 'camoufox-vnc' : 'playwright',
      browserBackend: settings.browserBackend,
+      viewerMode: VNC_VIEWER_SUPPORTED ? 'novnc' : 'window',
      available,
      playwrightInstalled: readiness.playwrightInstalled,
      chromiumInstalled: readiness.chromiumInstalled,
@@ -821,11 +861,7 @@ export const browserUseService = {

    const readiness = getRuntimeReadiness();
    const useVisibleBackend = useVisibleCamoufoxBackend(settings);
-    const visibleCamoufoxReady = useVisibleBackend
-      && VISIBLE_BROWSER_ENABLED
-      && Boolean(getCamoufoxExecutablePath())
-      && fs.existsSync(X11VNC_BIN)
-      && fs.existsSync(path.join(NOVNC_ROOT, 'vnc.html'));
+    const visibleCamoufoxReady = useVisibleBackend && isVisibleCamoufoxReady();
    if (!settings.enabled || !readiness.playwrightInstalled || !readiness.playwright || (useVisibleBackend ? !visibleCamoufoxReady : !readiness.chromiumInstalled)) {
      session.message = getSetupMessage(settings, readiness);
      sessions.set(session.id, session);
@@ -854,25 +890,31 @@ export const browserUseService = {
        if (!camoufoxExecutable) {
          throw new Error('Camoufox is not installed.');
        }
-        const runtime = await startVisibleRuntime();
-        viewer = {
-          display: runtime.display,
-          vncPort: runtime.vncPort,
-          websockifyPort: runtime.websockifyPort,
-          noVncRoot: runtime.noVncRoot,
-        };
-        processes = runtime.processes;
        launchOptions.executablePath = camoufoxExecutable;
-        launchOptions.env = {
-          ...process.env,
-          DISPLAY: runtime.display,
-          LD_LIBRARY_PATH: `${X11VNC_LIB_DIR}:${X11VNC_EXTRA_LIB_DIR}:${process.env.LD_LIBRARY_PATH || ''}`,
-        };
        launchOptions.args = [];
        session.backend = 'camoufox-vnc';
-        const viewerToken = createViewerToken(session.id);
-        session.viewerUrl = getViewerUrl(session.id, viewerToken);
-        session.viewerEmbedUrl = session.viewerUrl;
+
+        if (VNC_VIEWER_SUPPORTED) {
+          const runtime = await startVisibleRuntime();
+          viewer = {
+            display: runtime.display,
+            vncPort: runtime.vncPort,
+            websockifyPort: runtime.websockifyPort,
+            noVncRoot: runtime.noVncRoot,
+          };
+          processes = runtime.processes;
+          launchOptions.env = {
+            ...process.env,
+            DISPLAY: runtime.display,
+            LD_LIBRARY_PATH: `${X11VNC_LIB_DIR}:${X11VNC_EXTRA_LIB_DIR}:${process.env.LD_LIBRARY_PATH || ''}`,
+          };
+          const viewerToken = createViewerToken(session.id);
+          session.viewerUrl = getViewerUrl(session.id, viewerToken);
+          session.viewerEmbedUrl = session.viewerUrl;
+        } else {
+          // Without a VNC pipeline the browser opens as a normal window on the host desktop.
+          session.message = 'Browser window is open on the machine running CloudCLI.';
+        }
      }

      if (profileName) {
@@ -896,7 +938,7 @@ export const browserUseService = {
      throw error;
    }
    session.status = 'ready';
-    session.message = 'Browser session is ready.';
+    session.message = session.message || 'Browser session is ready.';
    sessions.set(session.id, session);
    handles.set(session.id, { browser, context, page, processes, viewer });
    await captureSession(session, page);
--- a/src/components/browser-use/view/BrowserUsePanel.tsx
+++ b/src/components/browser-use/view/BrowserUsePanel.tsx
@@ -28,6 +28,7 @@ type BrowserUseStatus = {
  available: boolean;
  backend: 'playwright' | 'camoufox-vnc';
  browserBackend: 'playwright' | 'camoufox-vnc';
+  viewerMode?: 'novnc' | 'window';
  playwrightInstalled: boolean;
  chromiumInstalled: boolean;
  installInProgress: boolean;
@@ -205,6 +206,7 @@ export default function BrowserUsePanel({ isVisible, projectId, onShowSettings }
  const isInitialLoading = isRefreshing && !hasLoadedOnce && sessions.length === 0;
  const isBackgroundRefreshing = isRefreshing && !isInitialLoading;
  const needsBrowserBinaries = Boolean(status?.enabled && !status.available);
+  const usesLocalWindowViewer = status?.viewerMode === 'window';
  const runtimeLabel = isInitialLoading
    ? 'Loading'
    : !status?.enabled
@@ -625,6 +627,14 @@ export default function BrowserUsePanel({ isVisible, projectId, onShowSettings }
                      Take control
                    </Button>
                  )}
+                  {usesLocalWindowViewer && selectedSession?.backend === 'camoufox-vnc' && !selectedSession.viewerUrl && selectedSession.status === 'ready' && (
+                    <span
+                      className="hidden rounded border border-border/70 bg-muted/30 px-2 py-1 text-[10px] text-muted-foreground md:inline"
+                      title="This visible session runs as a browser window on the machine running CloudCLI"
+                    >
+                      Window on host
+                    </span>
+                  )}
                  <Button variant="ghost" size="sm" className="h-8 w-8 p-0" onClick={() => setIsFullscreen(true)} disabled={!selectedSession?.screenshotDataUrl} title="Full screen" aria-label="Full screen">
                    <Expand className="h-4 w-4" />
                  </Button>
--- a/src/components/settings/view/tabs/browser-use-settings/BrowserUseSettingsTab.tsx
+++ b/src/components/settings/view/tabs/browser-use-settings/BrowserUseSettingsTab.tsx
@@ -22,6 +22,7 @@ type BrowserUseStatus = {
  available: boolean;
  backend: 'playwright' | 'camoufox-vnc';
  browserBackend: 'playwright' | 'camoufox-vnc';
+  viewerMode?: 'novnc' | 'window';
  playwrightInstalled: boolean;
  chromiumInstalled: boolean;
  camoufoxInstalled: boolean;
@@ -131,6 +132,8 @@ export default function BrowserUseSettingsTab() {
  const persistSessions = settings?.persistSessions === true;
  const selectedBackend = settings?.browserBackend || 'playwright';
  const effectiveBackend = status?.backend || 'playwright';
+  const usesLocalWindowViewer = status?.viewerMode === 'window';
+  const camoufoxLabel = usesLocalWindowViewer ? 'Camoufox (visible window)' : 'Camoufox + noVNC';
  const needsBrowserBinaries = Boolean(browserEnabled && status && !status.available);
  const runtimeLabel = (installed?: boolean) => {
    if (isStatusLoading && !status) {
@@ -205,8 +208,10 @@ export default function BrowserUseSettingsTab() {
                    },
                    {
                      value: 'camoufox-vnc' as const,
-                      label: 'Camoufox + noVNC',
-                      description: 'Best when a person may need to log in, approve a step, or watch the browser session live.',
+                      label: camoufoxLabel,
+                      description: usesLocalWindowViewer
+                        ? 'Best when a person may need to log in or approve a step. The browser opens as a window on the machine running CloudCLI.'
+                        : 'Best when a person may need to log in, approve a step, or watch the browser session live.',
                      icon: Eye,
                    },
                  ]).map((option) => {
@@ -288,7 +293,7 @@ export default function BrowserUseSettingsTab() {
          <div className="space-y-4 px-4 py-4">
            <div className="flex flex-wrap gap-2 text-xs text-muted-foreground">
              <span className="rounded-md border border-border px-2 py-1">
-                Backend: {effectiveBackend === 'camoufox-vnc' ? 'Camoufox + noVNC' : 'Playwright'}
+                Backend: {effectiveBackend === 'camoufox-vnc' ? camoufoxLabel : 'Playwright'}
              </span>
              <span className="rounded-md border border-border px-2 py-1">
                Playwright: {runtimeLabel(status?.playwrightInstalled)}
@@ -299,9 +304,11 @@ export default function BrowserUseSettingsTab() {
              <span className="rounded-md border border-border px-2 py-1">
                Camoufox: {runtimeLabel(status?.camoufoxInstalled)}
              </span>
-              <span className="rounded-md border border-border px-2 py-1">
-                noVNC: {runtimeLabel(status?.noVncInstalled)}
-              </span>
+              {!usesLocalWindowViewer && (
+                <span className="rounded-md border border-border px-2 py-1">
+                  noVNC: {runtimeLabel(status?.noVncInstalled)}
+                </span>
+              )}
              <span className="rounded-md border border-border px-2 py-1">
                Status: {isStatusLoading && !status ? 'checking...' : status?.available ? 'ready' : browserEnabled ? 'setup required' : 'disabled'}
              </span>