feat: add CloudCLI computer use semantics, desktop helper packaging, and permission onboarding

server/modules/computer-use/semantics/adapters/helper-semantic-adapter.ts

@@ -0,0 +1,82 @@
+import { SemanticHelperProcess } from '@/modules/computer-use/semantics/helpers/semantic-helper-process.js';
+import { resolveSemanticHelper } from '@/modules/computer-use/semantics/helpers/semantic-helper-resolver.js';
+import type { SemanticAdapter, SemanticAdapterCapabilities } from '@/modules/computer-use/semantics/adapters/semantic-adapter.js';
+import type { SemanticApp, SemanticAppState, SemanticToolInput } from '@/modules/computer-use/semantics/semantic-types.js';
+
+type HelperMethod =
+  | 'list_apps'
+  | 'get_app_state'
+  | 'click_element'
+  | 'perform_secondary_action'
+  | 'set_value'
+  | 'type_text'
+  | 'press_key'
+  | 'scroll_element'
+  | 'drag';
+
+export class HelperSemanticAdapter implements SemanticAdapter {
+  private helper: SemanticHelperProcess | null = null;
+
+  constructor(
+    private readonly platform: NodeJS.Platform,
+    private readonly arch: NodeJS.Architecture = process.arch,
+  ) {}
+
+  capabilities(): SemanticAdapterCapabilities {
+    return {
+      platform: this.platform,
+      appDiscovery: true,
+      accessibilityTree: true,
+      nativeElementActions: true,
+      nativeValueSetting: true,
+      targetedInput: true,
+    };
+  }
+
+  async listApps(): Promise<SemanticApp[]> {
+    return await this.request('list_apps', {}) as SemanticApp[];
+  }
+
+  async getAppState(input: SemanticToolInput): Promise<SemanticAppState> {
+    return await this.request('get_app_state', input) as SemanticAppState;
+  }
+
+  async clickElement(input: SemanticToolInput): Promise<SemanticAppState> {
+    return await this.request('click_element', input) as SemanticAppState;
+  }
+
+  async performSecondaryAction(input: SemanticToolInput): Promise<SemanticAppState> {
+    return await this.request('perform_secondary_action', input) as SemanticAppState;
+  }
+
+  async setValue(input: SemanticToolInput): Promise<SemanticAppState> {
+    return await this.request('set_value', input) as SemanticAppState;
+  }
+
+  async typeText(input: SemanticToolInput): Promise<SemanticAppState> {
+    return await this.request('type_text', input) as SemanticAppState;
+  }
+
+  async pressKey(input: SemanticToolInput): Promise<SemanticAppState> {
+    return await this.request('press_key', input) as SemanticAppState;
+  }
+
+  async scrollElement(input: SemanticToolInput): Promise<SemanticAppState> {
+    return await this.request('scroll_element', input) as SemanticAppState;
+  }
+
+  async drag(input: SemanticToolInput): Promise<SemanticAppState> {
+    return await this.request('drag', input) as SemanticAppState;
+  }
+
+  private async request(method: HelperMethod, params: Record<string, unknown>): Promise<unknown> {
+    if (!this.helper) {
+      const resolution = resolveSemanticHelper(this.platform, this.arch);
+      if (!resolution.available || !resolution.path) {
+        throw new Error(resolution.reason || `Semantic helper is unavailable for ${this.platform}-${this.arch}.`);
+      }
+      this.helper = new SemanticHelperProcess(resolution.path);
+    }
+    return this.helper.request(method, params);
+  }
+}