mirror of
https://github.com/siteboon/claudecodeui.git
synced 2026-06-30 08:45:28 +08:00
Compare commits
10 Commits
feat/pendi
...
fix/video-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6d5ed6fdd8 | ||
|
|
dc3a928bed | ||
|
|
6b5506087c | ||
|
|
de13eed72a | ||
|
|
e39de299c3 | ||
|
|
92b5b935fd | ||
|
|
1df336ca2d | ||
|
|
ed4ae3114a | ||
|
|
591e8e7642 | ||
|
|
c947eaaee5 |
@@ -61,6 +61,7 @@ import userRoutes from './routes/user.js';
|
||||
import geminiRoutes from './routes/gemini.js';
|
||||
import pluginsRoutes from './routes/plugins.js';
|
||||
import providerRoutes from './modules/providers/provider.routes.js';
|
||||
import voiceRoutes from './voice-proxy.js';
|
||||
import browserUseRoutes from './modules/browser-use/browser-use.routes.js';
|
||||
import browserUseMcpRoutes from './modules/browser-use/browser-use-mcp.routes.js';
|
||||
import { browserUseService } from './modules/browser-use/browser-use.service.js';
|
||||
@@ -222,6 +223,8 @@ app.use('/api/providers', authenticateToken, providerRoutes);
|
||||
// Agent API Routes (uses API key authentication)
|
||||
app.use('/api/agent', agentRoutes);
|
||||
|
||||
app.use('/api/voice', authenticateToken, voiceRoutes);
|
||||
|
||||
// Serve public files (like api-docs.html)
|
||||
app.use(express.static(path.join(APP_ROOT, 'public')));
|
||||
|
||||
|
||||
224
server/voice-proxy.js
Normal file
224
server/voice-proxy.js
Normal file
@@ -0,0 +1,224 @@
|
||||
// Optional voice proxy — forwards STT/TTS to an OpenAI-compatible audio backend.
|
||||
//
|
||||
// The backend is whatever the user points at: OpenAI, Groq, or a local server
|
||||
// (LocalAI / Speaches / Kokoro-FastAPI / openedai-speech / etc.). It must expose the
|
||||
// standard OpenAI audio endpoints:
|
||||
// POST {base}/audio/transcriptions (multipart 'file' + 'model') -> { text }
|
||||
// POST {base}/audio/speech ({ model, voice, input }) -> audio bytes
|
||||
//
|
||||
// Config is resolved per-request from headers (set by the client's voice settings),
|
||||
// falling back to server env defaults. Mounted at /api/voice behind authenticateToken.
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import express from 'express';
|
||||
|
||||
const ENV = {
|
||||
baseUrl: (process.env.VOICE_API_BASE_URL || '').replace(/\/$/, ''),
|
||||
apiKey: process.env.VOICE_API_KEY || '',
|
||||
sttModel: process.env.VOICE_STT_MODEL || 'whisper-1',
|
||||
ttsModel: process.env.VOICE_TTS_MODEL || 'tts-1',
|
||||
ttsVoice: process.env.VOICE_TTS_VOICE || 'alloy',
|
||||
};
|
||||
|
||||
/**
|
||||
* Resolve the voice backend config for a request. Client headers (set from the
|
||||
* user's in-app voice settings) take precedence over the server env defaults.
|
||||
* @param {import('express').Request} req
|
||||
* @returns {{baseUrl: string, apiKey: string, sttModel: string, ttsModel: string, ttsVoice: string, ttsFormat: string}}
|
||||
*/
|
||||
function resolveConfig(req) {
|
||||
const h = req.headers;
|
||||
return {
|
||||
// Security: do not allow clients to control the outbound backend host.
|
||||
// Always use the server-side configured base URL.
|
||||
baseUrl: ENV.baseUrl,
|
||||
apiKey: String(h['x-voice-api-key'] || '') || ENV.apiKey,
|
||||
sttModel: String(h['x-voice-stt-model'] || '') || ENV.sttModel,
|
||||
ttsModel: String(h['x-voice-tts-model'] || '') || ENV.ttsModel,
|
||||
ttsVoice: String(h['x-voice-tts-voice'] || '') || ENV.ttsVoice,
|
||||
ttsFormat: String(h['x-voice-tts-format'] || '').trim(),
|
||||
};
|
||||
}
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// Generous by default — local TTS can synthesize long messages at ~real-time on CPU.
|
||||
// Guard against a non-numeric/zero override that would make setTimeout fire immediately.
|
||||
const DEFAULT_VOICE_TIMEOUT_MS = 300000;
|
||||
const _parsedTimeout = Number(process.env.VOICE_TIMEOUT_MS);
|
||||
const VOICE_TIMEOUT_MS = Number.isFinite(_parsedTimeout) && _parsedTimeout > 0
|
||||
? _parsedTimeout
|
||||
: DEFAULT_VOICE_TIMEOUT_MS;
|
||||
|
||||
/**
|
||||
* fetch() with an AbortController timeout so a stalled backend can't hold the
|
||||
* request open indefinitely. Aborts after VOICE_TIMEOUT_MS.
|
||||
* @param {string} url
|
||||
* @param {RequestInit} [options]
|
||||
* @returns {Promise<Response>}
|
||||
*/
|
||||
async function fetchWithTimeout(url, options = {}) {
|
||||
const parsed = new URL(url);
|
||||
if (!['http:', 'https:'].includes(parsed.protocol) || !isAllowedBackendUrl(parsed.origin)) {
|
||||
throw new Error('Blocked outbound voice backend URL');
|
||||
}
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), VOICE_TIMEOUT_MS);
|
||||
try {
|
||||
return await fetch(parsed.toString(), { redirect: 'manual', ...options, signal: controller.signal });
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn a backend fetch failure into a clear, actionable client response:
|
||||
* 504 on timeout (AbortError), 502 otherwise.
|
||||
* @param {import('express').Response} res
|
||||
* @param {Error} e
|
||||
*/
|
||||
function backendError(res, e) {
|
||||
if (e && e.name === 'AbortError') {
|
||||
return res.status(504).json({
|
||||
error: `Voice backend timed out after ${Math.round(VOICE_TIMEOUT_MS / 1000)}s. Check your voice backend.`,
|
||||
});
|
||||
}
|
||||
return res.status(502).json({ error: `Voice backend unreachable: ${e.message}` });
|
||||
}
|
||||
|
||||
/**
|
||||
* SSRF guard for the user-configurable backend URL: allow http/https only and
|
||||
* block the link-local / cloud-metadata range (169.254.x). localhost and private
|
||||
* ranges are allowed on purpose so users can point at a local voice server
|
||||
* (LocalAI, Speaches, Kokoro-FastAPI, etc.).
|
||||
* @param {string} raw
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function isAllowedBackendUrl(raw) {
|
||||
let u;
|
||||
try {
|
||||
u = new URL(raw);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (u.protocol !== 'http:' && u.protocol !== 'https:') return false;
|
||||
if (u.hostname === '169.254.169.254' || u.hostname.startsWith('169.254.')) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Relay an upstream (backend) error to the client without making an upstream
|
||||
* 401/403 look like the user's own app login failed.
|
||||
* @param {import('express').Response} res
|
||||
* @param {number} status
|
||||
* @param {string} [text]
|
||||
*/
|
||||
function upstreamError(res, status, text) {
|
||||
if (status === 401 || status === 403) {
|
||||
return res.status(502).json({ error: 'Voice backend rejected the request (check the API key).' });
|
||||
}
|
||||
return res.status(status).json({ error: text || 'voice backend error' });
|
||||
}
|
||||
|
||||
let _upload = null;
|
||||
/**
|
||||
* Lazily build a memory-storage multer instance (25 MB cap) for audio uploads,
|
||||
* so multer is only imported when the voice feature is actually used.
|
||||
* @returns {Promise<import('multer').Multer>}
|
||||
*/
|
||||
async function getUpload() {
|
||||
if (!_upload) {
|
||||
const multer = (await import('multer')).default;
|
||||
_upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 25 * 1024 * 1024 } });
|
||||
}
|
||||
return _upload;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the Authorization header for the backend, or an empty object when no
|
||||
* key is configured (e.g. a local server that needs none).
|
||||
* @param {string} apiKey
|
||||
* @returns {Record<string, string>}
|
||||
*/
|
||||
function authHeader(apiKey) {
|
||||
return apiKey ? { Authorization: `Bearer ${apiKey}` } : {};
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/voice/health -> { configured } (true when a backend base URL is set).
|
||||
*/
|
||||
router.get('/health', (req, res) => {
|
||||
res.json({ configured: Boolean(resolveConfig(req).baseUrl) });
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /api/voice/transcribe (multipart 'audio') -> { text }.
|
||||
* Forwards the uploaded audio to the backend's /audio/transcriptions endpoint.
|
||||
*/
|
||||
router.post('/transcribe', async (req, res) => {
|
||||
const cfg = resolveConfig(req);
|
||||
if (!cfg.baseUrl) return res.status(503).json({ error: 'No voice backend configured' });
|
||||
if (!isAllowedBackendUrl(cfg.baseUrl)) return res.status(400).json({ error: 'Invalid voice backend URL.' });
|
||||
const upload = await getUpload();
|
||||
upload.single('audio')(req, res, async (err) => {
|
||||
if (err) return res.status(400).json({ error: err.message });
|
||||
if (!req.file) return res.status(400).json({ error: 'No audio uploaded' });
|
||||
try {
|
||||
const fd = new FormData();
|
||||
fd.append(
|
||||
'file',
|
||||
new Blob([req.file.buffer], { type: req.file.mimetype || 'audio/webm' }),
|
||||
req.file.originalname || 'recording.webm',
|
||||
);
|
||||
fd.append('model', cfg.sttModel);
|
||||
const r = await fetchWithTimeout(`${cfg.baseUrl}/audio/transcriptions`, {
|
||||
method: 'POST',
|
||||
headers: authHeader(cfg.apiKey),
|
||||
body: fd,
|
||||
});
|
||||
const text = await r.text();
|
||||
if (!r.ok) return upstreamError(res, r.status, text);
|
||||
let data;
|
||||
try { data = JSON.parse(text); } catch { data = { text }; }
|
||||
res.json({ text: data.text ?? '' });
|
||||
} catch (e) {
|
||||
backendError(res, e);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /api/voice/tts { text } -> audio bytes.
|
||||
* Forwards the text to the backend's /audio/speech endpoint and streams the audio back.
|
||||
*/
|
||||
router.post('/tts', async (req, res) => {
|
||||
const cfg = resolveConfig(req);
|
||||
if (!cfg.baseUrl) return res.status(503).json({ error: 'No voice backend configured' });
|
||||
if (!isAllowedBackendUrl(cfg.baseUrl)) return res.status(400).json({ error: 'Invalid voice backend URL.' });
|
||||
const text = req.body?.text;
|
||||
if (typeof text !== 'string' || !text.trim()) return res.status(400).json({ error: 'text required' });
|
||||
try {
|
||||
const r = await fetchWithTimeout(`${cfg.baseUrl}/audio/speech`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', ...authHeader(cfg.apiKey) },
|
||||
body: JSON.stringify({
|
||||
model: cfg.ttsModel,
|
||||
voice: cfg.ttsVoice,
|
||||
input: text,
|
||||
...(cfg.ttsFormat ? { response_format: cfg.ttsFormat } : {}),
|
||||
}),
|
||||
});
|
||||
if (!r.ok) {
|
||||
const errText = await r.text().catch(() => 'tts failed');
|
||||
return upstreamError(res, r.status, errText);
|
||||
}
|
||||
res.setHeader('Content-Type', r.headers.get('content-type') || 'audio/mpeg');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
if (!r.body) return res.end();
|
||||
Readable.fromWeb(r.body).on('error', (error) => res.destroy(error)).pipe(res);
|
||||
} catch (e) {
|
||||
backendError(res, e);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -775,6 +775,17 @@ export function useChatComposerState({
|
||||
handleSubmitRef.current = handleSubmit;
|
||||
}, [handleSubmit]);
|
||||
|
||||
// A voice transcript either fills the input (to edit before sending) or, when the
|
||||
// user tapped "stop and send", is submitted straight away. Mirror the value into
|
||||
// inputValueRef synchronously so handleSubmit reads the new text, not the stale state.
|
||||
const handleVoiceTranscript = useCallback((text: string, send?: boolean) => {
|
||||
const base = inputValueRef.current.trim();
|
||||
const next = base ? `${base} ${text}` : text;
|
||||
setInput(next);
|
||||
inputValueRef.current = next;
|
||||
if (send) handleSubmitRef.current?.(createFakeSubmitEvent());
|
||||
}, [setInput]);
|
||||
|
||||
useEffect(() => {
|
||||
inputValueRef.current = input;
|
||||
}, [input]);
|
||||
@@ -1013,6 +1024,7 @@ export function useChatComposerState({
|
||||
isDragActive,
|
||||
openImagePicker: open,
|
||||
handleSubmit,
|
||||
handleVoiceTranscript,
|
||||
handleInputChange,
|
||||
handleKeyDown,
|
||||
handlePaste,
|
||||
|
||||
@@ -114,7 +114,6 @@ export function useChatProviderState({ selectedSession, selectedProject }: UseCh
|
||||
const [providerModelsLoading, setProviderModelsLoading] = useState(true);
|
||||
const [providerModelsRefreshing, setProviderModelsRefreshing] = useState(false);
|
||||
|
||||
const lastProviderRef = useRef(provider);
|
||||
const providerModelsRequestIdRef = useRef(0);
|
||||
|
||||
const setStoredProviderModel = useCallback((targetProvider: LLMProvider, model: string) => {
|
||||
@@ -344,14 +343,8 @@ export function useChatProviderState({ selectedSession, selectedProject }: UseCh
|
||||
localStorage.setItem('selected-provider', selectedSession.__provider);
|
||||
}, [provider, selectedSession]);
|
||||
|
||||
useEffect(() => {
|
||||
if (lastProviderRef.current === provider) {
|
||||
return;
|
||||
}
|
||||
setPendingPermissionRequests([]);
|
||||
lastProviderRef.current = provider;
|
||||
}, [provider]);
|
||||
|
||||
// Permission prompts belong to a session, not to the transient provider
|
||||
// selection that is synchronized after navigation.
|
||||
useEffect(() => {
|
||||
setPendingPermissionRequests((previous) =>
|
||||
previous.filter((request) => !request.sessionId || request.sessionId === selectedSession?.id),
|
||||
|
||||
@@ -1,20 +1,29 @@
|
||||
import { useEffect } from 'react';
|
||||
import { useEffect, useRef } from 'react';
|
||||
import type { Dispatch, MutableRefObject, SetStateAction } from 'react';
|
||||
|
||||
import type { ServerEvent } from '../../../contexts/WebSocketContext';
|
||||
import { showCompletionTitleIndicator } from '../../../utils/pageTitleNotification';
|
||||
import { playChatCompletionSound } from '../../../utils/notificationSound';
|
||||
import { playChatCompletionSound, playNotificationSound } from '../../../utils/notificationSound';
|
||||
import type { MarkSessionIdle, MarkSessionProcessing } from '../../../hooks/useSessionProtection';
|
||||
import type { PendingPermissionRequest } from '../types/types';
|
||||
import type { ProjectSession, LLMProvider } from '../../../types/app';
|
||||
import type { SessionStore, NormalizedMessage } from '../../../stores/useSessionStore';
|
||||
|
||||
const isActionablePermissionRequest = (request: { toolName?: unknown } | null | undefined): boolean => {
|
||||
return request?.toolName !== 'ExitPlanMode' && request?.toolName !== 'exit_plan_mode';
|
||||
};
|
||||
|
||||
const hasActionablePermissionRequests = (requests: Array<{ toolName?: unknown }> | null | undefined): boolean => {
|
||||
return Array.isArray(requests) && requests.some((request) => isActionablePermissionRequest(request));
|
||||
};
|
||||
|
||||
interface UseChatRealtimeHandlersArgs {
|
||||
subscribe: (listener: (event: ServerEvent) => void) => () => void;
|
||||
provider: LLMProvider;
|
||||
selectedSession: ProjectSession | null;
|
||||
currentSessionId: string | null;
|
||||
setTokenBudget: (budget: Record<string, unknown> | null) => void;
|
||||
pendingPermissionRequests: PendingPermissionRequest[];
|
||||
setPendingPermissionRequests: Dispatch<SetStateAction<PendingPermissionRequest[]>>;
|
||||
streamTimerRef: MutableRefObject<number | null>;
|
||||
accumulatedStreamRef: MutableRefObject<string>;
|
||||
@@ -52,6 +61,7 @@ export function useChatRealtimeHandlers({
|
||||
selectedSession,
|
||||
currentSessionId,
|
||||
setTokenBudget,
|
||||
pendingPermissionRequests,
|
||||
setPendingPermissionRequests,
|
||||
streamTimerRef,
|
||||
accumulatedStreamRef,
|
||||
@@ -62,13 +72,29 @@ export function useChatRealtimeHandlers({
|
||||
onWebSocketReconnect,
|
||||
sessionStore,
|
||||
}: UseChatRealtimeHandlersArgs) {
|
||||
// Session switches can send `chat.subscribe` before this effect has a chance
|
||||
// to rebind the websocket listener. Read the visible session id from a ref
|
||||
// so a fast `chat_subscribed` ack is matched against the current view, not
|
||||
// the previous render's closed-over selection.
|
||||
const activeViewSessionIdRef = useRef<string | null>(selectedSession?.id || currentSessionId || null);
|
||||
activeViewSessionIdRef.current = selectedSession?.id || currentSessionId || null;
|
||||
|
||||
// Keep the latest pending-permission snapshot available to the websocket
|
||||
// listener so back-to-back permission events can dedupe and re-arm the
|
||||
// notification sound before React finishes a rerender.
|
||||
const pendingPermissionRequestsRef = useRef(pendingPermissionRequests);
|
||||
|
||||
useEffect(() => {
|
||||
pendingPermissionRequestsRef.current = pendingPermissionRequests;
|
||||
}, [pendingPermissionRequests]);
|
||||
|
||||
useEffect(() => {
|
||||
const handleEvent = (msg: ServerEvent) => {
|
||||
if (!msg.kind) {
|
||||
return;
|
||||
}
|
||||
|
||||
const activeViewSessionId = selectedSession?.id || currentSessionId || null;
|
||||
const activeViewSessionId = activeViewSessionIdRef.current;
|
||||
const sid = (typeof msg.sessionId === 'string' && msg.sessionId) || activeViewSessionId;
|
||||
|
||||
// Record replay progress for every sequenced live event.
|
||||
@@ -101,7 +127,16 @@ export function useChatRealtimeHandlers({
|
||||
|
||||
const isViewedSession = sid === activeViewSessionId;
|
||||
if (isViewedSession && Array.isArray(msg.pendingPermissions)) {
|
||||
setPendingPermissionRequests(msg.pendingPermissions as PendingPermissionRequest[]);
|
||||
const nextPendingPermissionRequests = msg.pendingPermissions as PendingPermissionRequest[];
|
||||
const hadActionablePermissionRequests = hasActionablePermissionRequests(pendingPermissionRequestsRef.current);
|
||||
const hasPendingActionablePermissionRequests = hasActionablePermissionRequests(nextPendingPermissionRequests);
|
||||
|
||||
pendingPermissionRequestsRef.current = nextPendingPermissionRequests;
|
||||
setPendingPermissionRequests(nextPendingPermissionRequests);
|
||||
|
||||
if (hasPendingActionablePermissionRequests && !hadActionablePermissionRequests) {
|
||||
void playNotificationSound();
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -203,6 +238,7 @@ export function useChatRealtimeHandlers({
|
||||
// hides it immediately and atomically.
|
||||
onSessionIdle?.(sid);
|
||||
if (sid === activeViewSessionId) {
|
||||
pendingPermissionRequestsRef.current = [];
|
||||
setPendingPermissionRequests([]);
|
||||
}
|
||||
|
||||
@@ -234,10 +270,14 @@ export function useChatRealtimeHandlers({
|
||||
|
||||
case 'permission_request': {
|
||||
if (!msg.requestId) break;
|
||||
if (isActionablePermissionRequest({ toolName: msg.toolName })) {
|
||||
void playNotificationSound();
|
||||
}
|
||||
|
||||
if (sid === activeViewSessionId) {
|
||||
setPendingPermissionRequests((prev) => {
|
||||
if (prev.some((r: PendingPermissionRequest) => r.requestId === msg.requestId)) return prev;
|
||||
return [...prev, {
|
||||
const previousPendingPermissionRequests = pendingPermissionRequestsRef.current;
|
||||
if (!previousPendingPermissionRequests.some((request) => request.requestId === msg.requestId)) {
|
||||
const nextPendingPermissionRequests = [...previousPendingPermissionRequests, {
|
||||
requestId: msg.requestId as string,
|
||||
toolName: (msg.toolName as string) || 'UnknownTool',
|
||||
input: msg.input,
|
||||
@@ -245,7 +285,10 @@ export function useChatRealtimeHandlers({
|
||||
sessionId: sid || null,
|
||||
receivedAt: new Date(),
|
||||
}];
|
||||
});
|
||||
|
||||
pendingPermissionRequestsRef.current = nextPendingPermissionRequests;
|
||||
setPendingPermissionRequests(nextPendingPermissionRequests);
|
||||
}
|
||||
}
|
||||
if (sid) {
|
||||
onSessionProcessing?.(sid);
|
||||
@@ -255,7 +298,12 @@ export function useChatRealtimeHandlers({
|
||||
|
||||
case 'permission_cancelled': {
|
||||
if (msg.requestId && sid === activeViewSessionId) {
|
||||
setPendingPermissionRequests((prev) => prev.filter((r: PendingPermissionRequest) => r.requestId !== msg.requestId));
|
||||
const nextPendingPermissionRequests = pendingPermissionRequestsRef.current.filter(
|
||||
(request: PendingPermissionRequest) => request.requestId !== msg.requestId,
|
||||
);
|
||||
|
||||
pendingPermissionRequestsRef.current = nextPendingPermissionRequests;
|
||||
setPendingPermissionRequests(nextPendingPermissionRequests);
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -286,6 +334,7 @@ export function useChatRealtimeHandlers({
|
||||
selectedSession,
|
||||
currentSessionId,
|
||||
setTokenBudget,
|
||||
pendingPermissionRequests,
|
||||
setPendingPermissionRequests,
|
||||
streamTimerRef,
|
||||
accumulatedStreamRef,
|
||||
|
||||
33
src/components/chat/hooks/useTts.ts
Normal file
33
src/components/chat/hooks/useTts.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
import { useCallback, useEffect, useState } from 'react';
|
||||
import { voicePlayer, voiceId, type VoiceSnapshot } from '../../../lib/voicePlayer';
|
||||
|
||||
export type TtsState = VoiceSnapshot['state'];
|
||||
|
||||
/**
|
||||
* Thin adapter over the app-level voicePlayer. Playback lives outside React (see
|
||||
* lib/voicePlayer), so switching chats or re-rendering a message no longer cuts the
|
||||
* audio off. This hook just reflects the player's state for one message and forwards taps.
|
||||
*/
|
||||
export function useTts(getText: () => string) {
|
||||
const content = getText();
|
||||
const id = voiceId(content);
|
||||
|
||||
const [snap, setSnap] = useState<VoiceSnapshot>(() => voicePlayer.getSnapshot(id));
|
||||
|
||||
useEffect(() => {
|
||||
const update = () =>
|
||||
setSnap((prev) => {
|
||||
const next = voicePlayer.getSnapshot(id);
|
||||
return prev.state === next.state && prev.error === next.error ? prev : next;
|
||||
});
|
||||
update();
|
||||
return voicePlayer.subscribe(update);
|
||||
}, [id]);
|
||||
|
||||
const toggle = useCallback(() => {
|
||||
voicePlayer.unlock(); // synchronous, within the click gesture (iOS)
|
||||
voicePlayer.toggle(content);
|
||||
}, [content]);
|
||||
|
||||
return { state: snap.state, toggle, error: snap.error };
|
||||
}
|
||||
85
src/components/chat/hooks/useVoiceAvailable.ts
Normal file
85
src/components/chat/hooks/useVoiceAvailable.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
|
||||
import { authenticatedFetch } from '../../../utils/api';
|
||||
import { readVoiceConfig, VOICE_CONFIG_SYNC_EVENT } from '../../../hooks/useVoiceConfig';
|
||||
|
||||
// Voice UI is gated on the `voiceEnabled` UI preference (toggled in Quick Settings /
|
||||
// the Settings modal) and a configured voice backend.
|
||||
const STORAGE_KEY = 'uiPreferences';
|
||||
const SYNC_EVENT = 'ui-preferences:sync';
|
||||
let healthRequest: Promise<boolean> | null = null;
|
||||
|
||||
function checkVoiceHealth(): Promise<boolean> {
|
||||
if (healthRequest) return healthRequest;
|
||||
const request = authenticatedFetch('/api/voice/health')
|
||||
.then(async (response) => {
|
||||
if (!response.ok) throw new Error(`Voice health check failed (${response.status})`);
|
||||
const data = await response.json();
|
||||
return data?.configured === true;
|
||||
})
|
||||
.finally(() => {
|
||||
healthRequest = null;
|
||||
});
|
||||
healthRequest = request;
|
||||
return request;
|
||||
}
|
||||
|
||||
function readVoiceEnabled(): boolean {
|
||||
try {
|
||||
const raw = localStorage.getItem(STORAGE_KEY);
|
||||
if (!raw) return false;
|
||||
const parsed = JSON.parse(raw);
|
||||
return parsed?.voiceEnabled === true || parsed?.voiceEnabled === 'true';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function useVoiceAvailable(): boolean {
|
||||
const [enabled, setEnabled] = useState<boolean>(() =>
|
||||
typeof window === 'undefined' ? false : readVoiceEnabled(),
|
||||
);
|
||||
const [available, setAvailable] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const update = () => setEnabled(readVoiceEnabled());
|
||||
window.addEventListener('storage', update);
|
||||
window.addEventListener(SYNC_EVENT, update as EventListener);
|
||||
return () => {
|
||||
window.removeEventListener('storage', update);
|
||||
window.removeEventListener(SYNC_EVENT, update as EventListener);
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
let active = true;
|
||||
let requestId = 0;
|
||||
|
||||
const check = async () => {
|
||||
if (!enabled) {
|
||||
setAvailable(false);
|
||||
return;
|
||||
}
|
||||
if (readVoiceConfig().baseUrl.trim()) {
|
||||
setAvailable(true);
|
||||
return;
|
||||
}
|
||||
const id = ++requestId;
|
||||
try {
|
||||
const result = await checkVoiceHealth();
|
||||
if (active && id === requestId) setAvailable(result);
|
||||
} catch {
|
||||
if (active && id === requestId) setAvailable(false);
|
||||
}
|
||||
};
|
||||
|
||||
void check();
|
||||
window.addEventListener(VOICE_CONFIG_SYNC_EVENT, check);
|
||||
return () => {
|
||||
active = false;
|
||||
window.removeEventListener(VOICE_CONFIG_SYNC_EVENT, check);
|
||||
};
|
||||
}, [enabled]);
|
||||
|
||||
return enabled && available;
|
||||
}
|
||||
149
src/components/chat/hooks/useVoiceInput.ts
Normal file
149
src/components/chat/hooks/useVoiceInput.ts
Normal file
@@ -0,0 +1,149 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
|
||||
import { transcribeVoice } from '../../../lib/voiceApi';
|
||||
|
||||
// Mobile-safe recording: iOS Safari 18.4+ supports webm/opus; older iOS needs mp4.
|
||||
const MIME_CANDIDATES = [
|
||||
'audio/webm;codecs=opus',
|
||||
'audio/webm',
|
||||
'audio/mp4',
|
||||
'audio/ogg;codecs=opus',
|
||||
'audio/ogg',
|
||||
];
|
||||
|
||||
function pickMime(): string {
|
||||
for (const t of MIME_CANDIDATES) {
|
||||
try {
|
||||
if (typeof MediaRecorder !== 'undefined' && MediaRecorder.isTypeSupported(t)) return t;
|
||||
} catch {
|
||||
/* isTypeSupported can throw on some iOS versions */
|
||||
}
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
export type VoiceInputState = 'idle' | 'recording' | 'transcribing';
|
||||
|
||||
/**
|
||||
* Push-to-talk dictation. Records the mic, uploads to /api/voice/transcribe
|
||||
* (an OpenAI-compatible speech-to-text backend via the Express proxy), and
|
||||
* returns the transcript through onTranscript.
|
||||
*/
|
||||
export function useVoiceInput(
|
||||
onTranscript: (text: string, send?: boolean) => void,
|
||||
onError?: (msg: string) => void,
|
||||
) {
|
||||
const [state, setState] = useState<VoiceInputState>('idle');
|
||||
const recorderRef = useRef<MediaRecorder | null>(null);
|
||||
const chunksRef = useRef<Blob[]>([]);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const cancelledRef = useRef(false);
|
||||
const startingRef = useRef(false);
|
||||
// Whether the in-progress stop should auto-send the transcript (vs just fill the box).
|
||||
const sendRef = useRef(false);
|
||||
|
||||
const stopTracks = () => {
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop());
|
||||
streamRef.current = null;
|
||||
};
|
||||
|
||||
// Stop the mic if the component unmounts mid-recording.
|
||||
useEffect(() => {
|
||||
cancelledRef.current = false;
|
||||
return () => {
|
||||
cancelledRef.current = true;
|
||||
startingRef.current = false;
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop());
|
||||
streamRef.current = null;
|
||||
recorderRef.current = null;
|
||||
};
|
||||
}, []);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (startingRef.current || (recorderRef.current && recorderRef.current.state !== 'inactive')) return;
|
||||
startingRef.current = true;
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { echoCancellation: true, noiseSuppression: true },
|
||||
});
|
||||
if (cancelledRef.current) {
|
||||
stream.getTracks().forEach((t) => t.stop());
|
||||
return;
|
||||
}
|
||||
streamRef.current = stream;
|
||||
const mimeType = pickMime();
|
||||
const rec = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream);
|
||||
recorderRef.current = rec;
|
||||
chunksRef.current = [];
|
||||
|
||||
rec.ondataavailable = (e) => {
|
||||
if (e.data.size > 0) chunksRef.current.push(e.data);
|
||||
};
|
||||
|
||||
rec.onstop = async () => {
|
||||
stopTracks();
|
||||
if (cancelledRef.current) return;
|
||||
// Capture and clear the send intent for this stop before any async work.
|
||||
const shouldSend = sendRef.current;
|
||||
sendRef.current = false;
|
||||
const type = rec.mimeType || 'audio/webm';
|
||||
const blob = new Blob(chunksRef.current, { type });
|
||||
if (blob.size < 800) {
|
||||
setState('idle');
|
||||
onError?.('Recording too short');
|
||||
return;
|
||||
}
|
||||
setState('transcribing');
|
||||
try {
|
||||
const ext = type.includes('mp4') ? 'm4a' : type.includes('ogg') ? 'ogg' : 'webm';
|
||||
const res = await transcribeVoice(blob, `recording.${ext}`);
|
||||
if (!res.ok) throw new Error(`transcribe ${res.status}`);
|
||||
const data = await res.json();
|
||||
if (cancelledRef.current) return;
|
||||
const text = String(data?.text || '').trim();
|
||||
if (text) onTranscript(text, shouldSend);
|
||||
else onError?.('No speech detected');
|
||||
} catch (e) {
|
||||
if (!cancelledRef.current) {
|
||||
onError?.(`Transcription failed: ${e instanceof Error ? e.message : String(e)}`);
|
||||
}
|
||||
} finally {
|
||||
if (!cancelledRef.current) setState('idle');
|
||||
}
|
||||
};
|
||||
|
||||
rec.start();
|
||||
setState('recording');
|
||||
} catch (e) {
|
||||
recorderRef.current = null;
|
||||
stopTracks();
|
||||
if (cancelledRef.current) return;
|
||||
const err = e as { name?: string; message?: string };
|
||||
let msg = `Mic error: ${err?.message || e}`;
|
||||
if (err?.name === 'NotAllowedError') msg = 'Microphone access denied.';
|
||||
else if (err?.name === 'NotFoundError') msg = 'No microphone found.';
|
||||
onError?.(msg);
|
||||
setState('idle');
|
||||
} finally {
|
||||
startingRef.current = false;
|
||||
}
|
||||
}, [onTranscript, onError]);
|
||||
|
||||
// Stop recording. Pass { send: true } to auto-send the transcript once it's ready.
|
||||
// Guard on the recorder's own state (not React state) so a double tap, or the mic
|
||||
// and Send buttons both firing, can't call stop() on an already-inactive recorder.
|
||||
const stop = useCallback((opts?: { send?: boolean }) => {
|
||||
const rec = recorderRef.current;
|
||||
if (rec && rec.state !== 'inactive') {
|
||||
sendRef.current = opts?.send ?? false;
|
||||
rec.stop();
|
||||
}
|
||||
}, []);
|
||||
|
||||
const toggle = useCallback(() => {
|
||||
if (state === 'recording') stop();
|
||||
else if (state === 'idle') start();
|
||||
}, [state, start, stop]);
|
||||
|
||||
return { state, toggle, stop };
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import React from 'react';
|
||||
import { renderToStaticMarkup } from 'react-dom/server';
|
||||
import { QuestionAnswerContent } from './QuestionAnswerContent';
|
||||
|
||||
// Regression coverage for the chat-interface crash where an AskUserQuestion
|
||||
// payload loaded from a session transcript arrives with a non-array `questions`
|
||||
// or a question missing its `options` array. Rendering must degrade gracefully
|
||||
// instead of throwing "TypeError: e.map is not a function".
|
||||
|
||||
test('renders without throwing when questions is a non-array value', () => {
|
||||
assert.doesNotThrow(() => {
|
||||
renderToStaticMarkup(
|
||||
React.createElement(QuestionAnswerContent, {
|
||||
// Malformed: object instead of an array
|
||||
questions: { 0: { question: 'q?', options: [{ label: 'a' }] } } as never,
|
||||
answers: {},
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('renders without throwing when a question is missing options[]', () => {
|
||||
assert.doesNotThrow(() => {
|
||||
renderToStaticMarkup(
|
||||
React.createElement(QuestionAnswerContent, {
|
||||
questions: [{ question: 'Pick one?', header: 'H' } as never],
|
||||
answers: { 'Pick one?': 'X' },
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('renders without throwing when options[] contains malformed entries', () => {
|
||||
assert.doesNotThrow(() => {
|
||||
renderToStaticMarkup(
|
||||
React.createElement(QuestionAnswerContent, {
|
||||
questions: [{ question: 'Pick one?', options: [null, 'oops', { label: 'A' }] } as never],
|
||||
answers: { 'Pick one?': 'A, Custom' },
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('renders without throwing when a questions entry is null/non-object', () => {
|
||||
assert.doesNotThrow(() => {
|
||||
renderToStaticMarkup(
|
||||
React.createElement(QuestionAnswerContent, {
|
||||
questions: [null, 'oops', { question: 'Ok?', options: [{ label: 'A' }] }] as never,
|
||||
answers: {},
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('renders without throwing when an answer is a non-string value', () => {
|
||||
assert.doesNotThrow(() => {
|
||||
renderToStaticMarkup(
|
||||
React.createElement(QuestionAnswerContent, {
|
||||
questions: [{ question: 'Pick one?', options: [{ label: 'A' }] }],
|
||||
// Malformed: answer is an object instead of the expected string
|
||||
answers: { 'Pick one?': { unexpected: true } } as never,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('still renders a well-formed question + answer', () => {
|
||||
const html = renderToStaticMarkup(
|
||||
React.createElement(QuestionAnswerContent, {
|
||||
questions: [{ question: 'Pick one?', header: 'H', options: [{ label: 'A' }, { label: 'B' }] }],
|
||||
answers: { 'Pick one?': 'A' },
|
||||
}),
|
||||
);
|
||||
assert.ok(html.includes('Pick one?'));
|
||||
});
|
||||
@@ -15,7 +15,11 @@ export const QuestionAnswerContent: React.FC<QuestionAnswerContentProps> = ({
|
||||
}) => {
|
||||
const [expandedIdx, setExpandedIdx] = useState<number | null>(null);
|
||||
|
||||
if (!questions || questions.length === 0) {
|
||||
// Tool inputs are runtime data loaded from session transcripts and may be
|
||||
// malformed (e.g. `questions` arriving as a non-array). Guard with
|
||||
// Array.isArray so a single bad payload can't crash the whole chat view
|
||||
// with "e.map is not a function".
|
||||
if (!Array.isArray(questions) || questions.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -24,11 +28,23 @@ export const QuestionAnswerContent: React.FC<QuestionAnswerContentProps> = ({
|
||||
|
||||
return (
|
||||
<div className={`space-y-2 ${className}`}>
|
||||
{questions.map((q, idx) => {
|
||||
{questions.map((rawQuestion, idx) => {
|
||||
// Entries come from session transcripts and may be malformed; skip
|
||||
// anything that isn't a proper question object with a string prompt.
|
||||
if (!rawQuestion || typeof rawQuestion !== 'object' || typeof rawQuestion.question !== 'string') {
|
||||
return null;
|
||||
}
|
||||
const q = rawQuestion;
|
||||
const answer = answers?.[q.question];
|
||||
const answerLabels = answer ? answer.split(', ') : [];
|
||||
// `answer` may be a non-string (or absent) in malformed payloads.
|
||||
const answerLabels = typeof answer === 'string' ? answer.split(', ') : [];
|
||||
const skipped = !answer;
|
||||
const isExpanded = expandedIdx === idx;
|
||||
// `options` is typed as an array but comes from untrusted runtime data;
|
||||
// keep only valid entries so `.some`/`.map` below never throw.
|
||||
const options = Array.isArray(q.options)
|
||||
? q.options.filter((opt) => opt && typeof opt === 'object' && typeof opt.label === 'string')
|
||||
: [];
|
||||
|
||||
return (
|
||||
<div
|
||||
@@ -74,7 +90,7 @@ export const QuestionAnswerContent: React.FC<QuestionAnswerContentProps> = ({
|
||||
{!isExpanded && answerLabels.length > 0 && (
|
||||
<div className="mt-1.5 flex flex-wrap gap-1">
|
||||
{answerLabels.map((lbl) => {
|
||||
const isCustom = !q.options.some(o => o.label === lbl);
|
||||
const isCustom = !options.some(o => o.label === lbl);
|
||||
return (
|
||||
<span
|
||||
key={lbl}
|
||||
@@ -110,7 +126,7 @@ export const QuestionAnswerContent: React.FC<QuestionAnswerContentProps> = ({
|
||||
{isExpanded && (
|
||||
<div className="border-t border-gray-100 px-3 pb-2.5 pt-0.5 dark:border-gray-700/40">
|
||||
<div className="ml-6.5 space-y-1">
|
||||
{q.options.map((opt) => {
|
||||
{options.map((opt) => {
|
||||
const wasSelected = answerLabels.includes(opt.label);
|
||||
return (
|
||||
<div
|
||||
@@ -148,7 +164,7 @@ export const QuestionAnswerContent: React.FC<QuestionAnswerContentProps> = ({
|
||||
);
|
||||
})}
|
||||
|
||||
{answerLabels.filter(lbl => !q.options.some(o => o.label === lbl)).map(lbl => (
|
||||
{answerLabels.filter(lbl => !options.some(o => o.label === lbl)).map(lbl => (
|
||||
<div
|
||||
key={lbl}
|
||||
className="flex items-start gap-2 rounded-lg border border-blue-200/60 bg-blue-50/80 px-2.5 py-1.5 text-[12px] dark:border-blue-800/40 dark:bg-blue-900/20"
|
||||
|
||||
@@ -173,6 +173,7 @@ function ChatInterface({
|
||||
isDragActive,
|
||||
openImagePicker,
|
||||
handleSubmit,
|
||||
handleVoiceTranscript,
|
||||
handleInputChange,
|
||||
handleKeyDown,
|
||||
handlePaste,
|
||||
@@ -239,6 +240,7 @@ function ChatInterface({
|
||||
selectedSession,
|
||||
currentSessionId,
|
||||
setTokenBudget,
|
||||
pendingPermissionRequests,
|
||||
setPendingPermissionRequests,
|
||||
streamTimerRef,
|
||||
accumulatedStreamRef,
|
||||
@@ -405,6 +407,7 @@ function ChatInterface({
|
||||
renderInputWithMentions={renderInputWithMentions}
|
||||
textareaRef={textareaRef}
|
||||
input={input}
|
||||
onVoiceTranscript={handleVoiceTranscript}
|
||||
onInputChange={handleInputChange}
|
||||
onTextareaClick={handleTextareaClick}
|
||||
onTextareaKeyDown={handleKeyDown}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import type {
|
||||
ChangeEvent,
|
||||
ClipboardEvent,
|
||||
@@ -9,8 +10,10 @@ import type {
|
||||
RefObject,
|
||||
TouchEvent,
|
||||
} from 'react';
|
||||
import { ImageIcon, MessageSquareIcon, XIcon, ArrowDownIcon } from 'lucide-react';
|
||||
import { ImageIcon, MessageSquareIcon, XIcon, ArrowDownIcon, Loader2 } from 'lucide-react';
|
||||
|
||||
import { useVoiceInput } from '../../hooks/useVoiceInput';
|
||||
import { useVoiceAvailable } from '../../hooks/useVoiceAvailable';
|
||||
import type { SessionActivity } from '../../../../hooks/useSessionProtection';
|
||||
import type { PendingPermissionRequest, PermissionMode } from '../../types/types';
|
||||
import {
|
||||
@@ -27,6 +30,7 @@ import {
|
||||
import CommandMenu from './CommandMenu';
|
||||
import ActivityIndicator from './ActivityIndicator';
|
||||
import ImageAttachment from './ImageAttachment';
|
||||
import VoiceInputButton from './VoiceInputButton';
|
||||
import PermissionRequestsBanner from './PermissionRequestsBanner';
|
||||
import TokenUsageSummary from './TokenUsageSummary';
|
||||
|
||||
@@ -89,6 +93,7 @@ interface ChatComposerProps {
|
||||
renderInputWithMentions: (text: string) => ReactNode;
|
||||
textareaRef: RefObject<HTMLTextAreaElement>;
|
||||
input: string;
|
||||
onVoiceTranscript?: (text: string, send?: boolean) => void;
|
||||
onInputChange: (event: ChangeEvent<HTMLTextAreaElement>) => void;
|
||||
onTextareaClick: (event: MouseEvent<HTMLTextAreaElement>) => void;
|
||||
onTextareaKeyDown: (event: KeyboardEvent<HTMLTextAreaElement>) => void;
|
||||
@@ -142,6 +147,7 @@ export default function ChatComposer({
|
||||
renderInputWithMentions,
|
||||
textareaRef,
|
||||
input,
|
||||
onVoiceTranscript,
|
||||
onInputChange,
|
||||
onTextareaClick,
|
||||
onTextareaKeyDown,
|
||||
@@ -154,6 +160,28 @@ export default function ChatComposer({
|
||||
sendByCtrlEnter,
|
||||
}: ChatComposerProps) {
|
||||
const { t } = useTranslation('chat');
|
||||
|
||||
// Voice state is hosted here (not in the mic button) so the main Send button can stop
|
||||
// recording and send the transcript in one tap, the way the mic button drops it in the box.
|
||||
const voiceAvailable = useVoiceAvailable();
|
||||
const [voiceError, setVoiceError] = useState<string | null>(null);
|
||||
const voiceErrorTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const handleVoiceError = useCallback((msg: string) => {
|
||||
setVoiceError(msg);
|
||||
if (voiceErrorTimer.current) clearTimeout(voiceErrorTimer.current);
|
||||
voiceErrorTimer.current = setTimeout(() => setVoiceError(null), 4000);
|
||||
}, []);
|
||||
useEffect(() => () => {
|
||||
if (voiceErrorTimer.current) clearTimeout(voiceErrorTimer.current);
|
||||
}, []);
|
||||
const noopTranscript = useCallback(() => {}, []);
|
||||
const { state: voiceState, toggle: voiceToggle, stop: voiceStop } = useVoiceInput(
|
||||
onVoiceTranscript ?? noopTranscript,
|
||||
handleVoiceError,
|
||||
);
|
||||
const isRecording = voiceState === 'recording';
|
||||
const isTranscribing = voiceState === 'transcribing';
|
||||
|
||||
const textareaRect = textareaRef.current?.getBoundingClientRect();
|
||||
const commandMenuPosition = {
|
||||
top: textareaRect ? Math.max(16, textareaRect.top - 316) : 0,
|
||||
@@ -309,6 +337,10 @@ export default function ChatComposer({
|
||||
<ImageIcon />
|
||||
</PromptInputButton>
|
||||
|
||||
{onVoiceTranscript && voiceAvailable && (
|
||||
<VoiceInputButton state={voiceState} onToggle={voiceToggle} errorMsg={voiceError} />
|
||||
)}
|
||||
|
||||
<button
|
||||
type="button"
|
||||
onClick={onModeSwitch}
|
||||
@@ -387,10 +419,21 @@ export default function ChatComposer({
|
||||
{sendByCtrlEnter ? t('input.hintText.ctrlEnter') : t('input.hintText.enter')}
|
||||
</div>
|
||||
<PromptInputSubmit
|
||||
onClick={isLoading ? onAbortSession : undefined}
|
||||
disabled={!isLoading && !input.trim()}
|
||||
onClick={
|
||||
isLoading
|
||||
? onAbortSession
|
||||
: isRecording
|
||||
? (e: MouseEvent<HTMLButtonElement>) => {
|
||||
e.preventDefault();
|
||||
voiceStop({ send: true });
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
disabled={isLoading ? false : isRecording ? false : isTranscribing ? true : !input.trim()}
|
||||
className="h-10 w-10 sm:h-10 sm:w-10"
|
||||
/>
|
||||
>
|
||||
{isTranscribing ? <Loader2 className="h-4 w-4 animate-spin" /> : undefined}
|
||||
</PromptInputSubmit>
|
||||
</div>
|
||||
</PromptInputFooter>
|
||||
</PromptInput>
|
||||
|
||||
@@ -15,6 +15,7 @@ import { Reasoning, ReasoningTrigger, ReasoningContent } from '../../../../share
|
||||
|
||||
import { Markdown } from './Markdown';
|
||||
import MessageCopyControl from './MessageCopyControl';
|
||||
import MessageSpeakControl from './MessageSpeakControl';
|
||||
|
||||
type DiffLine = {
|
||||
type: string;
|
||||
@@ -415,6 +416,9 @@ const MessageComponent = memo(({ message, prevMessage, createDiff, onFileOpen, a
|
||||
{shouldShowAssistantCopyControl && (
|
||||
<MessageCopyControl content={assistantCopyContent} messageType="assistant" />
|
||||
)}
|
||||
{shouldShowAssistantCopyControl && (
|
||||
<MessageSpeakControl content={assistantCopyContent} />
|
||||
)}
|
||||
{!isGrouped && <span>{formattedTime}</span>}
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import { Volume2, Loader2, Square } from 'lucide-react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { useTts } from '../../hooks/useTts';
|
||||
import { useVoiceAvailable } from '../../hooks/useVoiceAvailable';
|
||||
|
||||
// Tap-to-speak button beside the copy control on assistant messages.
|
||||
// Renders nothing unless the optional voice feature is enabled.
|
||||
const MessageSpeakControl = ({ content }: { content: string }) => {
|
||||
const { t } = useTranslation('chat');
|
||||
const available = useVoiceAvailable();
|
||||
const { state, toggle, error } = useTts(() => content);
|
||||
|
||||
if (!available) return null;
|
||||
|
||||
const title =
|
||||
state === 'playing' ? t('voice.stopSpeaking') : state === 'loading' ? t('voice.loading') : t('voice.speak');
|
||||
|
||||
return (
|
||||
<span className="relative inline-flex">
|
||||
{error && (
|
||||
<span className="absolute bottom-full left-1/2 z-10 mb-1 max-w-[240px] -translate-x-1/2 whitespace-normal rounded bg-red-600 px-2 py-1 text-center text-xs text-white shadow-lg">
|
||||
{error}
|
||||
</span>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
onClick={toggle}
|
||||
title={title}
|
||||
aria-label={title}
|
||||
className="inline-flex items-center gap-1 rounded px-1 py-0.5 text-gray-400 transition-colors hover:text-gray-600 dark:text-gray-500 dark:hover:text-gray-300"
|
||||
>
|
||||
{state === 'playing' ? (
|
||||
<Square className="h-3.5 w-3.5" />
|
||||
) : state === 'loading' ? (
|
||||
<Loader2 className="h-3.5 w-3.5 animate-spin" />
|
||||
) : (
|
||||
<Volume2 className="h-3.5 w-3.5" />
|
||||
)}
|
||||
</button>
|
||||
</span>
|
||||
);
|
||||
};
|
||||
|
||||
export default MessageSpeakControl;
|
||||
46
src/components/chat/view/subcomponents/VoiceInputButton.tsx
Normal file
46
src/components/chat/view/subcomponents/VoiceInputButton.tsx
Normal file
@@ -0,0 +1,46 @@
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { Mic, Square, Loader2 } from 'lucide-react';
|
||||
|
||||
import { PromptInputButton } from '../../../../shared/view/ui';
|
||||
import type { VoiceInputState } from '../../hooks/useVoiceInput';
|
||||
|
||||
type Props = {
|
||||
state: VoiceInputState;
|
||||
onToggle: () => void;
|
||||
errorMsg?: string | null;
|
||||
};
|
||||
|
||||
// Push-to-talk mic button (presentational). Recording state and the stop-and-send action
|
||||
// are owned by the composer so the main Send button can drive them too. This button just
|
||||
// starts recording and, while recording, stops and drops the transcript into the input box.
|
||||
export default function VoiceInputButton({ state, onToggle, errorMsg }: Props) {
|
||||
const { t } = useTranslation('chat');
|
||||
|
||||
const icon =
|
||||
state === 'recording' ? (
|
||||
<Square className="text-red-500" />
|
||||
) : state === 'transcribing' ? (
|
||||
<Loader2 className="animate-spin" />
|
||||
) : (
|
||||
<Mic />
|
||||
);
|
||||
|
||||
return (
|
||||
<span className="relative inline-flex">
|
||||
{errorMsg && (
|
||||
<span className="absolute bottom-full left-1/2 mb-1 -translate-x-1/2 whitespace-nowrap rounded bg-red-600 px-2 py-1 text-xs text-white shadow-lg">
|
||||
{errorMsg}
|
||||
</span>
|
||||
)}
|
||||
<PromptInputButton
|
||||
tooltip={{ content: state === 'recording' ? t('voice.stopRecording') : t('voice.input') }}
|
||||
onClick={(e: { preventDefault: () => void }) => {
|
||||
e.preventDefault();
|
||||
onToggle();
|
||||
}}
|
||||
>
|
||||
{icon}
|
||||
</PromptInputButton>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { useCallback, useEffect, useState } from 'react';
|
||||
import { api } from '../../../utils/api';
|
||||
import type { CodeEditorFile } from '../types/types';
|
||||
import { isBinaryFile } from '../utils/binaryFile';
|
||||
import { getPreviewKind } from '../utils/previewableFile';
|
||||
|
||||
type UseCodeEditorDocumentParams = {
|
||||
file: CodeEditorFile;
|
||||
@@ -23,6 +24,9 @@ export const useCodeEditorDocument = ({ file, projectPath }: UseCodeEditorDocume
|
||||
const [saveSuccess, setSaveSuccess] = useState(false);
|
||||
const [saveError, setSaveError] = useState<string | null>(null);
|
||||
const [isBinary, setIsBinary] = useState(false);
|
||||
// Some binaries (images, PDFs, audio, video) can be rendered natively, so the
|
||||
// editor shows an inline preview instead of the generic binary placeholder.
|
||||
const previewKind = getPreviewKind(file.name);
|
||||
// `fileProjectId` is the DB primary key passed down from the editor sidebar;
|
||||
// the fallback to `projectPath` preserves older callers that didn't yet
|
||||
// propagate the identifier.
|
||||
@@ -38,8 +42,19 @@ export const useCodeEditorDocument = ({ file, projectPath }: UseCodeEditorDocume
|
||||
setLoading(true);
|
||||
setIsBinary(false);
|
||||
|
||||
// Natively previewable media (image/pdf/audio/video) is rendered by
|
||||
// CodeEditorMediaPreview, so there is nothing to read as text here.
|
||||
// Clear any buffer left over from a previously opened text file so a
|
||||
// stray save can't write stale content over the binary file.
|
||||
if (getPreviewKind(file.name)) {
|
||||
setContent('');
|
||||
setLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if file is binary by extension
|
||||
if (isBinaryFile(file.name)) {
|
||||
setContent('');
|
||||
setIsBinary(true);
|
||||
setLoading(false);
|
||||
return;
|
||||
@@ -76,6 +91,12 @@ export const useCodeEditorDocument = ({ file, projectPath }: UseCodeEditorDocume
|
||||
}, [file.diffInfo, file.name, fileDiffNewString, fileDiffOldString, fileName, filePath, fileProjectId]);
|
||||
|
||||
const handleSave = useCallback(async () => {
|
||||
// Preview-only and binary files have no editable text buffer; never write
|
||||
// them back (e.g. via Cmd/Ctrl+S) or we'd corrupt the file on disk.
|
||||
if (previewKind || isBinaryFile(fileName)) {
|
||||
return;
|
||||
}
|
||||
|
||||
setSaving(true);
|
||||
setSaveError(null);
|
||||
|
||||
@@ -109,7 +130,7 @@ export const useCodeEditorDocument = ({ file, projectPath }: UseCodeEditorDocume
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
}, [content, filePath, fileProjectId]);
|
||||
}, [content, filePath, fileProjectId, previewKind, fileName]);
|
||||
|
||||
const handleDownload = useCallback(() => {
|
||||
const blob = new Blob([content], { type: 'text/plain' });
|
||||
@@ -134,6 +155,8 @@ export const useCodeEditorDocument = ({ file, projectPath }: UseCodeEditorDocume
|
||||
saveSuccess,
|
||||
saveError,
|
||||
isBinary,
|
||||
previewKind,
|
||||
fileProjectId,
|
||||
handleSave,
|
||||
handleDownload,
|
||||
};
|
||||
|
||||
63
src/components/code-editor/utils/previewableFile.ts
Normal file
63
src/components/code-editor/utils/previewableFile.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
// Some binary files can't be edited as text, but the browser can still render
|
||||
// them natively (images, PDFs, audio, video). For those we show an inline
|
||||
// preview instead of the generic "binary file" placeholder. Anything not listed
|
||||
// here (zip, exe, avi, mkv, fonts, ...) falls through to the binary message.
|
||||
|
||||
export type PreviewKind = 'image' | 'pdf' | 'video' | 'audio';
|
||||
|
||||
// Single source of truth: every extension the browser can preview, mapped to the
|
||||
// MIME type we apply when the server response has a missing/generic Content-Type.
|
||||
// The preview kind is derived from the MIME type so the two never drift apart.
|
||||
// Formats browsers generally can't play (avi, mkv, flv, wmv) are intentionally
|
||||
// absent and keep the binary fallback.
|
||||
const EXTENSION_MIME: Record<string, string> = {
|
||||
// Images
|
||||
png: 'image/png',
|
||||
jpg: 'image/jpeg',
|
||||
jpeg: 'image/jpeg',
|
||||
gif: 'image/gif',
|
||||
svg: 'image/svg+xml',
|
||||
webp: 'image/webp',
|
||||
ico: 'image/x-icon',
|
||||
bmp: 'image/bmp',
|
||||
avif: 'image/avif',
|
||||
apng: 'image/apng',
|
||||
// PDF
|
||||
pdf: 'application/pdf',
|
||||
// Video
|
||||
mp4: 'video/mp4',
|
||||
webm: 'video/webm',
|
||||
ogv: 'video/ogg',
|
||||
mov: 'video/quicktime',
|
||||
m4v: 'video/x-m4v',
|
||||
// Audio
|
||||
mp3: 'audio/mpeg',
|
||||
wav: 'audio/wav',
|
||||
m4a: 'audio/mp4',
|
||||
aac: 'audio/aac',
|
||||
flac: 'audio/flac',
|
||||
opus: 'audio/opus',
|
||||
oga: 'audio/ogg',
|
||||
ogg: 'audio/ogg',
|
||||
weba: 'audio/webm',
|
||||
};
|
||||
|
||||
const extensionOf = (filename: string): string => filename.split('.').pop()?.toLowerCase() ?? '';
|
||||
|
||||
const kindForMime = (mime: string): PreviewKind | null => {
|
||||
if (mime === 'application/pdf') return 'pdf';
|
||||
if (mime.startsWith('image/')) return 'image';
|
||||
if (mime.startsWith('video/')) return 'video';
|
||||
if (mime.startsWith('audio/')) return 'audio';
|
||||
return null;
|
||||
};
|
||||
|
||||
export const getPreviewKind = (filename: string): PreviewKind | null => {
|
||||
const mime = EXTENSION_MIME[extensionOf(filename)];
|
||||
return mime ? kindForMime(mime) : null;
|
||||
};
|
||||
|
||||
// MIME type to fall back to when the server returns no/generic Content-Type.
|
||||
// Returns undefined for non-previewable extensions.
|
||||
export const getPreviewMimeType = (filename: string): string | undefined =>
|
||||
EXTENSION_MIME[extensionOf(filename)];
|
||||
@@ -1,8 +1,9 @@
|
||||
import { EditorView } from '@codemirror/view';
|
||||
import { unifiedMergeView } from '@codemirror/merge';
|
||||
import type { Extension } from '@codemirror/state';
|
||||
import { useMemo, useState } from 'react';
|
||||
import { useCallback, useMemo, useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
|
||||
import { usePaletteOps } from '../../../contexts/PaletteOpsContext';
|
||||
import { useCodeEditorDocument } from '../hooks/useCodeEditorDocument';
|
||||
import { useCodeEditorSettings } from '../hooks/useCodeEditorSettings';
|
||||
@@ -11,11 +12,13 @@ import type { CodeEditorFile } from '../types/types';
|
||||
import { createMinimapExtension, createScrollToFirstChunkExtension, getLanguageExtensions } from '../utils/editorExtensions';
|
||||
import { getEditorStyles } from '../utils/editorStyles';
|
||||
import { createEditorToolbarPanelExtension } from '../utils/editorToolbarPanel';
|
||||
|
||||
import CodeEditorFooter from './subcomponents/CodeEditorFooter';
|
||||
import CodeEditorHeader from './subcomponents/CodeEditorHeader';
|
||||
import CodeEditorLoadingState from './subcomponents/CodeEditorLoadingState';
|
||||
import CodeEditorSurface from './subcomponents/CodeEditorSurface';
|
||||
import CodeEditorBinaryFile from './subcomponents/CodeEditorBinaryFile';
|
||||
import CodeEditorMediaPreview from './subcomponents/CodeEditorMediaPreview';
|
||||
|
||||
type CodeEditorProps = {
|
||||
file: CodeEditorFile;
|
||||
@@ -58,6 +61,8 @@ export default function CodeEditor({
|
||||
saveSuccess,
|
||||
saveError,
|
||||
isBinary,
|
||||
previewKind,
|
||||
fileProjectId,
|
||||
handleSave,
|
||||
handleDownload,
|
||||
} = useCodeEditorDocument({
|
||||
@@ -70,6 +75,29 @@ export default function CodeEditor({
|
||||
return extension === 'md' || extension === 'markdown';
|
||||
}, [file.name]);
|
||||
|
||||
const isHtmlPreviewFile = useMemo(() => {
|
||||
const extension = file.name.split('.').pop()?.toLowerCase();
|
||||
return extension === 'html' || extension === 'htm';
|
||||
}, [file.name]);
|
||||
|
||||
const openHtmlPreview = useCallback(() => {
|
||||
const previewWindow = window.open('', '_blank');
|
||||
if (!previewWindow) return;
|
||||
|
||||
previewWindow.opener = null;
|
||||
previewWindow.document.title = file.name;
|
||||
previewWindow.document.body.style.margin = '0';
|
||||
|
||||
const iframe = previewWindow.document.createElement('iframe');
|
||||
iframe.title = file.name;
|
||||
iframe.sandbox.add('allow-forms', 'allow-modals', 'allow-popups', 'allow-scripts');
|
||||
iframe.style.cssText = 'position:fixed;inset:0;width:100%;height:100%;border:0;background:white';
|
||||
|
||||
iframe.srcdoc = content;
|
||||
|
||||
previewWindow.document.body.appendChild(iframe);
|
||||
}, [content, file.name]);
|
||||
|
||||
const minimapExtension = useMemo(
|
||||
() => (
|
||||
createMinimapExtension({
|
||||
@@ -162,6 +190,30 @@ export default function CodeEditor({
|
||||
);
|
||||
}
|
||||
|
||||
// Natively previewable media (image/pdf/audio/video) is rendered inline
|
||||
// instead of showing the generic "cannot be displayed" placeholder.
|
||||
if (previewKind) {
|
||||
return (
|
||||
<CodeEditorMediaPreview
|
||||
file={file}
|
||||
kind={previewKind}
|
||||
projectId={fileProjectId}
|
||||
isSidebar={isSidebar}
|
||||
isFullscreen={isFullscreen}
|
||||
onClose={onClose}
|
||||
onToggleFullscreen={() => setIsFullscreen((previous) => !previous)}
|
||||
labels={{
|
||||
loading: t('filePreview.loading', 'Loading preview...'),
|
||||
error: t('filePreview.error', 'Unable to display this file.'),
|
||||
openInNewTab: t('filePreview.openInNewTab', 'Open in new tab'),
|
||||
fullscreen: t('actions.fullscreen', 'Fullscreen'),
|
||||
exitFullscreen: t('actions.exitFullscreen', 'Exit fullscreen'),
|
||||
close: t('actions.close', 'Close'),
|
||||
}}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
// Binary file display
|
||||
if (isBinary) {
|
||||
return (
|
||||
@@ -197,10 +249,12 @@ export default function CodeEditor({
|
||||
isSidebar={isSidebar}
|
||||
isFullscreen={isFullscreen}
|
||||
isMarkdownFile={isMarkdownFile}
|
||||
isHtmlPreviewFile={isHtmlPreviewFile}
|
||||
markdownPreview={markdownPreview}
|
||||
saving={saving}
|
||||
saveSuccess={saveSuccess}
|
||||
onToggleMarkdownPreview={() => setMarkdownPreview((previous) => !previous)}
|
||||
onOpenHtmlPreview={openHtmlPreview}
|
||||
onOpenSettings={() => paletteOps.openSettings('appearance')}
|
||||
onDownload={handleDownload}
|
||||
onSave={handleSave}
|
||||
@@ -210,6 +264,7 @@ export default function CodeEditor({
|
||||
showingChanges: t('header.showingChanges'),
|
||||
editMarkdown: t('actions.editMarkdown'),
|
||||
previewMarkdown: t('actions.previewMarkdown'),
|
||||
previewHtml: t('actions.previewHtml', 'Open HTML preview in new tab'),
|
||||
settings: t('toolbar.settings'),
|
||||
download: t('actions.download'),
|
||||
save: t('actions.save'),
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { Code2, Download, Eye, Maximize2, Minimize2, Save, Settings as SettingsIcon, X } from 'lucide-react';
|
||||
|
||||
import type { CodeEditorFile } from '../../types/types';
|
||||
|
||||
type CodeEditorHeaderProps = {
|
||||
@@ -6,10 +7,12 @@ type CodeEditorHeaderProps = {
|
||||
isSidebar: boolean;
|
||||
isFullscreen: boolean;
|
||||
isMarkdownFile: boolean;
|
||||
isHtmlPreviewFile: boolean;
|
||||
markdownPreview: boolean;
|
||||
saving: boolean;
|
||||
saveSuccess: boolean;
|
||||
onToggleMarkdownPreview: () => void;
|
||||
onOpenHtmlPreview: () => void;
|
||||
onOpenSettings: () => void;
|
||||
onDownload: () => void;
|
||||
onSave: () => void;
|
||||
@@ -19,6 +22,7 @@ type CodeEditorHeaderProps = {
|
||||
showingChanges: string;
|
||||
editMarkdown: string;
|
||||
previewMarkdown: string;
|
||||
previewHtml: string;
|
||||
settings: string;
|
||||
download: string;
|
||||
save: string;
|
||||
@@ -35,10 +39,12 @@ export default function CodeEditorHeader({
|
||||
isSidebar,
|
||||
isFullscreen,
|
||||
isMarkdownFile,
|
||||
isHtmlPreviewFile,
|
||||
markdownPreview,
|
||||
saving,
|
||||
saveSuccess,
|
||||
onToggleMarkdownPreview,
|
||||
onOpenHtmlPreview,
|
||||
onOpenSettings,
|
||||
onDownload,
|
||||
onSave,
|
||||
@@ -82,6 +88,17 @@ export default function CodeEditorHeader({
|
||||
</button>
|
||||
)}
|
||||
|
||||
{isHtmlPreviewFile && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onOpenHtmlPreview}
|
||||
className="flex items-center justify-center rounded-md p-1.5 text-gray-600 hover:bg-gray-100 hover:text-gray-900 dark:text-gray-400 dark:hover:bg-gray-800 dark:hover:text-white"
|
||||
title={labels.previewHtml}
|
||||
>
|
||||
<Eye className="h-4 w-4" />
|
||||
</button>
|
||||
)}
|
||||
|
||||
<button
|
||||
type="button"
|
||||
onClick={onOpenSettings}
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
|
||||
import { authenticatedFetch } from '../../../../utils/api';
|
||||
import type { CodeEditorFile } from '../../types/types';
|
||||
import { getPreviewMimeType, type PreviewKind } from '../../utils/previewableFile';
|
||||
|
||||
type CodeEditorMediaPreviewProps = {
|
||||
file: CodeEditorFile;
|
||||
kind: PreviewKind;
|
||||
// DB projectId used to build the raw-content URL; falls back to projectPath
|
||||
// for older callers, mirroring useCodeEditorDocument.
|
||||
projectId?: string;
|
||||
isSidebar: boolean;
|
||||
isFullscreen: boolean;
|
||||
onClose: () => void;
|
||||
onToggleFullscreen: () => void;
|
||||
labels: {
|
||||
loading: string;
|
||||
error: string;
|
||||
openInNewTab: string;
|
||||
fullscreen: string;
|
||||
exitFullscreen: string;
|
||||
close: string;
|
||||
};
|
||||
};
|
||||
|
||||
// Reject a "PDF" whose bytes aren't actually a PDF before handing it to the
|
||||
// same-origin iframe, so a mislabeled HTML/SVG file can't run in the app origin.
|
||||
const PDF_HEADER_SCAN_BYTES = 1024;
|
||||
|
||||
const looksLikePdf = async (blob: Blob): Promise<boolean> => {
|
||||
const header = await blob.slice(0, PDF_HEADER_SCAN_BYTES).arrayBuffer();
|
||||
// PDFs must contain the "%PDF-" marker at the very start of the file.
|
||||
return new TextDecoder('latin1').decode(header).includes('%PDF-');
|
||||
};
|
||||
|
||||
export default function CodeEditorMediaPreview({
|
||||
file,
|
||||
kind,
|
||||
projectId,
|
||||
isSidebar,
|
||||
isFullscreen,
|
||||
onClose,
|
||||
onToggleFullscreen,
|
||||
labels,
|
||||
}: CodeEditorMediaPreviewProps) {
|
||||
const [url, setUrl] = useState<string | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
// Identifies which file the current `url` was loaded for. Rendering is gated on
|
||||
// this so a blob from a previously-opened file can never show under the new
|
||||
// file (the editor reuses this component instance across files).
|
||||
const [loadedKey, setLoadedKey] = useState<string | null>(null);
|
||||
const sourceKey = `${projectId ?? ''}:${file.path}:${kind}`;
|
||||
|
||||
useEffect(() => {
|
||||
if (!projectId) {
|
||||
setUrl(null);
|
||||
setLoadedKey(null);
|
||||
setError(labels.error);
|
||||
setLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
let objectUrl: string | null = null;
|
||||
const controller = new AbortController();
|
||||
|
||||
const loadMedia = async () => {
|
||||
try {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
setUrl(null);
|
||||
|
||||
// The content endpoint requires the auth header, so we fetch the bytes
|
||||
// ourselves and hand the media element a blob URL instead of a bare src.
|
||||
// Fetching a blob (rather than streaming) also lets <video>/<audio> seek.
|
||||
const contentUrl = `/api/projects/${projectId}/files/content?path=${encodeURIComponent(file.path)}`;
|
||||
const response = await authenticatedFetch(contentUrl, { signal: controller.signal });
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Request failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
|
||||
// Pick the MIME type to expose to the browser. Preserve a valid
|
||||
// Content-Type from the server, but supply an extension-specific
|
||||
// default when it is missing or generic (application/octet-stream),
|
||||
// otherwise formats like webm/ogg/flac/svg won't render.
|
||||
const fallbackMime = getPreviewMimeType(file.name);
|
||||
const isGenericType = !blob.type || blob.type === 'application/octet-stream';
|
||||
const isMislabeledVideo = kind === 'video' && Boolean(fallbackMime) && !blob.type.startsWith('video/');
|
||||
let outType = isGenericType || isMislabeledVideo ? (fallbackMime ?? blob.type) : blob.type;
|
||||
|
||||
if (kind === 'pdf') {
|
||||
// The PDF renders in a same-origin <iframe>, so verify the bytes are
|
||||
// really a PDF and pin the type to application/pdf. That forces the
|
||||
// browser's PDF handler and prevents a mislabeled HTML/SVG file from
|
||||
// executing scripts in the app's origin.
|
||||
if (!(await looksLikePdf(blob))) {
|
||||
throw new Error('File is not a valid PDF');
|
||||
}
|
||||
outType = 'application/pdf';
|
||||
}
|
||||
|
||||
const typed = outType && outType !== blob.type ? new Blob([blob], { type: outType }) : blob;
|
||||
objectUrl = URL.createObjectURL(typed);
|
||||
|
||||
// The cleanup may have already run (deps changed during an await), in
|
||||
// which case it revoked nothing because objectUrl was still null. Don't
|
||||
// publish a URL the cleanup will never revoke — drop it ourselves.
|
||||
if (controller.signal.aborted) {
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
objectUrl = null;
|
||||
return;
|
||||
}
|
||||
|
||||
setUrl(objectUrl);
|
||||
setLoadedKey(sourceKey);
|
||||
} catch (loadError: unknown) {
|
||||
if (loadError instanceof Error && loadError.name === 'AbortError') {
|
||||
return;
|
||||
}
|
||||
console.error('Error loading preview:', loadError);
|
||||
setError(labels.error);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
loadMedia();
|
||||
|
||||
return () => {
|
||||
controller.abort();
|
||||
if (objectUrl) {
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
}
|
||||
};
|
||||
}, [file.path, file.name, projectId, kind, sourceKey, labels.error]);
|
||||
|
||||
// Only expose the blob once it matches the file currently being shown, so a
|
||||
// stale URL from the previous file is never rendered during a switch.
|
||||
const currentUrl = url && loadedKey === sourceKey ? url : null;
|
||||
|
||||
// SVGs render safely inline via <img> (scripts don't execute there), but the
|
||||
// open-in-new-tab link is a top-level navigation. A blob URL inherits the
|
||||
// app's origin, so a user-controlled SVG with an embedded <script> would run
|
||||
// as same-origin script. Withhold the new-tab action for SVGs.
|
||||
const isSvg = getPreviewMimeType(file.name) === 'image/svg+xml';
|
||||
const canOpenInNewTab = Boolean(currentUrl) && !isSvg;
|
||||
|
||||
const renderMedia = () => {
|
||||
if (!currentUrl) return null;
|
||||
switch (kind) {
|
||||
case 'image':
|
||||
return (
|
||||
<img
|
||||
src={currentUrl}
|
||||
alt={file.name}
|
||||
className="max-h-full max-w-full object-contain"
|
||||
/>
|
||||
);
|
||||
case 'pdf':
|
||||
// Not sandboxed on purpose: the browser's built-in PDF viewer refuses to
|
||||
// load inside a sandboxed frame (any `sandbox` value yields a broken
|
||||
// viewer). Script execution is instead prevented upstream by validating
|
||||
// the PDF magic bytes and pinning the blob's MIME type to application/pdf.
|
||||
return <iframe src={currentUrl} title={file.name} className="h-full w-full border-0 bg-white" />;
|
||||
case 'video':
|
||||
return (
|
||||
<video src={currentUrl} controls className="max-h-full max-w-full" autoPlay={false}>
|
||||
{labels.error}
|
||||
</video>
|
||||
);
|
||||
case 'audio':
|
||||
return (
|
||||
<div className="flex w-full max-w-xl flex-col items-center gap-4 px-6">
|
||||
<p className="max-w-full truncate text-sm text-muted-foreground">{file.name}</p>
|
||||
<audio src={currentUrl} controls className="w-full">
|
||||
{labels.error}
|
||||
</audio>
|
||||
</div>
|
||||
);
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const previewBody = (
|
||||
<div className="relative flex h-full w-full flex-col items-center justify-center bg-muted/30 p-2">
|
||||
{loading && (
|
||||
<div className="text-sm text-muted-foreground">{labels.loading}</div>
|
||||
)}
|
||||
|
||||
{!loading && currentUrl && renderMedia()}
|
||||
|
||||
{!loading && !currentUrl && (
|
||||
<div className="flex flex-col items-center gap-3 p-8 text-center text-muted-foreground">
|
||||
<p className="text-sm">{error || labels.error}</p>
|
||||
<p className="break-all text-xs">{file.path}</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
|
||||
const headerActions = (
|
||||
<div className="flex shrink-0 items-center gap-0.5">
|
||||
{canOpenInNewTab && currentUrl && (
|
||||
<a
|
||||
href={currentUrl}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="flex items-center justify-center rounded-md p-1.5 text-gray-600 hover:bg-gray-100 hover:text-gray-900 dark:text-gray-400 dark:hover:bg-gray-800 dark:hover:text-white"
|
||||
aria-label={labels.openInNewTab}
|
||||
title={labels.openInNewTab}
|
||||
>
|
||||
<svg aria-hidden="true" className="h-4 w-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14" />
|
||||
</svg>
|
||||
</a>
|
||||
)}
|
||||
{!isSidebar && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleFullscreen}
|
||||
className="flex items-center justify-center rounded-md p-1.5 text-gray-600 hover:bg-gray-100 hover:text-gray-900 dark:text-gray-400 dark:hover:bg-gray-800 dark:hover:text-white"
|
||||
aria-label={isFullscreen ? labels.exitFullscreen : labels.fullscreen}
|
||||
title={isFullscreen ? labels.exitFullscreen : labels.fullscreen}
|
||||
>
|
||||
{isFullscreen ? (
|
||||
<svg aria-hidden="true" className="h-4 w-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M9 9V4.5M9 9H4.5M9 9L3.5 3.5M9 15v4.5M9 15H4.5M9 15l-5.5 5.5M15 9h4.5M15 9V4.5M15 9l5.5-5.5M15 15h4.5M15 15v4.5m0-4.5l5.5 5.5" />
|
||||
</svg>
|
||||
) : (
|
||||
<svg aria-hidden="true" className="h-4 w-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 8V4m0 0h4M4 4l5 5m11-1V4m0 0h-4m4 0l-5 5M4 16v4m0 0h4m-4 0l5-5m11 5l-5-5m5 5v-4m0 4h-4" />
|
||||
</svg>
|
||||
)}
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
onClick={onClose}
|
||||
className="flex items-center justify-center rounded-md p-1.5 text-gray-600 hover:bg-gray-100 hover:text-gray-900 dark:text-gray-400 dark:hover:bg-gray-800 dark:hover:text-white"
|
||||
aria-label={labels.close}
|
||||
title={labels.close}
|
||||
>
|
||||
<svg aria-hidden="true" className="h-4 w-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M6 18L18 6M6 6l12 12" />
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
|
||||
const header = (
|
||||
<div className="flex flex-shrink-0 items-center justify-between border-b border-border px-3 py-1.5">
|
||||
<div className="flex min-w-0 flex-1 items-center gap-2">
|
||||
<h3 className="truncate text-sm font-medium text-gray-900 dark:text-white">{file.name}</h3>
|
||||
</div>
|
||||
{headerActions}
|
||||
</div>
|
||||
);
|
||||
|
||||
if (isSidebar) {
|
||||
return (
|
||||
<div className="flex h-full w-full flex-col bg-background">
|
||||
{header}
|
||||
{previewBody}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const containerClassName = isFullscreen
|
||||
? 'fixed inset-0 z-[9999] bg-background flex flex-col'
|
||||
: 'fixed inset-0 z-[9999] md:bg-black/50 md:flex md:items-center md:justify-center md:p-4';
|
||||
|
||||
const innerClassName = isFullscreen
|
||||
? 'bg-background flex flex-col w-full h-full'
|
||||
: 'bg-background shadow-2xl flex flex-col w-full h-full md:rounded-lg md:shadow-2xl md:w-full md:max-w-6xl md:h-[80vh] md:max-h-[80vh]';
|
||||
|
||||
return (
|
||||
<div className={containerClassName}>
|
||||
<div className={innerClassName}>
|
||||
{header}
|
||||
{previewBody}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
Eye,
|
||||
Languages,
|
||||
Maximize2,
|
||||
Mic,
|
||||
} from 'lucide-react';
|
||||
import type { PreferenceToggleItem } from './types';
|
||||
|
||||
@@ -54,4 +55,9 @@ export const INPUT_SETTING_TOGGLES: PreferenceToggleItem[] = [
|
||||
labelKey: 'quickSettings.sendByCtrlEnter',
|
||||
icon: Languages,
|
||||
},
|
||||
{
|
||||
key: 'voiceEnabled',
|
||||
labelKey: 'quickSettings.voiceEnabled',
|
||||
icon: Mic,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -6,7 +6,8 @@ export type PreferenceToggleKey =
|
||||
| 'showRawParameters'
|
||||
| 'showThinking'
|
||||
| 'autoScrollToBottom'
|
||||
| 'sendByCtrlEnter';
|
||||
| 'sendByCtrlEnter'
|
||||
| 'voiceEnabled';
|
||||
|
||||
export type QuickSettingsPreferences = Record<PreferenceToggleKey, boolean>;
|
||||
|
||||
|
||||
@@ -28,6 +28,9 @@ export default function QuickSettingsContent({
|
||||
onPreferenceChange,
|
||||
}: QuickSettingsContentProps) {
|
||||
const { t } = useTranslation('settings');
|
||||
const inputSettingToggles = preferences.voiceEnabled
|
||||
? INPUT_SETTING_TOGGLES
|
||||
: INPUT_SETTING_TOGGLES.filter(({ key }) => key !== 'voiceEnabled');
|
||||
|
||||
const renderToggleRows = (items: PreferenceToggleItem[]) => (
|
||||
items.map(({ key, labelKey, icon }) => (
|
||||
@@ -67,7 +70,7 @@ export default function QuickSettingsContent({
|
||||
</QuickSettingsSection>
|
||||
|
||||
<QuickSettingsSection title={t('quickSettings.sections.inputSettings')}>
|
||||
{renderToggleRows(INPUT_SETTING_TOGGLES)}
|
||||
{renderToggleRows(inputSettingToggles)}
|
||||
<p className="ml-3 text-xs text-gray-500 dark:text-gray-400">
|
||||
{t('quickSettings.sendByCtrlEnterDescription')}
|
||||
</p>
|
||||
|
||||
@@ -27,12 +27,14 @@ export default function QuickSettingsPanelView() {
|
||||
showThinking: preferences.showThinking,
|
||||
autoScrollToBottom: preferences.autoScrollToBottom,
|
||||
sendByCtrlEnter: preferences.sendByCtrlEnter,
|
||||
voiceEnabled: preferences.voiceEnabled,
|
||||
}), [
|
||||
preferences.autoExpandTools,
|
||||
preferences.autoScrollToBottom,
|
||||
preferences.sendByCtrlEnter,
|
||||
preferences.showRawParameters,
|
||||
preferences.showThinking,
|
||||
preferences.voiceEnabled,
|
||||
]);
|
||||
|
||||
const handlePreferenceChange = useCallback(
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { Dispatch, SetStateAction } from 'react';
|
||||
import type { LLMProvider } from '../../../types/app';
|
||||
import type { ProviderAuthStatus } from '../../provider-auth/types';
|
||||
|
||||
export type SettingsMainTab = 'agents' | 'appearance' | 'git' | 'api' | 'tasks' | 'browser' | 'notifications' | 'plugins' | 'about';
|
||||
export type SettingsMainTab = 'agents' | 'appearance' | 'git' | 'api' | 'voice' | 'tasks' | 'browser' | 'notifications' | 'plugins' | 'about';
|
||||
export type AgentProvider = LLMProvider;
|
||||
export type AgentCategory = 'account' | 'permissions' | 'mcp' | 'skills';
|
||||
export type ProjectSortOrder = 'name' | 'date';
|
||||
|
||||
@@ -7,6 +7,7 @@ import SettingsSidebar from '../view/SettingsSidebar';
|
||||
import AgentsSettingsTab from '../view/tabs/agents-settings/AgentsSettingsTab';
|
||||
import AppearanceSettingsTab from '../view/tabs/AppearanceSettingsTab';
|
||||
import CredentialsSettingsTab from '../view/tabs/api-settings/CredentialsSettingsTab';
|
||||
import VoiceSettingsTab from '../view/tabs/VoiceSettingsTab';
|
||||
import GitSettingsTab from '../view/tabs/git-settings/GitSettingsTab';
|
||||
import BrowserUseSettingsTab from '../view/tabs/browser-use-settings/BrowserUseSettingsTab';
|
||||
import NotificationsSettingsTab from '../view/tabs/NotificationsSettingsTab';
|
||||
@@ -157,6 +158,8 @@ function Settings({ isOpen, onClose, projects = [], initialTab = 'agents' }: Set
|
||||
|
||||
{activeTab === 'api' && <CredentialsSettingsTab />}
|
||||
|
||||
{activeTab === 'voice' && <VoiceSettingsTab />}
|
||||
|
||||
{activeTab === 'plugins' && <PluginSettingsTab />}
|
||||
|
||||
{activeTab === 'about' && <AboutTab />}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Bell, Bot, GitBranch, Info, Key, ListChecks, MonitorPlay, Palette, Puzzle } from 'lucide-react';
|
||||
import { Bell, Bot, GitBranch, Info, Key, ListChecks, Mic, MonitorPlay, Palette, Puzzle } from 'lucide-react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
|
||||
import { cn } from '../../../lib/utils';
|
||||
import { PillBar, Pill } from '../../../shared/view/ui';
|
||||
import type { SettingsMainTab } from '../types/types';
|
||||
@@ -20,6 +21,7 @@ const NAV_ITEMS: NavItem[] = [
|
||||
{ id: 'appearance', labelKey: 'mainTabs.appearance', icon: Palette },
|
||||
{ id: 'git', labelKey: 'mainTabs.git', icon: GitBranch },
|
||||
{ id: 'api', labelKey: 'mainTabs.apiTokens', icon: Key },
|
||||
{ id: 'voice', labelKey: 'mainTabs.voice', icon: Mic },
|
||||
{ id: 'tasks', labelKey: 'mainTabs.tasks', icon: ListChecks },
|
||||
{ id: 'browser', labelKey: 'mainTabs.browser', icon: MonitorPlay },
|
||||
{ id: 'plugins', labelKey: 'mainTabs.plugins', icon: Puzzle },
|
||||
|
||||
91
src/components/settings/view/tabs/VoiceSettingsTab.tsx
Normal file
91
src/components/settings/view/tabs/VoiceSettingsTab.tsx
Normal file
@@ -0,0 +1,91 @@
|
||||
import type { InputHTMLAttributes } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import SettingsSection from '../SettingsSection';
|
||||
import SettingsToggle from '../SettingsToggle';
|
||||
import { useUiPreferences } from '../../../../hooks/useUiPreferences';
|
||||
import { useVoiceConfig } from '../../../../hooks/useVoiceConfig';
|
||||
|
||||
const inputClass =
|
||||
'w-full rounded-md border border-border bg-background px-3 py-2 text-sm text-foreground placeholder:text-muted-foreground focus:outline-none focus:ring-2 focus:ring-ring';
|
||||
|
||||
function Field({ label, ...props }: { label: string } & InputHTMLAttributes<HTMLInputElement>) {
|
||||
return (
|
||||
<label className="block space-y-1">
|
||||
<span className="text-sm font-medium text-foreground">{label}</span>
|
||||
<input className={inputClass} {...props} />
|
||||
</label>
|
||||
);
|
||||
}
|
||||
|
||||
export default function VoiceSettingsTab() {
|
||||
const { t } = useTranslation('settings');
|
||||
const { preferences, setPreference } = useUiPreferences();
|
||||
const { config, update } = useVoiceConfig();
|
||||
const voiceEnabled = preferences.voiceEnabled;
|
||||
|
||||
return (
|
||||
<div className="space-y-8">
|
||||
<SettingsSection title={t('voiceSettings.title')} description={t('voiceSettings.description')}>
|
||||
<div className="flex items-center justify-between rounded-lg border border-border p-3">
|
||||
<div className="pr-3">
|
||||
<div className="text-sm font-medium text-foreground">{t('voiceSettings.enable')}</div>
|
||||
<div className="text-xs text-muted-foreground">{t('voiceSettings.enableDescription')}</div>
|
||||
</div>
|
||||
<SettingsToggle
|
||||
checked={voiceEnabled}
|
||||
onChange={(v) => setPreference('voiceEnabled', v)}
|
||||
ariaLabel={t('voiceSettings.enable')}
|
||||
/>
|
||||
</div>
|
||||
</SettingsSection>
|
||||
|
||||
{voiceEnabled && (
|
||||
<SettingsSection title={t('voiceSettings.backendTitle')} description={t('voiceSettings.backendDescription')}>
|
||||
<div className="space-y-4">
|
||||
<Field
|
||||
label={t('voiceSettings.baseUrl')}
|
||||
placeholder="https://api.openai.com/v1"
|
||||
value={config.baseUrl}
|
||||
onChange={(e) => update({ baseUrl: e.target.value })}
|
||||
/>
|
||||
<Field
|
||||
label={t('voiceSettings.apiKey')}
|
||||
type="password"
|
||||
autoComplete="off"
|
||||
placeholder="sk-…"
|
||||
value={config.apiKey}
|
||||
onChange={(e) => update({ apiKey: e.target.value })}
|
||||
/>
|
||||
<div className="grid grid-cols-1 gap-4 sm:grid-cols-4">
|
||||
<Field
|
||||
label={t('voiceSettings.sttModel')}
|
||||
placeholder="whisper-1"
|
||||
value={config.sttModel}
|
||||
onChange={(e) => update({ sttModel: e.target.value })}
|
||||
/>
|
||||
<Field
|
||||
label={t('voiceSettings.ttsModel')}
|
||||
placeholder="tts-1"
|
||||
value={config.ttsModel}
|
||||
onChange={(e) => update({ ttsModel: e.target.value })}
|
||||
/>
|
||||
<Field
|
||||
label={t('voiceSettings.voice')}
|
||||
placeholder="alloy"
|
||||
value={config.ttsVoice}
|
||||
onChange={(e) => update({ ttsVoice: e.target.value })}
|
||||
/>
|
||||
<Field
|
||||
label={t('voiceSettings.format')}
|
||||
placeholder="mp3"
|
||||
value={config.ttsFormat}
|
||||
onChange={(e) => update({ ttsFormat: e.target.value })}
|
||||
/>
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground">{t('voiceSettings.note')}</p>
|
||||
</div>
|
||||
</SettingsSection>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -7,6 +7,7 @@ type UiPreferences = {
|
||||
autoScrollToBottom: boolean;
|
||||
sendByCtrlEnter: boolean;
|
||||
sidebarVisible: boolean;
|
||||
voiceEnabled: boolean;
|
||||
};
|
||||
|
||||
type UiPreferenceKey = keyof UiPreferences;
|
||||
@@ -39,6 +40,7 @@ const DEFAULTS: UiPreferences = {
|
||||
autoScrollToBottom: true,
|
||||
sendByCtrlEnter: false,
|
||||
sidebarVisible: true,
|
||||
voiceEnabled: false,
|
||||
};
|
||||
|
||||
const PREFERENCE_KEYS = Object.keys(DEFAULTS) as UiPreferenceKey[];
|
||||
|
||||
68
src/hooks/useVoiceConfig.ts
Normal file
68
src/hooks/useVoiceConfig.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
import { useState } from 'react';
|
||||
|
||||
export type VoiceConfig = {
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
sttModel: string;
|
||||
ttsModel: string;
|
||||
ttsVoice: string;
|
||||
ttsFormat: string;
|
||||
};
|
||||
|
||||
const STORAGE_KEY = 'voiceConfig';
|
||||
export const VOICE_CONFIG_SYNC_EVENT = 'voice-config:sync';
|
||||
const DEFAULTS: VoiceConfig = { baseUrl: '', apiKey: '', sttModel: '', ttsModel: '', ttsVoice: '', ttsFormat: '' };
|
||||
|
||||
export function readVoiceConfig(): VoiceConfig {
|
||||
try {
|
||||
const raw = localStorage.getItem(STORAGE_KEY);
|
||||
if (!raw) return { ...DEFAULTS };
|
||||
const parsed = JSON.parse(raw);
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return { ...DEFAULTS };
|
||||
const config = { ...DEFAULTS };
|
||||
for (const key of Object.keys(DEFAULTS) as (keyof VoiceConfig)[]) {
|
||||
if (typeof parsed[key] === 'string') config[key] = parsed[key];
|
||||
}
|
||||
return config;
|
||||
} catch {
|
||||
return { ...DEFAULTS };
|
||||
}
|
||||
}
|
||||
|
||||
// Headers the voice proxy reads to target a per-user OpenAI-compatible backend.
|
||||
// Empty fields are omitted so the server's env defaults apply.
|
||||
export function voiceConfigHeaders(): Record<string, string> {
|
||||
if (typeof window === 'undefined') return {};
|
||||
const c = readVoiceConfig();
|
||||
const h: Record<string, string> = {};
|
||||
if (c.apiKey) h['x-voice-api-key'] = c.apiKey;
|
||||
if (c.sttModel) h['x-voice-stt-model'] = c.sttModel;
|
||||
if (c.ttsModel) h['x-voice-tts-model'] = c.ttsModel;
|
||||
if (c.ttsVoice) h['x-voice-tts-voice'] = c.ttsVoice;
|
||||
if (c.ttsFormat.trim()) h['x-voice-tts-format'] = c.ttsFormat.trim();
|
||||
return h;
|
||||
}
|
||||
|
||||
export function useVoiceConfig() {
|
||||
const [config, setConfig] = useState<VoiceConfig>(() =>
|
||||
typeof window === 'undefined' ? { ...DEFAULTS } : readVoiceConfig(),
|
||||
);
|
||||
|
||||
const update = (patch: Partial<VoiceConfig>) => {
|
||||
setConfig((prev) => {
|
||||
const next = { ...prev, ...patch };
|
||||
try {
|
||||
const stored: Partial<VoiceConfig> = { ...next };
|
||||
if (next.ttsFormat.trim()) stored.ttsFormat = next.ttsFormat.trim();
|
||||
else delete stored.ttsFormat;
|
||||
localStorage.setItem(STORAGE_KEY, JSON.stringify(stored));
|
||||
window.dispatchEvent(new Event(VOICE_CONFIG_SYNC_EVENT));
|
||||
} catch {
|
||||
/* ignore persistence errors */
|
||||
}
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
return { config, update };
|
||||
}
|
||||
@@ -122,6 +122,14 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"voice": {
|
||||
"input": "Voice input",
|
||||
"stopRecording": "Stop recording",
|
||||
"transcribing": "Transcribing…",
|
||||
"speak": "Read aloud",
|
||||
"stopSpeaking": "Stop",
|
||||
"loading": "Loading…"
|
||||
},
|
||||
"input": {
|
||||
"placeholder": "Type / for commands, @ for files, or ask {{provider}} anything...",
|
||||
"placeholderDefault": "Type your message...",
|
||||
|
||||
@@ -32,5 +32,10 @@
|
||||
"binaryFile": {
|
||||
"title": "Binary File",
|
||||
"message": "The file \"{{fileName}}\" cannot be displayed in the text editor because it is a binary file."
|
||||
},
|
||||
"filePreview": {
|
||||
"loading": "Loading preview...",
|
||||
"error": "Unable to display this file.",
|
||||
"openInNewTab": "Open in new tab"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,6 +50,21 @@
|
||||
"resetToDefaults": "Reset to Defaults",
|
||||
"cancelChanges": "Cancel Changes"
|
||||
},
|
||||
"voiceSettings": {
|
||||
"title": "Voice",
|
||||
"description": "Speech-to-text input and read-aloud, via an OpenAI-compatible audio backend.",
|
||||
"enable": "Enable voice",
|
||||
"enableDescription": "Show the mic button and the read-aloud button on messages.",
|
||||
"backendTitle": "Backend",
|
||||
"backendDescription": "Point at OpenAI, Groq, or a local server (LocalAI, Speaches, Kokoro-FastAPI). Leave blank to use the server default.",
|
||||
"baseUrl": "Base URL",
|
||||
"apiKey": "API key",
|
||||
"sttModel": "Speech-to-text model",
|
||||
"ttsModel": "Text-to-speech model",
|
||||
"voice": "Voice",
|
||||
"format": "Audio format",
|
||||
"note": "A custom base URL is called directly by your browser and must allow browser CORS requests. Leave it blank to use the server-configured backend."
|
||||
},
|
||||
"quickSettings": {
|
||||
"title": "Quick Settings",
|
||||
"sections": {
|
||||
@@ -64,6 +79,7 @@
|
||||
"showThinking": "Show thinking",
|
||||
"autoScrollToBottom": "Auto-scroll to bottom",
|
||||
"sendByCtrlEnter": "Send by Ctrl+Enter",
|
||||
"voiceEnabled": "Voice (mic + read aloud)",
|
||||
"sendByCtrlEnterDescription": "When enabled, pressing Ctrl+Enter will send the message instead of just Enter. This is useful for IME users to avoid accidental sends.",
|
||||
"dragHandle": {
|
||||
"dragging": "Dragging handle",
|
||||
@@ -94,6 +110,7 @@
|
||||
"appearance": "Appearance",
|
||||
"git": "Git",
|
||||
"apiTokens": "API & Tokens",
|
||||
"voice": "Voice",
|
||||
"tasks": "Tasks",
|
||||
"browser": "Browser",
|
||||
"notifications": "Notifications",
|
||||
@@ -114,7 +131,7 @@
|
||||
},
|
||||
"sound": {
|
||||
"title": "Sound",
|
||||
"description": "Play a short tone when a chat run finishes.",
|
||||
"description": "Play a short tone when a chat run finishes or needs tool approval.",
|
||||
"enabled": "Enabled",
|
||||
"test": "Test sound"
|
||||
},
|
||||
|
||||
60
src/lib/voiceApi.ts
Normal file
60
src/lib/voiceApi.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
import { authenticatedFetch } from '../utils/api';
|
||||
import { readVoiceConfig, voiceConfigHeaders } from '../hooks/useVoiceConfig';
|
||||
|
||||
function directUrl(baseUrl: string, path: string): string {
|
||||
return `${baseUrl.replace(/\/$/, '')}${path}`;
|
||||
}
|
||||
|
||||
export function voiceConfigSignature(): string {
|
||||
return JSON.stringify(readVoiceConfig());
|
||||
}
|
||||
|
||||
export function transcribeVoice(blob: Blob, filename: string): Promise<Response> {
|
||||
const config = readVoiceConfig();
|
||||
const body = new FormData();
|
||||
|
||||
if (config.baseUrl.trim()) {
|
||||
body.append('file', blob, filename);
|
||||
body.append('model', config.sttModel || 'whisper-1');
|
||||
return fetch(directUrl(config.baseUrl.trim(), '/audio/transcriptions'), {
|
||||
method: 'POST',
|
||||
headers: config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {},
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
body.append('audio', blob, filename);
|
||||
return authenticatedFetch('/api/voice/transcribe', {
|
||||
method: 'POST',
|
||||
headers: voiceConfigHeaders(),
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
export function synthesizeVoice(text: string, signal: AbortSignal): Promise<Response> {
|
||||
const config = readVoiceConfig();
|
||||
|
||||
if (config.baseUrl.trim()) {
|
||||
return fetch(directUrl(config.baseUrl.trim(), '/audio/speech'), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...(config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}),
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: config.ttsModel || 'tts-1',
|
||||
voice: config.ttsVoice || 'alloy',
|
||||
input: text,
|
||||
...(config.ttsFormat.trim() ? { response_format: config.ttsFormat.trim() } : {}),
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
}
|
||||
|
||||
return authenticatedFetch('/api/voice/tts', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ text }),
|
||||
headers: voiceConfigHeaders(),
|
||||
signal,
|
||||
});
|
||||
}
|
||||
196
src/lib/voicePlayer.ts
Normal file
196
src/lib/voicePlayer.ts
Normal file
@@ -0,0 +1,196 @@
|
||||
import { synthesizeVoice, voiceConfigSignature } from './voiceApi';
|
||||
|
||||
// A single app-level audio player for read-aloud. It owns one <audio> element, lives
|
||||
// outside the React tree, and caches generated audio by content. Because playback is not
|
||||
// tied to a component, switching chats or re-rendering a message can't revoke the blob URL
|
||||
// out from under it (the cause of mid-play cutoffs). v1 plays one message at a time
|
||||
// (a new play replaces the current one); the design leaves room for a queue later.
|
||||
|
||||
export type VoicePlayState = 'idle' | 'loading' | 'playing';
|
||||
|
||||
export type VoiceSnapshot = { state: VoicePlayState; error: string | null };
|
||||
|
||||
const IDLE: VoiceSnapshot = { state: 'idle', error: null };
|
||||
const CACHE_MAX = 24;
|
||||
const CLIENT_TIMEOUT_MS = 330000; // backstop; the server proxy already times out at 5 min
|
||||
|
||||
// Stable id / cache key from the text and voice settings that affect its audio (djb2).
|
||||
export function voiceId(content: string, signature = voiceConfigSignature()): string {
|
||||
const input = JSON.stringify([content, signature]);
|
||||
let h = 5381;
|
||||
for (let i = 0; i < input.length; i++) h = (((h << 5) + h) + input.charCodeAt(i)) | 0;
|
||||
return (h >>> 0).toString(36);
|
||||
}
|
||||
|
||||
class VoicePlayer {
|
||||
private audio: HTMLAudioElement | null = null;
|
||||
private unlocked = false;
|
||||
private cache = new Map<string, string>(); // id -> blob URL (insertion order = LRU)
|
||||
private currentId: string | null = null;
|
||||
private state: VoicePlayState = 'idle';
|
||||
private errorId: string | null = null;
|
||||
private errorMsg: string | null = null;
|
||||
private token = 0; // bumps to ignore stale in-flight results
|
||||
private activeController: AbortController | null = null; // aborts the in-flight TTS fetch
|
||||
private errorTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private listeners = new Set<() => void>();
|
||||
|
||||
subscribe(listener: () => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => {
|
||||
this.listeners.delete(listener);
|
||||
};
|
||||
}
|
||||
|
||||
private emit() {
|
||||
this.listeners.forEach((l) => l());
|
||||
}
|
||||
|
||||
getSnapshot(id: string): VoiceSnapshot {
|
||||
const state = this.currentId === id ? this.state : 'idle';
|
||||
const error = this.errorId === id ? this.errorMsg : null;
|
||||
if (state === 'idle' && error === null) return IDLE;
|
||||
return { state, error };
|
||||
}
|
||||
|
||||
private ensureAudio(): HTMLAudioElement {
|
||||
if (!this.audio) {
|
||||
const audio = new Audio();
|
||||
audio.addEventListener('ended', () => this.onEnded());
|
||||
audio.addEventListener('error', () => {
|
||||
// Only meaningful while we believe we're playing.
|
||||
if (this.state === 'playing') this.onEnded();
|
||||
});
|
||||
this.audio = audio;
|
||||
}
|
||||
return this.audio;
|
||||
}
|
||||
|
||||
// Call synchronously from the click handler so iOS grants the (reused) element playback.
|
||||
unlock() {
|
||||
if (this.unlocked) return;
|
||||
const audio = this.ensureAudio();
|
||||
try {
|
||||
const p = audio.play();
|
||||
if (p && typeof p.catch === 'function') p.catch(() => {});
|
||||
audio.pause();
|
||||
} catch {
|
||||
/* priming attempt; ignore */
|
||||
}
|
||||
this.unlocked = true;
|
||||
}
|
||||
|
||||
toggle(content: string) {
|
||||
const id = voiceId(content);
|
||||
if (this.currentId === id && (this.state === 'playing' || this.state === 'loading')) {
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
void this.play(id, content);
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.token++; // ignore any stale in-flight result
|
||||
this.abortActive(); // and actually cancel the network request
|
||||
if (this.audio) this.audio.pause();
|
||||
this.state = 'idle';
|
||||
this.currentId = null;
|
||||
this.emit();
|
||||
}
|
||||
|
||||
private abortActive() {
|
||||
if (this.activeController) {
|
||||
this.activeController.abort();
|
||||
this.activeController = null;
|
||||
}
|
||||
}
|
||||
|
||||
private onEnded() {
|
||||
this.state = 'idle';
|
||||
this.currentId = null;
|
||||
this.emit();
|
||||
// (queue auto-advance would hook in here)
|
||||
}
|
||||
|
||||
private setError(id: string, msg: string) {
|
||||
this.state = 'idle';
|
||||
this.currentId = id;
|
||||
this.errorId = id;
|
||||
this.errorMsg = msg;
|
||||
this.emit();
|
||||
if (this.errorTimer) clearTimeout(this.errorTimer);
|
||||
this.errorTimer = setTimeout(() => {
|
||||
if (this.errorId === id) {
|
||||
this.errorId = null;
|
||||
this.errorMsg = null;
|
||||
if (this.currentId === id) this.currentId = null;
|
||||
this.emit();
|
||||
}
|
||||
}, 6000);
|
||||
}
|
||||
|
||||
private async play(id: string, content: string) {
|
||||
const audio = this.ensureAudio();
|
||||
audio.pause();
|
||||
this.currentId = id;
|
||||
this.errorId = null;
|
||||
this.errorMsg = null;
|
||||
this.state = 'loading';
|
||||
this.emit();
|
||||
|
||||
const myToken = ++this.token;
|
||||
this.abortActive(); // cancel any request this play supersedes
|
||||
|
||||
try {
|
||||
let url = this.cache.get(id);
|
||||
if (!url) {
|
||||
const controller = new AbortController();
|
||||
this.activeController = controller;
|
||||
const timer = setTimeout(() => controller.abort(), CLIENT_TIMEOUT_MS);
|
||||
const res = await synthesizeVoice(content, controller.signal).finally(() => {
|
||||
clearTimeout(timer);
|
||||
if (this.activeController === controller) this.activeController = null;
|
||||
});
|
||||
if (myToken !== this.token) return; // superseded by another play/stop
|
||||
if (!res.ok) {
|
||||
let msg = `Read-aloud failed (${res.status})`;
|
||||
try {
|
||||
const j = await res.json();
|
||||
if (j?.error) msg = String(j.error);
|
||||
} catch {
|
||||
/* non-JSON error body */
|
||||
}
|
||||
throw new Error(msg);
|
||||
}
|
||||
const blob = await res.blob();
|
||||
if (myToken !== this.token) return;
|
||||
url = URL.createObjectURL(blob);
|
||||
this.cacheSet(id, url);
|
||||
}
|
||||
if (myToken !== this.token) return;
|
||||
audio.src = url;
|
||||
audio.load();
|
||||
await audio.play();
|
||||
if (myToken !== this.token) return;
|
||||
this.state = 'playing';
|
||||
this.emit();
|
||||
} catch (e) {
|
||||
if (myToken !== this.token) return;
|
||||
const aborted = e instanceof Error && e.name === 'AbortError';
|
||||
this.setError(id, aborted ? 'Read-aloud timed out.' : e instanceof Error ? e.message : 'Read-aloud failed');
|
||||
}
|
||||
}
|
||||
|
||||
private cacheSet(id: string, url: string) {
|
||||
this.cache.set(id, url);
|
||||
while (this.cache.size > CACHE_MAX) {
|
||||
const oldest = this.cache.keys().next().value as string | undefined;
|
||||
if (oldest === undefined) break;
|
||||
const oldUrl = this.cache.get(oldest);
|
||||
this.cache.delete(oldest);
|
||||
if (oldUrl && oldUrl !== this.audio?.src) URL.revokeObjectURL(oldUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const voicePlayer = new VoicePlayer();
|
||||
@@ -58,7 +58,7 @@ const playTone = (
|
||||
oscillator.stop(startsAt + duration + 0.02);
|
||||
};
|
||||
|
||||
export const playChatCompletionSound = async ({ force = false } = {}): Promise<void> => {
|
||||
export const playNotificationSound = async ({ force = false } = {}): Promise<void> => {
|
||||
if (!force && !isNotificationSoundEnabled()) {
|
||||
return;
|
||||
}
|
||||
@@ -81,3 +81,5 @@ export const playChatCompletionSound = async ({ force = false } = {}): Promise<v
|
||||
console.warn('Unable to play notification sound:', error);
|
||||
}
|
||||
};
|
||||
|
||||
export const playChatCompletionSound = (options = {}): Promise<void> => playNotificationSound(options);
|
||||
|
||||
Reference in New Issue
Block a user