import type { Scenario, Message, AnalysisResult } from '../../../types'; import { UserSettingsStore } from '../stores/UserSettingsStore.svelte'; import { streamOpenRouterChat, analyzeTurnWithOpenRouter, generateSessionSummaryReport, synthesizeSpeechWithOpenRouter } from '../../../services/openRouterService'; import { synthesizeSpeechWithGemini } from '../../../services/geminiVoiceService'; import { getOpenRouterApiKey, getGeminiApiKey } from '../../../services/apiConfig'; import { saveSession } from '../../../services/sessionStorage'; import { VOICE_PROFILES } from '../../../constants'; export type SessionStatus = 'idle' | 'connecting' | 'active' | 'analyzing' | 'finished' | 'error'; export class AudioEngine { status = $state('idle'); errorMessage = $state(null); isMuted = $state(false); micLevel = $state(0); aiAudioLevel = $state(0); isAiSpeaking = $state(false); isUserSpeaking = $state(false); messages = $state([]); liveInsight = $state(null); isAnalyzingTurn = $state(false); finalReport = $state(null); // Derived unified visualizer level activeLevel = $derived(this.isAiSpeaking ? this.aiAudioLevel : this.micLevel); private inputCtx: AudioContext | null = null; private outputCtx: AudioContext | null = null; private userAnalyser: AnalyserNode | null = null; private aiAnalyser: AnalyserNode | null = null; private currentAudioSource: AudioBufferSourceNode | null = null; private stream: MediaStream | null = null; private recognition: any = null; private animFrameId: number | null = null; private isProcessingTurn = false; constructor( private scenario: Scenario, private userSettings: UserSettingsStore ) {} async start() { const apiKey = getOpenRouterApiKey(this.userSettings.openRouterApiKey); if (!apiKey) { this.status = 'error'; this.errorMessage = 'OpenRouter API Key is missing. Please set your API key in Settings.'; return; } try { this.status = 'connecting'; this.errorMessage = null; // WebAudio Context Initialization const AudioCtxClass = typeof window !== 'undefined' ? (window.AudioContext || (window as any).webkitAudioContext) : (globalThis as any).AudioContext; if (!AudioCtxClass) { throw new Error('WebAudio AudioContext is not supported in this environment.'); } this.inputCtx = new AudioCtxClass(); this.outputCtx = new AudioCtxClass(); this.userAnalyser = this.inputCtx.createAnalyser(); this.userAnalyser.fftSize = 256; this.aiAnalyser = this.outputCtx.createAnalyser(); this.aiAnalyser.fftSize = 256; // Microphone Stream const constraints: MediaStreamConstraints = { audio: this.userSettings.preferredAudioDeviceId ? { deviceId: { exact: this.userSettings.preferredAudioDeviceId } } : true }; this.stream = await navigator.mediaDevices.getUserMedia(constraints); const micSource = this.inputCtx.createMediaStreamSource(this.stream); micSource.connect(this.userAnalyser); // Web Speech Recognition const SpeechRecognition = typeof window !== 'undefined' ? ((window as any).SpeechRecognition || (window as any).webkitSpeechRecognition) : (globalThis as any).SpeechRecognition; if (SpeechRecognition) { this.recognition = new SpeechRecognition(); this.recognition.continuous = true; this.recognition.interimResults = true; this.recognition.lang = 'en-US'; let speechDebounceTimer: any = null; let pendingUserSpeech = ''; this.recognition.onresult = (event: any) => { if (this.isMuted || this.isAiSpeaking) return; let interimTranscript = ''; for (let i = event.resultIndex; i < event.results.length; i++) { const transcript = event.results[i][0].transcript; if (event.results[i].isFinal) { pendingUserSpeech += ' ' + transcript; } else { interimTranscript += transcript; } } if (interimTranscript.trim() || pendingUserSpeech.trim()) { this.isUserSpeaking = true; } clearTimeout(speechDebounceTimer); speechDebounceTimer = setTimeout(() => { const finalSpeech = (pendingUserSpeech + ' ' + interimTranscript).trim(); if (finalSpeech.length > 2) { this.isUserSpeaking = false; this.handleUserTurn(finalSpeech); pendingUserSpeech = ''; } }, 1200); }; this.recognition.onerror = (e: any) => { if (e.error !== 'no-speech') { console.warn('Speech recognition warning:', e.error); } }; this.recognition.onend = () => { if (this.status === 'active' && !this.isMuted) { try { this.recognition?.start(); } catch (err) {} } }; try { this.recognition.start(); } catch (e) { console.warn('Speech recognition auto-start prevented', e); } } // Initial Message const initialGreeting: Message = { id: 'init', role: 'model', text: this.scenario.initialMessage, timestamp: new Date() }; this.messages = [initialGreeting]; this.status = 'active'; // Start Frequency Analysis Animation Loop this.startMeterLoop(); // Speak initial greeting await this.speakText(this.scenario.initialMessage); } catch (err: any) { console.error('Audio initialization failure', err); this.status = 'error'; this.errorMessage = err?.message || 'Failed to initialize audio input device.'; } } private startMeterLoop() { const userBuffer = new Uint8Array(this.userAnalyser ? this.userAnalyser.frequencyBinCount : 0); const aiBuffer = new Uint8Array(this.aiAnalyser ? this.aiAnalyser.frequencyBinCount : 0); const updateMeter = () => { // 1. Microphone Level if (this.userAnalyser && !this.isMuted && !this.isAiSpeaking) { this.userAnalyser.getByteFrequencyData(userBuffer); let sum = 0; for (let i = 0; i < userBuffer.length; i++) sum += userBuffer[i]; const avg = sum / userBuffer.length; this.micLevel = Math.min(100, Math.round((avg / 128) * 100)); } else { this.micLevel = 0; } // 2. AI Audio Level if (this.aiAnalyser && this.isAiSpeaking) { this.aiAnalyser.getByteFrequencyData(aiBuffer); let sum = 0; for (let i = 0; i < aiBuffer.length; i++) sum += aiBuffer[i]; const avg = sum / aiBuffer.length; this.aiAudioLevel = Math.min(100, Math.round((avg / 128) * 100)); } else { this.aiAudioLevel = 0; } this.animFrameId = requestAnimationFrame(updateMeter); }; this.animFrameId = requestAnimationFrame(updateMeter); } toggleMute() { this.isMuted = !this.isMuted; if (this.isMuted) { this.stopAllAudio(); try { this.recognition?.stop(); } catch (e) {} } else { try { this.recognition?.start(); } catch (e) {} } } async speakText(text: string) { this.stopAllAudio(); this.isAiSpeaking = true; const profile = VOICE_PROFILES.find( (p) => p.id.toLowerCase() === this.userSettings.voiceName.toLowerCase() ) || VOICE_PROFILES[0]; // 1. Google Gemini Native Voice (gemini-3.8-flash-tts with Fenrir, Puck, Kore, etc.) const geminiKey = getGeminiApiKey(this.userSettings.geminiApiKey); if (geminiKey) { try { const arrayBuffer = await synthesizeSpeechWithGemini( text, profile.id, geminiKey ); if (!this.outputCtx) return; if (this.outputCtx.state === 'suspended') await this.outputCtx.resume(); const buffer = await this.outputCtx.decodeAudioData(arrayBuffer); const source = this.outputCtx.createBufferSource(); source.buffer = buffer; if (this.aiAnalyser) { source.connect(this.aiAnalyser); this.aiAnalyser.connect(this.outputCtx.destination); } else { source.connect(this.outputCtx.destination); } source.onended = () => { this.isAiSpeaking = false; this.aiAudioLevel = 0; }; this.currentAudioSource = source; source.start(0); return; } catch (e) { console.warn('Gemini 3.8 native voice synthesis fallback to OpenRouter/WebSpeech:', e); } } // 2. OpenRouter TTS Fallback try { const arrayBuffer = await synthesizeSpeechWithOpenRouter( text, profile.openRouterVoice, this.userSettings.openRouterApiKey ); if (!this.outputCtx) return; if (this.outputCtx.state === 'suspended') await this.outputCtx.resume(); const buffer = await this.outputCtx.decodeAudioData(arrayBuffer); const source = this.outputCtx.createBufferSource(); source.buffer = buffer; if (this.aiAnalyser) { source.connect(this.aiAnalyser); this.aiAnalyser.connect(this.outputCtx.destination); } else { source.connect(this.outputCtx.destination); } source.onended = () => { this.isAiSpeaking = false; this.aiAudioLevel = 0; }; this.currentAudioSource = source; source.start(0); return; } catch (e) { console.warn('OpenRouter TTS fallback to Web Speech Synthesis', e); if (typeof window !== 'undefined' && 'speechSynthesis' in window) { const utterance = new SpeechSynthesisUtterance(text); utterance.rate = profile.webSpeechTuning.rate; utterance.pitch = profile.webSpeechTuning.pitch; utterance.onstart = () => { this.isAiSpeaking = true; }; utterance.onend = () => { this.isAiSpeaking = false; this.aiAudioLevel = 0; }; utterance.onerror = () => { this.isAiSpeaking = false; this.aiAudioLevel = 0; }; window.speechSynthesis.speak(utterance); } else { this.isAiSpeaking = false; } } } async handleUserTurn(userText: string) { if (this.isProcessingTurn || !userText.trim()) return; this.isProcessingTurn = true; this.stopAllAudio(); const userMsg: Message = { id: Date.now().toString(), role: 'user', text: userText.trim(), timestamp: new Date() }; const currentHistory = [...this.messages, userMsg]; this.messages = currentHistory; const botMsgId = (Date.now() + 1).toString(); const botMsg: Message = { id: botMsgId, role: 'model', text: '', timestamp: new Date() }; this.messages = [...currentHistory, botMsg]; const roleplaySystemInstruction = ` ${this.scenario.systemInstruction} [CONTEXT] You are participating in a clinical CBT roleplay simulation. The instructions above define your character, the scenario, and your internal CBT logic. [TASK] Respond to the user naturally in character to advance the roleplay scenario. [CONSTRAINTS] - CRITICAL: Stay strictly in character as the persona defined in the scenario. - CRITICAL: NEVER provide clinical feedback, advice, or meta-commentary about the user's progress. - CRITICAL: Keep dialogue natural and conversational (1-3 sentences per turn). - If the user successfully concludes the scenario, end the dialogue naturally. `; try { let accumulatedText = ''; await streamOpenRouterChat( currentHistory, roleplaySystemInstruction, { onChunk: (chunk) => { accumulatedText += chunk; this.messages = this.messages.map((m) => m.id === botMsgId ? { ...m, text: accumulatedText } : m ); }, onComplete: (fullText) => { this.speakText(fullText); // Parallel out-of-band turn supervisor this.isAnalyzingTurn = true; const turnContext = `User said: "${userMsg.text}". Roleplay partner said: "${fullText}".`; analyzeTurnWithOpenRouter( turnContext, this.scenario.title, this.scenario.description, this.scenario.cbtFocus, { apiKey: this.userSettings.openRouterApiKey, model: this.userSettings.supervisorModel } ).then((res) => { this.liveInsight = res; }).catch((err) => { console.error('Voice turn analysis error', err); }).finally(() => { this.isAnalyzingTurn = false; this.isProcessingTurn = false; }); }, onError: (err) => { console.error('OpenRouter voice stream failed', err); this.isProcessingTurn = false; } }, { apiKey: this.userSettings.openRouterApiKey, model: this.userSettings.roleplayModel } ); } catch (err) { console.error('User voice turn execution failed', err); this.isProcessingTurn = false; } } async finishSession() { this.status = 'analyzing'; this.stopAllAudio(); try { const report = await generateSessionSummaryReport( this.messages, this.scenario.title, this.scenario.description, { apiKey: this.userSettings.openRouterApiKey, model: this.userSettings.supervisorModel } ); this.finalReport = report; saveSession(this.scenario, 'voice', this.messages, report); this.status = 'finished'; } catch (err) { console.error('Post-session analysis error', err); this.status = 'error'; this.errorMessage = 'Failed to generate session report.'; } } stopAllAudio() { if (this.currentAudioSource) { try { this.currentAudioSource.stop(); } catch (e) {} this.currentAudioSource = null; } const synth = typeof window !== 'undefined' && 'speechSynthesis' in window ? window.speechSynthesis : typeof (globalThis as any).speechSynthesis !== 'undefined' ? (globalThis as any).speechSynthesis : null; if (synth) { synth.cancel(); } this.isAiSpeaking = false; this.aiAudioLevel = 0; } destroy() { this.stopAllAudio(); if (this.animFrameId) { cancelAnimationFrame(this.animFrameId); this.animFrameId = null; } try { this.recognition?.stop(); } catch (e) {} this.stream?.getTracks().forEach((t) => t.stop()); this.inputCtx?.close(); this.outputCtx?.close(); } }