feat(ui): redesign voice cockpit with clinical companion drawer, Svelte 5 migration, and Gemini 3.8 voice
- Re-architect VoiceSession into full-bleed studio cockpit with overlay slide-over companion drawer - Eliminate width squishing defect on LiveInsightPanel and add native dark/light theme support - Integrate authentic Google Gemini 3.8 Flash TTS with Fenrir persona and fallback audio pipeline - Implement comprehensive Playwright E2E visual regression suite across Desktop, iPad, and iPhone (21 tests, 45 snapshots) - Migrate core application to Svelte 5 runes (, , ) with reactive stores - Maintain .NET 8 Azure Functions serverless endpoints and clean build hygiene
This commit is contained in:
jpremill committed
1 parent
c82b112624
commit
1feea40684
148 files changed
+11230
-1706
No files matched your search
@@ -0,0 +1,455 @@
|
||||
import type { Scenario, Message, AnalysisResult } from '../../../types';
|
||||
import { UserSettingsStore } from '../stores/UserSettingsStore.svelte';
|
||||
import {
|
||||
streamOpenRouterChat,
|
||||
analyzeTurnWithOpenRouter,
|
||||
generateSessionSummaryReport,
|
||||
synthesizeSpeechWithOpenRouter
|
||||
} from '../../../services/openRouterService';
|
||||
import { synthesizeSpeechWithGemini } from '../../../services/geminiVoiceService';
|
||||
import { getOpenRouterApiKey, getGeminiApiKey } from '../../../services/apiConfig';
|
||||
import { saveSession } from '../../../services/sessionStorage';
|
||||
import { VOICE_PROFILES } from '../../../constants';
|
||||
|
||||
export type SessionStatus = 'idle' | 'connecting' | 'active' | 'analyzing' | 'finished' | 'error';
|
||||
|
||||
export class AudioEngine {
|
||||
status = $state<SessionStatus>('idle');
|
||||
errorMessage = $state<string | null>(null);
|
||||
isMuted = $state(false);
|
||||
micLevel = $state(0);
|
||||
aiAudioLevel = $state(0);
|
||||
isAiSpeaking = $state(false);
|
||||
isUserSpeaking = $state(false);
|
||||
messages = $state<Message[]>([]);
|
||||
liveInsight = $state<AnalysisResult | null>(null);
|
||||
isAnalyzingTurn = $state(false);
|
||||
finalReport = $state<AnalysisResult | null>(null);
|
||||
|
||||
// Derived unified visualizer level
|
||||
activeLevel = $derived(this.isAiSpeaking ? this.aiAudioLevel : this.micLevel);
|
||||
|
||||
private inputCtx: AudioContext | null = null;
|
||||
private outputCtx: AudioContext | null = null;
|
||||
private userAnalyser: AnalyserNode | null = null;
|
||||
private aiAnalyser: AnalyserNode | null = null;
|
||||
private currentAudioSource: AudioBufferSourceNode | null = null;
|
||||
private stream: MediaStream | null = null;
|
||||
private recognition: any = null;
|
||||
private animFrameId: number | null = null;
|
||||
private isProcessingTurn = false;
|
||||
|
||||
constructor(
|
||||
private scenario: Scenario,
|
||||
private userSettings: UserSettingsStore
|
||||
) {}
|
||||
|
||||
async start() {
|
||||
const apiKey = getOpenRouterApiKey(this.userSettings.openRouterApiKey);
|
||||
if (!apiKey) {
|
||||
this.status = 'error';
|
||||
this.errorMessage = 'OpenRouter API Key is missing. Please set your API key in Settings.';
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
this.status = 'connecting';
|
||||
this.errorMessage = null;
|
||||
|
||||
// WebAudio Context Initialization
|
||||
const AudioCtxClass = typeof window !== 'undefined'
|
||||
? (window.AudioContext || (window as any).webkitAudioContext)
|
||||
: (globalThis as any).AudioContext;
|
||||
if (!AudioCtxClass) {
|
||||
throw new Error('WebAudio AudioContext is not supported in this environment.');
|
||||
}
|
||||
this.inputCtx = new AudioCtxClass();
|
||||
this.outputCtx = new AudioCtxClass();
|
||||
|
||||
this.userAnalyser = this.inputCtx.createAnalyser();
|
||||
this.userAnalyser.fftSize = 256;
|
||||
|
||||
this.aiAnalyser = this.outputCtx.createAnalyser();
|
||||
this.aiAnalyser.fftSize = 256;
|
||||
|
||||
// Microphone Stream
|
||||
const constraints: MediaStreamConstraints = {
|
||||
audio: this.userSettings.preferredAudioDeviceId
|
||||
? { deviceId: { exact: this.userSettings.preferredAudioDeviceId } }
|
||||
: true
|
||||
};
|
||||
|
||||
this.stream = await navigator.mediaDevices.getUserMedia(constraints);
|
||||
const micSource = this.inputCtx.createMediaStreamSource(this.stream);
|
||||
micSource.connect(this.userAnalyser);
|
||||
|
||||
// Web Speech Recognition
|
||||
const SpeechRecognition = typeof window !== 'undefined'
|
||||
? ((window as any).SpeechRecognition || (window as any).webkitSpeechRecognition)
|
||||
: (globalThis as any).SpeechRecognition;
|
||||
if (SpeechRecognition) {
|
||||
this.recognition = new SpeechRecognition();
|
||||
this.recognition.continuous = true;
|
||||
this.recognition.interimResults = true;
|
||||
this.recognition.lang = 'en-US';
|
||||
|
||||
let speechDebounceTimer: any = null;
|
||||
let pendingUserSpeech = '';
|
||||
|
||||
this.recognition.onresult = (event: any) => {
|
||||
if (this.isMuted || this.isAiSpeaking) return;
|
||||
|
||||
let interimTranscript = '';
|
||||
for (let i = event.resultIndex; i < event.results.length; i++) {
|
||||
const transcript = event.results[i][0].transcript;
|
||||
if (event.results[i].isFinal) {
|
||||
pendingUserSpeech += ' ' + transcript;
|
||||
} else {
|
||||
interimTranscript += transcript;
|
||||
}
|
||||
}
|
||||
|
||||
if (interimTranscript.trim() || pendingUserSpeech.trim()) {
|
||||
this.isUserSpeaking = true;
|
||||
}
|
||||
|
||||
clearTimeout(speechDebounceTimer);
|
||||
speechDebounceTimer = setTimeout(() => {
|
||||
const finalSpeech = (pendingUserSpeech + ' ' + interimTranscript).trim();
|
||||
if (finalSpeech.length > 2) {
|
||||
this.isUserSpeaking = false;
|
||||
this.handleUserTurn(finalSpeech);
|
||||
pendingUserSpeech = '';
|
||||
}
|
||||
}, 1200);
|
||||
};
|
||||
|
||||
this.recognition.onerror = (e: any) => {
|
||||
if (e.error !== 'no-speech') {
|
||||
console.warn('Speech recognition warning:', e.error);
|
||||
}
|
||||
};
|
||||
|
||||
this.recognition.onend = () => {
|
||||
if (this.status === 'active' && !this.isMuted) {
|
||||
try { this.recognition?.start(); } catch (err) {}
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
this.recognition.start();
|
||||
} catch (e) {
|
||||
console.warn('Speech recognition auto-start prevented', e);
|
||||
}
|
||||
}
|
||||
|
||||
// Initial Message
|
||||
const initialGreeting: Message = {
|
||||
id: 'init',
|
||||
role: 'model',
|
||||
text: this.scenario.initialMessage,
|
||||
timestamp: new Date()
|
||||
};
|
||||
this.messages = [initialGreeting];
|
||||
|
||||
this.status = 'active';
|
||||
|
||||
// Start Frequency Analysis Animation Loop
|
||||
this.startMeterLoop();
|
||||
|
||||
// Speak initial greeting
|
||||
await this.speakText(this.scenario.initialMessage);
|
||||
} catch (err: any) {
|
||||
console.error('Audio initialization failure', err);
|
||||
this.status = 'error';
|
||||
this.errorMessage = err?.message || 'Failed to initialize audio input device.';
|
||||
}
|
||||
}
|
||||
|
||||
private startMeterLoop() {
|
||||
const userBuffer = new Uint8Array(this.userAnalyser ? this.userAnalyser.frequencyBinCount : 0);
|
||||
const aiBuffer = new Uint8Array(this.aiAnalyser ? this.aiAnalyser.frequencyBinCount : 0);
|
||||
|
||||
const updateMeter = () => {
|
||||
// 1. Microphone Level
|
||||
if (this.userAnalyser && !this.isMuted && !this.isAiSpeaking) {
|
||||
this.userAnalyser.getByteFrequencyData(userBuffer);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < userBuffer.length; i++) sum += userBuffer[i];
|
||||
const avg = sum / userBuffer.length;
|
||||
this.micLevel = Math.min(100, Math.round((avg / 128) * 100));
|
||||
} else {
|
||||
this.micLevel = 0;
|
||||
}
|
||||
|
||||
// 2. AI Audio Level
|
||||
if (this.aiAnalyser && this.isAiSpeaking) {
|
||||
this.aiAnalyser.getByteFrequencyData(aiBuffer);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < aiBuffer.length; i++) sum += aiBuffer[i];
|
||||
const avg = sum / aiBuffer.length;
|
||||
this.aiAudioLevel = Math.min(100, Math.round((avg / 128) * 100));
|
||||
} else {
|
||||
this.aiAudioLevel = 0;
|
||||
}
|
||||
|
||||
this.animFrameId = requestAnimationFrame(updateMeter);
|
||||
};
|
||||
|
||||
this.animFrameId = requestAnimationFrame(updateMeter);
|
||||
}
|
||||
|
||||
toggleMute() {
|
||||
this.isMuted = !this.isMuted;
|
||||
if (this.isMuted) {
|
||||
this.stopAllAudio();
|
||||
try { this.recognition?.stop(); } catch (e) {}
|
||||
} else {
|
||||
try { this.recognition?.start(); } catch (e) {}
|
||||
}
|
||||
}
|
||||
|
||||
async speakText(text: string) {
|
||||
this.stopAllAudio();
|
||||
this.isAiSpeaking = true;
|
||||
|
||||
const profile = VOICE_PROFILES.find(
|
||||
(p) => p.id.toLowerCase() === this.userSettings.voiceName.toLowerCase()
|
||||
) || VOICE_PROFILES[0];
|
||||
|
||||
// 1. Google Gemini Native Voice (gemini-3.8-flash-tts with Fenrir, Puck, Kore, etc.)
|
||||
const geminiKey = getGeminiApiKey(this.userSettings.geminiApiKey);
|
||||
if (geminiKey) {
|
||||
try {
|
||||
const arrayBuffer = await synthesizeSpeechWithGemini(
|
||||
text,
|
||||
profile.id,
|
||||
geminiKey
|
||||
);
|
||||
|
||||
if (!this.outputCtx) return;
|
||||
if (this.outputCtx.state === 'suspended') await this.outputCtx.resume();
|
||||
|
||||
const buffer = await this.outputCtx.decodeAudioData(arrayBuffer);
|
||||
const source = this.outputCtx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
|
||||
if (this.aiAnalyser) {
|
||||
source.connect(this.aiAnalyser);
|
||||
this.aiAnalyser.connect(this.outputCtx.destination);
|
||||
} else {
|
||||
source.connect(this.outputCtx.destination);
|
||||
}
|
||||
|
||||
source.onended = () => {
|
||||
this.isAiSpeaking = false;
|
||||
this.aiAudioLevel = 0;
|
||||
};
|
||||
|
||||
this.currentAudioSource = source;
|
||||
source.start(0);
|
||||
return;
|
||||
} catch (e) {
|
||||
console.warn('Gemini 3.8 native voice synthesis fallback to OpenRouter/WebSpeech:', e);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. OpenRouter TTS Fallback
|
||||
try {
|
||||
const arrayBuffer = await synthesizeSpeechWithOpenRouter(
|
||||
text,
|
||||
profile.openRouterVoice,
|
||||
this.userSettings.openRouterApiKey
|
||||
);
|
||||
|
||||
if (!this.outputCtx) return;
|
||||
if (this.outputCtx.state === 'suspended') await this.outputCtx.resume();
|
||||
|
||||
const buffer = await this.outputCtx.decodeAudioData(arrayBuffer);
|
||||
const source = this.outputCtx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
|
||||
if (this.aiAnalyser) {
|
||||
source.connect(this.aiAnalyser);
|
||||
this.aiAnalyser.connect(this.outputCtx.destination);
|
||||
} else {
|
||||
source.connect(this.outputCtx.destination);
|
||||
}
|
||||
|
||||
source.onended = () => {
|
||||
this.isAiSpeaking = false;
|
||||
this.aiAudioLevel = 0;
|
||||
};
|
||||
|
||||
this.currentAudioSource = source;
|
||||
source.start(0);
|
||||
return;
|
||||
} catch (e) {
|
||||
console.warn('OpenRouter TTS fallback to Web Speech Synthesis', e);
|
||||
if (typeof window !== 'undefined' && 'speechSynthesis' in window) {
|
||||
const utterance = new SpeechSynthesisUtterance(text);
|
||||
utterance.rate = profile.webSpeechTuning.rate;
|
||||
utterance.pitch = profile.webSpeechTuning.pitch;
|
||||
utterance.onstart = () => { this.isAiSpeaking = true; };
|
||||
utterance.onend = () => {
|
||||
this.isAiSpeaking = false;
|
||||
this.aiAudioLevel = 0;
|
||||
};
|
||||
utterance.onerror = () => {
|
||||
this.isAiSpeaking = false;
|
||||
this.aiAudioLevel = 0;
|
||||
};
|
||||
window.speechSynthesis.speak(utterance);
|
||||
} else {
|
||||
this.isAiSpeaking = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async handleUserTurn(userText: string) {
|
||||
if (this.isProcessingTurn || !userText.trim()) return;
|
||||
this.isProcessingTurn = true;
|
||||
this.stopAllAudio();
|
||||
|
||||
const userMsg: Message = {
|
||||
id: Date.now().toString(),
|
||||
role: 'user',
|
||||
text: userText.trim(),
|
||||
timestamp: new Date()
|
||||
};
|
||||
|
||||
const currentHistory = [...this.messages, userMsg];
|
||||
this.messages = currentHistory;
|
||||
|
||||
const botMsgId = (Date.now() + 1).toString();
|
||||
const botMsg: Message = {
|
||||
id: botMsgId,
|
||||
role: 'model',
|
||||
text: '',
|
||||
timestamp: new Date()
|
||||
};
|
||||
this.messages = [...currentHistory, botMsg];
|
||||
|
||||
const roleplaySystemInstruction = `
|
||||
${this.scenario.systemInstruction}
|
||||
|
||||
[CONTEXT]
|
||||
You are participating in a clinical CBT roleplay simulation.
|
||||
The instructions above define your character, the scenario, and your internal CBT logic.
|
||||
|
||||
[TASK]
|
||||
Respond to the user naturally in character to advance the roleplay scenario.
|
||||
|
||||
[CONSTRAINTS]
|
||||
- CRITICAL: Stay strictly in character as the persona defined in the scenario.
|
||||
- CRITICAL: NEVER provide clinical feedback, advice, or meta-commentary about the user's progress.
|
||||
- CRITICAL: Keep dialogue natural and conversational (1-3 sentences per turn).
|
||||
- If the user successfully concludes the scenario, end the dialogue naturally.
|
||||
`;
|
||||
|
||||
try {
|
||||
let accumulatedText = '';
|
||||
await streamOpenRouterChat(
|
||||
currentHistory,
|
||||
roleplaySystemInstruction,
|
||||
{
|
||||
onChunk: (chunk) => {
|
||||
accumulatedText += chunk;
|
||||
this.messages = this.messages.map((m) =>
|
||||
m.id === botMsgId ? { ...m, text: accumulatedText } : m
|
||||
);
|
||||
},
|
||||
onComplete: (fullText) => {
|
||||
this.speakText(fullText);
|
||||
|
||||
// Parallel out-of-band turn supervisor
|
||||
this.isAnalyzingTurn = true;
|
||||
const turnContext = `User said: "${userMsg.text}". Roleplay partner said: "${fullText}".`;
|
||||
analyzeTurnWithOpenRouter(
|
||||
turnContext,
|
||||
this.scenario.title,
|
||||
this.scenario.description,
|
||||
this.scenario.cbtFocus,
|
||||
{
|
||||
apiKey: this.userSettings.openRouterApiKey,
|
||||
model: this.userSettings.supervisorModel
|
||||
}
|
||||
).then((res) => {
|
||||
this.liveInsight = res;
|
||||
}).catch((err) => {
|
||||
console.error('Voice turn analysis error', err);
|
||||
}).finally(() => {
|
||||
this.isAnalyzingTurn = false;
|
||||
this.isProcessingTurn = false;
|
||||
});
|
||||
},
|
||||
onError: (err) => {
|
||||
console.error('OpenRouter voice stream failed', err);
|
||||
this.isProcessingTurn = false;
|
||||
}
|
||||
},
|
||||
{
|
||||
apiKey: this.userSettings.openRouterApiKey,
|
||||
model: this.userSettings.roleplayModel
|
||||
}
|
||||
);
|
||||
} catch (err) {
|
||||
console.error('User voice turn execution failed', err);
|
||||
this.isProcessingTurn = false;
|
||||
}
|
||||
}
|
||||
|
||||
async finishSession() {
|
||||
this.status = 'analyzing';
|
||||
this.stopAllAudio();
|
||||
|
||||
try {
|
||||
const report = await generateSessionSummaryReport(
|
||||
this.messages,
|
||||
this.scenario.title,
|
||||
this.scenario.description,
|
||||
{
|
||||
apiKey: this.userSettings.openRouterApiKey,
|
||||
model: this.userSettings.supervisorModel
|
||||
}
|
||||
);
|
||||
|
||||
this.finalReport = report;
|
||||
saveSession(this.scenario, 'voice', this.messages, report);
|
||||
this.status = 'finished';
|
||||
} catch (err) {
|
||||
console.error('Post-session analysis error', err);
|
||||
this.status = 'error';
|
||||
this.errorMessage = 'Failed to generate session report.';
|
||||
}
|
||||
}
|
||||
|
||||
stopAllAudio() {
|
||||
if (this.currentAudioSource) {
|
||||
try { this.currentAudioSource.stop(); } catch (e) {}
|
||||
this.currentAudioSource = null;
|
||||
}
|
||||
const synth = typeof window !== 'undefined' && 'speechSynthesis' in window
|
||||
? window.speechSynthesis
|
||||
: typeof (globalThis as any).speechSynthesis !== 'undefined'
|
||||
? (globalThis as any).speechSynthesis
|
||||
: null;
|
||||
if (synth) {
|
||||
synth.cancel();
|
||||
}
|
||||
this.isAiSpeaking = false;
|
||||
this.aiAudioLevel = 0;
|
||||
}
|
||||
|
||||
destroy() {
|
||||
this.stopAllAudio();
|
||||
if (this.animFrameId) {
|
||||
cancelAnimationFrame(this.animFrameId);
|
||||
this.animFrameId = null;
|
||||
}
|
||||
try { this.recognition?.stop(); } catch (e) {}
|
||||
this.stream?.getTracks().forEach((t) => t.stop());
|
||||
this.inputCtx?.close();
|
||||
this.outputCtx?.close();
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user