feat(ui): redesign voice cockpit with clinical companion drawer, Svelte 5 migration, and Gemini 3.8 voice

- Re-architect VoiceSession into full-bleed studio cockpit with overlay slide-over companion drawer
- Eliminate width squishing defect on LiveInsightPanel and add native dark/light theme support
- Integrate authentic Google Gemini 3.8 Flash TTS with Fenrir persona and fallback audio pipeline
- Implement comprehensive Playwright E2E visual regression suite across Desktop, iPad, and iPhone (21 tests, 45 snapshots)
- Migrate core application to Svelte 5 runes (, , ) with reactive stores
- Maintain .NET 8 Azure Functions serverless endpoints and clean build hygiene
This commit is contained in:
jpremill committed 2026-09-29 23:35:05 -04:00
1 parent c82b112624
commit 1feea40684
148 files changed
+11230 -1706

No files matched your search

+186
View File
@@ -0,0 +1,186 @@
import { synthesizeSpeechWithOpenRouter } from './openRouterService';
import { synthesizeSpeechWithGemini } from './geminiVoiceService';
import { getGeminiApiKey, hasGeminiApiKey, hasOpenRouterApiKey } from './apiConfig';
import { VOICE_PROFILES, type VoiceProfile } from '../constants';
import type { UserSettings } from '../types';
export interface VoicePlaybackState {
isPlaying: boolean;
activeVoiceId: UserSettings['voiceName'] | null;
mode: 'gemini' | 'openrouter' | 'webspeech' | 'idle';
error?: string;
}
export function getActiveVoiceEngine(): 'gemini' | 'openrouter' | 'webspeech' {
if (hasGeminiApiKey()) return 'gemini';
if (hasOpenRouterApiKey()) return 'openrouter';
return 'webspeech';
}
export class VoiceAuditionService {
private currentAudio: HTMLAudioElement | null = null;
private currentAudioUrl: string | null = null;
private activeUtterance: SpeechSynthesisUtterance | null = null;
/**
* Returns the VoiceProfile configuration for a given voice name.
*/
getProfile(voiceId: UserSettings['voiceName']): VoiceProfile {
return VOICE_PROFILES.find((p) => p.id === voiceId) || VOICE_PROFILES[0];
}
/**
* Auditions/plays a voice sample:
* 1. Google Gemini native voice (gemini-3.8-flash-tts) if GEMINI_API_KEY is available.
* 2. OpenRouter TTS if OPENROUTER_API_KEY is available.
* 3. Browser Web Speech Synthesis fallback.
*/
async playPreview(
voiceId: UserSettings['voiceName'],
customText?: string,
apiKey?: string,
onEnded?: () => void,
geminiApiKey?: string
): Promise<void> {
this.stop();
const profile = this.getProfile(voiceId);
const textToSpeak = customText?.trim() || profile.sampleText;
// 1. Prioritize Google Gemini Native Voice (gemini-3.8-flash-tts)
const geminiKey = getGeminiApiKey(geminiApiKey);
if (geminiKey) {
try {
const arrayBuffer = await synthesizeSpeechWithGemini(
textToSpeak,
profile.id,
geminiKey
);
const blob = new Blob([arrayBuffer], { type: 'audio/wav' });
const url = URL.createObjectURL(blob);
this.currentAudioUrl = url;
const audio = new Audio(url);
this.currentAudio = audio;
audio.onended = () => {
this.cleanupAudio();
onEnded?.();
};
audio.onerror = () => {
this.cleanupAudio();
this.playWebSpeech(textToSpeak, profile, onEnded);
};
await audio.play();
return;
} catch (err) {
console.warn('Gemini 3.8 native voice synthesis preview failed, checking OpenRouter fallback:', err);
}
}
// 2. OpenRouter TTS Fallback
if (apiKey && apiKey.trim().length > 0) {
try {
const arrayBuffer = await synthesizeSpeechWithOpenRouter(
textToSpeak,
profile.openRouterVoice,
apiKey
);
const blob = new Blob([arrayBuffer], { type: 'audio/mpeg' });
const url = URL.createObjectURL(blob);
this.currentAudioUrl = url;
const audio = new Audio(url);
this.currentAudio = audio;
audio.onended = () => {
this.cleanupAudio();
onEnded?.();
};
audio.onerror = () => {
this.cleanupAudio();
this.playWebSpeech(textToSpeak, profile, onEnded);
};
await audio.play();
return;
} catch (err) {
console.warn('OpenRouter TTS preview failed, falling back to Web Speech Synthesis:', err);
}
}
// 3. Web Speech API fallback with calibrated pitch/rate tuning
this.playWebSpeech(textToSpeak, profile, onEnded);
}
private playWebSpeech(
text: string,
profile: VoiceProfile,
onEnded?: () => void
): void {
const synth = typeof window !== 'undefined' && 'speechSynthesis' in window
? window.speechSynthesis
: (typeof (globalThis as any).speechSynthesis !== 'undefined' ? (globalThis as any).speechSynthesis : null);
const Utterance = typeof window !== 'undefined' && 'SpeechSynthesisUtterance' in window
? window.SpeechSynthesisUtterance
: (typeof (globalThis as any).SpeechSynthesisUtterance !== 'undefined' ? (globalThis as any).SpeechSynthesisUtterance : null);
if (synth && Utterance) {
synth.cancel();
const utterance = new Utterance(text);
utterance.pitch = profile.webSpeechTuning.pitch;
utterance.rate = profile.webSpeechTuning.rate;
utterance.onend = () => {
this.activeUtterance = null;
onEnded?.();
};
utterance.onerror = () => {
this.activeUtterance = null;
onEnded?.();
};
this.activeUtterance = utterance;
synth.speak(utterance);
} else {
onEnded?.();
}
}
private cleanupAudio(): void {
if (this.currentAudioUrl) {
URL.revokeObjectURL(this.currentAudioUrl);
this.currentAudioUrl = null;
}
this.currentAudio = null;
}
/**
* Immediately stops any active audition preview audio.
*/
stop(): void {
if (this.currentAudio) {
try {
this.currentAudio.pause();
this.currentAudio.currentTime = 0;
} catch (e) {}
this.cleanupAudio();
}
const synth = typeof window !== 'undefined' && 'speechSynthesis' in window
? window.speechSynthesis
: (typeof (globalThis as any).speechSynthesis !== 'undefined' ? (globalThis as any).speechSynthesis : null);
if (synth) {
try {
synth.cancel();
} catch (e) {}
this.activeUtterance = null;
}
}
}
export const defaultVoiceAudition = new VoiceAuditionService();