feat(ui): redesign voice cockpit with clinical companion drawer, Svelte 5 migration, and Gemini 3.8 voice

- Re-architect VoiceSession into full-bleed studio cockpit with overlay slide-over companion drawer
- Eliminate width squishing defect on LiveInsightPanel and add native dark/light theme support
- Integrate authentic Google Gemini 3.8 Flash TTS with Fenrir persona and fallback audio pipeline
- Implement comprehensive Playwright E2E visual regression suite across Desktop, iPad, and iPhone (21 tests, 45 snapshots)
- Migrate core application to Svelte 5 runes (, , ) with reactive stores
- Maintain .NET 8 Azure Functions serverless endpoints and clean build hygiene
This commit is contained in:
jpremill committed 2026-09-29 23:35:05 -04:00
1 parent c82b112624
commit 1feea40684
148 files changed
+11230 -1706

No files matched your search

+121
View File
@@ -0,0 +1,121 @@
/**
* Google Gemini Native Voice Synthesis Service
*
* Provides direct speech synthesis using Google's native Gemini voice models
* (gemini-3.8-flash-tts and gemini-2.5-flash-preview-tts) with official prebuilt
* voices: Fenrir (Gruff / Salt of the Earth), Puck, Charon, Kore, Aoede.
*/
import { getGeminiApiKey } from './apiConfig';
import { decodeBase64 } from './audioUtils';
const GEMINI_API_BASE = 'https://generativelanguage.googleapis.com/v1beta';
/**
* Packages raw PCM16 audio into a self-contained WAV ArrayBuffer.
*/
export function pcmToWav(pcmData: Uint8Array, sampleRate = 24000, numChannels = 1): ArrayBuffer {
const wavBuffer = new ArrayBuffer(44 + pcmData.byteLength);
const view = new DataView(wavBuffer);
// "RIFF"
view.setUint32(0, 0x52494646, false);
view.setUint32(4, 36 + pcmData.byteLength, true);
// "WAVE"
view.setUint32(8, 0x57415645, false);
// "fmt "
view.setUint32(12, 0x666d7420, false);
view.setUint32(16, 16, true);
view.setUint16(20, 1, true); // AudioFormat: PCM (1)
view.setUint16(22, numChannels, true);
view.setUint32(24, sampleRate, true);
view.setUint32(28, sampleRate * numChannels * 2, true);
view.setUint16(32, numChannels * 2, true);
view.setUint16(34, 16, true); // BitsPerSample: 16
// "data"
view.setUint32(36, 0x64617461, false);
view.setUint32(40, pcmData.byteLength, true);
new Uint8Array(wavBuffer, 44).set(pcmData);
return wavBuffer;
}
/**
* Synthesizes speech using Google Gemini's native voice model.
*
* @param text The text dialogue to speak.
* @param voiceName Prebuilt Gemini voice: 'Fenrir' | 'Puck' | 'Charon' | 'Kore' | 'Aoede' | 'Zephyr'
* @param apiKey Optional explicit Google Gemini API key.
* @returns ArrayBuffer containing standard decoded audio data (WAV container).
*/
export async function synthesizeSpeechWithGemini(
text: string,
voiceName = 'Fenrir',
apiKey?: string
): Promise<ArrayBuffer> {
const effectiveKey = getGeminiApiKey(apiKey);
if (!effectiveKey) {
throw new Error('Google Gemini API key is missing. Please set GEMINI_API_KEY in Settings or .env.');
}
// Normalize voice name to valid Gemini prebuilt voices (Zephyr -> Puck/Fenrir)
const validVoices = ['Fenrir', 'Puck', 'Charon', 'Kore', 'Aoede'];
const targetVoice = validVoices.includes(voiceName) ? voiceName : 'Fenrir';
const modelsToTry = [
'gemini-3.8-flash-tts',
'gemini-2.5-flash-preview-tts'
];
let lastError: any = null;
for (const model of modelsToTry) {
try {
const url = `${GEMINI_API_BASE}/models/${model}:generateContent?key=${encodeURIComponent(effectiveKey)}`;
const response = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
contents: [{ parts: [{ text }] }],
generationConfig: {
responseModalities: ['AUDIO'],
speechConfig: {
voiceConfig: {
prebuiltVoiceConfig: {
voiceName: targetVoice
}
}
}
}
})
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`Gemini ${model} TTS error (${response.status}): ${errorText}`);
}
const json = await response.json();
const part = json.candidates?.[0]?.content?.parts?.[0];
if (!part || !part.inlineData?.data) {
throw new Error(`Gemini ${model} returned no audio content in response candidates.`);
}
const mimeType = part.inlineData.mimeType || '';
const rawBytes = decodeBase64(part.inlineData.data);
if (mimeType.includes('wav')) {
const slice = rawBytes.buffer.slice(rawBytes.byteOffset, rawBytes.byteOffset + rawBytes.byteLength);
return slice as ArrayBuffer;
}
// If PCM (e.g. audio/L16;codec=pcm;rate=24000), wrap in WAV container for universal browser decoding
return pcmToWav(rawBytes, 24000, 1);
} catch (err: any) {
lastError = err;
console.warn(`Failed synthesis with Gemini model ${model}, checking next model...`, err);
}
}
throw lastError || new Error('All Gemini TTS models failed to generate audio.');
}