feat(ui): redesign voice cockpit with clinical companion drawer, Svelte 5 migration, and Gemini 3.8 voice
- Re-architect VoiceSession into full-bleed studio cockpit with overlay slide-over companion drawer - Eliminate width squishing defect on LiveInsightPanel and add native dark/light theme support - Integrate authentic Google Gemini 3.8 Flash TTS with Fenrir persona and fallback audio pipeline - Implement comprehensive Playwright E2E visual regression suite across Desktop, iPad, and iPhone (21 tests, 45 snapshots) - Migrate core application to Svelte 5 runes (, , ) with reactive stores - Maintain .NET 8 Azure Functions serverless endpoints and clean build hygiene
This commit is contained in:
1 parent
c82b112624
commit
1feea40684
148 files changed
+11230
-1706
No files matched your search
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* Google Gemini Native Voice Synthesis Service
|
||||
*
|
||||
* Provides direct speech synthesis using Google's native Gemini voice models
|
||||
* (gemini-3.8-flash-tts and gemini-2.5-flash-preview-tts) with official prebuilt
|
||||
* voices: Fenrir (Gruff / Salt of the Earth), Puck, Charon, Kore, Aoede.
|
||||
*/
|
||||
|
||||
import { getGeminiApiKey } from './apiConfig';
|
||||
import { decodeBase64 } from './audioUtils';
|
||||
|
||||
const GEMINI_API_BASE = 'https://generativelanguage.googleapis.com/v1beta';
|
||||
|
||||
/**
|
||||
* Packages raw PCM16 audio into a self-contained WAV ArrayBuffer.
|
||||
*/
|
||||
export function pcmToWav(pcmData: Uint8Array, sampleRate = 24000, numChannels = 1): ArrayBuffer {
|
||||
const wavBuffer = new ArrayBuffer(44 + pcmData.byteLength);
|
||||
const view = new DataView(wavBuffer);
|
||||
// "RIFF"
|
||||
view.setUint32(0, 0x52494646, false);
|
||||
view.setUint32(4, 36 + pcmData.byteLength, true);
|
||||
// "WAVE"
|
||||
view.setUint32(8, 0x57415645, false);
|
||||
// "fmt "
|
||||
view.setUint32(12, 0x666d7420, false);
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true); // AudioFormat: PCM (1)
|
||||
view.setUint16(22, numChannels, true);
|
||||
view.setUint32(24, sampleRate, true);
|
||||
view.setUint32(28, sampleRate * numChannels * 2, true);
|
||||
view.setUint16(32, numChannels * 2, true);
|
||||
view.setUint16(34, 16, true); // BitsPerSample: 16
|
||||
// "data"
|
||||
view.setUint32(36, 0x64617461, false);
|
||||
view.setUint32(40, pcmData.byteLength, true);
|
||||
new Uint8Array(wavBuffer, 44).set(pcmData);
|
||||
return wavBuffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Synthesizes speech using Google Gemini's native voice model.
|
||||
*
|
||||
* @param text The text dialogue to speak.
|
||||
* @param voiceName Prebuilt Gemini voice: 'Fenrir' | 'Puck' | 'Charon' | 'Kore' | 'Aoede' | 'Zephyr'
|
||||
* @param apiKey Optional explicit Google Gemini API key.
|
||||
* @returns ArrayBuffer containing standard decoded audio data (WAV container).
|
||||
*/
|
||||
export async function synthesizeSpeechWithGemini(
|
||||
text: string,
|
||||
voiceName = 'Fenrir',
|
||||
apiKey?: string
|
||||
): Promise<ArrayBuffer> {
|
||||
const effectiveKey = getGeminiApiKey(apiKey);
|
||||
if (!effectiveKey) {
|
||||
throw new Error('Google Gemini API key is missing. Please set GEMINI_API_KEY in Settings or .env.');
|
||||
}
|
||||
|
||||
// Normalize voice name to valid Gemini prebuilt voices (Zephyr -> Puck/Fenrir)
|
||||
const validVoices = ['Fenrir', 'Puck', 'Charon', 'Kore', 'Aoede'];
|
||||
const targetVoice = validVoices.includes(voiceName) ? voiceName : 'Fenrir';
|
||||
|
||||
const modelsToTry = [
|
||||
'gemini-3.8-flash-tts',
|
||||
'gemini-2.5-flash-preview-tts'
|
||||
];
|
||||
|
||||
let lastError: any = null;
|
||||
|
||||
for (const model of modelsToTry) {
|
||||
try {
|
||||
const url = `${GEMINI_API_BASE}/models/${model}:generateContent?key=${encodeURIComponent(effectiveKey)}`;
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
contents: [{ parts: [{ text }] }],
|
||||
generationConfig: {
|
||||
responseModalities: ['AUDIO'],
|
||||
speechConfig: {
|
||||
voiceConfig: {
|
||||
prebuiltVoiceConfig: {
|
||||
voiceName: targetVoice
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`Gemini ${model} TTS error (${response.status}): ${errorText}`);
|
||||
}
|
||||
|
||||
const json = await response.json();
|
||||
const part = json.candidates?.[0]?.content?.parts?.[0];
|
||||
if (!part || !part.inlineData?.data) {
|
||||
throw new Error(`Gemini ${model} returned no audio content in response candidates.`);
|
||||
}
|
||||
|
||||
const mimeType = part.inlineData.mimeType || '';
|
||||
const rawBytes = decodeBase64(part.inlineData.data);
|
||||
|
||||
if (mimeType.includes('wav')) {
|
||||
const slice = rawBytes.buffer.slice(rawBytes.byteOffset, rawBytes.byteOffset + rawBytes.byteLength);
|
||||
return slice as ArrayBuffer;
|
||||
}
|
||||
|
||||
// If PCM (e.g. audio/L16;codec=pcm;rate=24000), wrap in WAV container for universal browser decoding
|
||||
return pcmToWav(rawBytes, 24000, 1);
|
||||
} catch (err: any) {
|
||||
lastError = err;
|
||||
console.warn(`Failed synthesis with Gemini model ${model}, checking next model...`, err);
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError || new Error('All Gemini TTS models failed to generate audio.');
|
||||
}
|
||||
Reference in new issue
Block a user