jpremill
1feea40684
- Re-architect VoiceSession into full-bleed studio cockpit with overlay slide-over companion drawer - Eliminate width squishing defect on LiveInsightPanel and add native dark/light theme support - Integrate authentic Google Gemini 3.8 Flash TTS with Fenrir persona and fallback audio pipeline - Implement comprehensive Playwright E2E visual regression suite across Desktop, iPad, and iPhone (21 tests, 45 snapshots) - Migrate core application to Svelte 5 runes (, , ) with reactive stores - Maintain .NET 8 Azure Functions serverless endpoints and clean build hygiene
122 lines
4.2 KiB
TypeScript
122 lines
4.2 KiB
TypeScript
/**
|
|
* Google Gemini Native Voice Synthesis Service
|
|
*
|
|
* Provides direct speech synthesis using Google's native Gemini voice models
|
|
* (gemini-3.8-flash-tts and gemini-2.5-flash-preview-tts) with official prebuilt
|
|
* voices: Fenrir (Gruff / Salt of the Earth), Puck, Charon, Kore, Aoede.
|
|
*/
|
|
|
|
import { getGeminiApiKey } from './apiConfig';
|
|
import { decodeBase64 } from './audioUtils';
|
|
|
|
const GEMINI_API_BASE = 'https://generativelanguage.googleapis.com/v1beta';
|
|
|
|
/**
|
|
* Packages raw PCM16 audio into a self-contained WAV ArrayBuffer.
|
|
*/
|
|
export function pcmToWav(pcmData: Uint8Array, sampleRate = 24000, numChannels = 1): ArrayBuffer {
|
|
const wavBuffer = new ArrayBuffer(44 + pcmData.byteLength);
|
|
const view = new DataView(wavBuffer);
|
|
// "RIFF"
|
|
view.setUint32(0, 0x52494646, false);
|
|
view.setUint32(4, 36 + pcmData.byteLength, true);
|
|
// "WAVE"
|
|
view.setUint32(8, 0x57415645, false);
|
|
// "fmt "
|
|
view.setUint32(12, 0x666d7420, false);
|
|
view.setUint32(16, 16, true);
|
|
view.setUint16(20, 1, true); // AudioFormat: PCM (1)
|
|
view.setUint16(22, numChannels, true);
|
|
view.setUint32(24, sampleRate, true);
|
|
view.setUint32(28, sampleRate * numChannels * 2, true);
|
|
view.setUint16(32, numChannels * 2, true);
|
|
view.setUint16(34, 16, true); // BitsPerSample: 16
|
|
// "data"
|
|
view.setUint32(36, 0x64617461, false);
|
|
view.setUint32(40, pcmData.byteLength, true);
|
|
new Uint8Array(wavBuffer, 44).set(pcmData);
|
|
return wavBuffer;
|
|
}
|
|
|
|
/**
|
|
* Synthesizes speech using Google Gemini's native voice model.
|
|
*
|
|
* @param text The text dialogue to speak.
|
|
* @param voiceName Prebuilt Gemini voice: 'Fenrir' | 'Puck' | 'Charon' | 'Kore' | 'Aoede' | 'Zephyr'
|
|
* @param apiKey Optional explicit Google Gemini API key.
|
|
* @returns ArrayBuffer containing standard decoded audio data (WAV container).
|
|
*/
|
|
export async function synthesizeSpeechWithGemini(
|
|
text: string,
|
|
voiceName = 'Fenrir',
|
|
apiKey?: string
|
|
): Promise<ArrayBuffer> {
|
|
const effectiveKey = getGeminiApiKey(apiKey);
|
|
if (!effectiveKey) {
|
|
throw new Error('Google Gemini API key is missing. Please set GEMINI_API_KEY in Settings or .env.');
|
|
}
|
|
|
|
// Normalize voice name to valid Gemini prebuilt voices (Zephyr -> Puck/Fenrir)
|
|
const validVoices = ['Fenrir', 'Puck', 'Charon', 'Kore', 'Aoede'];
|
|
const targetVoice = validVoices.includes(voiceName) ? voiceName : 'Fenrir';
|
|
|
|
const modelsToTry = [
|
|
'gemini-3.8-flash-tts',
|
|
'gemini-2.5-flash-preview-tts'
|
|
];
|
|
|
|
let lastError: any = null;
|
|
|
|
for (const model of modelsToTry) {
|
|
try {
|
|
const url = `${GEMINI_API_BASE}/models/${model}:generateContent?key=${encodeURIComponent(effectiveKey)}`;
|
|
const response = await fetch(url, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({
|
|
contents: [{ parts: [{ text }] }],
|
|
generationConfig: {
|
|
responseModalities: ['AUDIO'],
|
|
speechConfig: {
|
|
voiceConfig: {
|
|
prebuiltVoiceConfig: {
|
|
voiceName: targetVoice
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
});
|
|
|
|
if (!response.ok) {
|
|
const errorText = await response.text();
|
|
throw new Error(`Gemini ${model} TTS error (${response.status}): ${errorText}`);
|
|
}
|
|
|
|
const json = await response.json();
|
|
const part = json.candidates?.[0]?.content?.parts?.[0];
|
|
if (!part || !part.inlineData?.data) {
|
|
throw new Error(`Gemini ${model} returned no audio content in response candidates.`);
|
|
}
|
|
|
|
const mimeType = part.inlineData.mimeType || '';
|
|
const rawBytes = decodeBase64(part.inlineData.data);
|
|
|
|
if (mimeType.includes('wav')) {
|
|
const slice = rawBytes.buffer.slice(rawBytes.byteOffset, rawBytes.byteOffset + rawBytes.byteLength);
|
|
return slice as ArrayBuffer;
|
|
}
|
|
|
|
// If PCM (e.g. audio/L16;codec=pcm;rate=24000), wrap in WAV container for universal browser decoding
|
|
return pcmToWav(rawBytes, 24000, 1);
|
|
} catch (err: any) {
|
|
lastError = err;
|
|
console.warn(`Failed synthesis with Gemini model ${model}, checking next model...`, err);
|
|
}
|
|
}
|
|
|
|
throw lastError || new Error('All Gemini TTS models failed to generate audio.');
|
|
}
|