/** * Google Gemini Native Voice Synthesis Service * * Provides direct speech synthesis using Google's native Gemini voice models * (gemini-3.8-flash-tts and gemini-2.5-flash-preview-tts) with official prebuilt * voices: Fenrir (Gruff / Salt of the Earth), Puck, Charon, Kore, Aoede. */ import { getGeminiApiKey } from './apiConfig'; import { decodeBase64 } from './audioUtils'; const GEMINI_API_BASE = 'https://generativelanguage.googleapis.com/v1beta'; /** * Packages raw PCM16 audio into a self-contained WAV ArrayBuffer. */ export function pcmToWav(pcmData: Uint8Array, sampleRate = 24000, numChannels = 1): ArrayBuffer { const wavBuffer = new ArrayBuffer(44 + pcmData.byteLength); const view = new DataView(wavBuffer); // "RIFF" view.setUint32(0, 0x52494646, false); view.setUint32(4, 36 + pcmData.byteLength, true); // "WAVE" view.setUint32(8, 0x57415645, false); // "fmt " view.setUint32(12, 0x666d7420, false); view.setUint32(16, 16, true); view.setUint16(20, 1, true); // AudioFormat: PCM (1) view.setUint16(22, numChannels, true); view.setUint32(24, sampleRate, true); view.setUint32(28, sampleRate * numChannels * 2, true); view.setUint16(32, numChannels * 2, true); view.setUint16(34, 16, true); // BitsPerSample: 16 // "data" view.setUint32(36, 0x64617461, false); view.setUint32(40, pcmData.byteLength, true); new Uint8Array(wavBuffer, 44).set(pcmData); return wavBuffer; } /** * Synthesizes speech using Google Gemini's native voice model. * * @param text The text dialogue to speak. * @param voiceName Prebuilt Gemini voice: 'Fenrir' | 'Puck' | 'Charon' | 'Kore' | 'Aoede' | 'Zephyr' * @param apiKey Optional explicit Google Gemini API key. * @returns ArrayBuffer containing standard decoded audio data (WAV container). */ export async function synthesizeSpeechWithGemini( text: string, voiceName = 'Fenrir', apiKey?: string ): Promise { const effectiveKey = getGeminiApiKey(apiKey); if (!effectiveKey) { throw new Error('Google Gemini API key is missing. Please set GEMINI_API_KEY in Settings or .env.'); } // Normalize voice name to valid Gemini prebuilt voices (Zephyr -> Puck/Fenrir) const validVoices = ['Fenrir', 'Puck', 'Charon', 'Kore', 'Aoede']; const targetVoice = validVoices.includes(voiceName) ? voiceName : 'Fenrir'; const modelsToTry = [ 'gemini-3.8-flash-tts', 'gemini-2.5-flash-preview-tts' ]; let lastError: any = null; for (const model of modelsToTry) { try { const url = `${GEMINI_API_BASE}/models/${model}:generateContent?key=${encodeURIComponent(effectiveKey)}`; const response = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ contents: [{ parts: [{ text }] }], generationConfig: { responseModalities: ['AUDIO'], speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: targetVoice } } } } }) }); if (!response.ok) { const errorText = await response.text(); throw new Error(`Gemini ${model} TTS error (${response.status}): ${errorText}`); } const json = await response.json(); const part = json.candidates?.[0]?.content?.parts?.[0]; if (!part || !part.inlineData?.data) { throw new Error(`Gemini ${model} returned no audio content in response candidates.`); } const mimeType = part.inlineData.mimeType || ''; const rawBytes = decodeBase64(part.inlineData.data); if (mimeType.includes('wav')) { const slice = rawBytes.buffer.slice(rawBytes.byteOffset, rawBytes.byteOffset + rawBytes.byteLength); return slice as ArrayBuffer; } // If PCM (e.g. audio/L16;codec=pcm;rate=24000), wrap in WAV container for universal browser decoding return pcmToWav(rawBytes, 24000, 1); } catch (err: any) { lastError = err; console.warn(`Failed synthesis with Gemini model ${model}, checking next model...`, err); } } throw lastError || new Error('All Gemini TTS models failed to generate audio.'); }