import React, { useEffect, useRef, useState, useCallback } from 'react'; import { Scenario, AnalysisResult, UserSettings } from '../types'; import { GoogleGenAI, LiveServerMessage, Modality, Chat } from "@google/genai"; import { Mic, MicOff, X, Radio, AlertCircle, Headphones, BrainCircuit, LayoutPanelLeft, Play, Info, CheckCircle2, MessageSquare, Sparkles, Activity } from 'lucide-react'; import { arrayBufferToBase64, base64ToUint8Array, decodeAudioData, floatTo16BitPCM } from '../services/audioUtils'; import { CBT_ANALYSIS_SCHEMA, SYSTEM_SUMMARY_ANALYSIS_PROMPT, SYSTEM_ANALYSIS_PROMPT } from '../constants'; // Sub-components import AudioVisualizer from './AudioVisualizer'; import LiveInsightPanel from './LiveInsightPanel'; import SessionReport from './SessionReport'; import Tooltip from './Tooltip'; import TranscriptPanel from './TranscriptPanel'; interface VoiceSessionProps { scenario: Scenario; onExit: () => void; voiceSettings: UserSettings; } interface DisplayMessage { id: string; role: 'user' | 'model'; text: string; } const VoiceSession: React.FC = ({ scenario, onExit, voiceSettings }) => { const [sessionPhase, setSessionPhase] = useState<'briefing' | 'session' | 'analyzing' | 'finished'>('briefing'); const [status, setStatus] = useState<'disconnected' | 'connecting' | 'active' | 'error'>('disconnected'); const [errorMessage, setErrorMessage] = useState(null); const [finalAnalysis, setFinalAnalysis] = useState(null); const [showLiveInsights, setShowLiveInsights] = useState(false); const [showTranscript, setShowTranscript] = useState(false); const [liveAnalysis, setLiveAnalysis] = useState(null); const [isAnalyzingTurn, setIsAnalyzingTurn] = useState(false); const [liveMessages, setLiveMessages] = useState([]); const inputContextRef = useRef(null); const outputContextRef = useRef(null); const nextStartTimeRef = useRef(0); const sourcesRef = useRef>(new Set()); const transcriptRef = useRef(""); const currentTurnUserText = useRef(""); const currentTurnPartnerText = useRef(""); const sessionRef = useRef(null); const analysisChatRef = useRef(null); const [volumeLevel, setVolumeLevel] = useState(0); useEffect(() => { return () => { sourcesRef.current.forEach(source => { try { source.stop(); } catch(e) {} }); }; }, []); const disconnectSession = useCallback(async () => { sourcesRef.current.forEach(source => { try { source.stop(); } catch(e) {} }); sourcesRef.current.clear(); try { if (inputContextRef.current && inputContextRef.current.state !== 'closed') await inputContextRef.current.close(); if (outputContextRef.current && outputContextRef.current.state !== 'closed') await outputContextRef.current.close(); } catch (e) { console.error("Error closing contexts", e); } inputContextRef.current = null; outputContextRef.current = null; if (sessionRef.current) { try { const session = await sessionRef.current; if (session && typeof session.close === 'function') session.close(); } catch (e) { console.error("Error closing session:", e); } sessionRef.current = null; } }, []); const handleTerminateAndAnalyze = async () => { await disconnectSession(); if (transcriptRef.current.length < 10) return onExit(); setSessionPhase('analyzing'); try { const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); const result = await ai.models.generateContent({ model: 'gemini-3-flash-preview', config: { systemInstruction: `${SYSTEM_SUMMARY_ANALYSIS_PROMPT}\n\nSCENARIO: ${scenario.title}\nCBT: ${scenario.cbtFocus.join(', ')}`, responseMimeType: 'application/json', responseSchema: CBT_ANALYSIS_SCHEMA, }, contents: [{ role: 'user', parts: [{ text: `TRANSCRIPT:\n${transcriptRef.current}` }] }] }); if (result.text) { setFinalAnalysis(JSON.parse(result.text)); setSessionPhase('finished'); } else { throw new Error("Analysis failed."); } } catch (err) { onExit(); } }; const startSession = async () => { try { setSessionPhase('session'); setStatus('connecting'); setErrorMessage(null); transcriptRef.current = ""; currentTurnUserText.current = ""; currentTurnPartnerText.current = ""; setLiveAnalysis(null); setLiveMessages([]); const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext; outputContextRef.current = new AudioContextClass({ sampleRate: 24000 }); inputContextRef.current = new AudioContextClass({ sampleRate: 16000 }); const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); analysisChatRef.current = ai.chats.create({ model: 'gemini-3-flash-preview', config: { systemInstruction: `${SYSTEM_ANALYSIS_PROMPT}\nSCENARIO: ${scenario.title}\nObjectives: ${scenario.cbtFocus.join(', ')}`, responseMimeType: 'application/json', responseSchema: CBT_ANALYSIS_SCHEMA, } }); const sessionPromise = ai.live.connect({ model: 'gemini-2.5-flash-native-audio-preview-09-2025', config: { responseModalities: [Modality.AUDIO], systemInstruction: ` ${scenario.systemInstruction} VOICE MODE: Keep responses short and empathetic. INITIATION: When the user says "Let's start", "Hello", or initiates speech, begin with your opening: "${scenario.initialMessage}". `, outputAudioTranscription: {}, inputAudioTranscription: {}, speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: voiceSettings.voiceName } } }, }, callbacks: { onopen: () => { setStatus('active'); if (!inputContextRef.current) return; const source = inputContextRef.current.createMediaStreamSource(stream); const processor = inputContextRef.current.createScriptProcessor(4096, 1, 1); processor.onaudioprocess = (e) => { const data = e.inputBuffer.getChannelData(0); let sum = 0; for(let i=0; i s.sendRealtimeInput({ media: { mimeType: 'audio/pcm;rate=16000', data: b64 } })); }; source.connect(processor); processor.connect(inputContextRef.current.destination); }, onmessage: async (msg: LiveServerMessage) => { if (msg.serverContent?.inputTranscription) { const text = msg.serverContent.inputTranscription.text; currentTurnUserText.current += text; setLiveMessages(prev => { const last = prev[prev.length - 1]; if (last && last.role === 'user') return [...prev.slice(0, -1), { ...last, text: last.text + text }]; return [...prev, { id: Date.now().toString(), role: 'user', text }]; }); } if (msg.serverContent?.outputTranscription) { const text = msg.serverContent.outputTranscription.text; currentTurnPartnerText.current += text; setLiveMessages(prev => { const last = prev[prev.length - 1]; if (last && last.role === 'model') return [...prev.slice(0, -1), { ...last, text: last.text + text }]; return [...prev, { id: Date.now().toString(), role: 'model', text }]; }); } const audioData = msg.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data; if (audioData && outputContextRef.current) { const bytes = base64ToUint8Array(audioData); try { const audioBuffer = await decodeAudioData(bytes, outputContextRef.current, 24000, 1); const source = outputContextRef.current.createBufferSource(); source.buffer = audioBuffer; source.connect(outputContextRef.current.destination); const now = outputContextRef.current.currentTime; if (nextStartTimeRef.current < now) nextStartTimeRef.current = now + 0.05; source.start(nextStartTimeRef.current); nextStartTimeRef.current += audioBuffer.duration; sourcesRef.current.add(source); source.onended = () => sourcesRef.current.delete(source); } catch (e) { console.error("Audio decoding error:", e); } } if (msg.serverContent?.interrupted) { sourcesRef.current.forEach(src => { try { src.stop(); } catch(e){} }); sourcesRef.current.clear(); if (outputContextRef.current) nextStartTimeRef.current = outputContextRef.current.currentTime; } if (msg.serverContent?.turnComplete) { transcriptRef.current += `[USER]: ${currentTurnUserText.current}\n[PARTNER]: ${currentTurnPartnerText.current}\n`; if (analysisChatRef.current) { setIsAnalyzingTurn(true); analysisChatRef.current.sendMessage({ message: `Evaluation Turn: User("${currentTurnUserText.current}") Partner("${currentTurnPartnerText.current}")` }).then(res => { if (res.text) setLiveAnalysis(JSON.parse(res.text)); setIsAnalyzingTurn(false); currentTurnUserText.current = ""; currentTurnPartnerText.current = ""; }).catch(() => setIsAnalyzingTurn(false)); } } }, onerror: () => { setStatus('error'); setErrorMessage("Voice bridge interrupted."); disconnectSession(); } } }); sessionRef.current = sessionPromise; } catch (err: any) { setStatus('error'); } }; if (sessionPhase === 'briefing') { return (

Ready to start?

Vocal Partner: {voiceSettings.voiceName} ({voiceSettings.vocalStyle})

Narrative Setting

"{scenario.description}"

Techniques

{scenario.cbtFocus.map(f => ({f}))}

Profile

{voiceSettings.genderPreference} Speaker
Protocol

Once active, say "Let's start" to begin the interaction with your {voiceSettings.vocalStyle.toLowerCase()} partner.

); } if (sessionPhase === 'analyzing') return (

Analyzing Session

Generating your clinical progress report...

); if (sessionPhase === 'finished' && finalAnalysis) return ; return (
{status === 'active' && (
)}

{scenario.title}

{status}
{status === 'connecting' &&

Initializing {voiceSettings.voiceName}...

} {status === 'active' && (

Listening...

Partner: {voiceSettings.voiceName}

)}
{status === 'active' && ()}
{showTranscript && setShowTranscript(false)} />}
{showLiveInsights && }
); }; export default VoiceSession;