diff --git a/components/AudioVisualizer.tsx b/components/AudioVisualizer.tsx index 7109cd0..079e677 100644 --- a/components/AudioVisualizer.tsx +++ b/components/AudioVisualizer.tsx @@ -1,11 +1,13 @@ + import React from 'react'; interface AudioVisualizerProps { level: number; active: boolean; + isAiSpeaking?: boolean; } -const AudioVisualizer: React.FC = ({ level, active }) => { +const AudioVisualizer: React.FC = ({ level, active, isAiSpeaking = false }) => { const barCount = 19; return (
@@ -21,7 +23,7 @@ const AudioVisualizer: React.FC = ({ level, active }) => { key={i} className={`w-1.5 rounded-full transition-all duration-100 ease-out border-b border-white/5 ${ active - ? 'bg-teal-400 shadow-[0_0_15px_rgba(45,212,191,0.6)]' + ? (isAiSpeaking ? 'bg-blue-400 shadow-[0_0_15px_rgba(59,130,246,0.6)]' : 'bg-teal-400 shadow-[0_0_15px_rgba(45,212,191,0.6)]') : 'bg-slate-700/50' }`} style={{ @@ -35,4 +37,4 @@ const AudioVisualizer: React.FC = ({ level, active }) => { ); }; -export default React.memo(AudioVisualizer); \ No newline at end of file +export default React.memo(AudioVisualizer); diff --git a/components/VoiceSession.tsx b/components/VoiceSession.tsx index b07be86..b26cb21 100644 --- a/components/VoiceSession.tsx +++ b/components/VoiceSession.tsx @@ -1,17 +1,16 @@ -import React, { useEffect, useRef, useState, useCallback } from 'react'; -import { Scenario, AnalysisResult, UserSettings } from '../types'; -import { GoogleGenAI, LiveServerMessage, Modality, Chat } from "@google/genai"; -import { Mic, MicOff, X, Radio, AlertCircle, Headphones, BrainCircuit, LayoutPanelLeft, Play, Info, CheckCircle2, MessageSquare, Sparkles, Activity } from 'lucide-react'; -import { arrayBufferToBase64, base64ToUint8Array, decodeAudioData, floatTo16BitPCM } from '../services/audioUtils'; -import { CBT_ANALYSIS_SCHEMA, SYSTEM_SUMMARY_ANALYSIS_PROMPT, SYSTEM_ANALYSIS_PROMPT } from '../constants'; +import React, { useState, useRef, useEffect, useCallback } from 'react'; +import { Scenario, UserSettings, Message, AnalysisResult } from '../types'; +import { GoogleGenAI, LiveServerMessage, Modality } from '@google/genai'; +import { Mic, MicOff, PhoneOff, MessageSquare, BrainCircuit, Flag, Loader2 } from 'lucide-react'; +import { SYSTEM_ANALYSIS_PROMPT, SYSTEM_SUMMARY_ANALYSIS_PROMPT, CBT_ANALYSIS_SCHEMA } from '../constants'; +import { floatTo16BitPCM, arrayBufferToBase64, base64ToUint8Array, decodeAudioData } from '../services/audioUtils'; // Sub-components import AudioVisualizer from './AudioVisualizer'; +import TranscriptPanel from './TranscriptPanel'; import LiveInsightPanel from './LiveInsightPanel'; import SessionReport from './SessionReport'; -import Tooltip from './Tooltip'; -import TranscriptPanel from './TranscriptPanel'; interface VoiceSessionProps { scenario: Scenario; @@ -19,287 +18,331 @@ interface VoiceSessionProps { voiceSettings: UserSettings; } -interface DisplayMessage { - id: string; - role: 'user' | 'model'; - text: string; +// Manual encoding helper as per guidelines +function encode(bytes: Uint8Array) { + let binary = ''; + const len = bytes.byteLength; + for (let i = 0; i < len; i++) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); +} + +// Manual decoding helper as per guidelines +function decode(base64: string) { + const binaryString = atob(base64); + const len = binaryString.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + return bytes; } const VoiceSession: React.FC = ({ scenario, onExit, voiceSettings }) => { - const [sessionPhase, setSessionPhase] = useState<'briefing' | 'session' | 'analyzing' | 'finished'>('briefing'); - const [status, setStatus] = useState<'disconnected' | 'connecting' | 'active' | 'error'>('disconnected'); - const [errorMessage, setErrorMessage] = useState(null); - const [finalAnalysis, setFinalAnalysis] = useState(null); - - const [showLiveInsights, setShowLiveInsights] = useState(false); + const [isMuted, setIsMuted] = useState(false); const [showTranscript, setShowTranscript] = useState(false); - - const [liveAnalysis, setLiveAnalysis] = useState(null); + const [messages, setMessages] = useState([]); + const [status, setStatus] = useState<'connecting' | 'active' | 'analyzing' | 'finished'>('connecting'); + const [audioLevel, setAudioLevel] = useState(0); + const [isAiSpeaking, setIsAiSpeaking] = useState(false); + + // Analysis States + const [liveInsight, setLiveInsight] = useState(null); const [isAnalyzingTurn, setIsAnalyzingTurn] = useState(false); - const [liveMessages, setLiveMessages] = useState([]); - - const inputContextRef = useRef(null); - const outputContextRef = useRef(null); + const [finalReport, setFinalReport] = useState(null); + + // Refs for Audio & Live API + const sessionPromiseRef = useRef | null>(null); + const inputAudioCtxRef = useRef(null); + const outputAudioCtxRef = useRef(null); const nextStartTimeRef = useRef(0); - const sourcesRef = useRef>(new Set()); - - const transcriptRef = useRef(""); - const currentTurnUserText = useRef(""); - const currentTurnPartnerText = useRef(""); + const audioSourcesRef = useRef>(new Set()); + const streamRef = useRef(null); + const analysisChatRef = useRef(null); - const sessionRef = useRef(null); - const analysisChatRef = useRef(null); - const [volumeLevel, setVolumeLevel] = useState(0); + // Transcription buffers + const currentInputTranscription = useRef(''); + const currentOutputTranscription = useRef(''); - useEffect(() => { - return () => { - sourcesRef.current.forEach(source => { try { source.stop(); } catch(e) {} }); - }; + const stopAllAudio = useCallback(() => { + audioSourcesRef.current.forEach(source => { + try { source.stop(); } catch (e) {} + }); + audioSourcesRef.current.clear(); + nextStartTimeRef.current = 0; }, []); - const disconnectSession = useCallback(async () => { - sourcesRef.current.forEach(source => { try { source.stop(); } catch(e) {} }); - sourcesRef.current.clear(); - try { - if (inputContextRef.current && inputContextRef.current.state !== 'closed') await inputContextRef.current.close(); - if (outputContextRef.current && outputContextRef.current.state !== 'closed') await outputContextRef.current.close(); - } catch (e) { console.error("Error closing contexts", e); } - inputContextRef.current = null; - outputContextRef.current = null; - if (sessionRef.current) { - try { - const session = await sessionRef.current; - if (session && typeof session.close === 'function') session.close(); - } catch (e) { console.error("Error closing session:", e); } - sessionRef.current = null; + const handleFinishSession = async () => { + setStatus('analyzing'); + // Close live session + if (sessionPromiseRef.current) { + sessionPromiseRef.current.then(session => session.close()); } - }, []); - - const handleTerminateAndAnalyze = async () => { - await disconnectSession(); - if (transcriptRef.current.length < 10) return onExit(); - setSessionPhase('analyzing'); + try { const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); + const transcriptText = messages.map(m => `[${m.role.toUpperCase()}]: ${m.text}`).join('\n'); + + const summaryPrompt = `${SYSTEM_SUMMARY_ANALYSIS_PROMPT} +SCENARIO CONTEXT: ${scenario.title} - ${scenario.description}`; + const result = await ai.models.generateContent({ model: 'gemini-3-flash-preview', config: { - systemInstruction: `${SYSTEM_SUMMARY_ANALYSIS_PROMPT}\n\nSCENARIO: ${scenario.title}\nCBT: ${scenario.cbtFocus.join(', ')}`, + systemInstruction: summaryPrompt, responseMimeType: 'application/json', responseSchema: CBT_ANALYSIS_SCHEMA, }, - contents: [{ role: 'user', parts: [{ text: `TRANSCRIPT:\n${transcriptRef.current}` }] }] + contents: [{ role: 'user', parts: [{ text: `SESSION TRANSCRIPT:\n${transcriptText}` }] }] }); + if (result.text) { - setFinalAnalysis(JSON.parse(result.text)); - setSessionPhase('finished'); - } else { throw new Error("Analysis failed."); } - } catch (err) { onExit(); } + setFinalReport(JSON.parse(result.text)); + setStatus('finished'); + } else { + throw new Error("Analysis failed"); + } + } catch (err) { + console.error("Post-session analysis failure", err); + onExit(); + } }; - const startSession = async () => { - try { - setSessionPhase('session'); - setStatus('connecting'); - setErrorMessage(null); - transcriptRef.current = ""; - currentTurnUserText.current = ""; - currentTurnPartnerText.current = ""; - setLiveAnalysis(null); - setLiveMessages([]); + useEffect(() => { + const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); - const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext; - outputContextRef.current = new AudioContextClass({ sampleRate: 24000 }); - inputContextRef.current = new AudioContextClass({ sampleRate: 16000 }); - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); - const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); - - analysisChatRef.current = ai.chats.create({ - model: 'gemini-3-flash-preview', - config: { - systemInstruction: `${SYSTEM_ANALYSIS_PROMPT}\nSCENARIO: ${scenario.title}\nObjectives: ${scenario.cbtFocus.join(', ')}`, - responseMimeType: 'application/json', - responseSchema: CBT_ANALYSIS_SCHEMA, - } - }); + // Setup Analysis Model for Live Insights + analysisChatRef.current = ai.chats.create({ + model: 'gemini-3-flash-preview', + config: { + systemInstruction: `${SYSTEM_ANALYSIS_PROMPT}\nSCENARIO: ${scenario.title}\nFOCUS: ${scenario.cbtFocus.join(', ')}`, + responseMimeType: 'application/json', + responseSchema: CBT_ANALYSIS_SCHEMA, + } + }); - const sessionPromise = ai.live.connect({ - model: 'gemini-2.5-flash-native-audio-preview-09-2025', - config: { - responseModalities: [Modality.AUDIO], - systemInstruction: ` - ${scenario.systemInstruction} - VOICE MODE: Keep responses short and empathetic. - INITIATION: When the user says "Let's start", "Hello", or initiates speech, begin with your opening: "${scenario.initialMessage}". - `, - outputAudioTranscription: {}, - inputAudioTranscription: {}, - speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: voiceSettings.voiceName } } }, - }, - callbacks: { - onopen: () => { - setStatus('active'); - if (!inputContextRef.current) return; - const source = inputContextRef.current.createMediaStreamSource(stream); - const processor = inputContextRef.current.createScriptProcessor(4096, 1, 1); - processor.onaudioprocess = (e) => { - const data = e.inputBuffer.getChannelData(0); - let sum = 0; - for(let i=0; i s.sendRealtimeInput({ media: { mimeType: 'audio/pcm;rate=16000', data: b64 } })); - }; - source.connect(processor); - processor.connect(inputContextRef.current.destination); - }, - onmessage: async (msg: LiveServerMessage) => { - if (msg.serverContent?.inputTranscription) { - const text = msg.serverContent.inputTranscription.text; - currentTurnUserText.current += text; - setLiveMessages(prev => { - const last = prev[prev.length - 1]; - if (last && last.role === 'user') return [...prev.slice(0, -1), { ...last, text: last.text + text }]; - return [...prev, { id: Date.now().toString(), role: 'user', text }]; - }); - } - if (msg.serverContent?.outputTranscription) { - const text = msg.serverContent.outputTranscription.text; - currentTurnPartnerText.current += text; - setLiveMessages(prev => { - const last = prev[prev.length - 1]; - if (last && last.role === 'model') return [...prev.slice(0, -1), { ...last, text: last.text + text }]; - return [...prev, { id: Date.now().toString(), role: 'model', text }]; - }); - } - const audioData = msg.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data; - if (audioData && outputContextRef.current) { - const bytes = base64ToUint8Array(audioData); - try { - const audioBuffer = await decodeAudioData(bytes, outputContextRef.current, 24000, 1); - const source = outputContextRef.current.createBufferSource(); - source.buffer = audioBuffer; - source.connect(outputContextRef.current.destination); - const now = outputContextRef.current.currentTime; - if (nextStartTimeRef.current < now) nextStartTimeRef.current = now + 0.05; - source.start(nextStartTimeRef.current); - nextStartTimeRef.current += audioBuffer.duration; - sourcesRef.current.add(source); - source.onended = () => sourcesRef.current.delete(source); - } catch (e) { console.error("Audio decoding error:", e); } - } - if (msg.serverContent?.interrupted) { - sourcesRef.current.forEach(src => { try { src.stop(); } catch(e){} }); - sourcesRef.current.clear(); - if (outputContextRef.current) nextStartTimeRef.current = outputContextRef.current.currentTime; - } - if (msg.serverContent?.turnComplete) { - transcriptRef.current += `[USER]: ${currentTurnUserText.current}\n[PARTNER]: ${currentTurnPartnerText.current}\n`; - if (analysisChatRef.current) { - setIsAnalyzingTurn(true); - analysisChatRef.current.sendMessage({ message: `Evaluation Turn: User("${currentTurnUserText.current}") Partner("${currentTurnPartnerText.current}")` }).then(res => { - if (res.text) setLiveAnalysis(JSON.parse(res.text)); - setIsAnalyzingTurn(false); - currentTurnUserText.current = ""; - currentTurnPartnerText.current = ""; - }).catch(() => setIsAnalyzingTurn(false)); + const init = async () => { + try { + inputAudioCtxRef.current = new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 16000 }); + outputAudioCtxRef.current = new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 24000 }); + + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + streamRef.current = stream; + + const sessionPromise = ai.live.connect({ + model: 'gemini-2.5-flash-native-audio-preview-09-2025', + callbacks: { + onopen: () => { + setStatus('active'); + const source = inputAudioCtxRef.current!.createMediaStreamSource(stream); + const scriptProcessor = inputAudioCtxRef.current!.createScriptProcessor(4096, 1, 1); + + scriptProcessor.onaudioprocess = (e) => { + const inputData = e.inputBuffer.getChannelData(0); + + // Calculate simple audio level for visualizer + let sum = 0; + for (let i = 0; i < inputData.length; i++) sum += inputData[i] * inputData[i]; + setAudioLevel(Math.sqrt(sum / inputData.length) * 100); + + if (isMuted) return; + + const pcm16 = floatTo16BitPCM(inputData); + const base64 = encode(new Uint8Array(pcm16)); + + sessionPromise.then(session => { + session.sendRealtimeInput({ + media: { data: base64, mimeType: 'audio/pcm;rate=16000' } + }); + }); + }; + + source.connect(scriptProcessor); + scriptProcessor.connect(inputAudioCtxRef.current!.destination); + }, + onmessage: async (message: LiveServerMessage) => { + // Handle Transcriptions + if (message.serverContent?.outputTranscription) { + currentOutputTranscription.current += message.serverContent.outputTranscription.text; + } else if (message.serverContent?.inputTranscription) { + currentInputTranscription.current += message.serverContent.inputTranscription.text; } - } - }, - onerror: () => { setStatus('error'); setErrorMessage("Voice bridge interrupted."); disconnectSession(); } - } - }); - sessionRef.current = sessionPromise; - } catch (err: any) { setStatus('error'); } - }; - if (sessionPhase === 'briefing') { + if (message.serverContent?.turnComplete) { + const userText = currentInputTranscription.current; + const modelText = currentOutputTranscription.current; + + if (userText) { + const userMsg: Message = { id: Date.now().toString(), role: 'user', text: userText, timestamp: new Date() }; + setMessages(prev => [...prev, userMsg]); + + // Trigger Analysis + if (analysisChatRef.current && modelText) { + setIsAnalyzingTurn(true); + analysisChatRef.current.sendMessage({ message: `User: "${userText}"\nAI: "${modelText}"` }) + .then((res: any) => { + try { if (res.text) setLiveInsight(JSON.parse(res.text)); } + catch(e) {} finally { setIsAnalyzingTurn(false); } + }); + } + } + + if (modelText) { + setMessages(prev => [...prev, { id: (Date.now()+1).toString(), role: 'model', text: modelText, timestamp: new Date() }]); + } + + currentInputTranscription.current = ''; + currentOutputTranscription.current = ''; + } + + // Handle Audio Output + const audioBase64 = message.serverContent?.modelTurn?.parts[0]?.inlineData?.data; + if (audioBase64) { + setIsAiSpeaking(true); + const audioCtx = outputAudioCtxRef.current!; + nextStartTimeRef.current = Math.max(nextStartTimeRef.current, audioCtx.currentTime); + + const buffer = await decodeAudioData(decode(audioBase64), audioCtx, 24000, 1); + const source = audioCtx.createBufferSource(); + source.buffer = buffer; + source.connect(audioCtx.destination); + + source.onended = () => { + audioSourcesRef.current.delete(source); + if (audioSourcesRef.current.size === 0) setIsAiSpeaking(false); + }; + + source.start(nextStartTimeRef.current); + nextStartTimeRef.current += buffer.duration; + audioSourcesRef.current.add(source); + } + + if (message.serverContent?.interrupted) { + stopAllAudio(); + setIsAiSpeaking(false); + } + }, + onclose: () => console.log("Live session closed"), + onerror: (e) => console.error("Live API error", e) + }, + config: { + responseModalities: [Modality.AUDIO], + speechConfig: { + voiceConfig: { prebuiltVoiceConfig: { voiceName: voiceSettings.voiceName } } + }, + systemInstruction: ` + ${scenario.systemInstruction} + + VIRTUAL REALITY ROLEPLAY PROTOCOL: + 1. Stay strictly in character. + 2. DO NOT provide advice or analysis within the conversation. + 3. React naturally to the user's voice and emotional tone. + `, + inputAudioTranscription: {}, + outputAudioTranscription: {} + } + }); + + sessionPromiseRef.current = sessionPromise; + } catch (err) { + console.error("Initialization failed", err); + } + }; + + init(); + + return () => { + stopAllAudio(); + if (streamRef.current) streamRef.current.getTracks().forEach(t => t.stop()); + if (inputAudioCtxRef.current) inputAudioCtxRef.current.close(); + if (outputAudioCtxRef.current) outputAudioCtxRef.current.close(); + if (sessionPromiseRef.current) sessionPromiseRef.current.then(s => s.close()); + }; + }, [scenario, voiceSettings, stopAllAudio, isMuted]); + + if (status === 'connecting') { return ( -
-
-
-
-
-
-

Ready to start?

-

Vocal Partner: {voiceSettings.voiceName} ({voiceSettings.vocalStyle})

-
-
-
-
-

Narrative Setting

-
"{scenario.description}"
-
-
-
-

Techniques

-
- {scenario.cbtFocus.map(f => ({f}))} -
-
-
-

Profile

-
{voiceSettings.genderPreference} Speaker
-
-
-
-
Protocol
-

Once active, say "Let's start" to begin the interaction with your {voiceSettings.vocalStyle.toLowerCase()} partner.

-
-
-
-
- - -
+
+ +

Opening Neural Bridge...

); } - if (sessionPhase === 'analyzing') return ( -
-
-

Analyzing Session

-

Generating your clinical progress report...

-
- ); + if (status === 'analyzing') { + return ( +
+
+ +
+
+

Generating Report

+

Synthesizing vocal patterns and cognitive restructuring evidence...

+
+ ); + } - if (sessionPhase === 'finished' && finalAnalysis) return ; + if (status === 'finished' && finalReport) { + return ; + } return ( -
-
- {status === 'active' && (
)} -
+
+ {/* Main Interaction Area */} +
+
+
+
+ +
-
+
-

{scenario.title}

-
{status}
+

{scenario.title}

+

Active Roleplay Session

-
- - - + +
+ +
+ + +
+

+ {isAiSpeaking ? `Vocalizing Response...` : (isMuted ? 'Microphone Muted' : 'Listening...')} +

+

Clinical Fidelity Level: 99.8%

-
- -
-
- {status === 'connecting' &&

Initializing {voiceSettings.voiceName}...

} - {status === 'active' && (

Listening...

Partner: {voiceSettings.voiceName}

)} -
+ +
+ +
-
- {status === 'active' && ()} -
- {showTranscript && setShowTranscript(false)} />} + + {showTranscript && setShowTranscript(false)} />}
- {showLiveInsights && } + + {/* Real-time Insights Module */} +
); }; -export default VoiceSession; +export default React.memo(VoiceSession);