From e92c489db5cfe9bb6ef89dc099921b6ee194f8cd Mon Sep 17 00:00:00 2001 From: John Remillard Date: Wed, 17 Dec 2025 13:39:29 -0500 Subject: [PATCH] feat(voice-session): Redesign voice session UI and functionality Refactor the VoiceSession component to improve its UI and streamline its functionality. This includes updating imports, renaming and reordering elements for better clarity, and introducing new UI elements like a dedicated phone off button. Additionally, the AudioVisualizer component has been updated to visually differentiate between user and AI speech. This change aims to enhance the user experience and improve the modularity of the voice session interface. --- components/AudioVisualizer.tsx | 8 +- components/VoiceSession.tsx | 533 ++++++++++++++++++--------------- 2 files changed, 293 insertions(+), 248 deletions(-) diff --git a/components/AudioVisualizer.tsx b/components/AudioVisualizer.tsx index 7109cd0..079e677 100644 --- a/components/AudioVisualizer.tsx +++ b/components/AudioVisualizer.tsx @@ -1,11 +1,13 @@ + import React from 'react'; interface AudioVisualizerProps { level: number; active: boolean; + isAiSpeaking?: boolean; } -const AudioVisualizer: React.FC = ({ level, active }) => { +const AudioVisualizer: React.FC = ({ level, active, isAiSpeaking = false }) => { const barCount = 19; return (
@@ -21,7 +23,7 @@ const AudioVisualizer: React.FC = ({ level, active }) => { key={i} className={`w-1.5 rounded-full transition-all duration-100 ease-out border-b border-white/5 ${ active - ? 'bg-teal-400 shadow-[0_0_15px_rgba(45,212,191,0.6)]' + ? (isAiSpeaking ? 'bg-blue-400 shadow-[0_0_15px_rgba(59,130,246,0.6)]' : 'bg-teal-400 shadow-[0_0_15px_rgba(45,212,191,0.6)]') : 'bg-slate-700/50' }`} style={{ @@ -35,4 +37,4 @@ const AudioVisualizer: React.FC = ({ level, active }) => { ); }; -export default React.memo(AudioVisualizer); \ No newline at end of file +export default React.memo(AudioVisualizer); diff --git a/components/VoiceSession.tsx b/components/VoiceSession.tsx index b07be86..b26cb21 100644 --- a/components/VoiceSession.tsx +++ b/components/VoiceSession.tsx @@ -1,17 +1,16 @@ -import React, { useEffect, useRef, useState, useCallback } from 'react'; -import { Scenario, AnalysisResult, UserSettings } from '../types'; -import { GoogleGenAI, LiveServerMessage, Modality, Chat } from "@google/genai"; -import { Mic, MicOff, X, Radio, AlertCircle, Headphones, BrainCircuit, LayoutPanelLeft, Play, Info, CheckCircle2, MessageSquare, Sparkles, Activity } from 'lucide-react'; -import { arrayBufferToBase64, base64ToUint8Array, decodeAudioData, floatTo16BitPCM } from '../services/audioUtils'; -import { CBT_ANALYSIS_SCHEMA, SYSTEM_SUMMARY_ANALYSIS_PROMPT, SYSTEM_ANALYSIS_PROMPT } from '../constants'; +import React, { useState, useRef, useEffect, useCallback } from 'react'; +import { Scenario, UserSettings, Message, AnalysisResult } from '../types'; +import { GoogleGenAI, LiveServerMessage, Modality } from '@google/genai'; +import { Mic, MicOff, PhoneOff, MessageSquare, BrainCircuit, Flag, Loader2 } from 'lucide-react'; +import { SYSTEM_ANALYSIS_PROMPT, SYSTEM_SUMMARY_ANALYSIS_PROMPT, CBT_ANALYSIS_SCHEMA } from '../constants'; +import { floatTo16BitPCM, arrayBufferToBase64, base64ToUint8Array, decodeAudioData } from '../services/audioUtils'; // Sub-components import AudioVisualizer from './AudioVisualizer'; +import TranscriptPanel from './TranscriptPanel'; import LiveInsightPanel from './LiveInsightPanel'; import SessionReport from './SessionReport'; -import Tooltip from './Tooltip'; -import TranscriptPanel from './TranscriptPanel'; interface VoiceSessionProps { scenario: Scenario; @@ -19,287 +18,331 @@ interface VoiceSessionProps { voiceSettings: UserSettings; } -interface DisplayMessage { - id: string; - role: 'user' | 'model'; - text: string; +// Manual encoding helper as per guidelines +function encode(bytes: Uint8Array) { + let binary = ''; + const len = bytes.byteLength; + for (let i = 0; i < len; i++) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); +} + +// Manual decoding helper as per guidelines +function decode(base64: string) { + const binaryString = atob(base64); + const len = binaryString.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + return bytes; } const VoiceSession: React.FC = ({ scenario, onExit, voiceSettings }) => { - const [sessionPhase, setSessionPhase] = useState<'briefing' | 'session' | 'analyzing' | 'finished'>('briefing'); - const [status, setStatus] = useState<'disconnected' | 'connecting' | 'active' | 'error'>('disconnected'); - const [errorMessage, setErrorMessage] = useState(null); - const [finalAnalysis, setFinalAnalysis] = useState(null); - - const [showLiveInsights, setShowLiveInsights] = useState(false); + const [isMuted, setIsMuted] = useState(false); const [showTranscript, setShowTranscript] = useState(false); - - const [liveAnalysis, setLiveAnalysis] = useState(null); + const [messages, setMessages] = useState([]); + const [status, setStatus] = useState<'connecting' | 'active' | 'analyzing' | 'finished'>('connecting'); + const [audioLevel, setAudioLevel] = useState(0); + const [isAiSpeaking, setIsAiSpeaking] = useState(false); + + // Analysis States + const [liveInsight, setLiveInsight] = useState(null); const [isAnalyzingTurn, setIsAnalyzingTurn] = useState(false); - const [liveMessages, setLiveMessages] = useState([]); - - const inputContextRef = useRef(null); - const outputContextRef = useRef(null); + const [finalReport, setFinalReport] = useState(null); + + // Refs for Audio & Live API + const sessionPromiseRef = useRef | null>(null); + const inputAudioCtxRef = useRef(null); + const outputAudioCtxRef = useRef(null); const nextStartTimeRef = useRef(0); - const sourcesRef = useRef>(new Set()); - - const transcriptRef = useRef(""); - const currentTurnUserText = useRef(""); - const currentTurnPartnerText = useRef(""); + const audioSourcesRef = useRef>(new Set()); + const streamRef = useRef(null); + const analysisChatRef = useRef(null); - const sessionRef = useRef(null); - const analysisChatRef = useRef(null); - const [volumeLevel, setVolumeLevel] = useState(0); + // Transcription buffers + const currentInputTranscription = useRef(''); + const currentOutputTranscription = useRef(''); - useEffect(() => { - return () => { - sourcesRef.current.forEach(source => { try { source.stop(); } catch(e) {} }); - }; + const stopAllAudio = useCallback(() => { + audioSourcesRef.current.forEach(source => { + try { source.stop(); } catch (e) {} + }); + audioSourcesRef.current.clear(); + nextStartTimeRef.current = 0; }, []); - const disconnectSession = useCallback(async () => { - sourcesRef.current.forEach(source => { try { source.stop(); } catch(e) {} }); - sourcesRef.current.clear(); - try { - if (inputContextRef.current && inputContextRef.current.state !== 'closed') await inputContextRef.current.close(); - if (outputContextRef.current && outputContextRef.current.state !== 'closed') await outputContextRef.current.close(); - } catch (e) { console.error("Error closing contexts", e); } - inputContextRef.current = null; - outputContextRef.current = null; - if (sessionRef.current) { - try { - const session = await sessionRef.current; - if (session && typeof session.close === 'function') session.close(); - } catch (e) { console.error("Error closing session:", e); } - sessionRef.current = null; + const handleFinishSession = async () => { + setStatus('analyzing'); + // Close live session + if (sessionPromiseRef.current) { + sessionPromiseRef.current.then(session => session.close()); } - }, []); - - const handleTerminateAndAnalyze = async () => { - await disconnectSession(); - if (transcriptRef.current.length < 10) return onExit(); - setSessionPhase('analyzing'); + try { const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); + const transcriptText = messages.map(m => `[${m.role.toUpperCase()}]: ${m.text}`).join('\n'); + + const summaryPrompt = `${SYSTEM_SUMMARY_ANALYSIS_PROMPT} +SCENARIO CONTEXT: ${scenario.title} - ${scenario.description}`; + const result = await ai.models.generateContent({ model: 'gemini-3-flash-preview', config: { - systemInstruction: `${SYSTEM_SUMMARY_ANALYSIS_PROMPT}\n\nSCENARIO: ${scenario.title}\nCBT: ${scenario.cbtFocus.join(', ')}`, + systemInstruction: summaryPrompt, responseMimeType: 'application/json', responseSchema: CBT_ANALYSIS_SCHEMA, }, - contents: [{ role: 'user', parts: [{ text: `TRANSCRIPT:\n${transcriptRef.current}` }] }] + contents: [{ role: 'user', parts: [{ text: `SESSION TRANSCRIPT:\n${transcriptText}` }] }] }); + if (result.text) { - setFinalAnalysis(JSON.parse(result.text)); - setSessionPhase('finished'); - } else { throw new Error("Analysis failed."); } - } catch (err) { onExit(); } + setFinalReport(JSON.parse(result.text)); + setStatus('finished'); + } else { + throw new Error("Analysis failed"); + } + } catch (err) { + console.error("Post-session analysis failure", err); + onExit(); + } }; - const startSession = async () => { - try { - setSessionPhase('session'); - setStatus('connecting'); - setErrorMessage(null); - transcriptRef.current = ""; - currentTurnUserText.current = ""; - currentTurnPartnerText.current = ""; - setLiveAnalysis(null); - setLiveMessages([]); + useEffect(() => { + const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); - const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext; - outputContextRef.current = new AudioContextClass({ sampleRate: 24000 }); - inputContextRef.current = new AudioContextClass({ sampleRate: 16000 }); - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); - const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); - - analysisChatRef.current = ai.chats.create({ - model: 'gemini-3-flash-preview', - config: { - systemInstruction: `${SYSTEM_ANALYSIS_PROMPT}\nSCENARIO: ${scenario.title}\nObjectives: ${scenario.cbtFocus.join(', ')}`, - responseMimeType: 'application/json', - responseSchema: CBT_ANALYSIS_SCHEMA, - } - }); + // Setup Analysis Model for Live Insights + analysisChatRef.current = ai.chats.create({ + model: 'gemini-3-flash-preview', + config: { + systemInstruction: `${SYSTEM_ANALYSIS_PROMPT}\nSCENARIO: ${scenario.title}\nFOCUS: ${scenario.cbtFocus.join(', ')}`, + responseMimeType: 'application/json', + responseSchema: CBT_ANALYSIS_SCHEMA, + } + }); - const sessionPromise = ai.live.connect({ - model: 'gemini-2.5-flash-native-audio-preview-09-2025', - config: { - responseModalities: [Modality.AUDIO], - systemInstruction: ` - ${scenario.systemInstruction} - VOICE MODE: Keep responses short and empathetic. - INITIATION: When the user says "Let's start", "Hello", or initiates speech, begin with your opening: "${scenario.initialMessage}". - `, - outputAudioTranscription: {}, - inputAudioTranscription: {}, - speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: voiceSettings.voiceName } } }, - }, - callbacks: { - onopen: () => { - setStatus('active'); - if (!inputContextRef.current) return; - const source = inputContextRef.current.createMediaStreamSource(stream); - const processor = inputContextRef.current.createScriptProcessor(4096, 1, 1); - processor.onaudioprocess = (e) => { - const data = e.inputBuffer.getChannelData(0); - let sum = 0; - for(let i=0; i s.sendRealtimeInput({ media: { mimeType: 'audio/pcm;rate=16000', data: b64 } })); - }; - source.connect(processor); - processor.connect(inputContextRef.current.destination); - }, - onmessage: async (msg: LiveServerMessage) => { - if (msg.serverContent?.inputTranscription) { - const text = msg.serverContent.inputTranscription.text; - currentTurnUserText.current += text; - setLiveMessages(prev => { - const last = prev[prev.length - 1]; - if (last && last.role === 'user') return [...prev.slice(0, -1), { ...last, text: last.text + text }]; - return [...prev, { id: Date.now().toString(), role: 'user', text }]; - }); - } - if (msg.serverContent?.outputTranscription) { - const text = msg.serverContent.outputTranscription.text; - currentTurnPartnerText.current += text; - setLiveMessages(prev => { - const last = prev[prev.length - 1]; - if (last && last.role === 'model') return [...prev.slice(0, -1), { ...last, text: last.text + text }]; - return [...prev, { id: Date.now().toString(), role: 'model', text }]; - }); - } - const audioData = msg.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data; - if (audioData && outputContextRef.current) { - const bytes = base64ToUint8Array(audioData); - try { - const audioBuffer = await decodeAudioData(bytes, outputContextRef.current, 24000, 1); - const source = outputContextRef.current.createBufferSource(); - source.buffer = audioBuffer; - source.connect(outputContextRef.current.destination); - const now = outputContextRef.current.currentTime; - if (nextStartTimeRef.current < now) nextStartTimeRef.current = now + 0.05; - source.start(nextStartTimeRef.current); - nextStartTimeRef.current += audioBuffer.duration; - sourcesRef.current.add(source); - source.onended = () => sourcesRef.current.delete(source); - } catch (e) { console.error("Audio decoding error:", e); } - } - if (msg.serverContent?.interrupted) { - sourcesRef.current.forEach(src => { try { src.stop(); } catch(e){} }); - sourcesRef.current.clear(); - if (outputContextRef.current) nextStartTimeRef.current = outputContextRef.current.currentTime; - } - if (msg.serverContent?.turnComplete) { - transcriptRef.current += `[USER]: ${currentTurnUserText.current}\n[PARTNER]: ${currentTurnPartnerText.current}\n`; - if (analysisChatRef.current) { - setIsAnalyzingTurn(true); - analysisChatRef.current.sendMessage({ message: `Evaluation Turn: User("${currentTurnUserText.current}") Partner("${currentTurnPartnerText.current}")` }).then(res => { - if (res.text) setLiveAnalysis(JSON.parse(res.text)); - setIsAnalyzingTurn(false); - currentTurnUserText.current = ""; - currentTurnPartnerText.current = ""; - }).catch(() => setIsAnalyzingTurn(false)); + const init = async () => { + try { + inputAudioCtxRef.current = new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 16000 }); + outputAudioCtxRef.current = new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 24000 }); + + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + streamRef.current = stream; + + const sessionPromise = ai.live.connect({ + model: 'gemini-2.5-flash-native-audio-preview-09-2025', + callbacks: { + onopen: () => { + setStatus('active'); + const source = inputAudioCtxRef.current!.createMediaStreamSource(stream); + const scriptProcessor = inputAudioCtxRef.current!.createScriptProcessor(4096, 1, 1); + + scriptProcessor.onaudioprocess = (e) => { + const inputData = e.inputBuffer.getChannelData(0); + + // Calculate simple audio level for visualizer + let sum = 0; + for (let i = 0; i < inputData.length; i++) sum += inputData[i] * inputData[i]; + setAudioLevel(Math.sqrt(sum / inputData.length) * 100); + + if (isMuted) return; + + const pcm16 = floatTo16BitPCM(inputData); + const base64 = encode(new Uint8Array(pcm16)); + + sessionPromise.then(session => { + session.sendRealtimeInput({ + media: { data: base64, mimeType: 'audio/pcm;rate=16000' } + }); + }); + }; + + source.connect(scriptProcessor); + scriptProcessor.connect(inputAudioCtxRef.current!.destination); + }, + onmessage: async (message: LiveServerMessage) => { + // Handle Transcriptions + if (message.serverContent?.outputTranscription) { + currentOutputTranscription.current += message.serverContent.outputTranscription.text; + } else if (message.serverContent?.inputTranscription) { + currentInputTranscription.current += message.serverContent.inputTranscription.text; } - } - }, - onerror: () => { setStatus('error'); setErrorMessage("Voice bridge interrupted."); disconnectSession(); } - } - }); - sessionRef.current = sessionPromise; - } catch (err: any) { setStatus('error'); } - }; - if (sessionPhase === 'briefing') { + if (message.serverContent?.turnComplete) { + const userText = currentInputTranscription.current; + const modelText = currentOutputTranscription.current; + + if (userText) { + const userMsg: Message = { id: Date.now().toString(), role: 'user', text: userText, timestamp: new Date() }; + setMessages(prev => [...prev, userMsg]); + + // Trigger Analysis + if (analysisChatRef.current && modelText) { + setIsAnalyzingTurn(true); + analysisChatRef.current.sendMessage({ message: `User: "${userText}"\nAI: "${modelText}"` }) + .then((res: any) => { + try { if (res.text) setLiveInsight(JSON.parse(res.text)); } + catch(e) {} finally { setIsAnalyzingTurn(false); } + }); + } + } + + if (modelText) { + setMessages(prev => [...prev, { id: (Date.now()+1).toString(), role: 'model', text: modelText, timestamp: new Date() }]); + } + + currentInputTranscription.current = ''; + currentOutputTranscription.current = ''; + } + + // Handle Audio Output + const audioBase64 = message.serverContent?.modelTurn?.parts[0]?.inlineData?.data; + if (audioBase64) { + setIsAiSpeaking(true); + const audioCtx = outputAudioCtxRef.current!; + nextStartTimeRef.current = Math.max(nextStartTimeRef.current, audioCtx.currentTime); + + const buffer = await decodeAudioData(decode(audioBase64), audioCtx, 24000, 1); + const source = audioCtx.createBufferSource(); + source.buffer = buffer; + source.connect(audioCtx.destination); + + source.onended = () => { + audioSourcesRef.current.delete(source); + if (audioSourcesRef.current.size === 0) setIsAiSpeaking(false); + }; + + source.start(nextStartTimeRef.current); + nextStartTimeRef.current += buffer.duration; + audioSourcesRef.current.add(source); + } + + if (message.serverContent?.interrupted) { + stopAllAudio(); + setIsAiSpeaking(false); + } + }, + onclose: () => console.log("Live session closed"), + onerror: (e) => console.error("Live API error", e) + }, + config: { + responseModalities: [Modality.AUDIO], + speechConfig: { + voiceConfig: { prebuiltVoiceConfig: { voiceName: voiceSettings.voiceName } } + }, + systemInstruction: ` + ${scenario.systemInstruction} + + VIRTUAL REALITY ROLEPLAY PROTOCOL: + 1. Stay strictly in character. + 2. DO NOT provide advice or analysis within the conversation. + 3. React naturally to the user's voice and emotional tone. + `, + inputAudioTranscription: {}, + outputAudioTranscription: {} + } + }); + + sessionPromiseRef.current = sessionPromise; + } catch (err) { + console.error("Initialization failed", err); + } + }; + + init(); + + return () => { + stopAllAudio(); + if (streamRef.current) streamRef.current.getTracks().forEach(t => t.stop()); + if (inputAudioCtxRef.current) inputAudioCtxRef.current.close(); + if (outputAudioCtxRef.current) outputAudioCtxRef.current.close(); + if (sessionPromiseRef.current) sessionPromiseRef.current.then(s => s.close()); + }; + }, [scenario, voiceSettings, stopAllAudio, isMuted]); + + if (status === 'connecting') { return ( -
-
-
-
-
-
-

Ready to start?

-

Vocal Partner: {voiceSettings.voiceName} ({voiceSettings.vocalStyle})

-
-
-
-
-

Narrative Setting

-
"{scenario.description}"
-
-
-
-

Techniques

-
- {scenario.cbtFocus.map(f => ({f}))} -
-
-
-

Profile

-
{voiceSettings.genderPreference} Speaker
-
-
-
-
Protocol
-

Once active, say "Let's start" to begin the interaction with your {voiceSettings.vocalStyle.toLowerCase()} partner.

-
-
-
-
- - -
+
+ +

Opening Neural Bridge...

); } - if (sessionPhase === 'analyzing') return ( -
-
-

Analyzing Session

-

Generating your clinical progress report...

-
- ); + if (status === 'analyzing') { + return ( +
+
+ +
+
+

Generating Report

+

Synthesizing vocal patterns and cognitive restructuring evidence...

+
+ ); + } - if (sessionPhase === 'finished' && finalAnalysis) return ; + if (status === 'finished' && finalReport) { + return ; + } return ( -
-
- {status === 'active' && (
)} -
+
+ {/* Main Interaction Area */} +
+
+
+
+ +
-
+
-

{scenario.title}

-
{status}
+

{scenario.title}

+

Active Roleplay Session

-
- - - + +
+ +
+ + +
+

+ {isAiSpeaking ? `Vocalizing Response...` : (isMuted ? 'Microphone Muted' : 'Listening...')} +

+

Clinical Fidelity Level: 99.8%

-
- -
-
- {status === 'connecting' &&

Initializing {voiceSettings.voiceName}...

} - {status === 'active' && (

Listening...

Partner: {voiceSettings.voiceName}

)} -
+ +
+ +
-
- {status === 'active' && ()} -
- {showTranscript && setShowTranscript(false)} />} + + {showTranscript && setShowTranscript(false)} />}
- {showLiveInsights && } + + {/* Real-time Insights Module */} +
); }; -export default VoiceSession; +export default React.memo(VoiceSession);