John Remillard
c78604de02
Sets up the basic project configuration for Cognitive Clarity. This includes: - Initializing `package.json` with necessary dependencies (React, Gemini API, Lucide React) and build scripts for Vite. - Creating a basic `index.html` with Tailwind CSS and Google Fonts integration. - Setting up `index.tsx` as the React application entry point. - Configuring `tsconfig.json` for TypeScript compilation with Vite. - Establishing `vite.config.ts` for development server setup and environment variable handling. - Adding a standard `.gitignore` file. - Updating `README.md` with local run instructions and AI Studio links. - Defining `metadata.json` for application details. - Creating core type definitions in `types.ts`. - Populating `constants.ts` with initial CBT glossary terms. - Adding utility functions for audio processing in `services/audioUtils.ts`.
387 lines
17 KiB
TypeScript
387 lines
17 KiB
TypeScript
import React, { useEffect, useRef, useState, useCallback } from 'react';
|
|
// Corrected import: AnalysisResult is defined in types.ts, not LiveInsightPanel.tsx
|
|
import { Scenario, AnalysisResult } from '../types';
|
|
import { GoogleGenAI, LiveServerMessage, Modality, Chat } from "@google/genai";
|
|
import { Mic, MicOff, X, Radio, AlertCircle, Headphones, BrainCircuit, LayoutPanelLeft } from 'lucide-react';
|
|
import { arrayBufferToBase64, base64ToUint8Array, decodeAudioData, floatTo16BitPCM } from '../services/audioUtils';
|
|
import { CBT_ANALYSIS_SCHEMA, SYSTEM_SUMMARY_ANALYSIS_PROMPT, SYSTEM_ANALYSIS_PROMPT } from '../constants';
|
|
|
|
// Sub-components
|
|
import AudioVisualizer from './AudioVisualizer';
|
|
// Corrected import: LiveInsightPanel is a default export, AnalysisResult is already imported from types
|
|
import LiveInsightPanel from './LiveInsightPanel';
|
|
import SessionReport from './SessionReport';
|
|
|
|
interface VoiceSessionProps {
|
|
scenario: Scenario;
|
|
onExit: () => void;
|
|
}
|
|
|
|
const VoiceSession: React.FC<VoiceSessionProps> = ({ scenario, onExit }) => {
|
|
const [status, setStatus] = useState<'disconnected' | 'connecting' | 'connected' | 'error' | 'analyzing' | 'finished'>('disconnected');
|
|
const [errorMessage, setErrorMessage] = useState<string | null>(null);
|
|
const [finalAnalysis, setFinalAnalysis] = useState<AnalysisResult | null>(null);
|
|
|
|
// Live Analysis State
|
|
const [liveAnalysis, setLiveAnalysis] = useState<AnalysisResult | null>(null);
|
|
const [isAnalyzingTurn, setIsAnalyzingTurn] = useState(false);
|
|
const [showLiveInsights, setShowLiveInsights] = useState(false);
|
|
|
|
// Audio Refs
|
|
const inputContextRef = useRef<AudioContext | null>(null);
|
|
const outputContextRef = useRef<AudioContext | null>(null);
|
|
const nextStartTimeRef = useRef<number>(0);
|
|
const sourcesRef = useRef<Set<AudioBufferSourceNode>>(new Set());
|
|
|
|
// Transcript accumulation
|
|
const transcriptRef = useRef<string>("");
|
|
const currentTurnUserText = useRef<string>("");
|
|
const currentTurnPartnerText = useRef<string>("");
|
|
|
|
// AI Refs
|
|
const sessionRef = useRef<any>(null);
|
|
const analysisChatRef = useRef<Chat | null>(null);
|
|
|
|
// Visualization
|
|
const [volumeLevel, setVolumeLevel] = useState(0);
|
|
|
|
useEffect(() => {
|
|
return () => {
|
|
sourcesRef.current.forEach(source => { try { source.stop(); } catch(e) {} });
|
|
};
|
|
}, []);
|
|
|
|
const disconnectSession = useCallback(async () => {
|
|
sourcesRef.current.forEach(source => {
|
|
try { source.stop(); } catch(e) {}
|
|
});
|
|
sourcesRef.current.clear();
|
|
|
|
try {
|
|
if (inputContextRef.current && inputContextRef.current.state !== 'closed') {
|
|
await inputContextRef.current.close();
|
|
}
|
|
if (outputContextRef.current && outputContextRef.current.state !== 'closed') {
|
|
await outputContextRef.current.close();
|
|
}
|
|
} catch (e) {
|
|
console.error("Error closing contexts", e);
|
|
}
|
|
inputContextRef.current = null;
|
|
outputContextRef.current = null;
|
|
|
|
if (sessionRef.current) {
|
|
try {
|
|
const session = await sessionRef.current;
|
|
if (session && typeof session.close === 'function') {
|
|
session.close();
|
|
}
|
|
} catch (e) {
|
|
console.error("Error closing session:", e);
|
|
}
|
|
sessionRef.current = null;
|
|
}
|
|
}, []);
|
|
|
|
const handleTerminateAndAnalyze = async () => {
|
|
await disconnectSession();
|
|
|
|
if (transcriptRef.current.length < 10) {
|
|
setStatus('disconnected');
|
|
onExit();
|
|
return;
|
|
}
|
|
|
|
setStatus('analyzing');
|
|
try {
|
|
const ai = new GoogleGenAI({ apiKey: process.env.API_KEY });
|
|
const systemPrompt = `${SYSTEM_SUMMARY_ANALYSIS_PROMPT}
|
|
|
|
SCENARIO CONTEXT:
|
|
Title: ${scenario.title}
|
|
Description: ${scenario.description}
|
|
CBT Focus: ${scenario.cbtFocus.join(', ')}`;
|
|
|
|
const result = await ai.models.generateContent({
|
|
model: 'gemini-3-flash-preview',
|
|
config: {
|
|
systemInstruction: systemPrompt,
|
|
responseMimeType: 'application/json',
|
|
responseSchema: CBT_ANALYSIS_SCHEMA,
|
|
},
|
|
contents: [{ role: 'user', parts: [{ text: `SESSION TRANSCRIPT:\n${transcriptRef.current}` }] }]
|
|
});
|
|
|
|
if (result.text) {
|
|
setFinalAnalysis(JSON.parse(result.text));
|
|
setStatus('finished');
|
|
} else {
|
|
throw new Error("Analysis synthesis failed.");
|
|
}
|
|
} catch (err) {
|
|
console.error("Post-session analysis error", err);
|
|
setStatus('disconnected');
|
|
onExit();
|
|
}
|
|
};
|
|
|
|
const startSession = async () => {
|
|
try {
|
|
setStatus('connecting');
|
|
setErrorMessage(null);
|
|
transcriptRef.current = "";
|
|
currentTurnUserText.current = "";
|
|
currentTurnPartnerText.current = "";
|
|
setLiveAnalysis(null);
|
|
|
|
const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext;
|
|
outputContextRef.current = new AudioContextClass({ sampleRate: 24000 });
|
|
inputContextRef.current = new AudioContextClass({ sampleRate: 16000 });
|
|
|
|
if (outputContextRef.current.state === 'suspended') await outputContextRef.current.resume();
|
|
if (inputContextRef.current.state === 'suspended') await inputContextRef.current.resume();
|
|
|
|
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
const ai = new GoogleGenAI({ apiKey: process.env.API_KEY });
|
|
|
|
analysisChatRef.current = ai.chats.create({
|
|
model: 'gemini-3-flash-preview',
|
|
config: {
|
|
systemInstruction: `${SYSTEM_ANALYSIS_PROMPT}\n\nSCENARIO CONTEXT:\nTitle: ${scenario.title}\nDescription: ${scenario.description}\nCBT Focus: ${scenario.cbtFocus.join(', ')}`,
|
|
responseMimeType: 'application/json',
|
|
responseSchema: CBT_ANALYSIS_SCHEMA,
|
|
}
|
|
});
|
|
|
|
const effectiveSystemInstruction = `
|
|
${scenario.systemInstruction}
|
|
IMPORTANT: This is a voice conversation. Keep responses concise (1-2 sentences). You are strictly in character.
|
|
START THE CONVERSATION IMMEDIATELY by saying: "${scenario.initialMessage}"
|
|
`;
|
|
|
|
const sessionPromise = ai.live.connect({
|
|
model: 'gemini-2.5-flash-native-audio-preview-09-2025',
|
|
config: {
|
|
responseModalities: [Modality.AUDIO],
|
|
systemInstruction: effectiveSystemInstruction,
|
|
outputAudioTranscription: {},
|
|
inputAudioTranscription: {},
|
|
speechConfig: {
|
|
voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Puck' } },
|
|
},
|
|
},
|
|
callbacks: {
|
|
onopen: () => {
|
|
setStatus('connected');
|
|
if (!inputContextRef.current) return;
|
|
|
|
const source = inputContextRef.current.createMediaStreamSource(stream);
|
|
const processor = inputContextRef.current.createScriptProcessor(4096, 1, 1);
|
|
const gainNode = inputContextRef.current.createGain();
|
|
gainNode.gain.value = 0;
|
|
|
|
processor.onaudioprocess = (e) => {
|
|
const inputData = e.inputBuffer.getChannelData(0);
|
|
let sum = 0;
|
|
for(let i=0; i<inputData.length; i+=10) sum += Math.abs(inputData[i]);
|
|
setVolumeLevel(Math.min((sum / (inputData.length/10)) * 500, 100));
|
|
|
|
const pcmData = floatTo16BitPCM(inputData);
|
|
const b64Data = arrayBufferToBase64(pcmData);
|
|
|
|
sessionPromise.then(session => {
|
|
session.sendRealtimeInput({
|
|
media: { mimeType: 'audio/pcm;rate=16000', data: b64Data }
|
|
});
|
|
}).catch(err => console.error("Error sending audio:", err));
|
|
};
|
|
|
|
source.connect(processor);
|
|
processor.connect(gainNode);
|
|
gainNode.connect(inputContextRef.current.destination);
|
|
},
|
|
onmessage: async (msg: LiveServerMessage) => {
|
|
if (msg.serverContent?.inputTranscription) {
|
|
const text = msg.serverContent.inputTranscription.text;
|
|
currentTurnUserText.current += text;
|
|
transcriptRef.current += `[USER]: ${text}\n`;
|
|
}
|
|
if (msg.serverContent?.outputTranscription) {
|
|
const text = msg.serverContent.outputTranscription.text;
|
|
currentTurnPartnerText.current += text;
|
|
transcriptRef.current += `[PARTNER]: ${text}\n`;
|
|
}
|
|
|
|
const audioData = msg.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data;
|
|
if (audioData && outputContextRef.current) {
|
|
const ctx = outputContextRef.current;
|
|
const bytes = base64ToUint8Array(audioData);
|
|
try {
|
|
const audioBuffer = await decodeAudioData(bytes, ctx, 24000, 1);
|
|
const source = ctx.createBufferSource();
|
|
source.buffer = audioBuffer;
|
|
source.connect(ctx.destination);
|
|
const now = ctx.currentTime;
|
|
if (nextStartTimeRef.current < now) nextStartTimeRef.current = now;
|
|
source.start(nextStartTimeRef.current);
|
|
nextStartTimeRef.current += audioBuffer.duration;
|
|
sourcesRef.current.add(source);
|
|
source.addEventListener('ended', () => sourcesRef.current.delete(source));
|
|
} catch (decodingErr) { console.error("Audio decoding error", decodingErr); }
|
|
}
|
|
|
|
if (msg.serverContent?.interrupted) {
|
|
sourcesRef.current.forEach(src => { try { src.stop(); } catch(e){} });
|
|
sourcesRef.current.clear();
|
|
if (outputContextRef.current) nextStartTimeRef.current = outputContextRef.current.currentTime;
|
|
}
|
|
|
|
if (msg.serverContent?.turnComplete) {
|
|
if (analysisChatRef.current && (currentTurnUserText.current || currentTurnPartnerText.current)) {
|
|
setIsAnalyzingTurn(true);
|
|
const turnContext = `User said: "${currentTurnUserText.current}". Roleplay partner said: "${currentTurnPartnerText.current}".`;
|
|
|
|
analysisChatRef.current.sendMessage({ message: turnContext }).then(res => {
|
|
try {
|
|
if (res.text) {
|
|
setLiveAnalysis(JSON.parse(res.text));
|
|
if (!showLiveInsights) setShowLiveInsights(true);
|
|
}
|
|
} catch (e) { console.error("Failed to parse live analysis", e); } finally {
|
|
setIsAnalyzingTurn(false);
|
|
currentTurnUserText.current = "";
|
|
currentTurnPartnerText.current = "";
|
|
}
|
|
});
|
|
}
|
|
}
|
|
},
|
|
onclose: () => { if (status !== 'error' && status !== 'analyzing' && status !== 'finished') setStatus('disconnected'); },
|
|
onerror: (err) => {
|
|
console.error("Session error", err);
|
|
setErrorMessage("Clinical bridge interrupted. Resetting connection.");
|
|
setStatus('error');
|
|
disconnectSession();
|
|
}
|
|
}
|
|
});
|
|
sessionRef.current = sessionPromise;
|
|
} catch (err: any) {
|
|
console.error("Failed to start session", err);
|
|
setErrorMessage(err.message || "Microphone access denied or link failure.");
|
|
setStatus('error');
|
|
}
|
|
};
|
|
|
|
if (status === 'analyzing') {
|
|
return (
|
|
<div className="flex flex-col items-center justify-center min-h-[500px] h-full bg-slate-50 rounded-3xl p-8 border border-slate-200">
|
|
<div className="bg-white p-12 rounded-full shadow-lg border border-teal-100 mb-8 relative">
|
|
<BrainCircuit size={64} className="text-teal-600 animate-pulse" />
|
|
<div className="absolute inset-0 border-4 border-teal-500 border-t-transparent rounded-full animate-spin"></div>
|
|
</div>
|
|
<h2 className="text-2xl font-bold text-slate-800 mb-2">Synthesizing Clinical Insights</h2>
|
|
<p className="text-slate-500 text-center max-w-sm">Please wait while the engine analyzes your vocal patterns and cognitive restructuring efficiency.</p>
|
|
</div>
|
|
);
|
|
}
|
|
|
|
if (status === 'finished' && finalAnalysis) {
|
|
return <SessionReport analysis={finalAnalysis} onExit={onExit} />;
|
|
}
|
|
|
|
return (
|
|
<div className={`flex flex-col lg:flex-row h-full max-h-[85vh] gap-4 transition-all duration-500`}>
|
|
{/* Main Voice Hub */}
|
|
<div className={`flex flex-col items-center justify-center bg-slate-900 rounded-3xl relative overflow-hidden text-white p-8 shadow-2xl border border-slate-800 transition-all duration-500 ${showLiveInsights ? 'lg:w-[65%]' : 'w-full'}`}>
|
|
|
|
{status === 'connected' && (
|
|
<div className="absolute inset-0 flex items-center justify-center pointer-events-none opacity-20">
|
|
<div className="w-80 h-80 bg-teal-500 rounded-full blur-[100px] transition-transform duration-150 ease-out" style={{ transform: `scale(${1 + volumeLevel/30})` }}></div>
|
|
</div>
|
|
)}
|
|
|
|
{/* Header */}
|
|
<div className="absolute top-8 left-8 right-8 flex justify-between items-start z-10">
|
|
<div className="flex items-center gap-4">
|
|
<div className="bg-teal-600/20 p-2.5 rounded-xl border border-teal-500/30">
|
|
<Headphones size={24} className="text-teal-400" />
|
|
</div>
|
|
<div>
|
|
<h2 className="text-xl font-bold text-white tracking-tight">{scenario.title}</h2>
|
|
<div className="flex items-center gap-2 mt-1.5">
|
|
<div className={`w-1.5 h-1.5 rounded-full ${status === 'connected' ? 'bg-teal-400 animate-pulse' : status === 'connecting' ? 'bg-amber-400 animate-bounce' : 'bg-rose-400'}`} />
|
|
<span className="text-[10px] font-bold text-slate-400 uppercase tracking-widest">{status}</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div className="flex gap-2">
|
|
{status === 'connected' && (
|
|
<button
|
|
onClick={() => setShowLiveInsights(!showLiveInsights)}
|
|
className={`p-2.5 rounded-xl transition-all backdrop-blur-md border ${showLiveInsights ? 'bg-teal-500 border-teal-400 text-white' : 'bg-white/5 border-white/10 text-white/60 hover:text-white'}`}
|
|
title="Toggle Live Insights"
|
|
>
|
|
<LayoutPanelLeft size={20} />
|
|
</button>
|
|
)}
|
|
<button
|
|
onClick={onExit}
|
|
className="bg-white/5 hover:bg-white/10 p-2.5 rounded-xl transition-all backdrop-blur-md border border-white/10 text-white/60 hover:text-white"
|
|
>
|
|
<X size={20} />
|
|
</button>
|
|
</div>
|
|
</div>
|
|
|
|
<div className="relative z-10 flex flex-col items-center gap-10 w-full">
|
|
<div className={`w-full max-w-lg mx-auto flex flex-col items-center gap-8 transition-all duration-700`}>
|
|
<AudioVisualizer level={volumeLevel} active={status === 'connected'} />
|
|
<div className={`w-32 h-32 rounded-full flex items-center justify-center transition-all duration-500 ${status === 'connected' ? 'bg-teal-500/10 ring-8 ring-teal-500/20' : 'bg-slate-800/50 ring-8 ring-slate-800'}`}>
|
|
<div className={`w-20 h-20 rounded-full flex items-center justify-center ${status === 'connected' ? 'bg-teal-500 shadow-[0_0_50px_rgba(20,184,166,0.3)]' : 'bg-slate-700'}`}>
|
|
<Radio size={32} className={`text-white ${status === 'connected' ? 'animate-pulse' : 'opacity-20'}`} />
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div className="text-center max-w-sm mt-4 min-h-[60px]">
|
|
{status === 'disconnected' && <p className="text-slate-400 text-sm font-light leading-relaxed">System ready for audio link.<br/>Please proceed when in a quiet environment.</p>}
|
|
{status === 'connecting' && <p className="text-teal-400 text-sm font-bold uppercase tracking-widest animate-pulse">Establishing Bi-Directional Bridge...</p>}
|
|
{status === 'connected' && <p className="text-white font-medium text-lg">Active Session. Listening...</p>}
|
|
{status === 'error' && (
|
|
<div className="flex items-center gap-2 text-rose-300 bg-rose-950/40 border border-rose-500/30 px-5 py-3 rounded-xl">
|
|
<AlertCircle size={18} />
|
|
<p className="text-sm font-medium">{errorMessage}</p>
|
|
</div>
|
|
)}
|
|
</div>
|
|
</div>
|
|
|
|
<div className="absolute bottom-12 z-10">
|
|
{status !== 'connected' ? (
|
|
<button
|
|
onClick={startSession}
|
|
disabled={status === 'connecting'}
|
|
className="flex items-center gap-3 bg-teal-600 hover:bg-teal-500 disabled:bg-teal-800 text-white px-10 py-4 rounded-2xl font-bold shadow-xl transition-all transform hover:scale-105 active:scale-95 border border-teal-400/20"
|
|
>
|
|
<Mic size={24} />
|
|
{status === 'connecting' ? 'SYNCING...' : 'INITIALIZE LINK'}
|
|
</button>
|
|
) : (
|
|
<button
|
|
onClick={handleTerminateAndAnalyze}
|
|
className="flex items-center gap-3 bg-slate-100 hover:bg-white text-slate-900 px-10 py-4 rounded-2xl font-bold shadow-xl transition-all active:scale-95"
|
|
>
|
|
<MicOff size={24} />
|
|
TERMINATE SESSION
|
|
</button>
|
|
)}
|
|
</div>
|
|
</div>
|
|
|
|
{showLiveInsights && <LiveInsightPanel analysis={liveAnalysis} isAnalyzing={isAnalyzingTurn} />}
|
|
</div>
|
|
);
|
|
};
|
|
|
|
export default VoiceSession; |