feat(ui): redesign voice cockpit with clinical companion drawer, Svelte 5 migration, and Gemini 3.8 voice
- Re-architect VoiceSession into full-bleed studio cockpit with overlay slide-over companion drawer - Eliminate width squishing defect on LiveInsightPanel and add native dark/light theme support - Integrate authentic Google Gemini 3.8 Flash TTS with Fenrir persona and fallback audio pipeline - Implement comprehensive Playwright E2E visual regression suite across Desktop, iPad, and iPhone (21 tests, 45 snapshots) - Migrate core application to Svelte 5 runes (, , ) with reactive stores - Maintain .NET 8 Azure Functions serverless endpoints and clean build hygiene
This commit is contained in:
jpremill committed
1 parent
c82b112624
commit
1feea40684
148 files changed
+11230
-1706
No files matched your search
+126
-160
@@ -1,11 +1,10 @@
|
||||
|
||||
import React, { useState, useCallback } from 'react';
|
||||
import { Scenario, UserSettings, AnalysisResult } from '../types';
|
||||
import { GoogleGenAI } from '@google/genai';
|
||||
import { SYSTEM_SUMMARY_ANALYSIS_PROMPT, CBT_ANALYSIS_SCHEMA } from '../constants';
|
||||
import { useLiveSession } from '../hooks/useLiveSession';
|
||||
import { AlertCircle, Settings2, X, BrainCircuit, Zap, Info } from 'lucide-react';
|
||||
import { AlertCircle, Settings2, X, AlertTriangle } from 'lucide-react';
|
||||
import { Button } from './ui/button';
|
||||
import { generateSessionSummaryReport } from '../services/openRouterService';
|
||||
import { saveSession } from '../services/sessionStorage';
|
||||
|
||||
// Sub-components
|
||||
import AudioVisualizer from './AudioVisualizer';
|
||||
@@ -34,6 +33,7 @@ const VoiceSession: React.FC<VoiceSessionProps> = ({ scenario, onExit, voiceSett
|
||||
const {
|
||||
status,
|
||||
setStatus,
|
||||
errorMessage,
|
||||
isMuted,
|
||||
toggleMute,
|
||||
audioLevel,
|
||||
@@ -44,7 +44,6 @@ const VoiceSession: React.FC<VoiceSessionProps> = ({ scenario, onExit, voiceSett
|
||||
liveInsight,
|
||||
isAnalyzingTurn,
|
||||
isSilenceDetected,
|
||||
sessionPromiseRef,
|
||||
startSession
|
||||
} = useLiveSession(scenario, voiceSettings);
|
||||
|
||||
@@ -58,188 +57,155 @@ const VoiceSession: React.FC<VoiceSessionProps> = ({ scenario, onExit, voiceSett
|
||||
const handleFinishSession = useCallback(async () => {
|
||||
setStatus('analyzing');
|
||||
|
||||
if (sessionPromiseRef.current) {
|
||||
sessionPromiseRef.current.then(session => session.close());
|
||||
}
|
||||
|
||||
try {
|
||||
const ai = new GoogleGenAI({ apiKey: process.env.API_KEY });
|
||||
const transcriptText = messages.map(m => `[${m.role.toUpperCase()}]: ${m.text}`).join('\n');
|
||||
const reportData = await generateSessionSummaryReport(
|
||||
messages,
|
||||
scenario.title,
|
||||
scenario.description,
|
||||
{
|
||||
apiKey: voiceSettings.openRouterApiKey,
|
||||
model: voiceSettings.supervisorModel
|
||||
}
|
||||
);
|
||||
|
||||
const summaryPrompt = `${SYSTEM_SUMMARY_ANALYSIS_PROMPT}
|
||||
SCENARIO CONTEXT: ${scenario.title} - ${scenario.description}`;
|
||||
|
||||
const result = await ai.models.generateContent({
|
||||
model: 'gemini-3-flash-preview',
|
||||
config: {
|
||||
systemInstruction: summaryPrompt,
|
||||
responseMimeType: 'application/json',
|
||||
responseSchema: CBT_ANALYSIS_SCHEMA,
|
||||
},
|
||||
contents: [{ role: 'user', parts: [{ text: `SESSION TRANSCRIPT:\n${transcriptText}` }] }]
|
||||
});
|
||||
|
||||
if (result.text) {
|
||||
setFinalReport(JSON.parse(result.text));
|
||||
setStatus('finished');
|
||||
} else {
|
||||
throw new Error("Analysis failed");
|
||||
}
|
||||
setFinalReport(reportData);
|
||||
saveSession(scenario, 'voice', messages, reportData);
|
||||
setStatus('finished');
|
||||
} catch (err) {
|
||||
console.error("Post-session analysis failure", err);
|
||||
onExit();
|
||||
}
|
||||
}, [messages, scenario, setStatus, sessionPromiseRef, onExit]);
|
||||
}, [messages, scenario, voiceSettings.openRouterApiKey, voiceSettings.supervisorModel, setStatus, onExit]);
|
||||
|
||||
if (status === 'connecting') return <ConnectionState />;
|
||||
if (status === 'analyzing') return <AnalyzingState />;
|
||||
if (status === 'finished' && finalReport) {
|
||||
return <SessionReport analysis={finalReport} onExit={onExit} />;
|
||||
return (
|
||||
<SessionReport
|
||||
analysis={finalReport}
|
||||
scenarioTitle={scenario.title}
|
||||
messages={messages}
|
||||
onExit={onExit}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex flex-col lg:flex-row h-full gap-4 relative overflow-hidden">
|
||||
{/* Main Interaction Area */}
|
||||
<div className="flex-grow bg-slate-900 rounded-3xl overflow-hidden flex flex-col relative shadow-2xl border border-white/5">
|
||||
{/* Animated Background Aura */}
|
||||
<div className="absolute inset-0 overflow-hidden pointer-events-none opacity-20">
|
||||
<div className={`absolute top-1/2 left-1/2 -translate-x-1/2 -translate-y-1/2 w-[80%] h-[80%] rounded-full blur-[100px] transition-all duration-1000 ${isAiSpeaking ? 'bg-blue-500 scale-110' : 'bg-teal-500 scale-100'}`}></div>
|
||||
</div>
|
||||
<div className="flex flex-col h-full bg-slate-900 rounded-3xl border border-white/10 overflow-hidden relative shadow-2xl">
|
||||
{/* Voice Session Header */}
|
||||
<VoiceHeader
|
||||
scenario={scenario}
|
||||
onExit={onExit}
|
||||
onOpenDeviceSelector={() => setShowDeviceSelector(true)}
|
||||
/>
|
||||
|
||||
<VoiceHeader
|
||||
title={scenario.title}
|
||||
onExit={onExit}
|
||||
onFinish={handleFinishSession}
|
||||
/>
|
||||
|
||||
<div className="flex-grow flex flex-col items-center justify-center p-12 relative z-10">
|
||||
{!hasStarted && status === 'active' ? (
|
||||
<div className="flex flex-col items-center animate-in fade-in zoom-in duration-700">
|
||||
<div className="w-24 h-24 bg-teal-500/10 rounded-full flex items-center justify-center mb-8 border border-teal-500/20 relative">
|
||||
<div className="absolute inset-0 bg-teal-500/20 rounded-full animate-ping"></div>
|
||||
<BrainCircuit size={40} className="text-teal-400 relative z-10" />
|
||||
{/* Main Interaction Stage */}
|
||||
<div className="flex-grow flex flex-col lg:flex-row items-center justify-between p-6 md:p-12 relative overflow-hidden gap-8">
|
||||
<div className="flex-grow flex flex-col items-center justify-center w-full relative z-10">
|
||||
{/* Error Banner */}
|
||||
{status === 'error' && (
|
||||
<div className="mb-6 p-4 rounded-2xl bg-rose-950/80 border border-rose-500/50 text-rose-200 flex items-center gap-3 max-w-md animate-in fade-in">
|
||||
<AlertTriangle className="text-rose-400 shrink-0" size={20} />
|
||||
<div className="text-xs">
|
||||
<p className="font-bold">Connection Warning</p>
|
||||
<p className="opacity-90">{errorMessage || 'Unable to connect to live voice session. Please verify your OpenRouter API key.'}</p>
|
||||
</div>
|
||||
<h3 className="text-2xl font-bold text-white mb-2">Neural Link Established</h3>
|
||||
<p className="text-slate-400 text-sm mb-8 max-w-xs text-center leading-relaxed">
|
||||
The simulation is ready. Press start to begin your interaction with <strong>{scenario.characterName}</strong>.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="bg-white/5 border border-white/10 rounded-2xl p-4 mb-8 flex items-center gap-3 max-w-sm">
|
||||
<div className="bg-teal-500/20 p-2 rounded-lg">
|
||||
<Info size={16} className="text-teal-400" />
|
||||
</div>
|
||||
<p className="text-[11px] text-slate-300 leading-relaxed">
|
||||
<strong>Greeting Hint:</strong> Say "Hello {scenario.characterName}" or just wait for them to speak first.
|
||||
</p>
|
||||
</div>
|
||||
{/* Silence Warning Notice */}
|
||||
{isSilenceDetected && !isMuted && status === 'active' && (
|
||||
<div className="absolute top-0 flex items-center gap-2 bg-amber-500/10 border border-amber-500/20 text-amber-300 px-4 py-2 rounded-full text-xs animate-bounce shadow-lg">
|
||||
<AlertCircle size={14} />
|
||||
<span>Low microphone input detected. Please speak closer to your mic.</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Audio Visualizer Orb */}
|
||||
<div className="relative mb-8 flex items-center justify-center">
|
||||
<AudioVisualizer
|
||||
level={activeLevel}
|
||||
isAiSpeaking={isAiSpeaking}
|
||||
isUserSpeaking={isUserSpeaking}
|
||||
isMuted={isMuted}
|
||||
/>
|
||||
|
||||
{!hasStarted && status === 'active' && (
|
||||
<Button
|
||||
onClick={handleStart}
|
||||
className="bg-teal-600 hover:bg-teal-500 text-white px-12 py-7 rounded-2xl font-bold transition-all shadow-xl shadow-teal-900/20 uppercase tracking-widest text-sm"
|
||||
className="absolute z-20 bg-teal-500 hover:bg-teal-400 text-slate-950 font-black px-8 py-6 rounded-full text-sm uppercase tracking-widest shadow-xl shadow-teal-500/20 hover:scale-105 transition-all"
|
||||
>
|
||||
<Zap size={18} fill="currentColor" className="mr-3" />
|
||||
Start Session
|
||||
Begin Voice Simulation
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Status Indicators */}
|
||||
<div className="flex flex-col items-center gap-2">
|
||||
<div className="flex items-center gap-2 px-4 py-1.5 rounded-full bg-white/5 border border-white/10 backdrop-blur-md">
|
||||
<span className={`w-2 h-2 rounded-full ${isAiSpeaking ? 'bg-blue-400 animate-ping' : isUserSpeaking ? 'bg-teal-400 animate-pulse' : 'bg-slate-500'}`} />
|
||||
<span className="text-[11px] font-bold uppercase tracking-widest text-slate-300">
|
||||
{isAiSpeaking ? `${scenario.characterName} is speaking` : isUserSpeaking ? 'Listening to you...' : 'Ready for dialogue'}
|
||||
</span>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
{isSilenceDetected && !isMuted && (
|
||||
<div className="absolute top-8 left-1/2 -translate-x-1/2 w-full max-w-md animate-in slide-in-from-top-4 duration-500">
|
||||
<div className="bg-amber-500/90 backdrop-blur-md border border-amber-400 text-white p-4 rounded-2xl shadow-xl flex items-start gap-4">
|
||||
<div className="bg-white/20 p-2 rounded-xl">
|
||||
<AlertCircle size={20} />
|
||||
</div>
|
||||
<div className="flex-grow">
|
||||
<h4 className="text-sm font-bold">No Audio Detected</h4>
|
||||
<p className="text-xs text-white/80 mt-1 leading-relaxed">We can't hear you. Please check your microphone or select a different input device.</p>
|
||||
<Button
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={() => setShowDeviceSelector(true)}
|
||||
className="mt-3 bg-white text-amber-600 px-4 py-1.5 rounded-lg text-[10px] font-black uppercase tracking-widest hover:bg-amber-50"
|
||||
>
|
||||
Change Input Device
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Speaker Indicator Badge */}
|
||||
<div className="mb-8 flex flex-col items-center">
|
||||
<div className={`px-4 py-1.5 rounded-full border transition-all duration-500 flex items-center gap-2 ${isAiSpeaking
|
||||
? 'bg-blue-500/20 border-blue-400/30 text-blue-300'
|
||||
: (isUserSpeaking ? 'bg-teal-500/20 border-teal-400/30 text-teal-300' : 'bg-white/5 border-white/10 text-white/40')
|
||||
}`}>
|
||||
<div className={`w-2 h-2 rounded-full ${isAiSpeaking ? 'bg-blue-400 animate-pulse' : (isUserSpeaking ? 'bg-teal-400 animate-pulse' : 'bg-white/20')
|
||||
}`} />
|
||||
<span className="text-[10px] font-black uppercase tracking-widest">
|
||||
{isAiSpeaking ? 'AI Speaking' : (isUserSpeaking ? 'You Speaking' : 'Waiting...')}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<AudioVisualizer level={activeLevel} active={true} isAiSpeaking={isAiSpeaking} />
|
||||
|
||||
<div className="mt-12 text-center">
|
||||
<p className={`text-sm transition-all duration-500 font-light ${isAiSpeaking ? 'text-blue-300' : 'text-teal-300'}`}>
|
||||
{isAiSpeaking ? `Vocalizing Response...` : (isMuted ? 'Microphone Muted' : 'Listening...')}
|
||||
</p>
|
||||
<p className="text-[10px] text-white/30 uppercase font-bold tracking-[0.3em] mt-4">Clinical Fidelity Level: 99.8%</p>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
<p className="text-xs text-slate-400 font-light mt-1">
|
||||
{voiceSettings.voiceName} Vocal Persona Active
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Device Selector Overlay */}
|
||||
{showDeviceSelector && (
|
||||
<div className="absolute inset-0 z-50 bg-slate-900/80 backdrop-blur-sm flex items-center justify-center p-6 animate-in fade-in duration-300">
|
||||
<div className="bg-white rounded-3xl p-6 w-full max-w-sm shadow-2xl border border-slate-100">
|
||||
<div className="flex justify-between items-center mb-6">
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="bg-slate-800 p-2 rounded-xl text-white">
|
||||
<Settings2 size={18} />
|
||||
</div>
|
||||
<h3 className="font-bold text-slate-800">Audio Input</h3>
|
||||
</div>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={() => setShowDeviceSelector(false)}
|
||||
className="text-slate-400 hover:bg-slate-100 rounded-xl"
|
||||
>
|
||||
<X size={20} />
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<AudioDeviceSelector
|
||||
selectedDeviceId={voiceSettings.preferredAudioDeviceId}
|
||||
onSelect={(deviceId) => {
|
||||
onUpdateSettings({ ...voiceSettings, preferredAudioDeviceId: deviceId });
|
||||
// Don't close immediately so user sees the checkmark
|
||||
setTimeout(() => setShowDeviceSelector(false), 300);
|
||||
}}
|
||||
/>
|
||||
|
||||
<p className="text-[10px] text-slate-400 mt-6 leading-relaxed italic">
|
||||
Changing your input device will restart the current neural link.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<VoiceControls
|
||||
isMuted={isMuted}
|
||||
onToggleMute={toggleMute}
|
||||
showTranscript={showTranscript}
|
||||
onToggleTranscript={() => setShowTranscript(prev => !prev)}
|
||||
/>
|
||||
|
||||
{showTranscript && <TranscriptPanel messages={messages} onClose={() => setShowTranscript(false)} />}
|
||||
{/* Live Insight Drawer */}
|
||||
<div className="w-full lg:w-96 flex-shrink-0 h-full max-h-[500px] lg:max-h-none flex flex-col justify-end">
|
||||
<LiveInsightPanel analysis={liveInsight} isAnalyzing={isAnalyzingTurn} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Real-time Insights Module */}
|
||||
<LiveInsightPanel analysis={liveInsight} isAnalyzing={isAnalyzingTurn} />
|
||||
{/* Persistent Audio Controls Footer */}
|
||||
<VoiceControls
|
||||
isMuted={isMuted}
|
||||
onToggleMute={toggleMute}
|
||||
onFinish={handleFinishSession}
|
||||
onToggleTranscript={() => setShowTranscript(prev => !prev)}
|
||||
showTranscript={showTranscript}
|
||||
messageCount={messages.length}
|
||||
/>
|
||||
|
||||
{/* Transcript Drawer Slideout */}
|
||||
{showTranscript && (
|
||||
<TranscriptPanel
|
||||
messages={messages}
|
||||
onClose={() => setShowTranscript(false)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Audio Hardware Selector Modal */}
|
||||
{showDeviceSelector && (
|
||||
<div className="fixed inset-0 z-[110] flex items-center justify-center p-4 bg-slate-950/80 backdrop-blur-md">
|
||||
<div className="bg-slate-900 border border-white/10 rounded-3xl p-6 max-w-md w-full shadow-2xl">
|
||||
<div className="flex justify-between items-center mb-6">
|
||||
<h3 className="text-sm font-bold uppercase tracking-widest text-white flex items-center gap-2">
|
||||
<Settings2 size={16} className="text-teal-400" />
|
||||
Select Audio Input Device
|
||||
</h3>
|
||||
<button
|
||||
onClick={() => setShowDeviceSelector(false)}
|
||||
className="text-slate-400 hover:text-white p-1"
|
||||
>
|
||||
<X size={18} />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<AudioDeviceSelector
|
||||
selectedDeviceId={voiceSettings.preferredAudioDeviceId}
|
||||
onSelect={(deviceId) => {
|
||||
onUpdateSettings({ ...voiceSettings, preferredAudioDeviceId: deviceId });
|
||||
setShowDeviceSelector(false);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user