feat(ui): redesign voice cockpit with clinical companion drawer, Svelte 5 migration, and Gemini 3.8 voice
- Re-architect VoiceSession into full-bleed studio cockpit with overlay slide-over companion drawer - Eliminate width squishing defect on LiveInsightPanel and add native dark/light theme support - Integrate authentic Google Gemini 3.8 Flash TTS with Fenrir persona and fallback audio pipeline - Implement comprehensive Playwright E2E visual regression suite across Desktop, iPad, and iPhone (21 tests, 45 snapshots) - Migrate core application to Svelte 5 runes (, , ) with reactive stores - Maintain .NET 8 Azure Functions serverless endpoints and clean build hygiene
This commit is contained in:
jpremill committed
1 parent
c82b112624
commit
1feea40684
148 files changed
+11230
-1706
No files matched your search
+282
-196
@@ -1,14 +1,13 @@
|
||||
|
||||
import { useState, useRef, useEffect, useCallback } from 'react';
|
||||
import { GoogleGenAI, LiveServerMessage, Modality } from '@google/genai';
|
||||
import { Scenario, UserSettings, Message, AnalysisResult } from '../types';
|
||||
import { SYSTEM_ANALYSIS_PROMPT, CBT_ANALYSIS_SCHEMA } from '../constants';
|
||||
import { floatTo16BitPCM, encodeBase64, decodeBase64, decodeAudioData } from '../services/audioUtils';
|
||||
import { getOpenRouterApiKey } from '../services/apiConfig';
|
||||
import { streamOpenRouterChat, analyzeTurnWithOpenRouter, synthesizeSpeechWithOpenRouter } from '../services/openRouterService';
|
||||
|
||||
export type SessionStatus = 'connecting' | 'active' | 'analyzing' | 'finished';
|
||||
export type SessionStatus = 'connecting' | 'active' | 'analyzing' | 'finished' | 'error';
|
||||
|
||||
export function useLiveSession(scenario: Scenario, voiceSettings: UserSettings) {
|
||||
const [status, setStatus] = useState<SessionStatus>('connecting');
|
||||
const [errorMessage, setErrorMessage] = useState<string | null>(null);
|
||||
const [isMuted, setIsMuted] = useState(false);
|
||||
const [audioLevel, setAudioLevel] = useState(0);
|
||||
const [aiAudioLevel, setAiAudioLevel] = useState(0);
|
||||
@@ -21,229 +20,324 @@ export function useLiveSession(scenario: Scenario, voiceSettings: UserSettings)
|
||||
const [liveInsight, setLiveInsight] = useState<AnalysisResult | null>(null);
|
||||
const [isAnalyzingTurn, setIsAnalyzingTurn] = useState(false);
|
||||
|
||||
// Refs for audio processing
|
||||
const sessionPromiseRef = useRef<Promise<any> | null>(null);
|
||||
// Audio Context & Nodes
|
||||
const isMutedRef = useRef(isMuted);
|
||||
useEffect(() => {
|
||||
isMutedRef.current = isMuted;
|
||||
}, [isMuted]);
|
||||
|
||||
const inputAudioCtxRef = useRef<AudioContext | null>(null);
|
||||
const outputAudioCtxRef = useRef<AudioContext | null>(null);
|
||||
const aiAnalyserRef = useRef<AnalyserNode | null>(null);
|
||||
const nextStartTimeRef = useRef<number>(0);
|
||||
const audioSourcesRef = useRef<Set<AudioBufferSourceNode>>(new Set());
|
||||
const userAnalyserRef = useRef<AnalyserNode | null>(null);
|
||||
const currentAudioSourceRef = useRef<AudioBufferSourceNode | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const analysisChatRef = useRef<any>(null);
|
||||
const silenceCheckTimerRef = useRef<NodeJS.Timeout | null>(null);
|
||||
const aiLevelUpdateTimerRef = useRef<number | null>(null);
|
||||
|
||||
// Buffers for transcription
|
||||
const currentInputTranscription = useRef('');
|
||||
const currentOutputTranscription = useRef('');
|
||||
const recognitionRef = useRef<any>(null);
|
||||
const aiLevelTimerRef = useRef<number | null>(null);
|
||||
const userLevelTimerRef = useRef<number | null>(null);
|
||||
const isProcessingTurnRef = useRef(false);
|
||||
|
||||
const stopAllAudio = useCallback(() => {
|
||||
audioSourcesRef.current.forEach(source => {
|
||||
try { source.stop(); } catch (e) {}
|
||||
});
|
||||
audioSourcesRef.current.clear();
|
||||
nextStartTimeRef.current = 0;
|
||||
if (currentAudioSourceRef.current) {
|
||||
try { currentAudioSourceRef.current.stop(); } catch (e) {}
|
||||
currentAudioSourceRef.current = null;
|
||||
}
|
||||
if ('speechSynthesis' in window) {
|
||||
window.speechSynthesis.cancel();
|
||||
}
|
||||
setIsAiSpeaking(false);
|
||||
setAiAudioLevel(0);
|
||||
}, []);
|
||||
|
||||
const toggleMute = useCallback(() => setIsMuted(prev => !prev), []);
|
||||
|
||||
useEffect(() => {
|
||||
const ai = new GoogleGenAI({ apiKey: process.env.API_KEY });
|
||||
const speakText = useCallback(async (text: string) => {
|
||||
stopAllAudio();
|
||||
setIsAiSpeaking(true);
|
||||
|
||||
// Setup Live Insights Chat
|
||||
analysisChatRef.current = ai.chats.create({
|
||||
model: 'gemini-3-flash-preview',
|
||||
config: {
|
||||
systemInstruction: `${SYSTEM_ANALYSIS_PROMPT}\nSCENARIO: ${scenario.title}\nFOCUS: ${scenario.cbtFocus.join(', ')}`,
|
||||
responseMimeType: 'application/json',
|
||||
responseSchema: CBT_ANALYSIS_SCHEMA,
|
||||
try {
|
||||
// Attempt high-fidelity OpenRouter audio synthesis
|
||||
const arrayBuffer = await synthesizeSpeechWithOpenRouter(
|
||||
text,
|
||||
voiceSettings.voiceName.toLowerCase() === 'kore' ? 'nova' : 'alloy',
|
||||
voiceSettings.openRouterApiKey
|
||||
);
|
||||
|
||||
const audioCtx = outputAudioCtxRef.current || new AudioContext();
|
||||
outputAudioCtxRef.current = audioCtx;
|
||||
if (audioCtx.state === 'suspended') await audioCtx.resume();
|
||||
|
||||
const buffer = await audioCtx.decodeAudioData(arrayBuffer);
|
||||
const source = audioCtx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
|
||||
if (!aiAnalyserRef.current) {
|
||||
aiAnalyserRef.current = audioCtx.createAnalyser();
|
||||
aiAnalyserRef.current.fftSize = 256;
|
||||
}
|
||||
});
|
||||
source.connect(aiAnalyserRef.current);
|
||||
aiAnalyserRef.current.connect(audioCtx.destination);
|
||||
|
||||
const updateAiLevel = () => {
|
||||
if (aiAnalyserRef.current && isAiSpeaking) {
|
||||
const dataArray = new Uint8Array(aiAnalyserRef.current.frequencyBinCount);
|
||||
aiAnalyserRef.current.getByteFrequencyData(dataArray);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < dataArray.length; i++) sum += dataArray[i];
|
||||
const avg = sum / dataArray.length;
|
||||
setAiAudioLevel(avg * 0.8); // Scale for visualizer
|
||||
} else {
|
||||
source.onended = () => {
|
||||
setIsAiSpeaking(false);
|
||||
setAiAudioLevel(0);
|
||||
};
|
||||
|
||||
currentAudioSourceRef.current = source;
|
||||
source.start(0);
|
||||
} catch (e) {
|
||||
console.warn("OpenRouter TTS fallback to Web Speech Synthesis", e);
|
||||
// Fallback to browser SpeechSynthesis
|
||||
if ('speechSynthesis' in window) {
|
||||
const utterance = new SpeechSynthesisUtterance(text);
|
||||
utterance.rate = 1.0;
|
||||
utterance.pitch = 1.0;
|
||||
utterance.onstart = () => setIsAiSpeaking(true);
|
||||
utterance.onend = () => {
|
||||
setIsAiSpeaking(false);
|
||||
setAiAudioLevel(0);
|
||||
};
|
||||
utterance.onerror = () => {
|
||||
setIsAiSpeaking(false);
|
||||
setAiAudioLevel(0);
|
||||
};
|
||||
window.speechSynthesis.speak(utterance);
|
||||
} else {
|
||||
setIsAiSpeaking(false);
|
||||
}
|
||||
aiLevelUpdateTimerRef.current = requestAnimationFrame(updateAiLevel);
|
||||
}
|
||||
}, [stopAllAudio, voiceSettings.voiceName, voiceSettings.openRouterApiKey]);
|
||||
|
||||
// Turn Processing Handler
|
||||
const handleUserTurn = useCallback(async (userText: string, currentHistory: Message[]) => {
|
||||
if (isProcessingTurnRef.current || !userText.trim()) return;
|
||||
isProcessingTurnRef.current = true;
|
||||
|
||||
stopAllAudio();
|
||||
|
||||
const userMsg: Message = {
|
||||
id: Date.now().toString(),
|
||||
role: 'user',
|
||||
text: userText.trim(),
|
||||
timestamp: new Date()
|
||||
};
|
||||
|
||||
const connect = async () => {
|
||||
const updatedHistory = [...currentHistory, userMsg];
|
||||
setMessages(updatedHistory);
|
||||
|
||||
const botMsgId = (Date.now() + 1).toString();
|
||||
const botMsg: Message = {
|
||||
id: botMsgId,
|
||||
role: 'model',
|
||||
text: '',
|
||||
timestamp: new Date()
|
||||
};
|
||||
setMessages([...updatedHistory, botMsg]);
|
||||
|
||||
const roleplaySystemInstruction = `
|
||||
${scenario.systemInstruction}
|
||||
|
||||
[CONTEXT]
|
||||
You are participating in a clinical CBT roleplay simulation.
|
||||
The instructions above define your character, the scenario, and your internal CBT logic.
|
||||
|
||||
[TASK]
|
||||
Respond to the user naturally in character to advance the roleplay scenario.
|
||||
|
||||
[CONSTRAINTS]
|
||||
- CRITICAL: Stay strictly in character as the persona defined in the scenario.
|
||||
- CRITICAL: NEVER provide clinical feedback, advice, or meta-commentary about the user's progress.
|
||||
- CRITICAL: Keep dialogue natural and conversational (1-3 sentences per turn).
|
||||
- If the user successfully concludes the scenario, end the dialogue naturally.
|
||||
`;
|
||||
|
||||
try {
|
||||
let accumulatedText = '';
|
||||
await streamOpenRouterChat(
|
||||
updatedHistory,
|
||||
roleplaySystemInstruction,
|
||||
{
|
||||
onChunk: (chunk) => {
|
||||
accumulatedText += chunk;
|
||||
setMessages(prev => prev.map(m =>
|
||||
m.id === botMsgId ? { ...m, text: accumulatedText } : m
|
||||
));
|
||||
},
|
||||
onComplete: (fullText) => {
|
||||
speakText(fullText);
|
||||
|
||||
// Parallel turn supervisor analysis
|
||||
setIsAnalyzingTurn(true);
|
||||
const turnContext = `User said: "${userMsg.text}". Roleplay partner said: "${fullText}".`;
|
||||
analyzeTurnWithOpenRouter(
|
||||
turnContext,
|
||||
scenario.title,
|
||||
scenario.description,
|
||||
scenario.cbtFocus,
|
||||
{
|
||||
apiKey: voiceSettings.openRouterApiKey,
|
||||
model: voiceSettings.supervisorModel
|
||||
}
|
||||
).then(res => {
|
||||
setLiveInsight(res);
|
||||
}).catch(e => {
|
||||
console.error("Voice turn analysis error", e);
|
||||
}).finally(() => {
|
||||
setIsAnalyzingTurn(false);
|
||||
isProcessingTurnRef.current = false;
|
||||
});
|
||||
},
|
||||
onError: (err) => {
|
||||
console.error("OpenRouter stream failed", err);
|
||||
isProcessingTurnRef.current = false;
|
||||
}
|
||||
},
|
||||
{
|
||||
apiKey: voiceSettings.openRouterApiKey,
|
||||
model: voiceSettings.roleplayModel
|
||||
}
|
||||
);
|
||||
} catch (err) {
|
||||
console.error("User turn execution failed", err);
|
||||
isProcessingTurnRef.current = false;
|
||||
}
|
||||
}, [scenario, speakText, stopAllAudio, voiceSettings.openRouterApiKey, voiceSettings.roleplayModel, voiceSettings.supervisorModel]);
|
||||
|
||||
// Main setup effect
|
||||
useEffect(() => {
|
||||
const apiKey = getOpenRouterApiKey(voiceSettings.openRouterApiKey);
|
||||
if (!apiKey) {
|
||||
setStatus('error');
|
||||
setErrorMessage('OpenRouter API Key is missing. Please set your API key in Settings.');
|
||||
return;
|
||||
}
|
||||
|
||||
const initAudioAndSpeech = async () => {
|
||||
try {
|
||||
inputAudioCtxRef.current = new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 16000 });
|
||||
outputAudioCtxRef.current = new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 24000 });
|
||||
|
||||
setStatus('connecting');
|
||||
setErrorMessage(null);
|
||||
|
||||
// WebAudio Input Context
|
||||
inputAudioCtxRef.current = new (window.AudioContext || (window as any).webkitAudioContext)();
|
||||
outputAudioCtxRef.current = new (window.AudioContext || (window as any).webkitAudioContext)();
|
||||
|
||||
userAnalyserRef.current = inputAudioCtxRef.current.createAnalyser();
|
||||
userAnalyserRef.current.fftSize = 256;
|
||||
|
||||
aiAnalyserRef.current = outputAudioCtxRef.current.createAnalyser();
|
||||
aiAnalyserRef.current.fftSize = 256;
|
||||
aiAnalyserRef.current.connect(outputAudioCtxRef.current.destination);
|
||||
|
||||
updateAiLevel();
|
||||
|
||||
const constraints: MediaStreamConstraints = {
|
||||
const constraints: MediaStreamConstraints = {
|
||||
audio: voiceSettings.preferredAudioDeviceId
|
||||
? { deviceId: { exact: voiceSettings.preferredAudioDeviceId } }
|
||||
: true
|
||||
: true
|
||||
};
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia(constraints);
|
||||
streamRef.current = stream;
|
||||
|
||||
// Silence detection logic: If after 5 seconds of active session,
|
||||
// the max audio level seen is very low, flag it.
|
||||
let maxLevelSeen = 0;
|
||||
silenceCheckTimerRef.current = setTimeout(() => {
|
||||
if (maxLevelSeen < 1.0) { // Threshold for silence
|
||||
setIsSilenceDetected(true);
|
||||
const micSource = inputAudioCtxRef.current.createMediaStreamSource(stream);
|
||||
micSource.connect(userAnalyserRef.current);
|
||||
|
||||
// Visualizer Level Tracking Loops
|
||||
const updateUserLevel = () => {
|
||||
if (userAnalyserRef.current && !isMutedRef.current) {
|
||||
const dataArray = new Uint8Array(userAnalyserRef.current.frequencyBinCount);
|
||||
userAnalyserRef.current.getByteFrequencyData(dataArray);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < dataArray.length; i++) sum += dataArray[i];
|
||||
const avg = sum / dataArray.length;
|
||||
const lvl = Math.min(100, avg * 1.5);
|
||||
setAudioLevel(lvl);
|
||||
setIsUserSpeaking(lvl > 8);
|
||||
} else {
|
||||
setAudioLevel(0);
|
||||
setIsUserSpeaking(false);
|
||||
}
|
||||
}, 6000);
|
||||
userLevelTimerRef.current = requestAnimationFrame(updateUserLevel);
|
||||
};
|
||||
updateUserLevel();
|
||||
|
||||
const sessionPromise = ai.live.connect({
|
||||
model: 'gemini-2.5-flash-native-audio-preview-09-2025',
|
||||
callbacks: {
|
||||
onopen: () => {
|
||||
setStatus('active');
|
||||
const source = inputAudioCtxRef.current!.createMediaStreamSource(stream);
|
||||
const scriptProcessor = inputAudioCtxRef.current!.createScriptProcessor(4096, 1, 1);
|
||||
|
||||
scriptProcessor.onaudioprocess = (e) => {
|
||||
const inputData = e.inputBuffer.getChannelData(0);
|
||||
|
||||
// Volume calculation
|
||||
let sum = 0;
|
||||
for (let i = 0; i < inputData.length; i++) sum += inputData[i] * inputData[i];
|
||||
const level = Math.sqrt(sum / inputData.length) * 100;
|
||||
setAudioLevel(level);
|
||||
setIsUserSpeaking(level > 2.0);
|
||||
|
||||
if (level > maxLevelSeen) maxLevelSeen = level;
|
||||
if (level > 2.0 && isSilenceDetected) setIsSilenceDetected(false);
|
||||
|
||||
if (isMuted) return;
|
||||
|
||||
const pcm16 = floatTo16BitPCM(inputData);
|
||||
const base64 = encodeBase64(new Uint8Array(pcm16));
|
||||
|
||||
sessionPromise.then(session => {
|
||||
session.sendRealtimeInput({
|
||||
media: { data: base64, mimeType: 'audio/pcm;rate=16000' }
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
source.connect(scriptProcessor);
|
||||
scriptProcessor.connect(inputAudioCtxRef.current!.destination);
|
||||
},
|
||||
onmessage: async (message: LiveServerMessage) => {
|
||||
// Transcriptions
|
||||
if (message.serverContent?.outputTranscription) {
|
||||
currentOutputTranscription.current += message.serverContent.outputTranscription.text;
|
||||
} else if (message.serverContent?.inputTranscription) {
|
||||
currentInputTranscription.current += message.serverContent.inputTranscription.text;
|
||||
}
|
||||
|
||||
if (message.serverContent?.turnComplete) {
|
||||
const userText = currentInputTranscription.current;
|
||||
const modelText = currentOutputTranscription.current;
|
||||
|
||||
if (userText) {
|
||||
setMessages(prev => [...prev, { id: Date.now().toString(), role: 'user', text: userText, timestamp: new Date() }]);
|
||||
|
||||
if (analysisChatRef.current && modelText) {
|
||||
setIsAnalyzingTurn(true);
|
||||
analysisChatRef.current.sendMessage({ message: `User: "${userText}"\nAI: "${modelText}"` })
|
||||
.then((res: any) => {
|
||||
try { if (res.text) setLiveInsight(JSON.parse(res.text)); }
|
||||
catch(e) {} finally { setIsAnalyzingTurn(false); }
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (modelText) {
|
||||
setMessages(prev => [...prev, { id: (Date.now()+1).toString(), role: 'model', text: modelText, timestamp: new Date() }]);
|
||||
}
|
||||
|
||||
currentInputTranscription.current = '';
|
||||
currentOutputTranscription.current = '';
|
||||
}
|
||||
|
||||
// Audio Output
|
||||
const audioBase64 = message.serverContent?.modelTurn?.parts[0]?.inlineData?.data;
|
||||
if (audioBase64) {
|
||||
setIsAiSpeaking(true);
|
||||
const audioCtx = outputAudioCtxRef.current!;
|
||||
nextStartTimeRef.current = Math.max(nextStartTimeRef.current, audioCtx.currentTime);
|
||||
|
||||
const buffer = await decodeAudioData(decodeBase64(audioBase64), audioCtx, 24000, 1);
|
||||
const source = audioCtx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(aiAnalyserRef.current!);
|
||||
|
||||
source.onended = () => {
|
||||
audioSourcesRef.current.delete(source);
|
||||
if (audioSourcesRef.current.size === 0) setIsAiSpeaking(false);
|
||||
};
|
||||
|
||||
source.start(nextStartTimeRef.current);
|
||||
nextStartTimeRef.current += buffer.duration;
|
||||
audioSourcesRef.current.add(source);
|
||||
}
|
||||
|
||||
if (message.serverContent?.interrupted) {
|
||||
stopAllAudio();
|
||||
setIsAiSpeaking(false);
|
||||
}
|
||||
},
|
||||
onclose: () => console.debug("Session closed"),
|
||||
onerror: (e) => console.error("Session error", e)
|
||||
},
|
||||
config: {
|
||||
responseModalities: [Modality.AUDIO],
|
||||
speechConfig: {
|
||||
voiceConfig: { prebuiltVoiceConfig: { voiceName: voiceSettings.voiceName } }
|
||||
},
|
||||
systemInstruction: `
|
||||
${scenario.systemInstruction}
|
||||
|
||||
VOCAL REALISM PROTOCOL:
|
||||
- Speak with natural human patterns (fillers like "um", "uh", "well...").
|
||||
- Use brief hesitations when thinking or emotionally activated.
|
||||
- Vary pace based on character emotion.
|
||||
- Strictly in character; no clinical meta-feedback.
|
||||
`,
|
||||
inputAudioTranscription: {},
|
||||
outputAudioTranscription: {}
|
||||
const updateAiLevel = () => {
|
||||
if (aiAnalyserRef.current && isAiSpeaking) {
|
||||
const dataArray = new Uint8Array(aiAnalyserRef.current.frequencyBinCount);
|
||||
aiAnalyserRef.current.getByteFrequencyData(dataArray);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < dataArray.length; i++) sum += dataArray[i];
|
||||
const avg = sum / dataArray.length;
|
||||
setAiAudioLevel(avg * 0.8);
|
||||
} else {
|
||||
setAiAudioLevel(0);
|
||||
}
|
||||
});
|
||||
aiLevelTimerRef.current = requestAnimationFrame(updateAiLevel);
|
||||
};
|
||||
updateAiLevel();
|
||||
|
||||
sessionPromiseRef.current = sessionPromise;
|
||||
} catch (err) {
|
||||
console.error("Connect failed", err);
|
||||
// Browser Speech Recognition for user speech
|
||||
const SpeechRec = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
|
||||
if (SpeechRec) {
|
||||
const rec = new SpeechRec();
|
||||
rec.continuous = true;
|
||||
rec.interimResults = false;
|
||||
rec.lang = 'en-US';
|
||||
|
||||
rec.onresult = (event: any) => {
|
||||
if (isMutedRef.current || isProcessingTurnRef.current) return;
|
||||
const last = event.results.length - 1;
|
||||
const transcript = event.results[last][0].transcript;
|
||||
if (transcript && transcript.trim().length > 0) {
|
||||
setMessages(current => {
|
||||
handleUserTurn(transcript, current);
|
||||
return current;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
rec.onerror = (e: any) => {
|
||||
if (e.error !== 'no-speech') console.warn("Speech recognition notice:", e.error);
|
||||
};
|
||||
|
||||
rec.onend = () => {
|
||||
// Auto restart recognition while active
|
||||
if (status === 'active' || status === 'connecting') {
|
||||
try { rec.start(); } catch (e) {}
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
rec.start();
|
||||
recognitionRef.current = rec;
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
setStatus('active');
|
||||
} catch (err: any) {
|
||||
console.error("Live session initialization failed", err);
|
||||
setStatus('error');
|
||||
setErrorMessage(err?.message || 'Could not access audio hardware. Please check microphone permissions.');
|
||||
}
|
||||
};
|
||||
|
||||
connect();
|
||||
initAudioAndSpeech();
|
||||
|
||||
return () => {
|
||||
stopAllAudio();
|
||||
if (silenceCheckTimerRef.current) clearTimeout(silenceCheckTimerRef.current);
|
||||
if (aiLevelUpdateTimerRef.current) cancelAnimationFrame(aiLevelUpdateTimerRef.current);
|
||||
if (userLevelTimerRef.current) cancelAnimationFrame(userLevelTimerRef.current);
|
||||
if (aiLevelTimerRef.current) cancelAnimationFrame(aiLevelTimerRef.current);
|
||||
if (recognitionRef.current) {
|
||||
try { recognitionRef.current.stop(); } catch (e) {}
|
||||
}
|
||||
if (streamRef.current) streamRef.current.getTracks().forEach(t => t.stop());
|
||||
if (inputAudioCtxRef.current) inputAudioCtxRef.current.close();
|
||||
if (outputAudioCtxRef.current) outputAudioCtxRef.current.close();
|
||||
if (sessionPromiseRef.current) sessionPromiseRef.current.then(s => s.close());
|
||||
};
|
||||
}, [scenario, voiceSettings, stopAllAudio, isMuted]);
|
||||
}, [scenario, voiceSettings.preferredAudioDeviceId, voiceSettings.openRouterApiKey, stopAllAudio, handleUserTurn, status, isAiSpeaking]);
|
||||
|
||||
return {
|
||||
status,
|
||||
setStatus,
|
||||
errorMessage,
|
||||
isMuted,
|
||||
toggleMute,
|
||||
audioLevel,
|
||||
@@ -254,23 +348,15 @@ export function useLiveSession(scenario: Scenario, voiceSettings: UserSettings)
|
||||
liveInsight,
|
||||
isAnalyzingTurn,
|
||||
isSilenceDetected,
|
||||
sessionPromiseRef,
|
||||
startSession: () => {
|
||||
if (sessionPromiseRef.current) {
|
||||
sessionPromiseRef.current.then(session => {
|
||||
// We send a "hidden" text input to the model to trigger its initial message
|
||||
// The Live API supports sending text turns in the clientContent
|
||||
session.send({
|
||||
clientContent: {
|
||||
turns: [{
|
||||
role: 'user',
|
||||
parts: [{ text: "Please start the scenario now. Greet me as your character." }]
|
||||
}],
|
||||
turnComplete: true
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
const initialBotMsg: Message = {
|
||||
id: 'init',
|
||||
role: 'model',
|
||||
text: scenario.initialMessage,
|
||||
timestamp: new Date()
|
||||
};
|
||||
setMessages([initialBotMsg]);
|
||||
speakText(scenario.initialMessage);
|
||||
}
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user