diff --git a/public/tutor_f.png b/public/tutor_f.png new file mode 100644 index 0000000..851aad8 Binary files /dev/null and b/public/tutor_f.png differ diff --git a/public/tutor_m.png b/public/tutor_m.png new file mode 100644 index 0000000..46a70d8 Binary files /dev/null and b/public/tutor_m.png differ diff --git a/src/App.css b/src/App.css index b9d355d..765dfe4 100644 --- a/src/App.css +++ b/src/App.css @@ -1,42 +1,137 @@ -#root { - max-width: 1280px; - margin: 0 auto; - padding: 2rem; +* { + box-sizing: border-box; +} + +:root { + +} + +body { + margin: 0; + padding: 0; + background: linear-gradient(to bottom, #bae6fd, #ffffff); + color: #1f2937; + height: 100vh; + overflow-x: hidden; +} + +.app-container { + min-height: 100vh; + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + padding: 1rem; +} + +.scene-wrapper { + position: relative; + max-width: 640px; + width: 100%; + padding: 1rem; +} + +.avatar { + position: absolute; + top: -80px; + left: 50%; + transform: translateX(-50%); + width: 128px; + height: 128px; + border-radius: 50%; + border: 4px solid white; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2); + object-fit: cover; + animation: popIn 0.4s ease; +} + +.dialogue-box { + background: rgba(255, 255, 255, 0.9); + border: 1px solid #d1d5db; + border-radius: 1.5rem; + padding: 1.5rem; + margin-top: 6rem; + box-shadow: 0 10px 20px rgba(0, 0, 0, 0.1); + animation: fadeIn 0.5s ease-in-out; +} + +.dialogue-text { + font-size: 1.125rem; + color: #374151; + margin-bottom: 1rem; +} + +.dialogue-text strong { + color: #111827; +} + +.control-button { + margin-top: 1.5rem; + padding: 0.75rem 1.5rem; + font-size: 1.125rem; + font-weight: 700; + color: white; + border: none; + border-radius: 9999px; + box-shadow: 0 6px 14px rgba(0, 0, 0, 0.1); + transition: background 0.3s ease; +} + +.control-button.recording { + background-color: #ef4444; +} + +.control-button.idle { + background-color: #3b82f6; +} + +.control-button.idle:hover { + background-color: #2563eb; +} + +.recording-indicator { + color: red; + font-weight: bold; + margin-top: 1rem; text-align: center; + animation: pulse 1s infinite; } -.logo { - height: 6em; - padding: 1.5em; - will-change: filter; - transition: filter 300ms; -} -.logo:hover { - filter: drop-shadow(0 0 2em #646cffaa); -} -.logo.react:hover { - filter: drop-shadow(0 0 2em #61dafbaa); +.waveform-canvas { + width: 300px; /* same as element width */ + height: 60px; /* same as element height */ + background-color: #000; + border-radius: 8px; + margin: 10px auto; + display: block; } -@keyframes logo-spin { + + +@keyframes pulse { + 0% { opacity: 1; } + 50% { opacity: 0.3; } + 100% { opacity: 1; } +} + +@keyframes fadeIn { from { - transform: rotate(0deg); + opacity: 0; + transform: translateY(10px); } to { - transform: rotate(360deg); + opacity: 1; + transform: translateY(0); } } -@media (prefers-reduced-motion: no-preference) { - a:nth-of-type(2) .logo { - animation: logo-spin infinite 20s linear; +@keyframes popIn { + 0% { + transform: scale(0.8); + opacity: 0; + } + 100% { + transform: scale(1); + opacity: 1; } } - -.card { - padding: 2em; -} - -.read-the-docs { - color: #888; -} diff --git a/src/App.jsx b/src/App.jsx index 4d2ec8f..521eeda 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -1,4 +1,5 @@ -import React, { useState, useRef } from 'react'; +// ai-tutor-poc/src/App.jsx +import React, { useState, useRef, useEffect } from 'react'; import './App.css'; const OPENAI_API_KEY = import.meta.env.VITE_OPENAI_API_KEY; @@ -9,111 +10,225 @@ function App() { const [isRecording, setIsRecording] = useState(false); const mediaRecorderRef = useRef(null); const audioChunksRef = useRef([]); + const audioStreamRef = useRef(null); + const silenceTimerRef = useRef(null); + const canvasRef = useRef(null); + const analyserRef = useRef(null); + const dataArrayRef = useRef(null); + const audioContextRef = useRef(null); - const startRecording = async () => { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); - mediaRecorderRef.current = new MediaRecorder(stream); - audioChunksRef.current = []; + useEffect(() => { + const initRecording = async () => { + audioStreamRef.current = await navigator.mediaDevices.getUserMedia({ audio: true }); + mediaRecorderRef.current = new MediaRecorder(audioStreamRef.current); - mediaRecorderRef.current.ondataavailable = (e) => { - audioChunksRef.current.push(e.data); + mediaRecorderRef.current.ondataavailable = (e) => { + audioChunksRef.current.push(e.data); + }; + + mediaRecorderRef.current.onstop = async () => { + const inputAudioBlob = new Blob(audioChunksRef.current, { type: 'audio/webm' }); + audioChunksRef.current = []; // Clear for next recording + const formData = new FormData(); + formData.append('file', inputAudioBlob, 'input.webm'); + formData.append('model', 'whisper-1'); + + const whisperRes = await fetch('https://api.openai.com/v1/audio/transcriptions', { + method: 'POST', + headers: { + Authorization: `Bearer ${OPENAI_API_KEY}` + }, + body: formData + }); + const { text } = await whisperRes.json(); + setTranscript(text); + + const chatRes = await fetch('https://api.openai.com/v1/chat/completions', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${OPENAI_API_KEY}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + model: 'gpt-4o', + messages: [ + { role: 'system', content: 'You are an English tutor that is kind, fun to be around and can teach English language lessons through adventurous stories very well. You are assigned to talk with a primary school EFL student about a movie they watched yesterday. The conversation will be in English.' }, + { role: 'user', content: text } + ] + }) + }); + const chatData = await chatRes.json(); + if (!chatData.choices || !chatData.choices[0]) { + console.error('Chat API response error:', chatData); + setAiReply('Sorry, something went wrong with the AI response.'); + return; + } + const reply = chatData.choices[0].message.content; + setAiReply(reply); + + const speechRes = await fetch('https://api.openai.com/v1/audio/speech', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${OPENAI_API_KEY}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + model: 'tts-1-hd', + voice: 'nova', + input: reply + }) + }); + const outputAudioBlob = await speechRes.blob(); + const audioUrl = URL.createObjectURL(outputAudioBlob); + const audio = new Audio(audioUrl); + audio.play(); + + audio.onended = () => { + if (mediaRecorderRef.current && audioStreamRef.current) { + audioChunksRef.current = []; + mediaRecorderRef.current.start(); + setIsRecording(true); + monitorSilence(); + } + }; + }; }; - mediaRecorderRef.current.onstop = async () => { - const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/webm' }); - const formData = new FormData(); - formData.append('file', audioBlob, 'input.webm'); - formData.append('model', 'whisper-1'); + initRecording(); + }, []); - const whisperRes = await fetch('https://api.openai.com/v1/audio/transcriptions', { - method: 'POST', - headers: { - Authorization: `Bearer ${OPENAI_API_KEY}` - }, - body: formData - }); - const { text } = await whisperRes.json(); - setTranscript(text); + const monitorSilence = () => { + if (!audioStreamRef.current) return; - const chatRes = await fetch('https://api.openai.com/v1/chat/completions', { - method: 'POST', - headers: { - 'Authorization': `Bearer ${OPENAI_API_KEY}`, - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - model: 'gpt-4o', - messages: [ - { role: 'system', content: 'You are an English tutor that is kind, fun to be around and can teach English language lessons through adventurous stories very well. You are assigned to talk with a primary school EFL student about a movie they watched yesterday.' }, - { role: 'user', content: text } - ] - }) - }); - const chatData = await chatRes.json(); - if (!chatData.choices || !chatData.choices[0]) { - console.error('Chat API response error:', chatData); - setAiReply('Sorry, something went wrong with the AI response.'); - return; + if (audioContextRef.current) { + audioContextRef.current.close(); + } + audioContextRef.current = new AudioContext(); + + // Resume AudioContext to avoid browser autoplay policy issues + audioContextRef.current.resume().then(() => { + const source = audioContextRef.current.createMediaStreamSource(audioStreamRef.current); + const analyser = audioContextRef.current.createAnalyser(); + analyser.fftSize = 2048; + analyserRef.current = analyser; + const bufferLength = analyser.frequencyBinCount; + const dataArray = new Uint8Array(bufferLength); + dataArrayRef.current = dataArray; + source.connect(analyser); + + const canvas = canvasRef.current; + const canvasCtx = canvas.getContext('2d'); + + const SILENCE_THRESHOLD = 0.02; // Adjust as needed (0 to 1 scale) + const SILENCE_TIMEOUT = 1500; // ms + + let silenceStart = null; + + const checkSilenceAndDraw = () => { + analyser.getByteTimeDomainData(dataArray); + + // Draw waveform (same as before) + canvasCtx.fillStyle = '#000'; + canvasCtx.fillRect(0, 0, canvas.width, canvas.height); + + canvasCtx.lineWidth = 2; + canvasCtx.strokeStyle = '#00ff00'; + canvasCtx.beginPath(); + + const sliceWidth = canvas.width / bufferLength; + let x = 0; + + for (let i = 0; i < bufferLength; i++) { + const v = dataArray[i] / 128.0; + const y = v * canvas.height / 2; + if (i === 0) { + canvasCtx.moveTo(x, y); + } else { + canvasCtx.lineTo(x, y); + } + x += sliceWidth; } - const reply = chatData.choices[0].message.content; - setAiReply(reply); + canvasCtx.lineTo(canvas.width, canvas.height / 2); + canvasCtx.stroke(); - const speechRes = await fetch('https://api.openai.com/v1/audio/speech', { - method: 'POST', - headers: { - 'Authorization': `Bearer ${OPENAI_API_KEY}`, - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - model: 'tts-1-hd', - voice: 'nova', - input: reply - }) - }); - const outputAudioBlob = await speechRes.blob(); - const audioUrl = URL.createObjectURL(outputAudioBlob); - const audio = new Audio(audioUrl); - audio.play(); + // RMS calculation + let sumSquares = 0; + for (let i = 0; i < bufferLength; i++) { + const normalized = (dataArray[i] - 128) / 128; + sumSquares += normalized * normalized; + } + const rms = Math.sqrt(sumSquares / bufferLength); + + if (rms < SILENCE_THRESHOLD) { + // Silence detected + if (!silenceStart) silenceStart = Date.now(); + else if (Date.now() - silenceStart > SILENCE_TIMEOUT) { + if (mediaRecorderRef.current && isRecording) { + mediaRecorderRef.current.stop(); + setIsRecording(false); + audioContextRef.current.close(); + } + silenceStart = null; // reset after stopping + return; // stop animation loop on silence stop + } + } else { + // Sound detected + silenceStart = null; + } + + requestAnimationFrame(checkSilenceAndDraw); }; - mediaRecorderRef.current.start(); - setIsRecording(true); - }; - const stopRecording = () => { - mediaRecorderRef.current.stop(); + }); +}; - // Stop all tracks on the media stream - if (mediaRecorderRef.current.stream) { - mediaRecorderRef.current.stream.getTracks().forEach(track => track.stop()); + const toggleRecording = () => { + if (!isRecording) { + audioChunksRef.current = []; + mediaRecorderRef.current.start(); + setIsRecording(true); + monitorSilence(); + } else { + mediaRecorderRef.current.stop(); + setIsRecording(false); } - - setIsRecording(false); }; return ( -