Files
maternal-app/maternal-web/components/voice/VoiceFloatingButton.tsx
Andrei 77f2c1d767
Some checks failed
CI/CD Pipeline / Lint and Test (push) Has been cancelled
CI/CD Pipeline / E2E Tests (push) Has been cancelled
CI/CD Pipeline / Build Application (push) Has been cancelled
Fix voice command data structure and prevent duplicate activities
Backend changes:
- Update LLM prompt to use correct field names matching frontend interfaces
  - Use 'diaperType' instead of 'type' for diaper activities
  - Use 'feedingType' instead of 'method' for feeding activities
  - Simplify sleep structure (duration, quality, location only)

Frontend changes:
- Add processedClassificationId tracking to prevent infinite loop
- Create unique ID for each classification to avoid duplicate processing
- Reset processed ID when dialog opens/closes or new recording starts

This fixes the issue where voice commands created multiple duplicate
activities and had mismatched data structures causing tracker warnings.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-10-02 10:44:52 +00:00

383 lines
13 KiB
TypeScript

'use client';
import React, { useState } from 'react';
import {
Fab,
Tooltip,
Snackbar,
Alert,
Dialog,
DialogTitle,
DialogContent,
DialogActions,
Button,
Box,
Typography,
CircularProgress,
Chip,
IconButton,
} from '@mui/material';
import MicIcon from '@mui/icons-material/Mic';
import MicOffIcon from '@mui/icons-material/MicOff';
import { useRouter } from 'next/navigation';
import { useVoiceInput } from '@/hooks/useVoiceInput';
import { useAuth } from '@/lib/auth/AuthContext';
import { trackingApi } from '@/lib/api/tracking';
import { childrenApi } from '@/lib/api/children';
/**
* Floating voice input button
*
* Always visible floating action button for quick voice commands.
* Positioned in bottom-right corner for easy thumb access.
*/
export function VoiceFloatingButton() {
const router = useRouter();
const { user } = useAuth();
const [open, setOpen] = useState(false);
const [isProcessing, setIsProcessing] = useState(false);
const [processingStatus, setProcessingStatus] = useState<'listening' | 'understanding' | 'saving' | null>(null);
const [identifiedActivity, setIdentifiedActivity] = useState<string>('');
const [classificationResult, setClassificationResult] = useState<any>(null);
const [processedClassificationId, setProcessedClassificationId] = useState<string | null>(null);
const [snackbar, setSnackbar] = useState<{
open: boolean;
message: string;
severity: 'success' | 'info' | 'warning' | 'error';
}>({
open: false,
message: '',
severity: 'info',
});
const familyId = user?.families?.[0]?.familyId;
const { isListening, isSupported, transcript, classification, error, usesFallback, startListening, stopListening, reset } =
useVoiceInput();
// Set status when listening starts/stops
React.useEffect(() => {
if (isListening) {
setProcessingStatus('listening');
} else if (processingStatus === 'listening' && transcript) {
// Transition from listening to understanding when we have a transcript
setProcessingStatus('understanding');
}
}, [isListening, transcript]);
// Auto-use classification from backend when transcription completes
// MediaRecorder sends audio to backend, which transcribes + classifies in one call
React.useEffect(() => {
// Create a unique ID for this classification based on transcript + type + timestamp
const classificationId = classification
? `${transcript}-${classification.type}-${classification.timestamp}`
: null;
// Only process if we haven't already processed this exact classification
if (classification && !isListening && !isProcessing && open && classificationId !== processedClassificationId) {
console.log('[Voice] New classification detected, processing...', classificationId);
setProcessedClassificationId(classificationId);
setClassificationResult(classification);
handleClassifiedIntent(classification);
}
}, [classification, isListening, isProcessing, open, transcript, processedClassificationId]);
const handleOpen = () => {
if (!isSupported) {
setSnackbar({
open: true,
message: 'Voice input not supported in this browser. Please use Chrome, Edge, or Safari.',
severity: 'error',
});
return;
}
setOpen(true);
reset();
setClassificationResult(null);
setProcessingStatus(null);
setIdentifiedActivity('');
setProcessedClassificationId(null);
};
const handleClose = () => {
if (isListening) {
stopListening();
}
setOpen(false);
reset();
setClassificationResult(null);
setProcessingStatus(null);
setIdentifiedActivity('');
setProcessedClassificationId(null);
};
const handleStartListening = () => {
reset();
setClassificationResult(null);
setProcessedClassificationId(null);
startListening();
};
const handleStopListening = () => {
stopListening();
};
const handleClassifiedIntent = async (result: any) => {
console.log('[Voice] handleClassifiedIntent called with result:', result);
if (result.error) {
console.log('[Voice] Result has error:', result.message);
setProcessingStatus(null);
setSnackbar({
open: true,
message: result.message,
severity: 'error',
});
return;
}
// Support both formats: backend returns 'type', frontend local classifier returns 'intent'
const activityType = result.type || result.intent;
console.log('[Voice] Activity type:', activityType);
// Set identified activity for status display
setIdentifiedActivity(activityType);
// Handle unknown or low confidence
if (activityType === 'unknown' || (result.confidence && result.confidence < 0.3)) {
console.log('[Voice] Unknown or low confidence:', activityType, result.confidence);
setProcessingStatus(null);
setSnackbar({
open: true,
message: 'Could not understand the command. Please try again or use manual entry.',
severity: 'warning',
});
return;
}
// Get the first child from the family
if (!familyId) {
console.log('[Voice] No familyId found');
setProcessingStatus(null);
setSnackbar({
open: true,
message: 'No family found. Please set up your profile first.',
severity: 'error',
});
return;
}
console.log('[Voice] Family ID:', familyId);
try {
setIsProcessing(true);
setProcessingStatus('saving');
// Fetch children
console.log('[Voice] Fetching children for family:', familyId);
const children = await childrenApi.getChildren(familyId);
console.log('[Voice] Children found:', children.length, children);
if (children.length === 0) {
setSnackbar({
open: true,
message: 'No children found. Please add a child first.',
severity: 'error',
});
setIsProcessing(false);
return;
}
// Use the first child (or you could enhance this to support child name matching)
const childId = children[0].id;
console.log('[Voice] Using child ID:', childId);
// Create the activity
const activityData = {
type: activityType,
timestamp: result.timestamp || new Date().toISOString(),
data: result.details || result.structuredData || {},
notes: result.details?.notes || result.structuredData?.notes || undefined,
};
console.log('[Voice] Creating activity with data:', JSON.stringify(activityData, null, 2));
const createdActivity = await trackingApi.createActivity(childId, activityData);
console.log('[Voice] Activity created successfully:', createdActivity);
// Show success message
const activityLabel = activityType.charAt(0).toUpperCase() + activityType.slice(1);
setSnackbar({
open: true,
message: `${activityLabel} activity saved successfully!`,
severity: 'success',
});
// Auto-close dialog
setTimeout(() => {
handleClose();
}, 1500);
} catch (error: any) {
console.error('[Voice] Failed to create activity - Full error:', error);
console.error('[Voice] Error response:', error.response);
console.error('[Voice] Error data:', error.response?.data);
setSnackbar({
open: true,
message: error.response?.data?.message || 'Failed to save activity. Please try again.',
severity: 'error',
});
} finally {
setIsProcessing(false);
}
};
const handleCloseSnackbar = () => {
setSnackbar(prev => ({ ...prev, open: false }));
};
return (
<>
{/* Floating button positioned in bottom-right */}
<Tooltip title="Voice Command (Beta)" placement="left">
<Fab
color="primary"
aria-label="voice input"
onClick={handleOpen}
disabled={!isSupported}
sx={{
position: 'fixed',
bottom: 24,
right: 24,
zIndex: 1000,
}}
>
<MicIcon />
</Fab>
</Tooltip>
{/* Voice input dialog */}
<Dialog open={open} onClose={handleClose} maxWidth="sm" fullWidth>
<DialogTitle>
Voice Command
{classificationResult && !classificationResult.error && (
<Chip
label={`${classificationResult.type || classificationResult.intent} (${classificationResult.confidenceLevel || Math.round((classificationResult.confidence || 0) * 100) + '%'})`}
color="success"
size="small"
sx={{ ml: 2 }}
/>
)}
</DialogTitle>
<DialogContent>
<Box sx={{ textAlign: 'center', py: 3 }}>
{/* Microphone animation */}
<Box sx={{ position: 'relative', display: 'inline-block', mb: 3 }}>
<IconButton
color={isListening ? 'error' : 'primary'}
onClick={isListening ? handleStopListening : handleStartListening}
sx={{
width: 80,
height: 80,
bgcolor: isListening ? 'error.light' : 'primary.light',
'&:hover': {
bgcolor: isListening ? 'error.main' : 'primary.main',
},
animation: isListening ? 'pulse 1.5s infinite' : 'none',
'@keyframes pulse': {
'0%': { transform: 'scale(1)', opacity: 1 },
'50%': { transform: 'scale(1.1)', opacity: 0.8 },
'100%': { transform: 'scale(1)', opacity: 1 },
},
}}
>
{isListening ? <MicIcon sx={{ fontSize: 48 }} /> : <MicOffIcon sx={{ fontSize: 48 }} />}
</IconButton>
</Box>
{/* Status text with detailed processing stages */}
<Typography variant="body1" color="text.secondary" gutterBottom>
{processingStatus === 'listening' && 'Listening... Speak now'}
{processingStatus === 'understanding' && 'Understanding your request...'}
{processingStatus === 'saving' && identifiedActivity && `Adding to ${identifiedActivity.charAt(0).toUpperCase() + identifiedActivity.slice(1)} tracker...`}
{!processingStatus && !isListening && 'Click the microphone to start'}
</Typography>
{/* Transcript */}
{transcript && (
<Box sx={{ mt: 3, p: 2, bgcolor: 'grey.100', borderRadius: 1 }}>
<Typography variant="body2" color="text.secondary" gutterBottom>
Transcript:
</Typography>
<Typography variant="body1">{transcript}</Typography>
</Box>
)}
{/* Processing indicator with status */}
{processingStatus && (
<Box sx={{ mt: 2, display: 'flex', alignItems: 'center', justifyContent: 'center' }}>
<CircularProgress size={20} sx={{ mr: 1 }} />
<Typography variant="body2" color="text.secondary">
{processingStatus === 'listening' && 'Listening...'}
{processingStatus === 'understanding' && 'Understanding...'}
{processingStatus === 'saving' && 'Saving...'}
</Typography>
</Box>
)}
{/* Classification result */}
{classificationResult && !classificationResult.error && (
<Alert severity="success" sx={{ mt: 2 }}>
<Typography variant="body2" gutterBottom>
<strong>Understood:</strong> {classificationResult.type || classificationResult.intent}
</Typography>
</Alert>
)}
{/* Error messages */}
{(error || (classificationResult && classificationResult.error)) && (
<Alert severity="error" sx={{ mt: 2 }}>
{error || classificationResult.message}
</Alert>
)}
{/* Examples */}
{!transcript && !isListening && (
<Box sx={{ mt: 3, textAlign: 'left' }}>
<Typography variant="caption" color="text.secondary" gutterBottom display="block">
Example commands:
</Typography>
<Typography variant="caption" color="text.secondary" component="div">
"Fed baby 120 ml"
<br />
"Nursed on left breast for 15 minutes"
<br />
"Changed wet diaper"
<br />
"Baby napped for 45 minutes"
</Typography>
</Box>
)}
</Box>
</DialogContent>
<DialogActions>
<Button onClick={handleClose}>Cancel</Button>
</DialogActions>
</Dialog>
{/* Snackbar for feedback */}
<Snackbar
open={snackbar.open}
autoHideDuration={3000}
onClose={handleCloseSnackbar}
anchorOrigin={{ vertical: 'bottom', horizontal: 'center' }}
>
<Alert onClose={handleCloseSnackbar} severity={snackbar.severity} sx={{ width: '100%' }}>
{snackbar.message}
</Alert>
</Snackbar>
</>
);
}