Add voice command review/edit system with user feedback tracking

Implemented complete review/edit workflow for voice commands with ML feedback collection: **Backend:** - Created V012 migration for voice_feedback table with user action tracking - Added VoiceFeedback entity with approval/edit/reject actions - Implemented voice feedback API endpoint (POST /api/v1/voice/feedback) - Fixed user ID extraction bug (req.user.userId vs req.user.sub) **Frontend:** - Built VoiceActivityReview component with field-specific editors - Integrated review dialog into voice command workflow - Added approve/edit/reject handlers with feedback submission - Fixed infinite loop by tracking processed classification IDs **Features:** - Users can review AI-extracted data before saving - Quick-edit capabilities for all activity fields - Feedback data stored for ML model improvement - Activity creation only happens after user approval/edit 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-10-02 11:03:54 +00:00
parent 77f2c1d767
commit e94a1018c4
10 changed files with 743 additions and 79 deletions
--- a/maternal-web/components/voice/VoiceFloatingButton.tsx
+++ b/maternal-web/components/voice/VoiceFloatingButton.tsx
@@ -24,6 +24,8 @@ import { useVoiceInput } from '@/hooks/useVoiceInput';
 import { useAuth } from '@/lib/auth/AuthContext';
 import { trackingApi } from '@/lib/api/tracking';
 import { childrenApi } from '@/lib/api/children';
+import { voiceApi } from '@/lib/api/voice';
+import { VoiceActivityReview } from './VoiceActivityReview';

 /**
 * Floating voice input button
@@ -40,6 +42,7 @@ export function VoiceFloatingButton() {
  const [identifiedActivity, setIdentifiedActivity] = useState<string>('');
  const [classificationResult, setClassificationResult] = useState<any>(null);
  const [processedClassificationId, setProcessedClassificationId] = useState<string | null>(null);
+  const [showReview, setShowReview] = useState(false);
  const [snackbar, setSnackbar] = useState<{
    open: boolean;
    message: string;
@@ -75,10 +78,23 @@ export function VoiceFloatingButton() {

    // Only process if we haven't already processed this exact classification
    if (classification && !isListening && !isProcessing && open && classificationId !== processedClassificationId) {
-      console.log('[Voice] New classification detected, processing...', classificationId);
+      console.log('[Voice] New classification detected, showing review...', classificationId);
      setProcessedClassificationId(classificationId);
      setClassificationResult(classification);
-      handleClassifiedIntent(classification);
+
+      // Show review dialog instead of immediately creating activity
+      if (classification.type !== 'unknown' && classification.confidence >= 0.3) {
+        setProcessingStatus(null);
+        setShowReview(true);
+      } else {
+        // For unknown or low confidence, show error
+        setProcessingStatus(null);
+        setSnackbar({
+          open: true,
+          message: 'Could not understand the command. Please try again or use manual entry.',
+          severity: 'warning',
+        });
+      }
    }
  }, [classification, isListening, isProcessing, open, transcript, processedClassificationId]);

@@ -122,91 +138,78 @@ export function VoiceFloatingButton() {
    stopListening();
  };

-  const handleClassifiedIntent = async (result: any) => {
-    console.log('[Voice] handleClassifiedIntent called with result:', result);
-
-    if (result.error) {
-      console.log('[Voice] Result has error:', result.message);
-      setProcessingStatus(null);
-      setSnackbar({
-        open: true,
-        message: result.message,
-        severity: 'error',
-      });
-      return;
-    }
-
-    // Support both formats: backend returns 'type', frontend local classifier returns 'intent'
-    const activityType = result.type || result.intent;
-    console.log('[Voice] Activity type:', activityType);
-
-    // Set identified activity for status display
-    setIdentifiedActivity(activityType);
-
-    // Handle unknown or low confidence
-    if (activityType === 'unknown' || (result.confidence && result.confidence < 0.3)) {
-      console.log('[Voice] Unknown or low confidence:', activityType, result.confidence);
-      setProcessingStatus(null);
-      setSnackbar({
-        open: true,
-        message: 'Could not understand the command. Please try again or use manual entry.',
-        severity: 'warning',
-      });
-      return;
-    }
-
+  const createActivity = async (activityType: string, activityDetails: Record<string, any>, activityTimestamp?: Date) => {
    // Get the first child from the family
    if (!familyId) {
      console.log('[Voice] No familyId found');
-      setProcessingStatus(null);
-      setSnackbar({
-        open: true,
-        message: 'No family found. Please set up your profile first.',
-        severity: 'error',
-      });
-      return;
+      throw new Error('No family found. Please set up your profile first.');
    }

    console.log('[Voice] Family ID:', familyId);

+    // Fetch children
+    console.log('[Voice] Fetching children for family:', familyId);
+    const children = await childrenApi.getChildren(familyId);
+    console.log('[Voice] Children found:', children.length, children);
+
+    if (children.length === 0) {
+      throw new Error('No children found. Please add a child first.');
+    }
+
+    // Use the first child
+    const childId = children[0].id;
+    console.log('[Voice] Using child ID:', childId);
+
+    // Create the activity
+    const activityData = {
+      type: activityType,
+      timestamp: activityTimestamp || new Date().toISOString(),
+      data: activityDetails,
+      notes: activityDetails.notes || undefined,
+    };
+
+    console.log('[Voice] Creating activity with data:', JSON.stringify(activityData, null, 2));
+
+    const createdActivity = await trackingApi.createActivity(childId, activityData);
+    console.log('[Voice] Activity created successfully:', createdActivity);
+
+    return { childId, activity: createdActivity };
+  };
+
+  const saveFeedback = async (action: 'approved' | 'edited' | 'rejected', finalData?: Record<string, any>, childId?: string, activityId?: string) => {
+    try {
+      await voiceApi.saveFeedback({
+        childId,
+        activityId,
+        transcript,
+        language: classificationResult?.language || 'en',
+        extractedType: classificationResult.type,
+        extractedData: classificationResult.details,
+        confidence: classificationResult.confidence,
+        action,
+        finalType: action === 'edited' ? classificationResult.type : undefined,
+        finalData: action === 'edited' ? finalData : undefined,
+      });
+      console.log(`[Voice] Feedback saved: ${action}`);
+    } catch (error) {
+      console.error('[Voice] Failed to save feedback:', error);
+      // Don't throw - feedback is nice-to-have, not critical
+    }
+  };
+
+  const handleApprove = async (data: any) => {
    try {
      setIsProcessing(true);
      setProcessingStatus('saving');
+      setShowReview(false);

-      // Fetch children
-      console.log('[Voice] Fetching children for family:', familyId);
-      const children = await childrenApi.getChildren(familyId);
-      console.log('[Voice] Children found:', children.length, children);
+      const { childId, activity } = await createActivity(data.type, data.details, data.timestamp);

-      if (children.length === 0) {
-        setSnackbar({
-          open: true,
-          message: 'No children found. Please add a child first.',
-          severity: 'error',
-        });
-        setIsProcessing(false);
-        return;
-      }
-
-      // Use the first child (or you could enhance this to support child name matching)
-      const childId = children[0].id;
-      console.log('[Voice] Using child ID:', childId);
-
-      // Create the activity
-      const activityData = {
-        type: activityType,
-        timestamp: result.timestamp || new Date().toISOString(),
-        data: result.details || result.structuredData || {},
-        notes: result.details?.notes || result.structuredData?.notes || undefined,
-      };
-
-      console.log('[Voice] Creating activity with data:', JSON.stringify(activityData, null, 2));
-
-      const createdActivity = await trackingApi.createActivity(childId, activityData);
-      console.log('[Voice] Activity created successfully:', createdActivity);
+      // Save feedback
+      await saveFeedback('approved', undefined, childId, activity.id);

      // Show success message
-      const activityLabel = activityType.charAt(0).toUpperCase() + activityType.slice(1);
+      const activityLabel = data.type.charAt(0).toUpperCase() + data.type.slice(1);
      setSnackbar({
        open: true,
        message: `${activityLabel} activity saved successfully!`,
@@ -218,16 +221,68 @@ export function VoiceFloatingButton() {
        handleClose();
      }, 1500);
    } catch (error: any) {
-      console.error('[Voice] Failed to create activity - Full error:', error);
-      console.error('[Voice] Error response:', error.response);
-      console.error('[Voice] Error data:', error.response?.data);
+      console.error('[Voice] Failed to create activity:', error);
      setSnackbar({
        open: true,
-        message: error.response?.data?.message || 'Failed to save activity. Please try again.',
+        message: error.message || 'Failed to save activity. Please try again.',
        severity: 'error',
      });
    } finally {
      setIsProcessing(false);
+      setProcessingStatus(null);
+    }
+  };
+
+  const handleEdit = async (editedData: any) => {
+    try {
+      setIsProcessing(true);
+      setProcessingStatus('saving');
+      setShowReview(false);
+
+      const { childId, activity } = await createActivity(editedData.type, editedData.details, editedData.timestamp);
+
+      // Save feedback with edited data
+      await saveFeedback('edited', editedData.details, childId, activity.id);
+
+      // Show success message
+      const activityLabel = editedData.type.charAt(0).toUpperCase() + editedData.type.slice(1);
+      setSnackbar({
+        open: true,
+        message: `${activityLabel} activity saved with your edits!`,
+        severity: 'success',
+      });
+
+      // Auto-close dialog
+      setTimeout(() => {
+        handleClose();
+      }, 1500);
+    } catch (error: any) {
+      console.error('[Voice] Failed to create edited activity:', error);
+      setSnackbar({
+        open: true,
+        message: error.message || 'Failed to save activity. Please try again.',
+        severity: 'error',
+      });
+    } finally {
+      setIsProcessing(false);
+      setProcessingStatus(null);
+    }
+  };
+
+  const handleReject = async () => {
+    try {
+      setShowReview(false);
+
+      // Save feedback
+      await saveFeedback('rejected');
+
+      setSnackbar({
+        open: true,
+        message: 'Voice command rejected. Try again or use manual entry.',
+        severity: 'info',
+      });
+    } catch (error) {
+      console.error('[Voice] Failed to save rejection feedback:', error);
    }
  };

@@ -366,6 +421,19 @@ export function VoiceFloatingButton() {
        </DialogActions>
      </Dialog>

+      {/* Review Dialog */}
+      {showReview && classificationResult && (
+        <VoiceActivityReview
+          open={showReview}
+          transcript={transcript}
+          classification={classificationResult}
+          onApprove={handleApprove}
+          onEdit={handleEdit}
+          onReject={handleReject}
+          onClose={() => setShowReview(false)}
+        />
+      )}
+
      {/* Snackbar for feedback */}
      <Snackbar
        open={snackbar.open}