diff --git a/maternal-app/maternal-app-backend/src/database/entities/index.ts b/maternal-app/maternal-app-backend/src/database/entities/index.ts index b5f9e03..fee2dd9 100644 --- a/maternal-app/maternal-app-backend/src/database/entities/index.ts +++ b/maternal-app/maternal-app-backend/src/database/entities/index.ts @@ -14,4 +14,5 @@ export { NotificationStatus, NotificationPriority, } from './notification.entity'; -export { Photo, PhotoType } from './photo.entity'; \ No newline at end of file +export { Photo, PhotoType } from './photo.entity'; +export { VoiceFeedback, VoiceFeedbackAction } from './voice-feedback.entity'; \ No newline at end of file diff --git a/maternal-app/maternal-app-backend/src/database/entities/voice-feedback.entity.ts b/maternal-app/maternal-app-backend/src/database/entities/voice-feedback.entity.ts new file mode 100644 index 0000000..5452a64 --- /dev/null +++ b/maternal-app/maternal-app-backend/src/database/entities/voice-feedback.entity.ts @@ -0,0 +1,76 @@ +import { Entity, Column, PrimaryColumn, ManyToOne, JoinColumn } from 'typeorm'; +import { User } from './user.entity'; +import { Child } from './child.entity'; +import { Activity } from './activity.entity'; + +export enum VoiceFeedbackAction { + APPROVED = 'approved', + EDITED = 'edited', + REJECTED = 'rejected', +} + +@Entity('voice_feedback') +export class VoiceFeedback { + @PrimaryColumn({ type: 'varchar', length: 20 }) + id: string; + + @Column({ name: 'user_id', type: 'varchar', length: 20 }) + userId: string; + + @Column({ name: 'child_id', type: 'varchar', length: 20, nullable: true }) + childId?: string; + + @Column({ name: 'activity_id', type: 'varchar', length: 20, nullable: true }) + activityId?: string; + + @Column({ type: 'text' }) + transcript: string; + + @Column({ type: 'varchar', length: 10, nullable: true }) + language?: string; + + @Column({ name: 'extracted_type', type: 'varchar', length: 20 }) + extractedType: string; + + @Column({ name: 'extracted_data', type: 'jsonb', default: {} }) + extractedData: Record; + + @Column({ type: 'decimal', precision: 3, scale: 2, nullable: true }) + confidence?: number; + + @Column({ + type: 'varchar', + length: 20, + enum: VoiceFeedbackAction, + }) + action: VoiceFeedbackAction; + + @Column({ name: 'final_type', type: 'varchar', length: 20, nullable: true }) + finalType?: string; + + @Column({ name: 'final_data', type: 'jsonb', nullable: true }) + finalData?: Record; + + @Column({ name: 'user_notes', type: 'text', nullable: true }) + userNotes?: string; + + @Column({ + name: 'created_at', + type: 'timestamp', + default: () => 'CURRENT_TIMESTAMP', + }) + createdAt: Date; + + // Relations + @ManyToOne(() => User, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'user_id' }) + user: User; + + @ManyToOne(() => Child, { nullable: true, onDelete: 'CASCADE' }) + @JoinColumn({ name: 'child_id' }) + child?: Child; + + @ManyToOne(() => Activity, { nullable: true, onDelete: 'CASCADE' }) + @JoinColumn({ name: 'activity_id' }) + activity?: Activity; +} diff --git a/maternal-app/maternal-app-backend/src/database/migrations/V012_create_voice_feedback.sql b/maternal-app/maternal-app-backend/src/database/migrations/V012_create_voice_feedback.sql new file mode 100644 index 0000000..2a4dfaa --- /dev/null +++ b/maternal-app/maternal-app-backend/src/database/migrations/V012_create_voice_feedback.sql @@ -0,0 +1,43 @@ +-- V012: Create voice feedback table for storing user corrections and approvals +-- This helps improve AI accuracy by learning from user edits + +CREATE TABLE IF NOT EXISTS voice_feedback ( + id VARCHAR(20) PRIMARY KEY, + user_id VARCHAR(20) NOT NULL REFERENCES users(id) ON DELETE CASCADE, + child_id VARCHAR(20) REFERENCES children(id) ON DELETE CASCADE, + activity_id VARCHAR(20) REFERENCES activities(id) ON DELETE CASCADE, + + -- Original voice input + transcript TEXT NOT NULL, + language VARCHAR(10), + + -- AI extracted data (before user edits) + extracted_type VARCHAR(20) NOT NULL, + extracted_data JSONB NOT NULL DEFAULT '{}', + confidence DECIMAL(3,2), + + -- User actions + action VARCHAR(20) NOT NULL CHECK (action IN ('approved', 'edited', 'rejected')), + + -- Final data (after user edits, if edited) + final_type VARCHAR(20), + final_data JSONB, + + -- User feedback + user_notes TEXT, + + -- Metadata + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Indexes for analytics +CREATE INDEX idx_voice_feedback_user ON voice_feedback(user_id, created_at DESC); +CREATE INDEX idx_voice_feedback_type ON voice_feedback(extracted_type, action); +CREATE INDEX idx_voice_feedback_confidence ON voice_feedback(confidence, action); +CREATE INDEX idx_voice_feedback_activity ON voice_feedback(activity_id); + +-- Add comment +COMMENT ON TABLE voice_feedback IS 'Stores user feedback on voice command accuracy for ML improvement'; +COMMENT ON COLUMN voice_feedback.action IS 'User action: approved (no changes), edited (modified data), rejected (discarded)'; +COMMENT ON COLUMN voice_feedback.extracted_data IS 'Original AI-extracted activity data before user edits'; +COMMENT ON COLUMN voice_feedback.final_data IS 'User-modified data if action=edited'; diff --git a/maternal-app/maternal-app-backend/src/modules/voice/dto/save-voice-feedback.dto.ts b/maternal-app/maternal-app-backend/src/modules/voice/dto/save-voice-feedback.dto.ts new file mode 100644 index 0000000..ac96e4e --- /dev/null +++ b/maternal-app/maternal-app-backend/src/modules/voice/dto/save-voice-feedback.dto.ts @@ -0,0 +1,50 @@ +import { IsString, IsNotEmpty, IsOptional, IsEnum, IsObject, IsNumber, Min, Max } from 'class-validator'; +import { VoiceFeedbackAction } from '../../../database/entities'; + +export class SaveVoiceFeedbackDto { + @IsString() + @IsOptional() + childId?: string; + + @IsString() + @IsOptional() + activityId?: string; + + @IsString() + @IsNotEmpty() + transcript: string; + + @IsString() + @IsOptional() + language?: string; + + @IsString() + @IsNotEmpty() + extractedType: string; + + @IsObject() + @IsNotEmpty() + extractedData: Record; + + @IsNumber() + @Min(0) + @Max(1) + @IsOptional() + confidence?: number; + + @IsEnum(VoiceFeedbackAction) + @IsNotEmpty() + action: VoiceFeedbackAction; + + @IsString() + @IsOptional() + finalType?: string; + + @IsObject() + @IsOptional() + finalData?: Record; + + @IsString() + @IsOptional() + userNotes?: string; +} diff --git a/maternal-app/maternal-app-backend/src/modules/voice/voice.controller.ts b/maternal-app/maternal-app-backend/src/modules/voice/voice.controller.ts index dae493c..6b7b22a 100644 --- a/maternal-app/maternal-app-backend/src/modules/voice/voice.controller.ts +++ b/maternal-app/maternal-app-backend/src/modules/voice/voice.controller.ts @@ -7,10 +7,13 @@ import { Req, BadRequestException, Logger, + UseGuards, } from '@nestjs/common'; import { FileInterceptor } from '@nestjs/platform-express'; +import { JwtAuthGuard } from '../auth/guards/jwt-auth.guard'; import { VoiceService } from './voice.service'; import { Public } from '../auth/decorators/public.decorator'; +import { SaveVoiceFeedbackDto } from './dto/save-voice-feedback.dto'; @Controller('api/v1/voice') export class VoiceController { @@ -183,4 +186,22 @@ export class VoiceController { classification: result, }; } + + @Post('feedback') + @UseGuards(JwtAuthGuard) + async saveFeedback( + @Req() req: any, + @Body() feedbackDto: SaveVoiceFeedbackDto, + ) { + const userId = req.user.userId; + + this.logger.log(`[Voice Feedback] User ${userId} submitting feedback: ${feedbackDto.action}`); + + const feedback = await this.voiceService.saveFeedback(userId, feedbackDto); + + return { + success: true, + data: feedback, + }; + } } \ No newline at end of file diff --git a/maternal-app/maternal-app-backend/src/modules/voice/voice.module.ts b/maternal-app/maternal-app-backend/src/modules/voice/voice.module.ts index 708e9cc..258443a 100644 --- a/maternal-app/maternal-app-backend/src/modules/voice/voice.module.ts +++ b/maternal-app/maternal-app-backend/src/modules/voice/voice.module.ts @@ -1,8 +1,11 @@ import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; import { VoiceService } from './voice.service'; import { VoiceController } from './voice.controller'; +import { VoiceFeedback } from '../../database/entities'; @Module({ + imports: [TypeOrmModule.forFeature([VoiceFeedback])], controllers: [VoiceController], providers: [VoiceService], exports: [VoiceService], diff --git a/maternal-app/maternal-app-backend/src/modules/voice/voice.service.ts b/maternal-app/maternal-app-backend/src/modules/voice/voice.service.ts index 6b0cdb0..0ddd311 100644 --- a/maternal-app/maternal-app-backend/src/modules/voice/voice.service.ts +++ b/maternal-app/maternal-app-backend/src/modules/voice/voice.service.ts @@ -1,8 +1,13 @@ import { Injectable, Logger, BadRequestException } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; import OpenAI from 'openai'; import * as fs from 'fs'; import * as path from 'path'; +import { nanoid } from 'nanoid'; +import { VoiceFeedback } from '../../database/entities'; +import { SaveVoiceFeedbackDto } from './dto/save-voice-feedback.dto'; export interface TranscriptionResult { text: string; @@ -26,7 +31,11 @@ export class VoiceService { // Supported languages for MVP private readonly SUPPORTED_LANGUAGES = ['en', 'es', 'fr', 'pt', 'zh']; - constructor(private configService: ConfigService) { + constructor( + private configService: ConfigService, + @InjectRepository(VoiceFeedback) + private readonly voiceFeedbackRepository: Repository, + ) { // Check if Azure OpenAI is enabled const azureEnabled = this.configService.get('AZURE_OPENAI_ENABLED'); @@ -328,4 +337,44 @@ Respond ONLY with the question text, no formatting.`; return 'Could you provide more details about this activity?'; } } + + /** + * Save user feedback on voice command accuracy + */ + async saveFeedback( + userId: string, + feedbackDto: SaveVoiceFeedbackDto, + ): Promise { + try { + const feedback = this.voiceFeedbackRepository.create({ + id: `vfb_${nanoid(16)}`, + userId, + childId: feedbackDto.childId, + activityId: feedbackDto.activityId, + transcript: feedbackDto.transcript, + language: feedbackDto.language, + extractedType: feedbackDto.extractedType, + extractedData: feedbackDto.extractedData, + confidence: feedbackDto.confidence, + action: feedbackDto.action, + finalType: feedbackDto.finalType, + finalData: feedbackDto.finalData, + userNotes: feedbackDto.userNotes, + }); + + const saved = await this.voiceFeedbackRepository.save(feedback); + + this.logger.log( + `[Voice Feedback] User ${userId} ${feedbackDto.action} voice command for type: ${feedbackDto.extractedType}`, + ); + + return saved; + } catch (error) { + this.logger.error( + `[Voice Feedback] Failed to save: ${error.message}`, + error.stack, + ); + throw new BadRequestException('Failed to save voice feedback'); + } + } } \ No newline at end of file diff --git a/maternal-web/components/voice/VoiceActivityReview.tsx b/maternal-web/components/voice/VoiceActivityReview.tsx new file mode 100644 index 0000000..7a55b39 --- /dev/null +++ b/maternal-web/components/voice/VoiceActivityReview.tsx @@ -0,0 +1,331 @@ +'use client'; + +import React, { useState } from 'react'; +import { + Dialog, + DialogTitle, + DialogContent, + DialogActions, + Button, + Box, + Typography, + TextField, + Select, + MenuItem, + FormControl, + InputLabel, + Alert, + Chip, + Grid, + InputAdornment, +} from '@mui/material'; +import CheckCircleIcon from '@mui/icons-material/CheckCircle'; +import EditIcon from '@mui/icons-material/Edit'; +import CloseIcon from '@mui/icons-material/Close'; + +interface VoiceActivityReviewProps { + open: boolean; + transcript: string; + classification: { + type: string; + details: Record; + confidence: number; + timestamp?: Date; + }; + onApprove: (data: any) => void; + onEdit: (editedData: any) => void; + onReject: () => void; + onClose: () => void; +} + +export function VoiceActivityReview({ + open, + transcript, + classification, + onApprove, + onEdit, + onReject, + onClose, +}: VoiceActivityReviewProps) { + const [isEditing, setIsEditing] = useState(false); + const [editedData, setEditedData] = useState>({}); + + React.useEffect(() => { + if (open) { + setEditedData(classification.details || {}); + setIsEditing(false); + } + }, [open, classification]); + + const handleApprove = () => { + onApprove({ + type: classification.type, + details: classification.details, + timestamp: classification.timestamp, + }); + }; + + const handleSaveEdit = () => { + onEdit({ + type: classification.type, + details: editedData, + timestamp: classification.timestamp, + }); + }; + + const renderFieldEditor = (key: string, value: any) => { + // Render different input types based on field + if (key === 'notes') { + return ( + setEditedData({ ...editedData, [key]: e.target.value })} + variant="outlined" + size="small" + /> + ); + } + + if (key === 'amount') { + return ( + setEditedData({ ...editedData, [key]: Number(e.target.value) })} + variant="outlined" + size="small" + InputProps={{ + endAdornment: {editedData.unit || 'ml'}, + }} + /> + ); + } + + if (key === 'duration') { + return ( + setEditedData({ ...editedData, [key]: Number(e.target.value) })} + variant="outlined" + size="small" + InputProps={{ + endAdornment: min, + }} + /> + ); + } + + if (key === 'feedingType') { + return ( + + Feeding Type + + + ); + } + + if (key === 'diaperType') { + return ( + + Diaper Type + + + ); + } + + if (key === 'quality') { + return ( + + Sleep Quality + + + ); + } + + if (key === 'location') { + return ( + + Location + + + ); + } + + if (key === 'side') { + return ( + + Side + + + ); + } + + // Default text input + return ( + setEditedData({ ...editedData, [key]: e.target.value })} + variant="outlined" + size="small" + /> + ); + }; + + const getActivityLabel = () => { + return classification.type.charAt(0).toUpperCase() + classification.type.slice(1); + }; + + const getConfidenceColor = () => { + if (classification.confidence >= 0.7) return 'success'; + if (classification.confidence >= 0.4) return 'warning'; + return 'error'; + }; + + return ( + + + Review Voice Command + + + + + + {/* Transcript */} + + + You said: "{transcript}" + + + + {/* Extracted Data */} + + {isEditing ? 'Edit Activity Details' : 'Extracted Details'} + + + + {isEditing ? ( + + {Object.entries(editedData).map(([key, value]) => ( + + {renderFieldEditor(key, value)} + + ))} + + ) : ( + + {Object.entries(classification.details).map(([key, value]) => ( + + + {key.charAt(0).toUpperCase() + key.slice(1).replace(/([A-Z])/g, ' $1')}: + + + {value === null || value === undefined ? '-' : String(value)} + + + ))} + + )} + + + {/* Help text */} + {!isEditing && ( + + + Please review the extracted information. Click Edit to make changes or Approve if it looks correct. + + + )} + + + + + + {isEditing ? ( + <> + + + + ) : ( + <> + + + + )} + + + ); +} diff --git a/maternal-web/components/voice/VoiceFloatingButton.tsx b/maternal-web/components/voice/VoiceFloatingButton.tsx index 309f4a2..e04b6ad 100644 --- a/maternal-web/components/voice/VoiceFloatingButton.tsx +++ b/maternal-web/components/voice/VoiceFloatingButton.tsx @@ -24,6 +24,8 @@ import { useVoiceInput } from '@/hooks/useVoiceInput'; import { useAuth } from '@/lib/auth/AuthContext'; import { trackingApi } from '@/lib/api/tracking'; import { childrenApi } from '@/lib/api/children'; +import { voiceApi } from '@/lib/api/voice'; +import { VoiceActivityReview } from './VoiceActivityReview'; /** * Floating voice input button @@ -40,6 +42,7 @@ export function VoiceFloatingButton() { const [identifiedActivity, setIdentifiedActivity] = useState(''); const [classificationResult, setClassificationResult] = useState(null); const [processedClassificationId, setProcessedClassificationId] = useState(null); + const [showReview, setShowReview] = useState(false); const [snackbar, setSnackbar] = useState<{ open: boolean; message: string; @@ -75,10 +78,23 @@ export function VoiceFloatingButton() { // Only process if we haven't already processed this exact classification if (classification && !isListening && !isProcessing && open && classificationId !== processedClassificationId) { - console.log('[Voice] New classification detected, processing...', classificationId); + console.log('[Voice] New classification detected, showing review...', classificationId); setProcessedClassificationId(classificationId); setClassificationResult(classification); - handleClassifiedIntent(classification); + + // Show review dialog instead of immediately creating activity + if (classification.type !== 'unknown' && classification.confidence >= 0.3) { + setProcessingStatus(null); + setShowReview(true); + } else { + // For unknown or low confidence, show error + setProcessingStatus(null); + setSnackbar({ + open: true, + message: 'Could not understand the command. Please try again or use manual entry.', + severity: 'warning', + }); + } } }, [classification, isListening, isProcessing, open, transcript, processedClassificationId]); @@ -122,91 +138,78 @@ export function VoiceFloatingButton() { stopListening(); }; - const handleClassifiedIntent = async (result: any) => { - console.log('[Voice] handleClassifiedIntent called with result:', result); - - if (result.error) { - console.log('[Voice] Result has error:', result.message); - setProcessingStatus(null); - setSnackbar({ - open: true, - message: result.message, - severity: 'error', - }); - return; - } - - // Support both formats: backend returns 'type', frontend local classifier returns 'intent' - const activityType = result.type || result.intent; - console.log('[Voice] Activity type:', activityType); - - // Set identified activity for status display - setIdentifiedActivity(activityType); - - // Handle unknown or low confidence - if (activityType === 'unknown' || (result.confidence && result.confidence < 0.3)) { - console.log('[Voice] Unknown or low confidence:', activityType, result.confidence); - setProcessingStatus(null); - setSnackbar({ - open: true, - message: 'Could not understand the command. Please try again or use manual entry.', - severity: 'warning', - }); - return; - } - + const createActivity = async (activityType: string, activityDetails: Record, activityTimestamp?: Date) => { // Get the first child from the family if (!familyId) { console.log('[Voice] No familyId found'); - setProcessingStatus(null); - setSnackbar({ - open: true, - message: 'No family found. Please set up your profile first.', - severity: 'error', - }); - return; + throw new Error('No family found. Please set up your profile first.'); } console.log('[Voice] Family ID:', familyId); + // Fetch children + console.log('[Voice] Fetching children for family:', familyId); + const children = await childrenApi.getChildren(familyId); + console.log('[Voice] Children found:', children.length, children); + + if (children.length === 0) { + throw new Error('No children found. Please add a child first.'); + } + + // Use the first child + const childId = children[0].id; + console.log('[Voice] Using child ID:', childId); + + // Create the activity + const activityData = { + type: activityType, + timestamp: activityTimestamp || new Date().toISOString(), + data: activityDetails, + notes: activityDetails.notes || undefined, + }; + + console.log('[Voice] Creating activity with data:', JSON.stringify(activityData, null, 2)); + + const createdActivity = await trackingApi.createActivity(childId, activityData); + console.log('[Voice] Activity created successfully:', createdActivity); + + return { childId, activity: createdActivity }; + }; + + const saveFeedback = async (action: 'approved' | 'edited' | 'rejected', finalData?: Record, childId?: string, activityId?: string) => { + try { + await voiceApi.saveFeedback({ + childId, + activityId, + transcript, + language: classificationResult?.language || 'en', + extractedType: classificationResult.type, + extractedData: classificationResult.details, + confidence: classificationResult.confidence, + action, + finalType: action === 'edited' ? classificationResult.type : undefined, + finalData: action === 'edited' ? finalData : undefined, + }); + console.log(`[Voice] Feedback saved: ${action}`); + } catch (error) { + console.error('[Voice] Failed to save feedback:', error); + // Don't throw - feedback is nice-to-have, not critical + } + }; + + const handleApprove = async (data: any) => { try { setIsProcessing(true); setProcessingStatus('saving'); + setShowReview(false); - // Fetch children - console.log('[Voice] Fetching children for family:', familyId); - const children = await childrenApi.getChildren(familyId); - console.log('[Voice] Children found:', children.length, children); + const { childId, activity } = await createActivity(data.type, data.details, data.timestamp); - if (children.length === 0) { - setSnackbar({ - open: true, - message: 'No children found. Please add a child first.', - severity: 'error', - }); - setIsProcessing(false); - return; - } - - // Use the first child (or you could enhance this to support child name matching) - const childId = children[0].id; - console.log('[Voice] Using child ID:', childId); - - // Create the activity - const activityData = { - type: activityType, - timestamp: result.timestamp || new Date().toISOString(), - data: result.details || result.structuredData || {}, - notes: result.details?.notes || result.structuredData?.notes || undefined, - }; - - console.log('[Voice] Creating activity with data:', JSON.stringify(activityData, null, 2)); - - const createdActivity = await trackingApi.createActivity(childId, activityData); - console.log('[Voice] Activity created successfully:', createdActivity); + // Save feedback + await saveFeedback('approved', undefined, childId, activity.id); // Show success message - const activityLabel = activityType.charAt(0).toUpperCase() + activityType.slice(1); + const activityLabel = data.type.charAt(0).toUpperCase() + data.type.slice(1); setSnackbar({ open: true, message: `${activityLabel} activity saved successfully!`, @@ -218,16 +221,68 @@ export function VoiceFloatingButton() { handleClose(); }, 1500); } catch (error: any) { - console.error('[Voice] Failed to create activity - Full error:', error); - console.error('[Voice] Error response:', error.response); - console.error('[Voice] Error data:', error.response?.data); + console.error('[Voice] Failed to create activity:', error); setSnackbar({ open: true, - message: error.response?.data?.message || 'Failed to save activity. Please try again.', + message: error.message || 'Failed to save activity. Please try again.', severity: 'error', }); } finally { setIsProcessing(false); + setProcessingStatus(null); + } + }; + + const handleEdit = async (editedData: any) => { + try { + setIsProcessing(true); + setProcessingStatus('saving'); + setShowReview(false); + + const { childId, activity } = await createActivity(editedData.type, editedData.details, editedData.timestamp); + + // Save feedback with edited data + await saveFeedback('edited', editedData.details, childId, activity.id); + + // Show success message + const activityLabel = editedData.type.charAt(0).toUpperCase() + editedData.type.slice(1); + setSnackbar({ + open: true, + message: `${activityLabel} activity saved with your edits!`, + severity: 'success', + }); + + // Auto-close dialog + setTimeout(() => { + handleClose(); + }, 1500); + } catch (error: any) { + console.error('[Voice] Failed to create edited activity:', error); + setSnackbar({ + open: true, + message: error.message || 'Failed to save activity. Please try again.', + severity: 'error', + }); + } finally { + setIsProcessing(false); + setProcessingStatus(null); + } + }; + + const handleReject = async () => { + try { + setShowReview(false); + + // Save feedback + await saveFeedback('rejected'); + + setSnackbar({ + open: true, + message: 'Voice command rejected. Try again or use manual entry.', + severity: 'info', + }); + } catch (error) { + console.error('[Voice] Failed to save rejection feedback:', error); } }; @@ -366,6 +421,19 @@ export function VoiceFloatingButton() { + {/* Review Dialog */} + {showReview && classificationResult && ( + setShowReview(false)} + /> + )} + {/* Snackbar for feedback */} ; + confidence?: number; + action: 'approved' | 'edited' | 'rejected'; + finalType?: string; + finalData?: Record; + userNotes?: string; +} + +export const voiceApi = { + async saveFeedback(feedbackData: VoiceFeedbackData) { + const response = await apiClient.post('/api/v1/voice/feedback', feedbackData); + return response.data; + }, +};