Add voice command review/edit system with user feedback tracking
Some checks failed
CI/CD Pipeline / Lint and Test (push) Has been cancelled
CI/CD Pipeline / E2E Tests (push) Has been cancelled
CI/CD Pipeline / Build Application (push) Has been cancelled

Implemented complete review/edit workflow for voice commands with ML feedback collection:

**Backend:**
- Created V012 migration for voice_feedback table with user action tracking
- Added VoiceFeedback entity with approval/edit/reject actions
- Implemented voice feedback API endpoint (POST /api/v1/voice/feedback)
- Fixed user ID extraction bug (req.user.userId vs req.user.sub)

**Frontend:**
- Built VoiceActivityReview component with field-specific editors
- Integrated review dialog into voice command workflow
- Added approve/edit/reject handlers with feedback submission
- Fixed infinite loop by tracking processed classification IDs

**Features:**
- Users can review AI-extracted data before saving
- Quick-edit capabilities for all activity fields
- Feedback data stored for ML model improvement
- Activity creation only happens after user approval/edit

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-10-02 11:03:54 +00:00
parent 77f2c1d767
commit e94a1018c4
10 changed files with 743 additions and 79 deletions

View File

@@ -14,4 +14,5 @@ export {
NotificationStatus,
NotificationPriority,
} from './notification.entity';
export { Photo, PhotoType } from './photo.entity';
export { Photo, PhotoType } from './photo.entity';
export { VoiceFeedback, VoiceFeedbackAction } from './voice-feedback.entity';

View File

@@ -0,0 +1,76 @@
import { Entity, Column, PrimaryColumn, ManyToOne, JoinColumn } from 'typeorm';
import { User } from './user.entity';
import { Child } from './child.entity';
import { Activity } from './activity.entity';
export enum VoiceFeedbackAction {
APPROVED = 'approved',
EDITED = 'edited',
REJECTED = 'rejected',
}
@Entity('voice_feedback')
export class VoiceFeedback {
@PrimaryColumn({ type: 'varchar', length: 20 })
id: string;
@Column({ name: 'user_id', type: 'varchar', length: 20 })
userId: string;
@Column({ name: 'child_id', type: 'varchar', length: 20, nullable: true })
childId?: string;
@Column({ name: 'activity_id', type: 'varchar', length: 20, nullable: true })
activityId?: string;
@Column({ type: 'text' })
transcript: string;
@Column({ type: 'varchar', length: 10, nullable: true })
language?: string;
@Column({ name: 'extracted_type', type: 'varchar', length: 20 })
extractedType: string;
@Column({ name: 'extracted_data', type: 'jsonb', default: {} })
extractedData: Record<string, any>;
@Column({ type: 'decimal', precision: 3, scale: 2, nullable: true })
confidence?: number;
@Column({
type: 'varchar',
length: 20,
enum: VoiceFeedbackAction,
})
action: VoiceFeedbackAction;
@Column({ name: 'final_type', type: 'varchar', length: 20, nullable: true })
finalType?: string;
@Column({ name: 'final_data', type: 'jsonb', nullable: true })
finalData?: Record<string, any>;
@Column({ name: 'user_notes', type: 'text', nullable: true })
userNotes?: string;
@Column({
name: 'created_at',
type: 'timestamp',
default: () => 'CURRENT_TIMESTAMP',
})
createdAt: Date;
// Relations
@ManyToOne(() => User, { onDelete: 'CASCADE' })
@JoinColumn({ name: 'user_id' })
user: User;
@ManyToOne(() => Child, { nullable: true, onDelete: 'CASCADE' })
@JoinColumn({ name: 'child_id' })
child?: Child;
@ManyToOne(() => Activity, { nullable: true, onDelete: 'CASCADE' })
@JoinColumn({ name: 'activity_id' })
activity?: Activity;
}

View File

@@ -0,0 +1,43 @@
-- V012: Create voice feedback table for storing user corrections and approvals
-- This helps improve AI accuracy by learning from user edits
CREATE TABLE IF NOT EXISTS voice_feedback (
id VARCHAR(20) PRIMARY KEY,
user_id VARCHAR(20) NOT NULL REFERENCES users(id) ON DELETE CASCADE,
child_id VARCHAR(20) REFERENCES children(id) ON DELETE CASCADE,
activity_id VARCHAR(20) REFERENCES activities(id) ON DELETE CASCADE,
-- Original voice input
transcript TEXT NOT NULL,
language VARCHAR(10),
-- AI extracted data (before user edits)
extracted_type VARCHAR(20) NOT NULL,
extracted_data JSONB NOT NULL DEFAULT '{}',
confidence DECIMAL(3,2),
-- User actions
action VARCHAR(20) NOT NULL CHECK (action IN ('approved', 'edited', 'rejected')),
-- Final data (after user edits, if edited)
final_type VARCHAR(20),
final_data JSONB,
-- User feedback
user_notes TEXT,
-- Metadata
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
-- Indexes for analytics
CREATE INDEX idx_voice_feedback_user ON voice_feedback(user_id, created_at DESC);
CREATE INDEX idx_voice_feedback_type ON voice_feedback(extracted_type, action);
CREATE INDEX idx_voice_feedback_confidence ON voice_feedback(confidence, action);
CREATE INDEX idx_voice_feedback_activity ON voice_feedback(activity_id);
-- Add comment
COMMENT ON TABLE voice_feedback IS 'Stores user feedback on voice command accuracy for ML improvement';
COMMENT ON COLUMN voice_feedback.action IS 'User action: approved (no changes), edited (modified data), rejected (discarded)';
COMMENT ON COLUMN voice_feedback.extracted_data IS 'Original AI-extracted activity data before user edits';
COMMENT ON COLUMN voice_feedback.final_data IS 'User-modified data if action=edited';

View File

@@ -0,0 +1,50 @@
import { IsString, IsNotEmpty, IsOptional, IsEnum, IsObject, IsNumber, Min, Max } from 'class-validator';
import { VoiceFeedbackAction } from '../../../database/entities';
export class SaveVoiceFeedbackDto {
@IsString()
@IsOptional()
childId?: string;
@IsString()
@IsOptional()
activityId?: string;
@IsString()
@IsNotEmpty()
transcript: string;
@IsString()
@IsOptional()
language?: string;
@IsString()
@IsNotEmpty()
extractedType: string;
@IsObject()
@IsNotEmpty()
extractedData: Record<string, any>;
@IsNumber()
@Min(0)
@Max(1)
@IsOptional()
confidence?: number;
@IsEnum(VoiceFeedbackAction)
@IsNotEmpty()
action: VoiceFeedbackAction;
@IsString()
@IsOptional()
finalType?: string;
@IsObject()
@IsOptional()
finalData?: Record<string, any>;
@IsString()
@IsOptional()
userNotes?: string;
}

View File

@@ -7,10 +7,13 @@ import {
Req,
BadRequestException,
Logger,
UseGuards,
} from '@nestjs/common';
import { FileInterceptor } from '@nestjs/platform-express';
import { JwtAuthGuard } from '../auth/guards/jwt-auth.guard';
import { VoiceService } from './voice.service';
import { Public } from '../auth/decorators/public.decorator';
import { SaveVoiceFeedbackDto } from './dto/save-voice-feedback.dto';
@Controller('api/v1/voice')
export class VoiceController {
@@ -183,4 +186,22 @@ export class VoiceController {
classification: result,
};
}
@Post('feedback')
@UseGuards(JwtAuthGuard)
async saveFeedback(
@Req() req: any,
@Body() feedbackDto: SaveVoiceFeedbackDto,
) {
const userId = req.user.userId;
this.logger.log(`[Voice Feedback] User ${userId} submitting feedback: ${feedbackDto.action}`);
const feedback = await this.voiceService.saveFeedback(userId, feedbackDto);
return {
success: true,
data: feedback,
};
}
}

View File

@@ -1,8 +1,11 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { VoiceService } from './voice.service';
import { VoiceController } from './voice.controller';
import { VoiceFeedback } from '../../database/entities';
@Module({
imports: [TypeOrmModule.forFeature([VoiceFeedback])],
controllers: [VoiceController],
providers: [VoiceService],
exports: [VoiceService],

View File

@@ -1,8 +1,13 @@
import { Injectable, Logger, BadRequestException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import OpenAI from 'openai';
import * as fs from 'fs';
import * as path from 'path';
import { nanoid } from 'nanoid';
import { VoiceFeedback } from '../../database/entities';
import { SaveVoiceFeedbackDto } from './dto/save-voice-feedback.dto';
export interface TranscriptionResult {
text: string;
@@ -26,7 +31,11 @@ export class VoiceService {
// Supported languages for MVP
private readonly SUPPORTED_LANGUAGES = ['en', 'es', 'fr', 'pt', 'zh'];
constructor(private configService: ConfigService) {
constructor(
private configService: ConfigService,
@InjectRepository(VoiceFeedback)
private readonly voiceFeedbackRepository: Repository<VoiceFeedback>,
) {
// Check if Azure OpenAI is enabled
const azureEnabled = this.configService.get<boolean>('AZURE_OPENAI_ENABLED');
@@ -328,4 +337,44 @@ Respond ONLY with the question text, no formatting.`;
return 'Could you provide more details about this activity?';
}
}
/**
* Save user feedback on voice command accuracy
*/
async saveFeedback(
userId: string,
feedbackDto: SaveVoiceFeedbackDto,
): Promise<VoiceFeedback> {
try {
const feedback = this.voiceFeedbackRepository.create({
id: `vfb_${nanoid(16)}`,
userId,
childId: feedbackDto.childId,
activityId: feedbackDto.activityId,
transcript: feedbackDto.transcript,
language: feedbackDto.language,
extractedType: feedbackDto.extractedType,
extractedData: feedbackDto.extractedData,
confidence: feedbackDto.confidence,
action: feedbackDto.action,
finalType: feedbackDto.finalType,
finalData: feedbackDto.finalData,
userNotes: feedbackDto.userNotes,
});
const saved = await this.voiceFeedbackRepository.save(feedback);
this.logger.log(
`[Voice Feedback] User ${userId} ${feedbackDto.action} voice command for type: ${feedbackDto.extractedType}`,
);
return saved;
} catch (error) {
this.logger.error(
`[Voice Feedback] Failed to save: ${error.message}`,
error.stack,
);
throw new BadRequestException('Failed to save voice feedback');
}
}
}

View File

@@ -0,0 +1,331 @@
'use client';
import React, { useState } from 'react';
import {
Dialog,
DialogTitle,
DialogContent,
DialogActions,
Button,
Box,
Typography,
TextField,
Select,
MenuItem,
FormControl,
InputLabel,
Alert,
Chip,
Grid,
InputAdornment,
} from '@mui/material';
import CheckCircleIcon from '@mui/icons-material/CheckCircle';
import EditIcon from '@mui/icons-material/Edit';
import CloseIcon from '@mui/icons-material/Close';
interface VoiceActivityReviewProps {
open: boolean;
transcript: string;
classification: {
type: string;
details: Record<string, any>;
confidence: number;
timestamp?: Date;
};
onApprove: (data: any) => void;
onEdit: (editedData: any) => void;
onReject: () => void;
onClose: () => void;
}
export function VoiceActivityReview({
open,
transcript,
classification,
onApprove,
onEdit,
onReject,
onClose,
}: VoiceActivityReviewProps) {
const [isEditing, setIsEditing] = useState(false);
const [editedData, setEditedData] = useState<Record<string, any>>({});
React.useEffect(() => {
if (open) {
setEditedData(classification.details || {});
setIsEditing(false);
}
}, [open, classification]);
const handleApprove = () => {
onApprove({
type: classification.type,
details: classification.details,
timestamp: classification.timestamp,
});
};
const handleSaveEdit = () => {
onEdit({
type: classification.type,
details: editedData,
timestamp: classification.timestamp,
});
};
const renderFieldEditor = (key: string, value: any) => {
// Render different input types based on field
if (key === 'notes') {
return (
<TextField
fullWidth
multiline
rows={2}
label="Notes"
value={editedData[key] || ''}
onChange={(e) => setEditedData({ ...editedData, [key]: e.target.value })}
variant="outlined"
size="small"
/>
);
}
if (key === 'amount') {
return (
<TextField
fullWidth
type="number"
label="Amount"
value={editedData[key] || ''}
onChange={(e) => setEditedData({ ...editedData, [key]: Number(e.target.value) })}
variant="outlined"
size="small"
InputProps={{
endAdornment: <InputAdornment position="end">{editedData.unit || 'ml'}</InputAdornment>,
}}
/>
);
}
if (key === 'duration') {
return (
<TextField
fullWidth
type="number"
label="Duration"
value={editedData[key] || ''}
onChange={(e) => setEditedData({ ...editedData, [key]: Number(e.target.value) })}
variant="outlined"
size="small"
InputProps={{
endAdornment: <InputAdornment position="end">min</InputAdornment>,
}}
/>
);
}
if (key === 'feedingType') {
return (
<FormControl fullWidth size="small">
<InputLabel>Feeding Type</InputLabel>
<Select
value={editedData[key] || ''}
onChange={(e) => setEditedData({ ...editedData, [key]: e.target.value })}
label="Feeding Type"
>
<MenuItem value="bottle">Bottle</MenuItem>
<MenuItem value="breast">Breast</MenuItem>
<MenuItem value="solids">Solids</MenuItem>
</Select>
</FormControl>
);
}
if (key === 'diaperType') {
return (
<FormControl fullWidth size="small">
<InputLabel>Diaper Type</InputLabel>
<Select
value={editedData[key] || ''}
onChange={(e) => setEditedData({ ...editedData, [key]: e.target.value })}
label="Diaper Type"
>
<MenuItem value="wet">Wet</MenuItem>
<MenuItem value="dirty">Dirty</MenuItem>
<MenuItem value="both">Both</MenuItem>
</Select>
</FormControl>
);
}
if (key === 'quality') {
return (
<FormControl fullWidth size="small">
<InputLabel>Sleep Quality</InputLabel>
<Select
value={editedData[key] || ''}
onChange={(e) => setEditedData({ ...editedData, [key]: e.target.value })}
label="Sleep Quality"
>
<MenuItem value="peaceful">Peaceful</MenuItem>
<MenuItem value="restless">Restless</MenuItem>
<MenuItem value="fussy">Fussy</MenuItem>
</Select>
</FormControl>
);
}
if (key === 'location') {
return (
<FormControl fullWidth size="small">
<InputLabel>Location</InputLabel>
<Select
value={editedData[key] || ''}
onChange={(e) => setEditedData({ ...editedData, [key]: e.target.value })}
label="Location"
>
<MenuItem value="crib">Crib</MenuItem>
<MenuItem value="bassinet">Bassinet</MenuItem>
<MenuItem value="arms">Arms</MenuItem>
<MenuItem value="bed">Bed</MenuItem>
<MenuItem value="stroller">Stroller</MenuItem>
<MenuItem value="car seat">Car Seat</MenuItem>
</Select>
</FormControl>
);
}
if (key === 'side') {
return (
<FormControl fullWidth size="small">
<InputLabel>Side</InputLabel>
<Select
value={editedData[key] || ''}
onChange={(e) => setEditedData({ ...editedData, [key]: e.target.value })}
label="Side"
>
<MenuItem value="left">Left</MenuItem>
<MenuItem value="right">Right</MenuItem>
<MenuItem value="both">Both</MenuItem>
</Select>
</FormControl>
);
}
// Default text input
return (
<TextField
fullWidth
label={key.charAt(0).toUpperCase() + key.slice(1)}
value={editedData[key] || ''}
onChange={(e) => setEditedData({ ...editedData, [key]: e.target.value })}
variant="outlined"
size="small"
/>
);
};
const getActivityLabel = () => {
return classification.type.charAt(0).toUpperCase() + classification.type.slice(1);
};
const getConfidenceColor = () => {
if (classification.confidence >= 0.7) return 'success';
if (classification.confidence >= 0.4) return 'warning';
return 'error';
};
return (
<Dialog open={open} onClose={onClose} maxWidth="sm" fullWidth>
<DialogTitle>
Review Voice Command
<Chip
label={`${getActivityLabel()}`}
color="primary"
size="small"
sx={{ ml: 2 }}
/>
<Chip
label={`${Math.round(classification.confidence * 100)}% confidence`}
color={getConfidenceColor()}
size="small"
sx={{ ml: 1 }}
/>
</DialogTitle>
<DialogContent>
{/* Transcript */}
<Alert severity="info" sx={{ mb: 2 }}>
<Typography variant="body2" color="text.secondary">
<strong>You said:</strong> "{transcript}"
</Typography>
</Alert>
{/* Extracted Data */}
<Typography variant="subtitle2" gutterBottom>
{isEditing ? 'Edit Activity Details' : 'Extracted Details'}
</Typography>
<Box sx={{ mt: 2 }}>
{isEditing ? (
<Grid container spacing={2}>
{Object.entries(editedData).map(([key, value]) => (
<Grid item xs={12} sm={6} key={key}>
{renderFieldEditor(key, value)}
</Grid>
))}
</Grid>
) : (
<Box>
{Object.entries(classification.details).map(([key, value]) => (
<Box key={key} sx={{ display: 'flex', justifyContent: 'space-between', mb: 1 }}>
<Typography variant="body2" color="text.secondary">
{key.charAt(0).toUpperCase() + key.slice(1).replace(/([A-Z])/g, ' $1')}:
</Typography>
<Typography variant="body2">
{value === null || value === undefined ? '-' : String(value)}
</Typography>
</Box>
))}
</Box>
)}
</Box>
{/* Help text */}
{!isEditing && (
<Alert severity="warning" sx={{ mt: 2 }}>
<Typography variant="caption">
Please review the extracted information. Click <strong>Edit</strong> to make changes or <strong>Approve</strong> if it looks correct.
</Typography>
</Alert>
)}
</DialogContent>
<DialogActions>
<Button onClick={onReject} startIcon={<CloseIcon />} color="error">
Reject
</Button>
{isEditing ? (
<>
<Button onClick={() => setIsEditing(false)} color="inherit">
Cancel Edit
</Button>
<Button onClick={handleSaveEdit} variant="contained" startIcon={<CheckCircleIcon />}>
Save & Approve
</Button>
</>
) : (
<>
<Button onClick={() => setIsEditing(true)} startIcon={<EditIcon />} color="inherit">
Edit
</Button>
<Button onClick={handleApprove} variant="contained" startIcon={<CheckCircleIcon />} color="success">
Approve
</Button>
</>
)}
</DialogActions>
</Dialog>
);
}

View File

@@ -24,6 +24,8 @@ import { useVoiceInput } from '@/hooks/useVoiceInput';
import { useAuth } from '@/lib/auth/AuthContext';
import { trackingApi } from '@/lib/api/tracking';
import { childrenApi } from '@/lib/api/children';
import { voiceApi } from '@/lib/api/voice';
import { VoiceActivityReview } from './VoiceActivityReview';
/**
* Floating voice input button
@@ -40,6 +42,7 @@ export function VoiceFloatingButton() {
const [identifiedActivity, setIdentifiedActivity] = useState<string>('');
const [classificationResult, setClassificationResult] = useState<any>(null);
const [processedClassificationId, setProcessedClassificationId] = useState<string | null>(null);
const [showReview, setShowReview] = useState(false);
const [snackbar, setSnackbar] = useState<{
open: boolean;
message: string;
@@ -75,10 +78,23 @@ export function VoiceFloatingButton() {
// Only process if we haven't already processed this exact classification
if (classification && !isListening && !isProcessing && open && classificationId !== processedClassificationId) {
console.log('[Voice] New classification detected, processing...', classificationId);
console.log('[Voice] New classification detected, showing review...', classificationId);
setProcessedClassificationId(classificationId);
setClassificationResult(classification);
handleClassifiedIntent(classification);
// Show review dialog instead of immediately creating activity
if (classification.type !== 'unknown' && classification.confidence >= 0.3) {
setProcessingStatus(null);
setShowReview(true);
} else {
// For unknown or low confidence, show error
setProcessingStatus(null);
setSnackbar({
open: true,
message: 'Could not understand the command. Please try again or use manual entry.',
severity: 'warning',
});
}
}
}, [classification, isListening, isProcessing, open, transcript, processedClassificationId]);
@@ -122,91 +138,78 @@ export function VoiceFloatingButton() {
stopListening();
};
const handleClassifiedIntent = async (result: any) => {
console.log('[Voice] handleClassifiedIntent called with result:', result);
if (result.error) {
console.log('[Voice] Result has error:', result.message);
setProcessingStatus(null);
setSnackbar({
open: true,
message: result.message,
severity: 'error',
});
return;
}
// Support both formats: backend returns 'type', frontend local classifier returns 'intent'
const activityType = result.type || result.intent;
console.log('[Voice] Activity type:', activityType);
// Set identified activity for status display
setIdentifiedActivity(activityType);
// Handle unknown or low confidence
if (activityType === 'unknown' || (result.confidence && result.confidence < 0.3)) {
console.log('[Voice] Unknown or low confidence:', activityType, result.confidence);
setProcessingStatus(null);
setSnackbar({
open: true,
message: 'Could not understand the command. Please try again or use manual entry.',
severity: 'warning',
});
return;
}
const createActivity = async (activityType: string, activityDetails: Record<string, any>, activityTimestamp?: Date) => {
// Get the first child from the family
if (!familyId) {
console.log('[Voice] No familyId found');
setProcessingStatus(null);
setSnackbar({
open: true,
message: 'No family found. Please set up your profile first.',
severity: 'error',
});
return;
throw new Error('No family found. Please set up your profile first.');
}
console.log('[Voice] Family ID:', familyId);
// Fetch children
console.log('[Voice] Fetching children for family:', familyId);
const children = await childrenApi.getChildren(familyId);
console.log('[Voice] Children found:', children.length, children);
if (children.length === 0) {
throw new Error('No children found. Please add a child first.');
}
// Use the first child
const childId = children[0].id;
console.log('[Voice] Using child ID:', childId);
// Create the activity
const activityData = {
type: activityType,
timestamp: activityTimestamp || new Date().toISOString(),
data: activityDetails,
notes: activityDetails.notes || undefined,
};
console.log('[Voice] Creating activity with data:', JSON.stringify(activityData, null, 2));
const createdActivity = await trackingApi.createActivity(childId, activityData);
console.log('[Voice] Activity created successfully:', createdActivity);
return { childId, activity: createdActivity };
};
const saveFeedback = async (action: 'approved' | 'edited' | 'rejected', finalData?: Record<string, any>, childId?: string, activityId?: string) => {
try {
await voiceApi.saveFeedback({
childId,
activityId,
transcript,
language: classificationResult?.language || 'en',
extractedType: classificationResult.type,
extractedData: classificationResult.details,
confidence: classificationResult.confidence,
action,
finalType: action === 'edited' ? classificationResult.type : undefined,
finalData: action === 'edited' ? finalData : undefined,
});
console.log(`[Voice] Feedback saved: ${action}`);
} catch (error) {
console.error('[Voice] Failed to save feedback:', error);
// Don't throw - feedback is nice-to-have, not critical
}
};
const handleApprove = async (data: any) => {
try {
setIsProcessing(true);
setProcessingStatus('saving');
setShowReview(false);
// Fetch children
console.log('[Voice] Fetching children for family:', familyId);
const children = await childrenApi.getChildren(familyId);
console.log('[Voice] Children found:', children.length, children);
const { childId, activity } = await createActivity(data.type, data.details, data.timestamp);
if (children.length === 0) {
setSnackbar({
open: true,
message: 'No children found. Please add a child first.',
severity: 'error',
});
setIsProcessing(false);
return;
}
// Use the first child (or you could enhance this to support child name matching)
const childId = children[0].id;
console.log('[Voice] Using child ID:', childId);
// Create the activity
const activityData = {
type: activityType,
timestamp: result.timestamp || new Date().toISOString(),
data: result.details || result.structuredData || {},
notes: result.details?.notes || result.structuredData?.notes || undefined,
};
console.log('[Voice] Creating activity with data:', JSON.stringify(activityData, null, 2));
const createdActivity = await trackingApi.createActivity(childId, activityData);
console.log('[Voice] Activity created successfully:', createdActivity);
// Save feedback
await saveFeedback('approved', undefined, childId, activity.id);
// Show success message
const activityLabel = activityType.charAt(0).toUpperCase() + activityType.slice(1);
const activityLabel = data.type.charAt(0).toUpperCase() + data.type.slice(1);
setSnackbar({
open: true,
message: `${activityLabel} activity saved successfully!`,
@@ -218,16 +221,68 @@ export function VoiceFloatingButton() {
handleClose();
}, 1500);
} catch (error: any) {
console.error('[Voice] Failed to create activity - Full error:', error);
console.error('[Voice] Error response:', error.response);
console.error('[Voice] Error data:', error.response?.data);
console.error('[Voice] Failed to create activity:', error);
setSnackbar({
open: true,
message: error.response?.data?.message || 'Failed to save activity. Please try again.',
message: error.message || 'Failed to save activity. Please try again.',
severity: 'error',
});
} finally {
setIsProcessing(false);
setProcessingStatus(null);
}
};
const handleEdit = async (editedData: any) => {
try {
setIsProcessing(true);
setProcessingStatus('saving');
setShowReview(false);
const { childId, activity } = await createActivity(editedData.type, editedData.details, editedData.timestamp);
// Save feedback with edited data
await saveFeedback('edited', editedData.details, childId, activity.id);
// Show success message
const activityLabel = editedData.type.charAt(0).toUpperCase() + editedData.type.slice(1);
setSnackbar({
open: true,
message: `${activityLabel} activity saved with your edits!`,
severity: 'success',
});
// Auto-close dialog
setTimeout(() => {
handleClose();
}, 1500);
} catch (error: any) {
console.error('[Voice] Failed to create edited activity:', error);
setSnackbar({
open: true,
message: error.message || 'Failed to save activity. Please try again.',
severity: 'error',
});
} finally {
setIsProcessing(false);
setProcessingStatus(null);
}
};
const handleReject = async () => {
try {
setShowReview(false);
// Save feedback
await saveFeedback('rejected');
setSnackbar({
open: true,
message: 'Voice command rejected. Try again or use manual entry.',
severity: 'info',
});
} catch (error) {
console.error('[Voice] Failed to save rejection feedback:', error);
}
};
@@ -366,6 +421,19 @@ export function VoiceFloatingButton() {
</DialogActions>
</Dialog>
{/* Review Dialog */}
{showReview && classificationResult && (
<VoiceActivityReview
open={showReview}
transcript={transcript}
classification={classificationResult}
onApprove={handleApprove}
onEdit={handleEdit}
onReject={handleReject}
onClose={() => setShowReview(false)}
/>
)}
{/* Snackbar for feedback */}
<Snackbar
open={snackbar.open}

View File

@@ -0,0 +1,22 @@
import apiClient from './client';
export interface VoiceFeedbackData {
childId?: string;
activityId?: string;
transcript: string;
language?: string;
extractedType: string;
extractedData: Record<string, any>;
confidence?: number;
action: 'approved' | 'edited' | 'rejected';
finalType?: string;
finalData?: Record<string, any>;
userNotes?: string;
}
export const voiceApi = {
async saveFeedback(feedbackData: VoiceFeedbackData) {
const response = await apiClient.post('/api/v1/voice/feedback', feedbackData);
return response.data;
},
};