Add voice command review/edit system with user feedback tracking
Implemented complete review/edit workflow for voice commands with ML feedback collection: **Backend:** - Created V012 migration for voice_feedback table with user action tracking - Added VoiceFeedback entity with approval/edit/reject actions - Implemented voice feedback API endpoint (POST /api/v1/voice/feedback) - Fixed user ID extraction bug (req.user.userId vs req.user.sub) **Frontend:** - Built VoiceActivityReview component with field-specific editors - Integrated review dialog into voice command workflow - Added approve/edit/reject handlers with feedback submission - Fixed infinite loop by tracking processed classification IDs **Features:** - Users can review AI-extracted data before saving - Quick-edit capabilities for all activity fields - Feedback data stored for ML model improvement - Activity creation only happens after user approval/edit 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -14,4 +14,5 @@ export {
|
||||
NotificationStatus,
|
||||
NotificationPriority,
|
||||
} from './notification.entity';
|
||||
export { Photo, PhotoType } from './photo.entity';
|
||||
export { Photo, PhotoType } from './photo.entity';
|
||||
export { VoiceFeedback, VoiceFeedbackAction } from './voice-feedback.entity';
|
||||
@@ -0,0 +1,76 @@
|
||||
import { Entity, Column, PrimaryColumn, ManyToOne, JoinColumn } from 'typeorm';
|
||||
import { User } from './user.entity';
|
||||
import { Child } from './child.entity';
|
||||
import { Activity } from './activity.entity';
|
||||
|
||||
export enum VoiceFeedbackAction {
|
||||
APPROVED = 'approved',
|
||||
EDITED = 'edited',
|
||||
REJECTED = 'rejected',
|
||||
}
|
||||
|
||||
@Entity('voice_feedback')
|
||||
export class VoiceFeedback {
|
||||
@PrimaryColumn({ type: 'varchar', length: 20 })
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'user_id', type: 'varchar', length: 20 })
|
||||
userId: string;
|
||||
|
||||
@Column({ name: 'child_id', type: 'varchar', length: 20, nullable: true })
|
||||
childId?: string;
|
||||
|
||||
@Column({ name: 'activity_id', type: 'varchar', length: 20, nullable: true })
|
||||
activityId?: string;
|
||||
|
||||
@Column({ type: 'text' })
|
||||
transcript: string;
|
||||
|
||||
@Column({ type: 'varchar', length: 10, nullable: true })
|
||||
language?: string;
|
||||
|
||||
@Column({ name: 'extracted_type', type: 'varchar', length: 20 })
|
||||
extractedType: string;
|
||||
|
||||
@Column({ name: 'extracted_data', type: 'jsonb', default: {} })
|
||||
extractedData: Record<string, any>;
|
||||
|
||||
@Column({ type: 'decimal', precision: 3, scale: 2, nullable: true })
|
||||
confidence?: number;
|
||||
|
||||
@Column({
|
||||
type: 'varchar',
|
||||
length: 20,
|
||||
enum: VoiceFeedbackAction,
|
||||
})
|
||||
action: VoiceFeedbackAction;
|
||||
|
||||
@Column({ name: 'final_type', type: 'varchar', length: 20, nullable: true })
|
||||
finalType?: string;
|
||||
|
||||
@Column({ name: 'final_data', type: 'jsonb', nullable: true })
|
||||
finalData?: Record<string, any>;
|
||||
|
||||
@Column({ name: 'user_notes', type: 'text', nullable: true })
|
||||
userNotes?: string;
|
||||
|
||||
@Column({
|
||||
name: 'created_at',
|
||||
type: 'timestamp',
|
||||
default: () => 'CURRENT_TIMESTAMP',
|
||||
})
|
||||
createdAt: Date;
|
||||
|
||||
// Relations
|
||||
@ManyToOne(() => User, { onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'user_id' })
|
||||
user: User;
|
||||
|
||||
@ManyToOne(() => Child, { nullable: true, onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'child_id' })
|
||||
child?: Child;
|
||||
|
||||
@ManyToOne(() => Activity, { nullable: true, onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'activity_id' })
|
||||
activity?: Activity;
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
-- V012: Create voice feedback table for storing user corrections and approvals
|
||||
-- This helps improve AI accuracy by learning from user edits
|
||||
|
||||
CREATE TABLE IF NOT EXISTS voice_feedback (
|
||||
id VARCHAR(20) PRIMARY KEY,
|
||||
user_id VARCHAR(20) NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
child_id VARCHAR(20) REFERENCES children(id) ON DELETE CASCADE,
|
||||
activity_id VARCHAR(20) REFERENCES activities(id) ON DELETE CASCADE,
|
||||
|
||||
-- Original voice input
|
||||
transcript TEXT NOT NULL,
|
||||
language VARCHAR(10),
|
||||
|
||||
-- AI extracted data (before user edits)
|
||||
extracted_type VARCHAR(20) NOT NULL,
|
||||
extracted_data JSONB NOT NULL DEFAULT '{}',
|
||||
confidence DECIMAL(3,2),
|
||||
|
||||
-- User actions
|
||||
action VARCHAR(20) NOT NULL CHECK (action IN ('approved', 'edited', 'rejected')),
|
||||
|
||||
-- Final data (after user edits, if edited)
|
||||
final_type VARCHAR(20),
|
||||
final_data JSONB,
|
||||
|
||||
-- User feedback
|
||||
user_notes TEXT,
|
||||
|
||||
-- Metadata
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Indexes for analytics
|
||||
CREATE INDEX idx_voice_feedback_user ON voice_feedback(user_id, created_at DESC);
|
||||
CREATE INDEX idx_voice_feedback_type ON voice_feedback(extracted_type, action);
|
||||
CREATE INDEX idx_voice_feedback_confidence ON voice_feedback(confidence, action);
|
||||
CREATE INDEX idx_voice_feedback_activity ON voice_feedback(activity_id);
|
||||
|
||||
-- Add comment
|
||||
COMMENT ON TABLE voice_feedback IS 'Stores user feedback on voice command accuracy for ML improvement';
|
||||
COMMENT ON COLUMN voice_feedback.action IS 'User action: approved (no changes), edited (modified data), rejected (discarded)';
|
||||
COMMENT ON COLUMN voice_feedback.extracted_data IS 'Original AI-extracted activity data before user edits';
|
||||
COMMENT ON COLUMN voice_feedback.final_data IS 'User-modified data if action=edited';
|
||||
@@ -0,0 +1,50 @@
|
||||
import { IsString, IsNotEmpty, IsOptional, IsEnum, IsObject, IsNumber, Min, Max } from 'class-validator';
|
||||
import { VoiceFeedbackAction } from '../../../database/entities';
|
||||
|
||||
export class SaveVoiceFeedbackDto {
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
childId?: string;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
activityId?: string;
|
||||
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
transcript: string;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
language?: string;
|
||||
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
extractedType: string;
|
||||
|
||||
@IsObject()
|
||||
@IsNotEmpty()
|
||||
extractedData: Record<string, any>;
|
||||
|
||||
@IsNumber()
|
||||
@Min(0)
|
||||
@Max(1)
|
||||
@IsOptional()
|
||||
confidence?: number;
|
||||
|
||||
@IsEnum(VoiceFeedbackAction)
|
||||
@IsNotEmpty()
|
||||
action: VoiceFeedbackAction;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
finalType?: string;
|
||||
|
||||
@IsObject()
|
||||
@IsOptional()
|
||||
finalData?: Record<string, any>;
|
||||
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
userNotes?: string;
|
||||
}
|
||||
@@ -7,10 +7,13 @@ import {
|
||||
Req,
|
||||
BadRequestException,
|
||||
Logger,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { FileInterceptor } from '@nestjs/platform-express';
|
||||
import { JwtAuthGuard } from '../auth/guards/jwt-auth.guard';
|
||||
import { VoiceService } from './voice.service';
|
||||
import { Public } from '../auth/decorators/public.decorator';
|
||||
import { SaveVoiceFeedbackDto } from './dto/save-voice-feedback.dto';
|
||||
|
||||
@Controller('api/v1/voice')
|
||||
export class VoiceController {
|
||||
@@ -183,4 +186,22 @@ export class VoiceController {
|
||||
classification: result,
|
||||
};
|
||||
}
|
||||
|
||||
@Post('feedback')
|
||||
@UseGuards(JwtAuthGuard)
|
||||
async saveFeedback(
|
||||
@Req() req: any,
|
||||
@Body() feedbackDto: SaveVoiceFeedbackDto,
|
||||
) {
|
||||
const userId = req.user.userId;
|
||||
|
||||
this.logger.log(`[Voice Feedback] User ${userId} submitting feedback: ${feedbackDto.action}`);
|
||||
|
||||
const feedback = await this.voiceService.saveFeedback(userId, feedbackDto);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: feedback,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { VoiceService } from './voice.service';
|
||||
import { VoiceController } from './voice.controller';
|
||||
import { VoiceFeedback } from '../../database/entities';
|
||||
|
||||
@Module({
|
||||
imports: [TypeOrmModule.forFeature([VoiceFeedback])],
|
||||
controllers: [VoiceController],
|
||||
providers: [VoiceService],
|
||||
exports: [VoiceService],
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
import { Injectable, Logger, BadRequestException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import OpenAI from 'openai';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { nanoid } from 'nanoid';
|
||||
import { VoiceFeedback } from '../../database/entities';
|
||||
import { SaveVoiceFeedbackDto } from './dto/save-voice-feedback.dto';
|
||||
|
||||
export interface TranscriptionResult {
|
||||
text: string;
|
||||
@@ -26,7 +31,11 @@ export class VoiceService {
|
||||
// Supported languages for MVP
|
||||
private readonly SUPPORTED_LANGUAGES = ['en', 'es', 'fr', 'pt', 'zh'];
|
||||
|
||||
constructor(private configService: ConfigService) {
|
||||
constructor(
|
||||
private configService: ConfigService,
|
||||
@InjectRepository(VoiceFeedback)
|
||||
private readonly voiceFeedbackRepository: Repository<VoiceFeedback>,
|
||||
) {
|
||||
// Check if Azure OpenAI is enabled
|
||||
const azureEnabled = this.configService.get<boolean>('AZURE_OPENAI_ENABLED');
|
||||
|
||||
@@ -328,4 +337,44 @@ Respond ONLY with the question text, no formatting.`;
|
||||
return 'Could you provide more details about this activity?';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save user feedback on voice command accuracy
|
||||
*/
|
||||
async saveFeedback(
|
||||
userId: string,
|
||||
feedbackDto: SaveVoiceFeedbackDto,
|
||||
): Promise<VoiceFeedback> {
|
||||
try {
|
||||
const feedback = this.voiceFeedbackRepository.create({
|
||||
id: `vfb_${nanoid(16)}`,
|
||||
userId,
|
||||
childId: feedbackDto.childId,
|
||||
activityId: feedbackDto.activityId,
|
||||
transcript: feedbackDto.transcript,
|
||||
language: feedbackDto.language,
|
||||
extractedType: feedbackDto.extractedType,
|
||||
extractedData: feedbackDto.extractedData,
|
||||
confidence: feedbackDto.confidence,
|
||||
action: feedbackDto.action,
|
||||
finalType: feedbackDto.finalType,
|
||||
finalData: feedbackDto.finalData,
|
||||
userNotes: feedbackDto.userNotes,
|
||||
});
|
||||
|
||||
const saved = await this.voiceFeedbackRepository.save(feedback);
|
||||
|
||||
this.logger.log(
|
||||
`[Voice Feedback] User ${userId} ${feedbackDto.action} voice command for type: ${feedbackDto.extractedType}`,
|
||||
);
|
||||
|
||||
return saved;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`[Voice Feedback] Failed to save: ${error.message}`,
|
||||
error.stack,
|
||||
);
|
||||
throw new BadRequestException('Failed to save voice feedback');
|
||||
}
|
||||
}
|
||||
}
|
||||
331
maternal-web/components/voice/VoiceActivityReview.tsx
Normal file
331
maternal-web/components/voice/VoiceActivityReview.tsx
Normal file
@@ -0,0 +1,331 @@
|
||||
'use client';
|
||||
|
||||
import React, { useState } from 'react';
|
||||
import {
|
||||
Dialog,
|
||||
DialogTitle,
|
||||
DialogContent,
|
||||
DialogActions,
|
||||
Button,
|
||||
Box,
|
||||
Typography,
|
||||
TextField,
|
||||
Select,
|
||||
MenuItem,
|
||||
FormControl,
|
||||
InputLabel,
|
||||
Alert,
|
||||
Chip,
|
||||
Grid,
|
||||
InputAdornment,
|
||||
} from '@mui/material';
|
||||
import CheckCircleIcon from '@mui/icons-material/CheckCircle';
|
||||
import EditIcon from '@mui/icons-material/Edit';
|
||||
import CloseIcon from '@mui/icons-material/Close';
|
||||
|
||||
interface VoiceActivityReviewProps {
|
||||
open: boolean;
|
||||
transcript: string;
|
||||
classification: {
|
||||
type: string;
|
||||
details: Record<string, any>;
|
||||
confidence: number;
|
||||
timestamp?: Date;
|
||||
};
|
||||
onApprove: (data: any) => void;
|
||||
onEdit: (editedData: any) => void;
|
||||
onReject: () => void;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
export function VoiceActivityReview({
|
||||
open,
|
||||
transcript,
|
||||
classification,
|
||||
onApprove,
|
||||
onEdit,
|
||||
onReject,
|
||||
onClose,
|
||||
}: VoiceActivityReviewProps) {
|
||||
const [isEditing, setIsEditing] = useState(false);
|
||||
const [editedData, setEditedData] = useState<Record<string, any>>({});
|
||||
|
||||
React.useEffect(() => {
|
||||
if (open) {
|
||||
setEditedData(classification.details || {});
|
||||
setIsEditing(false);
|
||||
}
|
||||
}, [open, classification]);
|
||||
|
||||
const handleApprove = () => {
|
||||
onApprove({
|
||||
type: classification.type,
|
||||
details: classification.details,
|
||||
timestamp: classification.timestamp,
|
||||
});
|
||||
};
|
||||
|
||||
const handleSaveEdit = () => {
|
||||
onEdit({
|
||||
type: classification.type,
|
||||
details: editedData,
|
||||
timestamp: classification.timestamp,
|
||||
});
|
||||
};
|
||||
|
||||
const renderFieldEditor = (key: string, value: any) => {
|
||||
// Render different input types based on field
|
||||
if (key === 'notes') {
|
||||
return (
|
||||
<TextField
|
||||
fullWidth
|
||||
multiline
|
||||
rows={2}
|
||||
label="Notes"
|
||||
value={editedData[key] || ''}
|
||||
onChange={(e) => setEditedData({ ...editedData, [key]: e.target.value })}
|
||||
variant="outlined"
|
||||
size="small"
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (key === 'amount') {
|
||||
return (
|
||||
<TextField
|
||||
fullWidth
|
||||
type="number"
|
||||
label="Amount"
|
||||
value={editedData[key] || ''}
|
||||
onChange={(e) => setEditedData({ ...editedData, [key]: Number(e.target.value) })}
|
||||
variant="outlined"
|
||||
size="small"
|
||||
InputProps={{
|
||||
endAdornment: <InputAdornment position="end">{editedData.unit || 'ml'}</InputAdornment>,
|
||||
}}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (key === 'duration') {
|
||||
return (
|
||||
<TextField
|
||||
fullWidth
|
||||
type="number"
|
||||
label="Duration"
|
||||
value={editedData[key] || ''}
|
||||
onChange={(e) => setEditedData({ ...editedData, [key]: Number(e.target.value) })}
|
||||
variant="outlined"
|
||||
size="small"
|
||||
InputProps={{
|
||||
endAdornment: <InputAdornment position="end">min</InputAdornment>,
|
||||
}}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (key === 'feedingType') {
|
||||
return (
|
||||
<FormControl fullWidth size="small">
|
||||
<InputLabel>Feeding Type</InputLabel>
|
||||
<Select
|
||||
value={editedData[key] || ''}
|
||||
onChange={(e) => setEditedData({ ...editedData, [key]: e.target.value })}
|
||||
label="Feeding Type"
|
||||
>
|
||||
<MenuItem value="bottle">Bottle</MenuItem>
|
||||
<MenuItem value="breast">Breast</MenuItem>
|
||||
<MenuItem value="solids">Solids</MenuItem>
|
||||
</Select>
|
||||
</FormControl>
|
||||
);
|
||||
}
|
||||
|
||||
if (key === 'diaperType') {
|
||||
return (
|
||||
<FormControl fullWidth size="small">
|
||||
<InputLabel>Diaper Type</InputLabel>
|
||||
<Select
|
||||
value={editedData[key] || ''}
|
||||
onChange={(e) => setEditedData({ ...editedData, [key]: e.target.value })}
|
||||
label="Diaper Type"
|
||||
>
|
||||
<MenuItem value="wet">Wet</MenuItem>
|
||||
<MenuItem value="dirty">Dirty</MenuItem>
|
||||
<MenuItem value="both">Both</MenuItem>
|
||||
</Select>
|
||||
</FormControl>
|
||||
);
|
||||
}
|
||||
|
||||
if (key === 'quality') {
|
||||
return (
|
||||
<FormControl fullWidth size="small">
|
||||
<InputLabel>Sleep Quality</InputLabel>
|
||||
<Select
|
||||
value={editedData[key] || ''}
|
||||
onChange={(e) => setEditedData({ ...editedData, [key]: e.target.value })}
|
||||
label="Sleep Quality"
|
||||
>
|
||||
<MenuItem value="peaceful">Peaceful</MenuItem>
|
||||
<MenuItem value="restless">Restless</MenuItem>
|
||||
<MenuItem value="fussy">Fussy</MenuItem>
|
||||
</Select>
|
||||
</FormControl>
|
||||
);
|
||||
}
|
||||
|
||||
if (key === 'location') {
|
||||
return (
|
||||
<FormControl fullWidth size="small">
|
||||
<InputLabel>Location</InputLabel>
|
||||
<Select
|
||||
value={editedData[key] || ''}
|
||||
onChange={(e) => setEditedData({ ...editedData, [key]: e.target.value })}
|
||||
label="Location"
|
||||
>
|
||||
<MenuItem value="crib">Crib</MenuItem>
|
||||
<MenuItem value="bassinet">Bassinet</MenuItem>
|
||||
<MenuItem value="arms">Arms</MenuItem>
|
||||
<MenuItem value="bed">Bed</MenuItem>
|
||||
<MenuItem value="stroller">Stroller</MenuItem>
|
||||
<MenuItem value="car seat">Car Seat</MenuItem>
|
||||
</Select>
|
||||
</FormControl>
|
||||
);
|
||||
}
|
||||
|
||||
if (key === 'side') {
|
||||
return (
|
||||
<FormControl fullWidth size="small">
|
||||
<InputLabel>Side</InputLabel>
|
||||
<Select
|
||||
value={editedData[key] || ''}
|
||||
onChange={(e) => setEditedData({ ...editedData, [key]: e.target.value })}
|
||||
label="Side"
|
||||
>
|
||||
<MenuItem value="left">Left</MenuItem>
|
||||
<MenuItem value="right">Right</MenuItem>
|
||||
<MenuItem value="both">Both</MenuItem>
|
||||
</Select>
|
||||
</FormControl>
|
||||
);
|
||||
}
|
||||
|
||||
// Default text input
|
||||
return (
|
||||
<TextField
|
||||
fullWidth
|
||||
label={key.charAt(0).toUpperCase() + key.slice(1)}
|
||||
value={editedData[key] || ''}
|
||||
onChange={(e) => setEditedData({ ...editedData, [key]: e.target.value })}
|
||||
variant="outlined"
|
||||
size="small"
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
const getActivityLabel = () => {
|
||||
return classification.type.charAt(0).toUpperCase() + classification.type.slice(1);
|
||||
};
|
||||
|
||||
const getConfidenceColor = () => {
|
||||
if (classification.confidence >= 0.7) return 'success';
|
||||
if (classification.confidence >= 0.4) return 'warning';
|
||||
return 'error';
|
||||
};
|
||||
|
||||
return (
|
||||
<Dialog open={open} onClose={onClose} maxWidth="sm" fullWidth>
|
||||
<DialogTitle>
|
||||
Review Voice Command
|
||||
<Chip
|
||||
label={`${getActivityLabel()}`}
|
||||
color="primary"
|
||||
size="small"
|
||||
sx={{ ml: 2 }}
|
||||
/>
|
||||
<Chip
|
||||
label={`${Math.round(classification.confidence * 100)}% confidence`}
|
||||
color={getConfidenceColor()}
|
||||
size="small"
|
||||
sx={{ ml: 1 }}
|
||||
/>
|
||||
</DialogTitle>
|
||||
|
||||
<DialogContent>
|
||||
{/* Transcript */}
|
||||
<Alert severity="info" sx={{ mb: 2 }}>
|
||||
<Typography variant="body2" color="text.secondary">
|
||||
<strong>You said:</strong> "{transcript}"
|
||||
</Typography>
|
||||
</Alert>
|
||||
|
||||
{/* Extracted Data */}
|
||||
<Typography variant="subtitle2" gutterBottom>
|
||||
{isEditing ? 'Edit Activity Details' : 'Extracted Details'}
|
||||
</Typography>
|
||||
|
||||
<Box sx={{ mt: 2 }}>
|
||||
{isEditing ? (
|
||||
<Grid container spacing={2}>
|
||||
{Object.entries(editedData).map(([key, value]) => (
|
||||
<Grid item xs={12} sm={6} key={key}>
|
||||
{renderFieldEditor(key, value)}
|
||||
</Grid>
|
||||
))}
|
||||
</Grid>
|
||||
) : (
|
||||
<Box>
|
||||
{Object.entries(classification.details).map(([key, value]) => (
|
||||
<Box key={key} sx={{ display: 'flex', justifyContent: 'space-between', mb: 1 }}>
|
||||
<Typography variant="body2" color="text.secondary">
|
||||
{key.charAt(0).toUpperCase() + key.slice(1).replace(/([A-Z])/g, ' $1')}:
|
||||
</Typography>
|
||||
<Typography variant="body2">
|
||||
{value === null || value === undefined ? '-' : String(value)}
|
||||
</Typography>
|
||||
</Box>
|
||||
))}
|
||||
</Box>
|
||||
)}
|
||||
</Box>
|
||||
|
||||
{/* Help text */}
|
||||
{!isEditing && (
|
||||
<Alert severity="warning" sx={{ mt: 2 }}>
|
||||
<Typography variant="caption">
|
||||
Please review the extracted information. Click <strong>Edit</strong> to make changes or <strong>Approve</strong> if it looks correct.
|
||||
</Typography>
|
||||
</Alert>
|
||||
)}
|
||||
</DialogContent>
|
||||
|
||||
<DialogActions>
|
||||
<Button onClick={onReject} startIcon={<CloseIcon />} color="error">
|
||||
Reject
|
||||
</Button>
|
||||
|
||||
{isEditing ? (
|
||||
<>
|
||||
<Button onClick={() => setIsEditing(false)} color="inherit">
|
||||
Cancel Edit
|
||||
</Button>
|
||||
<Button onClick={handleSaveEdit} variant="contained" startIcon={<CheckCircleIcon />}>
|
||||
Save & Approve
|
||||
</Button>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Button onClick={() => setIsEditing(true)} startIcon={<EditIcon />} color="inherit">
|
||||
Edit
|
||||
</Button>
|
||||
<Button onClick={handleApprove} variant="contained" startIcon={<CheckCircleIcon />} color="success">
|
||||
Approve
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
</DialogActions>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -24,6 +24,8 @@ import { useVoiceInput } from '@/hooks/useVoiceInput';
|
||||
import { useAuth } from '@/lib/auth/AuthContext';
|
||||
import { trackingApi } from '@/lib/api/tracking';
|
||||
import { childrenApi } from '@/lib/api/children';
|
||||
import { voiceApi } from '@/lib/api/voice';
|
||||
import { VoiceActivityReview } from './VoiceActivityReview';
|
||||
|
||||
/**
|
||||
* Floating voice input button
|
||||
@@ -40,6 +42,7 @@ export function VoiceFloatingButton() {
|
||||
const [identifiedActivity, setIdentifiedActivity] = useState<string>('');
|
||||
const [classificationResult, setClassificationResult] = useState<any>(null);
|
||||
const [processedClassificationId, setProcessedClassificationId] = useState<string | null>(null);
|
||||
const [showReview, setShowReview] = useState(false);
|
||||
const [snackbar, setSnackbar] = useState<{
|
||||
open: boolean;
|
||||
message: string;
|
||||
@@ -75,10 +78,23 @@ export function VoiceFloatingButton() {
|
||||
|
||||
// Only process if we haven't already processed this exact classification
|
||||
if (classification && !isListening && !isProcessing && open && classificationId !== processedClassificationId) {
|
||||
console.log('[Voice] New classification detected, processing...', classificationId);
|
||||
console.log('[Voice] New classification detected, showing review...', classificationId);
|
||||
setProcessedClassificationId(classificationId);
|
||||
setClassificationResult(classification);
|
||||
handleClassifiedIntent(classification);
|
||||
|
||||
// Show review dialog instead of immediately creating activity
|
||||
if (classification.type !== 'unknown' && classification.confidence >= 0.3) {
|
||||
setProcessingStatus(null);
|
||||
setShowReview(true);
|
||||
} else {
|
||||
// For unknown or low confidence, show error
|
||||
setProcessingStatus(null);
|
||||
setSnackbar({
|
||||
open: true,
|
||||
message: 'Could not understand the command. Please try again or use manual entry.',
|
||||
severity: 'warning',
|
||||
});
|
||||
}
|
||||
}
|
||||
}, [classification, isListening, isProcessing, open, transcript, processedClassificationId]);
|
||||
|
||||
@@ -122,91 +138,78 @@ export function VoiceFloatingButton() {
|
||||
stopListening();
|
||||
};
|
||||
|
||||
const handleClassifiedIntent = async (result: any) => {
|
||||
console.log('[Voice] handleClassifiedIntent called with result:', result);
|
||||
|
||||
if (result.error) {
|
||||
console.log('[Voice] Result has error:', result.message);
|
||||
setProcessingStatus(null);
|
||||
setSnackbar({
|
||||
open: true,
|
||||
message: result.message,
|
||||
severity: 'error',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Support both formats: backend returns 'type', frontend local classifier returns 'intent'
|
||||
const activityType = result.type || result.intent;
|
||||
console.log('[Voice] Activity type:', activityType);
|
||||
|
||||
// Set identified activity for status display
|
||||
setIdentifiedActivity(activityType);
|
||||
|
||||
// Handle unknown or low confidence
|
||||
if (activityType === 'unknown' || (result.confidence && result.confidence < 0.3)) {
|
||||
console.log('[Voice] Unknown or low confidence:', activityType, result.confidence);
|
||||
setProcessingStatus(null);
|
||||
setSnackbar({
|
||||
open: true,
|
||||
message: 'Could not understand the command. Please try again or use manual entry.',
|
||||
severity: 'warning',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const createActivity = async (activityType: string, activityDetails: Record<string, any>, activityTimestamp?: Date) => {
|
||||
// Get the first child from the family
|
||||
if (!familyId) {
|
||||
console.log('[Voice] No familyId found');
|
||||
setProcessingStatus(null);
|
||||
setSnackbar({
|
||||
open: true,
|
||||
message: 'No family found. Please set up your profile first.',
|
||||
severity: 'error',
|
||||
});
|
||||
return;
|
||||
throw new Error('No family found. Please set up your profile first.');
|
||||
}
|
||||
|
||||
console.log('[Voice] Family ID:', familyId);
|
||||
|
||||
// Fetch children
|
||||
console.log('[Voice] Fetching children for family:', familyId);
|
||||
const children = await childrenApi.getChildren(familyId);
|
||||
console.log('[Voice] Children found:', children.length, children);
|
||||
|
||||
if (children.length === 0) {
|
||||
throw new Error('No children found. Please add a child first.');
|
||||
}
|
||||
|
||||
// Use the first child
|
||||
const childId = children[0].id;
|
||||
console.log('[Voice] Using child ID:', childId);
|
||||
|
||||
// Create the activity
|
||||
const activityData = {
|
||||
type: activityType,
|
||||
timestamp: activityTimestamp || new Date().toISOString(),
|
||||
data: activityDetails,
|
||||
notes: activityDetails.notes || undefined,
|
||||
};
|
||||
|
||||
console.log('[Voice] Creating activity with data:', JSON.stringify(activityData, null, 2));
|
||||
|
||||
const createdActivity = await trackingApi.createActivity(childId, activityData);
|
||||
console.log('[Voice] Activity created successfully:', createdActivity);
|
||||
|
||||
return { childId, activity: createdActivity };
|
||||
};
|
||||
|
||||
const saveFeedback = async (action: 'approved' | 'edited' | 'rejected', finalData?: Record<string, any>, childId?: string, activityId?: string) => {
|
||||
try {
|
||||
await voiceApi.saveFeedback({
|
||||
childId,
|
||||
activityId,
|
||||
transcript,
|
||||
language: classificationResult?.language || 'en',
|
||||
extractedType: classificationResult.type,
|
||||
extractedData: classificationResult.details,
|
||||
confidence: classificationResult.confidence,
|
||||
action,
|
||||
finalType: action === 'edited' ? classificationResult.type : undefined,
|
||||
finalData: action === 'edited' ? finalData : undefined,
|
||||
});
|
||||
console.log(`[Voice] Feedback saved: ${action}`);
|
||||
} catch (error) {
|
||||
console.error('[Voice] Failed to save feedback:', error);
|
||||
// Don't throw - feedback is nice-to-have, not critical
|
||||
}
|
||||
};
|
||||
|
||||
const handleApprove = async (data: any) => {
|
||||
try {
|
||||
setIsProcessing(true);
|
||||
setProcessingStatus('saving');
|
||||
setShowReview(false);
|
||||
|
||||
// Fetch children
|
||||
console.log('[Voice] Fetching children for family:', familyId);
|
||||
const children = await childrenApi.getChildren(familyId);
|
||||
console.log('[Voice] Children found:', children.length, children);
|
||||
const { childId, activity } = await createActivity(data.type, data.details, data.timestamp);
|
||||
|
||||
if (children.length === 0) {
|
||||
setSnackbar({
|
||||
open: true,
|
||||
message: 'No children found. Please add a child first.',
|
||||
severity: 'error',
|
||||
});
|
||||
setIsProcessing(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Use the first child (or you could enhance this to support child name matching)
|
||||
const childId = children[0].id;
|
||||
console.log('[Voice] Using child ID:', childId);
|
||||
|
||||
// Create the activity
|
||||
const activityData = {
|
||||
type: activityType,
|
||||
timestamp: result.timestamp || new Date().toISOString(),
|
||||
data: result.details || result.structuredData || {},
|
||||
notes: result.details?.notes || result.structuredData?.notes || undefined,
|
||||
};
|
||||
|
||||
console.log('[Voice] Creating activity with data:', JSON.stringify(activityData, null, 2));
|
||||
|
||||
const createdActivity = await trackingApi.createActivity(childId, activityData);
|
||||
console.log('[Voice] Activity created successfully:', createdActivity);
|
||||
// Save feedback
|
||||
await saveFeedback('approved', undefined, childId, activity.id);
|
||||
|
||||
// Show success message
|
||||
const activityLabel = activityType.charAt(0).toUpperCase() + activityType.slice(1);
|
||||
const activityLabel = data.type.charAt(0).toUpperCase() + data.type.slice(1);
|
||||
setSnackbar({
|
||||
open: true,
|
||||
message: `${activityLabel} activity saved successfully!`,
|
||||
@@ -218,16 +221,68 @@ export function VoiceFloatingButton() {
|
||||
handleClose();
|
||||
}, 1500);
|
||||
} catch (error: any) {
|
||||
console.error('[Voice] Failed to create activity - Full error:', error);
|
||||
console.error('[Voice] Error response:', error.response);
|
||||
console.error('[Voice] Error data:', error.response?.data);
|
||||
console.error('[Voice] Failed to create activity:', error);
|
||||
setSnackbar({
|
||||
open: true,
|
||||
message: error.response?.data?.message || 'Failed to save activity. Please try again.',
|
||||
message: error.message || 'Failed to save activity. Please try again.',
|
||||
severity: 'error',
|
||||
});
|
||||
} finally {
|
||||
setIsProcessing(false);
|
||||
setProcessingStatus(null);
|
||||
}
|
||||
};
|
||||
|
||||
const handleEdit = async (editedData: any) => {
|
||||
try {
|
||||
setIsProcessing(true);
|
||||
setProcessingStatus('saving');
|
||||
setShowReview(false);
|
||||
|
||||
const { childId, activity } = await createActivity(editedData.type, editedData.details, editedData.timestamp);
|
||||
|
||||
// Save feedback with edited data
|
||||
await saveFeedback('edited', editedData.details, childId, activity.id);
|
||||
|
||||
// Show success message
|
||||
const activityLabel = editedData.type.charAt(0).toUpperCase() + editedData.type.slice(1);
|
||||
setSnackbar({
|
||||
open: true,
|
||||
message: `${activityLabel} activity saved with your edits!`,
|
||||
severity: 'success',
|
||||
});
|
||||
|
||||
// Auto-close dialog
|
||||
setTimeout(() => {
|
||||
handleClose();
|
||||
}, 1500);
|
||||
} catch (error: any) {
|
||||
console.error('[Voice] Failed to create edited activity:', error);
|
||||
setSnackbar({
|
||||
open: true,
|
||||
message: error.message || 'Failed to save activity. Please try again.',
|
||||
severity: 'error',
|
||||
});
|
||||
} finally {
|
||||
setIsProcessing(false);
|
||||
setProcessingStatus(null);
|
||||
}
|
||||
};
|
||||
|
||||
const handleReject = async () => {
|
||||
try {
|
||||
setShowReview(false);
|
||||
|
||||
// Save feedback
|
||||
await saveFeedback('rejected');
|
||||
|
||||
setSnackbar({
|
||||
open: true,
|
||||
message: 'Voice command rejected. Try again or use manual entry.',
|
||||
severity: 'info',
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[Voice] Failed to save rejection feedback:', error);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -366,6 +421,19 @@ export function VoiceFloatingButton() {
|
||||
</DialogActions>
|
||||
</Dialog>
|
||||
|
||||
{/* Review Dialog */}
|
||||
{showReview && classificationResult && (
|
||||
<VoiceActivityReview
|
||||
open={showReview}
|
||||
transcript={transcript}
|
||||
classification={classificationResult}
|
||||
onApprove={handleApprove}
|
||||
onEdit={handleEdit}
|
||||
onReject={handleReject}
|
||||
onClose={() => setShowReview(false)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Snackbar for feedback */}
|
||||
<Snackbar
|
||||
open={snackbar.open}
|
||||
|
||||
22
maternal-web/lib/api/voice.ts
Normal file
22
maternal-web/lib/api/voice.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import apiClient from './client';
|
||||
|
||||
export interface VoiceFeedbackData {
|
||||
childId?: string;
|
||||
activityId?: string;
|
||||
transcript: string;
|
||||
language?: string;
|
||||
extractedType: string;
|
||||
extractedData: Record<string, any>;
|
||||
confidence?: number;
|
||||
action: 'approved' | 'edited' | 'rejected';
|
||||
finalType?: string;
|
||||
finalData?: Record<string, any>;
|
||||
userNotes?: string;
|
||||
}
|
||||
|
||||
export const voiceApi = {
|
||||
async saveFeedback(feedbackData: VoiceFeedbackData) {
|
||||
const response = await apiClient.post('/api/v1/voice/feedback', feedbackData);
|
||||
return response.data;
|
||||
},
|
||||
};
|
||||
Reference in New Issue
Block a user