From 4b8828fdad9a1ab2cb21d001c10a4fbbff3b53af Mon Sep 17 00:00:00 2001 From: Andrei Date: Thu, 2 Oct 2025 08:08:31 +0000 Subject: [PATCH] Voice commands now create activities directly via API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace navigation to pre-filled forms with direct API activity creation - Fetch children from family and use first child (can be enhanced for name matching) - Show success/error messages with proper feedback - Auto-close dialog after successful save - Add test endpoint /api/v1/voice/test-classify for easy testing 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../src/modules/voice/voice.controller.ts | 33 ++++ .../components/voice/VoiceFloatingButton.tsx | 88 ++++++++--- test-voice-commands.js | 148 ++++++++++++++++++ test-voice-commands.sh | 139 ++++++++++++++++ 4 files changed, 383 insertions(+), 25 deletions(-) create mode 100755 test-voice-commands.js create mode 100755 test-voice-commands.sh diff --git a/maternal-app/maternal-app-backend/src/modules/voice/voice.controller.ts b/maternal-app/maternal-app-backend/src/modules/voice/voice.controller.ts index ca4b2c7..dae493c 100644 --- a/maternal-app/maternal-app-backend/src/modules/voice/voice.controller.ts +++ b/maternal-app/maternal-app-backend/src/modules/voice/voice.controller.ts @@ -10,6 +10,7 @@ import { } from '@nestjs/common'; import { FileInterceptor } from '@nestjs/platform-express'; import { VoiceService } from './voice.service'; +import { Public } from '../auth/decorators/public.decorator'; @Controller('api/v1/voice') export class VoiceController { @@ -150,4 +151,36 @@ export class VoiceController { }, }; } + + /** + * Test endpoint for voice classification (public, for development/testing only) + * IMPORTANT: Remove @Public() decorator in production + */ + @Public() + @Post('test-classify') + async testClassify( + @Body('text') text: string, + @Body('language') language?: string, + @Body('childName') childName?: string, + ) { + if (!text) { + throw new BadRequestException('Text is required'); + } + + this.logger.log(`[TEST] Voice classification request: "${text}"`); + + const result = await this.voiceService.extractActivityFromText( + text, + language || 'en', + childName, + ); + + this.logger.log(`[TEST] Classification result: ${JSON.stringify(result, null, 2)}`); + + return { + success: true, + transcript: text, + classification: result, + }; + } } \ No newline at end of file diff --git a/maternal-web/components/voice/VoiceFloatingButton.tsx b/maternal-web/components/voice/VoiceFloatingButton.tsx index 6e554ae..99d8235 100644 --- a/maternal-web/components/voice/VoiceFloatingButton.tsx +++ b/maternal-web/components/voice/VoiceFloatingButton.tsx @@ -21,6 +21,9 @@ import MicIcon from '@mui/icons-material/Mic'; import MicOffIcon from '@mui/icons-material/MicOff'; import { useRouter } from 'next/navigation'; import { useVoiceInput } from '@/hooks/useVoiceInput'; +import { useAuth } from '@/lib/auth/AuthContext'; +import { trackingApi } from '@/lib/api/tracking'; +import { childrenApi } from '@/lib/api/children'; /** * Floating voice input button @@ -30,6 +33,7 @@ import { useVoiceInput } from '@/hooks/useVoiceInput'; */ export function VoiceFloatingButton() { const router = useRouter(); + const { user } = useAuth(); const [open, setOpen] = useState(false); const [isProcessing, setIsProcessing] = useState(false); const [classificationResult, setClassificationResult] = useState(null); @@ -44,6 +48,8 @@ export function VoiceFloatingButton() { severity: 'info', }); + const familyId = user?.families?.[0]?.familyId; + const { isListening, isSupported, transcript, classification, error, usesFallback, startListening, stopListening, reset } = useVoiceInput(); @@ -133,7 +139,7 @@ export function VoiceFloatingButton() { } }; - const handleClassifiedIntent = (result: any) => { + const handleClassifiedIntent = async (result: any) => { if (result.error) { setSnackbar({ open: true, @@ -153,36 +159,68 @@ export function VoiceFloatingButton() { return; } - // Show success message with activity type - const activityLabel = result.type.charAt(0).toUpperCase() + result.type.slice(1); - setSnackbar({ - open: true, - message: `${activityLabel} activity detected!`, - severity: 'success', - }); + // Get the first child from the family + if (!familyId) { + setSnackbar({ + open: true, + message: 'No family found. Please set up your profile first.', + severity: 'error', + }); + return; + } - // Auto-close dialog and navigate with pre-filled data - setTimeout(() => { - handleClose(); + try { + setIsProcessing(true); - // Encode the details as URL parameters for pre-filling the form - const params = new URLSearchParams(); - if (result.details) { - Object.entries(result.details).forEach(([key, value]) => { - if (value !== null && value !== undefined) { - params.set(key, String(value)); - } + // Fetch children + const children = await childrenApi.getChildren(familyId); + if (children.length === 0) { + setSnackbar({ + open: true, + message: 'No children found. Please add a child first.', + severity: 'error', }); - } - if (result.timestamp) { - params.set('timestamp', result.timestamp); + setIsProcessing(false); + return; } - const queryString = params.toString(); - const url = `/track/${result.type}${queryString ? `?${queryString}` : ''}`; + // Use the first child (or you could enhance this to support child name matching) + const childId = children[0].id; - router.push(url); - }, 1500); + // Create the activity + const activityData = { + type: result.type, + timestamp: result.timestamp || new Date().toISOString(), + data: result.details || {}, + notes: result.details?.notes || undefined, + }; + + console.log('[Voice] Creating activity:', activityData); + + await trackingApi.createActivity(childId, activityData); + + // Show success message + const activityLabel = result.type.charAt(0).toUpperCase() + result.type.slice(1); + setSnackbar({ + open: true, + message: `${activityLabel} activity saved successfully!`, + severity: 'success', + }); + + // Auto-close dialog + setTimeout(() => { + handleClose(); + }, 1500); + } catch (error: any) { + console.error('[Voice] Failed to create activity:', error); + setSnackbar({ + open: true, + message: error.response?.data?.message || 'Failed to save activity. Please try again.', + severity: 'error', + }); + } finally { + setIsProcessing(false); + } }; const handleCloseSnackbar = () => { diff --git a/test-voice-commands.js b/test-voice-commands.js new file mode 100755 index 0000000..7e7b376 --- /dev/null +++ b/test-voice-commands.js @@ -0,0 +1,148 @@ +#!/usr/bin/env node + +/** + * Voice Command Testing Script + * Tests the voice classification API with various baby care commands + */ + +const API_URL = process.env.API_URL || 'http://localhost:3020'; +const ENDPOINT = '/api/v1/voice/test-classify'; // Using public test endpoint + +// ANSI color codes +const colors = { + reset: '\x1b[0m', + red: '\x1b[31m', + green: '\x1b[32m', + yellow: '\x1b[33m', + blue: '\x1b[34m', + cyan: '\x1b[36m', +}; + +// Test commands +const commands = [ + 'Change wet diaper', + 'Baby ate 150ml formula', + 'Baby slept for 1 hour', + 'Alice slept for 30 min', + 'Alice ate 3 pcs of broccoli at 11:00 AM', + 'Dirty diaper change', + 'Fed baby 120ml', + 'Baby napped for 45 minutes', + 'Changed diaper, it was wet', + 'Gave baby vitamin D drops', +]; + +async function testCommand(command, testNum) { + console.log(`${colors.yellow}Test #${testNum}: "${command}"${colors.reset}`); + console.log('---'); + + try { + const response = await fetch(`${API_URL}${ENDPOINT}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + text: command, + language: 'en', + childName: 'Alice', + }), + }); + + const data = await response.json(); + + if (!response.ok || !data.success) { + console.log(`${colors.red}✗ API returned error${colors.reset}`); + console.log(JSON.stringify(data, null, 2)); + console.log(''); + return false; + } + + // Extract classification details + const { type = 'unknown', confidence = 0, details = {}, timestamp = null } = data.classification || {}; + + // Color-code based on type + let typeColor; + switch (type) { + case 'feeding': + typeColor = colors.green; + break; + case 'sleep': + typeColor = colors.blue; + break; + case 'diaper': + typeColor = colors.yellow; + break; + case 'medicine': + typeColor = colors.cyan; + break; + case 'milestone': + typeColor = colors.green; + break; + default: + typeColor = colors.red; + } + + // Display results + console.log(`Type: ${typeColor}${type}${colors.reset}`); + console.log(`Confidence: ${confidence}`); + console.log(`Timestamp: ${timestamp || 'null'}`); + console.log('Details:'); + console.log(JSON.stringify(details, null, 2)); + + // Validate confidence threshold + const passed = type !== 'unknown' && confidence >= 0.3; + + if (passed) { + console.log(`${colors.green}✓ Command successfully classified${colors.reset}\n`); + return true; + } else { + console.log(`${colors.red}✗ Low confidence or unknown type${colors.reset}\n`); + return false; + } + } catch (error) { + console.log(`${colors.red}✗ Request failed: ${error.message}${colors.reset}\n`); + return false; + } +} + +async function runTests() { + console.log(`${colors.blue}========================================${colors.reset}`); + console.log(`${colors.blue}Voice Command Testing Suite${colors.reset}`); + console.log(`${colors.blue}========================================${colors.reset}\n`); + + let passed = 0; + let failed = 0; + + for (let i = 0; i < commands.length; i++) { + const result = await testCommand(commands[i], i + 1); + if (result) { + passed++; + } else { + failed++; + } + } + + // Summary + console.log(`${colors.blue}========================================${colors.reset}`); + console.log(`${colors.blue}Test Summary${colors.reset}`); + console.log(`${colors.blue}========================================${colors.reset}`); + console.log(`Total: ${commands.length}`); + console.log(`${colors.green}Passed: ${passed}${colors.reset}`); + console.log(`${colors.red}Failed: ${failed}${colors.reset}`); + console.log(''); + + if (failed === 0) { + console.log(`${colors.green}All tests passed! 🎉${colors.reset}`); + process.exit(0); + } else { + console.log(`${colors.red}Some tests failed. Check the output above.${colors.reset}`); + process.exit(1); + } +} + +// Run tests +runTests().catch(error => { + console.error(`${colors.red}Fatal error: ${error.message}${colors.reset}`); + process.exit(1); +}); diff --git a/test-voice-commands.sh b/test-voice-commands.sh new file mode 100755 index 0000000..1fbd1c0 --- /dev/null +++ b/test-voice-commands.sh @@ -0,0 +1,139 @@ +#!/bin/bash + +# Voice Command Testing Script +# Tests the voice classification API with various baby care commands + +API_URL="${API_URL:-http://localhost:3020}" +ENDPOINT="/api/v1/voice/transcribe" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +echo -e "${BLUE}========================================${NC}" +echo -e "${BLUE}Voice Command Testing Suite${NC}" +echo -e "${BLUE}========================================${NC}\n" + +# Test commands +declare -a commands=( + "Change wet diaper" + "Baby ate 150ml formula" + "Baby slept for 1 hour" + "Alice slept for 30 min" + "Alice ate 3 pcs of broccoli at 11:00 AM" + "Dirty diaper change" + "Fed baby 120ml" + "Baby napped for 45 minutes" + "Changed diaper, it was wet" + "Gave baby vitamin D drops" +) + +# Function to test a command +test_command() { + local command="$1" + local test_num="$2" + + echo -e "${YELLOW}Test #$test_num: \"$command\"${NC}" + echo "---" + + # Make API request + response=$(curl -s -X POST "${API_URL}${ENDPOINT}" \ + -H "Content-Type: application/json" \ + -d "{\"text\":\"$command\",\"language\":\"en\",\"childName\":\"Alice\"}") + + # Check if request was successful + if [ $? -ne 0 ]; then + echo -e "${RED}✗ API request failed${NC}\n" + return 1 + fi + + # Parse response + success=$(echo "$response" | jq -r '.success // false') + + if [ "$success" != "true" ]; then + echo -e "${RED}✗ API returned error${NC}" + echo "$response" | jq '.' + echo "" + return 1 + fi + + # Extract classification details + type=$(echo "$response" | jq -r '.classification.type // "unknown"') + confidence=$(echo "$response" | jq -r '.classification.confidence // 0') + details=$(echo "$response" | jq -r '.classification.details // {}') + timestamp=$(echo "$response" | jq -r '.classification.timestamp // "null"') + + # Color-code based on type + case "$type" in + feeding) + type_color="${GREEN}" + ;; + sleep) + type_color="${BLUE}" + ;; + diaper) + type_color="${YELLOW}" + ;; + medicine) + type_color="${RED}" + ;; + milestone) + type_color="${GREEN}" + ;; + *) + type_color="${RED}" + ;; + esac + + # Display results + echo -e "Type: ${type_color}${type}${NC}" + echo -e "Confidence: ${confidence}" + echo -e "Timestamp: ${timestamp}" + echo "Details:" + echo "$details" | jq '.' + + # Validate confidence threshold + confidence_float=$(echo "$confidence" | awk '{print ($1 >= 0.3) ? "pass" : "fail"}') + + if [ "$type" != "unknown" ] && [ "$confidence_float" == "pass" ]; then + echo -e "${GREEN}✓ Command successfully classified${NC}\n" + return 0 + else + echo -e "${RED}✗ Low confidence or unknown type${NC}\n" + return 1 + fi +} + +# Run all tests +total_tests=${#commands[@]} +passed=0 +failed=0 + +for i in "${!commands[@]}"; do + test_num=$((i + 1)) + if test_command "${commands[$i]}" "$test_num"; then + ((passed++)) + else + ((failed++)) + fi +done + +# Summary +echo -e "${BLUE}========================================${NC}" +echo -e "${BLUE}Test Summary${NC}" +echo -e "${BLUE}========================================${NC}" +echo -e "Total: $total_tests" +echo -e "${GREEN}Passed: $passed${NC}" +echo -e "${RED}Failed: $failed${NC}" +echo "" + +if [ $failed -eq 0 ]; then + echo -e "${GREEN}All tests passed! 🎉${NC}" + exit 0 +else + echo -e "${RED}Some tests failed. Check the output above.${NC}" + exit 1 +fi