Add voice command auto-fill and server-side logging
- Add URL parameter reading to diaper tracking page for voice-extracted data - Add comprehensive server-side logging in voice controller and service - Log request type (Web Speech API vs MediaRecorder), input text/audio, GPT calls, and classification results - Enable automatic form pre-filling when voice commands navigate to tracking pages 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -6,12 +6,15 @@ import {
|
||||
Body,
|
||||
Req,
|
||||
BadRequestException,
|
||||
Logger,
|
||||
} from '@nestjs/common';
|
||||
import { FileInterceptor } from '@nestjs/platform-express';
|
||||
import { VoiceService } from './voice.service';
|
||||
|
||||
@Controller('api/v1/voice')
|
||||
export class VoiceController {
|
||||
private readonly logger = new Logger('VoiceController');
|
||||
|
||||
constructor(private readonly voiceService: VoiceService) {}
|
||||
|
||||
@Post('transcribe')
|
||||
@@ -22,14 +25,24 @@ export class VoiceController {
|
||||
@Body('language') language?: string,
|
||||
@Body('childName') childName?: string,
|
||||
) {
|
||||
this.logger.log('=== Voice Transcribe Request ===');
|
||||
this.logger.log(`Mode: ${text ? 'Text Classification (Web Speech API)' : 'Audio Transcription (MediaRecorder)'}`);
|
||||
this.logger.log(`Language: ${language || 'en'}`);
|
||||
this.logger.log(`Child Name: ${childName || 'none'}`);
|
||||
|
||||
// If text is provided (from Web Speech API), classify it directly
|
||||
if (text) {
|
||||
this.logger.log(`Input Text: "${text}"`);
|
||||
|
||||
const result = await this.voiceService.extractActivityFromText(
|
||||
text,
|
||||
language || 'en',
|
||||
childName,
|
||||
);
|
||||
|
||||
this.logger.log(`Classification Result: ${JSON.stringify(result, null, 2)}`);
|
||||
this.logger.log('=== Request Complete ===\n');
|
||||
|
||||
return {
|
||||
success: true,
|
||||
transcript: text,
|
||||
@@ -39,14 +52,19 @@ export class VoiceController {
|
||||
|
||||
// Otherwise, transcribe the audio file
|
||||
if (!file) {
|
||||
this.logger.error('No audio file or text provided');
|
||||
throw new BadRequestException('Audio file or text is required');
|
||||
}
|
||||
|
||||
this.logger.log(`Audio File: ${file.originalname} (${file.size} bytes, ${file.mimetype})`);
|
||||
|
||||
const transcription = await this.voiceService.transcribeAudio(
|
||||
file.buffer,
|
||||
language,
|
||||
);
|
||||
|
||||
this.logger.log(`Transcription: "${transcription.text}" (${transcription.language})`);
|
||||
|
||||
// Also classify the transcription
|
||||
const classification = await this.voiceService.extractActivityFromText(
|
||||
transcription.text,
|
||||
@@ -54,6 +72,9 @@ export class VoiceController {
|
||||
childName,
|
||||
);
|
||||
|
||||
this.logger.log(`Classification Result: ${JSON.stringify(classification, null, 2)}`);
|
||||
this.logger.log('=== Request Complete ===\n');
|
||||
|
||||
return {
|
||||
success: true,
|
||||
transcript: transcription.text,
|
||||
|
||||
@@ -141,33 +141,92 @@ export class VoiceService {
|
||||
throw new BadRequestException('Chat service not configured');
|
||||
}
|
||||
|
||||
this.logger.log(`[Activity Extraction] Starting extraction for: "${text}"`);
|
||||
this.logger.log(`[Activity Extraction] Language: ${language}, Child: ${childName || 'none'}`);
|
||||
|
||||
try {
|
||||
const systemPrompt = `You are an assistant that extracts baby care activity information from natural language.
|
||||
const systemPrompt = `You are an intelligent assistant that interprets natural language commands related to baby care and extracts structured activity data.
|
||||
|
||||
Extract activity details from the user's text and respond ONLY with valid JSON (no markdown, no explanations).
|
||||
|
||||
Activity types:
|
||||
- feeding: nursing, bottle, solids (include amount, duration, notes)
|
||||
- sleep: start time, end time (or duration), quality
|
||||
- diaper: type (wet/dirty/both), notes
|
||||
- medicine: name, dosage, time
|
||||
- milestone: description, date
|
||||
**Supported Activity Types:**
|
||||
|
||||
Response format:
|
||||
1. **feeding** - Any mention of eating, drinking, nursing, bottle, breastfeeding, formula, solids, meals
|
||||
- Extract: amount (ml/oz), method (bottle/breast/solids), duration (minutes), side (left/right/both for breastfeeding), notes
|
||||
|
||||
2. **sleep** - Any mention of sleeping, napping, bedtime, waking up, dozed off
|
||||
- Extract: start_time, end_time, duration (minutes), quality (peaceful/restless/fussy), location (crib/bassinet/arms), notes
|
||||
|
||||
3. **diaper** - Any mention of diaper change, wet, dirty, bowel movement, pee, poop
|
||||
- Extract: type (wet/dirty/both), color, consistency, rash (true/false), notes
|
||||
|
||||
4. **medicine** - Any mention of medication, vitamin, supplement, drops, dose
|
||||
- Extract: name, dosage, unit, notes
|
||||
|
||||
5. **milestone** - Any mention of first time events, developmental progress, achievements
|
||||
- Extract: description, category (motor/social/cognitive/language), notes
|
||||
|
||||
**Response Format:**
|
||||
{
|
||||
"type": "activity_type",
|
||||
"timestamp": "ISO 8601 datetime if mentioned, null otherwise",
|
||||
"details": {...activity specific fields...},
|
||||
"confidence": 0-1
|
||||
"type": "feeding|sleep|diaper|medicine|milestone|unknown",
|
||||
"timestamp": "ISO 8601 datetime if mentioned (e.g., '3pm', '30 minutes ago'), otherwise use current time",
|
||||
"details": {
|
||||
// For feeding:
|
||||
"amount": number or null,
|
||||
"unit": "ml|oz" or null,
|
||||
"method": "bottle|breast|solids" or null,
|
||||
"side": "left|right|both" or null,
|
||||
"duration": number (minutes) or null,
|
||||
"notes": string or null
|
||||
|
||||
// For sleep:
|
||||
"start_time": "ISO 8601" or null,
|
||||
"end_time": "ISO 8601" or null,
|
||||
"duration": number (minutes) or null,
|
||||
"quality": "peaceful|restless|fussy" or null,
|
||||
"location": string or null,
|
||||
"notes": string or null
|
||||
|
||||
// For diaper:
|
||||
"type": "wet|dirty|both",
|
||||
"color": string or null,
|
||||
"consistency": string or null,
|
||||
"rash": boolean or null,
|
||||
"notes": string or null
|
||||
|
||||
// For medicine:
|
||||
"name": string,
|
||||
"dosage": number or string,
|
||||
"unit": string or null,
|
||||
"notes": string or null
|
||||
|
||||
// For milestone:
|
||||
"description": string,
|
||||
"category": "motor|social|cognitive|language" or null,
|
||||
"notes": string or null
|
||||
},
|
||||
"confidence": 0.0-1.0,
|
||||
"action": "create_activity|unknown"
|
||||
}
|
||||
|
||||
If the text doesn't describe a trackable activity, respond with:
|
||||
{"type": "unknown", "details": {}, "confidence": 0}`;
|
||||
**Important:**
|
||||
- Be smart about interpreting casual language (e.g., "fed him" = feeding activity)
|
||||
- Infer reasonable defaults (e.g., if amount is mentioned without unit and sounds like bottles, assume "ml")
|
||||
- Use "unknown" type only if the text is clearly NOT about baby care
|
||||
- Set confidence based on how clearly the activity is described
|
||||
- Assume current time for timestamp if no time is mentioned
|
||||
- Be flexible with language variations and typos
|
||||
|
||||
If the text doesn't describe a trackable baby care activity:
|
||||
{"type": "unknown", "details": {}, "confidence": 0, "action": "unknown"}`;
|
||||
|
||||
const userPrompt = childName
|
||||
? `Child name: ${childName}\nUser said: "${text}"`
|
||||
: `User said: "${text}"`;
|
||||
|
||||
this.logger.log(`[Activity Extraction] Calling GPT-4o-mini with user prompt: ${userPrompt}`);
|
||||
|
||||
const startTime = Date.now();
|
||||
const completion = await this.chatOpenAI.chat.completions.create({
|
||||
model: 'gpt-4o-mini',
|
||||
messages: [
|
||||
@@ -176,11 +235,18 @@ If the text doesn't describe a trackable activity, respond with:
|
||||
],
|
||||
response_format: { type: 'json_object' },
|
||||
});
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
this.logger.log(`[Activity Extraction] GPT response received in ${duration}ms`);
|
||||
this.logger.log(`[Activity Extraction] Raw GPT response: ${completion.choices[0].message.content}`);
|
||||
|
||||
const result = JSON.parse(completion.choices[0].message.content);
|
||||
|
||||
this.logger.log(
|
||||
`Activity extracted: ${result.type} (confidence: ${result.confidence})`,
|
||||
`[Activity Extraction] Extracted activity: ${result.type} (confidence: ${result.confidence})`,
|
||||
);
|
||||
this.logger.log(
|
||||
`[Activity Extraction] Details: ${JSON.stringify(result.details || {})}`,
|
||||
);
|
||||
|
||||
return {
|
||||
@@ -191,7 +257,7 @@ If the text doesn't describe a trackable activity, respond with:
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Activity extraction failed: ${error.message}`,
|
||||
`[Activity Extraction] Failed: ${error.message}`,
|
||||
error.stack,
|
||||
);
|
||||
throw new BadRequestException('Failed to extract activity from text');
|
||||
|
||||
Reference in New Issue
Block a user