Improve voice command UX and add desktop home navigation
Some checks failed
CI/CD Pipeline / Lint and Test (push) Has been cancelled
CI/CD Pipeline / E2E Tests (push) Has been cancelled
CI/CD Pipeline / Build Application (push) Has been cancelled

Voice command improvements:
- Auto-start listening when voice dialog opens (removes extra tap/click)
- Added Activity option to unknown intent dialog
- Users can now speak immediately after clicking the mic button

Desktop navigation:
- Added Home icon button in top bar/header for quick navigation to main page
- Positioned between app title and user avatar

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-10-02 11:52:06 +00:00
parent 26306d7ed8
commit e79eda6a7d
5 changed files with 242 additions and 0 deletions

View File

@@ -0,0 +1,5 @@
# Update Azure OpenAI Chat credentials
AZURE_OPENAI_CHAT_ENDPOINT=https://footprints-open-ai.openai.azure.com
AZURE_OPENAI_CHAT_DEPLOYMENT=gpt-5-mini
AZURE_OPENAI_CHAT_API_VERSION=2025-04-01-preview
AZURE_OPENAI_CHAT_API_KEY=a5f7e3e70a454a399f9216853b45e18b

Binary file not shown.

View File

@@ -63,6 +63,14 @@ export const MobileNav = () => {
<Typography variant="h6" component="div" sx={{ flexGrow: 1, color: 'primary.main', fontWeight: 600 }}>
Maternal
</Typography>
<IconButton
color="primary"
aria-label="home"
onClick={() => router.push('/')}
sx={{ mr: 1 }}
>
<Home />
</IconButton>
<Avatar sx={{ bgcolor: 'primary.main' }}>U</Avatar>
</Toolbar>
</AppBar>

View File

@@ -121,6 +121,11 @@ export function VoiceFloatingButton() {
setProcessingStatus(null);
setIdentifiedActivity('');
setProcessedClassificationId(null);
// Auto-start listening after dialog opens
setTimeout(() => {
startListening();
}, 300);
};
const handleClose = () => {
@@ -484,6 +489,7 @@ export function VoiceFloatingButton() {
<MenuItem value="sleep">Sleep</MenuItem>
<MenuItem value="diaper">Diaper Change</MenuItem>
<MenuItem value="medicine">Medicine</MenuItem>
<MenuItem value="activity">Activity</MenuItem>
</Select>
</FormControl>
</DialogContent>

223
test-voice-e2e.js Executable file
View File

@@ -0,0 +1,223 @@
#!/usr/bin/env node
/**
* End-to-End Voice Command Test
* Tests the full voice flow: classify + create activity in database
*/
const API_URL = process.env.API_URL || 'http://localhost:3020';
// ANSI color codes
const colors = {
reset: '\x1b[0m',
red: '\x1b[31m',
green: '\x1b[32m',
yellow: '\x1b[33m',
blue: '\x1b[34m',
cyan: '\x1b[36m',
};
// Test credentials
const TEST_USER = {
email: 'andrei@cloudz.ro',
password: 'Test1234!',
};
// Test commands
const commands = [
'Change wet diaper',
'Baby ate 150ml formula',
'Baby slept for 1 hour',
'Alice ate 3 pcs of broccoli at 11:00 AM',
];
let accessToken = null;
let childId = null;
async function login() {
console.log(`${colors.blue}Logging in...${colors.reset}`);
const response = await fetch(`${API_URL}/api/v1/auth/login`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(TEST_USER),
});
const data = await response.json();
if (!response.ok || !data.success) {
throw new Error(`Login failed: ${data.message || 'Unknown error'}`);
}
accessToken = data.data.accessToken;
console.log(`${colors.green}✓ Logged in successfully${colors.reset}\n`);
}
async function getChild() {
console.log(`${colors.blue}Fetching children...${colors.reset}`);
const response = await fetch(`${API_URL}/api/v1/children`, {
headers: { 'Authorization': `Bearer ${accessToken}` },
});
const data = await response.json();
if (!response.ok || !data.success || data.data.length === 0) {
throw new Error('No children found');
}
childId = data.data[0].id;
console.log(`${colors.green}✓ Found child: ${data.data[0].name} (${childId})${colors.reset}\n`);
}
async function getActivitiesCount(type = null) {
let url = `${API_URL}/api/v1/activities?childId=${childId}&limit=1000`;
if (type) url += `&type=${type}`;
const response = await fetch(url, {
headers: { 'Authorization': `Bearer ${accessToken}` },
});
const data = await response.json();
return data.success ? data.data.length : 0;
}
async function classifyAndCreateActivity(text) {
console.log(`${colors.yellow}Processing: "${text}"${colors.reset}`);
// Step 1: Classify the command
const classifyResponse = await fetch(`${API_URL}/api/v1/voice/transcribe`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${accessToken}`,
},
body: JSON.stringify({
text,
language: 'en',
childName: 'Alice',
}),
});
const classifyData = await classifyResponse.json();
if (!classifyResponse.ok || !classifyData.success) {
console.log(`${colors.red}✗ Classification failed${colors.reset}`);
console.log(JSON.stringify(classifyData, null, 2));
return false;
}
const { type, details, timestamp, confidence } = classifyData.classification;
console.log(` Type: ${type} (confidence: ${confidence})`);
console.log(` Details: ${JSON.stringify(details)}`);
if (type === 'unknown' || confidence < 0.3) {
console.log(`${colors.red}✗ Low confidence or unknown type${colors.reset}\n`);
return false;
}
// Step 2: Create the activity
const activityData = {
type,
timestamp: timestamp || new Date().toISOString(),
data: details || {},
notes: details?.notes || undefined,
};
const createResponse = await fetch(`${API_URL}/api/v1/activities`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${accessToken}`,
},
body: JSON.stringify({
childId,
...activityData,
}),
});
const createData = await createResponse.json();
if (!createResponse.ok || !createData.success) {
console.log(`${colors.red}✗ Failed to create activity${colors.reset}`);
console.log(JSON.stringify(createData, null, 2));
return false;
}
console.log(`${colors.green}✓ Activity created: ${createData.data.id}${colors.reset}\n`);
return true;
}
async function runTests() {
console.log(`${colors.blue}========================================${colors.reset}`);
console.log(`${colors.blue}Voice E2E Test Suite${colors.reset}`);
console.log(`${colors.blue}========================================${colors.reset}\n`);
try {
// Login and get child
await login();
await getChild();
// Get initial counts
const initialCounts = {
total: await getActivitiesCount(),
diaper: await getActivitiesCount('diaper'),
feeding: await getActivitiesCount('feeding'),
sleep: await getActivitiesCount('sleep'),
};
console.log(`${colors.cyan}Initial activity counts:${colors.reset}`);
console.log(` Total: ${initialCounts.total}`);
console.log(` Diapers: ${initialCounts.diaper}`);
console.log(` Feedings: ${initialCounts.feeding}`);
console.log(` Sleep: ${initialCounts.sleep}\n`);
// Run tests
let passed = 0;
let failed = 0;
for (const command of commands) {
const result = await classifyAndCreateActivity(command);
if (result) {
passed++;
} else {
failed++;
}
}
// Get final counts
const finalCounts = {
total: await getActivitiesCount(),
diaper: await getActivitiesCount('diaper'),
feeding: await getActivitiesCount('feeding'),
sleep: await getActivitiesCount('sleep'),
};
console.log(`${colors.cyan}Final activity counts:${colors.reset}`);
console.log(` Total: ${finalCounts.total} (+${finalCounts.total - initialCounts.total})`);
console.log(` Diapers: ${finalCounts.diaper} (+${finalCounts.diaper - initialCounts.diaper})`);
console.log(` Feedings: ${finalCounts.feeding} (+${finalCounts.feeding - initialCounts.feeding})`);
console.log(` Sleep: ${finalCounts.sleep} (+${finalCounts.sleep - initialCounts.sleep})\n`);
// Summary
console.log(`${colors.blue}========================================${colors.reset}`);
console.log(`${colors.blue}Test Summary${colors.reset}`);
console.log(`${colors.blue}========================================${colors.reset}`);
console.log(`Total: ${commands.length}`);
console.log(`${colors.green}Passed: ${passed}${colors.reset}`);
console.log(`${colors.red}Failed: ${failed}${colors.reset}`);
console.log('');
if (failed === 0) {
console.log(`${colors.green}All tests passed! Activities saved to database. 🎉${colors.reset}`);
process.exit(0);
} else {
console.log(`${colors.red}Some tests failed. Check the output above.${colors.reset}`);
process.exit(1);
}
} catch (error) {
console.error(`${colors.red}Fatal error: ${error.message}${colors.reset}`);
console.error(error.stack);
process.exit(1);
}
}
// Run tests
runTests();