feat(phase-6): Bulk CSV processing and background worker implementation
- Add BulkJob model to Prisma schema with relations - Implement BulkProcessorService for CSV parsing and job management - Create BulkTrackingWorker for background processing with BullMQ - Add comprehensive bulk API routes (upload, jobs, progress, export) - Integrate multer for CSV file uploads with validation - Add job progress tracking and estimation - Implement CSV export functionality for results - Add queue statistics and cleanup endpoints - Create shared types for bulk processing - Add comprehensive test suite for all bulk functionality - Implement graceful worker shutdown and error handling - Add rate limiting and authentication for all bulk endpoints Backward compatibility: Maintained for /api/track and /api/v1/track
This commit is contained in:
Andrei
471
test-phase-6.js
Normal file
471
test-phase-6.js
Normal file
@@ -0,0 +1,471 @@
|
||||
/**
|
||||
* Test script for Phase 6: Bulk CSV + Worker
|
||||
* Tests bulk processing functionality, CSV upload, and worker integration
|
||||
*/
|
||||
|
||||
const http = require('http');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const BASE_URL = 'http://localhost:3333';
|
||||
|
||||
// Helper function to make HTTP requests
|
||||
function makeRequest(options, data = null) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = http.request(options, (res) => {
|
||||
let body = '';
|
||||
res.on('data', (chunk) => body += chunk);
|
||||
res.on('end', () => {
|
||||
try {
|
||||
const result = {
|
||||
statusCode: res.statusCode,
|
||||
headers: res.headers,
|
||||
body: res.headers['content-type'] && res.headers['content-type'].includes('application/json')
|
||||
? JSON.parse(body)
|
||||
: body
|
||||
};
|
||||
resolve(result);
|
||||
} catch (error) {
|
||||
resolve({
|
||||
statusCode: res.statusCode,
|
||||
headers: res.headers,
|
||||
body: body
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
req.on('error', reject);
|
||||
|
||||
if (data) {
|
||||
if (typeof data === 'string') {
|
||||
req.write(data);
|
||||
} else {
|
||||
req.write(JSON.stringify(data));
|
||||
}
|
||||
}
|
||||
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
// Helper function to create test CSV file
|
||||
function createTestCSV() {
|
||||
const csvContent = `url,label,method,max_hops,enable_ssl
|
||||
https://httpbin.org/redirect/1,Test Redirect 1,GET,5,true
|
||||
https://httpbin.org/redirect/2,Test Redirect 2,GET,10,true
|
||||
https://example.com,Example Domain,GET,3,false
|
||||
https://httpbin.org/status/302,Status 302,GET,5,true
|
||||
invalid-url,Invalid URL,GET,5,true`;
|
||||
|
||||
const filePath = path.join(__dirname, 'test-urls.csv');
|
||||
fs.writeFileSync(filePath, csvContent);
|
||||
return filePath;
|
||||
}
|
||||
|
||||
// Helper function to create multipart form data
|
||||
function createMultipartData(filePath, options = {}) {
|
||||
const boundary = '----formdata-test-' + Math.random().toString(36);
|
||||
const fileName = path.basename(filePath);
|
||||
const fileContent = fs.readFileSync(filePath);
|
||||
|
||||
let data = '';
|
||||
|
||||
// Add file field
|
||||
data += `--${boundary}\r\n`;
|
||||
data += `Content-Disposition: form-data; name="file"; filename="${fileName}"\r\n`;
|
||||
data += `Content-Type: text/csv\r\n\r\n`;
|
||||
data += fileContent;
|
||||
data += `\r\n`;
|
||||
|
||||
// Add options field
|
||||
if (Object.keys(options).length > 0) {
|
||||
data += `--${boundary}\r\n`;
|
||||
data += `Content-Disposition: form-data; name="options"\r\n\r\n`;
|
||||
data += JSON.stringify(options);
|
||||
data += `\r\n`;
|
||||
}
|
||||
|
||||
data += `--${boundary}--\r\n`;
|
||||
|
||||
return {
|
||||
data: Buffer.from(data),
|
||||
boundary: boundary
|
||||
};
|
||||
}
|
||||
|
||||
async function runTests() {
|
||||
console.log('🧪 Starting Phase 6: Bulk CSV + Worker Tests\n');
|
||||
|
||||
let authToken = null;
|
||||
let testJobId = null;
|
||||
const csvFilePath = createTestCSV();
|
||||
|
||||
// Test 1: User Registration
|
||||
console.log('1️⃣ Testing user registration...');
|
||||
try {
|
||||
const registerResult = await makeRequest({
|
||||
hostname: 'localhost',
|
||||
port: 3333,
|
||||
path: '/api/v1/auth/register',
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}, {
|
||||
email: 'bulk-test@example.com',
|
||||
name: 'Bulk Test User',
|
||||
password: 'bulktest123',
|
||||
organizationName: 'Bulk Test Org'
|
||||
});
|
||||
|
||||
if (registerResult.statusCode === 201 && registerResult.body.success) {
|
||||
authToken = registerResult.body.data.token;
|
||||
console.log('✅ User registration successful');
|
||||
} else if (registerResult.statusCode === 409) {
|
||||
console.log('ℹ️ User already exists, attempting login...');
|
||||
|
||||
// Try to login if user exists
|
||||
const loginResult = await makeRequest({
|
||||
hostname: 'localhost',
|
||||
port: 3333,
|
||||
path: '/api/v1/auth/login',
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}, {
|
||||
email: 'bulk-test@example.com',
|
||||
password: 'bulktest123'
|
||||
});
|
||||
|
||||
if (loginResult.statusCode === 200 && loginResult.body.success) {
|
||||
authToken = loginResult.body.data.token;
|
||||
console.log('✅ User login successful');
|
||||
} else {
|
||||
console.log('❌ Login failed:', loginResult.body);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
console.log('❌ Registration failed:', registerResult.body);
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('❌ Registration/login error:', error.message);
|
||||
return;
|
||||
}
|
||||
|
||||
// Test 2: Get queue stats (should work before any jobs)
|
||||
console.log('\n2️⃣ Testing queue statistics...');
|
||||
try {
|
||||
const statsResult = await makeRequest({
|
||||
hostname: 'localhost',
|
||||
port: 3333,
|
||||
path: '/api/v2/bulk/stats',
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${authToken}`,
|
||||
},
|
||||
});
|
||||
|
||||
console.log('Queue stats response:', statsResult.statusCode);
|
||||
if (statsResult.statusCode === 200) {
|
||||
console.log('✅ Queue stats retrieved:', statsResult.body.data);
|
||||
} else {
|
||||
console.log('⚠️ Queue stats failed:', statsResult.body);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('❌ Queue stats error:', error.message);
|
||||
}
|
||||
|
||||
// Test 3: Create bulk job with JSON payload
|
||||
console.log('\n3️⃣ Testing bulk job creation with JSON...');
|
||||
try {
|
||||
const createJobResult = await makeRequest({
|
||||
hostname: 'localhost',
|
||||
port: 3333,
|
||||
path: '/api/v2/bulk/jobs',
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${authToken}`,
|
||||
},
|
||||
}, {
|
||||
urls: [
|
||||
{ url: 'https://httpbin.org/redirect/1', label: 'JSON Test 1' },
|
||||
{ url: 'https://example.com', label: 'JSON Test 2' }
|
||||
],
|
||||
options: {
|
||||
method: 'GET',
|
||||
maxHops: 5,
|
||||
timeout: 10000,
|
||||
enableSSLAnalysis: true,
|
||||
enableSEOAnalysis: false,
|
||||
enableSecurityAnalysis: true
|
||||
}
|
||||
});
|
||||
|
||||
if (createJobResult.statusCode === 201 && createJobResult.body.success) {
|
||||
testJobId = createJobResult.body.data.jobId;
|
||||
console.log('✅ Bulk job created:', testJobId);
|
||||
console.log('Job status:', createJobResult.body.data.status);
|
||||
console.log('URL count:', createJobResult.body.data.urls);
|
||||
} else {
|
||||
console.log('❌ Bulk job creation failed:', createJobResult.body);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('❌ Bulk job creation error:', error.message);
|
||||
}
|
||||
|
||||
// Test 4: Upload CSV file
|
||||
console.log('\n4️⃣ Testing CSV upload...');
|
||||
try {
|
||||
const multipartData = createMultipartData(csvFilePath, {
|
||||
method: 'GET',
|
||||
maxHops: 8,
|
||||
timeout: 15000
|
||||
});
|
||||
|
||||
const uploadResult = await makeRequest({
|
||||
hostname: 'localhost',
|
||||
port: 3333,
|
||||
path: '/api/v2/bulk/upload',
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': `multipart/form-data; boundary=${multipartData.boundary}`,
|
||||
'Authorization': `Bearer ${authToken}`,
|
||||
'Content-Length': multipartData.data.length,
|
||||
},
|
||||
}, multipartData.data);
|
||||
|
||||
if (uploadResult.statusCode === 200 && uploadResult.body.success) {
|
||||
console.log('✅ CSV upload successful');
|
||||
console.log('Job ID:', uploadResult.body.data.jobId);
|
||||
console.log('Job status:', uploadResult.body.data.status);
|
||||
|
||||
// Use this job for further tests if we don't have one from JSON
|
||||
if (!testJobId) {
|
||||
testJobId = uploadResult.body.data.jobId;
|
||||
}
|
||||
} else {
|
||||
console.log('❌ CSV upload failed:', uploadResult.body);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('❌ CSV upload error:', error.message);
|
||||
}
|
||||
|
||||
// Test 5: Get user bulk jobs list
|
||||
console.log('\n5️⃣ Testing bulk jobs list...');
|
||||
try {
|
||||
const jobsListResult = await makeRequest({
|
||||
hostname: 'localhost',
|
||||
port: 3333,
|
||||
path: '/api/v2/bulk/jobs?limit=10&offset=0',
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${authToken}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (jobsListResult.statusCode === 200 && jobsListResult.body.success) {
|
||||
console.log('✅ Jobs list retrieved');
|
||||
console.log('Job count:', jobsListResult.body.data.length);
|
||||
jobsListResult.body.data.forEach((job, index) => {
|
||||
console.log(` Job ${index + 1}: ${job.id} - ${job.status} (${job.urlCount} URLs)`);
|
||||
});
|
||||
} else {
|
||||
console.log('❌ Jobs list failed:', jobsListResult.body);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('❌ Jobs list error:', error.message);
|
||||
}
|
||||
|
||||
// Test 6: Get specific job details
|
||||
if (testJobId) {
|
||||
console.log('\n6️⃣ Testing job details retrieval...');
|
||||
try {
|
||||
const jobDetailResult = await makeRequest({
|
||||
hostname: 'localhost',
|
||||
port: 3333,
|
||||
path: `/api/v2/bulk/jobs/${testJobId}`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${authToken}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (jobDetailResult.statusCode === 200 && jobDetailResult.body.success) {
|
||||
console.log('✅ Job details retrieved');
|
||||
console.log('Job status:', jobDetailResult.body.data.status);
|
||||
console.log('Progress:', jobDetailResult.body.data.progress);
|
||||
|
||||
if (jobDetailResult.body.data.estimatedCompletionAt) {
|
||||
console.log('Estimated completion:', jobDetailResult.body.data.estimatedCompletionAt);
|
||||
}
|
||||
} else {
|
||||
console.log('❌ Job details failed:', jobDetailResult.body);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('❌ Job details error:', error.message);
|
||||
}
|
||||
|
||||
// Test 7: Monitor job progress
|
||||
console.log('\n7️⃣ Monitoring job progress...');
|
||||
let attempts = 0;
|
||||
const maxAttempts = 30; // Wait up to 30 seconds
|
||||
|
||||
while (attempts < maxAttempts) {
|
||||
try {
|
||||
const progressResult = await makeRequest({
|
||||
hostname: 'localhost',
|
||||
port: 3333,
|
||||
path: `/api/v2/bulk/jobs/${testJobId}`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${authToken}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (progressResult.statusCode === 200 && progressResult.body.success) {
|
||||
const job = progressResult.body.data;
|
||||
console.log(`Progress: ${job.progress.processed}/${job.progress.total} (${job.status})`);
|
||||
|
||||
if (job.status === 'completed' || job.status === 'failed') {
|
||||
console.log('✅ Job completed!');
|
||||
console.log('Final stats:', job.progress);
|
||||
|
||||
if (job.results) {
|
||||
console.log('Results available:', job.results.length);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
attempts++;
|
||||
if (attempts < maxAttempts) {
|
||||
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait 1 second
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('❌ Progress monitoring error:', error.message);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (attempts >= maxAttempts) {
|
||||
console.log('⚠️ Job monitoring timed out');
|
||||
}
|
||||
|
||||
// Test 8: Export results (if job completed)
|
||||
console.log('\n8️⃣ Testing results export...');
|
||||
try {
|
||||
const exportResult = await makeRequest({
|
||||
hostname: 'localhost',
|
||||
port: 3333,
|
||||
path: `/api/v2/bulk/jobs/${testJobId}/export/csv`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${authToken}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (exportResult.statusCode === 200) {
|
||||
console.log('✅ Results export successful');
|
||||
console.log('Content-Type:', exportResult.headers['content-type']);
|
||||
console.log('File size:', exportResult.body.length, 'bytes');
|
||||
} else {
|
||||
console.log('⚠️ Results export failed:', exportResult.statusCode);
|
||||
if (typeof exportResult.body === 'object') {
|
||||
console.log('Error:', exportResult.body);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('❌ Results export error:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
// Test 9: Test validation errors
|
||||
console.log('\n9️⃣ Testing validation errors...');
|
||||
try {
|
||||
// Test with invalid URL
|
||||
const invalidJobResult = await makeRequest({
|
||||
hostname: 'localhost',
|
||||
port: 3333,
|
||||
path: '/api/v2/bulk/jobs',
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${authToken}`,
|
||||
},
|
||||
}, {
|
||||
urls: [
|
||||
{ url: 'not-a-valid-url', label: 'Invalid URL' }
|
||||
]
|
||||
});
|
||||
|
||||
if (invalidJobResult.statusCode === 400) {
|
||||
console.log('✅ Validation correctly rejected invalid URL');
|
||||
} else {
|
||||
console.log('⚠️ Validation did not catch invalid URL:', invalidJobResult.body);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('❌ Validation test error:', error.message);
|
||||
}
|
||||
|
||||
// Test 10: Test unauthorized access
|
||||
console.log('\n🔟 Testing unauthorized access...');
|
||||
try {
|
||||
const unauthorizedResult = await makeRequest({
|
||||
hostname: 'localhost',
|
||||
port: 3333,
|
||||
path: '/api/v2/bulk/jobs',
|
||||
method: 'GET',
|
||||
headers: {
|
||||
// No authorization header
|
||||
},
|
||||
});
|
||||
|
||||
if (unauthorizedResult.statusCode === 401) {
|
||||
console.log('✅ Unauthorized access correctly blocked');
|
||||
} else {
|
||||
console.log('⚠️ Unauthorized access was not blocked:', unauthorizedResult.statusCode);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('❌ Unauthorized test error:', error.message);
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
try {
|
||||
fs.unlinkSync(csvFilePath);
|
||||
console.log('\n🧹 Test CSV file cleaned up');
|
||||
} catch (error) {
|
||||
console.log('\n⚠️ Failed to cleanup test file:', error.message);
|
||||
}
|
||||
|
||||
console.log('\n🎉 Phase 6 testing completed!');
|
||||
console.log('\nKey features tested:');
|
||||
console.log('✓ Bulk job creation with JSON payload');
|
||||
console.log('✓ CSV file upload and parsing');
|
||||
console.log('✓ Job progress monitoring');
|
||||
console.log('✓ Results export to CSV');
|
||||
console.log('✓ Queue statistics');
|
||||
console.log('✓ Input validation');
|
||||
console.log('✓ Authentication/authorization');
|
||||
console.log('✓ Error handling');
|
||||
}
|
||||
|
||||
// Error handling
|
||||
process.on('uncaughtException', (error) => {
|
||||
console.log('\n💥 Uncaught Exception:', error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
process.on('unhandledRejection', (reason) => {
|
||||
console.log('\n💥 Unhandled Rejection:', reason);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
// Run tests
|
||||
runTests().catch(error => {
|
||||
console.log('\n💥 Test execution failed:', error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user