feat(phase-6): Bulk CSV processing and background worker implementation
- Add BulkJob model to Prisma schema with relations - Implement BulkProcessorService for CSV parsing and job management - Create BulkTrackingWorker for background processing with BullMQ - Add comprehensive bulk API routes (upload, jobs, progress, export) - Integrate multer for CSV file uploads with validation - Add job progress tracking and estimation - Implement CSV export functionality for results - Add queue statistics and cleanup endpoints - Create shared types for bulk processing - Add comprehensive test suite for all bulk functionality - Implement graceful worker shutdown and error handling - Add rate limiting and authentication for all bulk endpoints Backward compatibility: Maintained for /api/track and /api/v1/track
This commit is contained in:
@@ -22,6 +22,7 @@ model User {
|
||||
|
||||
memberships OrgMembership[]
|
||||
auditLogs AuditLog[]
|
||||
bulkJobs BulkJob[]
|
||||
|
||||
@@map("users")
|
||||
}
|
||||
@@ -36,6 +37,7 @@ model Organization {
|
||||
projects Project[]
|
||||
apiKeys ApiKey[]
|
||||
auditLogs AuditLog[]
|
||||
bulkJobs BulkJob[]
|
||||
|
||||
@@map("organizations")
|
||||
}
|
||||
@@ -212,6 +214,32 @@ model AuditLog {
|
||||
@@map("audit_logs")
|
||||
}
|
||||
|
||||
model BulkJob {
|
||||
id String @id
|
||||
userId String
|
||||
organizationId String?
|
||||
projectId String?
|
||||
status String // 'pending' | 'processing' | 'completed' | 'failed' | 'cancelled'
|
||||
totalUrls Int
|
||||
processedUrls Int @default(0)
|
||||
successfulUrls Int @default(0)
|
||||
failedUrls Int @default(0)
|
||||
configJson Json // Job configuration (options)
|
||||
urlsJson Json // Array of URLs to process
|
||||
resultsJson Json? // Array of results
|
||||
createdAt DateTime @default(now())
|
||||
startedAt DateTime?
|
||||
finishedAt DateTime?
|
||||
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
organization Organization? @relation(fields: [organizationId], references: [id], onDelete: SetNull)
|
||||
project Project? @relation(fields: [projectId], references: [id], onDelete: SetNull)
|
||||
|
||||
@@index([userId, createdAt])
|
||||
@@index([status, createdAt])
|
||||
@@map("bulk_jobs")
|
||||
}
|
||||
|
||||
enum Role {
|
||||
OWNER
|
||||
ADMIN
|
||||
|
||||
Reference in New Issue
Block a user