feat(phase-6): Bulk CSV processing and background worker implementation

- Add BulkJob model to Prisma schema with relations
- Implement BulkProcessorService for CSV parsing and job management
- Create BulkTrackingWorker for background processing with BullMQ
- Add comprehensive bulk API routes (upload, jobs, progress, export)
- Integrate multer for CSV file uploads with validation
- Add job progress tracking and estimation
- Implement CSV export functionality for results
- Add queue statistics and cleanup endpoints
- Create shared types for bulk processing
- Add comprehensive test suite for all bulk functionality
- Implement graceful worker shutdown and error handling
- Add rate limiting and authentication for all bulk endpoints

Backward compatibility: Maintained for /api/track and /api/v1/track

packages/database/prisma/schema.prisma

@@ -22,6 +22,7 @@ model User {
   
   memberships   OrgMembership[]
   auditLogs     AuditLog[]
+  bulkJobs      BulkJob[]
   
   @@map("users")
 }
@@ -36,6 +37,7 @@ model Organization {
   projects    Project[]
   apiKeys     ApiKey[]
   auditLogs   AuditLog[]
+  bulkJobs    BulkJob[]
   
   @@map("organizations")
 }
@@ -212,6 +214,32 @@ model AuditLog {
   @@map("audit_logs")
 }
 
+model BulkJob {
+  id             String    @id
+  userId         String
+  organizationId String?
+  projectId      String?
+  status         String    // 'pending' | 'processing' | 'completed' | 'failed' | 'cancelled'
+  totalUrls      Int
+  processedUrls  Int       @default(0)
+  successfulUrls Int       @default(0)
+  failedUrls     Int       @default(0)
+  configJson     Json      // Job configuration (options)
+  urlsJson       Json      // Array of URLs to process
+  resultsJson    Json?     // Array of results
+  createdAt      DateTime  @default(now())
+  startedAt      DateTime?
+  finishedAt     DateTime?
+  
+  user         User          @relation(fields: [userId], references: [id], onDelete: Cascade)
+  organization Organization? @relation(fields: [organizationId], references: [id], onDelete: SetNull)
+  project      Project?      @relation(fields: [projectId], references: [id], onDelete: SetNull)
+  
+  @@index([userId, createdAt])
+  @@index([status, createdAt])
+  @@map("bulk_jobs")
+}
+
 enum Role {
   OWNER
   ADMIN

packages/shared/src/types/api.ts

@@ -222,4 +222,92 @@ export const ErrorResponseSchema = z.object({
   details: z.any().optional(),
 });
 
-export type ErrorResponse = z.infer<typeof ErrorResponseSchema>;
+export type ErrorResponse = z.infer<typeof ErrorResponseSchema>;
+
+// ============================================================================
+// BULK PROCESSING TYPES
+// ============================================================================
+
+export const BulkJobStatusSchema = z.enum(['pending', 'processing', 'completed', 'failed', 'cancelled']);
+
+export const BulkJobProgressSchema = z.object({
+  total: z.number(),
+  processed: z.number(),
+  successful: z.number(),
+  failed: z.number(),
+});
+
+export const BulkJobResultSchema = z.object({
+  url: z.string(),
+  label: z.string().optional(),
+  checkId: z.string().optional(),
+  status: z.enum(['success', 'failed']),
+  error: z.string().optional(),
+  timing: z.object({
+    startedAt: z.date(),
+    finishedAt: z.date().optional(),
+    durationMs: z.number().optional(),
+  }),
+});
+
+export const BulkJobSchema = z.object({
+  id: z.string(),
+  userId: z.string(),
+  organizationId: z.string().optional(),
+  projectId: z.string().optional(),
+  status: BulkJobStatusSchema,
+  progress: BulkJobProgressSchema,
+  createdAt: z.date(),
+  startedAt: z.date().optional(),
+  finishedAt: z.date().optional(),
+  estimatedCompletionAt: z.date().optional(),
+  urlCount: z.number(),
+  options: z.object({
+    method: z.enum(['GET', 'POST', 'HEAD']),
+    userAgent: z.string().optional(),
+    maxHops: z.number(),
+    timeout: z.number(),
+    enableSSLAnalysis: z.boolean(),
+    enableSEOAnalysis: z.boolean(),
+    enableSecurityAnalysis: z.boolean(),
+    headers: z.record(z.string()).optional(),
+  }),
+  results: z.array(BulkJobResultSchema).optional(),
+});
+
+export const CreateBulkJobRequestSchema = z.object({
+  projectId: z.string().optional(),
+  urls: z.array(z.object({
+    url: z.string().url(),
+    label: z.string().optional(),
+    metadata: z.record(z.any()).optional(),
+  })).min(1).max(1000),
+  options: z.object({
+    method: z.enum(['GET', 'POST', 'HEAD']).default('GET'),
+    userAgent: z.string().optional(),
+    maxHops: z.number().min(1).max(20).default(10),
+    timeout: z.number().min(1000).max(30000).default(15000),
+    enableSSLAnalysis: z.boolean().default(true),
+    enableSEOAnalysis: z.boolean().default(true),
+    enableSecurityAnalysis: z.boolean().default(true),
+    headers: z.record(z.string()).optional(),
+  }).default({}),
+});
+
+export const BulkStatsSchema = z.object({
+  queue: z.object({
+    waiting: z.number(),
+    active: z.number(),
+    completed: z.number(),
+    failed: z.number(),
+    delayed: z.number(),
+  }),
+  timestamp: z.string(),
+});
+
+export type BulkJobStatus = z.infer<typeof BulkJobStatusSchema>;
+export type BulkJobProgress = z.infer<typeof BulkJobProgressSchema>;
+export type BulkJobResult = z.infer<typeof BulkJobResultSchema>;
+export type BulkJob = z.infer<typeof BulkJobSchema>;
+export type CreateBulkJobRequest = z.infer<typeof CreateBulkJobRequestSchema>;
+export type BulkStats = z.infer<typeof BulkStatsSchema>;