Fix bulk CSV processing and improve user registration
- Fix bulk CSV upload functionality that was returning HTML errors - Implement proper project/organization handling for logged-in vs anonymous users - Update user registration to create unique Default Organization and Default Project - Fix frontend API URL configuration for bulk upload endpoints - Resolve foreign key constraint violations in bulk processing - Update BulkProcessorService to use in-memory processing instead of Redis - Fix redirect-tracker service to handle missing project IDs properly - Update Prisma schema for optional project relationships in bulk jobs - Improve registration form UI with better password validation and alignment
This commit is contained in:
Andrei
126
apps/api/dist/routes/bulk.routes.js
vendored
126
apps/api/dist/routes/bulk.routes.js
vendored
@@ -1,4 +1,37 @@
|
||||
"use strict";
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
var desc = Object.getOwnPropertyDescriptor(m, k);
|
||||
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
||||
desc = { enumerable: true, get: function() { return m[k]; } };
|
||||
}
|
||||
Object.defineProperty(o, k2, desc);
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || (function () {
|
||||
var ownKeys = function(o) {
|
||||
ownKeys = Object.getOwnPropertyNames || function (o) {
|
||||
var ar = [];
|
||||
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
||||
return ar;
|
||||
};
|
||||
return ownKeys(o);
|
||||
};
|
||||
return function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
})();
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
@@ -11,6 +44,7 @@ const auth_middleware_1 = require("../middleware/auth.middleware");
|
||||
const rate_limit_middleware_1 = require("../middleware/rate-limit.middleware");
|
||||
const bulk_processor_service_1 = require("../services/bulk-processor.service");
|
||||
const logger_1 = require("../lib/logger");
|
||||
const prisma_1 = require("../lib/prisma");
|
||||
const router = express_1.default.Router();
|
||||
const bulkProcessor = new bulk_processor_service_1.BulkProcessorService();
|
||||
const upload = (0, multer_1.default)({
|
||||
@@ -63,20 +97,100 @@ router.post('/upload', auth_middleware_1.requireAuth, rate_limit_middleware_1.bu
|
||||
}
|
||||
const userId = req.user.id;
|
||||
const organizationId = req.user.memberships?.[0]?.organizationId;
|
||||
const projectId = req.body.projectId || 'default-project';
|
||||
let projectId = req.body.projectId;
|
||||
if (!projectId && organizationId) {
|
||||
let defaultProject = await prisma_1.prisma.project.findFirst({
|
||||
where: { orgId: organizationId }
|
||||
});
|
||||
if (!defaultProject) {
|
||||
defaultProject = await prisma_1.prisma.project.create({
|
||||
data: {
|
||||
name: 'Default Project',
|
||||
orgId: organizationId,
|
||||
settingsJson: {}
|
||||
}
|
||||
});
|
||||
}
|
||||
projectId = defaultProject.id;
|
||||
}
|
||||
if (!projectId) {
|
||||
logger_1.logger.warn('No valid project ID found for bulk processing', { userId, organizationId });
|
||||
}
|
||||
const options = req.body.options ? JSON.parse(req.body.options) : {};
|
||||
logger_1.logger.info(`Processing CSV upload for user: ${userId}`, {
|
||||
filename: req.file.originalname,
|
||||
size: req.file.size,
|
||||
});
|
||||
const job = await bulkProcessor.createBulkJobFromCsv(userId, organizationId, req.file.path, projectId, options);
|
||||
const jobId = `bulk_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
||||
const csvContent = await promises_1.default.readFile(req.file.path, 'utf-8');
|
||||
const lines = csvContent.split('\n').filter(line => line.trim() && !line.startsWith('url'));
|
||||
const urls = lines.map(line => {
|
||||
const [url, label] = line.split(',').map(s => s.trim());
|
||||
return { url, label: label || '' };
|
||||
}).filter(item => item.url && item.url.startsWith('http'));
|
||||
logger_1.logger.info(`Bulk upload processing: ${urls.length} URLs for user ${userId}`, {
|
||||
projectId,
|
||||
organizationId,
|
||||
hasProject: !!projectId
|
||||
});
|
||||
const results = [];
|
||||
let processed = 0;
|
||||
let successful = 0;
|
||||
let failed = 0;
|
||||
for (const urlData of urls) {
|
||||
try {
|
||||
const { RedirectTrackerService } = await Promise.resolve().then(() => __importStar(require('../services/redirect-tracker.service')));
|
||||
const tracker = new RedirectTrackerService();
|
||||
const trackingRequest = {
|
||||
url: urlData.url,
|
||||
method: 'GET',
|
||||
maxHops: 10,
|
||||
timeout: 15000,
|
||||
enableSSLAnalysis: true,
|
||||
enableSEOAnalysis: true,
|
||||
enableSecurityAnalysis: true,
|
||||
};
|
||||
if (projectId) {
|
||||
trackingRequest.projectId = projectId;
|
||||
}
|
||||
const result = await tracker.trackUrl(trackingRequest, userId);
|
||||
results.push({
|
||||
url: urlData.url,
|
||||
label: urlData.label,
|
||||
status: 'success',
|
||||
checkId: result.id,
|
||||
finalUrl: result.finalUrl,
|
||||
redirectCount: result.redirectCount,
|
||||
});
|
||||
successful++;
|
||||
}
|
||||
catch (error) {
|
||||
logger_1.logger.error(`Failed to process URL ${urlData.url}:`, error);
|
||||
results.push({
|
||||
url: urlData.url,
|
||||
label: urlData.label,
|
||||
status: 'failed',
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
});
|
||||
failed++;
|
||||
}
|
||||
processed++;
|
||||
}
|
||||
await promises_1.default.unlink(req.file.path).catch(() => { });
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
jobId: job.id,
|
||||
status: job.status,
|
||||
progress: job.progress,
|
||||
estimatedCompletionAt: job.estimatedCompletionAt,
|
||||
jobId: jobId,
|
||||
status: 'COMPLETED',
|
||||
progress: {
|
||||
total: urls.length,
|
||||
processed: processed,
|
||||
successful: successful,
|
||||
failed: failed,
|
||||
},
|
||||
results: results,
|
||||
message: `Bulk processing completed: ${successful} successful, ${failed} failed out of ${urls.length} URLs.`,
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user