url_tracker_tool/test-phase-6.js

/**
 * Test script for Phase 6: Bulk CSV + Worker
 * Tests bulk processing functionality, CSV upload, and worker integration
 */

const http = require('http');
const fs = require('fs');
const path = require('path');

const BASE_URL = 'http://localhost:3333';

// Helper function to make HTTP requests
function makeRequest(options, data = null) {
  return new Promise((resolve, reject) => {
    const req = http.request(options, (res) => {
      let body = '';
      res.on('data', (chunk) => body += chunk);
      res.on('end', () => {
        try {
          const result = {
            statusCode: res.statusCode,
            headers: res.headers,
            body: res.headers['content-type'] && res.headers['content-type'].includes('application/json') 
              ? JSON.parse(body) 
              : body
          };
          resolve(result);
        } catch (error) {
          resolve({
            statusCode: res.statusCode,
            headers: res.headers,
            body: body
          });
        }
      });
    });

    req.on('error', reject);
    
    if (data) {
      if (typeof data === 'string') {
        req.write(data);
      } else {
        req.write(JSON.stringify(data));
      }
    }
    
    req.end();
  });
}

// Helper function to create test CSV file
function createTestCSV() {
  const csvContent = `url,label,method,max_hops,enable_ssl
https://httpbin.org/redirect/1,Test Redirect 1,GET,5,true
https://httpbin.org/redirect/2,Test Redirect 2,GET,10,true
https://example.com,Example Domain,GET,3,false
https://httpbin.org/status/302,Status 302,GET,5,true
invalid-url,Invalid URL,GET,5,true`;

  const filePath = path.join(__dirname, 'test-urls.csv');
  fs.writeFileSync(filePath, csvContent);
  return filePath;
}

// Helper function to create multipart form data
function createMultipartData(filePath, options = {}) {
  const boundary = '----formdata-test-' + Math.random().toString(36);
  const fileName = path.basename(filePath);
  const fileContent = fs.readFileSync(filePath);
  
  let data = '';
  
  // Add file field
  data += `--${boundary}\r\n`;
  data += `Content-Disposition: form-data; name="file"; filename="${fileName}"\r\n`;
  data += `Content-Type: text/csv\r\n\r\n`;
  data += fileContent;
  data += `\r\n`;
  
  // Add options field
  if (Object.keys(options).length > 0) {
    data += `--${boundary}\r\n`;
    data += `Content-Disposition: form-data; name="options"\r\n\r\n`;
    data += JSON.stringify(options);
    data += `\r\n`;
  }
  
  data += `--${boundary}--\r\n`;
  
  return {
    data: Buffer.from(data),
    boundary: boundary
  };
}

async function runTests() {
  console.log('🧪 Starting Phase 6: Bulk CSV + Worker Tests\n');

  let authToken = null;
  let testJobId = null;
  const csvFilePath = createTestCSV();

  // Test 1: User Registration
  console.log('1️⃣ Testing user registration...');
  try {
    const registerResult = await makeRequest({
      hostname: 'localhost',
      port: 3333,
      path: '/api/v1/auth/register',
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
      },
    }, {
      email: 'bulk-test@example.com',
      name: 'Bulk Test User',
      password: 'bulktest123',
      organizationName: 'Bulk Test Org'
    });

    if (registerResult.statusCode === 201 && registerResult.body.success) {
      authToken = registerResult.body.data.token;
      console.log('✅ User registration successful');
    } else if (registerResult.statusCode === 409) {
      console.log('ℹ️ User already exists, attempting login...');
      
      // Try to login if user exists
      const loginResult = await makeRequest({
        hostname: 'localhost',
        port: 3333,
        path: '/api/v1/auth/login',
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
        },
      }, {
        email: 'bulk-test@example.com',
        password: 'bulktest123'
      });

      if (loginResult.statusCode === 200 && loginResult.body.success) {
        authToken = loginResult.body.data.token;
        console.log('✅ User login successful');
      } else {
        console.log('❌ Login failed:', loginResult.body);
        return;
      }
    } else {
      console.log('❌ Registration failed:', registerResult.body);
      return;
    }
  } catch (error) {
    console.log('❌ Registration/login error:', error.message);
    return;
  }

  // Test 2: Get queue stats (should work before any jobs)
  console.log('\n2️⃣ Testing queue statistics...');
  try {
    const statsResult = await makeRequest({
      hostname: 'localhost',
      port: 3333,
      path: '/api/v2/bulk/stats',
      method: 'GET',
      headers: {
        'Authorization': `Bearer ${authToken}`,
      },
    });

    console.log('Queue stats response:', statsResult.statusCode);
    if (statsResult.statusCode === 200) {
      console.log('✅ Queue stats retrieved:', statsResult.body.data);
    } else {
      console.log('⚠️ Queue stats failed:', statsResult.body);
    }
  } catch (error) {
    console.log('❌ Queue stats error:', error.message);
  }

  // Test 3: Create bulk job with JSON payload
  console.log('\n3️⃣ Testing bulk job creation with JSON...');
  try {
    const createJobResult = await makeRequest({
      hostname: 'localhost',
      port: 3333,
      path: '/api/v2/bulk/jobs',
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${authToken}`,
      },
    }, {
      urls: [
        { url: 'https://httpbin.org/redirect/1', label: 'JSON Test 1' },
        { url: 'https://example.com', label: 'JSON Test 2' }
      ],
      options: {
        method: 'GET',
        maxHops: 5,
        timeout: 10000,
        enableSSLAnalysis: true,
        enableSEOAnalysis: false,
        enableSecurityAnalysis: true
      }
    });

    if (createJobResult.statusCode === 201 && createJobResult.body.success) {
      testJobId = createJobResult.body.data.jobId;
      console.log('✅ Bulk job created:', testJobId);
      console.log('Job status:', createJobResult.body.data.status);
      console.log('URL count:', createJobResult.body.data.urls);
    } else {
      console.log('❌ Bulk job creation failed:', createJobResult.body);
    }
  } catch (error) {
    console.log('❌ Bulk job creation error:', error.message);
  }

  // Test 4: Upload CSV file
  console.log('\n4️⃣ Testing CSV upload...');
  try {
    const multipartData = createMultipartData(csvFilePath, {
      method: 'GET',
      maxHops: 8,
      timeout: 15000
    });

    const uploadResult = await makeRequest({
      hostname: 'localhost',
      port: 3333,
      path: '/api/v2/bulk/upload',
      method: 'POST',
      headers: {
        'Content-Type': `multipart/form-data; boundary=${multipartData.boundary}`,
        'Authorization': `Bearer ${authToken}`,
        'Content-Length': multipartData.data.length,
      },
    }, multipartData.data);

    if (uploadResult.statusCode === 200 && uploadResult.body.success) {
      console.log('✅ CSV upload successful');
      console.log('Job ID:', uploadResult.body.data.jobId);
      console.log('Job status:', uploadResult.body.data.status);
      
      // Use this job for further tests if we don't have one from JSON
      if (!testJobId) {
        testJobId = uploadResult.body.data.jobId;
      }
    } else {
      console.log('❌ CSV upload failed:', uploadResult.body);
    }
  } catch (error) {
    console.log('❌ CSV upload error:', error.message);
  }

  // Test 5: Get user bulk jobs list
  console.log('\n5️⃣ Testing bulk jobs list...');
  try {
    const jobsListResult = await makeRequest({
      hostname: 'localhost',
      port: 3333,
      path: '/api/v2/bulk/jobs?limit=10&offset=0',
      method: 'GET',
      headers: {
        'Authorization': `Bearer ${authToken}`,
      },
    });

    if (jobsListResult.statusCode === 200 && jobsListResult.body.success) {
      console.log('✅ Jobs list retrieved');
      console.log('Job count:', jobsListResult.body.data.length);
      jobsListResult.body.data.forEach((job, index) => {
        console.log(`  Job ${index + 1}: ${job.id} - ${job.status} (${job.urlCount} URLs)`);
      });
    } else {
      console.log('❌ Jobs list failed:', jobsListResult.body);
    }
  } catch (error) {
    console.log('❌ Jobs list error:', error.message);
  }

  // Test 6: Get specific job details
  if (testJobId) {
    console.log('\n6️⃣ Testing job details retrieval...');
    try {
      const jobDetailResult = await makeRequest({
        hostname: 'localhost',
        port: 3333,
        path: `/api/v2/bulk/jobs/${testJobId}`,
        method: 'GET',
        headers: {
          'Authorization': `Bearer ${authToken}`,
        },
      });

      if (jobDetailResult.statusCode === 200 && jobDetailResult.body.success) {
        console.log('✅ Job details retrieved');
        console.log('Job status:', jobDetailResult.body.data.status);
        console.log('Progress:', jobDetailResult.body.data.progress);
        
        if (jobDetailResult.body.data.estimatedCompletionAt) {
          console.log('Estimated completion:', jobDetailResult.body.data.estimatedCompletionAt);
        }
      } else {
        console.log('❌ Job details failed:', jobDetailResult.body);
      }
    } catch (error) {
      console.log('❌ Job details error:', error.message);
    }

    // Test 7: Monitor job progress
    console.log('\n7️⃣ Monitoring job progress...');
    let attempts = 0;
    const maxAttempts = 30; // Wait up to 30 seconds
    
    while (attempts < maxAttempts) {
      try {
        const progressResult = await makeRequest({
          hostname: 'localhost',
          port: 3333,
          path: `/api/v2/bulk/jobs/${testJobId}`,
          method: 'GET',
          headers: {
            'Authorization': `Bearer ${authToken}`,
          },
        });

        if (progressResult.statusCode === 200 && progressResult.body.success) {
          const job = progressResult.body.data;
          console.log(`Progress: ${job.progress.processed}/${job.progress.total} (${job.status})`);
          
          if (job.status === 'completed' || job.status === 'failed') {
            console.log('✅ Job completed!');
            console.log('Final stats:', job.progress);
            
            if (job.results) {
              console.log('Results available:', job.results.length);
            }
            break;
          }
        }
        
        attempts++;
        if (attempts < maxAttempts) {
          await new Promise(resolve => setTimeout(resolve, 1000)); // Wait 1 second
        }
      } catch (error) {
        console.log('❌ Progress monitoring error:', error.message);
        break;
      }
    }

    if (attempts >= maxAttempts) {
      console.log('⚠️ Job monitoring timed out');
    }

    // Test 8: Export results (if job completed)
    console.log('\n8️⃣ Testing results export...');
    try {
      const exportResult = await makeRequest({
        hostname: 'localhost',
        port: 3333,
        path: `/api/v2/bulk/jobs/${testJobId}/export/csv`,
        method: 'GET',
        headers: {
          'Authorization': `Bearer ${authToken}`,
        },
      });

      if (exportResult.statusCode === 200) {
        console.log('✅ Results export successful');
        console.log('Content-Type:', exportResult.headers['content-type']);
        console.log('File size:', exportResult.body.length, 'bytes');
      } else {
        console.log('⚠️ Results export failed:', exportResult.statusCode);
        if (typeof exportResult.body === 'object') {
          console.log('Error:', exportResult.body);
        }
      }
    } catch (error) {
      console.log('❌ Results export error:', error.message);
    }
  }

  // Test 9: Test validation errors
  console.log('\n9️⃣ Testing validation errors...');
  try {
    // Test with invalid URL
    const invalidJobResult = await makeRequest({
      hostname: 'localhost',
      port: 3333,
      path: '/api/v2/bulk/jobs',
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${authToken}`,
      },
    }, {
      urls: [
        { url: 'not-a-valid-url', label: 'Invalid URL' }
      ]
    });

    if (invalidJobResult.statusCode === 400) {
      console.log('✅ Validation correctly rejected invalid URL');
    } else {
      console.log('⚠️ Validation did not catch invalid URL:', invalidJobResult.body);
    }
  } catch (error) {
    console.log('❌ Validation test error:', error.message);
  }

  // Test 10: Test unauthorized access
  console.log('\n🔟 Testing unauthorized access...');
  try {
    const unauthorizedResult = await makeRequest({
      hostname: 'localhost',
      port: 3333,
      path: '/api/v2/bulk/jobs',
      method: 'GET',
      headers: {
        // No authorization header
      },
    });

    if (unauthorizedResult.statusCode === 401) {
      console.log('✅ Unauthorized access correctly blocked');
    } else {
      console.log('⚠️ Unauthorized access was not blocked:', unauthorizedResult.statusCode);
    }
  } catch (error) {
    console.log('❌ Unauthorized test error:', error.message);
  }

  // Cleanup
  try {
    fs.unlinkSync(csvFilePath);
    console.log('\n🧹 Test CSV file cleaned up');
  } catch (error) {
    console.log('\n⚠️ Failed to cleanup test file:', error.message);
  }

  console.log('\n🎉 Phase 6 testing completed!');
  console.log('\nKey features tested:');
  console.log('✓ Bulk job creation with JSON payload');
  console.log('✓ CSV file upload and parsing');
  console.log('✓ Job progress monitoring');
  console.log('✓ Results export to CSV');
  console.log('✓ Queue statistics');
  console.log('✓ Input validation');
  console.log('✓ Authentication/authorization');
  console.log('✓ Error handling');
}

// Error handling
process.on('uncaughtException', (error) => {
  console.log('\n💥 Uncaught Exception:', error.message);
  process.exit(1);
});

process.on('unhandledRejection', (reason) => {
  console.log('\n💥 Unhandled Rejection:', reason);
  process.exit(1);
});

// Run tests
runTests().catch(error => {
  console.log('\n💥 Test execution failed:', error.message);
  process.exit(1);
});