import fs from 'fs' import path from 'path' interface BibleMetadata { Country: string flag_Image: string Language: string Language_English: string Vernacular_Bible_Title: string English_Bible_Title: string file_ID: string } interface ParsedVerse { book: string chapter: number verse: number text: string } interface JsonVerse { verseNum: number text: string } interface JsonChapter { chapterNum: number verses: JsonVerse[] } interface JsonBook { bookKey: string name: string testament: string orderNum: number chapters: JsonChapter[] } interface JsonBibleVersion { name: string abbreviation: string language: string description: string country: string englishTitle: string zipFileUrl: string flagImageUrl: string isDefault: boolean books: JsonBook[] } // Book name mappings from VPL format to normalized keys const BOOK_MAPPINGS: Record = { // Old Testament 'GEN': 'genesis', 'EXO': 'exodus', 'LEV': 'leviticus', 'NUM': 'numbers', 'DEU': 'deuteronomy', 'JOS': 'joshua', 'JDG': 'judges', 'RUT': 'ruth', '1SA': '1-samuel', '2SA': '2-samuel', '1KI': '1-kings', '2KI': '2-kings', '1CH': '1-chronicles', '2CH': '2-chronicles', 'EZR': 'ezra', 'NEH': 'nehemiah', 'EST': 'esther', 'JOB': 'job', 'PSA': 'psalms', 'PRO': 'proverbs', 'ECC': 'ecclesiastes', 'SNG': 'song-of-songs', 'ISA': 'isaiah', 'JER': 'jeremiah', 'LAM': 'lamentations', 'EZK': 'ezekiel', 'EZE': 'ezekiel', 'DAN': 'daniel', 'HOS': 'hosea', 'JOL': 'joel', 'JOE': 'joel', 'AMO': 'amos', 'OBA': 'obadiah', 'JON': 'jonah', 'MIC': 'micah', 'NAM': 'nahum', 'NAH': 'nahum', 'HAB': 'habakkuk', 'ZEP': 'zephaniah', 'HAG': 'haggai', 'ZEC': 'zechariah', 'MAL': 'malachi', // New Testament 'MAT': 'matthew', 'MRK': 'mark', 'MAR': 'mark', 'LUK': 'luke', 'JHN': 'john', 'JOH': 'john', 'ACT': 'acts', 'ROM': 'romans', '1CO': '1-corinthians', '2CO': '2-corinthians', 'GAL': 'galatians', 'EPH': 'ephesians', 'PHP': 'philippians', 'PHI': 'philippians', 'COL': 'colossians', '1TH': '1-thessalonians', '2TH': '2-thessalonians', '1TI': '1-timothy', '2TI': '2-timothy', 'TIT': 'titus', 'PHM': 'philemon', 'HEB': 'hebrews', 'JAS': 'james', 'JAM': 'james', '1PE': '1-peter', '2PE': '2-peter', '1JN': '1-john', '1JO': '1-john', '2JN': '2-john', '2JO': '2-john', '3JN': '3-john', '3JO': '3-john', 'JUD': 'jude', 'REV': 'revelation', 'SOL': 'song-of-songs' } // Book order numbers (1-66) const BOOK_ORDER: Record = { 'genesis': 1, 'exodus': 2, 'leviticus': 3, 'numbers': 4, 'deuteronomy': 5, 'joshua': 6, 'judges': 7, 'ruth': 8, '1-samuel': 9, '2-samuel': 10, '1-kings': 11, '2-kings': 12, '1-chronicles': 13, '2-chronicles': 14, 'ezra': 15, 'nehemiah': 16, 'esther': 17, 'job': 18, 'psalms': 19, 'proverbs': 20, 'ecclesiastes': 21, 'song-of-songs': 22, 'isaiah': 23, 'jeremiah': 24, 'lamentations': 25, 'ezekiel': 26, 'daniel': 27, 'hosea': 28, 'joel': 29, 'amos': 30, 'obadiah': 31, 'jonah': 32, 'micah': 33, 'nahum': 34, 'habakkuk': 35, 'zephaniah': 36, 'haggai': 37, 'zechariah': 38, 'malachi': 39, 'matthew': 40, 'mark': 41, 'luke': 42, 'john': 43, 'acts': 44, 'romans': 45, '1-corinthians': 46, '2-corinthians': 47, 'galatians': 48, 'ephesians': 49, 'philippians': 50, 'colossians': 51, '1-thessalonians': 52, '2-thessalonians': 53, '1-timothy': 54, '2-timothy': 55, 'titus': 56, 'philemon': 57, 'hebrews': 58, 'james': 59, '1-peter': 60, '2-peter': 61, '1-john': 62, '2-john': 63, '3-john': 64, 'jude': 65, 'revelation': 66 } function parseVplFile(filePath: string): ParsedVerse[] { const content = fs.readFileSync(filePath, 'utf-8') const lines = content.split('\n').filter(line => line.trim()) const verses: ParsedVerse[] = [] for (const line of lines) { const match = line.match(/^(\w+)\s+(\d+):(\d+)\s+(.+)$/) if (match) { const [, bookCode, chapterStr, verseStr, text] = match const chapter = parseInt(chapterStr, 10) const verse = parseInt(verseStr, 10) if (BOOK_MAPPINGS[bookCode]) { verses.push({ book: BOOK_MAPPINGS[bookCode], chapter, verse, text: text.trim() }) } else { console.warn(`Unknown book code: ${bookCode} in ${filePath}`) } } } return verses } function loadBibleMetadata(): BibleMetadata[] { const csvPath = path.join(process.cwd(), 'bibles', 'bibles_list.csv') const content = fs.readFileSync(csvPath, 'utf-8') const lines = content.split('\n').filter(line => line.trim()) const results: BibleMetadata[] = [] // Parse CSV manually handling quoted fields for (let i = 1; i < lines.length; i++) { const line = lines[i] const values: string[] = [] let current = '' let inQuotes = false for (let j = 0; j < line.length; j++) { const char = line[j] if (char === '"') { inQuotes = !inQuotes } else if (char === ',' && !inQuotes) { values.push(current.trim()) current = '' } else { current += char } } values.push(current.trim()) // Add the last value if (values.length >= 7) { results.push({ Country: values[0] || '', flag_Image: values[1] || '', Language: values[2] || '', Language_English: values[3] || '', Vernacular_Bible_Title: values[4] || '', English_Bible_Title: values[5] || '', file_ID: values[6] || '' }) } } return results } function getTestament(bookKey: string): string { const orderNum = BOOK_ORDER[bookKey] return orderNum <= 39 ? 'Old Testament' : 'New Testament' } function getLanguageCode(language: string): string { // Try to extract ISO language code from language string const langMap: Record = { 'english': 'en', 'spanish': 'es', 'french': 'fr', 'german': 'de', 'portuguese': 'pt', 'italian': 'it', 'dutch': 'nl', 'russian': 'ru', 'chinese': 'zh', 'japanese': 'ja', 'korean': 'ko', 'arabic': 'ar', 'hindi': 'hi', 'romanian': 'ro' } const lowerLang = language.toLowerCase() for (const [key, code] of Object.entries(langMap)) { if (lowerLang.includes(key)) { return code } } // Default to first 2 characters if no mapping found return lowerLang.substring(0, 2) } function convertVplToJson(metadata: BibleMetadata): JsonBibleVersion | null { const vplPath = path.join(process.cwd(), 'bibles', 'ebible_vpl', `${metadata.file_ID}_vpl.txt`) if (!fs.existsSync(vplPath)) { console.warn(`VPL file not found: ${vplPath}`) return null } console.log(`Converting ${metadata.Vernacular_Bible_Title} (${metadata.file_ID})...`) const verses = parseVplFile(vplPath) if (verses.length === 0) { console.warn(`No verses found in ${vplPath}`) return null } // Create Bible version metadata const languageCode = getLanguageCode(metadata.Language_English) const zipFileUrl = `https://ebible.org/Scriptures/${metadata.file_ID}_vpl.zip` const bibleVersion: JsonBibleVersion = { name: metadata.Vernacular_Bible_Title, abbreviation: metadata.file_ID.toUpperCase(), language: languageCode, description: `${metadata.English_Bible_Title} - ${metadata.Country}`, country: metadata.Country.trim(), englishTitle: metadata.English_Bible_Title, zipFileUrl: zipFileUrl, flagImageUrl: metadata.flag_Image, isDefault: false, books: [] } // Group verses by book const bookGroups = new Map() for (const verse of verses) { if (!bookGroups.has(verse.book)) { bookGroups.set(verse.book, []) } bookGroups.get(verse.book)!.push(verse) } // Convert each book for (const [bookKey, bookVerses] of bookGroups) { const orderNum = BOOK_ORDER[bookKey] const testament = getTestament(bookKey) const book: JsonBook = { bookKey: bookKey, name: bookKey.charAt(0).toUpperCase() + bookKey.slice(1).replace('-', ' '), testament: testament, orderNum: orderNum, chapters: [] } // Group verses by chapter const chapterGroups = new Map() for (const verse of bookVerses) { if (!chapterGroups.has(verse.chapter)) { chapterGroups.set(verse.chapter, []) } chapterGroups.get(verse.chapter)!.push(verse) } // Convert each chapter for (const [chapterNum, chapterVerses] of chapterGroups) { const chapter: JsonChapter = { chapterNum: chapterNum, verses: [] } // Sort verses by verse number chapterVerses.sort((a, b) => a.verse - b.verse) // Convert verses for (const verse of chapterVerses) { chapter.verses.push({ verseNum: verse.verse, text: verse.text }) } book.chapters.push(chapter) } // Sort chapters by chapter number book.chapters.sort((a, b) => a.chapterNum - b.chapterNum) bibleVersion.books.push(book) } // Sort books by order number bibleVersion.books.sort((a, b) => a.orderNum - b.orderNum) console.log(`āœ… Converted ${verses.length} verses in ${bibleVersion.books.length} books`) return bibleVersion } async function main() { console.log('šŸš€ Starting VPL to JSON conversion...') const outputDir = path.join(process.cwd(), 'bibles', 'json') const logFilePath = path.join(outputDir, 'conversion_log.txt') const errorLogPath = path.join(outputDir, 'conversion_errors.txt') // Initialize log files const logMessage = (message: string) => { const timestamp = new Date().toISOString() const logEntry = `[${timestamp}] ${message}\n` console.log(message) fs.appendFileSync(logFilePath, logEntry, 'utf-8') } const logError = (message: string, error?: any) => { const timestamp = new Date().toISOString() const errorEntry = `[${timestamp}] ERROR: ${message}\n${error ? `Details: ${error.toString()}\n` : ''}\n` console.error(message, error || '') fs.appendFileSync(errorLogPath, errorEntry, 'utf-8') } try { // Create output directory if (!fs.existsSync(outputDir)) { fs.mkdirSync(outputDir, { recursive: true }) logMessage(`šŸ“ Created output directory: ${outputDir}`) } // Initialize log files fs.writeFileSync(logFilePath, `VPL to JSON Conversion Log - Started at ${new Date().toISOString()}\n`, 'utf-8') fs.writeFileSync(errorLogPath, `VPL to JSON Conversion Errors - Started at ${new Date().toISOString()}\n`, 'utf-8') // Load Bible metadata logMessage('šŸ“‹ Loading Bible metadata...') const metadata = loadBibleMetadata() logMessage(`Found ${metadata.length} Bible versions to convert`) // Force re-conversion of all files for end-to-end testing logMessage(`Force re-converting all files for complete end-to-end process`) // Convert each Bible version let converted = 0 let skipped = 0 let resumed = 0 for (const bibleData of metadata) { try { const jsonBible = convertVplToJson(bibleData) if (jsonBible) { // Save individual Bible JSON file const filename = `${bibleData.file_ID}_bible.json` const filepath = path.join(outputDir, filename) fs.writeFileSync(filepath, JSON.stringify(jsonBible, null, 2), 'utf-8') logMessage(`šŸ’¾ Saved: ${filename}`) converted++ // Progress update every 10 conversions if (converted % 10 === 0) { logMessage(`šŸ“ˆ Progress: ${converted}/${metadata.length} converted...`) // Force garbage collection to prevent memory buildup if (global.gc) { global.gc() } } } else { skipped++ logError(`Skipped ${bibleData.file_ID}: No valid Bible data found`) } } catch (error) { logError(`Failed to convert ${bibleData.file_ID}`, error) skipped++ } } // Skip creating large master file to prevent memory issues logMessage('\nšŸ“¦ Skipping master file creation to prevent memory issues...') // Final summary const totalProcessed = converted + skipped + resumed logMessage('\nāœ… VPL to JSON conversion completed!') logMessage(`šŸ“Š Final Summary:`) logMessage(` - Successfully converted: ${converted}`) logMessage(` - Already converted (resumed): ${resumed}`) logMessage(` - Skipped due to errors: ${skipped}`) logMessage(` - Total processed: ${totalProcessed}`) logMessage(` - Output directory: ${outputDir}`) logMessage(` - Individual files: ${converted + resumed} Bible JSON files`) logMessage(` - Log file: conversion_log.txt`) logMessage(` - Error file: conversion_errors.txt`) // Create a simple summary without loading all data into memory const allJsonFiles = fs.readdirSync(outputDir).filter(f => f.endsWith('_bible.json')) const summaryData = { conversionSummary: { totalJsonFiles: allJsonFiles.length, totalAttempted: metadata.length, successfullyConverted: converted, alreadyExisted: resumed, skippedDueToErrors: skipped, completedAt: new Date().toISOString() }, availableFiles: allJsonFiles.sort() } const summaryFile = path.join(outputDir, 'conversion_summary.json') fs.writeFileSync(summaryFile, JSON.stringify(summaryData, null, 2), 'utf-8') logMessage(`šŸ“Š Summary file saved: conversion_summary.json`) } catch (error) { logError('Conversion failed with fatal error', error) throw error } } main() .then(() => { console.log('šŸŽ‰ Conversion process completed successfully!') process.exit(0) }) .catch((error) => { console.error('šŸ’„ Conversion failed:', error) process.exit(1) })