biblical-guide.com/scripts/convert_bibles_to_json.ts

import fs from 'fs'
import path from 'path'

interface BibleMetadata {
  Country: string
  flag_Image: string
  Language: string
  Language_English: string
  Vernacular_Bible_Title: string
  English_Bible_Title: string
  file_ID: string
}

interface ParsedVerse {
  book: string
  chapter: number
  verse: number
  text: string
}

interface JsonVerse {
  verseNum: number
  text: string
}

interface JsonChapter {
  chapterNum: number
  verses: JsonVerse[]
}

interface JsonBook {
  bookKey: string
  name: string
  testament: string
  orderNum: number
  chapters: JsonChapter[]
}

interface JsonBibleVersion {
  name: string
  abbreviation: string
  language: string
  description: string
  country: string
  englishTitle: string
  zipFileUrl: string
  flagImageUrl: string
  isDefault: boolean
  books: JsonBook[]
}

// Book name mappings from VPL format to normalized keys
const BOOK_MAPPINGS: Record<string, string> = {
  // Old Testament
  'GEN': 'genesis', 'EXO': 'exodus', 'LEV': 'leviticus', 'NUM': 'numbers', 'DEU': 'deuteronomy',
  'JOS': 'joshua', 'JDG': 'judges', 'RUT': 'ruth', '1SA': '1-samuel', '2SA': '2-samuel',
  '1KI': '1-kings', '2KI': '2-kings', '1CH': '1-chronicles', '2CH': '2-chronicles',
  'EZR': 'ezra', 'NEH': 'nehemiah', 'EST': 'esther', 'JOB': 'job', 'PSA': 'psalms',
  'PRO': 'proverbs', 'ECC': 'ecclesiastes', 'SNG': 'song-of-songs', 'ISA': 'isaiah',
  'JER': 'jeremiah', 'LAM': 'lamentations', 'EZK': 'ezekiel', 'EZE': 'ezekiel', 'DAN': 'daniel',
  'HOS': 'hosea', 'JOL': 'joel', 'JOE': 'joel', 'AMO': 'amos', 'OBA': 'obadiah', 'JON': 'jonah',
  'MIC': 'micah', 'NAM': 'nahum', 'NAH': 'nahum', 'HAB': 'habakkuk', 'ZEP': 'zephaniah',
  'HAG': 'haggai', 'ZEC': 'zechariah', 'MAL': 'malachi',

  // New Testament
  'MAT': 'matthew', 'MRK': 'mark', 'MAR': 'mark', 'LUK': 'luke', 'JHN': 'john', 'JOH': 'john', 'ACT': 'acts',
  'ROM': 'romans', '1CO': '1-corinthians', '2CO': '2-corinthians', 'GAL': 'galatians',
  'EPH': 'ephesians', 'PHP': 'philippians', 'PHI': 'philippians', 'COL': 'colossians', '1TH': '1-thessalonians',
  '2TH': '2-thessalonians', '1TI': '1-timothy', '2TI': '2-timothy', 'TIT': 'titus',
  'PHM': 'philemon', 'HEB': 'hebrews', 'JAS': 'james', 'JAM': 'james', '1PE': '1-peter', '2PE': '2-peter',
  '1JN': '1-john', '1JO': '1-john', '2JN': '2-john', '2JO': '2-john', '3JN': '3-john',
  '3JO': '3-john', 'JUD': 'jude', 'REV': 'revelation', 'SOL': 'song-of-songs'
}

// Book order numbers (1-66)
const BOOK_ORDER: Record<string, number> = {
  'genesis': 1, 'exodus': 2, 'leviticus': 3, 'numbers': 4, 'deuteronomy': 5,
  'joshua': 6, 'judges': 7, 'ruth': 8, '1-samuel': 9, '2-samuel': 10,
  '1-kings': 11, '2-kings': 12, '1-chronicles': 13, '2-chronicles': 14,
  'ezra': 15, 'nehemiah': 16, 'esther': 17, 'job': 18, 'psalms': 19,
  'proverbs': 20, 'ecclesiastes': 21, 'song-of-songs': 22, 'isaiah': 23,
  'jeremiah': 24, 'lamentations': 25, 'ezekiel': 26, 'daniel': 27,
  'hosea': 28, 'joel': 29, 'amos': 30, 'obadiah': 31, 'jonah': 32,
  'micah': 33, 'nahum': 34, 'habakkuk': 35, 'zephaniah': 36,
  'haggai': 37, 'zechariah': 38, 'malachi': 39,

  'matthew': 40, 'mark': 41, 'luke': 42, 'john': 43, 'acts': 44,
  'romans': 45, '1-corinthians': 46, '2-corinthians': 47, 'galatians': 48,
  'ephesians': 49, 'philippians': 50, 'colossians': 51, '1-thessalonians': 52,
  '2-thessalonians': 53, '1-timothy': 54, '2-timothy': 55, 'titus': 56,
  'philemon': 57, 'hebrews': 58, 'james': 59, '1-peter': 60, '2-peter': 61,
  '1-john': 62, '2-john': 63, '3-john': 64, 'jude': 65, 'revelation': 66
}

function parseVplFile(filePath: string): ParsedVerse[] {
  const content = fs.readFileSync(filePath, 'utf-8')
  const lines = content.split('\n').filter(line => line.trim())
  const verses: ParsedVerse[] = []

  for (const line of lines) {
    const match = line.match(/^(\w+)\s+(\d+):(\d+)\s+(.+)$/)
    if (match) {
      const [, bookCode, chapterStr, verseStr, text] = match
      const chapter = parseInt(chapterStr, 10)
      const verse = parseInt(verseStr, 10)

      if (BOOK_MAPPINGS[bookCode]) {
        verses.push({
          book: BOOK_MAPPINGS[bookCode],
          chapter,
          verse,
          text: text.trim()
        })
      } else {
        console.warn(`Unknown book code: ${bookCode} in ${filePath}`)
      }
    }
  }

  return verses
}

function loadBibleMetadata(): BibleMetadata[] {
  const csvPath = path.join(process.cwd(), 'bibles', 'bibles_list.csv')
  const content = fs.readFileSync(csvPath, 'utf-8')
  const lines = content.split('\n').filter(line => line.trim())
  const results: BibleMetadata[] = []

  // Parse CSV manually handling quoted fields
  for (let i = 1; i < lines.length; i++) {
    const line = lines[i]
    const values: string[] = []
    let current = ''
    let inQuotes = false

    for (let j = 0; j < line.length; j++) {
      const char = line[j]
      if (char === '"') {
        inQuotes = !inQuotes
      } else if (char === ',' && !inQuotes) {
        values.push(current.trim())
        current = ''
      } else {
        current += char
      }
    }
    values.push(current.trim()) // Add the last value

    if (values.length >= 7) {
      results.push({
        Country: values[0] || '',
        flag_Image: values[1] || '',
        Language: values[2] || '',
        Language_English: values[3] || '',
        Vernacular_Bible_Title: values[4] || '',
        English_Bible_Title: values[5] || '',
        file_ID: values[6] || ''
      })
    }
  }

  return results
}

function getTestament(bookKey: string): string {
  const orderNum = BOOK_ORDER[bookKey]
  return orderNum <= 39 ? 'Old Testament' : 'New Testament'
}

function getLanguageCode(language: string): string {
  // Try to extract ISO language code from language string
  const langMap: Record<string, string> = {
    'english': 'en',
    'spanish': 'es',
    'french': 'fr',
    'german': 'de',
    'portuguese': 'pt',
    'italian': 'it',
    'dutch': 'nl',
    'russian': 'ru',
    'chinese': 'zh',
    'japanese': 'ja',
    'korean': 'ko',
    'arabic': 'ar',
    'hindi': 'hi',
    'romanian': 'ro'
  }

  const lowerLang = language.toLowerCase()
  for (const [key, code] of Object.entries(langMap)) {
    if (lowerLang.includes(key)) {
      return code
    }
  }

  // Default to first 2 characters if no mapping found
  return lowerLang.substring(0, 2)
}

function convertVplToJson(metadata: BibleMetadata): JsonBibleVersion | null {
  const vplPath = path.join(process.cwd(), 'bibles', 'ebible_vpl', `${metadata.file_ID}_vpl.txt`)

  if (!fs.existsSync(vplPath)) {
    console.warn(`VPL file not found: ${vplPath}`)
    return null
  }

  console.log(`Converting ${metadata.Vernacular_Bible_Title} (${metadata.file_ID})...`)

  const verses = parseVplFile(vplPath)
  if (verses.length === 0) {
    console.warn(`No verses found in ${vplPath}`)
    return null
  }

  // Create Bible version metadata
  const languageCode = getLanguageCode(metadata.Language_English)
  const zipFileUrl = `https://ebible.org/Scriptures/${metadata.file_ID}_vpl.zip`

  const bibleVersion: JsonBibleVersion = {
    name: metadata.Vernacular_Bible_Title,
    abbreviation: metadata.file_ID.toUpperCase(),
    language: languageCode,
    description: `${metadata.English_Bible_Title} - ${metadata.Country}`,
    country: metadata.Country.trim(),
    englishTitle: metadata.English_Bible_Title,
    zipFileUrl: zipFileUrl,
    flagImageUrl: metadata.flag_Image,
    isDefault: false,
    books: []
  }

  // Group verses by book
  const bookGroups = new Map<string, ParsedVerse[]>()
  for (const verse of verses) {
    if (!bookGroups.has(verse.book)) {
      bookGroups.set(verse.book, [])
    }
    bookGroups.get(verse.book)!.push(verse)
  }

  // Convert each book
  for (const [bookKey, bookVerses] of bookGroups) {
    const orderNum = BOOK_ORDER[bookKey]
    const testament = getTestament(bookKey)

    const book: JsonBook = {
      bookKey: bookKey,
      name: bookKey.charAt(0).toUpperCase() + bookKey.slice(1).replace('-', ' '),
      testament: testament,
      orderNum: orderNum,
      chapters: []
    }

    // Group verses by chapter
    const chapterGroups = new Map<number, ParsedVerse[]>()
    for (const verse of bookVerses) {
      if (!chapterGroups.has(verse.chapter)) {
        chapterGroups.set(verse.chapter, [])
      }
      chapterGroups.get(verse.chapter)!.push(verse)
    }

    // Convert each chapter
    for (const [chapterNum, chapterVerses] of chapterGroups) {
      const chapter: JsonChapter = {
        chapterNum: chapterNum,
        verses: []
      }

      // Sort verses by verse number
      chapterVerses.sort((a, b) => a.verse - b.verse)

      // Convert verses
      for (const verse of chapterVerses) {
        chapter.verses.push({
          verseNum: verse.verse,
          text: verse.text
        })
      }

      book.chapters.push(chapter)
    }

    // Sort chapters by chapter number
    book.chapters.sort((a, b) => a.chapterNum - b.chapterNum)
    bibleVersion.books.push(book)
  }

  // Sort books by order number
  bibleVersion.books.sort((a, b) => a.orderNum - b.orderNum)

  console.log(`✅ Converted ${verses.length} verses in ${bibleVersion.books.length} books`)
  return bibleVersion
}

async function main() {
  console.log('🚀 Starting VPL to JSON conversion...')

  const outputDir = path.join(process.cwd(), 'bibles', 'json')
  const logFilePath = path.join(outputDir, 'conversion_log.txt')
  const errorLogPath = path.join(outputDir, 'conversion_errors.txt')

  // Initialize log files
  const logMessage = (message: string) => {
    const timestamp = new Date().toISOString()
    const logEntry = `[${timestamp}] ${message}\n`
    console.log(message)
    fs.appendFileSync(logFilePath, logEntry, 'utf-8')
  }

  const logError = (message: string, error?: any) => {
    const timestamp = new Date().toISOString()
    const errorEntry = `[${timestamp}] ERROR: ${message}\n${error ? `Details: ${error.toString()}\n` : ''}\n`
    console.error(message, error || '')
    fs.appendFileSync(errorLogPath, errorEntry, 'utf-8')
  }

  try {
    // Create output directory
    if (!fs.existsSync(outputDir)) {
      fs.mkdirSync(outputDir, { recursive: true })
      logMessage(`📁 Created output directory: ${outputDir}`)
    }

    // Initialize log files
    fs.writeFileSync(logFilePath, `VPL to JSON Conversion Log - Started at ${new Date().toISOString()}\n`, 'utf-8')
    fs.writeFileSync(errorLogPath, `VPL to JSON Conversion Errors - Started at ${new Date().toISOString()}\n`, 'utf-8')

    // Load Bible metadata
    logMessage('📋 Loading Bible metadata...')
    const metadata = loadBibleMetadata()
    logMessage(`Found ${metadata.length} Bible versions to convert`)

    // Force re-conversion of all files for end-to-end testing
    logMessage(`Force re-converting all files for complete end-to-end process`)

    // Convert each Bible version
    let converted = 0
    let skipped = 0
    let resumed = 0

    for (const bibleData of metadata) {

      try {
        const jsonBible = convertVplToJson(bibleData)
        if (jsonBible) {
          // Save individual Bible JSON file
          const filename = `${bibleData.file_ID}_bible.json`
          const filepath = path.join(outputDir, filename)

          fs.writeFileSync(filepath, JSON.stringify(jsonBible, null, 2), 'utf-8')
          logMessage(`💾 Saved: ${filename}`)
          converted++

          // Progress update every 10 conversions
          if (converted % 10 === 0) {
            logMessage(`📈 Progress: ${converted}/${metadata.length} converted...`)

            // Force garbage collection to prevent memory buildup
            if (global.gc) {
              global.gc()
            }
          }
        } else {
          skipped++
          logError(`Skipped ${bibleData.file_ID}: No valid Bible data found`)
        }
      } catch (error) {
        logError(`Failed to convert ${bibleData.file_ID}`, error)
        skipped++
      }
    }

    // Skip creating large master file to prevent memory issues
    logMessage('\n📦 Skipping master file creation to prevent memory issues...')

    // Final summary
    const totalProcessed = converted + skipped + resumed
    logMessage('\n✅ VPL to JSON conversion completed!')
    logMessage(`📊 Final Summary:`)
    logMessage(`   - Successfully converted: ${converted}`)
    logMessage(`   - Already converted (resumed): ${resumed}`)
    logMessage(`   - Skipped due to errors: ${skipped}`)
    logMessage(`   - Total processed: ${totalProcessed}`)
    logMessage(`   - Output directory: ${outputDir}`)
    logMessage(`   - Individual files: ${converted + resumed} Bible JSON files`)
    logMessage(`   - Log file: conversion_log.txt`)
    logMessage(`   - Error file: conversion_errors.txt`)

    // Create a simple summary without loading all data into memory
    const allJsonFiles = fs.readdirSync(outputDir).filter(f => f.endsWith('_bible.json'))
    const summaryData = {
      conversionSummary: {
        totalJsonFiles: allJsonFiles.length,
        totalAttempted: metadata.length,
        successfullyConverted: converted,
        alreadyExisted: resumed,
        skippedDueToErrors: skipped,
        completedAt: new Date().toISOString()
      },
      availableFiles: allJsonFiles.sort()
    }

    const summaryFile = path.join(outputDir, 'conversion_summary.json')
    fs.writeFileSync(summaryFile, JSON.stringify(summaryData, null, 2), 'utf-8')
    logMessage(`📊 Summary file saved: conversion_summary.json`)

  } catch (error) {
    logError('Conversion failed with fatal error', error)
    throw error
  }
}

main()
  .then(() => {
    console.log('🎉 Conversion process completed successfully!')
    process.exit(0)
  })
  .catch((error) => {
    console.error('💥 Conversion failed:', error)
    process.exit(1)
  })