Add comprehensive page management system to admin dashboard
Features added: - Database schema for pages and media files with content types (Rich Text, HTML, Markdown) - Admin API routes for full page CRUD operations - Image upload functionality with file management - Rich text editor using TinyMCE with image insertion - Admin interface for creating/editing pages with SEO options - Dynamic navigation and footer integration - Public page display routes with proper SEO metadata - Support for featured images and content excerpts Admin features: - Create/edit/delete pages with rich content editor - Upload and manage images through media library - Configure pages to appear in navigation or footer - Set page status (Draft, Published, Archived) - SEO title and description management - Real-time preview of content changes 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
423
scripts/convert_bibles_to_json.ts
Normal file
423
scripts/convert_bibles_to_json.ts
Normal file
@@ -0,0 +1,423 @@
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
|
||||
interface BibleMetadata {
|
||||
Country: string
|
||||
flag_Image: string
|
||||
Language: string
|
||||
Language_English: string
|
||||
Vernacular_Bible_Title: string
|
||||
English_Bible_Title: string
|
||||
file_ID: string
|
||||
}
|
||||
|
||||
interface ParsedVerse {
|
||||
book: string
|
||||
chapter: number
|
||||
verse: number
|
||||
text: string
|
||||
}
|
||||
|
||||
interface JsonVerse {
|
||||
verseNum: number
|
||||
text: string
|
||||
}
|
||||
|
||||
interface JsonChapter {
|
||||
chapterNum: number
|
||||
verses: JsonVerse[]
|
||||
}
|
||||
|
||||
interface JsonBook {
|
||||
bookKey: string
|
||||
name: string
|
||||
testament: string
|
||||
orderNum: number
|
||||
chapters: JsonChapter[]
|
||||
}
|
||||
|
||||
interface JsonBibleVersion {
|
||||
name: string
|
||||
abbreviation: string
|
||||
language: string
|
||||
description: string
|
||||
country: string
|
||||
englishTitle: string
|
||||
zipFileUrl: string
|
||||
flagImageUrl: string
|
||||
isDefault: boolean
|
||||
books: JsonBook[]
|
||||
}
|
||||
|
||||
// Book name mappings from VPL format to normalized keys
|
||||
const BOOK_MAPPINGS: Record<string, string> = {
|
||||
// Old Testament
|
||||
'GEN': 'genesis', 'EXO': 'exodus', 'LEV': 'leviticus', 'NUM': 'numbers', 'DEU': 'deuteronomy',
|
||||
'JOS': 'joshua', 'JDG': 'judges', 'RUT': 'ruth', '1SA': '1-samuel', '2SA': '2-samuel',
|
||||
'1KI': '1-kings', '2KI': '2-kings', '1CH': '1-chronicles', '2CH': '2-chronicles',
|
||||
'EZR': 'ezra', 'NEH': 'nehemiah', 'EST': 'esther', 'JOB': 'job', 'PSA': 'psalms',
|
||||
'PRO': 'proverbs', 'ECC': 'ecclesiastes', 'SNG': 'song-of-songs', 'ISA': 'isaiah',
|
||||
'JER': 'jeremiah', 'LAM': 'lamentations', 'EZK': 'ezekiel', 'EZE': 'ezekiel', 'DAN': 'daniel',
|
||||
'HOS': 'hosea', 'JOL': 'joel', 'JOE': 'joel', 'AMO': 'amos', 'OBA': 'obadiah', 'JON': 'jonah',
|
||||
'MIC': 'micah', 'NAM': 'nahum', 'NAH': 'nahum', 'HAB': 'habakkuk', 'ZEP': 'zephaniah',
|
||||
'HAG': 'haggai', 'ZEC': 'zechariah', 'MAL': 'malachi',
|
||||
|
||||
// New Testament
|
||||
'MAT': 'matthew', 'MRK': 'mark', 'MAR': 'mark', 'LUK': 'luke', 'JHN': 'john', 'JOH': 'john', 'ACT': 'acts',
|
||||
'ROM': 'romans', '1CO': '1-corinthians', '2CO': '2-corinthians', 'GAL': 'galatians',
|
||||
'EPH': 'ephesians', 'PHP': 'philippians', 'PHI': 'philippians', 'COL': 'colossians', '1TH': '1-thessalonians',
|
||||
'2TH': '2-thessalonians', '1TI': '1-timothy', '2TI': '2-timothy', 'TIT': 'titus',
|
||||
'PHM': 'philemon', 'HEB': 'hebrews', 'JAS': 'james', 'JAM': 'james', '1PE': '1-peter', '2PE': '2-peter',
|
||||
'1JN': '1-john', '1JO': '1-john', '2JN': '2-john', '2JO': '2-john', '3JN': '3-john',
|
||||
'3JO': '3-john', 'JUD': 'jude', 'REV': 'revelation', 'SOL': 'song-of-songs'
|
||||
}
|
||||
|
||||
// Book order numbers (1-66)
|
||||
const BOOK_ORDER: Record<string, number> = {
|
||||
'genesis': 1, 'exodus': 2, 'leviticus': 3, 'numbers': 4, 'deuteronomy': 5,
|
||||
'joshua': 6, 'judges': 7, 'ruth': 8, '1-samuel': 9, '2-samuel': 10,
|
||||
'1-kings': 11, '2-kings': 12, '1-chronicles': 13, '2-chronicles': 14,
|
||||
'ezra': 15, 'nehemiah': 16, 'esther': 17, 'job': 18, 'psalms': 19,
|
||||
'proverbs': 20, 'ecclesiastes': 21, 'song-of-songs': 22, 'isaiah': 23,
|
||||
'jeremiah': 24, 'lamentations': 25, 'ezekiel': 26, 'daniel': 27,
|
||||
'hosea': 28, 'joel': 29, 'amos': 30, 'obadiah': 31, 'jonah': 32,
|
||||
'micah': 33, 'nahum': 34, 'habakkuk': 35, 'zephaniah': 36,
|
||||
'haggai': 37, 'zechariah': 38, 'malachi': 39,
|
||||
|
||||
'matthew': 40, 'mark': 41, 'luke': 42, 'john': 43, 'acts': 44,
|
||||
'romans': 45, '1-corinthians': 46, '2-corinthians': 47, 'galatians': 48,
|
||||
'ephesians': 49, 'philippians': 50, 'colossians': 51, '1-thessalonians': 52,
|
||||
'2-thessalonians': 53, '1-timothy': 54, '2-timothy': 55, 'titus': 56,
|
||||
'philemon': 57, 'hebrews': 58, 'james': 59, '1-peter': 60, '2-peter': 61,
|
||||
'1-john': 62, '2-john': 63, '3-john': 64, 'jude': 65, 'revelation': 66
|
||||
}
|
||||
|
||||
function parseVplFile(filePath: string): ParsedVerse[] {
|
||||
const content = fs.readFileSync(filePath, 'utf-8')
|
||||
const lines = content.split('\n').filter(line => line.trim())
|
||||
const verses: ParsedVerse[] = []
|
||||
|
||||
for (const line of lines) {
|
||||
const match = line.match(/^(\w+)\s+(\d+):(\d+)\s+(.+)$/)
|
||||
if (match) {
|
||||
const [, bookCode, chapterStr, verseStr, text] = match
|
||||
const chapter = parseInt(chapterStr, 10)
|
||||
const verse = parseInt(verseStr, 10)
|
||||
|
||||
if (BOOK_MAPPINGS[bookCode]) {
|
||||
verses.push({
|
||||
book: BOOK_MAPPINGS[bookCode],
|
||||
chapter,
|
||||
verse,
|
||||
text: text.trim()
|
||||
})
|
||||
} else {
|
||||
console.warn(`Unknown book code: ${bookCode} in ${filePath}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return verses
|
||||
}
|
||||
|
||||
function loadBibleMetadata(): BibleMetadata[] {
|
||||
const csvPath = path.join(process.cwd(), 'bibles', 'bibles_list.csv')
|
||||
const content = fs.readFileSync(csvPath, 'utf-8')
|
||||
const lines = content.split('\n').filter(line => line.trim())
|
||||
const results: BibleMetadata[] = []
|
||||
|
||||
// Parse CSV manually handling quoted fields
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const line = lines[i]
|
||||
const values: string[] = []
|
||||
let current = ''
|
||||
let inQuotes = false
|
||||
|
||||
for (let j = 0; j < line.length; j++) {
|
||||
const char = line[j]
|
||||
if (char === '"') {
|
||||
inQuotes = !inQuotes
|
||||
} else if (char === ',' && !inQuotes) {
|
||||
values.push(current.trim())
|
||||
current = ''
|
||||
} else {
|
||||
current += char
|
||||
}
|
||||
}
|
||||
values.push(current.trim()) // Add the last value
|
||||
|
||||
if (values.length >= 7) {
|
||||
results.push({
|
||||
Country: values[0] || '',
|
||||
flag_Image: values[1] || '',
|
||||
Language: values[2] || '',
|
||||
Language_English: values[3] || '',
|
||||
Vernacular_Bible_Title: values[4] || '',
|
||||
English_Bible_Title: values[5] || '',
|
||||
file_ID: values[6] || ''
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
function getTestament(bookKey: string): string {
|
||||
const orderNum = BOOK_ORDER[bookKey]
|
||||
return orderNum <= 39 ? 'Old Testament' : 'New Testament'
|
||||
}
|
||||
|
||||
function getLanguageCode(language: string): string {
|
||||
// Try to extract ISO language code from language string
|
||||
const langMap: Record<string, string> = {
|
||||
'english': 'en',
|
||||
'spanish': 'es',
|
||||
'french': 'fr',
|
||||
'german': 'de',
|
||||
'portuguese': 'pt',
|
||||
'italian': 'it',
|
||||
'dutch': 'nl',
|
||||
'russian': 'ru',
|
||||
'chinese': 'zh',
|
||||
'japanese': 'ja',
|
||||
'korean': 'ko',
|
||||
'arabic': 'ar',
|
||||
'hindi': 'hi',
|
||||
'romanian': 'ro'
|
||||
}
|
||||
|
||||
const lowerLang = language.toLowerCase()
|
||||
for (const [key, code] of Object.entries(langMap)) {
|
||||
if (lowerLang.includes(key)) {
|
||||
return code
|
||||
}
|
||||
}
|
||||
|
||||
// Default to first 2 characters if no mapping found
|
||||
return lowerLang.substring(0, 2)
|
||||
}
|
||||
|
||||
function convertVplToJson(metadata: BibleMetadata): JsonBibleVersion | null {
|
||||
const vplPath = path.join(process.cwd(), 'bibles', 'ebible_vpl', `${metadata.file_ID}_vpl.txt`)
|
||||
|
||||
if (!fs.existsSync(vplPath)) {
|
||||
console.warn(`VPL file not found: ${vplPath}`)
|
||||
return null
|
||||
}
|
||||
|
||||
console.log(`Converting ${metadata.Vernacular_Bible_Title} (${metadata.file_ID})...`)
|
||||
|
||||
const verses = parseVplFile(vplPath)
|
||||
if (verses.length === 0) {
|
||||
console.warn(`No verses found in ${vplPath}`)
|
||||
return null
|
||||
}
|
||||
|
||||
// Create Bible version metadata
|
||||
const languageCode = getLanguageCode(metadata.Language_English)
|
||||
const zipFileUrl = `https://ebible.org/Scriptures/${metadata.file_ID}_vpl.zip`
|
||||
|
||||
const bibleVersion: JsonBibleVersion = {
|
||||
name: metadata.Vernacular_Bible_Title,
|
||||
abbreviation: metadata.file_ID.toUpperCase(),
|
||||
language: languageCode,
|
||||
description: `${metadata.English_Bible_Title} - ${metadata.Country}`,
|
||||
country: metadata.Country.trim(),
|
||||
englishTitle: metadata.English_Bible_Title,
|
||||
zipFileUrl: zipFileUrl,
|
||||
flagImageUrl: metadata.flag_Image,
|
||||
isDefault: false,
|
||||
books: []
|
||||
}
|
||||
|
||||
// Group verses by book
|
||||
const bookGroups = new Map<string, ParsedVerse[]>()
|
||||
for (const verse of verses) {
|
||||
if (!bookGroups.has(verse.book)) {
|
||||
bookGroups.set(verse.book, [])
|
||||
}
|
||||
bookGroups.get(verse.book)!.push(verse)
|
||||
}
|
||||
|
||||
// Convert each book
|
||||
for (const [bookKey, bookVerses] of bookGroups) {
|
||||
const orderNum = BOOK_ORDER[bookKey]
|
||||
const testament = getTestament(bookKey)
|
||||
|
||||
const book: JsonBook = {
|
||||
bookKey: bookKey,
|
||||
name: bookKey.charAt(0).toUpperCase() + bookKey.slice(1).replace('-', ' '),
|
||||
testament: testament,
|
||||
orderNum: orderNum,
|
||||
chapters: []
|
||||
}
|
||||
|
||||
// Group verses by chapter
|
||||
const chapterGroups = new Map<number, ParsedVerse[]>()
|
||||
for (const verse of bookVerses) {
|
||||
if (!chapterGroups.has(verse.chapter)) {
|
||||
chapterGroups.set(verse.chapter, [])
|
||||
}
|
||||
chapterGroups.get(verse.chapter)!.push(verse)
|
||||
}
|
||||
|
||||
// Convert each chapter
|
||||
for (const [chapterNum, chapterVerses] of chapterGroups) {
|
||||
const chapter: JsonChapter = {
|
||||
chapterNum: chapterNum,
|
||||
verses: []
|
||||
}
|
||||
|
||||
// Sort verses by verse number
|
||||
chapterVerses.sort((a, b) => a.verse - b.verse)
|
||||
|
||||
// Convert verses
|
||||
for (const verse of chapterVerses) {
|
||||
chapter.verses.push({
|
||||
verseNum: verse.verse,
|
||||
text: verse.text
|
||||
})
|
||||
}
|
||||
|
||||
book.chapters.push(chapter)
|
||||
}
|
||||
|
||||
// Sort chapters by chapter number
|
||||
book.chapters.sort((a, b) => a.chapterNum - b.chapterNum)
|
||||
bibleVersion.books.push(book)
|
||||
}
|
||||
|
||||
// Sort books by order number
|
||||
bibleVersion.books.sort((a, b) => a.orderNum - b.orderNum)
|
||||
|
||||
console.log(`✅ Converted ${verses.length} verses in ${bibleVersion.books.length} books`)
|
||||
return bibleVersion
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('🚀 Starting VPL to JSON conversion...')
|
||||
|
||||
const outputDir = path.join(process.cwd(), 'bibles', 'json')
|
||||
const logFilePath = path.join(outputDir, 'conversion_log.txt')
|
||||
const errorLogPath = path.join(outputDir, 'conversion_errors.txt')
|
||||
|
||||
// Initialize log files
|
||||
const logMessage = (message: string) => {
|
||||
const timestamp = new Date().toISOString()
|
||||
const logEntry = `[${timestamp}] ${message}\n`
|
||||
console.log(message)
|
||||
fs.appendFileSync(logFilePath, logEntry, 'utf-8')
|
||||
}
|
||||
|
||||
const logError = (message: string, error?: any) => {
|
||||
const timestamp = new Date().toISOString()
|
||||
const errorEntry = `[${timestamp}] ERROR: ${message}\n${error ? `Details: ${error.toString()}\n` : ''}\n`
|
||||
console.error(message, error || '')
|
||||
fs.appendFileSync(errorLogPath, errorEntry, 'utf-8')
|
||||
}
|
||||
|
||||
try {
|
||||
// Create output directory
|
||||
if (!fs.existsSync(outputDir)) {
|
||||
fs.mkdirSync(outputDir, { recursive: true })
|
||||
logMessage(`📁 Created output directory: ${outputDir}`)
|
||||
}
|
||||
|
||||
// Initialize log files
|
||||
fs.writeFileSync(logFilePath, `VPL to JSON Conversion Log - Started at ${new Date().toISOString()}\n`, 'utf-8')
|
||||
fs.writeFileSync(errorLogPath, `VPL to JSON Conversion Errors - Started at ${new Date().toISOString()}\n`, 'utf-8')
|
||||
|
||||
// Load Bible metadata
|
||||
logMessage('📋 Loading Bible metadata...')
|
||||
const metadata = loadBibleMetadata()
|
||||
logMessage(`Found ${metadata.length} Bible versions to convert`)
|
||||
|
||||
// Force re-conversion of all files for end-to-end testing
|
||||
logMessage(`Force re-converting all files for complete end-to-end process`)
|
||||
|
||||
// Convert each Bible version
|
||||
let converted = 0
|
||||
let skipped = 0
|
||||
let resumed = 0
|
||||
|
||||
for (const bibleData of metadata) {
|
||||
|
||||
try {
|
||||
const jsonBible = convertVplToJson(bibleData)
|
||||
if (jsonBible) {
|
||||
// Save individual Bible JSON file
|
||||
const filename = `${bibleData.file_ID}_bible.json`
|
||||
const filepath = path.join(outputDir, filename)
|
||||
|
||||
fs.writeFileSync(filepath, JSON.stringify(jsonBible, null, 2), 'utf-8')
|
||||
logMessage(`💾 Saved: ${filename}`)
|
||||
converted++
|
||||
|
||||
// Progress update every 10 conversions
|
||||
if (converted % 10 === 0) {
|
||||
logMessage(`📈 Progress: ${converted}/${metadata.length} converted...`)
|
||||
|
||||
// Force garbage collection to prevent memory buildup
|
||||
if (global.gc) {
|
||||
global.gc()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
skipped++
|
||||
logError(`Skipped ${bibleData.file_ID}: No valid Bible data found`)
|
||||
}
|
||||
} catch (error) {
|
||||
logError(`Failed to convert ${bibleData.file_ID}`, error)
|
||||
skipped++
|
||||
}
|
||||
}
|
||||
|
||||
// Skip creating large master file to prevent memory issues
|
||||
logMessage('\n📦 Skipping master file creation to prevent memory issues...')
|
||||
|
||||
// Final summary
|
||||
const totalProcessed = converted + skipped + resumed
|
||||
logMessage('\n✅ VPL to JSON conversion completed!')
|
||||
logMessage(`📊 Final Summary:`)
|
||||
logMessage(` - Successfully converted: ${converted}`)
|
||||
logMessage(` - Already converted (resumed): ${resumed}`)
|
||||
logMessage(` - Skipped due to errors: ${skipped}`)
|
||||
logMessage(` - Total processed: ${totalProcessed}`)
|
||||
logMessage(` - Output directory: ${outputDir}`)
|
||||
logMessage(` - Individual files: ${converted + resumed} Bible JSON files`)
|
||||
logMessage(` - Log file: conversion_log.txt`)
|
||||
logMessage(` - Error file: conversion_errors.txt`)
|
||||
|
||||
// Create a simple summary without loading all data into memory
|
||||
const allJsonFiles = fs.readdirSync(outputDir).filter(f => f.endsWith('_bible.json'))
|
||||
const summaryData = {
|
||||
conversionSummary: {
|
||||
totalJsonFiles: allJsonFiles.length,
|
||||
totalAttempted: metadata.length,
|
||||
successfullyConverted: converted,
|
||||
alreadyExisted: resumed,
|
||||
skippedDueToErrors: skipped,
|
||||
completedAt: new Date().toISOString()
|
||||
},
|
||||
availableFiles: allJsonFiles.sort()
|
||||
}
|
||||
|
||||
const summaryFile = path.join(outputDir, 'conversion_summary.json')
|
||||
fs.writeFileSync(summaryFile, JSON.stringify(summaryData, null, 2), 'utf-8')
|
||||
logMessage(`📊 Summary file saved: conversion_summary.json`)
|
||||
|
||||
} catch (error) {
|
||||
logError('Conversion failed with fatal error', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => {
|
||||
console.log('🎉 Conversion process completed successfully!')
|
||||
process.exit(0)
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error('💥 Conversion failed:', error)
|
||||
process.exit(1)
|
||||
})
|
||||
353
scripts/import_json_bibles.py
Normal file
353
scripts/import_json_bibles.py
Normal file
@@ -0,0 +1,353 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import JSON Bible files into the database.
|
||||
Skips files under 500KB and handles database constraints properly.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import psycopg
|
||||
from urllib.parse import urlparse
|
||||
from dotenv import load_dotenv
|
||||
from typing import Dict, List, Optional
|
||||
import sys
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
|
||||
# Load environment variables
|
||||
load_dotenv()
|
||||
|
||||
def get_db_connection():
|
||||
"""Get connection to biblical-guide database"""
|
||||
db_url = os.getenv("DATABASE_URL")
|
||||
if not db_url:
|
||||
raise ValueError("DATABASE_URL environment variable not found")
|
||||
|
||||
parsed = urlparse(db_url)
|
||||
conn_str = f"host={parsed.hostname} port={parsed.port or 5432} user={parsed.username} password={parsed.password} dbname=biblical-guide"
|
||||
return psycopg.connect(conn_str)
|
||||
|
||||
def get_file_size_kb(file_path: str) -> float:
|
||||
"""Get file size in KB"""
|
||||
return os.path.getsize(file_path) / 1024
|
||||
|
||||
def load_json_file(file_path: str) -> Optional[Dict]:
|
||||
"""Load and parse JSON file"""
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
print(f"❌ Error loading {file_path}: {e}")
|
||||
return None
|
||||
|
||||
def get_language_code(language: str) -> str:
|
||||
"""Convert language to proper ISO code"""
|
||||
lang_map = {
|
||||
'english': 'en',
|
||||
'spanish': 'es',
|
||||
'french': 'fr',
|
||||
'german': 'de',
|
||||
'portuguese': 'pt',
|
||||
'italian': 'it',
|
||||
'dutch': 'nl',
|
||||
'russian': 'ru',
|
||||
'chinese': 'zh',
|
||||
'japanese': 'ja',
|
||||
'korean': 'ko',
|
||||
'arabic': 'ar',
|
||||
'hindi': 'hi',
|
||||
'romanian': 'ro'
|
||||
}
|
||||
|
||||
lower_lang = language.lower()
|
||||
for key, code in lang_map.items():
|
||||
if key in lower_lang:
|
||||
return code
|
||||
|
||||
# Default to first 2 characters if no mapping found
|
||||
return lower_lang[:2] if len(lower_lang) >= 2 else 'xx'
|
||||
|
||||
def bible_version_exists(conn, abbreviation: str, language: str) -> bool:
|
||||
"""Check if Bible version already exists"""
|
||||
with conn.cursor() as cur:
|
||||
cur.execute('''
|
||||
SELECT COUNT(*) FROM "BibleVersion"
|
||||
WHERE abbreviation = %s AND language = %s
|
||||
''', (abbreviation, language))
|
||||
return cur.fetchone()[0] > 0
|
||||
|
||||
def import_bible_version(conn, bible_data: Dict) -> Optional[str]:
|
||||
"""Import a Bible version and return its ID"""
|
||||
try:
|
||||
# Extract and clean data
|
||||
name = bible_data.get('name', '').strip()
|
||||
abbreviation = bible_data.get('abbreviation', '').strip()
|
||||
language = get_language_code(bible_data.get('language', ''))
|
||||
description = bible_data.get('description', '').strip()
|
||||
country = bible_data.get('country', '').strip()
|
||||
english_title = bible_data.get('englishTitle', '').strip()
|
||||
zip_file_url = bible_data.get('zipFileUrl', '').strip()
|
||||
flag_image_url = bible_data.get('flagImageUrl', '').strip()
|
||||
is_default = bible_data.get('isDefault', False)
|
||||
|
||||
# Validate required fields
|
||||
if not name or not abbreviation:
|
||||
print(f"⚠️ Skipping Bible: missing name or abbreviation")
|
||||
return None
|
||||
|
||||
# Check for duplicates
|
||||
if bible_version_exists(conn, abbreviation, language):
|
||||
print(f"⚠️ Bible version {abbreviation} ({language}) already exists, skipping...")
|
||||
return None
|
||||
|
||||
# Insert Bible version
|
||||
version_id = str(uuid.uuid4())
|
||||
with conn.cursor() as cur:
|
||||
cur.execute('''
|
||||
INSERT INTO "BibleVersion" (
|
||||
id, name, abbreviation, language, description, country,
|
||||
"englishTitle", "zipFileUrl", "flagImageUrl", "isDefault",
|
||||
"createdAt", "updatedAt"
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW(), NOW())
|
||||
''', (
|
||||
version_id, name, abbreviation, language, description, country,
|
||||
english_title, zip_file_url, flag_image_url, is_default
|
||||
))
|
||||
|
||||
conn.commit()
|
||||
print(f"✅ Created Bible version: {name} ({abbreviation})")
|
||||
return version_id
|
||||
|
||||
except Exception as e:
|
||||
conn.rollback()
|
||||
print(f"❌ Error importing Bible version: {e}")
|
||||
return None
|
||||
|
||||
def import_bible_books(conn, version_id: str, books_data: List[Dict]) -> int:
|
||||
"""Import Bible books for a version"""
|
||||
imported_count = 0
|
||||
|
||||
try:
|
||||
for book_data in books_data:
|
||||
book_key = book_data.get('bookKey', '').strip()
|
||||
name = book_data.get('name', '').strip()
|
||||
testament = book_data.get('testament', '').strip()
|
||||
order_num = book_data.get('orderNum', 0)
|
||||
chapters_data = book_data.get('chapters', [])
|
||||
|
||||
if not book_key or not name or not testament:
|
||||
print(f"⚠️ Skipping book: missing required fields")
|
||||
continue
|
||||
|
||||
# Insert book
|
||||
book_id = str(uuid.uuid4())
|
||||
with conn.cursor() as cur:
|
||||
cur.execute('''
|
||||
INSERT INTO "BibleBook" (
|
||||
id, "versionId", name, testament, "orderNum", "bookKey"
|
||||
) VALUES (%s, %s, %s, %s, %s, %s)
|
||||
''', (book_id, version_id, name, testament, order_num, book_key))
|
||||
|
||||
# Import chapters for this book
|
||||
chapters_imported = import_bible_chapters(conn, book_id, chapters_data)
|
||||
|
||||
if chapters_imported > 0:
|
||||
imported_count += 1
|
||||
print(f" 📖 {name}: {chapters_imported} chapters")
|
||||
|
||||
conn.commit()
|
||||
return imported_count
|
||||
|
||||
except Exception as e:
|
||||
conn.rollback()
|
||||
print(f"❌ Error importing books: {e}")
|
||||
return 0
|
||||
|
||||
def import_bible_chapters(conn, book_id: str, chapters_data: List[Dict]) -> int:
|
||||
"""Import Bible chapters for a book"""
|
||||
imported_count = 0
|
||||
|
||||
try:
|
||||
for chapter_data in chapters_data:
|
||||
chapter_num = chapter_data.get('chapterNum', 0)
|
||||
verses_data = chapter_data.get('verses', [])
|
||||
|
||||
if chapter_num <= 0:
|
||||
print(f"⚠️ Skipping chapter: invalid chapter number")
|
||||
continue
|
||||
|
||||
# Insert chapter
|
||||
chapter_id = str(uuid.uuid4())
|
||||
with conn.cursor() as cur:
|
||||
cur.execute('''
|
||||
INSERT INTO "BibleChapter" (
|
||||
id, "bookId", "chapterNum"
|
||||
) VALUES (%s, %s, %s)
|
||||
''', (chapter_id, book_id, chapter_num))
|
||||
|
||||
# Import verses for this chapter
|
||||
verses_imported = import_bible_verses(conn, chapter_id, verses_data)
|
||||
|
||||
if verses_imported > 0:
|
||||
imported_count += 1
|
||||
|
||||
return imported_count
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error importing chapters: {e}")
|
||||
return 0
|
||||
|
||||
def import_bible_verses(conn, chapter_id: str, verses_data: List[Dict]) -> int:
|
||||
"""Import Bible verses for a chapter"""
|
||||
imported_count = 0
|
||||
|
||||
try:
|
||||
# Batch insert verses for better performance
|
||||
verses_to_insert = []
|
||||
|
||||
for verse_data in verses_data:
|
||||
verse_num = verse_data.get('verseNum', 0)
|
||||
text = verse_data.get('text', '').strip()
|
||||
|
||||
if verse_num <= 0 or not text:
|
||||
continue
|
||||
|
||||
verse_id = str(uuid.uuid4())
|
||||
verses_to_insert.append((verse_id, chapter_id, verse_num, text))
|
||||
|
||||
if verses_to_insert:
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('''
|
||||
INSERT INTO "BibleVerse" (
|
||||
id, "chapterId", "verseNum", text
|
||||
) VALUES (%s, %s, %s, %s)
|
||||
''', verses_to_insert)
|
||||
imported_count = len(verses_to_insert)
|
||||
|
||||
return imported_count
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error importing verses: {e}")
|
||||
return 0
|
||||
|
||||
def main():
|
||||
"""Main import function"""
|
||||
print("🚀 Starting JSON Bible import...")
|
||||
|
||||
json_dir = os.path.join(os.getcwd(), 'bibles', 'json')
|
||||
if not os.path.exists(json_dir):
|
||||
print(f"❌ JSON directory not found: {json_dir}")
|
||||
sys.exit(1)
|
||||
|
||||
# Get all JSON Bible files
|
||||
json_files = [f for f in os.listdir(json_dir) if f.endswith('_bible.json')]
|
||||
print(f"📁 Found {len(json_files)} JSON Bible files")
|
||||
|
||||
# Filter by file size (skip files under 500KB)
|
||||
valid_files = []
|
||||
skipped_small = 0
|
||||
|
||||
for file in json_files:
|
||||
file_path = os.path.join(json_dir, file)
|
||||
size_kb = get_file_size_kb(file_path)
|
||||
|
||||
if size_kb >= 500:
|
||||
valid_files.append((file, file_path, size_kb))
|
||||
else:
|
||||
skipped_small += 1
|
||||
|
||||
print(f"📏 Filtered files: {len(valid_files)} valid (≥500KB), {skipped_small} skipped (<500KB)")
|
||||
|
||||
# Sort by file size (largest first for better progress visibility)
|
||||
valid_files.sort(key=lambda x: x[2], reverse=True)
|
||||
|
||||
# Connect to database
|
||||
try:
|
||||
conn = get_db_connection()
|
||||
print("🔗 Connected to database")
|
||||
except Exception as e:
|
||||
print(f"❌ Database connection failed: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# Import statistics
|
||||
stats = {
|
||||
'total_files': len(valid_files),
|
||||
'imported': 0,
|
||||
'skipped': 0,
|
||||
'errors': 0,
|
||||
'total_books': 0,
|
||||
'total_chapters': 0,
|
||||
'total_verses': 0
|
||||
}
|
||||
|
||||
# Process each file
|
||||
for i, (filename, file_path, size_kb) in enumerate(valid_files, 1):
|
||||
print(f"\n📖 [{i}/{len(valid_files)}] Processing {filename} ({size_kb:.1f}KB)")
|
||||
|
||||
try:
|
||||
# Load JSON data
|
||||
bible_data = load_json_file(file_path)
|
||||
if not bible_data:
|
||||
stats['errors'] += 1
|
||||
continue
|
||||
|
||||
# Import Bible version
|
||||
version_id = import_bible_version(conn, bible_data)
|
||||
if not version_id:
|
||||
stats['skipped'] += 1
|
||||
continue
|
||||
|
||||
# Import books
|
||||
books_data = bible_data.get('books', [])
|
||||
books_imported = import_bible_books(conn, version_id, books_data)
|
||||
|
||||
if books_imported > 0:
|
||||
stats['imported'] += 1
|
||||
stats['total_books'] += books_imported
|
||||
|
||||
# Count chapters and verses
|
||||
for book in books_data:
|
||||
chapters = book.get('chapters', [])
|
||||
stats['total_chapters'] += len(chapters)
|
||||
for chapter in chapters:
|
||||
stats['total_verses'] += len(chapter.get('verses', []))
|
||||
|
||||
print(f"✅ Successfully imported {books_imported} books")
|
||||
else:
|
||||
stats['errors'] += 1
|
||||
|
||||
# Progress update every 10 files
|
||||
if i % 10 == 0:
|
||||
progress = (i / len(valid_files)) * 100
|
||||
print(f"\n📈 Progress: {progress:.1f}% ({stats['imported']} imported, {stats['skipped']} skipped, {stats['errors']} errors)")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error processing {filename}: {e}")
|
||||
stats['errors'] += 1
|
||||
|
||||
# Close database connection
|
||||
conn.close()
|
||||
|
||||
# Final summary
|
||||
print(f"\n🎉 JSON Bible import completed!")
|
||||
print(f"📊 Final Statistics:")
|
||||
print(f" - Total files processed: {stats['total_files']}")
|
||||
print(f" - Successfully imported: {stats['imported']}")
|
||||
print(f" - Skipped (duplicates): {stats['skipped']}")
|
||||
print(f" - Errors: {stats['errors']}")
|
||||
print(f" - Files skipped (<500KB): {skipped_small}")
|
||||
print(f" - Total books imported: {stats['total_books']}")
|
||||
print(f" - Total chapters imported: {stats['total_chapters']}")
|
||||
print(f" - Total verses imported: {stats['total_verses']}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print("\n⚠️ Import interrupted by user")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"❌ Fatal error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user