Create comprehensive bookmarks management page

Add complete bookmarks page with navigation functionality:

Features:
- Dedicated /bookmarks page for viewing all saved bookmarks
- Support for both chapter and verse bookmarks in unified view
- Statistics dashboard showing total, chapter, and verse bookmark counts
- Tabbed filtering (All, Chapters, Verses) for easy organization
- Direct navigation to Bible reading page with URL parameters
- Delete functionality for individual bookmarks
- Empty state with call-to-action to start reading

Navigation Integration:
- Add Bookmarks to main navigation menu (authenticated users only)
- Add Bookmarks to user profile dropdown menu
- Dynamic navigation based on authentication state

Bible Page Enhancements:
- URL parameter support for bookmark navigation (book, chapter, verse)
- Verse highlighting when navigating from bookmarks
- Auto-clear highlight after 3 seconds for better UX

API Endpoints:
- /api/bookmarks/all - Unified endpoint for all user bookmarks
- Returns transformed data optimized for frontend consumption

Multilingual Support:
- Full Romanian and English translations
- Consistent messaging across all bookmark interfaces

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
andupetcu
2025-09-21 01:29:46 +03:00
parent 1b43b4e1e3
commit 686f498300
12 changed files with 31800 additions and 30905 deletions

View File

@@ -34,6 +34,7 @@ import {
import { useState, useEffect } from 'react'
import { useTranslations, useLocale } from 'next-intl'
import { useAuth } from '@/hooks/use-auth'
import { useSearchParams } from 'next/navigation'
interface BibleVerse {
id: string
@@ -60,6 +61,7 @@ export default function BiblePage() {
const theme = useTheme()
const t = useTranslations('pages.bible')
const locale = useLocale()
const searchParams = useSearchParams()
const [books, setBooks] = useState<BibleBook[]>([])
const [selectedBook, setSelectedBook] = useState<string>('')
const [selectedChapter, setSelectedChapter] = useState<number>(1)
@@ -69,6 +71,7 @@ export default function BiblePage() {
const [bookmarkLoading, setBookmarkLoading] = useState(false)
const [verseBookmarks, setVerseBookmarks] = useState<{[key: string]: any}>({})
const [verseBookmarkLoading, setVerseBookmarkLoading] = useState<{[key: string]: boolean}>({})
const [highlightedVerse, setHighlightedVerse] = useState<number | null>(null)
const { user } = useAuth()
// Fetch available books
@@ -88,6 +91,38 @@ export default function BiblePage() {
})
}, [])
// Handle URL parameters for navigation from bookmarks
useEffect(() => {
if (books.length > 0) {
const bookParam = searchParams.get('book')
const chapterParam = searchParams.get('chapter')
const verseParam = searchParams.get('verse')
if (bookParam) {
const book = books.find(b => b.id === bookParam)
if (book) {
setSelectedBook(bookParam)
if (chapterParam) {
const chapter = parseInt(chapterParam)
if (chapter > 0) {
setSelectedChapter(chapter)
}
}
if (verseParam) {
const verse = parseInt(verseParam)
if (verse > 0) {
setHighlightedVerse(verse)
// Clear highlight after 3 seconds
setTimeout(() => setHighlightedVerse(null), 3000)
}
}
}
}
}
}, [books, searchParams])
// Fetch verses when book/chapter changes
useEffect(() => {
if (selectedBook && selectedChapter) {
@@ -445,9 +480,16 @@ export default function BiblePage() {
sx={{
lineHeight: 1.8,
fontSize: '1.1rem',
bgcolor: isVerseBookmarked ? 'warning.light' : 'transparent',
borderRadius: isVerseBookmarked ? 1 : 0,
p: isVerseBookmarked ? 1 : 0,
bgcolor: highlightedVerse === verse.verseNum
? 'primary.light'
: isVerseBookmarked
? 'warning.light'
: 'transparent',
borderRadius: (isVerseBookmarked || highlightedVerse === verse.verseNum) ? 1 : 0,
p: (isVerseBookmarked || highlightedVerse === verse.verseNum) ? 1 : 0,
transition: 'all 0.3s ease',
border: highlightedVerse === verse.verseNum ? '2px solid' : 'none',
borderColor: 'primary.main',
}}
>
<Typography

View File

@@ -0,0 +1,402 @@
'use client'
import { useState, useEffect } from 'react'
import { useTranslations, useLocale } from 'next-intl'
import { useAuth } from '@/hooks/use-auth'
import { ProtectedRoute } from '@/components/auth/protected-route'
import { useRouter } from 'next/navigation'
import {
Container,
Paper,
Box,
Typography,
Card,
CardContent,
CardActions,
Button,
Chip,
Divider,
Grid,
IconButton,
Alert,
CircularProgress,
Tab,
Tabs,
List,
ListItem,
ListItemIcon,
ListItemText,
ListItemSecondary
} from '@mui/material'
import {
Bookmark,
BookmarkBorder,
MenuBook,
Article,
Delete,
Launch,
AccessTime,
Note
} from '@mui/icons-material'
interface BookmarkItem {
id: string
type: 'chapter' | 'verse'
title: string
subtitle: string
note?: string
createdAt: string
color?: string
text?: string
navigation: {
bookId: string
chapterNum: number
verseNum?: number
}
verse?: {
id: string
}
}
interface BookmarkStats {
total: number
chapters: number
verses: number
}
export default function BookmarksPage() {
const { user } = useAuth()
const t = useTranslations('bookmarks')
const locale = useLocale()
const router = useRouter()
const [bookmarks, setBookmarks] = useState<BookmarkItem[]>([])
const [stats, setStats] = useState<BookmarkStats>({ total: 0, chapters: 0, verses: 0 })
const [loading, setLoading] = useState(true)
const [error, setError] = useState('')
const [activeTab, setActiveTab] = useState(0)
const [deletingIds, setDeletingIds] = useState<Set<string>>(new Set())
// Fetch bookmarks
useEffect(() => {
if (user) {
fetchBookmarks()
}
}, [user])
const fetchBookmarks = async () => {
try {
setLoading(true)
setError('')
const token = localStorage.getItem('authToken')
if (!token) {
setError(t('authRequired'))
return
}
const response = await fetch(`/api/bookmarks/all?locale=${locale}`, {
headers: {
'Authorization': `Bearer ${token}`
}
})
if (response.ok) {
const data = await response.json()
setBookmarks(data.bookmarks || [])
setStats(data.stats || { total: 0, chapters: 0, verses: 0 })
} else {
const data = await response.json()
setError(data.error || t('loadError'))
}
} catch (error) {
console.error('Error fetching bookmarks:', error)
setError(t('loadError'))
} finally {
setLoading(false)
}
}
const handleNavigateToBookmark = (bookmark: BookmarkItem) => {
const params = new URLSearchParams({
book: bookmark.navigation.bookId,
chapter: bookmark.navigation.chapterNum.toString()
})
if (bookmark.navigation.verseNum) {
params.set('verse', bookmark.navigation.verseNum.toString())
}
router.push(`/${locale}/bible?${params.toString()}`)
}
const handleDeleteBookmark = async (bookmark: BookmarkItem) => {
if (!user) return
setDeletingIds(prev => new Set(prev).add(bookmark.id))
const token = localStorage.getItem('authToken')
if (!token) {
setDeletingIds(prev => {
const newSet = new Set(prev)
newSet.delete(bookmark.id)
return newSet
})
return
}
try {
const endpoint = bookmark.type === 'chapter'
? `/api/bookmarks/chapter?bookId=${bookmark.navigation.bookId}&chapterNum=${bookmark.navigation.chapterNum}&locale=${locale}`
: `/api/bookmarks/verse?verseId=${bookmark.verse.id}&locale=${locale}`
const response = await fetch(endpoint, {
method: 'DELETE',
headers: {
'Authorization': `Bearer ${token}`
}
})
if (response.ok) {
// Remove from local state
setBookmarks(prev => prev.filter(b => b.id !== bookmark.id))
setStats(prev => ({
total: prev.total - 1,
chapters: bookmark.type === 'chapter' ? prev.chapters - 1 : prev.chapters,
verses: bookmark.type === 'verse' ? prev.verses - 1 : prev.verses
}))
}
} catch (error) {
console.error('Error deleting bookmark:', error)
} finally {
setDeletingIds(prev => {
const newSet = new Set(prev)
newSet.delete(bookmark.id)
return newSet
})
}
}
const formatDate = (dateString: string) => {
const date = new Date(dateString)
return date.toLocaleDateString(locale, {
year: 'numeric',
month: 'short',
day: 'numeric'
})
}
const filteredBookmarks = () => {
switch (activeTab) {
case 1: return bookmarks.filter(b => b.type === 'chapter')
case 2: return bookmarks.filter(b => b.type === 'verse')
default: return bookmarks
}
}
if (loading) {
return (
<ProtectedRoute>
<Container maxWidth="lg" sx={{ py: 4 }}>
<Box sx={{ display: 'flex', justifyContent: 'center', alignItems: 'center', minHeight: 400 }}>
<CircularProgress size={48} />
</Box>
</Container>
</ProtectedRoute>
)
}
return (
<ProtectedRoute>
<Container maxWidth="lg" sx={{ py: 4 }}>
<Paper elevation={3} sx={{ p: 4 }}>
{/* Header */}
<Box textAlign="center" mb={4}>
<Typography variant="h3" component="h1" gutterBottom>
<Bookmark sx={{ fontSize: 40, mr: 2, verticalAlign: 'middle' }} />
{t('title')}
</Typography>
<Typography variant="body1" color="text.secondary">
{t('subtitle')}
</Typography>
</Box>
{/* Stats */}
<Grid container spacing={3} sx={{ mb: 4 }}>
<Grid item xs={12} sm={4}>
<Card variant="outlined">
<CardContent sx={{ textAlign: 'center' }}>
<Typography variant="h4" color="primary">
{stats.total}
</Typography>
<Typography variant="body2" color="text.secondary">
{t('totalBookmarks')}
</Typography>
</CardContent>
</Card>
</Grid>
<Grid item xs={12} sm={4}>
<Card variant="outlined">
<CardContent sx={{ textAlign: 'center' }}>
<Typography variant="h4" color="success.main">
{stats.chapters}
</Typography>
<Typography variant="body2" color="text.secondary">
{t('chapterBookmarks')}
</Typography>
</CardContent>
</Card>
</Grid>
<Grid item xs={12} sm={4}>
<Card variant="outlined">
<CardContent sx={{ textAlign: 'center' }}>
<Typography variant="h4" color="warning.main">
{stats.verses}
</Typography>
<Typography variant="body2" color="text.secondary">
{t('verseBookmarks')}
</Typography>
</CardContent>
</Card>
</Grid>
</Grid>
<Divider sx={{ mb: 3 }} />
{error && (
<Alert severity="error" sx={{ mb: 3 }}>
{error}
</Alert>
)}
{/* Filter Tabs */}
<Box sx={{ borderBottom: 1, borderColor: 'divider', mb: 3 }}>
<Tabs value={activeTab} onChange={(_, newValue) => setActiveTab(newValue)}>
<Tab
label={`${t('allBookmarks')} (${stats.total})`}
icon={<Bookmark />}
iconPosition="start"
/>
<Tab
label={`${t('chapters')} (${stats.chapters})`}
icon={<MenuBook />}
iconPosition="start"
/>
<Tab
label={`${t('verses')} (${stats.verses})`}
icon={<Article />}
iconPosition="start"
/>
</Tabs>
</Box>
{/* Bookmarks List */}
{filteredBookmarks().length === 0 ? (
<Box textAlign="center" py={6}>
<BookmarkBorder sx={{ fontSize: 64, color: 'text.secondary', mb: 2 }} />
<Typography variant="h6" color="text.secondary" gutterBottom>
{t('noBookmarks')}
</Typography>
<Typography variant="body2" color="text.secondary" sx={{ mb: 3 }}>
{t('noBookmarksDescription')}
</Typography>
<Button
variant="contained"
onClick={() => router.push(`/${locale}/bible`)}
startIcon={<MenuBook />}
>
{t('startReading')}
</Button>
</Box>
) : (
<Grid container spacing={3}>
{filteredBookmarks().map((bookmark) => (
<Grid item xs={12} key={bookmark.id}>
<Card variant="outlined" sx={{ transition: 'all 0.2s', '&:hover': { elevation: 2 } }}>
<CardContent>
<Box display="flex" justifyContent="space-between" alignItems="flex-start">
<Box flex={1}>
<Box display="flex" alignItems="center" gap={1} mb={1}>
{bookmark.type === 'chapter' ? (
<MenuBook color="primary" fontSize="small" />
) : (
<Article color="warning" fontSize="small" />
)}
<Typography variant="h6" component="h3">
{bookmark.title}
</Typography>
<Chip
label={bookmark.type === 'chapter' ? t('chapter') : t('verse')}
size="small"
color={bookmark.type === 'chapter' ? 'primary' : 'warning'}
variant="outlined"
/>
</Box>
<Typography variant="body2" color="text.secondary" gutterBottom>
{bookmark.subtitle}
</Typography>
{bookmark.text && (
<Typography variant="body1" sx={{
my: 1,
p: 1,
bgcolor: 'grey.50',
borderRadius: 1,
fontStyle: 'italic'
}}>
"{bookmark.text}"
</Typography>
)}
{bookmark.note && (
<Box display="flex" alignItems="center" gap={1} mt={1}>
<Note fontSize="small" color="action" />
<Typography variant="body2" color="text.secondary">
{bookmark.note}
</Typography>
</Box>
)}
<Box display="flex" alignItems="center" gap={1} mt={1}>
<AccessTime fontSize="small" color="action" />
<Typography variant="caption" color="text.secondary">
{formatDate(bookmark.createdAt)}
</Typography>
</Box>
</Box>
<Box display="flex" flexDirection="column" gap={1}>
<Button
variant="contained"
size="small"
startIcon={<Launch />}
onClick={() => handleNavigateToBookmark(bookmark)}
>
{t('goTo')}
</Button>
<IconButton
size="small"
color="error"
onClick={() => handleDeleteBookmark(bookmark)}
disabled={deletingIds.has(bookmark.id)}
>
{deletingIds.has(bookmark.id) ? (
<CircularProgress size={20} />
) : (
<Delete />
)}
</IconButton>
</Box>
</Box>
</CardContent>
</Card>
</Grid>
))}
</Grid>
)}
</Paper>
</Container>
</ProtectedRoute>
)
}

View File

@@ -0,0 +1,137 @@
import { NextResponse } from 'next/server'
import { getUserFromToken } from '@/lib/auth'
import { prisma } from '@/lib/db'
export const runtime = 'nodejs'
function getErrorMessages(locale: string = 'ro') {
const messages = {
ro: {
unauthorized: 'Nu ești autentificat',
bookmarkError: 'Eroare la încărcarea bookmark-urilor'
},
en: {
unauthorized: 'Unauthorized',
bookmarkError: 'Error loading bookmarks'
}
}
return messages[locale as keyof typeof messages] || messages.ro
}
// GET - Get all user's bookmarks (both chapter and verse)
export async function GET(request: Request) {
try {
const url = new URL(request.url)
const locale = url.searchParams.get('locale') || 'ro'
const messages = getErrorMessages(locale)
// Get token from authorization header
const authHeader = request.headers.get('authorization')
const token = authHeader?.replace('Bearer ', '')
if (!token) {
return NextResponse.json({ error: messages.unauthorized }, { status: 401 })
}
// Verify token and get user
const user = await getUserFromToken(token)
if (!user) {
return NextResponse.json({ error: messages.unauthorized }, { status: 401 })
}
// Get chapter bookmarks
const chapterBookmarks = await prisma.chapterBookmark.findMany({
where: {
userId: user.id
},
include: {
book: {
include: {
version: true
}
}
},
orderBy: {
createdAt: 'desc'
}
})
// Get verse bookmarks
const verseBookmarks = await prisma.bookmark.findMany({
where: {
userId: user.id
},
include: {
verse: {
include: {
chapter: {
include: {
book: {
include: {
version: true
}
}
}
}
}
}
},
orderBy: {
createdAt: 'desc'
}
})
// Transform the data for easier frontend consumption
const transformedChapterBookmarks = chapterBookmarks.map(bookmark => ({
id: bookmark.id,
type: 'chapter' as const,
title: `${bookmark.book.name} ${bookmark.chapterNum}`,
subtitle: bookmark.book.version.name,
note: bookmark.note,
createdAt: bookmark.createdAt,
navigation: {
bookId: bookmark.bookId,
chapterNum: bookmark.chapterNum
},
book: bookmark.book
}))
const transformedVerseBookmarks = verseBookmarks.map(bookmark => ({
id: bookmark.id,
type: 'verse' as const,
title: `${bookmark.verse.chapter.book.name} ${bookmark.verse.chapter.chapterNum}:${bookmark.verse.verseNum}`,
subtitle: bookmark.verse.chapter.book.version.name,
note: bookmark.note,
createdAt: bookmark.createdAt,
color: bookmark.color,
text: bookmark.verse.text.substring(0, 100) + (bookmark.verse.text.length > 100 ? '...' : ''),
navigation: {
bookId: bookmark.verse.chapter.bookId,
chapterNum: bookmark.verse.chapter.chapterNum,
verseNum: bookmark.verse.verseNum
},
verse: bookmark.verse
}))
// Combine and sort by creation date
const allBookmarks = [...transformedChapterBookmarks, ...transformedVerseBookmarks]
.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime())
return NextResponse.json({
bookmarks: allBookmarks,
stats: {
total: allBookmarks.length,
chapters: chapterBookmarks.length,
verses: verseBookmarks.length
}
})
} catch (error) {
console.error('All bookmarks fetch error:', error)
const url = new URL(request.url)
const locale = url.searchParams.get('locale') || 'ro'
const messages = getErrorMessages(locale)
return NextResponse.json({ error: messages.bookmarkError }, { status: 500 })
}
}

View File

@@ -31,6 +31,7 @@ import {
Settings,
Logout,
Login,
Bookmark,
} from '@mui/icons-material'
import { useRouter } from 'next/navigation'
import { useTranslations, useLocale } from 'next-intl'
@@ -55,7 +56,13 @@ export function Navigation() {
{ name: t('search'), path: '/search', icon: <Search /> },
]
const authenticatedPages = [
...pages,
{ name: t('bookmarks'), path: '/bookmarks', icon: <Bookmark /> },
]
const settings = [
{ name: t('bookmarks'), icon: <Bookmark />, action: 'bookmarks' },
{ name: t('profile'), icon: <AccountCircle />, action: 'profile' },
{ name: t('settings'), icon: <Settings />, action: 'settings' },
{ name: t('logout'), icon: <Logout />, action: 'logout' },
@@ -88,6 +95,9 @@ export function Navigation() {
handleCloseUserMenu()
switch (action) {
case 'bookmarks':
router.push(`/${locale}/bookmarks`)
break
case 'profile':
router.push(`/${locale}/profile`)
break
@@ -114,7 +124,7 @@ export function Navigation() {
const DrawerList = (
<Box sx={{ width: 250 }} role="presentation">
<List>
{pages.map((page) => (
{(isAuthenticated ? authenticatedPages : pages).map((page) => (
<ListItem key={page.name} disablePadding>
<ListItemButton onClick={() => handleNavigate(page.path)}>
<ListItemIcon sx={{ color: 'primary.main' }}>
@@ -190,7 +200,7 @@ export function Navigation() {
{/* Desktop Menu */}
<Box sx={{ flexGrow: 1, display: { xs: 'none', md: 'flex' } }}>
{pages.map((page) => (
{(isAuthenticated ? authenticatedPages : pages).map((page) => (
<Button
key={page.name}
onClick={() => handleNavigate(page.path)}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -4,6 +4,7 @@
"bible": "Bible",
"prayers": "Prayers",
"search": "Search",
"bookmarks": "Bookmarks",
"profile": "Profile",
"settings": "Settings",
"logout": "Logout",
@@ -271,6 +272,24 @@
"en": "English"
}
},
"bookmarks": {
"title": "My Bookmarks",
"subtitle": "Your saved verses and chapters",
"totalBookmarks": "Total bookmarks",
"chapterBookmarks": "Saved chapters",
"verseBookmarks": "Saved verses",
"allBookmarks": "All",
"chapters": "Chapters",
"verses": "Verses",
"chapter": "Chapter",
"verse": "Verse",
"noBookmarks": "You don't have any bookmarks yet",
"noBookmarksDescription": "Start reading the Bible and save your favorite chapters or verses",
"startReading": "Start reading",
"goTo": "Go to",
"authRequired": "You must be authenticated",
"loadError": "Error loading bookmarks"
},
"common": {
"loading": "Loading...",
"error": "An error occurred",

View File

@@ -4,6 +4,7 @@
"bible": "Biblia",
"prayers": "Rugăciuni",
"search": "Căutare",
"bookmarks": "Bookmark-uri",
"profile": "Profil",
"settings": "Setări",
"logout": "Deconectare",
@@ -271,6 +272,24 @@
"en": "English"
}
},
"bookmarks": {
"title": "Bookmark-urile mele",
"subtitle": "Versetele și capitolele tale salvate",
"totalBookmarks": "Total bookmark-uri",
"chapterBookmarks": "Capitole salvate",
"verseBookmarks": "Versete salvate",
"allBookmarks": "Toate",
"chapters": "Capitole",
"verses": "Versete",
"chapter": "Capitol",
"verse": "Verset",
"noBookmarks": "Nu ai încă bookmark-uri",
"noBookmarksDescription": "Începe să citești Biblia și salvează capitolele sau versetele tale preferate",
"startReading": "Începe să citești",
"goTo": "Mergi la",
"authRequired": "Trebuie să fii autentificat",
"loadError": "Eroare la încărcarea bookmark-urilor"
},
"common": {
"loading": "Se încarcă...",
"error": "A apărut o eroare",

View File

@@ -0,0 +1,47 @@
#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const ABBR = (process.env.EN_ABBR || 'WEB').toUpperCase();
const ROOT = process.env.OUTPUT_DIR || path.join('data','en_bible', ABBR);
function cleanText(s){
return s
// remove \+w/\w wrappers and closing tags
.replace(/\\\+?w\s+/gi,'')
.replace(/\|strong="[^"]*"/gi,'')
.replace(/\\\+?w\*/gi,'')
// remove footnotes / cross-refs blocks
.replace(/\\f\s+.*?\\f\*/gis,' ')
.replace(/\\x\s+.*?\\x\*/gis,' ')
// remove +wh blocks and similar wrappers
.replace(/\\\+wh\s+.*?\\\+wh\*/gis,' ')
// remove inline verse-note blocks like "+ 1:1 ... *"
.replace(/\+\s*\d+:\d+.*?\*/g,' ')
// remove stray asterisks left after stripping tags
.replace(/\*/g,'')
// remove any other inline tags like \\qs, \\add, etc.
.replace(/\\[a-z0-9-]+\s*/gi,' ')
.replace(/\s+/g,' ').trim();
}
function processFile(file){
const p = path.join(ROOT, file);
if(!fs.existsSync(p)){
console.error('Missing', p);
process.exit(1);
}
const j = JSON.parse(fs.readFileSync(p,'utf-8'));
for(const b of j.books||[]){
for(const c of b.chapters||[]){
for(const v of c.verses||[]){
if(v.text) v.text = cleanText(String(v.text));
}
}
}
fs.writeFileSync(p, JSON.stringify(j,null,2),'utf-8');
console.log('Cleaned', p);
}
processFile('old_testament.json');
processFile('new_testament.json');

View File

@@ -0,0 +1,169 @@
import os, json, re, asyncio
from pathlib import Path
from typing import List, Dict
from dotenv import load_dotenv
import httpx
import psycopg
load_dotenv()
AZ_ENDPOINT = os.getenv("AZURE_OPENAI_ENDPOINT", "").rstrip("/")
AZ_API_KEY = os.getenv("AZURE_OPENAI_KEY")
AZ_API_VER = os.getenv("AZURE_OPENAI_API_VERSION", "2024-05-01-preview")
AZ_DEPLOYMENT = os.getenv("AZURE_OPENAI_EMBED_DEPLOYMENT", "embed-3")
EMBED_DIMS = int(os.getenv("EMBED_DIMS", "3072"))
DB_URL = os.getenv("DATABASE_URL")
VECTOR_SCHEMA = os.getenv("VECTOR_SCHEMA", "ai_bible")
LANG_CODE = os.getenv("LANG_CODE", "en")
TRANSLATION = os.getenv("TRANSLATION_CODE", "WEB")
JSON_DIR = os.getenv("JSON_DIR", f"data/en_bible/{TRANSLATION}")
assert AZ_ENDPOINT and AZ_API_KEY and DB_URL and JSON_DIR, "Missing required env vars"
EMBED_URL = f"{AZ_ENDPOINT}/openai/deployments/{AZ_DEPLOYMENT}/embeddings?api-version={AZ_API_VER}"
def safe_ident(s: str) -> str:
return re.sub(r"[^a-z0-9_]+", "_", s.lower()).strip("_")
TABLE_BASENAME = f"bv_{safe_ident(LANG_CODE)}_{safe_ident(TRANSLATION)}"
TABLE_FQN = f'"{VECTOR_SCHEMA}"."{TABLE_BASENAME}"'
def create_table_sql() -> str:
return f"""
CREATE SCHEMA IF NOT EXISTS "{VECTOR_SCHEMA}";
CREATE TABLE IF NOT EXISTS {TABLE_FQN} (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
testament TEXT NOT NULL,
book TEXT NOT NULL,
chapter INT NOT NULL,
verse INT NOT NULL,
ref TEXT GENERATED ALWAYS AS (book || ' ' || chapter || ':' || verse) STORED,
text_raw TEXT NOT NULL,
text_norm TEXT NOT NULL,
tsv tsvector,
embedding vector({EMBED_DIMS}),
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
"""
def create_indexes_sql() -> str:
return f"""
CREATE UNIQUE INDEX IF NOT EXISTS ux_ref_{TABLE_BASENAME} ON {TABLE_FQN} (book, chapter, verse);
CREATE INDEX IF NOT EXISTS idx_tsv_{TABLE_BASENAME} ON {TABLE_FQN} USING GIN (tsv);
CREATE INDEX IF NOT EXISTS idx_book_ch_{TABLE_BASENAME} ON {TABLE_FQN} (book, chapter);
CREATE INDEX IF NOT EXISTS idx_testament_{TABLE_BASENAME} ON {TABLE_FQN} (testament);
"""
def upsert_sql() -> str:
return f"""
INSERT INTO {TABLE_FQN} (testament, book, chapter, verse, text_raw, text_norm, tsv, embedding)
VALUES (%(testament)s, %(book)s, %(chapter)s, %(verse)s, %(text_raw)s, %(text_norm)s,
to_tsvector(COALESCE(%(ts_lang)s,'simple')::regconfig, %(text_norm)s), %(embedding)s)
ON CONFLICT (book, chapter, verse) DO UPDATE
SET text_raw=EXCLUDED.text_raw,
text_norm=EXCLUDED.text_norm,
tsv=EXCLUDED.tsv,
embedding=EXCLUDED.embedding,
updated_at=now();
"""
def normalize(s: str) -> str:
s = re.sub(r"\s+", " ", s.strip())
return s
async def embed_batch(client: httpx.AsyncClient, inputs: List[str]) -> List[List[float]]:
payload = {"input": inputs}
headers = {"api-key": AZ_API_KEY, "Content-Type": "application/json"}
for attempt in range(6):
try:
r = await client.post(EMBED_URL, headers=headers, json=payload, timeout=60)
if r.status_code == 200:
data = r.json()
ordered = sorted(data["data"], key=lambda x: x["index"])
return [d["embedding"] for d in ordered]
elif r.status_code in (429, 500, 502, 503):
backoff = 2 ** attempt + (0.25 * attempt)
print(f"Rate/Server limited ({r.status_code}), waiting {backoff:.1f}s...")
await asyncio.sleep(backoff)
else:
raise RuntimeError(f"Embedding error {r.status_code}: {r.text}")
except Exception as e:
backoff = 2 ** attempt + (0.25 * attempt)
print(f"Error on attempt {attempt + 1}: {e}, waiting {backoff:.1f}s...")
await asyncio.sleep(backoff)
raise RuntimeError("Failed to embed after retries")
def load_json() -> List[Dict]:
ot = json.loads(Path(Path(JSON_DIR)/'old_testament.json').read_text('utf-8'))
nt = json.loads(Path(Path(JSON_DIR)/'new_testament.json').read_text('utf-8'))
verses = []
for test in (ot, nt):
testament = test.get('testament')
for book in test.get('books', []):
bname = book.get('name')
for ch in book.get('chapters', []):
cnum = int(ch.get('chapterNum'))
for v in ch.get('verses', []):
vnum = int(v.get('verseNum'))
text = str(v.get('text') or '').strip()
if text:
verses.append({
'testament': testament,
'book': bname,
'chapter': cnum,
'verse': vnum,
'text_raw': text,
'text_norm': normalize(text),
})
return verses
async def main():
print("Starting JSON embedding ingestion...", JSON_DIR)
verses = load_json()
print("Verses loaded:", len(verses))
batch_size = int(os.getenv('BATCH_SIZE', '128'))
# Prepare schema/table
with psycopg.connect(DB_URL) as conn:
with conn.cursor() as cur:
print(f"Ensuring schema/table {TABLE_FQN} ...")
cur.execute("CREATE EXTENSION IF NOT EXISTS vector;")
cur.execute(create_table_sql())
cur.execute(create_indexes_sql())
conn.commit()
async with httpx.AsyncClient() as client:
with psycopg.connect(DB_URL, autocommit=False) as conn:
with conn.cursor() as cur:
for i in range(0, len(verses), batch_size):
batch = verses[i:i+batch_size]
inputs = [v['text_norm'] for v in batch]
embs = await embed_batch(client, inputs)
rows = []
ts_lang = 'english' if LANG_CODE.lower().startswith('en') else 'simple'
for v, e in zip(batch, embs):
rows.append({ **v, 'ts_lang': ts_lang, 'embedding': e })
cur.executemany(upsert_sql(), rows)
conn.commit()
print(f"Upserted {len(rows)} verses... {i+len(rows)}/{len(verses)}")
print("Creating IVFFLAT index...")
with psycopg.connect(DB_URL, autocommit=True) as conn:
with conn.cursor() as cur:
cur.execute(f"VACUUM ANALYZE {TABLE_FQN};")
try:
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_vec_ivfflat_{TABLE_BASENAME}
ON {TABLE_FQN} USING ivfflat (embedding vector_cosine_ops)
WITH (lists = 200);
""")
except Exception as e:
print('IVFFLAT creation skipped (tune maintenance_work_mem):', e)
print("✅ JSON embedding ingestion completed successfully!")
if __name__ == '__main__':
asyncio.run(main())

View File

@@ -0,0 +1,40 @@
import { PrismaClient } from '@prisma/client'
import path from 'path'
import fs from 'fs'
const prisma = new PrismaClient()
async function main() {
try {
const abbr = (process.env.EN_ABBR || 'WEB').toUpperCase()
const lang = 'en'
const dir = process.env.INPUT_DIR || path.join('data', 'en_bible', abbr)
if (!fs.existsSync(path.join(dir, 'old_testament.json')) || !fs.existsSync(path.join(dir, 'new_testament.json'))) {
console.error('Missing OT/NT JSON in', dir)
process.exit(1)
}
// Ensure version exists
let version = await prisma.bibleVersion.findUnique({ where: { abbreviation_language: { abbreviation: abbr, language: lang } } })
if (!version) {
version = await prisma.bibleVersion.create({ data: { name: abbr, abbreviation: abbr, language: lang, description: `English Bible (${abbr})`, isDefault: true } })
console.log('Created version', version.id)
} else {
// Make this the default and disable others
await prisma.bibleVersion.updateMany({ where: { language: lang }, data: { isDefault: false } })
await prisma.bibleVersion.update({ where: { id: version.id }, data: { isDefault: true } })
}
// Wipe current WEB content for a clean import
const delVerses = await prisma.bibleVerse.deleteMany({ where: { chapter: { book: { versionId: version.id } } } })
console.log('Deleted verses for', abbr, ':', delVerses.count)
} catch (e) {
console.error('Reset failed:', e)
process.exit(1)
} finally {
await prisma.$disconnect()
}
}
main()

View File

@@ -99,8 +99,18 @@ function parseUsfmFile(file: string): Book | null {
if (m) {
const verseNum = parseInt(m[1], 10)
let text = m[2]
// Strip inline USFM markers (basic)
text = text.replace(/\\[a-z0-9-]+\s*/gi,'').trim()
// Strip inline USFM markup, preserving words
// Remove word wrappers: \w Word|strong="..."\w* and \+w ... \+w*
text = text.replace(/\\\+?w\s+/gi, '')
.replace(/\|strong="[^"]*"/gi, '')
.replace(/\\\+?w\*/gi, '')
// Remove footnotes / cross-refs blocks: \f ... \f* and \x ... \x*
text = text.replace(/\\f\s+.*?\\f\*/gis, ' ')
.replace(/\\x\s+.*?\\x\*/gis, ' ')
// Remove any remaining inline tags like \\add, \\nd, \\qs, etc.
text = text.replace(/\\[a-z0-9-]+\s*/gi, ' ')
// Collapse whitespace
text = text.replace(/\s+/g, ' ').trim()
currentVerses.push({ verseNum, text })
}
continue