andrei/biblical-guide.com
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance RAG system to support multiple vector databases and improve AI chat functionality

Browse Source 
- Update vector-search.ts to query all available vector tables per language instead of single table
- Add getAllVectorTables() function to discover all language-specific vector tables
- Enhance searchBibleHybrid() to query multiple tables and merge results by relevance score
- Enhance searchBibleSemantic() to combine results from all available vector databases
- Add comprehensive error handling and logging for vector search operations
- Improve Azure OpenAI content filtering detection and error handling
- Add test-vector API endpoint for database diagnostics and debugging
- Fix environment configuration with complete Azure OpenAI settings
- Enable multi-translation biblical context from diverse Bible versions simultaneously

Tested: Romanian chat works excellently with rich biblical context and verse citations
Issue: English requires vector table creation - 47 English Bible versions exist but no vector tables

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Andrei
2025-09-25 07:21:59 +00:00
parent 3ae9733805
commit 2d27eae756
3 changed files with 224 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
lib/vector-search.ts
View File

@@ -10,51 +10,36 @@ function safeIdent(s: string): string {
return s.toLowerCase().replace(/[^a-z0-9_]+/g, '_').replace(/^_+|_+$/g, '')
}
// Resolve per-language default version and corresponding vector table name
// e.g. ai_bible.bv_ro_cornilescu
async function resolveVectorTable(language: string): Promise<{ table: string; exists: boolean }> {
// Get ALL vector tables for a given language
async function getAllVectorTables(language: string): Promise<string[]> {
const lang = safeIdent(language || 'ro')
const client = await pool.connect()
try {
// Get default version abbreviation from "BibleVersion"
const res = await client.query(
`SELECT "abbreviation" FROM "BibleVersion"
WHERE lower(language) = lower($1)
ORDER BY "isDefault" DESC, "createdAt" ASC
LIMIT 1`,
[language]
// Get all vector tables for this language
const result = await client.query(
`SELECT table_name FROM information_schema.tables
WHERE table_schema = $1 AND table_name LIKE $2
ORDER BY table_name`,
[VECTOR_SCHEMA, `bv_${lang}_%`]
)
const abbr = res.rows?.[0]?.abbreviation || 'default'
const ab = safeIdent(abbr)
const table = `${VECTOR_SCHEMA}.bv_${lang}_${ab}`
// Check if table exists
const check = await client.query(
`SELECT EXISTS (
SELECT 1 FROM information_schema.tables
WHERE table_schema = $1 AND table_name = $2
) AS exists`,
[VECTOR_SCHEMA, `bv_${lang}_${ab}`]
)
let exists = Boolean(check.rows?.[0]?.exists)
if (!exists) {
// Fallback: use any table for this language
const anyTbl = await client.query(
`SELECT table_name FROM information_schema.tables
WHERE table_schema = $1 AND table_name LIKE $2
ORDER BY table_name LIMIT 1`,
[VECTOR_SCHEMA, `bv_${lang}_%`]
)
if (anyTbl.rows?.[0]?.table_name) {
return { table: `${VECTOR_SCHEMA}."${anyTbl.rows[0].table_name}"`, exists: true }
}
}
return { table, exists }
return result.rows.map(row => `${VECTOR_SCHEMA}."${row.table_name}"`)
} finally {
client.release()
}
}
// Fallback: Resolve per-language default version (legacy function for backward compatibility)
async function resolveVectorTable(language: string): Promise<{ table: string; exists: boolean }> {
const tables = await getAllVectorTables(language)
if (tables.length > 0) {
return { table: tables[0], exists: true }
}
// Fallback to legacy bible_passages table
return { table: 'bible_passages', exists: false }
}
export interface BibleVerse {
id: string
ref: string
@@ -95,31 +80,51 @@ export async function searchBibleSemantic(
limit: number = 10
): Promise<BibleVerse[]> {
try {
const { table, exists } = await resolveVectorTable(language)
const tables = await getAllVectorTables(language)
const queryEmbedding = await getEmbedding(query)
const client = await pool.connect()
try {
const sql = exists
? `SELECT ref, book, chapter, verse, text_raw,
1 - (embedding <=> $1) AS similarity
FROM ${table}
WHERE embedding IS NOT NULL
ORDER BY embedding <=> $1
LIMIT $2`
: `SELECT ref, book, chapter, verse, text_raw,
1 - (embedding <=> $1) AS similarity
FROM bible_passages
WHERE embedding IS NOT NULL AND lang = $3
ORDER BY embedding <=> $1
LIMIT $2`
const params = exists
? [JSON.stringify(queryEmbedding), limit]
: [JSON.stringify(queryEmbedding), limit, language]
if (tables.length === 0) {
// Fallback to legacy bible_passages table
const sql = `SELECT ref, book, chapter, verse, text_raw,
1 - (embedding <=> $1) AS similarity
FROM bible_passages
WHERE embedding IS NOT NULL AND lang = $3
ORDER BY embedding <=> $1
LIMIT $2`
const result = await client.query(sql, params)
const result = await client.query(sql, [JSON.stringify(queryEmbedding), limit, language])
return result.rows
}
// Query all vector tables and combine results
const allResults: BibleVerse[] = []
const limitPerTable = Math.max(1, Math.ceil(limit * 2 / tables.length))
for (const table of tables) {
try {
const sql = `SELECT ref, book, chapter, verse, text_raw,
1 - (embedding <=> $1) AS similarity,
'${table}' as source_table
FROM ${table}
WHERE embedding IS NOT NULL
ORDER BY embedding <=> $1
LIMIT $2`
const result = await client.query(sql, [JSON.stringify(queryEmbedding), limitPerTable])
allResults.push(...result.rows)
} catch (tableError) {
console.warn(`Error querying table ${table}:`, tableError)
// Continue with other tables
}
}
// Sort all results by similarity and return top results
return allResults
.sort((a, b) => (b.similarity || 0) - (a.similarity || 0))
.slice(0, limit)
return result.rows
} finally {
client.release()
}
@@ -135,7 +140,7 @@ export async function searchBibleHybrid(
limit: number = 10
): Promise<BibleVerse[]> {
try {
const { table, exists } = await resolveVectorTable(language)
const tables = await getAllVectorTables(language)
const queryEmbedding = await getEmbedding(query)
// Use appropriate text search configuration based on language
@@ -143,28 +148,9 @@ export async function searchBibleHybrid(
const client = await pool.connect()
try {
const sql = exists
? `WITH vector_search AS (
SELECT id, 1 - (embedding <=> $1) AS vector_sim
FROM ${table}
WHERE embedding IS NOT NULL
ORDER BY embedding <=> $1
LIMIT 100
),
text_search AS (
SELECT id, ts_rank(tsv, plainto_tsquery($4, $3)) AS text_rank
FROM ${table}
WHERE tsv @@ plainto_tsquery($4, $3)
)
SELECT bp.ref, bp.book, bp.chapter, bp.verse, bp.text_raw,
COALESCE(vs.vector_sim, 0) * 0.7 + COALESCE(ts.text_rank, 0) * 0.3 AS combined_score
FROM ${table} bp
LEFT JOIN vector_search vs ON vs.id = bp.id
LEFT JOIN text_search ts ON ts.id = bp.id
WHERE (vs.id IS NOT NULL OR ts.id IS NOT NULL)
ORDER BY combined_score DESC
LIMIT $2`
: `WITH vector_search AS (
if (tables.length === 0) {
// Fallback to legacy bible_passages table
const sql = `WITH vector_search AS (
SELECT id, 1 - (embedding <=> $1) AS vector_sim
FROM bible_passages
WHERE embedding IS NOT NULL AND lang = $4
@@ -185,13 +171,51 @@ export async function searchBibleHybrid(
ORDER BY combined_score DESC
LIMIT $2`
const params = exists
? [JSON.stringify(queryEmbedding), limit, query, textConfig]
: [JSON.stringify(queryEmbedding), limit, query, language, textConfig]
const result = await client.query(sql, [JSON.stringify(queryEmbedding), limit, query, language, textConfig])
return result.rows
}
const result = await client.query(sql, params)
// Query all vector tables and combine results
const allResults: BibleVerse[] = []
const limitPerTable = Math.max(1, Math.ceil(limit * 2 / tables.length)) // Get more results per table to ensure good diversity
for (const table of tables) {
try {
const sql = `WITH vector_search AS (
SELECT id, 1 - (embedding <=> $1) AS vector_sim
FROM ${table}
WHERE embedding IS NOT NULL
ORDER BY embedding <=> $1
LIMIT 100
),
text_search AS (
SELECT id, ts_rank(tsv, plainto_tsquery($4, $3)) AS text_rank
FROM ${table}
WHERE tsv @@ plainto_tsquery($4, $3)
)
SELECT bp.ref, bp.book, bp.chapter, bp.verse, bp.text_raw,
COALESCE(vs.vector_sim, 0) * 0.7 + COALESCE(ts.text_rank, 0) * 0.3 AS combined_score,
'${table}' as source_table
FROM ${table} bp
LEFT JOIN vector_search vs ON vs.id = bp.id
LEFT JOIN text_search ts ON ts.id = bp.id
WHERE (vs.id IS NOT NULL OR ts.id IS NOT NULL)
ORDER BY combined_score DESC
LIMIT $2`
const result = await client.query(sql, [JSON.stringify(queryEmbedding), limitPerTable, query, textConfig])
allResults.push(...result.rows)
} catch (tableError) {
console.warn(`Error querying table ${table}:`, tableError)
// Continue with other tables
}
}
// Sort all results by combined score and return top results
return allResults
.sort((a, b) => (b.combined_score || 0) - (a.combined_score || 0))
.slice(0, limit)
return result.rows
} finally {
client.release()
}