From e1e8d9ac33917ae8da651e35490f19d6175bd6d9 Mon Sep 17 00:00:00 2001 From: w-dan Date: Fri, 26 Dec 2025 22:56:01 +0100 Subject: [PATCH 1/5] Fix multilingual card search to support foreign language names - Updated import script to use fd.name instead of fd.faceName for foreign card names - Added index on foreign_name for better search performance - Modified searchCards to query card_foreign_data table - Searches now work with Spanish, Japanese, German, French, etc. card names --- scripts/import-mtgjson.js | 2 +- src/db/migrations/010-add-foreign-data.js | 1 + src/services/cardService.js | 24 ++++++++++++----------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/scripts/import-mtgjson.js b/scripts/import-mtgjson.js index 675daeb..5c9966a 100644 --- a/scripts/import-mtgjson.js +++ b/scripts/import-mtgjson.js @@ -444,7 +444,7 @@ async function importCards(sourceDb, targetDb) { // Import foreign data console.log('Importing foreign card data...'); const foreignData = srcDb.prepare(` - SELECT c.name, fd.language, fd.faceName, fd.text, fd.type, fd.flavorText + SELECT c.name, fd.language, fd.name as faceName, fd.text, fd.type, fd.flavorText FROM cardForeignData fd JOIN cards c ON fd.uuid = c.uuid WHERE c.name IS NOT NULL AND fd.language IS NOT NULL diff --git a/src/db/migrations/010-add-foreign-data.js b/src/db/migrations/010-add-foreign-data.js index ff57b59..89fdb78 100644 --- a/src/db/migrations/010-add-foreign-data.js +++ b/src/db/migrations/010-add-foreign-data.js @@ -12,6 +12,7 @@ export function up(db) { ); CREATE INDEX idx_foreign_card_name ON card_foreign_data(card_name); CREATE INDEX idx_foreign_language ON card_foreign_data(language); + CREATE INDEX idx_foreign_name ON card_foreign_data(foreign_name); `); console.log('✓ Created card_foreign_data table'); } diff --git a/src/services/cardService.js b/src/services/cardService.js index c8b4a15..f622b08 100644 --- a/src/services/cardService.js +++ b/src/services/cardService.js @@ -19,26 +19,28 @@ function generateImageUrls(uuid) { /** * Search cards by name (for autocomplete) + * Supports both English names and foreign language names */ export function searchCards(query, limit = 20) { const searchTerm = `%${query}%`; - // Update the query to include image_url + // Search both English and foreign card names const cards = db.all( - `SELECT c.id, c.name, c.mana_cost, c.cmc, c.colors, c.type_line, c.oracle_text, + `SELECT DISTINCT c.id, c.name, c.mana_cost, c.cmc, c.colors, c.type_line, c.oracle_text, p.image_url, - (SELECT p.uuid FROM printings p WHERE p.card_id = c.id LIMIT 1) as sample_uuid + (SELECT p.uuid FROM printings p WHERE p.card_id = c.id LIMIT 1) as sample_uuid, + CASE + WHEN c.name LIKE ? THEN 0 + WHEN f.foreign_name LIKE ? THEN 1 + ELSE 2 + END as match_priority FROM cards c LEFT JOIN printings p ON p.card_id = c.id - WHERE c.name LIKE ? - ORDER BY - CASE - WHEN c.name LIKE ? THEN 0 - ELSE 1 - END, - c.name + LEFT JOIN card_foreign_data f ON f.card_name = c.name + WHERE c.name LIKE ? OR f.foreign_name LIKE ? + ORDER BY match_priority, c.name LIMIT ?`, - [searchTerm, `${query}%`, limit] + [`${query}%`, `${query}%`, searchTerm, searchTerm, limit] ); // Add image URLs from database From 1609cd33086973d21dcfbd7e7a9e3ac0db258dab Mon Sep 17 00:00:00 2001 From: Dani Date: Mon, 29 Dec 2025 11:27:35 +0100 Subject: [PATCH 2/5] Optimize multilingual queries - Fixed foreign name field - Added index on foreign name + FTS fallback --- scripts/benchmark-search.js | 37 ++++++++++++++ scripts/import-mtgjson.js | 21 ++++++++ src/db/migrations/017-add-card-search-fts.js | 16 ++++++ src/services/cardService.js | 54 ++++++++++++++++---- 4 files changed, 119 insertions(+), 9 deletions(-) create mode 100644 scripts/benchmark-search.js create mode 100644 src/db/migrations/017-add-card-search-fts.js diff --git a/scripts/benchmark-search.js b/scripts/benchmark-search.js new file mode 100644 index 0000000..040bd8f --- /dev/null +++ b/scripts/benchmark-search.js @@ -0,0 +1,37 @@ +import { performance } from 'perf_hooks'; +import { searchCards } from '../src/services/cardService.js'; + +async function run() { + const queries = ['lightning', 'island', 'sol', 'dragon', 'guild', 'counterspell']; + console.log('Running search benchmarks (warmup + timed runs)...'); + + // Warmup + for (const q of queries) { + searchCards(q, 10); + } + + const results = []; + for (const q of queries) { + const runs = 5; + let total = 0; + for (let i = 0; i < runs; i++) { + const t0 = performance.now(); + const res = searchCards(q, 20); + const t1 = performance.now(); + const ms = t1 - t0; + total += ms; + } + const avg = total / runs; + results.push({ query: q, avg_ms: avg }); + } + + console.log('Benchmark results:'); + for (const r of results) { + console.log(`- ${r.query}: ${r.avg_ms.toFixed(2)} ms (avg)`); + } +} + +run().catch(err => { + console.error('Benchmark failed:', err); + process.exit(1); +}); diff --git a/scripts/import-mtgjson.js b/scripts/import-mtgjson.js index 5c9966a..52caf86 100644 --- a/scripts/import-mtgjson.js +++ b/scripts/import-mtgjson.js @@ -479,6 +479,27 @@ async function importCards(sourceDb, targetDb) { const insertedCount = insertForeignMany(foreignData); console.log(`✓ Imported ${insertedCount} foreign card translations`); + // Populate FTS5 search table if it exists + try { + console.log('Populating card_search FTS5 table...'); + const cardsToIndex = targetDb.prepare(`SELECT id, name FROM cards`).all(); + const getForeignNames = targetDb.prepare(`SELECT GROUP_CONCAT(foreign_name, ' || ') as foreign_names FROM card_foreign_data WHERE card_name = ?`); + const insertSearch = targetDb.prepare(`INSERT OR REPLACE INTO card_search (name, foreign_names, card_id) VALUES (?, ?, ?)`); + + const populateMany = targetDb.transaction((rows) => { + for (const r of rows) { + const fn = getForeignNames.get(r.name); + insertSearch.run(r.name, fn ? fn.foreign_names : null, r.id); + } + }); + + populateMany(cardsToIndex); + console.log(`✓ Populated card_search for ${cardsToIndex.length} cards`); + } catch (e) { + // If the FTS5 table or extension is not available, skip gracefully + console.log('Skipping card_search population (FTS5 unavailable):', e.message); + } + srcDb.close(); console.log('✓ Import complete'); } diff --git a/src/db/migrations/017-add-card-search-fts.js b/src/db/migrations/017-add-card-search-fts.js new file mode 100644 index 0000000..916aa4b --- /dev/null +++ b/src/db/migrations/017-add-card-search-fts.js @@ -0,0 +1,16 @@ +export function up(db) { + // FTS5 table for fast multilingual searching across card names and foreign names + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS card_search USING fts5( + name, + foreign_names, + card_id UNINDEXED + ); + `); + console.log('Created card_search FTS5 table'); +} + +export function down(db) { + db.exec(`DROP TABLE IF EXISTS card_search;`); + console.log('Dropped card_search FTS5 table'); +} diff --git a/src/services/cardService.js b/src/services/cardService.js index f622b08..a0f7a41 100644 --- a/src/services/cardService.js +++ b/src/services/cardService.js @@ -22,25 +22,61 @@ function generateImageUrls(uuid) { * Supports both English names and foreign language names */ export function searchCards(query, limit = 20) { - const searchTerm = `%${query}%`; + const prefix = `${query}%`; + + // Try fast path using FTS5 if available (populated during import) + const ftsQuery = query.replace(/"/g, '').trim(); + if (ftsQuery) { + try { + const ftsParam = `${ftsQuery}*`; + const cards = db.all( + `SELECT c.id, c.name, c.mana_cost, c.cmc, c.colors, c.type_line, c.oracle_text, + (SELECT p.image_url FROM printings p WHERE p.card_id = c.id AND p.image_url IS NOT NULL LIMIT 1) as image_url, + (SELECT p.uuid FROM printings p WHERE p.card_id = c.id LIMIT 1) as sample_uuid, + cs.foreign_names, + CASE + WHEN c.name LIKE ? THEN 0 + WHEN cs.foreign_names LIKE ? THEN 1 + ELSE 2 + END as match_priority + FROM card_search cs + JOIN cards c ON c.id = cs.card_id + WHERE cs MATCH ? + ORDER BY match_priority, c.name + LIMIT ?`, + [prefix, prefix, ftsParam, limit] + ); + + return cards.map(card => ({ + ...card, + image_url: card.image_url, + large_image_url: card.image_url ? card.image_url.replace('/normal/', '/large/') : null, + art_crop_url: card.image_url ? card.image_url.replace('/normal/', '/art_crop/') : null + })); + } catch (e) { + // FTS unavailable or query failed; fall through to indexed prefix/exist strategy + console.log('FTS search failed, falling back to indexed LIKE/EXISTS:', e.message); + } + } - // Search both English and foreign card names + // Fallback: Use subqueries and EXISTS to avoid joining printings and foreign data + // which can multiply rows and slow searches. Use prefix matching so + // indexes on `cards.name` and `card_foreign_data(foreign_name)` can be used. const cards = db.all( - `SELECT DISTINCT c.id, c.name, c.mana_cost, c.cmc, c.colors, c.type_line, c.oracle_text, - p.image_url, + `SELECT c.id, c.name, c.mana_cost, c.cmc, c.colors, c.type_line, c.oracle_text, + (SELECT p.image_url FROM printings p WHERE p.card_id = c.id AND p.image_url IS NOT NULL LIMIT 1) as image_url, (SELECT p.uuid FROM printings p WHERE p.card_id = c.id LIMIT 1) as sample_uuid, CASE WHEN c.name LIKE ? THEN 0 - WHEN f.foreign_name LIKE ? THEN 1 + WHEN EXISTS(SELECT 1 FROM card_foreign_data f WHERE f.card_name = c.name AND f.foreign_name LIKE ? LIMIT 1) THEN 1 ELSE 2 END as match_priority FROM cards c - LEFT JOIN printings p ON p.card_id = c.id - LEFT JOIN card_foreign_data f ON f.card_name = c.name - WHERE c.name LIKE ? OR f.foreign_name LIKE ? + WHERE c.name LIKE ? + OR EXISTS(SELECT 1 FROM card_foreign_data f WHERE f.card_name = c.name AND f.foreign_name LIKE ? LIMIT 1) ORDER BY match_priority, c.name LIMIT ?`, - [`${query}%`, `${query}%`, searchTerm, searchTerm, limit] + [prefix, prefix, prefix, prefix, limit] ); // Add image URLs from database From 5cac99cd924bdbd6fe1ba1e5fb6b2f3cf5ff64d5 Mon Sep 17 00:00:00 2001 From: w-dan Date: Mon, 29 Dec 2025 16:28:18 +0100 Subject: [PATCH 3/5] Removed FTS search (found to be unnecessary) --- scripts/import-mtgjson.js | 21 -------------------- src/services/cardService.js | 38 +------------------------------------ 2 files changed, 1 insertion(+), 58 deletions(-) diff --git a/scripts/import-mtgjson.js b/scripts/import-mtgjson.js index 52caf86..5c9966a 100644 --- a/scripts/import-mtgjson.js +++ b/scripts/import-mtgjson.js @@ -479,27 +479,6 @@ async function importCards(sourceDb, targetDb) { const insertedCount = insertForeignMany(foreignData); console.log(`✓ Imported ${insertedCount} foreign card translations`); - // Populate FTS5 search table if it exists - try { - console.log('Populating card_search FTS5 table...'); - const cardsToIndex = targetDb.prepare(`SELECT id, name FROM cards`).all(); - const getForeignNames = targetDb.prepare(`SELECT GROUP_CONCAT(foreign_name, ' || ') as foreign_names FROM card_foreign_data WHERE card_name = ?`); - const insertSearch = targetDb.prepare(`INSERT OR REPLACE INTO card_search (name, foreign_names, card_id) VALUES (?, ?, ?)`); - - const populateMany = targetDb.transaction((rows) => { - for (const r of rows) { - const fn = getForeignNames.get(r.name); - insertSearch.run(r.name, fn ? fn.foreign_names : null, r.id); - } - }); - - populateMany(cardsToIndex); - console.log(`✓ Populated card_search for ${cardsToIndex.length} cards`); - } catch (e) { - // If the FTS5 table or extension is not available, skip gracefully - console.log('Skipping card_search population (FTS5 unavailable):', e.message); - } - srcDb.close(); console.log('✓ Import complete'); } diff --git a/src/services/cardService.js b/src/services/cardService.js index a0f7a41..b2fe520 100644 --- a/src/services/cardService.js +++ b/src/services/cardService.js @@ -24,44 +24,8 @@ function generateImageUrls(uuid) { export function searchCards(query, limit = 20) { const prefix = `${query}%`; - // Try fast path using FTS5 if available (populated during import) - const ftsQuery = query.replace(/"/g, '').trim(); - if (ftsQuery) { - try { - const ftsParam = `${ftsQuery}*`; - const cards = db.all( - `SELECT c.id, c.name, c.mana_cost, c.cmc, c.colors, c.type_line, c.oracle_text, - (SELECT p.image_url FROM printings p WHERE p.card_id = c.id AND p.image_url IS NOT NULL LIMIT 1) as image_url, - (SELECT p.uuid FROM printings p WHERE p.card_id = c.id LIMIT 1) as sample_uuid, - cs.foreign_names, - CASE - WHEN c.name LIKE ? THEN 0 - WHEN cs.foreign_names LIKE ? THEN 1 - ELSE 2 - END as match_priority - FROM card_search cs - JOIN cards c ON c.id = cs.card_id - WHERE cs MATCH ? - ORDER BY match_priority, c.name - LIMIT ?`, - [prefix, prefix, ftsParam, limit] - ); - - return cards.map(card => ({ - ...card, - image_url: card.image_url, - large_image_url: card.image_url ? card.image_url.replace('/normal/', '/large/') : null, - art_crop_url: card.image_url ? card.image_url.replace('/normal/', '/art_crop/') : null - })); - } catch (e) { - // FTS unavailable or query failed; fall through to indexed prefix/exist strategy - console.log('FTS search failed, falling back to indexed LIKE/EXISTS:', e.message); - } - } - // Fallback: Use subqueries and EXISTS to avoid joining printings and foreign data - // which can multiply rows and slow searches. Use prefix matching so - // indexes on `cards.name` and `card_foreign_data(foreign_name)` can be used. + // Prefix matching so indexes on `cards.name` and `card_foreign_data(foreign_name)` can be used. const cards = db.all( `SELECT c.id, c.name, c.mana_cost, c.cmc, c.colors, c.type_line, c.oracle_text, (SELECT p.image_url FROM printings p WHERE p.card_id = c.id AND p.image_url IS NOT NULL LIMIT 1) as image_url, From 5ac904b0797f9b3938f7d47769bbc2e064caa443 Mon Sep 17 00:00:00 2001 From: w-dan Date: Mon, 29 Dec 2025 17:20:39 +0100 Subject: [PATCH 4/5] Fixed split card search --- scripts/import-mtgjson.js | 8 ++++---- src/services/cardService.js | 15 +++++++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/scripts/import-mtgjson.js b/scripts/import-mtgjson.js index 5c9966a..99dd8d9 100644 --- a/scripts/import-mtgjson.js +++ b/scripts/import-mtgjson.js @@ -444,7 +444,7 @@ async function importCards(sourceDb, targetDb) { // Import foreign data console.log('Importing foreign card data...'); const foreignData = srcDb.prepare(` - SELECT c.name, fd.language, fd.name as faceName, fd.text, fd.type, fd.flavorText + SELECT c.name as english_name, fd.language, fd.name as faceName, fd.text, fd.type, fd.flavorText FROM cardForeignData fd JOIN cards c ON fd.uuid = c.uuid WHERE c.name IS NOT NULL AND fd.language IS NOT NULL @@ -460,12 +460,12 @@ async function importCards(sourceDb, targetDb) { let inserted = 0; for (const fd of data) { // Only insert if the card exists in our target database - const cardExists = targetDb.prepare('SELECT 1 FROM cards WHERE name = ? LIMIT 1').get(fd.name); + const cardExists = targetDb.prepare('SELECT 1 FROM cards WHERE name = ? LIMIT 1').get(fd.english_name); if (cardExists) { insertForeign.run( - fd.name, + fd.english_name, // card_name: English card name fd.language, - fd.faceName, + fd.faceName, // foreign_name: translated name fd.text, fd.type, fd.flavorText diff --git a/src/services/cardService.js b/src/services/cardService.js index b2fe520..f2be3d2 100644 --- a/src/services/cardService.js +++ b/src/services/cardService.js @@ -20,27 +20,34 @@ function generateImageUrls(uuid) { /** * Search cards by name (for autocomplete) * Supports both English names and foreign language names + * Also matches second part of split cards (e.g., "Tear" in "Wear // Tear") + * Works for both English and foreign split cards (e.g., "Borbotear" in "Alquimista de agua // Borbotear") */ export function searchCards(query, limit = 20) { const prefix = `${query}%`; + const infix = `%${query}%`; // Fallback: Use subqueries and EXISTS to avoid joining printings and foreign data - // Prefix matching so indexes on `cards.name` and `card_foreign_data(foreign_name)` can be used. + // Prefix matching on card names, infix for split card second parts, and foreign data const cards = db.all( `SELECT c.id, c.name, c.mana_cost, c.cmc, c.colors, c.type_line, c.oracle_text, (SELECT p.image_url FROM printings p WHERE p.card_id = c.id AND p.image_url IS NOT NULL LIMIT 1) as image_url, (SELECT p.uuid FROM printings p WHERE p.card_id = c.id LIMIT 1) as sample_uuid, CASE WHEN c.name LIKE ? THEN 0 - WHEN EXISTS(SELECT 1 FROM card_foreign_data f WHERE f.card_name = c.name AND f.foreign_name LIKE ? LIMIT 1) THEN 1 - ELSE 2 + WHEN c.name LIKE '%//%' AND c.name LIKE ? THEN 1 + WHEN EXISTS(SELECT 1 FROM card_foreign_data f WHERE f.card_name = c.name AND f.foreign_name LIKE ? LIMIT 1) THEN 2 + WHEN EXISTS(SELECT 1 FROM card_foreign_data f WHERE f.card_name = c.name AND f.foreign_name LIKE '%//%' AND f.foreign_name LIKE ? LIMIT 1) THEN 3 + ELSE 4 END as match_priority FROM cards c WHERE c.name LIKE ? + OR (c.name LIKE '%//%' AND c.name LIKE ?) -- Second part of split cards OR EXISTS(SELECT 1 FROM card_foreign_data f WHERE f.card_name = c.name AND f.foreign_name LIKE ? LIMIT 1) + OR EXISTS(SELECT 1 FROM card_foreign_data f WHERE f.card_name = c.name AND f.foreign_name LIKE '%//%' AND f.foreign_name LIKE ? LIMIT 1) ORDER BY match_priority, c.name LIMIT ?`, - [prefix, prefix, prefix, prefix, limit] + [prefix, infix, prefix, infix, prefix, infix, prefix, infix, limit] ); // Add image URLs from database From 9173293f9f2442610b39bbd7e6a0352d3a5bf3d6 Mon Sep 17 00:00:00 2001 From: w-dan Date: Mon, 29 Dec 2025 22:57:03 +0100 Subject: [PATCH 5/5] Fixed all prints not apprearing on search --- src/services/cardService.js | 39 +++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/services/cardService.js b/src/services/cardService.js index f2be3d2..06395bf 100644 --- a/src/services/cardService.js +++ b/src/services/cardService.js @@ -18,21 +18,21 @@ function generateImageUrls(uuid) { } /** - * Search cards by name (for autocomplete) - * Supports both English names and foreign language names - * Also matches second part of split cards (e.g., "Tear" in "Wear // Tear") - * Works for both English and foreign split cards (e.g., "Borbotear" in "Alquimista de agua // Borbotear") + * Search cards by name (autocomplete) - returns individual printings + * Supports English and foreign names, plus second parts of split cards. */ export function searchCards(query, limit = 20) { const prefix = `${query}%`; const infix = `%${query}%`; - // Fallback: Use subqueries and EXISTS to avoid joining printings and foreign data - // Prefix matching on card names, infix for split card second parts, and foreign data - const cards = db.all( + const rows = db.all( `SELECT c.id, c.name, c.mana_cost, c.cmc, c.colors, c.type_line, c.oracle_text, - (SELECT p.image_url FROM printings p WHERE p.card_id = c.id AND p.image_url IS NOT NULL LIMIT 1) as image_url, - (SELECT p.uuid FROM printings p WHERE p.card_id = c.id LIMIT 1) as sample_uuid, + p.image_url, + p.set_code, + s.name as set_name, + p.collector_number, + p.rarity, + p.uuid as sample_uuid, CASE WHEN c.name LIKE ? THEN 0 WHEN c.name LIKE '%//%' AND c.name LIKE ? THEN 1 @@ -41,21 +41,22 @@ export function searchCards(query, limit = 20) { ELSE 4 END as match_priority FROM cards c + LEFT JOIN printings p ON p.card_id = c.id + LEFT JOIN sets s ON p.set_code = s.code WHERE c.name LIKE ? - OR (c.name LIKE '%//%' AND c.name LIKE ?) -- Second part of split cards - OR EXISTS(SELECT 1 FROM card_foreign_data f WHERE f.card_name = c.name AND f.foreign_name LIKE ? LIMIT 1) - OR EXISTS(SELECT 1 FROM card_foreign_data f WHERE f.card_name = c.name AND f.foreign_name LIKE '%//%' AND f.foreign_name LIKE ? LIMIT 1) - ORDER BY match_priority, c.name + OR (c.name LIKE '%//%' AND c.name LIKE ?) + OR EXISTS(SELECT 1 FROM card_foreign_data f WHERE f.card_name = c.name AND f.foreign_name LIKE ? LIMIT 1) + OR EXISTS(SELECT 1 FROM card_foreign_data f WHERE f.card_name = c.name AND f.foreign_name LIKE '%//%' AND f.foreign_name LIKE ? LIMIT 1) + ORDER BY match_priority, c.name, p.set_code, p.collector_number LIMIT ?`, [prefix, infix, prefix, infix, prefix, infix, prefix, infix, limit] ); - // Add image URLs from database - return cards.map(card => ({ - ...card, - image_url: card.image_url, - large_image_url: card.image_url ? card.image_url.replace('/normal/', '/large/') : null, - art_crop_url: card.image_url ? card.image_url.replace('/normal/', '/art_crop/') : null + return rows.map(row => ({ + ...row, + image_url: row.image_url, + large_image_url: row.image_url ? row.image_url.replace('/normal/', '/large/') : null, + art_crop_url: row.image_url ? row.image_url.replace('/normal/', '/art_crop/') : null })); }