From cbe0675b070ec5b9fac4351f10700bcdf72593fe Mon Sep 17 00:00:00 2001 From: Yash Date: Sat, 24 Jan 2026 12:56:14 +0700 Subject: [PATCH 1/4] perf(docs): optimize document processing and fix --ignore flag - Batch database inserts (500 rows per batch) - Preload file/document maps for lookups - Process symbols once against full content vs per-line - Fix --ignore patterns not passed to scanner - Add object params style guide to CLAUDE.md Co-Authored-By: Claude Sonnet 4.5 --- CLAUDE.md | 64 ++ src/commands/index.ts | 221 ++++--- src/converter/convert.ts | 24 +- src/converter/documents.ts | 159 +++-- src/utils/fileScanner.ts | 10 + test/converter/batch-processing.test.ts | 397 ++++++------ test/converter/convert.test.ts | 773 +++++++++++++----------- 7 files changed, 917 insertions(+), 731 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 65f5e7a..84eea0a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -109,6 +109,70 @@ export function getDependencies(db: Database, path: string): DependencyNode[] { **Note:** Generated files (like `scip_pb.ts`) are exempt from these rules. +### Function Parameters + +Functions with more than one parameter must use a single object parameter instead of multiple positional parameters. + +**Rules:** + +1. **Single parameter functions** - Can use a simple parameter type +2. **Multiple parameters** - Must use a single object parameter with named properties + +**Good:** + +```typescript +// Single parameter - OK +function getSymbol(id: number): Symbol { + // Implementation +} + +// Multiple parameters - use object +function createDocument(params: { + path: string; + type: string; + content: string; + mtime: number; +}): Document { + // Implementation +} + +// Better with type alias +type CreateDocumentParams = { + path: string; + type: string; + content: string; + mtime: number; +}; + +function createDocument(params: CreateDocumentParams): Document { + // Implementation +} +``` + +**Bad:** + +```typescript +// BAD - Multiple positional parameters +function createDocument( + path: string, + type: string, + content: string, + mtime: number, +): Document { + // Implementation +} + +// BAD - Two or more parameters +function batchInsert( + db: Database, + table: string, + columns: string[], + rows: Array>, +): void { + // Implementation +} +``` + ## Directory Structure Note: Example scip files in `example` folder diff --git a/src/commands/index.ts b/src/commands/index.ts index 4d8794e..1c3385f 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -9,127 +9,122 @@ import { outputJson } from "../utils/output.ts"; import { resolveAbsolute } from "../utils/paths.ts"; export interface IndexOptions { - full?: boolean; - skipScip?: boolean; - ignore?: string[]; + full?: boolean; + skipScip?: boolean; + ignore?: string[]; } export async function index(options: IndexOptions = {}): Promise { - const startTime = Date.now(); - - debugIndex("Loading configuration..."); - const config = await loadConfig(); - debugIndex( - `Config loaded: root=${config.root}, scip=${config.scip}, db=${config.db}`, - ); - - const ignorePatterns = [ - ...(config.ignore || []), - ...(options.ignore || []), - ]; - - if (ignorePatterns.length > 0) { - debugIndex( - `Ignore patterns configured: ${ignorePatterns.join(", ")}`, - ); - } - - const scipPath = resolveAbsolute(config.root, config.scip); - const databasePath = resolveAbsolute(config.root, config.db); - debugIndex( - `Resolved paths: scipPath=${scipPath}, databasePath=${databasePath}`, - ); - - if (options.skipScip) { - debugIndex("Skipping SCIP indexer (--skip-scip flag set)"); - } else if (config.commands?.index) { - debugIndex(`Running SCIP indexer: ${config.commands.index}`); - await runCommand(config.commands.index, config.root, "Indexing"); - debugIndex("SCIP indexer completed successfully"); - } else { - debugIndex( - "No index command configured, checking for existing SCIP file...", - ); - if (!existsSync(scipPath)) { - throw new CtxError( - `No SCIP index found at ${scipPath} and no index command configured. ` + - `Either:\n` + - `1. Run your SCIP indexer manually to create ${config.scip}\n` + - `2. Configure commands.index in .dora/config.json to run your indexer automatically\n\n` + - `Example config:\n` + - `{\n` + - ` "commands": {\n` + - ` "index": "scip-typescript index --output .dora/index.scip"\n` + - ` }\n` + - `}`, - ); - } - } - - debugIndex("Verifying SCIP file exists..."); - if (!existsSync(scipPath)) { - throw new CtxError( - `SCIP file not created at ${scipPath}. Check your commands configuration.`, - ); - } - debugIndex("SCIP file verified"); - - debugIndex( - `Starting conversion to database (mode: ${options.full ? "full" : "auto"})`, - ); - const conversionStats = await convertToDatabase( - scipPath, - databasePath, - config.root, - { - force: options.full, - ignore: ignorePatterns, - }, - ); - debugIndex( - `Conversion completed: ${conversionStats.mode} mode, ${conversionStats.total_files} files, ${conversionStats.total_symbols} symbols`, - ); - - debugIndex("Closing database connection..."); - closeDb(); - - debugIndex("Updating config with last indexed timestamp..."); - config.lastIndexed = new Date().toISOString(); - await saveConfig(config); - - const time_ms = Date.now() - startTime; - - const result: IndexResult = { - success: true, - file_count: conversionStats.total_files, - symbol_count: conversionStats.total_symbols, - time_ms, - mode: conversionStats.mode, - changed_files: conversionStats.changed_files, - }; - - debugIndex(`Indexing completed successfully in ${time_ms}ms`); - outputJson(result); + const startTime = Date.now(); + + debugIndex("Loading configuration..."); + const config = await loadConfig(); + debugIndex( + `Config loaded: root=${config.root}, scip=${config.scip}, db=${config.db}` + ); + + const ignorePatterns = [...(config.ignore || []), ...(options.ignore || [])]; + + if (ignorePatterns.length > 0) { + debugIndex(`Ignore patterns configured: ${ignorePatterns.join(", ")}`); + } + + const scipPath = resolveAbsolute(config.root, config.scip); + const databasePath = resolveAbsolute(config.root, config.db); + debugIndex( + `Resolved paths: scipPath=${scipPath}, databasePath=${databasePath}` + ); + + if (options.skipScip) { + debugIndex("Skipping SCIP indexer (--skip-scip flag set)"); + } else if (config.commands?.index) { + debugIndex(`Running SCIP indexer: ${config.commands.index}`); + await runCommand(config.commands.index, config.root, "Indexing"); + debugIndex("SCIP indexer completed successfully"); + } else { + debugIndex( + "No index command configured, checking for existing SCIP file..." + ); + if (!existsSync(scipPath)) { + throw new CtxError( + `No SCIP index found at ${scipPath} and no index command configured. ` + + `Either:\n` + + `1. Run your SCIP indexer manually to create ${config.scip}\n` + + `2. Configure commands.index in .dora/config.json to run your indexer automatically\n\n` + + `Example config:\n` + + `{\n` + + ` "commands": {\n` + + ` "index": "scip-typescript index --output .dora/index.scip"\n` + + ` }\n` + + `}` + ); + } + } + + debugIndex("Verifying SCIP file exists..."); + if (!existsSync(scipPath)) { + throw new CtxError( + `SCIP file not created at ${scipPath}. Check your commands configuration.` + ); + } + debugIndex("SCIP file verified"); + + debugIndex( + `Starting conversion to database (mode: ${options.full ? "full" : "auto"})` + ); + const conversionStats = await convertToDatabase( + scipPath, + databasePath, + config.root, + { + force: options.full, + ignore: ignorePatterns, + } + ); + debugIndex( + `Conversion completed: ${conversionStats.mode} mode, ${conversionStats.total_files} files, ${conversionStats.total_symbols} symbols` + ); + + debugIndex("Closing database connection..."); + closeDb(); + + debugIndex("Updating config with last indexed timestamp..."); + config.lastIndexed = new Date().toISOString(); + await saveConfig(config); + + const time_ms = Date.now() - startTime; + + const result: IndexResult = { + success: true, + file_count: conversionStats.total_files, + symbol_count: conversionStats.total_symbols, + time_ms, + mode: conversionStats.mode, + changed_files: conversionStats.changed_files, + }; + + debugIndex(`Indexing completed successfully in ${time_ms}ms`); + outputJson(result); } /** * Run a shell command */ async function runCommand( - command: string, - cwd: string, - label: string, + command: string, + cwd: string, + label: string ): Promise { - const proc = Bun.spawn(command.split(" "), { - cwd, - stdout: "pipe", - stderr: "pipe", - }); - - const exitCode = await proc.exited; - - if (exitCode !== 0) { - const stderr = await new Response(proc.stderr).text(); - throw new CtxError(`${label} failed: ${stderr || "Unknown error"}`); - } + const proc = Bun.spawn(command.split(" "), { + cwd, + stdout: "pipe", + stderr: "pipe", + }); + + const exitCode = await proc.exited; + + if (exitCode !== 0) { + const stderr = await new Response(proc.stderr).text(); + throw new CtxError(`${label} failed: ${stderr || "Unknown error"}`); + } } diff --git a/src/converter/convert.ts b/src/converter/convert.ts index 1a8ec7f..f19358e 100644 --- a/src/converter/convert.ts +++ b/src/converter/convert.ts @@ -215,12 +215,17 @@ function chunkArray(array: T[], chunkSize: number): T[][] { * @returns Conversion statistics including mode (full/incremental), file counts, and timing * @throws {Error} If SCIP file cannot be parsed or database cannot be created */ -export async function convertToDatabase( - scipPath: string, - databasePath: string, - repoRoot: string, - options: ConversionOptions = {} -): Promise { +export async function convertToDatabase({ + scipPath, + databasePath, + repoRoot, + options = {}, +}: { + scipPath: string; + databasePath: string; + repoRoot: string; + options?: ConversionOptions; +}): Promise { const startTime = Date.now(); // Parse SCIP protobuf file @@ -334,7 +339,12 @@ export async function convertToDatabase( // Process documentation files debugConverter("Processing documentation files..."); const { processDocuments } = await import("./documents.js"); - const docStats = await processDocuments(db, repoRoot, mode); + const docStats = await processDocuments( + db, + repoRoot, + mode, + options.ignore || [] + ); debugConverter( `Documentation processing complete: ${docStats.processed} processed, ${docStats.skipped} skipped` ); diff --git a/src/converter/documents.ts b/src/converter/documents.ts index e55e30a..be60702 100644 --- a/src/converter/documents.ts +++ b/src/converter/documents.ts @@ -41,13 +41,18 @@ export async function processDocuments( db: Database, repoRoot: string, mode: "full" | "incremental", + ignorePatterns: string[] = [], ): Promise { debugDocs("Starting document processing in %s mode", mode); const startTime = Date.now(); // Scan for document files - const scannedDocs = await scanDocumentFiles(repoRoot); + const scannedDocs = await scanDocumentFiles( + repoRoot, + [".md", ".txt"], + ignorePatterns, + ); debugDocs("Scanned %d document files", scannedDocs.length); let docsToProcess: DocumentFile[]; @@ -98,23 +103,10 @@ export async function processDocuments( VALUES (?, ?, ?, ?, ?, ?, ?, ?) `); - const insertSymRefStmt = db.prepare(` - INSERT INTO document_symbol_refs (document_id, symbol_id, line) - VALUES (?, ?, ?) - `); - - const insertFileRefStmt = db.prepare(` - INSERT INTO document_file_refs (document_id, file_id, line) - VALUES (?, ?, ?) - `); - - const insertDocRefStmt = db.prepare(` - INSERT INTO document_document_refs (document_id, referenced_document_id, line) - VALUES (?, ?, ?) - `); - const getDocIdStmt = db.prepare("SELECT id FROM documents WHERE path = ?"); + const BATCH_SIZE = 500; + for (const doc of docsToProcess) { try { const fullPath = join(repoRoot, doc.path); @@ -154,20 +146,29 @@ export async function processDocuments( docRow.id, ]); - // Insert symbol references - for (const symRef of refs.symbolRefs) { - insertSymRefStmt.run(docRow.id, symRef.symbolId, symRef.line); - } + batchInsert( + db, + "document_symbol_refs", + ["document_id", "symbol_id", "line"], + refs.symbolRefs.map((r) => [docRow.id, r.symbolId, r.line]), + BATCH_SIZE, + ); - // Insert file references - for (const fileRef of refs.fileRefs) { - insertFileRefStmt.run(docRow.id, fileRef.fileId, fileRef.line); - } + batchInsert( + db, + "document_file_refs", + ["document_id", "file_id", "line"], + refs.fileRefs.map((r) => [docRow.id, r.fileId, r.line]), + BATCH_SIZE, + ); - // Insert document references - for (const docRef of refs.docRefs) { - insertDocRefStmt.run(docRow.id, docRef.docId, docRef.line); - } + batchInsert( + db, + "document_document_refs", + ["document_id", "referenced_document_id", "line"], + refs.docRefs.map((r) => [docRow.id, r.docId, r.line]), + BATCH_SIZE, + ); processed++; } catch (error) { @@ -221,22 +222,41 @@ function extractReferences(content: string, db: Database): DocumentReference { path: string; }>; - // Process each line - for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { - const lineContent = lines[lineIndex]; - const lineNumber = lineIndex + 1; // 1-indexed + const filePathMap = new Map(); + for (const file of files) { + filePathMap.set(file.path, file.id); + } - // Match symbol names with word boundaries - for (const sym of symbols) { - const escapedName = escapeRegex(sym.name); - const regex = new RegExp(`\\b${escapedName}\\b`); - if (regex.test(lineContent)) { - if (!symbolRefsMap.has(sym.id)) { - symbolRefsMap.set(sym.id, new Set()); - } - symbolRefsMap.get(sym.id)!.add(lineNumber); + const docPathMap = new Map(); + const docs = db.query("SELECT id, path FROM documents").all() as Array<{ + id: number; + path: string; + }>; + for (const doc of docs) { + docPathMap.set(doc.path, doc.id); + } + + const allContent = content; + for (const sym of symbols) { + const escapedName = escapeRegex(sym.name); + const regex = new RegExp(`\\b${escapedName}\\b`, "gm"); + let match; + + while ((match = regex.exec(allContent)) !== null) { + const lineNumber = + allContent.substring(0, match.index).split("\n").length; + + if (!symbolRefsMap.has(sym.id)) { + symbolRefsMap.set(sym.id, new Set()); } + symbolRefsMap.get(sym.id)!.add(lineNumber); } + } + + // Process each line for file path matching + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const lineContent = lines[lineIndex]; + const lineNumber = lineIndex + 1; // 1-indexed // Match file paths (direct mentions) for (const file of files) { @@ -262,29 +282,21 @@ function extractReferences(content: string, db: Database): DocumentReference { // Normalize path const normalized = normalizePath(linkPath); - // Check if it's a code file - const fileRow = db - .query("SELECT id FROM files WHERE path = ?") - .get(normalized) as { id: number } | null; - - if (fileRow) { - if (!fileRefsMap.has(fileRow.id)) { - fileRefsMap.set(fileRow.id, new Set()); + const fileId = filePathMap.get(normalized); + if (fileId !== undefined) { + if (!fileRefsMap.has(fileId)) { + fileRefsMap.set(fileId, new Set()); } - fileRefsMap.get(fileRow.id)!.add(lineNumber); + fileRefsMap.get(fileId)!.add(lineNumber); continue; } - // Check if it's a document file - const docRow = db - .query("SELECT id FROM documents WHERE path = ?") - .get(normalized) as { id: number } | null; - - if (docRow) { - if (!docRefsMap.has(docRow.id)) { - docRefsMap.set(docRow.id, new Set()); + const docId = docPathMap.get(normalized); + if (docId !== undefined) { + if (!docRefsMap.has(docId)) { + docRefsMap.set(docId, new Set()); } - docRefsMap.get(docRow.id)!.add(lineNumber); + docRefsMap.get(docId)!.add(lineNumber); } } } @@ -356,3 +368,32 @@ function normalizePath(path: string): string { return path; } + +/** + * Insert rows in batches for better performance + */ +function batchInsert( + db: Database, + table: string, + columns: string[], + rows: Array>, + batchSize: number, +): void { + if (rows.length === 0) { + return; + } + + const columnList = columns.join(", "); + + for (let i = 0; i < rows.length; i += batchSize) { + const batch = rows.slice(i, i + batchSize); + const valuePlaceholders = batch + .map(() => `(${columns.map(() => "?").join(", ")})`) + .join(", "); + + const sql = `INSERT INTO ${table} (${columnList}) VALUES ${valuePlaceholders}`; + const flatValues = batch.flat(); + + db.run(sql, flatValues); + } +} diff --git a/src/utils/fileScanner.ts b/src/utils/fileScanner.ts index cdb2e63..9c8900c 100644 --- a/src/utils/fileScanner.ts +++ b/src/utils/fileScanner.ts @@ -15,9 +15,13 @@ export interface DocumentFile { export async function scanDocumentFiles( repoRoot: string, extensions: string[] = [".md", ".txt"], + ignorePatterns: string[] = [], ): Promise { debugScanner("Scanning for document files in %s", repoRoot); debugScanner("Extensions: %o", extensions); + if (ignorePatterns.length > 0) { + debugScanner("User ignore patterns: %o", ignorePatterns); + } // Load .gitignore patterns const ig = await loadGitignorePatterns(repoRoot); @@ -36,6 +40,12 @@ export async function scanDocumentFiles( "*.log", ]); + // Add user-provided ignore patterns + if (ignorePatterns.length > 0) { + ig.add(ignorePatterns); + debugScanner("Added %d user ignore patterns", ignorePatterns.length); + } + const documents: DocumentFile[] = []; await walkDirectory(repoRoot, repoRoot, extensions, ig, documents); diff --git a/test/converter/batch-processing.test.ts b/test/converter/batch-processing.test.ts index 7649a5d..4fbb4cd 100644 --- a/test/converter/batch-processing.test.ts +++ b/test/converter/batch-processing.test.ts @@ -5,228 +5,237 @@ import { join } from "path"; import { convertToDatabase } from "../../src/converter/convert.ts"; describe("Batch Processing - Duplicate File Paths", () => { - const testDir = join(process.cwd(), ".test-batch-regression"); - const scipPath = join(process.cwd(), "test", "fixtures", "index.scip"); - const dbPath = join(testDir, "test.db"); - const repoRoot = join(process.cwd(), "test", "fixtures"); - - const skipTests = !existsSync(scipPath); - - beforeEach(() => { - if (existsSync(testDir)) { - rmSync(testDir, { recursive: true, force: true }); - } - mkdirSync(testDir, { recursive: true }); - }); - - afterEach(() => { - if (existsSync(testDir)) { - rmSync(testDir, { recursive: true, force: true }); - } - }); - - test("should not create duplicate files in database", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - await convertToDatabase(scipPath, dbPath, repoRoot); - - const db = new Database(dbPath); - - const files = db.query("SELECT path FROM files").all() as Array<{ - path: string; - }>; - - const paths = files.map((f) => f.path); - const uniquePaths = new Set(paths); - - expect(paths.length).toBe(uniquePaths.size); - - for (const path of paths) { - const count = paths.filter((p) => p === path).length; - if (count > 1) { - throw new Error(`Duplicate file path found: ${path} (${count} times)`); - } - } - - db.close(); - }); - - test("should handle batch processing correctly", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const stats = await convertToDatabase(scipPath, dbPath, repoRoot); - - expect(stats.mode).toBe("full"); - expect(stats.total_files).toBeGreaterThan(0); - - const db = new Database(dbPath); - - const duplicateCheck = db - .query( - ` + const testDir = join(process.cwd(), ".test-batch-regression"); + const scipPath = join(process.cwd(), "test", "fixtures", "index.scip"); + const dbPath = join(testDir, "test.db"); + const repoRoot = join(process.cwd(), "test", "fixtures"); + + const skipTests = !existsSync(scipPath); + + beforeEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + mkdirSync(testDir, { recursive: true }); + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + test("should not create duplicate files in database", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + await convertToDatabase({ scipPath, databasePath: dbPath, repoRoot }); + + const db = new Database(dbPath); + + const files = db.query("SELECT path FROM files").all() as Array<{ + path: string; + }>; + + const paths = files.map((f) => f.path); + const uniquePaths = new Set(paths); + + expect(paths.length).toBe(uniquePaths.size); + + for (const path of paths) { + const count = paths.filter((p) => p === path).length; + if (count > 1) { + throw new Error(`Duplicate file path found: ${path} (${count} times)`); + } + } + + db.close(); + }); + + test("should handle batch processing correctly", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const stats = await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot, + }); + + expect(stats.mode).toBe("full"); + expect(stats.total_files).toBeGreaterThan(0); + + const db = new Database(dbPath); + + const duplicateCheck = db + .query( + ` SELECT path, COUNT(*) as count FROM files GROUP BY path HAVING COUNT(*) > 1 - `, - ) - .all() as Array<{ - path: string; - count: number; - }>; + ` + ) + .all() as Array<{ + path: string; + count: number; + }>; - expect(duplicateCheck.length).toBe(0); + expect(duplicateCheck.length).toBe(0); - db.close(); - }); + db.close(); + }); - test("should maintain UNIQUE constraint on files.path", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } + test("should maintain UNIQUE constraint on files.path", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } - await convertToDatabase(scipPath, dbPath, repoRoot); + await convertToDatabase({ scipPath, databasePath: dbPath, repoRoot }); - const db = new Database(dbPath); + const db = new Database(dbPath); - const indexes = db - .query( - ` + const indexes = db + .query( + ` SELECT sql FROM sqlite_master WHERE type='table' AND name='files' - `, - ) - .all() as Array<{ - sql: string; - }>; - - expect(indexes.length).toBeGreaterThan(0); - expect(indexes[0].sql).toContain("UNIQUE"); - - db.close(); - }); - - test("should handle full rebuild without errors", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - let error: Error | null = null; - try { - await convertToDatabase(scipPath, dbPath, repoRoot, { force: true }); - } catch (e) { - error = e as Error; - } - - expect(error).toBeNull(); - - const db = new Database(dbPath); - - const files = db.query("SELECT COUNT(*) as c FROM files").get() as { - c: number; - }; - expect(files.c).toBeGreaterThan(0); - - const duplicates = db - .query( - ` + ` + ) + .all() as Array<{ + sql: string; + }>; + + expect(indexes.length).toBeGreaterThan(0); + expect(indexes[0].sql).toContain("UNIQUE"); + + db.close(); + }); + + test("should handle full rebuild without errors", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + let error: Error | null = null; + try { + await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot, + options: { force: true }, + }); + } catch (e) { + error = e as Error; + } + + expect(error).toBeNull(); + + const db = new Database(dbPath); + + const files = db.query("SELECT COUNT(*) as c FROM files").get() as { + c: number; + }; + expect(files.c).toBeGreaterThan(0); + + const duplicates = db + .query( + ` SELECT path, COUNT(*) as count FROM files GROUP BY path HAVING COUNT(*) > 1 - `, - ) - .all() as Array<{ - path: string; - count: number; - }>; + ` + ) + .all() as Array<{ + path: string; + count: number; + }>; - expect(duplicates.length).toBe(0); + expect(duplicates.length).toBe(0); - db.close(); - }); + db.close(); + }); - test("should properly track inserted files across batches", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } + test("should properly track inserted files across batches", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } - await convertToDatabase(scipPath, dbPath, repoRoot); + await convertToDatabase({ scipPath, databasePath: dbPath, repoRoot }); - const db = new Database(dbPath); + const db = new Database(dbPath); - const fileIdCheck = db - .query( - ` + const fileIdCheck = db + .query( + ` SELECT f.id, f.path, COUNT(s.id) as symbol_count FROM files f LEFT JOIN symbols s ON s.file_id = f.id GROUP BY f.id - `, - ) - .all() as Array<{ - id: number; - path: string; - symbol_count: number; - }>; - - for (const file of fileIdCheck) { - expect(file.id).toBeGreaterThan(0); - expect(typeof file.path).toBe("string"); - expect(file.symbol_count).toBeGreaterThanOrEqual(0); - } - - const orphanedSymbols = db - .query( - ` + ` + ) + .all() as Array<{ + id: number; + path: string; + symbol_count: number; + }>; + + for (const file of fileIdCheck) { + expect(file.id).toBeGreaterThan(0); + expect(typeof file.path).toBe("string"); + expect(file.symbol_count).toBeGreaterThanOrEqual(0); + } + + const orphanedSymbols = db + .query( + ` SELECT COUNT(*) as c FROM symbols s WHERE NOT EXISTS (SELECT 1 FROM files f WHERE f.id = s.file_id) - `, - ) - .get() as { c: number }; - - expect(orphanedSymbols.c).toBe(0); - - db.close(); - }); - - test("regression: UNIQUE constraint should not fail on first run", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - let uniqueConstraintError = false; - try { - await convertToDatabase(scipPath, dbPath, repoRoot); - } catch (error) { - if ( - error instanceof Error && - error.message.includes("UNIQUE constraint failed") - ) { - uniqueConstraintError = true; - } - throw error; - } - - expect(uniqueConstraintError).toBe(false); - - const db = new Database(dbPath); - const fileCount = ( - db.query("SELECT COUNT(*) as c FROM files").get() as { c: number } - ).c; - expect(fileCount).toBeGreaterThan(0); - db.close(); - }); + ` + ) + .get() as { c: number }; + + expect(orphanedSymbols.c).toBe(0); + + db.close(); + }); + + test("regression: UNIQUE constraint should not fail on first run", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + let uniqueConstraintError = false; + try { + await convertToDatabase({ scipPath, databasePath: dbPath, repoRoot }); + } catch (error) { + if ( + error instanceof Error && + error.message.includes("UNIQUE constraint failed") + ) { + uniqueConstraintError = true; + } + throw error; + } + + expect(uniqueConstraintError).toBe(false); + + const db = new Database(dbPath); + const fileCount = ( + db.query("SELECT COUNT(*) as c FROM files").get() as { c: number } + ).c; + expect(fileCount).toBeGreaterThan(0); + db.close(); + }); }); diff --git a/test/converter/convert.test.ts b/test/converter/convert.test.ts index 276449b..183bd7f 100644 --- a/test/converter/convert.test.ts +++ b/test/converter/convert.test.ts @@ -5,362 +5,419 @@ import { join } from "path"; import { convertToDatabase } from "../../src/converter/convert.ts"; describe("Database Converter", () => { - const testDir = join(process.cwd(), "test", "fixtures"); - const scipPath = join(testDir, "index.scip"); - const testDbDir = join(process.cwd(), ".test-db"); - const dbPath = join(testDbDir, "test.db"); - - const skipTests = !existsSync(scipPath); - - beforeEach(() => { - if (!existsSync(testDbDir)) { - mkdirSync(testDbDir, { recursive: true }); - } - }); - - afterEach(() => { - if (existsSync(testDbDir)) { - rmSync(testDbDir, { recursive: true, force: true }); - } - }); - - test("should convert SCIP to database successfully", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const stats = await convertToDatabase(scipPath, dbPath, testDir); - - expect(stats).toBeDefined(); - expect(stats.mode).toBeDefined(); - expect(stats.total_files).toBeGreaterThan(0); - expect(stats.total_symbols).toBeGreaterThan(0); - expect(stats.time_ms).toBeGreaterThan(0); - - expect(existsSync(dbPath)).toBe(true); - }); - - test("should create database with correct schema", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - await convertToDatabase(scipPath, dbPath, testDir); - - const db = new Database(dbPath); - - const tables = db - .query( - "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name", - ) - .all() as Array<{ name: string }>; - - const tableNames = tables.map((t) => t.name); - - expect(tableNames).toContain("files"); - expect(tableNames).toContain("symbols"); - expect(tableNames).toContain("dependencies"); - expect(tableNames).toContain("symbol_references"); - expect(tableNames).toContain("packages"); - expect(tableNames).toContain("metadata"); - expect(tableNames).toContain("documents"); - - db.close(); - }); - - test("should insert files without duplicates", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - await convertToDatabase(scipPath, dbPath, testDir); - - const db = new Database(dbPath); - - const files = db.query("SELECT path FROM files ORDER BY path").all() as Array<{ - path: string; - }>; - - const paths = files.map((f) => f.path); - const uniquePaths = new Set(paths); - - expect(paths.length).toBe(uniquePaths.size); - - db.close(); - }); - - test("should handle symbols correctly", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - await convertToDatabase(scipPath, dbPath, testDir); - - const db = new Database(dbPath); - - const symbols = db - .query("SELECT id, file_id, name, kind FROM symbols LIMIT 10") - .all() as Array<{ - id: number; - file_id: number; - name: string; - kind: string; - }>; - - expect(symbols.length).toBeGreaterThan(0); - - for (const sym of symbols) { - expect(sym.id).toBeGreaterThan(0); - expect(sym.file_id).toBeGreaterThan(0); - expect(typeof sym.name).toBe("string"); - expect(typeof sym.kind).toBe("string"); - } - - db.close(); - }); - - test("should handle dependencies correctly", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - await convertToDatabase(scipPath, dbPath, testDir); - - const db = new Database(dbPath); - - const deps = db - .query( - "SELECT from_file_id, to_file_id, symbol_count FROM dependencies LIMIT 10", - ) - .all() as Array<{ - from_file_id: number; - to_file_id: number; - symbol_count: number; - }>; - - for (const dep of deps) { - expect(dep.from_file_id).toBeGreaterThan(0); - expect(dep.to_file_id).toBeGreaterThan(0); - expect(dep.symbol_count).toBeGreaterThan(0); - expect(dep.from_file_id).not.toBe(dep.to_file_id); - } - - db.close(); - }); - - test("should update denormalized fields correctly", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - await convertToDatabase(scipPath, dbPath, testDir); - - const db = new Database(dbPath); - - const files = db - .query( - "SELECT id, path, symbol_count, dependency_count, dependent_count FROM files", - ) - .all() as Array<{ - id: number; - path: string; - symbol_count: number; - dependency_count: number; - dependent_count: number; - }>; - - for (const file of files) { - const actualSymbolCount = ( - db - .query("SELECT COUNT(*) as c FROM symbols WHERE file_id = ?") - .get(file.id) as { c: number } - ).c; - expect(file.symbol_count).toBe(actualSymbolCount); - - const actualDependencyCount = ( - db - .query( - "SELECT COUNT(DISTINCT to_file_id) as c FROM dependencies WHERE from_file_id = ?", - ) - .get(file.id) as { c: number } - ).c; - expect(file.dependency_count).toBe(actualDependencyCount); - - const actualDependentCount = ( - db - .query( - "SELECT COUNT(DISTINCT from_file_id) as c FROM dependencies WHERE to_file_id = ?", - ) - .get(file.id) as { c: number } - ).c; - expect(file.dependent_count).toBe(actualDependentCount); - } - - db.close(); - }); - - test("should handle incremental builds", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const stats1 = await convertToDatabase(scipPath, dbPath, testDir); - expect(stats1.mode).toBe("full"); - - await Bun.sleep(100); - - const stats2 = await convertToDatabase(scipPath, dbPath, testDir); - expect(stats2.mode).toBe("incremental"); - expect(stats2.changed_files).toBe(0); - expect(stats2.deleted_files).toBe(0); - }); - - test("should force full rebuild with force option", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const stats1 = await convertToDatabase(scipPath, dbPath, testDir); - expect(stats1.mode).toBe("full"); - - await Bun.sleep(100); - - const stats2 = await convertToDatabase(scipPath, dbPath, testDir, { - force: true, - }); - expect(stats2.mode).toBe("full"); - }); - - test("should handle symbol references correctly", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - await convertToDatabase(scipPath, dbPath, testDir); - - const db = new Database(dbPath); - - const refs = db - .query( - "SELECT symbol_id, file_id, line FROM symbol_references LIMIT 10", - ) - .all() as Array<{ - symbol_id: number; - file_id: number; - line: number; - }>; - - for (const ref of refs) { - expect(ref.symbol_id).toBeGreaterThan(0); - expect(ref.file_id).toBeGreaterThan(0); - expect(ref.line).toBeGreaterThanOrEqual(0); - - const symbol = db - .query("SELECT id FROM symbols WHERE id = ?") - .get(ref.symbol_id) as { id: number } | undefined; - expect(symbol).toBeDefined(); - - const file = db - .query("SELECT id FROM files WHERE id = ?") - .get(ref.file_id) as { id: number } | undefined; - expect(file).toBeDefined(); - } - - db.close(); - }); - - test("should filter local symbols correctly", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - await convertToDatabase(scipPath, dbPath, testDir); - - const db = new Database(dbPath); - - const localSymbols = db - .query("SELECT name, scip_symbol FROM symbols WHERE is_local = 1 LIMIT 10") - .all() as Array<{ - name: string; - scip_symbol: string; - }>; - - for (const sym of localSymbols) { - expect(sym.scip_symbol).toContain("local"); - } - - db.close(); - }); - - test("should handle packages correctly", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - await convertToDatabase(scipPath, dbPath, testDir); - - const db = new Database(dbPath); - - const packages = db - .query("SELECT name, manager, symbol_count FROM packages") - .all() as Array<{ - name: string; - manager: string; - symbol_count: number; - }>; - - for (const pkg of packages) { - expect(typeof pkg.name).toBe("string"); - expect(pkg.name.length).toBeGreaterThan(0); - expect(pkg.manager).toBe("npm"); - expect(pkg.symbol_count).toBeGreaterThan(0); - } - - db.close(); - }); - - test("should store metadata correctly", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - await convertToDatabase(scipPath, dbPath, testDir); - - const db = new Database(dbPath); - - const metadata = db - .query("SELECT key, value FROM metadata") - .all() as Array<{ - key: string; - value: string; - }>; - - const metadataMap = new Map(metadata.map((m) => [m.key, m.value])); - - expect(metadataMap.has("last_indexed")).toBe(true); - expect(metadataMap.has("total_files")).toBe(true); - expect(metadataMap.has("total_symbols")).toBe(true); - - const totalFiles = Number.parseInt(metadataMap.get("total_files") || "0"); - const totalSymbols = Number.parseInt( - metadataMap.get("total_symbols") || "0", - ); - - expect(totalFiles).toBeGreaterThan(0); - expect(totalSymbols).toBeGreaterThan(0); - - db.close(); - }); + const testDir = join(process.cwd(), "test", "fixtures"); + const scipPath = join(testDir, "index.scip"); + const testDbDir = join(process.cwd(), ".test-db"); + const dbPath = join(testDbDir, "test.db"); + + const skipTests = !existsSync(scipPath); + + beforeEach(() => { + if (!existsSync(testDbDir)) { + mkdirSync(testDbDir, { recursive: true }); + } + }); + + afterEach(() => { + if (existsSync(testDbDir)) { + rmSync(testDbDir, { recursive: true, force: true }); + } + }); + + test("should convert SCIP to database successfully", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const stats = await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot: testDir, + }); + + expect(stats).toBeDefined(); + expect(stats.mode).toBeDefined(); + expect(stats.total_files).toBeGreaterThan(0); + expect(stats.total_symbols).toBeGreaterThan(0); + expect(stats.time_ms).toBeGreaterThan(0); + + expect(existsSync(dbPath)).toBe(true); + }); + + test("should create database with correct schema", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot: testDir, + }); + + const db = new Database(dbPath); + + const tables = db + .query("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name") + .all() as Array<{ name: string }>; + + const tableNames = tables.map((t) => t.name); + + expect(tableNames).toContain("files"); + expect(tableNames).toContain("symbols"); + expect(tableNames).toContain("dependencies"); + expect(tableNames).toContain("symbol_references"); + expect(tableNames).toContain("packages"); + expect(tableNames).toContain("metadata"); + expect(tableNames).toContain("documents"); + + db.close(); + }); + + test("should insert files without duplicates", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot: testDir, + }); + + const db = new Database(dbPath); + + const files = db + .query("SELECT path FROM files ORDER BY path") + .all() as Array<{ + path: string; + }>; + + const paths = files.map((f) => f.path); + const uniquePaths = new Set(paths); + + expect(paths.length).toBe(uniquePaths.size); + + db.close(); + }); + + test("should handle symbols correctly", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot: testDir, + }); + + const db = new Database(dbPath); + + const symbols = db + .query("SELECT id, file_id, name, kind FROM symbols LIMIT 10") + .all() as Array<{ + id: number; + file_id: number; + name: string; + kind: string; + }>; + + expect(symbols.length).toBeGreaterThan(0); + + for (const sym of symbols) { + expect(sym.id).toBeGreaterThan(0); + expect(sym.file_id).toBeGreaterThan(0); + expect(typeof sym.name).toBe("string"); + expect(typeof sym.kind).toBe("string"); + } + + db.close(); + }); + + test("should handle dependencies correctly", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot: testDir, + }); + + const db = new Database(dbPath); + + const deps = db + .query( + "SELECT from_file_id, to_file_id, symbol_count FROM dependencies LIMIT 10" + ) + .all() as Array<{ + from_file_id: number; + to_file_id: number; + symbol_count: number; + }>; + + for (const dep of deps) { + expect(dep.from_file_id).toBeGreaterThan(0); + expect(dep.to_file_id).toBeGreaterThan(0); + expect(dep.symbol_count).toBeGreaterThan(0); + expect(dep.from_file_id).not.toBe(dep.to_file_id); + } + + db.close(); + }); + + test("should update denormalized fields correctly", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot: testDir, + }); + + const db = new Database(dbPath); + + const files = db + .query( + "SELECT id, path, symbol_count, dependency_count, dependent_count FROM files" + ) + .all() as Array<{ + id: number; + path: string; + symbol_count: number; + dependency_count: number; + dependent_count: number; + }>; + + for (const file of files) { + const actualSymbolCount = ( + db + .query("SELECT COUNT(*) as c FROM symbols WHERE file_id = ?") + .get(file.id) as { c: number } + ).c; + expect(file.symbol_count).toBe(actualSymbolCount); + + const actualDependencyCount = ( + db + .query( + "SELECT COUNT(DISTINCT to_file_id) as c FROM dependencies WHERE from_file_id = ?" + ) + .get(file.id) as { c: number } + ).c; + expect(file.dependency_count).toBe(actualDependencyCount); + + const actualDependentCount = ( + db + .query( + "SELECT COUNT(DISTINCT from_file_id) as c FROM dependencies WHERE to_file_id = ?" + ) + .get(file.id) as { c: number } + ).c; + expect(file.dependent_count).toBe(actualDependentCount); + } + + db.close(); + }); + + test("should handle incremental builds", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const stats1 = await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot: testDir, + }); + expect(stats1.mode).toBe("full"); + + await Bun.sleep(100); + + const stats2 = await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot: testDir, + }); + expect(stats2.mode).toBe("incremental"); + expect(stats2.changed_files).toBe(0); + expect(stats2.deleted_files).toBe(0); + }); + + test("should force full rebuild with force option", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const stats1 = await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot: testDir, + }); + expect(stats1.mode).toBe("full"); + + await Bun.sleep(100); + + const stats2 = await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot: testDir, + options: { + force: true, + }, + }); + expect(stats2.mode).toBe("full"); + }); + + test("should handle symbol references correctly", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot: testDir, + }); + + const db = new Database(dbPath); + + const refs = db + .query("SELECT symbol_id, file_id, line FROM symbol_references LIMIT 10") + .all() as Array<{ + symbol_id: number; + file_id: number; + line: number; + }>; + + for (const ref of refs) { + expect(ref.symbol_id).toBeGreaterThan(0); + expect(ref.file_id).toBeGreaterThan(0); + expect(ref.line).toBeGreaterThanOrEqual(0); + + const symbol = db + .query("SELECT id FROM symbols WHERE id = ?") + .get(ref.symbol_id) as { id: number } | undefined; + expect(symbol).toBeDefined(); + + const file = db + .query("SELECT id FROM files WHERE id = ?") + .get(ref.file_id) as { id: number } | undefined; + expect(file).toBeDefined(); + } + + db.close(); + }); + + test("should filter local symbols correctly", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot: testDir, + }); + + const db = new Database(dbPath); + + const localSymbols = db + .query( + "SELECT name, scip_symbol FROM symbols WHERE is_local = 1 LIMIT 10" + ) + .all() as Array<{ + name: string; + scip_symbol: string; + }>; + + for (const sym of localSymbols) { + expect(sym.scip_symbol).toContain("local"); + } + + db.close(); + }); + + test("should handle packages correctly", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot: testDir, + }); + + const db = new Database(dbPath); + + const packages = db + .query("SELECT name, manager, symbol_count FROM packages") + .all() as Array<{ + name: string; + manager: string; + symbol_count: number; + }>; + + for (const pkg of packages) { + expect(typeof pkg.name).toBe("string"); + expect(pkg.name.length).toBeGreaterThan(0); + expect(pkg.manager).toBe("npm"); + expect(pkg.symbol_count).toBeGreaterThan(0); + } + + db.close(); + }); + + test("should store metadata correctly", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + await convertToDatabase({ + scipPath, + databasePath: dbPath, + repoRoot: testDir, + }); + + const db = new Database(dbPath); + + const metadata = db + .query("SELECT key, value FROM metadata") + .all() as Array<{ + key: string; + value: string; + }>; + + const metadataMap = new Map(metadata.map((m) => [m.key, m.value])); + + expect(metadataMap.has("last_indexed")).toBe(true); + expect(metadataMap.has("total_files")).toBe(true); + expect(metadataMap.has("total_symbols")).toBe(true); + + const totalFiles = Number.parseInt(metadataMap.get("total_files") || "0"); + const totalSymbols = Number.parseInt( + metadataMap.get("total_symbols") || "0" + ); + + expect(totalFiles).toBeGreaterThan(0); + expect(totalSymbols).toBeGreaterThan(0); + + db.close(); + }); }); From 508b50d9b8ff9473e90e411e09df690a94931c37 Mon Sep 17 00:00:00 2001 From: Yash Date: Sat, 24 Jan 2026 13:16:54 +0700 Subject: [PATCH 2/4] refactor: convert multi-param functions to object params (excluding db queries) - Change functions with 2+ params to use single object parameter - Update call sites in commands, converter, utils, and tests - DB queries refactoring deferred Co-Authored-By: Claude Sonnet 4.5 --- src/commands/adventure.ts | 8 +- src/commands/changes.ts | 6 +- src/commands/complexity.ts | 6 +- src/commands/cookbook.ts | 108 +- src/commands/coupling.ts | 2 +- src/commands/cycles.ts | 2 +- src/commands/deps.ts | 8 +- src/commands/docs/search.ts | 6 +- src/commands/exports.ts | 4 +- src/commands/file.ts | 2 +- src/commands/graph.ts | 16 +- src/commands/imports.ts | 2 +- src/commands/index.ts | 222 ++-- src/commands/leaves.ts | 8 +- src/commands/lost.ts | 6 +- src/commands/ls.ts | 4 +- src/commands/rdeps.ts | 8 +- src/commands/refs.ts | 8 +- src/commands/shared.ts | 65 +- src/commands/symbol.ts | 8 +- src/commands/treasure.ts | 6 +- src/converter/convert.ts | 1724 ++++++++++++++-------------- src/converter/documents.ts | 102 +- src/converter/scip-parser.ts | 37 +- src/db/connection.ts | 9 +- src/db/queries.ts | 780 ++++++------- src/index.ts | 308 ++--- src/utils/changeDetection.ts | 55 +- src/utils/config.ts | 22 +- src/utils/fileScanner.ts | 25 +- src/utils/paths.ts | 26 +- src/utils/templates.ts | 22 +- test/commands/adventure.test.ts | 348 +++--- test/converter/documents.test.ts | 20 +- test/converter/scip-parser.test.ts | 828 ++++++------- test/db/queries.test.ts | 358 +++--- test/utils/fileScanner.test.ts | 193 ++-- 37 files changed, 2799 insertions(+), 2563 deletions(-) diff --git a/src/commands/adventure.ts b/src/commands/adventure.ts index 121ac38..6107f12 100644 --- a/src/commands/adventure.ts +++ b/src/commands/adventure.ts @@ -12,8 +12,8 @@ import { export async function adventure(from: string, to: string): Promise { const ctx = await setupCommand(); - const fromPath = resolveAndValidatePath(ctx, from); - const toPath = resolveAndValidatePath(ctx, to); + const fromPath = resolveAndValidatePath({ ctx, inputPath: from }); + const toPath = resolveAndValidatePath({ ctx, inputPath: to }); // If same file, return direct path if (fromPath === toPath) { @@ -73,7 +73,7 @@ function findShortestPath( } // Get reverse dependencies from 'to' file - const reverseDeps = getReverseDependencies(db, to, depth); + const reverseDeps = getReverseDependencies({ db, relativePath: to, depth }); const reverseSet = new Set(reverseDeps.map((d) => d.path)); // Check if 'from' is in reverse dependencies @@ -142,7 +142,7 @@ function reconstructPath( // Get neighbors const neighbors = forward ? getDependencies(db, current.file, 1) - : getReverseDependencies(db, current.file, 1); + : getReverseDependencies({ db, relativePath: current.file, depth: 1 }); for (const neighbor of neighbors) { if (!visited.has(neighbor.path)) { diff --git a/src/commands/changes.ts b/src/commands/changes.ts index f63ae8e..70809f6 100644 --- a/src/commands/changes.ts +++ b/src/commands/changes.ts @@ -20,7 +20,11 @@ export async function changes( for (const file of changedFiles) { try { - const rdeps = getReverseDependencies(ctx.db, file, DEFAULTS.DEPTH); + const rdeps = getReverseDependencies({ + db: ctx.db, + relativePath: file, + depth: DEFAULTS.DEPTH, + }); rdeps.forEach((dep) => impacted.add(dep.path)); } catch {} } diff --git a/src/commands/complexity.ts b/src/commands/complexity.ts index 5ab2205..a38a2fe 100644 --- a/src/commands/complexity.ts +++ b/src/commands/complexity.ts @@ -11,7 +11,11 @@ export async function complexity( ): Promise { const { db } = await setupCommand(); - const sortBy = parseStringFlag(flags, "sort", "complexity"); + const sortBy = parseStringFlag({ + flags, + key: "sort", + defaultValue: "complexity", + }); // Validate sort option if (!["complexity", "symbols", "stability"].includes(sortBy)) { diff --git a/src/commands/cookbook.ts b/src/commands/cookbook.ts index fbceb4a..ac29506 100644 --- a/src/commands/cookbook.ts +++ b/src/commands/cookbook.ts @@ -5,72 +5,72 @@ import { getDoraDir } from "../utils/paths.ts"; import { outputJson } from "./shared.ts"; type CookbookOptions = { - format?: "json" | "markdown"; + format?: "json" | "markdown"; }; function getAvailableRecipes(cookbookDir: string): string[] { - try { - const files = readdirSync(cookbookDir); - return files - .filter((file) => file.endsWith(".md") && file !== "index.md") - .map((file) => file.replace(".md", "")) - .sort(); - } catch { - return []; - } + try { + const files = readdirSync(cookbookDir); + return files + .filter((file) => file.endsWith(".md") && file !== "index.md") + .map((file) => file.replace(".md", "")) + .sort(); + } catch { + return []; + } } export async function cookbookList( - options: CookbookOptions = {} + options: CookbookOptions = {}, ): Promise { - const config = await loadConfig(); - const cookbookDir = join(getDoraDir(config.root), "cookbook"); - const format = options.format || "json"; - const recipes = getAvailableRecipes(cookbookDir); + const config = await loadConfig(); + const cookbookDir = join(getDoraDir(config.root), "cookbook"); + const format = options.format || "json"; + const recipes = getAvailableRecipes(cookbookDir); - if (format === "markdown") { - console.log("Available recipes:\n"); - for (const r of recipes) { - console.log(` - ${r}`); - } - console.log("\nView a recipe: dora cookbook show "); - console.log("Example: dora cookbook show quickstart"); - } else { - outputJson({ - recipes, - total: recipes.length, - }); - } + if (format === "markdown") { + console.log("Available recipes:\n"); + for (const r of recipes) { + console.log(` - ${r}`); + } + console.log("\nView a recipe: dora cookbook show "); + console.log("Example: dora cookbook show quickstart"); + } else { + outputJson({ + recipes, + total: recipes.length, + }); + } } export async function cookbookShow( - recipe: string = "", - options: CookbookOptions = {} + recipe: string = "", + options: CookbookOptions = {}, ): Promise { - const config = await loadConfig(); - const cookbookDir = join(getDoraDir(config.root), "cookbook"); - const format = options.format || "json"; - const templateName = recipe ? `${recipe}.md` : "index.md"; - const templatePath = join(cookbookDir, templateName); + const config = await loadConfig(); + const cookbookDir = join(getDoraDir(config.root), "cookbook"); + const format = options.format || "json"; + const templateName = recipe ? `${recipe}.md` : "index.md"; + const templatePath = join(cookbookDir, templateName); - try { - const content = readFileSync(templatePath, "utf-8"); + try { + const content = readFileSync(templatePath, "utf-8"); - if (format === "markdown") { - console.log(content.trim()); - } else { - outputJson({ - recipe: recipe || "index", - content: content.trim(), - }); - } - } catch (error) { - if (error instanceof Error && error.message.includes("ENOENT")) { - const availableRecipes = getAvailableRecipes(cookbookDir); - throw new Error( - `Recipe '${recipe}' not found. Available recipes: ${availableRecipes.join(", ")}\n\nCookbook files missing. Run 'dora init' to restore them.` - ); - } - throw error; - } + if (format === "markdown") { + console.log(content.trim()); + } else { + outputJson({ + recipe: recipe || "index", + content: content.trim(), + }); + } + } catch (error) { + if (error instanceof Error && error.message.includes("ENOENT")) { + const availableRecipes = getAvailableRecipes(cookbookDir); + throw new Error( + `Recipe '${recipe}' not found. Available recipes: ${availableRecipes.join(", ")}\n\nCookbook files missing. Run 'dora init' to restore them.`, + ); + } + throw error; + } } diff --git a/src/commands/coupling.ts b/src/commands/coupling.ts index 2951c43..1f7495e 100644 --- a/src/commands/coupling.ts +++ b/src/commands/coupling.ts @@ -11,7 +11,7 @@ export async function coupling( ): Promise { const { db } = await setupCommand(); - const threshold = parseIntFlag(flags, "threshold", 5); + const threshold = parseIntFlag({ flags, key: "threshold", defaultValue: 5 }); // Get coupled files const coupledFiles = getCoupledFiles(db, threshold); diff --git a/src/commands/cycles.ts b/src/commands/cycles.ts index 037fead..7bde2d6 100644 --- a/src/commands/cycles.ts +++ b/src/commands/cycles.ts @@ -11,7 +11,7 @@ export async function cycles( ): Promise { const { db } = await setupCommand(); - const limit = parseIntFlag(flags, "limit", 50); + const limit = parseIntFlag({ flags, key: "limit", defaultValue: 50 }); // Get bidirectional dependencies const cyclesList = getCycles(db, limit); diff --git a/src/commands/deps.ts b/src/commands/deps.ts index b3bb36b..a9893b3 100644 --- a/src/commands/deps.ts +++ b/src/commands/deps.ts @@ -13,8 +13,12 @@ export async function deps( flags: Record = {}, ): Promise { const ctx = await setupCommand(); - const depth = parseIntFlag(flags, "depth", DEFAULTS.DEPTH); - const relativePath = resolveAndValidatePath(ctx, path); + const depth = parseIntFlag({ + flags, + key: "depth", + defaultValue: DEFAULTS.DEPTH, + }); + const relativePath = resolveAndValidatePath({ ctx, inputPath: path }); const dependencies = getDependencies(ctx.db, relativePath, depth); diff --git a/src/commands/docs/search.ts b/src/commands/docs/search.ts index c5c3646..a0d7399 100644 --- a/src/commands/docs/search.ts +++ b/src/commands/docs/search.ts @@ -9,7 +9,11 @@ export async function docsSearch( ): Promise { const ctx = await setupCommand(); const db = ctx.db; - const limit = parseIntFlag(flags, "limit", DEFAULT_LIMIT); + const limit = parseIntFlag({ + flags, + key: "limit", + defaultValue: DEFAULT_LIMIT, + }); if (limit <= 0) { throw new Error("Limit must be a positive number"); diff --git a/src/commands/exports.ts b/src/commands/exports.ts index a52295e..c011341 100644 --- a/src/commands/exports.ts +++ b/src/commands/exports.ts @@ -14,9 +14,9 @@ export async function exports( const ctx = await setupCommand(); // Try as file path first - const relativePath = resolvePath(ctx, target); + const relativePath = resolvePath({ ctx: { ctx, inputPath: target } }); - if (fileExists(ctx.db, relativePath)) { + if (fileExists({ db: ctx.db, relativePath })) { const exportedSymbols = getFileExports(ctx.db, relativePath); if (exportedSymbols.length > 0) { const result: ExportsResult = { diff --git a/src/commands/file.ts b/src/commands/file.ts index fe819b4..13f78e4 100644 --- a/src/commands/file.ts +++ b/src/commands/file.ts @@ -8,7 +8,7 @@ import { outputJson, resolveAndValidatePath, setupCommand } from "./shared.ts"; export async function file(path: string): Promise { const ctx = await setupCommand(); - const relativePath = resolveAndValidatePath(ctx, path); + const relativePath = resolveAndValidatePath({ ctx, inputPath: path }); const symbols = getFileSymbols(ctx.db, relativePath); const depends_on = getFileDependencies(ctx.db, relativePath); diff --git a/src/commands/graph.ts b/src/commands/graph.ts index 93c06e8..9e56535 100644 --- a/src/commands/graph.ts +++ b/src/commands/graph.ts @@ -17,8 +17,16 @@ export async function graph( flags: Record = {}, ): Promise { const ctx = await setupCommand(); - const depth = parseIntFlag(flags, "depth", DEFAULTS.DEPTH); - const direction = parseStringFlag(flags, "direction", "both"); + const depth = parseIntFlag({ + flags, + key: "depth", + defaultValue: DEFAULTS.DEPTH, + }); + const direction = parseStringFlag({ + flags, + key: "direction", + defaultValue: "both", + }); if ( !VALID_DIRECTIONS.includes(direction as (typeof VALID_DIRECTIONS)[number]) @@ -28,7 +36,7 @@ export async function graph( ); } - const relativePath = resolveAndValidatePath(ctx, path); + const relativePath = resolveAndValidatePath({ ctx, inputPath: path }); // Build graph const nodes = new Set(); @@ -45,7 +53,7 @@ export async function graph( } if (direction === "rdeps" || direction === "both") { - const rdeps = getReverseDependencies(ctx.db, relativePath, depth); + const rdeps = getReverseDependencies({ db: ctx.db, relativePath, depth }); rdeps.forEach((rdep) => { nodes.add(rdep.path); edges.push({ from: rdep.path, to: relativePath }); diff --git a/src/commands/imports.ts b/src/commands/imports.ts index d199f3a..9cc5323 100644 --- a/src/commands/imports.ts +++ b/src/commands/imports.ts @@ -7,7 +7,7 @@ export async function imports( _flags: Record = {}, ): Promise { const ctx = await setupCommand(); - const relativePath = resolveAndValidatePath(ctx, path); + const relativePath = resolveAndValidatePath({ ctx, inputPath: path }); const importsList = getFileImports(ctx.db, relativePath); diff --git a/src/commands/index.ts b/src/commands/index.ts index 1c3385f..0413a9e 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -9,122 +9,128 @@ import { outputJson } from "../utils/output.ts"; import { resolveAbsolute } from "../utils/paths.ts"; export interface IndexOptions { - full?: boolean; - skipScip?: boolean; - ignore?: string[]; + full?: boolean; + skipScip?: boolean; + ignore?: string[]; } export async function index(options: IndexOptions = {}): Promise { - const startTime = Date.now(); - - debugIndex("Loading configuration..."); - const config = await loadConfig(); - debugIndex( - `Config loaded: root=${config.root}, scip=${config.scip}, db=${config.db}` - ); - - const ignorePatterns = [...(config.ignore || []), ...(options.ignore || [])]; - - if (ignorePatterns.length > 0) { - debugIndex(`Ignore patterns configured: ${ignorePatterns.join(", ")}`); - } - - const scipPath = resolveAbsolute(config.root, config.scip); - const databasePath = resolveAbsolute(config.root, config.db); - debugIndex( - `Resolved paths: scipPath=${scipPath}, databasePath=${databasePath}` - ); - - if (options.skipScip) { - debugIndex("Skipping SCIP indexer (--skip-scip flag set)"); - } else if (config.commands?.index) { - debugIndex(`Running SCIP indexer: ${config.commands.index}`); - await runCommand(config.commands.index, config.root, "Indexing"); - debugIndex("SCIP indexer completed successfully"); - } else { - debugIndex( - "No index command configured, checking for existing SCIP file..." - ); - if (!existsSync(scipPath)) { - throw new CtxError( - `No SCIP index found at ${scipPath} and no index command configured. ` + - `Either:\n` + - `1. Run your SCIP indexer manually to create ${config.scip}\n` + - `2. Configure commands.index in .dora/config.json to run your indexer automatically\n\n` + - `Example config:\n` + - `{\n` + - ` "commands": {\n` + - ` "index": "scip-typescript index --output .dora/index.scip"\n` + - ` }\n` + - `}` - ); - } - } - - debugIndex("Verifying SCIP file exists..."); - if (!existsSync(scipPath)) { - throw new CtxError( - `SCIP file not created at ${scipPath}. Check your commands configuration.` - ); - } - debugIndex("SCIP file verified"); - - debugIndex( - `Starting conversion to database (mode: ${options.full ? "full" : "auto"})` - ); - const conversionStats = await convertToDatabase( - scipPath, - databasePath, - config.root, - { - force: options.full, - ignore: ignorePatterns, - } - ); - debugIndex( - `Conversion completed: ${conversionStats.mode} mode, ${conversionStats.total_files} files, ${conversionStats.total_symbols} symbols` - ); - - debugIndex("Closing database connection..."); - closeDb(); - - debugIndex("Updating config with last indexed timestamp..."); - config.lastIndexed = new Date().toISOString(); - await saveConfig(config); - - const time_ms = Date.now() - startTime; - - const result: IndexResult = { - success: true, - file_count: conversionStats.total_files, - symbol_count: conversionStats.total_symbols, - time_ms, - mode: conversionStats.mode, - changed_files: conversionStats.changed_files, - }; - - debugIndex(`Indexing completed successfully in ${time_ms}ms`); - outputJson(result); + const startTime = Date.now(); + + debugIndex("Loading configuration..."); + const config = await loadConfig(); + debugIndex( + `Config loaded: root=${config.root}, scip=${config.scip}, db=${config.db}`, + ); + + const ignorePatterns = [...(config.ignore || []), ...(options.ignore || [])]; + + if (ignorePatterns.length > 0) { + debugIndex(`Ignore patterns configured: ${ignorePatterns.join(", ")}`); + } + + const scipPath = resolveAbsolute({ + root: config.root, + relativePath: config.scip, + }); + const databasePath = resolveAbsolute({ + root: config.root, + relativePath: config.db, + }); + debugIndex( + `Resolved paths: scipPath=${scipPath}, databasePath=${databasePath}`, + ); + + if (options.skipScip) { + debugIndex("Skipping SCIP indexer (--skip-scip flag set)"); + } else if (config.commands?.index) { + debugIndex(`Running SCIP indexer: ${config.commands.index}`); + await runCommand(config.commands.index, config.root, "Indexing"); + debugIndex("SCIP indexer completed successfully"); + } else { + debugIndex( + "No index command configured, checking for existing SCIP file...", + ); + if (!existsSync(scipPath)) { + throw new CtxError( + `No SCIP index found at ${scipPath} and no index command configured. ` + + `Either:\n` + + `1. Run your SCIP indexer manually to create ${config.scip}\n` + + `2. Configure commands.index in .dora/config.json to run your indexer automatically\n\n` + + `Example config:\n` + + `{\n` + + ` "commands": {\n` + + ` "index": "scip-typescript index --output .dora/index.scip"\n` + + ` }\n` + + `}`, + ); + } + } + + debugIndex("Verifying SCIP file exists..."); + if (!existsSync(scipPath)) { + throw new CtxError( + `SCIP file not created at ${scipPath}. Check your commands configuration.`, + ); + } + debugIndex("SCIP file verified"); + + debugIndex( + `Starting conversion to database (mode: ${options.full ? "full" : "auto"})`, + ); + const conversionStats = await convertToDatabase( + scipPath, + databasePath, + config.root, + { + force: options.full, + ignore: ignorePatterns, + }, + ); + debugIndex( + `Conversion completed: ${conversionStats.mode} mode, ${conversionStats.total_files} files, ${conversionStats.total_symbols} symbols`, + ); + + debugIndex("Closing database connection..."); + closeDb(); + + debugIndex("Updating config with last indexed timestamp..."); + config.lastIndexed = new Date().toISOString(); + await saveConfig(config); + + const time_ms = Date.now() - startTime; + + const result: IndexResult = { + success: true, + file_count: conversionStats.total_files, + symbol_count: conversionStats.total_symbols, + time_ms, + mode: conversionStats.mode, + changed_files: conversionStats.changed_files, + }; + + debugIndex(`Indexing completed successfully in ${time_ms}ms`); + outputJson(result); } /** * Run a shell command */ async function runCommand( - command: string, - cwd: string, - label: string + command: string, + cwd: string, + label: string, ): Promise { - const proc = Bun.spawn(command.split(" "), { - cwd, - stdout: "pipe", - stderr: "pipe", - }); - - const exitCode = await proc.exited; - - if (exitCode !== 0) { - const stderr = await new Response(proc.stderr).text(); - throw new CtxError(`${label} failed: ${stderr || "Unknown error"}`); - } + const proc = Bun.spawn(command.split(" "), { + cwd, + stdout: "pipe", + stderr: "pipe", + }); + + const exitCode = await proc.exited; + + if (exitCode !== 0) { + const stderr = await new Response(proc.stderr).text(); + throw new CtxError(`${label} failed: ${stderr || "Unknown error"}`); + } } diff --git a/src/commands/leaves.ts b/src/commands/leaves.ts index ecdeaad..239014a 100644 --- a/src/commands/leaves.ts +++ b/src/commands/leaves.ts @@ -6,11 +6,11 @@ export async function leaves( flags: Record = {}, ): Promise { const ctx = await setupCommand(); - const maxDependents = parseIntFlag( + const maxDependents = parseIntFlag({ flags, - "max-dependents", - DEFAULTS.LEAF_MAX_DEPENDENTS, - ); + key: "max-dependents", + defaultValue: DEFAULTS.LEAF_MAX_DEPENDENTS, + }); const leafNodes = getLeafNodes(ctx.db, maxDependents); diff --git a/src/commands/lost.ts b/src/commands/lost.ts index e3c0ea6..77cb492 100644 --- a/src/commands/lost.ts +++ b/src/commands/lost.ts @@ -6,7 +6,11 @@ export async function lost( flags: Record = {}, ): Promise { const ctx = await setupCommand(); - const limit = parseIntFlag(flags, "limit", DEFAULTS.UNUSED_LIMIT); + const limit = parseIntFlag({ + flags, + key: "limit", + defaultValue: DEFAULTS.UNUSED_LIMIT, + }); const unusedSymbols = getUnusedSymbols(ctx.db, limit); diff --git a/src/commands/ls.ts b/src/commands/ls.ts index 8638f72..5e45b83 100644 --- a/src/commands/ls.ts +++ b/src/commands/ls.ts @@ -84,8 +84,8 @@ export async function ls( ): Promise { const ctx = await setupCommand(); - const limit = parseIntFlag(flags, "limit", 100); - const sort = parseStringFlag(flags, "sort", "path"); + const limit = parseIntFlag({ flags, key: "limit", defaultValue: 100 }); + const sort = parseStringFlag({ flags, key: "sort", defaultValue: "path" }); // Validate sort option if (!["path", "symbols", "deps", "rdeps"].includes(sort)) { diff --git a/src/commands/rdeps.ts b/src/commands/rdeps.ts index a4c5fc4..3a51d67 100644 --- a/src/commands/rdeps.ts +++ b/src/commands/rdeps.ts @@ -13,8 +13,12 @@ export async function rdeps( flags: Record = {}, ): Promise { const ctx = await setupCommand(); - const depth = parseIntFlag(flags, "depth", DEFAULTS.DEPTH); - const relativePath = resolveAndValidatePath(ctx, path); + const depth = parseIntFlag({ + flags, + key: "depth", + defaultValue: DEFAULTS.DEPTH, + }); + const relativePath = resolveAndValidatePath({ ctx, inputPath: path }); const dependents = getReverseDependencies(ctx.db, relativePath, depth); diff --git a/src/commands/refs.ts b/src/commands/refs.ts index 5b47d73..54323aa 100644 --- a/src/commands/refs.ts +++ b/src/commands/refs.ts @@ -13,8 +13,12 @@ export async function refs( flags: Record = {}, ): Promise { const ctx = await setupCommand(); - const kind = parseOptionalStringFlag(flags, "kind"); - const limit = parseIntFlag(flags, "limit", DEFAULTS.REFS_LIMIT); + const kind = parseOptionalStringFlag({ flags, key: "kind" }); + const limit = parseIntFlag({ + flags, + key: "limit", + defaultValue: DEFAULTS.REFS_LIMIT, + }); const results = getSymbolReferences(ctx.db, query, { kind, limit }); diff --git a/src/commands/shared.ts b/src/commands/shared.ts index 3474834..506d0c5 100644 --- a/src/commands/shared.ts +++ b/src/commands/shared.ts @@ -42,11 +42,15 @@ export async function setupCommand(): Promise { /** * Parse an integer flag with a default value. */ -export function parseIntFlag( - flags: Record, - key: string, - defaultValue: number, -): number { +export function parseIntFlag({ + flags, + key, + defaultValue, +}: { + flags: Record; + key: string; + defaultValue: number; +}): number { const value = flags[key]; if (value === undefined || value === true) { return defaultValue; @@ -58,11 +62,15 @@ export function parseIntFlag( /** * Parse a string flag with a default value. */ -export function parseStringFlag( - flags: Record, - key: string, - defaultValue: string, -): string { +export function parseStringFlag({ + flags, + key, + defaultValue, +}: { + flags: Record; + key: string; + defaultValue: string; +}): string { const value = flags[key]; if (value === undefined || value === true) { return defaultValue; @@ -73,10 +81,13 @@ export function parseStringFlag( /** * Parse an optional string flag (returns undefined if not provided). */ -export function parseOptionalStringFlag( - flags: Record, - key: string, -): string | undefined { +export function parseOptionalStringFlag({ + flags, + key, +}: { + flags: Record; + key: string; +}): string | undefined { const value = flags[key]; if (value === undefined || value === true) { return undefined; @@ -93,11 +104,17 @@ export function parseOptionalStringFlag( * If path is a typo with suggestions, outputs suggestions and exits. * Otherwise throws CtxError. */ -export function resolveAndValidatePath( - ctx: CommandContext, - inputPath: string, -): string { - const relativePath = normalizeToRelative(ctx.config.root, inputPath); +export function resolveAndValidatePath({ + ctx, + inputPath, +}: { + ctx: CommandContext; + inputPath: string; +}): string { + const relativePath = normalizeToRelative({ + root: ctx.config.root, + inputPath, + }); if (fileExists(ctx.db, relativePath)) { return relativePath; @@ -147,8 +164,14 @@ export function resolveAndValidatePath( /** * Normalize path without validation (for cases where we want to check existence separately). */ -export function resolvePath(ctx: CommandContext, inputPath: string): string { - return normalizeToRelative(ctx.config.root, inputPath); +export function resolvePath({ + ctx, + inputPath, +}: { + ctx: CommandContext; + inputPath: string; +}): string { + return normalizeToRelative({ root: ctx.config.root, inputPath }); } export { outputJson }; diff --git a/src/commands/symbol.ts b/src/commands/symbol.ts index 119fe68..94d3d53 100644 --- a/src/commands/symbol.ts +++ b/src/commands/symbol.ts @@ -13,8 +13,12 @@ export async function symbol( flags: Record = {}, ): Promise { const ctx = await setupCommand(); - const limit = parseIntFlag(flags, "limit", DEFAULTS.SYMBOL_LIMIT); - const kind = parseOptionalStringFlag(flags, "kind"); + const limit = parseIntFlag({ + flags, + key: "limit", + defaultValue: DEFAULTS.SYMBOL_LIMIT, + }); + const kind = parseOptionalStringFlag({ flags, key: "kind" }); const results = searchSymbols(ctx.db, query, { kind, limit }); diff --git a/src/commands/treasure.ts b/src/commands/treasure.ts index 6034e91..8bb5c65 100644 --- a/src/commands/treasure.ts +++ b/src/commands/treasure.ts @@ -9,7 +9,11 @@ export async function treasure( flags: Record = {}, ): Promise { const ctx = await setupCommand(); - const limit = parseIntFlag(flags, "limit", DEFAULTS.HOTSPOTS_LIMIT); + const limit = parseIntFlag({ + flags, + key: "limit", + defaultValue: DEFAULTS.HOTSPOTS_LIMIT, + }); const mostReferenced = getMostReferencedFiles(ctx.db, limit); const mostDependencies = getMostDependentFiles(ctx.db, limit); diff --git a/src/converter/convert.ts b/src/converter/convert.ts index f19358e..f9fbd0d 100644 --- a/src/converter/convert.ts +++ b/src/converter/convert.ts @@ -1,23 +1,22 @@ import { Database } from "bun:sqlite"; -import { existsSync } from "fs"; import ignore from "ignore"; import path from "path"; import { debugConverter } from "../utils/logger.ts"; +import { processDocuments } from "./documents"; import { - extractKindFromDocumentation, - extractNameFromScip, - extractPackageFromScip, - symbolKindToString, + extractKindFromDocumentation, + extractNameFromScip, + extractPackageFromScip, + symbolKindToString, } from "./helpers"; import { - buildLookupMaps, - extractDefinitions, - extractReferences, - getFileDependencies, - type ParsedDocument, - type ParsedSymbol, - parseScipFile, - type ScipData, + extractDefinitions, + extractReferences, + getFileDependencies, + type ParsedDocument, + type ParsedSymbol, + parseScipFile, + type ScipData, } from "./scip-parser"; // Batch size for processing documents to avoid memory exhaustion @@ -171,35 +170,35 @@ CREATE INDEX IF NOT EXISTS idx_document_document_refs_referenced ON document_doc CREATE INDEX IF NOT EXISTS idx_document_document_refs_line ON document_document_refs(line);`; export interface ConversionOptions { - force?: boolean; - ignore?: string[]; + force?: boolean; + ignore?: string[]; } export interface ConversionStats { - mode: "full" | "incremental"; - total_files: number; - total_symbols: number; - changed_files: number; - deleted_files: number; - time_ms: number; - total_documents?: number; - processed_documents?: number; + mode: "full" | "incremental"; + total_files: number; + total_symbols: number; + changed_files: number; + deleted_files: number; + time_ms: number; + total_documents?: number; + processed_documents?: number; } interface ChangedFile { - path: string; - mtime: number; + path: string; + mtime: number; } /** * Helper function to chunk an array into smaller batches */ function chunkArray(array: T[], chunkSize: number): T[][] { - const chunks: T[][] = []; - for (let i = 0; i < array.length; i += chunkSize) { - chunks.push(array.slice(i, i + chunkSize)); - } - return chunks; + const chunks: T[][] = []; + for (let i = 0; i < array.length; i += chunkSize) { + chunks.push(array.slice(i, i + chunkSize)); + } + return chunks; } /** @@ -216,527 +215,548 @@ function chunkArray(array: T[], chunkSize: number): T[][] { * @throws {Error} If SCIP file cannot be parsed or database cannot be created */ export async function convertToDatabase({ - scipPath, - databasePath, - repoRoot, - options = {}, + scipPath, + databasePath, + repoRoot, + options = {}, }: { - scipPath: string; - databasePath: string; - repoRoot: string; - options?: ConversionOptions; + scipPath: string; + databasePath: string; + repoRoot: string; + options?: ConversionOptions; }): Promise { - const startTime = Date.now(); - - // Parse SCIP protobuf file - debugConverter(`Parsing SCIP file at ${scipPath}...`); - let scipData: ScipData; - try { - scipData = await parseScipFile(scipPath); - debugConverter(`Parsed SCIP file: ${scipData.documents.length} documents`); - } catch (error) { - throw new Error(`Failed to parse SCIP file at ${scipPath}: ${error}`); - } - - // Open database - debugConverter(`Opening database at ${databasePath}...`); - let db: Database; - try { - db = new Database(databasePath, { create: true }); - debugConverter("Database opened successfully"); - } catch (error) { - throw new Error( - `Failed to open/create database at ${databasePath}: ${error}` - ); - } - - // Initialize schema - debugConverter("Initializing database schema..."); - initializeSchema(db); - debugConverter("Schema initialized"); - - // Optimize database for bulk writes - optimizeDatabaseForWrites(db); - - // Determine if this is a full or incremental build - const isFirstRun = !hasExistingData(db); - const isForceFull = options.force === true; - const mode = isFirstRun || isForceFull ? "full" : "incremental"; - debugConverter( - `Build mode: ${mode} (firstRun=${isFirstRun}, force=${isForceFull})` - ); - - // Create ignore matcher if patterns are provided - const ig = ignore(); - if (options.ignore && options.ignore.length > 0) { - ig.add(options.ignore); - debugConverter(`Filtering with ${options.ignore.length} ignore patterns`); - } - - // Build a quick document lookup (lightweight - just paths) - const documentsByPath = new Map( - scipData.documents.map((doc) => [doc.relativePath, doc]) - ); - - let changedFiles: ChangedFile[]; - let deletedFiles: string[]; - - if (mode === "full") { - // Full rebuild: get all files from SCIP data - debugConverter("Getting all files for full rebuild..."); - changedFiles = await getAllFiles(documentsByPath, repoRoot, ig); - deletedFiles = []; - debugConverter(`Full rebuild: processing ${changedFiles.length} files`); - - // Clear existing data - debugConverter("Clearing existing database data..."); - clearAllData(db); - debugConverter("Existing data cleared"); - } else { - // Incremental: detect changes via filesystem scan - debugConverter("Detecting changed and deleted files..."); - const changes = await detectChangedFiles(documentsByPath, db, repoRoot, ig); - changedFiles = changes.changed; - deletedFiles = changes.deleted; - debugConverter( - `Incremental build: ${changedFiles.length} changed, ${deletedFiles.length} deleted` - ); - } - - // Delete old data - if (deletedFiles.length > 0) { - debugConverter( - `Deleting ${deletedFiles.length} old files from database...` - ); - deleteOldData(db, deletedFiles, []); - debugConverter(`Deleted ${deletedFiles.length} files from database`); - } - - if (changedFiles.length > 0) { - // Delete old versions of changed files - debugConverter( - `Removing old versions of ${changedFiles.length} changed files...` - ); - deleteOldData(db, [], changedFiles); - - // Process files in batches to avoid memory exhaustion - await processBatches(db, scipData, changedFiles, repoRoot); - } - - // Restore database settings - restoreDatabaseSettings(db); - - // Update packages (skip if no files changed) - debugConverter("Updating packages table..."); - updatePackages(db, changedFiles.length === 0 && deletedFiles.length === 0); - debugConverter("Packages table updated"); - - // Update denormalized fields - debugConverter("Updating denormalized fields..."); - updateDenormalizedFields(db); - debugConverter("Denormalized fields updated"); - - // Process documentation files - debugConverter("Processing documentation files..."); - const { processDocuments } = await import("./documents.js"); - const docStats = await processDocuments( - db, - repoRoot, - mode, - options.ignore || [] - ); - debugConverter( - `Documentation processing complete: ${docStats.processed} processed, ${docStats.skipped} skipped` - ); - - // Update metadata and get stats - debugConverter("Updating metadata..."); - const stats = updateMetadata( - db, - mode, - changedFiles.length, - deletedFiles.length - ); - debugConverter( - `Metadata updated: ${stats.total_files} total files, ${stats.total_symbols} total symbols` - ); - - // Close database - debugConverter("Closing database..."); - db.close(); - - const timeMs = Date.now() - startTime; - - return { - ...stats, - time_ms: timeMs, - total_documents: docStats.total, - processed_documents: docStats.processed, - }; + const startTime = Date.now(); + + // Parse SCIP protobuf file + debugConverter(`Parsing SCIP file at ${scipPath}...`); + let scipData: ScipData; + try { + scipData = await parseScipFile(scipPath); + debugConverter(`Parsed SCIP file: ${scipData.documents.length} documents`); + } catch (error) { + throw new Error(`Failed to parse SCIP file at ${scipPath}: ${error}`); + } + + // Open database + debugConverter(`Opening database at ${databasePath}...`); + let db: Database; + try { + db = new Database(databasePath, { create: true }); + debugConverter("Database opened successfully"); + } catch (error) { + throw new Error( + `Failed to open/create database at ${databasePath}: ${error}`, + ); + } + + // Initialize schema + debugConverter("Initializing database schema..."); + initializeSchema(db); + debugConverter("Schema initialized"); + + // Optimize database for bulk writes + optimizeDatabaseForWrites(db); + + // Determine if this is a full or incremental build + const isFirstRun = !hasExistingData(db); + const isForceFull = options.force === true; + const mode = isFirstRun || isForceFull ? "full" : "incremental"; + debugConverter( + `Build mode: ${mode} (firstRun=${isFirstRun}, force=${isForceFull})`, + ); + + // Create ignore matcher if patterns are provided + const ig = ignore(); + if (options.ignore && options.ignore.length > 0) { + ig.add(options.ignore); + debugConverter(`Filtering with ${options.ignore.length} ignore patterns`); + } + + // Build a quick document lookup (lightweight - just paths) + const documentsByPath = new Map( + scipData.documents.map((doc) => [doc.relativePath, doc]), + ); + + let changedFiles: ChangedFile[]; + let deletedFiles: string[]; + + if (mode === "full") { + // Full rebuild: get all files from SCIP data + debugConverter("Getting all files for full rebuild..."); + changedFiles = await getAllFiles({ documentsByPath, repoRoot, ig }); + deletedFiles = []; + debugConverter(`Full rebuild: processing ${changedFiles.length} files`); + + // Clear existing data + debugConverter("Clearing existing database data..."); + clearAllData(db); + debugConverter("Existing data cleared"); + } else { + // Incremental: detect changes via filesystem scan + debugConverter("Detecting changed and deleted files..."); + const changes = await detectChangedFiles({ + documentsByPath, + db, + repoRoot, + ig, + }); + changedFiles = changes.changed; + deletedFiles = changes.deleted; + debugConverter( + `Incremental build: ${changedFiles.length} changed, ${deletedFiles.length} deleted`, + ); + } + + // Delete old data + if (deletedFiles.length > 0) { + debugConverter( + `Deleting ${deletedFiles.length} old files from database...`, + ); + deleteOldData(db, deletedFiles, []); + debugConverter(`Deleted ${deletedFiles.length} files from database`); + } + + if (changedFiles.length > 0) { + // Delete old versions of changed files + debugConverter( + `Removing old versions of ${changedFiles.length} changed files...`, + ); + deleteOldData(db, [], changedFiles); + + // Process files in batches to avoid memory exhaustion + await processBatches({ db, scipData, changedFiles, repoRoot }); + } + + // Restore database settings + restoreDatabaseSettings(db); + + // Update packages (skip if no files changed) + debugConverter("Updating packages table..."); + updatePackages({ + db, + skipIfNoChanges: changedFiles.length === 0 && deletedFiles.length === 0, + }); + debugConverter("Packages table updated"); + + // Update denormalized fields + debugConverter("Updating denormalized fields..."); + updateDenormalizedFields(db); + debugConverter("Denormalized fields updated"); + + // Process documentation files + debugConverter("Processing documentation files..."); + const docStats = await processDocuments({ + db, + repoRoot, + mode, + ignorePatterns: options.ignore || [], + }); + debugConverter( + `Documentation processing complete: ${docStats.processed} processed, ${docStats.skipped} skipped`, + ); + + // Update metadata and get stats + debugConverter("Updating metadata..."); + const stats = updateMetadata({ + db, + mode, + changedFiles: changedFiles.length, + deletedFiles: deletedFiles.length, + }); + debugConverter( + `Metadata updated: ${stats.total_files} total files, ${stats.total_symbols} total symbols`, + ); + + // Close database + debugConverter("Closing database..."); + db.close(); + + const timeMs = Date.now() - startTime; + + return { + ...stats, + time_ms: timeMs, + total_documents: docStats.total, + processed_documents: docStats.processed, + }; } /** * Process files in batches to avoid memory exhaustion */ -async function processBatches( - db: Database, - scipData: ScipData, - changedFiles: ChangedFile[], - repoRoot: string -): Promise { - const timestamp = Math.floor(Date.now() / 1000); - - const insertedFiles = new Set(); - - // Create a set of changed paths for quick lookup - const changedPathsSet = new Set(changedFiles.map((f) => f.path)); - - // Filter scipData documents to only include changed files - const docsToProcess = scipData.documents.filter((doc) => - changedPathsSet.has(doc.relativePath) - ); - - debugConverter( - `Processing ${docsToProcess.length} documents in batches of ${BATCH_SIZE}...` - ); - - // Build LIGHTWEIGHT global definition map (only symbol -> file path) - debugConverter("Building lightweight global definition map..."); - const globalDefinitionsBySymbol = new Map< - string, - { file: string; definition: any } - >(); - const externalSymbols = scipData.externalSymbols; - - // Process documents in chunks to build definition map without keeping all in memory - for (const doc of scipData.documents) { - // Extract only the symbol IDs and file path (very lightweight) - for (const occ of doc.occurrences) { - if (occ.symbolRoles & 0x1) { - // Definition bit - // Store minimal info - we'll get full details from documentsByPath later - if (!globalDefinitionsBySymbol.has(occ.symbol)) { - globalDefinitionsBySymbol.set(occ.symbol, { - file: doc.relativePath, - definition: { symbol: occ.symbol, range: occ.range }, - }); - } - } - } - } - debugConverter( - `Global definition map built: ${globalDefinitionsBySymbol.size} definitions` - ); - - // Build LIGHTWEIGHT global symbols map (only external symbols + doc symbols, no duplication) - debugConverter("Building global symbols map..."); - const globalSymbolsById = new Map(); - - // Add external symbols first (these are small, usually < 10K) - for (const sym of externalSymbols) { - globalSymbolsById.set(sym.symbol, sym); - } - - // Add document symbols efficiently (no deep copies) - for (const doc of scipData.documents) { - for (const sym of doc.symbols) { - if (!globalSymbolsById.has(sym.symbol)) { - globalSymbolsById.set(sym.symbol, sym); - } - } - } - debugConverter(`Global symbols map built: ${globalSymbolsById.size} symbols`); - - // Chunk documents into batches - const batches = chunkArray(docsToProcess, BATCH_SIZE); - - // Clear scipData external symbols reference (we copied it) - scipData.externalSymbols = []; - debugConverter("Cleared scipData external symbols"); - - const totalBatches = batches.length; - let processedFiles = 0; - const totalFiles = docsToProcess.length; - const progressStartTime = Date.now(); - - for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) { - const batch = batches[batchIndex]; - const batchNum = batchIndex + 1; - - // Calculate progress - const percent = Math.floor((processedFiles / totalFiles) * 100); - const elapsed = (Date.now() - progressStartTime) / 1000; - const rate = processedFiles / elapsed || 0; - const remaining = totalFiles - processedFiles; - const eta = rate > 0 ? Math.ceil(remaining / rate) : 0; - - debugConverter( - `\rIndexing: ${percent}% (${processedFiles}/${totalFiles} files, batch ${batchNum}/${totalBatches}, ETA: ${eta}s) ` - ); - - // Build lightweight document map for this batch - const documentsByPath = new Map( - batch.map((doc) => [doc.relativePath, doc]) - ); - - // Get ChangedFile objects for this batch - const batchChangedFiles = changedFiles.filter((f) => - batch.some((doc) => doc.relativePath === f.path) - ); - - // Convert files in this batch - await convertFiles( - documentsByPath, - globalSymbolsById, - db, - batchChangedFiles, - timestamp, - insertedFiles - ); - - // Update dependencies for this batch (uses global maps for cross-batch deps) - await updateDependencies( - documentsByPath, - globalSymbolsById, - globalDefinitionsBySymbol, - db, - batchChangedFiles - ); - - // Update symbol references for this batch - await updateSymbolReferences( - documentsByPath, - globalSymbolsById, - db, - batchChangedFiles - ); - - processedFiles += batch.length; - } - - process.stderr.write("\n"); - debugConverter( - `Batch processing complete: ${processedFiles} files processed` - ); +async function processBatches({ + db, + scipData, + changedFiles, + repoRoot, +}: { + db: Database; + scipData: ScipData; + changedFiles: ChangedFile[]; + repoRoot: string; +}): Promise { + const timestamp = Math.floor(Date.now() / 1000); + + const insertedFiles = new Set(); + + // Create a set of changed paths for quick lookup + const changedPathsSet = new Set(changedFiles.map((f) => f.path)); + + // Filter scipData documents to only include changed files + const docsToProcess = scipData.documents.filter((doc) => + changedPathsSet.has(doc.relativePath), + ); + + debugConverter( + `Processing ${docsToProcess.length} documents in batches of ${BATCH_SIZE}...`, + ); + + // Build LIGHTWEIGHT global definition map (only symbol -> file path) + debugConverter("Building lightweight global definition map..."); + const globalDefinitionsBySymbol = new Map< + string, + { file: string; definition: any } + >(); + const externalSymbols = scipData.externalSymbols; + + // Process documents in chunks to build definition map without keeping all in memory + for (const doc of scipData.documents) { + // Extract only the symbol IDs and file path (very lightweight) + for (const occ of doc.occurrences) { + if (occ.symbolRoles & 0x1) { + // Definition bit + // Store minimal info - we'll get full details from documentsByPath later + if (!globalDefinitionsBySymbol.has(occ.symbol)) { + globalDefinitionsBySymbol.set(occ.symbol, { + file: doc.relativePath, + definition: { symbol: occ.symbol, range: occ.range }, + }); + } + } + } + } + debugConverter( + `Global definition map built: ${globalDefinitionsBySymbol.size} definitions`, + ); + + // Build LIGHTWEIGHT global symbols map (only external symbols + doc symbols, no duplication) + debugConverter("Building global symbols map..."); + const globalSymbolsById = new Map(); + + // Add external symbols first (these are small, usually < 10K) + for (const sym of externalSymbols) { + globalSymbolsById.set(sym.symbol, sym); + } + + // Add document symbols efficiently (no deep copies) + for (const doc of scipData.documents) { + for (const sym of doc.symbols) { + if (!globalSymbolsById.has(sym.symbol)) { + globalSymbolsById.set(sym.symbol, sym); + } + } + } + debugConverter(`Global symbols map built: ${globalSymbolsById.size} symbols`); + + // Chunk documents into batches + const batches = chunkArray(docsToProcess, BATCH_SIZE); + + // Clear scipData external symbols reference (we copied it) + scipData.externalSymbols = []; + debugConverter("Cleared scipData external symbols"); + + const totalBatches = batches.length; + let processedFiles = 0; + const totalFiles = docsToProcess.length; + const progressStartTime = Date.now(); + + for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) { + const batch = batches[batchIndex]; + const batchNum = batchIndex + 1; + + // Calculate progress + const percent = Math.floor((processedFiles / totalFiles) * 100); + const elapsed = (Date.now() - progressStartTime) / 1000; + const rate = processedFiles / elapsed || 0; + const remaining = totalFiles - processedFiles; + const eta = rate > 0 ? Math.ceil(remaining / rate) : 0; + + debugConverter( + `\rIndexing: ${percent}% (${processedFiles}/${totalFiles} files, batch ${batchNum}/${totalBatches}, ETA: ${eta}s) `, + ); + + // Build lightweight document map for this batch + const documentsByPath = new Map( + batch.map((doc) => [doc.relativePath, doc]), + ); + + // Get ChangedFile objects for this batch + const batchChangedFiles = changedFiles.filter((f) => + batch.some((doc) => doc.relativePath === f.path), + ); + + // Convert files in this batch + await convertFiles( + documentsByPath, + globalSymbolsById, + db, + batchChangedFiles, + timestamp, + insertedFiles, + ); + + // Update dependencies for this batch (uses global maps for cross-batch deps) + await updateDependencies( + documentsByPath, + globalSymbolsById, + globalDefinitionsBySymbol, + db, + batchChangedFiles, + ); + + // Update symbol references for this batch + await updateSymbolReferences({ + documentsByPath, + symbolsById: globalSymbolsById, + db, + changedFiles: batchChangedFiles, + }); + + processedFiles += batch.length; + } + + process.stderr.write("\n"); + debugConverter( + `Batch processing complete: ${processedFiles} files processed`, + ); } /** * Initialize database schema */ function initializeSchema(db: Database): void { - // Check if all tables exist (including new ones like documents) - // We always run the schema if any table is missing - try { - const filesCheck = db - .query( - "SELECT name FROM sqlite_master WHERE type='table' AND name='files'" - ) - .get(); - - const documentsCheck = db - .query( - "SELECT name FROM sqlite_master WHERE type='table' AND name='documents'" - ) - .get(); - - if (filesCheck && documentsCheck) { - // All tables exist, skip initialization - return; - } - } catch { - // Continue with initialization - } - - // Execute schema (multiple statements) - const statements = SCHEMA_SQL.split(";") - .map((s) => s.trim()) - .filter((s) => s.length > 0); - - for (const stmt of statements) { - db.run(stmt); - } + // Check if all tables exist (including new ones like documents) + // We always run the schema if any table is missing + try { + const filesCheck = db + .query( + "SELECT name FROM sqlite_master WHERE type='table' AND name='files'", + ) + .get(); + + const documentsCheck = db + .query( + "SELECT name FROM sqlite_master WHERE type='table' AND name='documents'", + ) + .get(); + + if (filesCheck && documentsCheck) { + // All tables exist, skip initialization + return; + } + } catch { + // Continue with initialization + } + + // Execute schema (multiple statements) + const statements = SCHEMA_SQL.split(";") + .map((s) => s.trim()) + .filter((s) => s.length > 0); + + for (const stmt of statements) { + db.run(stmt); + } } /** * Optimize database for bulk writes */ function optimizeDatabaseForWrites(db: Database): void { - debugConverter("Optimizing database for bulk writes..."); + debugConverter("Optimizing database for bulk writes..."); - // Disable synchronous writes (much faster, but less crash-safe during indexing) - db.run("PRAGMA synchronous = OFF"); + // Disable synchronous writes (much faster, but less crash-safe during indexing) + db.run("PRAGMA synchronous = OFF"); - // Use memory for journal (faster than disk) - db.run("PRAGMA journal_mode = MEMORY"); + // Use memory for journal (faster than disk) + db.run("PRAGMA journal_mode = MEMORY"); - // Increase cache size (10MB) - db.run("PRAGMA cache_size = -10000"); + // Increase cache size (10MB) + db.run("PRAGMA cache_size = -10000"); - debugConverter("Database optimizations applied"); + debugConverter("Database optimizations applied"); } /** * Restore normal database settings after bulk writes */ function restoreDatabaseSettings(db: Database): void { - debugConverter("Restoring normal database settings..."); + debugConverter("Restoring normal database settings..."); - // Re-enable synchronous writes - db.run("PRAGMA synchronous = FULL"); + // Re-enable synchronous writes + db.run("PRAGMA synchronous = FULL"); - // Switch back to WAL mode - db.run("PRAGMA journal_mode = WAL"); + // Switch back to WAL mode + db.run("PRAGMA journal_mode = WAL"); - debugConverter("Database settings restored"); + debugConverter("Database settings restored"); } /** * Check if database has existing data */ function hasExistingData(db: Database): boolean { - try { - const result = db.query("SELECT COUNT(*) as count FROM files").get() as { - count: number; - }; - return result.count > 0; - } catch { - return false; - } + try { + const result = db.query("SELECT COUNT(*) as count FROM files").get() as { + count: number; + }; + return result.count > 0; + } catch { + return false; + } } /** * Clear all data from database (for full rebuild) */ function clearAllData(db: Database): void { - db.run("BEGIN TRANSACTION"); - db.run("DELETE FROM symbol_references"); - db.run("DELETE FROM dependencies"); - db.run("DELETE FROM symbols"); - db.run("DELETE FROM files"); - db.run("DELETE FROM packages"); - db.run("DELETE FROM metadata"); - db.run("COMMIT"); + db.run("BEGIN TRANSACTION"); + db.run("DELETE FROM symbol_references"); + db.run("DELETE FROM dependencies"); + db.run("DELETE FROM symbols"); + db.run("DELETE FROM files"); + db.run("DELETE FROM packages"); + db.run("DELETE FROM metadata"); + db.run("COMMIT"); } /** * Get all files from SCIP data (for full rebuild) */ -async function getAllFiles( - documentsByPath: Map, - repoRoot: string, - ig: ReturnType -): Promise { - const files: ChangedFile[] = []; - - for (const [relativePath, doc] of documentsByPath) { - if (ig.ignores(relativePath)) { - continue; - } - - const fullPath = path.join(repoRoot, relativePath); - try { - const stat = await Bun.file(fullPath).stat(); - const mtime = Math.floor(stat.mtime.getTime() / 1000); - files.push({ path: relativePath, mtime }); - } catch {} - } - - return files; +async function getAllFiles({ + documentsByPath, + repoRoot, + ig, +}: { + documentsByPath: Map; + repoRoot: string; + ig: ReturnType; +}): Promise { + const files: ChangedFile[] = []; + + for (const [relativePath, doc] of documentsByPath) { + if (ig.ignores(relativePath)) { + continue; + } + + const fullPath = path.join(repoRoot, relativePath); + try { + const stat = await Bun.file(fullPath).stat(); + const mtime = Math.floor(stat.mtime.getTime() / 1000); + files.push({ path: relativePath, mtime }); + } catch {} + } + + return files; } /** * Detect changed and deleted files (for incremental rebuild) */ -async function detectChangedFiles( - documentsByPath: Map, - db: Database, - repoRoot: string, - ig: ReturnType -): Promise<{ changed: ChangedFile[]; deleted: string[] }> { - // Get existing files from database with mtime - const existingFiles = new Map( - ( - db.query("SELECT path, mtime FROM files").all() as Array<{ - path: string; - mtime: number; - }> - ).map((f) => [f.path, f.mtime]) - ); - - const changed: ChangedFile[] = []; - const deleted = new Set(existingFiles.keys()); - - for (const [relativePath, doc] of documentsByPath) { - if (ig.ignores(relativePath)) { - continue; - } - - deleted.delete(relativePath); - - // Get current mtime from filesystem - const fullPath = path.join(repoRoot, relativePath); - try { - const stat = await Bun.file(fullPath).stat(); - const currentMtime = Math.floor(stat.mtime.getTime() / 1000); - - const existingMtime = existingFiles.get(relativePath); - - // File is new or modified - if (!existingMtime || currentMtime > existingMtime) { - changed.push({ path: relativePath, mtime: currentMtime }); - } - } catch {} - } - - return { changed, deleted: Array.from(deleted) }; +async function detectChangedFiles({ + documentsByPath, + db, + repoRoot, + ig, +}: { + documentsByPath: Map; + db: Database; + repoRoot: string; + ig: ReturnType; +}): Promise<{ changed: ChangedFile[]; deleted: string[] }> { + // Get existing files from database with mtime + const existingFiles = new Map( + ( + db.query("SELECT path, mtime FROM files").all() as Array<{ + path: string; + mtime: number; + }> + ).map((f) => [f.path, f.mtime]), + ); + + const changed: ChangedFile[] = []; + const deleted = new Set(existingFiles.keys()); + + for (const [relativePath, doc] of documentsByPath) { + if (ig.ignores(relativePath)) { + continue; + } + + deleted.delete(relativePath); + + // Get current mtime from filesystem + const fullPath = path.join(repoRoot, relativePath); + try { + const stat = await Bun.file(fullPath).stat(); + const currentMtime = Math.floor(stat.mtime.getTime() / 1000); + + const existingMtime = existingFiles.get(relativePath); + + // File is new or modified + if (!existingMtime || currentMtime > existingMtime) { + changed.push({ path: relativePath, mtime: currentMtime }); + } + } catch {} + } + + return { changed, deleted: Array.from(deleted) }; } /** * Delete old data for deleted or changed files */ function deleteOldData( - db: Database, - deletedFiles: string[], - changedFiles: ChangedFile[] + db: Database, + deletedFiles: string[], + changedFiles: ChangedFile[], ): void { - const allFilesToRemove = [ - ...deletedFiles, - ...changedFiles.map((f) => f.path), - ]; + const allFilesToRemove = [ + ...deletedFiles, + ...changedFiles.map((f) => f.path), + ]; - if (allFilesToRemove.length === 0) return; + if (allFilesToRemove.length === 0) return; - db.run("BEGIN TRANSACTION"); + db.run("BEGIN TRANSACTION"); - const stmt = db.prepare("DELETE FROM files WHERE path = ?"); - for (const filePath of allFilesToRemove) { - stmt.run(filePath); - } + const stmt = db.prepare("DELETE FROM files WHERE path = ?"); + for (const filePath of allFilesToRemove) { + stmt.run(filePath); + } - db.run("COMMIT"); + db.run("COMMIT"); } /** * Convert changed files from SCIP data to database */ async function convertFiles( - documentsByPath: Map, - symbolsById: Map, - db: Database, - changedFiles: ChangedFile[], - timestamp: number, - insertedFiles: Set + documentsByPath: Map, + symbolsById: Map, + db: Database, + changedFiles: ChangedFile[], + timestamp: number, + insertedFiles: Set, ): Promise { - if (changedFiles.length === 0) return; + if (changedFiles.length === 0) return; - debugConverter("Starting database transaction for file conversion..."); - db.run("BEGIN TRANSACTION"); + debugConverter("Starting database transaction for file conversion..."); + db.run("BEGIN TRANSACTION"); - const fileStmt = db.prepare( - "INSERT INTO files (path, language, mtime, indexed_at) VALUES (?, ?, ?, ?)" - ); + const fileStmt = db.prepare( + "INSERT INTO files (path, language, mtime, indexed_at) VALUES (?, ?, ?, ?)", + ); - const symbolStmt = db.prepare(` + const symbolStmt = db.prepare(` INSERT INTO symbols ( file_id, name, scip_symbol, kind, start_line, end_line, start_char, end_char, @@ -744,341 +764,346 @@ async function convertFiles( ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); - let processedCount = 0; - const logInterval = Math.max(1, Math.floor(changedFiles.length / 10)); // Log every 10% - - for (const { path: filePath, mtime } of changedFiles) { - processedCount++; - - if ( - processedCount % logInterval === 0 || - processedCount === changedFiles.length - ) { - debugConverter( - `Converting files: ${processedCount}/${ - changedFiles.length - } (${Math.floor((processedCount / changedFiles.length) * 100)}%)` - ); - } - - if (insertedFiles.has(filePath)) { - continue; - } - - // Get document from parsed SCIP data - const doc = documentsByPath.get(filePath); - if (!doc) continue; - - // Insert file - fileStmt.run(filePath, doc.language, mtime, timestamp); - - insertedFiles.add(filePath); - - // Get file_id from database - const fileRecord = db - .query("SELECT id FROM files WHERE path = ?") - .get(filePath) as { id: number } | undefined; - - if (!fileRecord) continue; - - const fileId = fileRecord.id; - - // Extract symbol definitions from occurrences - const definitions = extractDefinitions(doc); - - // Insert symbols (batch) - for (const def of definitions) { - const symbolInfo = symbolsById.get(def.symbol); - - // Get symbol metadata - let kind = symbolKindToString(symbolInfo?.kind ?? 0); - - // Fallback: If kind is unknown, try to extract from documentation - if (kind === "unknown" && symbolInfo?.documentation) { - kind = extractKindFromDocumentation(symbolInfo.documentation); - } - - const pkg = extractPackageFromScip(def.symbol); - const name = symbolInfo?.displayName || extractNameFromScip(def.symbol); - const documentation = symbolInfo?.documentation?.join("\n"); - - // Detect if symbol is local (function parameters, closure variables, etc.) - const isLocal = def.symbol.includes("local") ? 1 : 0; - - symbolStmt.run( - fileId, - name, - def.symbol, - kind, - def.range[0], // start_line - def.range[2], // end_line - def.range[1], // start_char - def.range[3], // end_char - documentation || null, - pkg, - isLocal - ); - } - } - - debugConverter(`Committing transaction for ${changedFiles.length} files...`); - db.run("COMMIT"); - debugConverter("Transaction committed successfully"); + let processedCount = 0; + const logInterval = Math.max(1, Math.floor(changedFiles.length / 10)); // Log every 10% + + for (const { path: filePath, mtime } of changedFiles) { + processedCount++; + + if ( + processedCount % logInterval === 0 || + processedCount === changedFiles.length + ) { + debugConverter( + `Converting files: ${processedCount}/${ + changedFiles.length + } (${Math.floor((processedCount / changedFiles.length) * 100)}%)`, + ); + } + + if (insertedFiles.has(filePath)) { + continue; + } + + // Get document from parsed SCIP data + const doc = documentsByPath.get(filePath); + if (!doc) continue; + + // Insert file + fileStmt.run(filePath, doc.language, mtime, timestamp); + + insertedFiles.add(filePath); + + // Get file_id from database + const fileRecord = db + .query("SELECT id FROM files WHERE path = ?") + .get(filePath) as { id: number } | undefined; + + if (!fileRecord) continue; + + const fileId = fileRecord.id; + + // Extract symbol definitions from occurrences + const definitions = extractDefinitions(doc); + + // Insert symbols (batch) + for (const def of definitions) { + const symbolInfo = symbolsById.get(def.symbol); + + // Get symbol metadata + let kind = symbolKindToString(symbolInfo?.kind ?? 0); + + // Fallback: If kind is unknown, try to extract from documentation + if (kind === "unknown" && symbolInfo?.documentation) { + kind = extractKindFromDocumentation(symbolInfo.documentation); + } + + const pkg = extractPackageFromScip(def.symbol); + const name = symbolInfo?.displayName || extractNameFromScip(def.symbol); + const documentation = symbolInfo?.documentation?.join("\n"); + + // Detect if symbol is local (function parameters, closure variables, etc.) + const isLocal = def.symbol.includes("local") ? 1 : 0; + + symbolStmt.run( + fileId, + name, + def.symbol, + kind, + def.range[0], // start_line + def.range[2], // end_line + def.range[1], // start_char + def.range[3], // end_char + documentation || null, + pkg, + isLocal, + ); + } + } + + debugConverter(`Committing transaction for ${changedFiles.length} files...`); + db.run("COMMIT"); + debugConverter("Transaction committed successfully"); } /** * Update dependencies for changed files */ async function updateDependencies( - documentsByPath: Map, - symbolsById: Map, - definitionsBySymbol: Map, - db: Database, - changedFiles: ChangedFile[] + documentsByPath: Map, + symbolsById: Map, + definitionsBySymbol: Map, + db: Database, + changedFiles: ChangedFile[], ): Promise { - if (changedFiles.length === 0) return; - - const changedPaths = changedFiles.map((f) => f.path); - debugConverter( - `Finding affected files for ${changedPaths.length} changed files...` - ); - - // Get affected files (changed + their dependents) - const affectedFiles = new Set(changedPaths); - - // Find files that import changed files - if (changedPaths.length > 0) { - const placeholders = changedPaths.map(() => "?").join(","); - const dependents = db - .query( - ` + if (changedFiles.length === 0) return; + + const changedPaths = changedFiles.map((f) => f.path); + debugConverter( + `Finding affected files for ${changedPaths.length} changed files...`, + ); + + // Get affected files (changed + their dependents) + const affectedFiles = new Set(changedPaths); + + // Find files that import changed files + if (changedPaths.length > 0) { + const placeholders = changedPaths.map(() => "?").join(","); + const dependents = db + .query( + ` SELECT DISTINCT f.path FROM dependencies d JOIN files f ON f.id = d.from_file_id JOIN files f2 ON f2.id = d.to_file_id WHERE f2.path IN (${placeholders}) - ` - ) - .all(...changedPaths) as Array<{ path: string }>; - - for (const { path } of dependents) { - affectedFiles.add(path); - } - debugConverter( - `Found ${dependents.length} dependent files, total affected: ${affectedFiles.size}` - ); - } - - // Delete old dependencies for affected files - debugConverter("Starting transaction for dependencies update..."); - db.run("BEGIN TRANSACTION"); - - const deleteStmt = db.prepare(` + `, + ) + .all(...changedPaths) as Array<{ path: string }>; + + for (const { path } of dependents) { + affectedFiles.add(path); + } + debugConverter( + `Found ${dependents.length} dependent files, total affected: ${affectedFiles.size}`, + ); + } + + // Delete old dependencies for affected files + debugConverter("Starting transaction for dependencies update..."); + db.run("BEGIN TRANSACTION"); + + const deleteStmt = db.prepare(` DELETE FROM dependencies WHERE from_file_id IN (SELECT id FROM files WHERE path = ?) `); - for (const filePath of affectedFiles) { - deleteStmt.run(filePath); - } - debugConverter(`Deleted old dependencies for ${affectedFiles.size} files`); + for (const filePath of affectedFiles) { + deleteStmt.run(filePath); + } + debugConverter(`Deleted old dependencies for ${affectedFiles.size} files`); - // Recompute dependencies from SCIP data - debugConverter(`Recomputing dependencies for ${affectedFiles.size} files...`); - const insertStmt = db.prepare(` + // Recompute dependencies from SCIP data + debugConverter(`Recomputing dependencies for ${affectedFiles.size} files...`); + const insertStmt = db.prepare(` INSERT INTO dependencies (from_file_id, to_file_id, symbol_count, symbols) VALUES (?, ?, ?, ?) `); - let processedCount = 0; - const logInterval = Math.max(1, Math.floor(affectedFiles.size / 10)); - - for (const fromPath of affectedFiles) { - processedCount++; - - if ( - processedCount % logInterval === 0 || - processedCount === affectedFiles.size - ) { - debugConverter( - `Processing dependencies: ${processedCount}/${ - affectedFiles.size - } (${Math.floor((processedCount / affectedFiles.size) * 100)}%)` - ); - } - const doc = documentsByPath.get(fromPath); - if (!doc) continue; - - // Get file dependencies - const depsByFile = getFileDependencies( - doc, - documentsByPath, - symbolsById, - definitionsBySymbol - ); - - const fromFileRecord = db - .query("SELECT id FROM files WHERE path = ?") - .get(fromPath) as { id: number } | undefined; - - if (!fromFileRecord) continue; - - const fromFileId = fromFileRecord.id; - - for (const [toPath, symbols] of depsByFile) { - const toFileRecord = db - .query("SELECT id FROM files WHERE path = ?") - .get(toPath) as { id: number } | undefined; - - if (!toFileRecord) continue; - - // Extract symbol names - const symbolNames = Array.from( - new Set( - Array.from(symbols) - .filter((scipSymbol) => !scipSymbol.includes("local")) // Filter out local symbols - .map((scipSymbol) => { - const symbolInfo = symbolsById.get(scipSymbol); - return symbolInfo?.displayName || extractNameFromScip(scipSymbol); - }) - .filter((name) => name && name !== "unknown") - ) - ); - - if (symbolNames.length === 0) continue; - - insertStmt.run( - fromFileId, - toFileRecord.id, - symbolNames.length, - JSON.stringify(symbolNames) - ); - } - } - - debugConverter("Committing dependencies transaction..."); - db.run("COMMIT"); - debugConverter("Dependencies transaction committed"); + let processedCount = 0; + const logInterval = Math.max(1, Math.floor(affectedFiles.size / 10)); + + for (const fromPath of affectedFiles) { + processedCount++; + + if ( + processedCount % logInterval === 0 || + processedCount === affectedFiles.size + ) { + debugConverter( + `Processing dependencies: ${processedCount}/${ + affectedFiles.size + } (${Math.floor((processedCount / affectedFiles.size) * 100)}%)`, + ); + } + const doc = documentsByPath.get(fromPath); + if (!doc) continue; + + // Get file dependencies + const depsByFile = getFileDependencies({ + doc, + documentsByPath, + symbolsById, + definitionsBySymbol, + }); + + const fromFileRecord = db + .query("SELECT id FROM files WHERE path = ?") + .get(fromPath) as { id: number } | undefined; + + if (!fromFileRecord) continue; + + const fromFileId = fromFileRecord.id; + + for (const [toPath, symbols] of depsByFile) { + const toFileRecord = db + .query("SELECT id FROM files WHERE path = ?") + .get(toPath) as { id: number } | undefined; + + if (!toFileRecord) continue; + + // Extract symbol names + const symbolNames = Array.from( + new Set( + Array.from(symbols) + .filter((scipSymbol) => !scipSymbol.includes("local")) // Filter out local symbols + .map((scipSymbol) => { + const symbolInfo = symbolsById.get(scipSymbol); + return symbolInfo?.displayName || extractNameFromScip(scipSymbol); + }) + .filter((name) => name && name !== "unknown"), + ), + ); + + if (symbolNames.length === 0) continue; + + insertStmt.run( + fromFileId, + toFileRecord.id, + symbolNames.length, + JSON.stringify(symbolNames), + ); + } + } + + debugConverter("Committing dependencies transaction..."); + db.run("COMMIT"); + debugConverter("Dependencies transaction committed"); } /** * Update symbol references for changed files */ -async function updateSymbolReferences( - documentsByPath: Map, - symbolsById: Map, - db: Database, - changedFiles: ChangedFile[] -): Promise { - if (changedFiles.length === 0) return; - - const affectedFiles = changedFiles.map((f) => f.path); - debugConverter( - `Updating symbol references for ${affectedFiles.length} files...` - ); - - db.run("BEGIN TRANSACTION"); - - // Delete old references from changed files - const deleteStmt = db.prepare(` +async function updateSymbolReferences({ + documentsByPath, + symbolsById, + db, + changedFiles, +}: { + documentsByPath: Map; + symbolsById: Map; + db: Database; + changedFiles: ChangedFile[]; +}): Promise { + if (changedFiles.length === 0) return; + + const affectedFiles = changedFiles.map((f) => f.path); + debugConverter( + `Updating symbol references for ${affectedFiles.length} files...`, + ); + + db.run("BEGIN TRANSACTION"); + + // Delete old references from changed files + const deleteStmt = db.prepare(` DELETE FROM symbol_references WHERE file_id IN (SELECT id FROM files WHERE path = ?) `); - for (const filePath of affectedFiles) { - deleteStmt.run(filePath); - } - debugConverter(`Deleted old references for ${affectedFiles.length} files`); - - // Build symbol lookup map (scip_symbol -> id) for fast lookups - debugConverter("Building symbol ID lookup map..."); - const symbolIdMap = new Map(); - const allSymbols = db - .query("SELECT id, scip_symbol FROM symbols") - .all() as Array<{ - id: number; - scip_symbol: string; - }>; - for (const sym of allSymbols) { - symbolIdMap.set(sym.scip_symbol, sym.id); - } - debugConverter(`Symbol lookup map built: ${symbolIdMap.size} symbols`); - - // Build file ID lookup map for fast lookups - debugConverter("Building file ID lookup map..."); - const fileIdMap = new Map(); - for (const filePath of affectedFiles) { - const fileRecord = db - .query("SELECT id FROM files WHERE path = ?") - .get(filePath) as { id: number } | undefined; - if (fileRecord) { - fileIdMap.set(filePath, fileRecord.id); - } - } - debugConverter(`File lookup map built: ${fileIdMap.size} files`); - - // Insert new references from changed files - const insertStmt = db.prepare(` + for (const filePath of affectedFiles) { + deleteStmt.run(filePath); + } + debugConverter(`Deleted old references for ${affectedFiles.length} files`); + + // Build symbol lookup map (scip_symbol -> id) for fast lookups + debugConverter("Building symbol ID lookup map..."); + const symbolIdMap = new Map(); + const allSymbols = db + .query("SELECT id, scip_symbol FROM symbols") + .all() as Array<{ + id: number; + scip_symbol: string; + }>; + for (const sym of allSymbols) { + symbolIdMap.set(sym.scip_symbol, sym.id); + } + debugConverter(`Symbol lookup map built: ${symbolIdMap.size} symbols`); + + // Build file ID lookup map for fast lookups + debugConverter("Building file ID lookup map..."); + const fileIdMap = new Map(); + for (const filePath of affectedFiles) { + const fileRecord = db + .query("SELECT id FROM files WHERE path = ?") + .get(filePath) as { id: number } | undefined; + if (fileRecord) { + fileIdMap.set(filePath, fileRecord.id); + } + } + debugConverter(`File lookup map built: ${fileIdMap.size} files`); + + // Insert new references from changed files + const insertStmt = db.prepare(` INSERT INTO symbol_references (symbol_id, file_id, line) VALUES (?, ?, ?) `); - let processedCount = 0; - let totalReferences = 0; - const logInterval = Math.max(1, Math.floor(affectedFiles.length / 10)); - - for (const fromPath of affectedFiles) { - processedCount++; - - if ( - processedCount % logInterval === 0 || - processedCount === affectedFiles.length - ) { - debugConverter( - `Processing references: ${processedCount}/${ - affectedFiles.length - } (${Math.floor( - (processedCount / affectedFiles.length) * 100 - )}%) - ${totalReferences} refs inserted` - ); - } - - const doc = documentsByPath.get(fromPath); - if (!doc) continue; - - const fromFileId = fileIdMap.get(fromPath); - if (!fromFileId) continue; - - // Extract references from occurrences - const references = extractReferences(doc); - - // For each reference, look up symbol ID from map - for (const ref of references) { - // Skip local symbols - if (ref.symbol.includes("local")) continue; - - const symbolId = symbolIdMap.get(ref.symbol); - if (!symbolId) continue; - - insertStmt.run(symbolId, fromFileId, ref.line); - totalReferences++; - } - } - - debugConverter(`Total references inserted: ${totalReferences}`); - - debugConverter("Committing symbol references transaction..."); - db.run("COMMIT"); - debugConverter("Symbol references transaction committed"); + let processedCount = 0; + let totalReferences = 0; + const logInterval = Math.max(1, Math.floor(affectedFiles.length / 10)); + + for (const fromPath of affectedFiles) { + processedCount++; + + if ( + processedCount % logInterval === 0 || + processedCount === affectedFiles.length + ) { + debugConverter( + `Processing references: ${processedCount}/${ + affectedFiles.length + } (${Math.floor( + (processedCount / affectedFiles.length) * 100, + )}%) - ${totalReferences} refs inserted`, + ); + } + + const doc = documentsByPath.get(fromPath); + if (!doc) continue; + + const fromFileId = fileIdMap.get(fromPath); + if (!fromFileId) continue; + + // Extract references from occurrences + const references = extractReferences(doc); + + // For each reference, look up symbol ID from map + for (const ref of references) { + // Skip local symbols + if (ref.symbol.includes("local")) continue; + + const symbolId = symbolIdMap.get(ref.symbol); + if (!symbolId) continue; + + insertStmt.run(symbolId, fromFileId, ref.line); + totalReferences++; + } + } + + debugConverter(`Total references inserted: ${totalReferences}`); + + debugConverter("Committing symbol references transaction..."); + db.run("COMMIT"); + debugConverter("Symbol references transaction committed"); } /** * Update denormalized fields for performance */ function updateDenormalizedFields(db: Database): void { - // Update file symbol counts - debugConverter("Computing file symbol counts..."); - db.run(` + // Update file symbol counts + debugConverter("Computing file symbol counts..."); + db.run(` UPDATE files SET symbol_count = ( SELECT COUNT(*) @@ -1086,11 +1111,11 @@ function updateDenormalizedFields(db: Database): void { WHERE s.file_id = files.id ) `); - debugConverter("File symbol counts updated"); + debugConverter("File symbol counts updated"); - // Update symbol reference counts - debugConverter("Computing symbol reference counts..."); - db.run(` + // Update symbol reference counts + debugConverter("Computing symbol reference counts..."); + db.run(` UPDATE symbols SET reference_count = ( SELECT COUNT(*) @@ -1098,11 +1123,11 @@ function updateDenormalizedFields(db: Database): void { WHERE sr.symbol_id = symbols.id ) `); - debugConverter("Symbol reference counts updated"); + debugConverter("Symbol reference counts updated"); - // Update file dependency counts (outgoing dependencies) - debugConverter("Computing file dependency counts..."); - db.run(` + // Update file dependency counts (outgoing dependencies) + debugConverter("Computing file dependency counts..."); + db.run(` UPDATE files SET dependency_count = ( SELECT COUNT(DISTINCT to_file_id) @@ -1110,11 +1135,11 @@ function updateDenormalizedFields(db: Database): void { WHERE d.from_file_id = files.id ) `); - debugConverter("File dependency counts updated"); + debugConverter("File dependency counts updated"); - // Update file dependent counts (incoming dependencies / fan-in) - debugConverter("Computing file dependent counts..."); - db.run(` + // Update file dependent counts (incoming dependencies / fan-in) + debugConverter("Computing file dependent counts..."); + db.run(` UPDATE files SET dependent_count = ( SELECT COUNT(DISTINCT from_file_id) @@ -1122,37 +1147,43 @@ function updateDenormalizedFields(db: Database): void { WHERE d.to_file_id = files.id ) `); - debugConverter("File dependent counts updated"); + debugConverter("File dependent counts updated"); } /** * Update packages table */ -function updatePackages(db: Database, skipIfNoChanges: boolean = false): void { - if (skipIfNoChanges) { - // Check if packages table needs update - const packageCount = ( - db.query("SELECT COUNT(*) as c FROM packages").get() as { - c: number; - } - ).c; - const symbolPackageCount = ( - db - .query( - "SELECT COUNT(DISTINCT package) as c FROM symbols WHERE package IS NOT NULL" - ) - .get() as { c: number } - ).c; - - // Skip if counts match (no new packages) - if (packageCount === symbolPackageCount) { - return; - } - } - - db.run("DELETE FROM packages"); - - db.run(` +function updatePackages({ + db, + skipIfNoChanges = false, +}: { + db: Database; + skipIfNoChanges?: boolean; +}): void { + if (skipIfNoChanges) { + // Check if packages table needs update + const packageCount = ( + db.query("SELECT COUNT(*) as c FROM packages").get() as { + c: number; + } + ).c; + const symbolPackageCount = ( + db + .query( + "SELECT COUNT(DISTINCT package) as c FROM symbols WHERE package IS NOT NULL", + ) + .get() as { c: number } + ).c; + + // Skip if counts match (no new packages) + if (packageCount === symbolPackageCount) { + return; + } + } + + db.run("DELETE FROM packages"); + + db.run(` INSERT INTO packages (name, manager, symbol_count) SELECT package, @@ -1167,39 +1198,44 @@ function updatePackages(db: Database, skipIfNoChanges: boolean = false): void { /** * Update metadata table */ -function updateMetadata( - db: Database, - mode: string, - changedFiles: number, - deletedFiles: number -): ConversionStats { - const totalFiles = ( - db.query("SELECT COUNT(*) as c FROM files").get() as { c: number } - ).c; - const totalSymbols = ( - db.query("SELECT COUNT(*) as c FROM symbols").get() as { c: number } - ).c; - - const metadata = { - last_indexed: new Date().toISOString(), - total_files: totalFiles.toString(), - total_symbols: totalSymbols.toString(), - }; - - for (const [key, value] of Object.entries(metadata)) { - db.run( - "INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)", - key, - value - ); - } - - return { - mode: mode as "full" | "incremental", - total_files: totalFiles, - total_symbols: totalSymbols, - changed_files: changedFiles, - deleted_files: deletedFiles, - time_ms: 0, // Will be set by caller - }; +function updateMetadata({ + db, + mode, + changedFiles, + deletedFiles, +}: { + db: Database; + mode: string; + changedFiles: number; + deletedFiles: number; +}): ConversionStats { + const totalFiles = ( + db.query("SELECT COUNT(*) as c FROM files").get() as { c: number } + ).c; + const totalSymbols = ( + db.query("SELECT COUNT(*) as c FROM symbols").get() as { c: number } + ).c; + + const metadata = { + last_indexed: new Date().toISOString(), + total_files: totalFiles.toString(), + total_symbols: totalSymbols.toString(), + }; + + for (const [key, value] of Object.entries(metadata)) { + db.run( + "INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)", + key, + value, + ); + } + + return { + mode: mode as "full" | "incremental", + total_files: totalFiles, + total_symbols: totalSymbols, + changed_files: changedFiles, + deleted_files: deletedFiles, + time_ms: 0, // Will be set by caller + }; } diff --git a/src/converter/documents.ts b/src/converter/documents.ts index be60702..5a8b23b 100644 --- a/src/converter/documents.ts +++ b/src/converter/documents.ts @@ -37,22 +37,27 @@ interface DocumentReference { /** * Process documentation files and index them in the database */ -export async function processDocuments( - db: Database, - repoRoot: string, - mode: "full" | "incremental", - ignorePatterns: string[] = [], -): Promise { +export async function processDocuments({ + db, + repoRoot, + mode, + ignorePatterns = [], +}: { + db: Database; + repoRoot: string; + mode: "full" | "incremental"; + ignorePatterns?: string[]; +}): Promise { debugDocs("Starting document processing in %s mode", mode); const startTime = Date.now(); // Scan for document files - const scannedDocs = await scanDocumentFiles( + const scannedDocs = await scanDocumentFiles({ repoRoot, - [".md", ".txt"], + extensions: [".md", ".txt"], ignorePatterns, - ); + }); debugDocs("Scanned %d document files", scannedDocs.length); let docsToProcess: DocumentFile[]; @@ -70,7 +75,7 @@ export async function processDocuments( } // Filter to only changed documents - docsToProcess = filterChangedDocuments(existingDocs, scannedDocs); + docsToProcess = filterChangedDocuments({ existingDocs, scannedDocs }); debugDocs("Incremental mode: %d documents changed", docsToProcess.length); // Remove documents that no longer exist @@ -113,7 +118,7 @@ export async function processDocuments( const content = await Bun.file(fullPath).text(); // Extract references from content - const refs = extractReferences(content, db); + const refs = extractReferences({ content, db }); // Insert document const now = Date.now(); @@ -146,29 +151,29 @@ export async function processDocuments( docRow.id, ]); - batchInsert( + batchInsert({ db, - "document_symbol_refs", - ["document_id", "symbol_id", "line"], - refs.symbolRefs.map((r) => [docRow.id, r.symbolId, r.line]), - BATCH_SIZE, - ); + table: "document_symbol_refs", + columns: ["document_id", "symbol_id", "line"], + rows: refs.symbolRefs.map((r) => [docRow.id, r.symbolId, r.line]), + batchSize: BATCH_SIZE, + }); - batchInsert( + batchInsert({ db, - "document_file_refs", - ["document_id", "file_id", "line"], - refs.fileRefs.map((r) => [docRow.id, r.fileId, r.line]), - BATCH_SIZE, - ); + table: "document_file_refs", + columns: ["document_id", "file_id", "line"], + rows: refs.fileRefs.map((r) => [docRow.id, r.fileId, r.line]), + batchSize: BATCH_SIZE, + }); - batchInsert( + batchInsert({ db, - "document_document_refs", - ["document_id", "referenced_document_id", "line"], - refs.docRefs.map((r) => [docRow.id, r.docId, r.line]), - BATCH_SIZE, - ); + table: "document_document_refs", + columns: ["document_id", "referenced_document_id", "line"], + rows: refs.docRefs.map((r) => [docRow.id, r.docId, r.line]), + batchSize: BATCH_SIZE, + }); processed++; } catch (error) { @@ -194,7 +199,13 @@ export async function processDocuments( /** * Extract references to symbols and files from document content with line numbers */ -function extractReferences(content: string, db: Database): DocumentReference { +function extractReferences({ + content, + db, +}: { + content: string; + db: Database; +}): DocumentReference { const symbolRefs: SymbolReference[] = []; const fileRefs: FileReference[] = []; const docRefs: DocReference[] = []; @@ -208,12 +219,14 @@ function extractReferences(content: string, db: Database): DocumentReference { // Get eligible symbols (non-local, specific kinds only) // Order by name length DESC to match longer names first (e.g., "AuthService" before "Auth") const symbols = db - .query(` + .query( + ` SELECT id, name FROM symbols WHERE is_local = 0 AND kind IN ('class', 'function', 'interface', 'method', 'type', 'type_alias', 'enum') ORDER BY LENGTH(name) DESC - `) + `, + ) .all() as Array<{ id: number; name: string }>; // Get all indexed files @@ -243,8 +256,9 @@ function extractReferences(content: string, db: Database): DocumentReference { let match; while ((match = regex.exec(allContent)) !== null) { - const lineNumber = - allContent.substring(0, match.index).split("\n").length; + const lineNumber = allContent + .substring(0, match.index) + .split("\n").length; if (!symbolRefsMap.has(sym.id)) { symbolRefsMap.set(sym.id, new Set()); @@ -372,13 +386,19 @@ function normalizePath(path: string): string { /** * Insert rows in batches for better performance */ -function batchInsert( - db: Database, - table: string, - columns: string[], - rows: Array>, - batchSize: number, -): void { +function batchInsert({ + db, + table, + columns, + rows, + batchSize, +}: { + db: Database; + table: string; + columns: string[]; + rows: Array>; + batchSize: number; +}): void { if (rows.length === 0) { return; } diff --git a/src/converter/scip-parser.ts b/src/converter/scip-parser.ts index df999b9..6ae96ca 100644 --- a/src/converter/scip-parser.ts +++ b/src/converter/scip-parser.ts @@ -6,7 +6,6 @@ import { fromBinary } from "@bufbuild/protobuf"; import { type Document, - Index, IndexSchema, type Occurrence, type SymbolInformation, @@ -256,11 +255,13 @@ export function extractReferences(doc: ParsedDocument): SymbolReference[] { * @param symbols Map of symbol -> ParsedSymbol (includes external symbols) * @returns The relative path of the file where the symbol is defined, or null if not found */ -export function findDefinitionFile( - symbol: string, - documents: Map, - symbols: Map, -): string | null { +export function findDefinitionFile({ + symbol, + documents, +}: { + symbol: string; + documents: Map; +}): string | null { // First, check if it's a local symbol (contains 'local') if (symbol.includes("local")) { // Local symbols are defined in the same file they're used in @@ -353,10 +354,13 @@ export function buildLookupMaps(scipData: ScipData): { /** * Get all symbols defined in a document */ -export function getDocumentSymbols( - doc: ParsedDocument, - symbolsById: Map, -): Array<{ +export function getDocumentSymbols({ + doc, + symbolsById, +}: { + doc: ParsedDocument; + symbolsById: Map; +}): Array<{ symbol: string; name?: string; kind: number; @@ -381,15 +385,16 @@ export function getDocumentSymbols( * Get file dependencies for a document * Returns a map of file path -> set of symbols used from that file */ -export function getFileDependencies( - doc: ParsedDocument, - documentsByPath: Map, - symbolsById: Map, +export function getFileDependencies({ + doc, + definitionsBySymbol, +}: { + doc: ParsedDocument; definitionsBySymbol: Map< string, { file: string; definition: SymbolDefinition } - >, -): Map> { + >; +}): Map> { const references = extractReferences(doc); const depsByFile = new Map>(); diff --git a/src/db/connection.ts b/src/db/connection.ts index 087df96..a210165 100644 --- a/src/db/connection.ts +++ b/src/db/connection.ts @@ -14,7 +14,10 @@ let currentDbPath: string | null = null; * Always uses the dora database */ export function getDb(config: Config): Database { - const dbPath = resolveAbsolute(config.root, config.db); + const dbPath = resolveAbsolute({ + root: config.root, + relativePath: config.db, + }); // Check if database exists if (!existsSync(dbPath)) { @@ -40,7 +43,9 @@ export function getDb(config: Config): Database { return dbInstance; } catch (error) { throw new CtxError( - `Failed to open database: ${error instanceof Error ? error.message : String(error)}`, + `Failed to open database: ${ + error instanceof Error ? error.message : String(error) + }`, ); } } diff --git a/src/db/queries.ts b/src/db/queries.ts index 462e2c4..fde8dbc 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -2,47 +2,47 @@ import type { Database } from "bun:sqlite"; import type { - ComplexityMetric, - CoupledFiles, - Cycle, - DependencyNode, - Document, - DocumentDocRef, - DocumentFileRef, - DocumentSymbolRef, - ExportedSymbol, - FileDependency, - FileDependent, - FileSymbol, - Hotspot, - ImportedFile, - SymbolResult, - UnusedSymbol, + ComplexityMetric, + CoupledFiles, + Cycle, + DependencyNode, + Document, + DocumentDocRef, + DocumentFileRef, + DocumentSymbolRef, + ExportedSymbol, + FileDependency, + FileDependent, + FileSymbol, + Hotspot, + ImportedFile, + SymbolResult, + UnusedSymbol, } from "../types.ts"; export function getFileCount(db: Database): number { - const result = db.query("SELECT COUNT(*) as count FROM files").get() as { - count: number; - }; - return result.count; + const result = db.query("SELECT COUNT(*) as count FROM files").get() as { + count: number; + }; + return result.count; } export function getSymbolCount(db: Database): number { - const result = db.query("SELECT COUNT(*) as count FROM symbols").get() as { - count: number; - }; - return result.count; + const result = db.query("SELECT COUNT(*) as count FROM symbols").get() as { + count: number; + }; + return result.count; } export function getPackages(db: Database): string[] { - const query = ` + const query = ` SELECT name FROM packages ORDER BY name `; - const results = db.query(query).all() as { name: string }[]; - return results.map((r) => r.name); + const results = db.query(query).all() as { name: string }[]; + return results.map((r) => r.name); } /** @@ -50,10 +50,10 @@ export function getPackages(db: Database): string[] { * Leaf nodes are files that few things depend on, but which have their own dependencies */ export function getLeafNodes( - db: Database, - maxDependents: number = 0 + db: Database, + maxDependents: number = 0, ): string[] { - const query = ` + const query = ` SELECT f.path, COUNT(DISTINCT d_out.to_file_id) as dependencies_count, @@ -78,15 +78,15 @@ export function getLeafNodes( LIMIT 100 `; - const results = db.query(query).all(maxDependents) as { path: string }[]; - return results.map((r) => r.path); + const results = db.query(query).all(maxDependents) as { path: string }[]; + return results.map((r) => r.path); } export function getFileSymbols( - db: Database, - relativePath: string + db: Database, + relativePath: string, ): FileSymbol[] { - const query = ` + const query = ` SELECT s.name, s.kind, @@ -98,25 +98,25 @@ export function getFileSymbols( ORDER BY s.start_line `; - const results = db.query(query).all(relativePath) as { - name: string; - kind: string; - start_line: number; - end_line: number; - }[]; - - return results.map((r) => ({ - name: r.name, - kind: r.kind, - lines: [r.start_line, r.end_line] as [number, number], - })); + const results = db.query(query).all(relativePath) as { + name: string; + kind: string; + start_line: number; + end_line: number; + }[]; + + return results.map((r) => ({ + name: r.name, + kind: r.kind, + lines: [r.start_line, r.end_line] as [number, number], + })); } export function getFileDependencies( - db: Database, - relativePath: string + db: Database, + relativePath: string, ): FileDependency[] { - const query = ` + const query = ` SELECT f.path as depends_on, d.symbols as symbols_used @@ -126,22 +126,22 @@ export function getFileDependencies( ORDER BY f.path `; - const results = db.query(query).all(relativePath) as { - depends_on: string; - symbols_used: string | null; - }[]; + const results = db.query(query).all(relativePath) as { + depends_on: string; + symbols_used: string | null; + }[]; - return results.map((r) => ({ - path: r.depends_on, - symbols: r.symbols_used ? JSON.parse(r.symbols_used) : undefined, - })); + return results.map((r) => ({ + path: r.depends_on, + symbols: r.symbols_used ? JSON.parse(r.symbols_used) : undefined, + })); } export function getFileDependents( - db: Database, - relativePath: string + db: Database, + relativePath: string, ): FileDependent[] { - const query = ` + const query = ` SELECT f.path as dependent, d.symbol_count as ref_count @@ -151,26 +151,26 @@ export function getFileDependents( ORDER BY d.symbol_count DESC `; - const results = db.query(query).all(relativePath) as { - dependent: string; - ref_count: number; - }[]; + const results = db.query(query).all(relativePath) as { + dependent: string; + ref_count: number; + }[]; - return results.map((r) => ({ - path: r.dependent, - refs: r.ref_count, - })); + return results.map((r) => ({ + path: r.dependent, + refs: r.ref_count, + })); } export function searchSymbols( - db: Database, - searchQuery: string, - options: { kind?: string; limit?: number } = {} + db: Database, + searchQuery: string, + options: { kind?: string; limit?: number } = {}, ): SymbolResult[] { - const limit = options.limit || 20; - const kindFilter = options.kind !== undefined ? "AND s.kind = ?" : ""; + const limit = options.limit || 20; + const kindFilter = options.kind !== undefined ? "AND s.kind = ?" : ""; - const query = ` + const query = ` SELECT s.name, s.kind, @@ -185,50 +185,50 @@ export function searchSymbols( LIMIT ? `; - const params = - options.kind !== undefined - ? [`%${searchQuery}%`, options.kind, limit] - : [`%${searchQuery}%`, limit]; - - const results = db.query(query).all(...params) as { - name: string; - kind: string; - path: string; - start_line: number; - end_line: number; - }[]; - - return results.map((r) => ({ - name: r.name, - kind: r.kind, - path: r.path, - lines: [r.start_line, r.end_line] as [number, number], - })); + const params = + options.kind !== undefined + ? [`%${searchQuery}%`, options.kind, limit] + : [`%${searchQuery}%`, limit]; + + const results = db.query(query).all(...params) as { + name: string; + kind: string; + path: string; + start_line: number; + end_line: number; + }[]; + + return results.map((r) => ({ + name: r.name, + kind: r.kind, + path: r.path, + lines: [r.start_line, r.end_line] as [number, number], + })); } export interface SymbolReferencesResult { - symbol_id: number; - name: string; - kind: string; - definition: { - path: string; - line: number; - }; - references: string[]; + symbol_id: number; + name: string; + kind: string; + definition: { + path: string; + line: number; + }; + references: string[]; } /** * Get all references to a symbol by name */ export function getSymbolReferences( - db: Database, - symbolName: string, - options: { kind?: string; limit?: number } = {} + db: Database, + symbolName: string, + options: { kind?: string; limit?: number } = {}, ): SymbolReferencesResult[] { - const limit = options.limit || 100; - const kindFilter = options.kind !== undefined ? "AND s.kind = ?" : ""; + const limit = options.limit || 100; + const kindFilter = options.kind !== undefined ? "AND s.kind = ?" : ""; - const query = ` + const query = ` SELECT s.id, s.name, @@ -250,39 +250,39 @@ export function getSymbolReferences( LIMIT ? `; - const params = - options.kind !== undefined - ? [`%${symbolName}%`, options.kind, limit] - : [`%${symbolName}%`, limit]; - - const rows = db.query(query).all(...params) as { - id: number; - name: string; - kind: string; - def_path: string; - def_line: number; - reference_count: number; - ref_paths: string | null; - }[]; - - return rows.map((row) => ({ - symbol_id: row.id, - name: row.name, - kind: row.kind, - definition: { - path: row.def_path, - line: row.def_line, - }, - references: row.ref_paths ? row.ref_paths.split(",") : [], - })); + const params = + options.kind !== undefined + ? [`%${symbolName}%`, options.kind, limit] + : [`%${symbolName}%`, limit]; + + const rows = db.query(query).all(...params) as { + id: number; + name: string; + kind: string; + def_path: string; + def_line: number; + reference_count: number; + ref_paths: string | null; + }[]; + + return rows.map((row) => ({ + symbol_id: row.id, + name: row.name, + kind: row.kind, + definition: { + path: row.def_path, + line: row.def_line, + }, + references: row.ref_paths ? row.ref_paths.split(",") : [], + })); } export function getDependencies( - db: Database, - relativePath: string, - depth: number + db: Database, + relativePath: string, + depth: number, ): DependencyNode[] { - const query = ` + const query = ` WITH RECURSIVE dep_tree AS ( -- Base case: start with the target file SELECT id, path, 0 as depth @@ -305,23 +305,23 @@ export function getDependencies( ORDER BY depth, path `; - const results = db.query(query).all(relativePath, depth) as { - path: string; - depth: number; - }[]; + const results = db.query(query).all(relativePath, depth) as { + path: string; + depth: number; + }[]; - return results.map((r) => ({ - path: r.path, - depth: r.depth, - })); + return results.map((r) => ({ + path: r.path, + depth: r.depth, + })); } export function getReverseDependencies( - db: Database, - relativePath: string, - depth: number + db: Database, + relativePath: string, + depth: number, ): DependencyNode[] { - const query = ` + const query = ` WITH RECURSIVE rdep_tree AS ( -- Base case: start with the target file SELECT id, path, 0 as depth @@ -344,42 +344,42 @@ export function getReverseDependencies( ORDER BY depth, path `; - const results = db.query(query).all(relativePath, depth) as { - path: string; - depth: number; - }[]; + const results = db.query(query).all(relativePath, depth) as { + path: string; + depth: number; + }[]; - return results.map((r) => ({ - path: r.path, - depth: r.depth, - })); + return results.map((r) => ({ + path: r.path, + depth: r.depth, + })); } /** * Check if a file exists in the database */ export function fileExists(db: Database, relativePath: string): boolean { - const query = "SELECT 1 FROM files WHERE path = ? LIMIT 1"; - const result = db.query(query).get(relativePath); - return result !== null; + const query = "SELECT 1 FROM files WHERE path = ? LIMIT 1"; + const result = db.query(query).get(relativePath); + return result !== null; } /** * Get files in a directory (or matching a path prefix) */ export function getFilesInDirectory( - db: Database, - directoryPath: string, - options: { limit?: number; exactMatch?: boolean } = {} + db: Database, + directoryPath: string, + options: { limit?: number; exactMatch?: boolean } = {}, ): string[] { - const limit = options.limit || 50; + const limit = options.limit || 50; - // For exact directory matching, look for files like "dir/%" but not "dir-other/%" - const pattern = options.exactMatch - ? `${directoryPath}/%` - : `${directoryPath}%`; + // For exact directory matching, look for files like "dir/%" but not "dir-other/%" + const pattern = options.exactMatch + ? `${directoryPath}/%` + : `${directoryPath}%`; - const query = ` + const query = ` SELECT path FROM files WHERE path LIKE ? @@ -387,8 +387,8 @@ export function getFilesInDirectory( LIMIT ? `; - const results = db.query(query).all(pattern, limit) as { path: string }[]; - return results.map((r) => r.path); + const results = db.query(query).all(pattern, limit) as { path: string }[]; + return results.map((r) => r.path); } /** @@ -397,41 +397,41 @@ export function getFilesInDirectory( * Returns the path if found, or null if not found. */ export function findIndexFile( - db: Database, - directoryPath: string + db: Database, + directoryPath: string, ): string | null { - // Common index file patterns (order matters - prefer TypeScript) - const indexPatterns = [ - `${directoryPath}/index.ts`, - `${directoryPath}/index.tsx`, - `${directoryPath}/index.js`, - `${directoryPath}/index.jsx`, - `${directoryPath}/index.mts`, - `${directoryPath}/index.mjs`, - `${directoryPath}/index.cjs`, - `${directoryPath}/index.py`, - `${directoryPath}/index.go`, - `${directoryPath}/mod.rs`, // Rust uses mod.rs - `${directoryPath}/lib.rs`, // Rust lib crates - ]; - - // Try each pattern in order - for (const pattern of indexPatterns) { - const query = "SELECT path FROM files WHERE path = ? LIMIT 1"; - const result = db.query(query).get(pattern) as { path: string } | null; - if (result) { - return result.path; - } - } - - return null; + // Common index file patterns (order matters - prefer TypeScript) + const indexPatterns = [ + `${directoryPath}/index.ts`, + `${directoryPath}/index.tsx`, + `${directoryPath}/index.js`, + `${directoryPath}/index.jsx`, + `${directoryPath}/index.mts`, + `${directoryPath}/index.mjs`, + `${directoryPath}/index.cjs`, + `${directoryPath}/index.py`, + `${directoryPath}/index.go`, + `${directoryPath}/mod.rs`, // Rust uses mod.rs + `${directoryPath}/lib.rs`, // Rust lib crates + ]; + + // Try each pattern in order + for (const pattern of indexPatterns) { + const query = "SELECT path FROM files WHERE path = ? LIMIT 1"; + const result = db.query(query).get(pattern) as { path: string } | null; + if (result) { + return result.path; + } + } + + return null; } export function getFileExports( - db: Database, - relativePath: string + db: Database, + relativePath: string, ): ExportedSymbol[] { - const query = ` + const query = ` SELECT s.name, s.kind, @@ -446,25 +446,25 @@ export function getFileExports( ORDER BY start_line `; - const results = db.query(query).all(relativePath) as { - name: string; - kind: string; - start_line: number; - end_line: number; - }[]; - - return results.map((r) => ({ - name: r.name, - kind: r.kind, - lines: [r.start_line, r.end_line] as [number, number], - })); + const results = db.query(query).all(relativePath) as { + name: string; + kind: string; + start_line: number; + end_line: number; + }[]; + + return results.map((r) => ({ + name: r.name, + kind: r.kind, + lines: [r.start_line, r.end_line] as [number, number], + })); } export function getPackageExports( - db: Database, - packageName: string + db: Database, + packageName: string, ): ExportedSymbol[] { - const query = ` + const query = ` SELECT s.name, s.kind, @@ -480,27 +480,27 @@ export function getPackageExports( ORDER BY f.path, start_line `; - const results = db.query(query).all(packageName) as { - name: string; - kind: string; - file: string; - start_line: number; - end_line: number; - }[]; - - return results.map((r) => ({ - name: r.name, - kind: r.kind, - file: r.file, - lines: [r.start_line, r.end_line] as [number, number], - })); + const results = db.query(query).all(packageName) as { + name: string; + kind: string; + file: string; + start_line: number; + end_line: number; + }[]; + + return results.map((r) => ({ + name: r.name, + kind: r.kind, + file: r.file, + lines: [r.start_line, r.end_line] as [number, number], + })); } export function getFileImports( - db: Database, - relativePath: string + db: Database, + relativePath: string, ): ImportedFile[] { - const query = ` + const query = ` SELECT f.path as file, d.symbols @@ -510,19 +510,19 @@ export function getFileImports( ORDER BY f.path `; - const results = db.query(query).all(relativePath) as { - file: string; - symbols: string | null; - }[]; + const results = db.query(query).all(relativePath) as { + file: string; + symbols: string | null; + }[]; - return results.map((r) => ({ - file: r.file, - symbols: r.symbols ? JSON.parse(r.symbols) : [], - })); + return results.map((r) => ({ + file: r.file, + symbols: r.symbols ? JSON.parse(r.symbols) : [], + })); } export function getUnusedSymbols(db: Database, limit: number): UnusedSymbol[] { - const query = ` + const query = ` SELECT s.name, s.kind, @@ -539,24 +539,24 @@ export function getUnusedSymbols(db: Database, limit: number): UnusedSymbol[] { LIMIT ? `; - const results = db.query(query).all(limit) as { - name: string; - kind: string; - file: string; - start_line: number; - end_line: number; - }[]; - - return results.map((r) => ({ - name: r.name, - file: r.file, - lines: [r.start_line, r.end_line] as [number, number], - kind: r.kind, - })); + const results = db.query(query).all(limit) as { + name: string; + kind: string; + file: string; + start_line: number; + end_line: number; + }[]; + + return results.map((r) => ({ + name: r.name, + file: r.file, + lines: [r.start_line, r.end_line] as [number, number], + kind: r.kind, + })); } export function getMostReferencedFiles(db: Database, limit: number): Hotspot[] { - const query = ` + const query = ` SELECT f.path as file, f.dependent_count as referenced_by @@ -566,19 +566,19 @@ export function getMostReferencedFiles(db: Database, limit: number): Hotspot[] { LIMIT ? `; - const results = db.query(query).all(limit) as { - file: string; - referenced_by: number; - }[]; + const results = db.query(query).all(limit) as { + file: string; + referenced_by: number; + }[]; - return results.map((r) => ({ - file: r.file, - count: r.referenced_by, - })); + return results.map((r) => ({ + file: r.file, + count: r.referenced_by, + })); } export function getMostDependentFiles(db: Database, limit: number): Hotspot[] { - const query = ` + const query = ` SELECT f.path as file, f.dependency_count as depends_on @@ -588,19 +588,19 @@ export function getMostDependentFiles(db: Database, limit: number): Hotspot[] { LIMIT ? `; - const results = db.query(query).all(limit) as { - file: string; - depends_on: number; - }[]; + const results = db.query(query).all(limit) as { + file: string; + depends_on: number; + }[]; - return results.map((r) => ({ - file: r.file, - count: r.depends_on, - })); + return results.map((r) => ({ + file: r.file, + count: r.depends_on, + })); } export function getCycles(db: Database, limit: number = 50): Cycle[] { - const query = ` + const query = ` SELECT f1.path as path1, f2.path as path2 @@ -614,22 +614,22 @@ export function getCycles(db: Database, limit: number = 50): Cycle[] { LIMIT ? `; - const results = db.query(query).all(limit) as { - path1: string; - path2: string; - }[]; + const results = db.query(query).all(limit) as { + path1: string; + path2: string; + }[]; - return results.map((r) => ({ - files: [r.path1, r.path2, r.path1], - length: 2, - })); + return results.map((r) => ({ + files: [r.path1, r.path2, r.path1], + length: 2, + })); } export function getCoupledFiles( - db: Database, - threshold: number = 5 + db: Database, + threshold: number = 5, ): CoupledFiles[] { - const query = ` + const query = ` SELECT f1.path as file1, f2.path as file2, @@ -646,21 +646,21 @@ export function getCoupledFiles( ORDER BY total_coupling DESC `; - return db.query(query).all(threshold) as CoupledFiles[]; + return db.query(query).all(threshold) as CoupledFiles[]; } export function getComplexityMetrics( - db: Database, - sortBy: string = "complexity" + db: Database, + sortBy: string = "complexity", ): ComplexityMetric[] { - const orderByClause = - sortBy === "symbols" - ? "f.symbol_count DESC" - : sortBy === "stability" - ? "stability_ratio DESC" - : "complexity_score DESC"; - - const query = ` + const orderByClause = + sortBy === "symbols" + ? "f.symbol_count DESC" + : sortBy === "stability" + ? "stability_ratio DESC" + : "complexity_score DESC"; + + const query = ` SELECT f.path, f.symbol_count, @@ -673,17 +673,17 @@ export function getComplexityMetrics( LIMIT 20 `; - return db.query(query).all() as ComplexityMetric[]; + return db.query(query).all() as ComplexityMetric[]; } /** * Get documents referencing a symbol */ export function getDocumentsForSymbol( - db: Database, - symbolId: number + db: Database, + symbolId: number, ): Document[] { - const query = ` + const query = ` SELECT d.path, d.type FROM documents d JOIN document_symbol_refs dsr ON dsr.document_id = d.id @@ -691,14 +691,14 @@ export function getDocumentsForSymbol( ORDER BY d.path `; - return db.query(query).all(symbolId) as Document[]; + return db.query(query).all(symbolId) as Document[]; } /** * Get documents referencing a file */ export function getDocumentsForFile(db: Database, fileId: number): Document[] { - const query = ` + const query = ` SELECT d.path, d.type FROM documents d JOIN document_file_refs dfr ON dfr.document_id = d.id @@ -706,22 +706,22 @@ export function getDocumentsForFile(db: Database, fileId: number): Document[] { ORDER BY d.path `; - return db.query(query).all(fileId) as Document[]; + return db.query(query).all(fileId) as Document[]; } /** * Get symbols and files referenced by a document with line numbers */ export function getDocumentReferences( - db: Database, - docPath: string + db: Database, + docPath: string, ): { - symbols: DocumentSymbolRef[]; - files: DocumentFileRef[]; - documents: DocumentDocRef[]; + symbols: DocumentSymbolRef[]; + files: DocumentFileRef[]; + documents: DocumentDocRef[]; } { - // Get symbols with aggregated line numbers - const symbolQuery = ` + // Get symbols with aggregated line numbers + const symbolQuery = ` SELECT s.name, s.kind, @@ -737,8 +737,8 @@ export function getDocumentReferences( ORDER BY s.name `; - // Get files with aggregated line numbers - const fileQuery = ` + // Get files with aggregated line numbers + const fileQuery = ` SELECT f.path, GROUP_CONCAT(dfr.line) as lines @@ -750,8 +750,8 @@ export function getDocumentReferences( ORDER BY f.path `; - // Get documents with aggregated line numbers - const docQuery = ` + // Get documents with aggregated line numbers + const docQuery = ` SELECT d2.path, GROUP_CONCAT(ddr.line) as lines @@ -763,115 +763,115 @@ export function getDocumentReferences( ORDER BY d2.path `; - const symbolRows = db.query(symbolQuery).all(docPath) as Array<{ - name: string; - kind: string; - path: string; - start_line: number; - lines: string; - }>; - - const fileRows = db.query(fileQuery).all(docPath) as Array<{ - path: string; - lines: string; - }>; - - const docRows = db.query(docQuery).all(docPath) as Array<{ - path: string; - lines: string; - }>; - - const symbols = symbolRows.map((row) => ({ - name: row.name, - kind: row.kind, - path: row.path, - start_line: row.start_line, - lines: row.lines.split(",").map((l) => parseInt(l, 10)), - })); - - const files = fileRows.map((row) => ({ - path: row.path, - lines: row.lines.split(",").map((l) => parseInt(l, 10)), - })); - - const documents = docRows.map((row) => ({ - path: row.path, - lines: row.lines.split(",").map((l) => parseInt(l, 10)), - })); - - return { symbols, files, documents }; + const symbolRows = db.query(symbolQuery).all(docPath) as Array<{ + name: string; + kind: string; + path: string; + start_line: number; + lines: string; + }>; + + const fileRows = db.query(fileQuery).all(docPath) as Array<{ + path: string; + lines: string; + }>; + + const docRows = db.query(docQuery).all(docPath) as Array<{ + path: string; + lines: string; + }>; + + const symbols = symbolRows.map((row) => ({ + name: row.name, + kind: row.kind, + path: row.path, + start_line: row.start_line, + lines: row.lines.split(",").map((l) => parseInt(l, 10)), + })); + + const files = fileRows.map((row) => ({ + path: row.path, + lines: row.lines.split(",").map((l) => parseInt(l, 10)), + })); + + const documents = docRows.map((row) => ({ + path: row.path, + lines: row.lines.split(",").map((l) => parseInt(l, 10)), + })); + + return { symbols, files, documents }; } /** * Get document content and metadata */ export function getDocumentContent( - db: Database, - docPath: string + db: Database, + docPath: string, ): { - path: string; - type: string; - content: string; - symbol_count: number; - file_count: number; - document_count: number; + path: string; + type: string; + content: string; + symbol_count: number; + file_count: number; + document_count: number; } | null { - const query = ` + const query = ` SELECT path, type, content, symbol_count, file_count, document_count FROM documents WHERE path = ? `; - return db.query(query).get(docPath) as { - path: string; - type: string; - content: string; - symbol_count: number; - file_count: number; - document_count: number; - } | null; + return db.query(query).get(docPath) as { + path: string; + type: string; + content: string; + symbol_count: number; + file_count: number; + document_count: number; + } | null; } /** * Get document count */ export function getDocumentCount(db: Database): number { - const result = db.query("SELECT COUNT(*) as count FROM documents").get() as { - count: number; - }; - return result.count; + const result = db.query("SELECT COUNT(*) as count FROM documents").get() as { + count: number; + }; + return result.count; } /** * Get document counts by type */ export function getDocumentCountsByType( - db: Database + db: Database, ): Array<{ type: string; count: number }> { - const query = ` + const query = ` SELECT type, COUNT(*) as count FROM documents GROUP BY type ORDER BY count DESC `; - return db.query(query).all() as Array<{ type: string; count: number }>; + return db.query(query).all() as Array<{ type: string; count: number }>; } /** * Search documents by content (case-insensitive LIKE search) */ export function searchDocumentContent( - db: Database, - searchQuery: string, - limit: number = 20 + db: Database, + searchQuery: string, + limit: number = 20, ): Array<{ - path: string; - type: string; - symbol_count: number; - file_count: number; + path: string; + type: string; + symbol_count: number; + file_count: number; }> { - const query = ` + const query = ` SELECT path, type, symbol_count, file_count FROM documents WHERE content LIKE ? @@ -879,10 +879,10 @@ export function searchDocumentContent( LIMIT ? `; - return db.query(query).all(`%${searchQuery}%`, limit) as Array<{ - path: string; - type: string; - symbol_count: number; - file_count: number; - }>; + return db.query(query).all(`%${searchQuery}%`, limit) as Array<{ + path: string; + type: string; + symbol_count: number; + file_count: number; + }>; } diff --git a/src/index.ts b/src/index.ts index a0d7872..bcbe297 100755 --- a/src/index.ts +++ b/src/index.ts @@ -35,211 +35,223 @@ import packageJson from "../package.json"; const program = new Command(); program - .name("dora") - .description("Code Context CLI for AI Agents") - .version(packageJson.version); + .name("dora") + .description("Code Context CLI for AI Agents") + .version(packageJson.version); program - .command("init") - .description("Initialize dora in the current repository") - .action(wrapCommand(init)); + .command("init") + .description("Initialize dora in the current repository") + .action(wrapCommand(init)); program - .command("index") - .description("Run SCIP indexing (requires configured commands)") - .option("--full", "Force full rebuild") - .option("--skip-scip", "Skip running SCIP indexer (use existing .scip file)") - .option("--ignore ", "Ignore files matching pattern (can be repeated)", (value: string, previous: string[]) => previous.concat([value]), []) - .action( - wrapCommand(async (options) => { - await index({ full: options.full, skipScip: options.skipScip, ignore: options.ignore }); - }) - ); + .command("index") + .description("Run SCIP indexing (requires configured commands)") + .option("--full", "Force full rebuild") + .option("--skip-scip", "Skip running SCIP indexer (use existing .scip file)") + .option( + "--ignore ", + "Ignore files matching pattern (can be repeated)", + (value: string, previous: string[]) => previous.concat([value]), + [], + ) + .action( + wrapCommand(async (options) => { + await index({ + full: options.full, + skipScip: options.skipScip, + ignore: options.ignore, + }); + }), + ); program - .command("status") - .description("Show index status and statistics") - .action(wrapCommand(status)); + .command("status") + .description("Show index status and statistics") + .action(wrapCommand(status)); program - .command("map") - .description("Show high-level codebase map") - .action(wrapCommand(map)); + .command("map") + .description("Show high-level codebase map") + .action(wrapCommand(map)); program - .command("ls") - .description("List files in a directory from the index") - .argument("[directory]", "Directory path (optional, defaults to all files)") - .option("--limit ", "Maximum number of results (default: 100)") - .option( - "--sort ", - "Sort by: path, symbols, deps, or rdeps (default: path)" - ) - .action(wrapCommand(ls)); + .command("ls") + .description("List files in a directory from the index") + .argument("[directory]", "Directory path (optional, defaults to all files)") + .option("--limit ", "Maximum number of results (default: 100)") + .option( + "--sort ", + "Sort by: path, symbols, deps, or rdeps (default: path)", + ) + .action(wrapCommand(ls)); program - .command("file") - .description("Analyze a specific file with symbols and dependencies") - .argument("", "File path to analyze") - .action(wrapCommand(file)); + .command("file") + .description("Analyze a specific file with symbols and dependencies") + .argument("", "File path to analyze") + .action(wrapCommand(file)); program - .command("symbol") - .description("Search for symbols by name") - .argument("", "Symbol name to search for") - .option("--limit ", "Maximum number of results") - .option( - "--kind ", - "Filter by symbol kind (type, class, function, interface)" - ) - .action(wrapCommand(symbol)); + .command("symbol") + .description("Search for symbols by name") + .argument("", "Symbol name to search for") + .option("--limit ", "Maximum number of results") + .option( + "--kind ", + "Filter by symbol kind (type, class, function, interface)", + ) + .action(wrapCommand(symbol)); program - .command("refs") - .description("Find all references to a symbol") - .argument("", "Symbol name to find references for") - .option("--kind ", "Filter by symbol kind") - .option("--limit ", "Maximum number of results") - .action(wrapCommand(refs)); + .command("refs") + .description("Find all references to a symbol") + .argument("", "Symbol name to find references for") + .option("--kind ", "Filter by symbol kind") + .option("--limit ", "Maximum number of results") + .action(wrapCommand(refs)); program - .command("deps") - .description("Show file dependencies") - .argument("", "File path to analyze") - .option("--depth ", "Recursion depth (default: 1)") - .action(wrapCommand(deps)); + .command("deps") + .description("Show file dependencies") + .argument("", "File path to analyze") + .option("--depth ", "Recursion depth (default: 1)") + .action(wrapCommand(deps)); program - .command("rdeps") - .description("Show reverse dependencies (what depends on this file)") - .argument("", "File path to analyze") - .option("--depth ", "Recursion depth (default: 1)") - .action(wrapCommand(rdeps)); + .command("rdeps") + .description("Show reverse dependencies (what depends on this file)") + .argument("", "File path to analyze") + .option("--depth ", "Recursion depth (default: 1)") + .action(wrapCommand(rdeps)); program - .command("adventure") - .description("Find shortest adventure between two files") - .argument("", "Source file path") - .argument("", "Target file path") - .action(wrapCommand(adventure)); + .command("adventure") + .description("Find shortest adventure between two files") + .argument("", "Source file path") + .argument("", "Target file path") + .action(wrapCommand(adventure)); program - .command("leaves") - .description("Find leaf nodes - files with few dependents") - .option( - "--max-dependents ", - "Maximum number of dependents (default: 0)" - ) - .action(wrapCommand(leaves)); + .command("leaves") + .description("Find leaf nodes - files with few dependents") + .option( + "--max-dependents ", + "Maximum number of dependents (default: 0)", + ) + .action(wrapCommand(leaves)); program - .command("exports") - .description("List exported symbols from a file or package") - .argument("", "File path or package name") - .action(wrapCommand(exports)); + .command("exports") + .description("List exported symbols from a file or package") + .argument("", "File path or package name") + .action(wrapCommand(exports)); program - .command("imports") - .description("Show what a file imports (direct dependencies)") - .argument("", "File path to analyze") - .action(wrapCommand(imports)); + .command("imports") + .description("Show what a file imports (direct dependencies)") + .argument("", "File path to analyze") + .action(wrapCommand(imports)); program - .command("lost") - .description("Find lost symbols (potentially unused)") - .option("--limit ", "Maximum number of results (default: 50)") - .action(wrapCommand(lost)); + .command("lost") + .description("Find lost symbols (potentially unused)") + .option("--limit ", "Maximum number of results (default: 50)") + .action(wrapCommand(lost)); program - .command("treasure") - .description("Find treasure (most referenced files and largest dependencies)") - .option("--limit ", "Maximum number of results (default: 10)") - .action(wrapCommand(treasure)); + .command("treasure") + .description("Find treasure (most referenced files and largest dependencies)") + .option("--limit ", "Maximum number of results (default: 10)") + .action(wrapCommand(treasure)); program - .command("changes") - .description("Show files changed since git ref and their impact") - .argument("", "Git ref to compare against (e.g., main, HEAD~5)") - .action(wrapCommand(changes)); + .command("changes") + .description("Show files changed since git ref and their impact") + .argument("", "Git ref to compare against (e.g., main, HEAD~5)") + .action(wrapCommand(changes)); program - .command("graph") - .description("Generate dependency graph") - .argument("", "File path to analyze") - .option("--depth ", "Graph depth (default: 1)") - .option( - "--direction ", - "Graph direction: deps, rdeps, or both (default: both)" - ) - .action(wrapCommand(graph)); + .command("graph") + .description("Generate dependency graph") + .argument("", "File path to analyze") + .option("--depth ", "Graph depth (default: 1)") + .option( + "--direction ", + "Graph direction: deps, rdeps, or both (default: both)", + ) + .action(wrapCommand(graph)); program - .command("cycles") - .description("Find bidirectional dependencies (A imports B, B imports A)") - .option("--limit ", "Maximum number of results (default: 50)") - .action(wrapCommand(cycles)); + .command("cycles") + .description("Find bidirectional dependencies (A imports B, B imports A)") + .option("--limit ", "Maximum number of results (default: 50)") + .action(wrapCommand(cycles)); program - .command("coupling") - .description("Find tightly coupled file pairs") - .option("--threshold ", "Minimum total coupling score (default: 5)") - .action(wrapCommand(coupling)); + .command("coupling") + .description("Find tightly coupled file pairs") + .option("--threshold ", "Minimum total coupling score (default: 5)") + .action(wrapCommand(coupling)); program - .command("complexity") - .description("Show file complexity metrics") - .option( - "--sort ", - "Sort by: complexity, symbols, or stability (default: complexity)" - ) - .action(wrapCommand(complexity)); + .command("complexity") + .description("Show file complexity metrics") + .option( + "--sort ", + "Sort by: complexity, symbols, or stability (default: complexity)", + ) + .action(wrapCommand(complexity)); program - .command("schema") - .description("Show database schema (tables, columns, indexes)") - .action(wrapCommand(schema)); + .command("schema") + .description("Show database schema (tables, columns, indexes)") + .action(wrapCommand(schema)); program - .command("query") - .description("Execute raw SQL query (read-only)") - .argument("", "SQL query to execute") - .action(wrapCommand(query)); + .command("query") + .description("Execute raw SQL query (read-only)") + .argument("", "SQL query to execute") + .action(wrapCommand(query)); const cookbook = program - .command("cookbook") - .description("Query pattern cookbook and recipes"); + .command("cookbook") + .description("Query pattern cookbook and recipes"); cookbook - .command("list") - .description("List all available recipes") - .option("-f, --format ", "Output format: json or markdown", "json") - .action(wrapCommand(cookbookList)); + .command("list") + .description("List all available recipes") + .option("-f, --format ", "Output format: json or markdown", "json") + .action(wrapCommand(cookbookList)); cookbook - .command("show") - .argument("[recipe]", "Recipe name (quickstart, methods, references, exports)") - .description("Show a recipe or index") - .option("-f, --format ", "Output format: json or markdown", "json") - .action(wrapCommand(cookbookShow)); + .command("show") + .argument( + "[recipe]", + "Recipe name (quickstart, methods, references, exports)", + ) + .description("Show a recipe or index") + .option("-f, --format ", "Output format: json or markdown", "json") + .action(wrapCommand(cookbookShow)); const docs = program - .command("docs") - .description("List, search, and view documentation files") - .option("-t, --type ", "Filter by document type (md, txt)") - .action(wrapCommand((options) => docsList(options))); + .command("docs") + .description("List, search, and view documentation files") + .option("-t, --type ", "Filter by document type (md, txt)") + .action(wrapCommand((options) => docsList(options))); docs - .command("search") - .argument("", "Text to search for in documentation") - .option("-l, --limit ", "Maximum number of results (default: 20)") - .description("Search through documentation content") - .action(wrapCommand(docsSearch)); + .command("search") + .argument("", "Text to search for in documentation") + .option("-l, --limit ", "Maximum number of results (default: 20)") + .description("Search through documentation content") + .action(wrapCommand(docsSearch)); docs - .command("show") - .argument("", "Document path") - .option("-c, --content", "Include full document content") - .description("Show document metadata and references") - .action(wrapCommand(docsShow)); + .command("show") + .argument("", "Document path") + .option("-c, --content", "Include full document content") + .description("Show document metadata and references") + .action(wrapCommand(docsShow)); program.parse(); diff --git a/src/utils/changeDetection.ts b/src/utils/changeDetection.ts index 976910b..449c760 100644 --- a/src/utils/changeDetection.ts +++ b/src/utils/changeDetection.ts @@ -29,10 +29,13 @@ async function getFileMtime(filePath: string): Promise { * Main decision function: should we reindex? * Returns decision with reason and optionally a list of changed files */ -export async function shouldReindex( - config: Config, - force: boolean = false, -): Promise { +export async function shouldReindex({ + config, + force = false, +}: { + config: Config; + force?: boolean; +}): Promise { // Force flag bypasses all checks if (force) { return { shouldReindex: true, reason: "forced" }; @@ -119,8 +122,14 @@ async function mtimeBasedDetection( config: Config, ): Promise { try { - const scipPath = resolveAbsolute(config.root, config.scip); - const databasePath = resolveAbsolute(config.root, config.db); + const scipPath = resolveAbsolute({ + root: config.root, + relativePath: config.scip, + }); + const databasePath = resolveAbsolute({ + root: config.root, + relativePath: config.db, + }); // Check if files exist if (!existsSync(scipPath)) { @@ -166,19 +175,29 @@ async function mtimeBasedDetection( * Get current index state after indexing * This should be called after successful indexing to save the state */ -export async function getCurrentIndexState( - config: Config, - fileCount: number, - symbolCount: number, -) { +export async function getCurrentIndexState({ + config, + fileCount, + symbolCount, +}: { + config: Config; + fileCount: number; + symbolCount: number; +}) { const inGitRepo = await isGitRepo(); if (inGitRepo) { try { const gitCommit = await getCurrentGitCommit(); const gitHasUncommitted = await hasUncommittedChanges(); - const scipPath = resolveAbsolute(config.root, config.scip); - const databasePath = resolveAbsolute(config.root, config.db); + const scipPath = resolveAbsolute({ + root: config.root, + relativePath: config.scip, + }); + const databasePath = resolveAbsolute({ + root: config.root, + relativePath: config.db, + }); return { gitCommit, @@ -194,8 +213,14 @@ export async function getCurrentIndexState( } // Non-git repo or git failed - const scipPath = resolveAbsolute(config.root, config.scip); - const databasePath = resolveAbsolute(config.root, config.db); + const scipPath = resolveAbsolute({ + root: config.root, + relativePath: config.scip, + }); + const databasePath = resolveAbsolute({ + root: config.root, + relativePath: config.db, + }); return { gitCommit: "", diff --git a/src/utils/config.ts b/src/utils/config.ts index fae70d3..dba197b 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -4,7 +4,12 @@ import { existsSync, readFileSync } from "fs"; import { join } from "path"; import { ZodError, z } from "zod"; import { CtxError } from "./errors.ts"; -import { getConfigPath, getDoraDir } from "./paths.ts"; +import { + findRepoRoot, + getConfigPath, + getDoraDir, + resolveAbsolute, +} from "./paths.ts"; // Zod schemas for configuration validation @@ -40,7 +45,6 @@ export type Config = z.infer; */ export async function loadConfig(root?: string): Promise { if (!root) { - const { findRepoRoot } = await import("./paths.ts"); root = await findRepoRoot(); } @@ -58,7 +62,9 @@ export async function loadConfig(root?: string): Promise { return validateConfig(data); } catch (error) { throw new CtxError( - `Failed to read config: ${error instanceof Error ? error.message : String(error)}`, + `Failed to read config: ${ + error instanceof Error ? error.message : String(error) + }`, ); } } @@ -73,7 +79,9 @@ export async function saveConfig(config: Config): Promise { await Bun.write(configPath, JSON.stringify(config, null, 2) + "\n"); } catch (error) { throw new CtxError( - `Failed to write config: ${error instanceof Error ? error.message : String(error)}`, + `Failed to write config: ${ + error instanceof Error ? error.message : String(error) + }`, ); } } @@ -229,7 +237,9 @@ export function isInitialized(root: string): boolean { * Check if repository is indexed (has database file) */ export async function isIndexed(config: Config): Promise { - const { resolveAbsolute } = await import("./paths.ts"); - const dbPath = resolveAbsolute(config.root, config.db); + const dbPath = resolveAbsolute({ + root: config.root, + relativePath: config.db, + }); return existsSync(dbPath); } diff --git a/src/utils/fileScanner.ts b/src/utils/fileScanner.ts index 9c8900c..bbaf265 100644 --- a/src/utils/fileScanner.ts +++ b/src/utils/fileScanner.ts @@ -12,11 +12,15 @@ export interface DocumentFile { /** * Scan for documentation files in the repository with .gitignore support */ -export async function scanDocumentFiles( - repoRoot: string, - extensions: string[] = [".md", ".txt"], - ignorePatterns: string[] = [], -): Promise { +export async function scanDocumentFiles({ + repoRoot, + extensions = [".md", ".txt"], + ignorePatterns = [], +}: { + repoRoot: string; + extensions?: string[]; + ignorePatterns?: string[]; +}): Promise { debugScanner("Scanning for document files in %s", repoRoot); debugScanner("Extensions: %o", extensions); if (ignorePatterns.length > 0) { @@ -131,10 +135,13 @@ async function walkDirectory( /** * Filter documents based on modification time (for incremental indexing) */ -export function filterChangedDocuments( - existingDocs: Map, // path -> mtime - scannedDocs: DocumentFile[], -): DocumentFile[] { +export function filterChangedDocuments({ + existingDocs, + scannedDocs, +}: { + existingDocs: Map; + scannedDocs: DocumentFile[]; +}): DocumentFile[] { return scannedDocs.filter((doc) => { const existingMtime = existingDocs.get(doc.path); if (!existingMtime) { diff --git a/src/utils/paths.ts b/src/utils/paths.ts index d7fbb7e..e165b6f 100644 --- a/src/utils/paths.ts +++ b/src/utils/paths.ts @@ -44,14 +44,26 @@ export async function findRepoRoot( /** * Resolve path to absolute from repo root */ -export function resolveAbsolute(root: string, relativePath: string): string { +export function resolveAbsolute({ + root, + relativePath, +}: { + root: string; + relativePath: string; +}): string { return resolve(root, relativePath); } /** * Convert absolute path to relative from repo root */ -export function resolveRelative(root: string, absolutePath: string): string { +export function resolveRelative({ + root, + absolutePath, +}: { + root: string; + absolutePath: string; +}): string { return relative(root, absolutePath); } @@ -74,9 +86,15 @@ export function getConfigPath(root: string): string { * If the path is already relative, returns it unchanged. * If the path is absolute, converts it to relative. */ -export function normalizeToRelative(root: string, inputPath: string): string { +export function normalizeToRelative({ + root, + inputPath, +}: { + root: string; + inputPath: string; +}): string { if (inputPath.startsWith("/")) { - return resolveRelative(root, inputPath); + return resolveRelative({ root, absolutePath: inputPath }); } return inputPath; } diff --git a/src/utils/templates.ts b/src/utils/templates.ts index 4e249e5..b746649 100644 --- a/src/utils/templates.ts +++ b/src/utils/templates.ts @@ -5,7 +5,9 @@ import { join } from "path"; import snippetMd from "../templates/docs/SNIPPET.md" with { type: "text" }; import skillMd from "../templates/docs/SKILL.md" with { type: "text" }; -import cookbookIndexMd from "../templates/cookbook/index.md" with { type: "text" }; +import cookbookIndexMd from "../templates/cookbook/index.md" with { + type: "text", +}; import cookbookQuickstartMd from "../templates/cookbook/quickstart.md" with { type: "text", }; @@ -23,10 +25,13 @@ import cookbookExportsMd from "../templates/cookbook/exports.md" with { * Copy a single file if it doesn't exist at target * @returns true if copied, false if skipped */ -async function copyFileIfNotExists( - content: string, - targetPath: string, -): Promise { +async function copyFileIfNotExists({ + content, + targetPath, +}: { + content: string; + targetPath: string; +}): Promise { // Check if target file already exists const targetFile = Bun.file(targetPath); if (await targetFile.exists()) { @@ -72,7 +77,10 @@ export async function copyTemplates(targetDoraDir: string): Promise { ]; // Create subdirectories - const subdirs = [join(targetDoraDir, "docs"), join(targetDoraDir, "cookbook")]; + const subdirs = [ + join(targetDoraDir, "docs"), + join(targetDoraDir, "cookbook"), + ]; for (const dir of subdirs) { try { @@ -84,6 +92,6 @@ export async function copyTemplates(targetDoraDir: string): Promise { // Copy each template file for (const { content, target } of templates) { - await copyFileIfNotExists(content, target); + await copyFileIfNotExists({ content, targetPath: target }); } } diff --git a/test/commands/adventure.test.ts b/test/commands/adventure.test.ts index 787da9c..5ccb411 100644 --- a/test/commands/adventure.test.ts +++ b/test/commands/adventure.test.ts @@ -3,19 +3,19 @@ import { Database } from "bun:sqlite"; import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import { - getDependencies, - getReverseDependencies, + getDependencies, + getReverseDependencies, } from "../../src/db/queries.ts"; describe("Adventure Command - Pathfinding Algorithm", () => { - let db: Database; + let db: Database; - beforeAll(() => { - // Create in-memory test database with dependency graph - db = new Database(":memory:"); + beforeAll(() => { + // Create in-memory test database with dependency graph + db = new Database(":memory:"); - // Create schema - db.exec(` + // Create schema + db.exec(` CREATE TABLE files ( id INTEGER PRIMARY KEY, path TEXT UNIQUE NOT NULL, @@ -38,179 +38,179 @@ describe("Adventure Command - Pathfinding Algorithm", () => { ); `); - // Create test dependency graph: - // A -> B -> C -> D - // | | - // v v - // E F - // G (isolated) - - const files = [ - { id: 1, path: "a.ts" }, - { id: 2, path: "b.ts" }, - { id: 3, path: "c.ts" }, - { id: 4, path: "d.ts" }, - { id: 5, path: "e.ts" }, - { id: 6, path: "f.ts" }, - { id: 7, path: "g.ts" }, // isolated - ]; - - for (const file of files) { - db.run( - "INSERT INTO files (id, path, language, mtime, indexed_at) VALUES (?, ?, 'typescript', 1000, 1000)", - [file.id, file.path], - ); - } - - // Create dependencies - const deps = [ - { from: 1, to: 2 }, // A -> B - { from: 2, to: 3 }, // B -> C - { from: 3, to: 4 }, // C -> D - { from: 2, to: 5 }, // B -> E - { from: 3, to: 6 }, // C -> F - ]; - - for (const dep of deps) { - db.run( - "INSERT INTO dependencies (from_file_id, to_file_id, symbol_count) VALUES (?, ?, 1)", - [dep.from, dep.to], - ); - } - }); - - afterAll(() => { - db.close(); - }); - - describe("Direct dependencies", () => { - test("should find direct dependency at depth 1", () => { - const deps = getDependencies(db, "a.ts", 1); - - expect(deps).toHaveLength(1); - expect(deps[0].path).toBe("b.ts"); - expect(deps[0].depth).toBe(1); - }); - - test("should find multiple direct dependencies", () => { - const deps = getDependencies(db, "b.ts", 1); - - expect(deps).toHaveLength(2); - const paths = deps.map((d) => d.path).sort(); - expect(paths).toEqual(["c.ts", "e.ts"]); - }); - }); - - describe("Multi-hop dependencies", () => { - test("should find dependencies at depth 2", () => { - const deps = getDependencies(db, "a.ts", 2); - - expect(deps.length).toBeGreaterThanOrEqual(2); - const depMap = new Map(deps.map((d) => [d.path, d.depth])); - - // Direct: A -> B - expect(depMap.get("b.ts")).toBe(1); - - // 2-hop: A -> B -> C and A -> B -> E - expect(depMap.get("c.ts")).toBe(2); - expect(depMap.get("e.ts")).toBe(2); - }); - - test("should find dependencies at depth 3", () => { - const deps = getDependencies(db, "a.ts", 3); - - const depMap = new Map(deps.map((d) => [d.path, d.depth])); - - // Should have all reachable nodes - expect(depMap.get("b.ts")).toBe(1); - expect(depMap.get("c.ts")).toBe(2); - expect(depMap.get("e.ts")).toBe(2); - expect(depMap.get("d.ts")).toBe(3); - expect(depMap.get("f.ts")).toBe(3); - }); - }); - - describe("Reverse dependencies", () => { - test("should find direct reverse dependency", () => { - const rdeps = getReverseDependencies(db, "b.ts", 1); - - expect(rdeps).toHaveLength(1); - expect(rdeps[0].path).toBe("a.ts"); - expect(rdeps[0].depth).toBe(1); - }); - - test("should find multi-hop reverse dependencies", () => { - const rdeps = getReverseDependencies(db, "c.ts", 2); - - const depMap = new Map(rdeps.map((d) => [d.path, d.depth])); - - // Direct: B -> C - expect(depMap.get("b.ts")).toBe(1); - - // 2-hop: A -> B -> C - expect(depMap.get("a.ts")).toBe(2); - }); - }); - - describe("Isolated nodes", () => { - test("should return empty for isolated node dependencies", () => { - const deps = getDependencies(db, "g.ts", 5); - - expect(deps).toHaveLength(0); - }); - - test("should return empty for isolated node reverse deps", () => { - const rdeps = getReverseDependencies(db, "g.ts", 5); - - expect(rdeps).toHaveLength(0); - }); - }); - - describe("Leaf nodes", () => { - test("should return empty for leaf node (no outgoing deps)", () => { - const deps = getDependencies(db, "d.ts", 2); - - expect(deps).toHaveLength(0); - }); - - test("should find reverse deps for leaf node", () => { - const rdeps = getReverseDependencies(db, "d.ts", 3); + // Create test dependency graph: + // A -> B -> C -> D + // | | + // v v + // E F + // G (isolated) + + const files = [ + { id: 1, path: "a.ts" }, + { id: 2, path: "b.ts" }, + { id: 3, path: "c.ts" }, + { id: 4, path: "d.ts" }, + { id: 5, path: "e.ts" }, + { id: 6, path: "f.ts" }, + { id: 7, path: "g.ts" }, // isolated + ]; + + for (const file of files) { + db.run( + "INSERT INTO files (id, path, language, mtime, indexed_at) VALUES (?, ?, 'typescript', 1000, 1000)", + [file.id, file.path] + ); + } + + // Create dependencies + const deps = [ + { from: 1, to: 2 }, // A -> B + { from: 2, to: 3 }, // B -> C + { from: 3, to: 4 }, // C -> D + { from: 2, to: 5 }, // B -> E + { from: 3, to: 6 }, // C -> F + ]; + + for (const dep of deps) { + db.run( + "INSERT INTO dependencies (from_file_id, to_file_id, symbol_count) VALUES (?, ?, 1)", + [dep.from, dep.to] + ); + } + }); + + afterAll(() => { + db.close(); + }); + + describe("Direct dependencies", () => { + test("should find direct dependency at depth 1", () => { + const deps = getDependencies(db, "a.ts", 1); + + expect(deps).toHaveLength(1); + expect(deps[0].path).toBe("b.ts"); + expect(deps[0].depth).toBe(1); + }); + + test("should find multiple direct dependencies", () => { + const deps = getDependencies(db, "b.ts", 1); + + expect(deps).toHaveLength(2); + const paths = deps.map((d) => d.path).sort(); + expect(paths).toEqual(["c.ts", "e.ts"]); + }); + }); + + describe("Multi-hop dependencies", () => { + test("should find dependencies at depth 2", () => { + const deps = getDependencies(db, "a.ts", 2); + + expect(deps.length).toBeGreaterThanOrEqual(2); + const depMap = new Map(deps.map((d) => [d.path, d.depth])); + + // Direct: A -> B + expect(depMap.get("b.ts")).toBe(1); + + // 2-hop: A -> B -> C and A -> B -> E + expect(depMap.get("c.ts")).toBe(2); + expect(depMap.get("e.ts")).toBe(2); + }); + + test("should find dependencies at depth 3", () => { + const deps = getDependencies(db, "a.ts", 3); + + const depMap = new Map(deps.map((d) => [d.path, d.depth])); + + // Should have all reachable nodes + expect(depMap.get("b.ts")).toBe(1); + expect(depMap.get("c.ts")).toBe(2); + expect(depMap.get("e.ts")).toBe(2); + expect(depMap.get("d.ts")).toBe(3); + expect(depMap.get("f.ts")).toBe(3); + }); + }); + + describe("Reverse dependencies", () => { + test("should find direct reverse dependency", () => { + const rdeps = getReverseDependencies(db, "b.ts", 1); + + expect(rdeps).toHaveLength(1); + expect(rdeps[0].path).toBe("a.ts"); + expect(rdeps[0].depth).toBe(1); + }); + + test("should find multi-hop reverse dependencies", () => { + const rdeps = getReverseDependencies(db, "c.ts", 2); + + const depMap = new Map(rdeps.map((d) => [d.path, d.depth])); + + // Direct: B -> C + expect(depMap.get("b.ts")).toBe(1); + + // 2-hop: A -> B -> C + expect(depMap.get("a.ts")).toBe(2); + }); + }); + + describe("Isolated nodes", () => { + test("should return empty for isolated node dependencies", () => { + const deps = getDependencies(db, "g.ts", 5); + + expect(deps).toHaveLength(0); + }); + + test("should return empty for isolated node reverse deps", () => { + const rdeps = getReverseDependencies(db, "g.ts", 5); + + expect(rdeps).toHaveLength(0); + }); + }); + + describe("Leaf nodes", () => { + test("should return empty for leaf node (no outgoing deps)", () => { + const deps = getDependencies(db, "d.ts", 2); + + expect(deps).toHaveLength(0); + }); + + test("should find reverse deps for leaf node", () => { + const rdeps = getReverseDependencies(db, "d.ts", 3); - expect(rdeps.length).toBeGreaterThan(0); - const depMap = new Map(rdeps.map((d) => [d.path, d.depth])); + expect(rdeps.length).toBeGreaterThan(0); + const depMap = new Map(rdeps.map((d) => [d.path, d.depth])); - // D is reachable from C -> B -> A - expect(depMap.get("c.ts")).toBe(1); - expect(depMap.get("b.ts")).toBe(2); - expect(depMap.get("a.ts")).toBe(3); - }); - }); + // D is reachable from C -> B -> A + expect(depMap.get("c.ts")).toBe(1); + expect(depMap.get("b.ts")).toBe(2); + expect(depMap.get("a.ts")).toBe(3); + }); + }); - describe("Depth limiting", () => { - test("should respect depth limit", () => { - const deps = getDependencies(db, "a.ts", 1); + describe("Depth limiting", () => { + test("should respect depth limit", () => { + const deps = getDependencies(db, "a.ts", 1); - // Should only get direct dependency (B), not transitive ones - expect(deps).toHaveLength(1); - expect(deps[0].path).toBe("b.ts"); + // Should only get direct dependency (B), not transitive ones + expect(deps).toHaveLength(1); + expect(deps[0].path).toBe("b.ts"); - // Should not include C, D, E, F - const paths = deps.map((d) => d.path); - expect(paths).not.toContain("c.ts"); - expect(paths).not.toContain("d.ts"); - }); + // Should not include C, D, E, F + const paths = deps.map((d) => d.path); + expect(paths).not.toContain("c.ts"); + expect(paths).not.toContain("d.ts"); + }); - test("should find shortest path (min depth) when multiple paths exist", () => { - // Both B and C depend on different files, but B is closer - const deps = getDependencies(db, "a.ts", 2); + test("should find shortest path (min depth) when multiple paths exist", () => { + // Both B and C depend on different files, but B is closer + const deps = getDependencies(db, "a.ts", 2); - const depMap = new Map(deps.map((d) => [d.path, d.depth])); + const depMap = new Map(deps.map((d) => [d.path, d.depth])); - // B should be at depth 1 (shortest) - expect(depMap.get("b.ts")).toBe(1); + // B should be at depth 1 (shortest) + expect(depMap.get("b.ts")).toBe(1); - // C should be at depth 2 (A -> B -> C) - expect(depMap.get("c.ts")).toBe(2); - }); - }); + // C should be at depth 2 (A -> B -> C) + expect(depMap.get("c.ts")).toBe(2); + }); + }); }); diff --git a/test/converter/documents.test.ts b/test/converter/documents.test.ts index a090009..1aab718 100644 --- a/test/converter/documents.test.ts +++ b/test/converter/documents.test.ts @@ -117,14 +117,18 @@ See src/logger.ts for implementation. }); test("should process documents and extract references", async () => { - const stats = await processDocuments(db, testDir, "full"); + const stats = await processDocuments({ + db, + repoRoot: testDir, + mode: "full", + }); expect(stats.processed).toBe(1); expect(stats.total).toBe(1); }); test("should store document content", async () => { - await processDocuments(db, testDir, "full"); + await processDocuments({ db, repoRoot: testDir, mode: "full" }); const doc = db .query("SELECT * FROM documents WHERE path = ?") @@ -137,7 +141,7 @@ See src/logger.ts for implementation. }); test("should extract symbol references", async () => { - await processDocuments(db, testDir, "full"); + await processDocuments({ db, repoRoot: testDir, mode: "full" }); const refs = db .query( @@ -157,7 +161,7 @@ See src/logger.ts for implementation. }); test("should extract file references", async () => { - await processDocuments(db, testDir, "full"); + await processDocuments({ db, repoRoot: testDir, mode: "full" }); const refs = db .query( @@ -177,10 +181,14 @@ See src/logger.ts for implementation. test("should support incremental mode", async () => { // First full process - await processDocuments(db, testDir, "full"); + await processDocuments({ db, repoRoot: testDir, mode: "full" }); // Second incremental process (no changes) - const stats = await processDocuments(db, testDir, "incremental"); + const stats = await processDocuments({ + db, + repoRoot: testDir, + mode: "incremental", + }); // Should skip unchanged files expect(stats.skipped).toBeGreaterThan(0); diff --git a/test/converter/scip-parser.test.ts b/test/converter/scip-parser.test.ts index 18c3e3e..b9f2fa6 100644 --- a/test/converter/scip-parser.test.ts +++ b/test/converter/scip-parser.test.ts @@ -4,418 +4,424 @@ import { describe, expect, test } from "bun:test"; import { existsSync } from "fs"; import { join } from "path"; import { - buildLookupMaps, - extractDefinitions, - extractReferences, - findDefinitionFile, - getDocumentSymbols, - getFileDependencies, - type ParsedDocument, - parseScipFile, - type ScipData, + buildLookupMaps, + extractDefinitions, + extractReferences, + findDefinitionFile, + getDocumentSymbols, + getFileDependencies, + type ParsedDocument, + parseScipFile, + type ScipData, } from "../../src/converter/scip-parser.ts"; describe("SCIP Parser", () => { - const exampleScipPath = join(process.cwd(), "test", "fixtures", "index.scip"); - const skipTests = !existsSync(exampleScipPath); - - test("should parse SCIP file successfully", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - - expect(scipData).toBeDefined(); - expect(scipData.documents).toBeDefined(); - expect(scipData.documents.length).toBeGreaterThan(0); - expect(scipData.externalSymbols).toBeDefined(); - }); - - test("should parse document with correct structure", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - const sampleDoc = scipData.documents[0]; - - if (!sampleDoc) { - console.log("Skipping test: no documents in SCIP file"); - return; - } - - expect(sampleDoc).toHaveProperty("relativePath"); - expect(sampleDoc).toHaveProperty("language"); - expect(sampleDoc).toHaveProperty("occurrences"); - expect(sampleDoc).toHaveProperty("symbols"); - expect(typeof sampleDoc.relativePath).toBe("string"); - expect(typeof sampleDoc.language).toBe("string"); - expect(Array.isArray(sampleDoc.occurrences)).toBe(true); - expect(Array.isArray(sampleDoc.symbols)).toBe(true); - }); - - test("should parse occurrences with correct range format", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - const sampleDoc = scipData.documents[0]; - - if (!sampleDoc || sampleDoc.occurrences.length === 0) { - console.log("Skipping test: no occurrences in SCIP file"); - return; - } - - const occ = sampleDoc.occurrences[0]; - expect(occ).toHaveProperty("range"); - expect(occ).toHaveProperty("symbol"); - expect(occ).toHaveProperty("symbolRoles"); - - // Range should be [startLine, startChar, endLine, endChar] - expect(occ.range.length).toBe(4); - expect(typeof occ.range[0]).toBe("number"); // startLine - expect(typeof occ.range[1]).toBe("number"); // startChar - expect(typeof occ.range[2]).toBe("number"); // endLine - expect(typeof occ.range[3]).toBe("number"); // endChar - }); - - test("should extract definitions from document", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - const sampleDoc = scipData.documents[0]; - - if (!sampleDoc) { - console.log("Skipping test: no documents in SCIP file"); - return; - } - - const definitions = extractDefinitions(sampleDoc); - expect(Array.isArray(definitions)).toBe(true); - - if (definitions.length > 0) { - const def = definitions[0]; - expect(def).toHaveProperty("symbol"); - expect(def).toHaveProperty("range"); - expect(def.range.length).toBe(4); - expect(typeof def.symbol).toBe("string"); - } - }); - - test("should extract references from document", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - const sampleDoc = scipData.documents[0]; - - if (!sampleDoc) { - console.log("Skipping test: no documents in SCIP file"); - return; - } - - const references = extractReferences(sampleDoc); - expect(Array.isArray(references)).toBe(true); - - if (references.length > 0) { - const ref = references[0]; - expect(ref).toHaveProperty("symbol"); - expect(ref).toHaveProperty("range"); - expect(ref).toHaveProperty("line"); - expect(ref.range.length).toBe(4); - expect(typeof ref.symbol).toBe("string"); - expect(typeof ref.line).toBe("number"); - } - }); - - test("should not have overlap between definitions and references", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - const sampleDoc = scipData.documents[0]; - - if (!sampleDoc) { - console.log("Skipping test: no documents in SCIP file"); - return; - } - - const definitions = extractDefinitions(sampleDoc); - const references = extractReferences(sampleDoc); - - // Build sets of occurrence indices - const defIndices = new Set( - sampleDoc.occurrences - .map((occ, idx) => (occ.symbolRoles & 0x1 ? idx : -1)) - .filter((idx) => idx !== -1), - ); - - const refIndices = new Set( - sampleDoc.occurrences - .map((occ, idx) => (!(occ.symbolRoles & 0x1) ? idx : -1)) - .filter((idx) => idx !== -1), - ); - - // Should have no overlap - for (const defIdx of defIndices) { - expect(refIndices.has(defIdx)).toBe(false); - } - }); - - test("should build lookup maps correctly", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - const maps = buildLookupMaps(scipData); - - expect(maps).toHaveProperty("documentsByPath"); - expect(maps).toHaveProperty("symbolsById"); - expect(maps).toHaveProperty("definitionsBySymbol"); - - expect(maps.documentsByPath instanceof Map).toBe(true); - expect(maps.symbolsById instanceof Map).toBe(true); - expect(maps.definitionsBySymbol instanceof Map).toBe(true); - - // Documents map should have all documents - expect(maps.documentsByPath.size).toBe(scipData.documents.length); - - // Symbol map should have at least external symbols - expect(maps.symbolsById.size).toBeGreaterThanOrEqual( - scipData.externalSymbols.length, - ); - }); - - test("should find definition file for a symbol", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - const sampleDoc = scipData.documents[0]; - - if (!sampleDoc) { - console.log("Skipping test: no documents in SCIP file"); - return; - } - - const maps = buildLookupMaps(scipData); - const definitions = extractDefinitions(sampleDoc); - - if (definitions.length > 0) { - const def = definitions[0]; - const defFile = findDefinitionFile( - def.symbol, - maps.documentsByPath, - maps.symbolsById, - ); - - // Should find the file (or null for external symbols) - expect(defFile === null || typeof defFile === "string").toBe(true); - } - }); - - test("should get document symbols with metadata", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - const sampleDoc = scipData.documents[0]; - - if (!sampleDoc) { - console.log("Skipping test: no documents in SCIP file"); - return; - } - - const maps = buildLookupMaps(scipData); - const docSymbols = getDocumentSymbols(sampleDoc, maps.symbolsById); - - expect(Array.isArray(docSymbols)).toBe(true); - - if (docSymbols.length > 0) { - const sym = docSymbols[0]; - expect(sym).toHaveProperty("symbol"); - expect(sym).toHaveProperty("kind"); - expect(sym).toHaveProperty("range"); - expect(typeof sym.symbol).toBe("string"); - expect(typeof sym.kind).toBe("number"); - expect(sym.range.length).toBe(4); - } - }); - - test("should get file dependencies excluding self-references", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - const sampleDoc = scipData.documents[0]; - - if (!sampleDoc) { - console.log("Skipping test: no documents in SCIP file"); - return; - } - - const maps = buildLookupMaps(scipData); - const deps = getFileDependencies( - sampleDoc, - maps.documentsByPath, - maps.symbolsById, - maps.definitionsBySymbol, - ); - - expect(deps instanceof Map).toBe(true); - - // Verify no self-references - expect(deps.has(sampleDoc.relativePath)).toBe(false); - - // Dependencies should map file path -> set of symbols - for (const [depPath, symbols] of deps) { - expect(typeof depPath).toBe("string"); - expect(symbols instanceof Set).toBe(true); - expect(depPath).not.toBe(sampleDoc.relativePath); - - // All symbols should be non-empty strings - for (const sym of symbols) { - expect(typeof sym).toBe("string"); - expect(sym.length).toBeGreaterThan(0); - } - } - }); - - test("should handle SCIP metadata", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - - if (scipData.metadata) { - expect(scipData.metadata).toHaveProperty("toolName"); - expect(scipData.metadata).toHaveProperty("toolVersion"); - - if (scipData.metadata.toolName) { - expect(typeof scipData.metadata.toolName).toBe("string"); - } - } - }); - - test("should parse external symbols", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - - expect(Array.isArray(scipData.externalSymbols)).toBe(true); - - if (scipData.externalSymbols.length > 0) { - const extSym = scipData.externalSymbols[0]; - expect(extSym).toHaveProperty("symbol"); - expect(extSym).toHaveProperty("kind"); - expect(typeof extSym.symbol).toBe("string"); - expect(typeof extSym.kind).toBe("number"); - } - }); - - test("should correctly identify local symbols", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - const maps = buildLookupMaps(scipData); - - // Find a local symbol (contains 'local' in symbol identifier) - let foundLocalSymbol = false; - for (const doc of scipData.documents) { - const definitions = extractDefinitions(doc); - for (const def of definitions) { - if (def.symbol.includes("local")) { - foundLocalSymbol = true; - - // Local symbols should be defined in the same file - const defFile = findDefinitionFile( - def.symbol, - maps.documentsByPath, - maps.symbolsById, - ); - - // Should find in current document or return null - expect(defFile === null || defFile === doc.relativePath).toBe(true); - break; - } - } - if (foundLocalSymbol) break; - } - }); - - test("should handle documents with no symbols", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - - // Find or create a document with no symbols - const emptyDoc: ParsedDocument = { - relativePath: "test/empty.ts", - language: "TypeScript", - occurrences: [], - symbols: [], - }; - - const definitions = extractDefinitions(emptyDoc); - const references = extractReferences(emptyDoc); - - expect(definitions.length).toBe(0); - expect(references.length).toBe(0); - - const maps = buildLookupMaps(scipData); - const docSymbols = getDocumentSymbols(emptyDoc, maps.symbolsById); - expect(docSymbols.length).toBe(0); - - const deps = getFileDependencies( - emptyDoc, - maps.documentsByPath, - maps.symbolsById, - maps.definitionsBySymbol, - ); - expect(deps.size).toBe(0); - }); - - test("should handle 3-element ranges (same line)", async () => { - if (skipTests) { - console.log("Skipping test: example SCIP file not found"); - return; - } - - const scipData = await parseScipFile(exampleScipPath); - - // This is tested implicitly in the parsing - if we got here without errors, - // the parser handled both 3 and 4 element ranges correctly - expect(scipData.documents.length).toBeGreaterThan(0); - }); + const exampleScipPath = join(process.cwd(), "test", "fixtures", "index.scip"); + const skipTests = !existsSync(exampleScipPath); + + test("should parse SCIP file successfully", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + + expect(scipData).toBeDefined(); + expect(scipData.documents).toBeDefined(); + expect(scipData.documents.length).toBeGreaterThan(0); + expect(scipData.externalSymbols).toBeDefined(); + }); + + test("should parse document with correct structure", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + const sampleDoc = scipData.documents[0]; + + if (!sampleDoc) { + console.log("Skipping test: no documents in SCIP file"); + return; + } + + expect(sampleDoc).toHaveProperty("relativePath"); + expect(sampleDoc).toHaveProperty("language"); + expect(sampleDoc).toHaveProperty("occurrences"); + expect(sampleDoc).toHaveProperty("symbols"); + expect(typeof sampleDoc.relativePath).toBe("string"); + expect(typeof sampleDoc.language).toBe("string"); + expect(Array.isArray(sampleDoc.occurrences)).toBe(true); + expect(Array.isArray(sampleDoc.symbols)).toBe(true); + }); + + test("should parse occurrences with correct range format", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + const sampleDoc = scipData.documents[0]; + + if (!sampleDoc || sampleDoc.occurrences.length === 0) { + console.log("Skipping test: no occurrences in SCIP file"); + return; + } + + const occ = sampleDoc.occurrences[0]; + expect(occ).toHaveProperty("range"); + expect(occ).toHaveProperty("symbol"); + expect(occ).toHaveProperty("symbolRoles"); + + // Range should be [startLine, startChar, endLine, endChar] + expect(occ.range.length).toBe(4); + expect(typeof occ.range[0]).toBe("number"); // startLine + expect(typeof occ.range[1]).toBe("number"); // startChar + expect(typeof occ.range[2]).toBe("number"); // endLine + expect(typeof occ.range[3]).toBe("number"); // endChar + }); + + test("should extract definitions from document", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + const sampleDoc = scipData.documents[0]; + + if (!sampleDoc) { + console.log("Skipping test: no documents in SCIP file"); + return; + } + + const definitions = extractDefinitions(sampleDoc); + expect(Array.isArray(definitions)).toBe(true); + + if (definitions.length > 0) { + const def = definitions[0]; + expect(def).toHaveProperty("symbol"); + expect(def).toHaveProperty("range"); + expect(def.range.length).toBe(4); + expect(typeof def.symbol).toBe("string"); + } + }); + + test("should extract references from document", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + const sampleDoc = scipData.documents[0]; + + if (!sampleDoc) { + console.log("Skipping test: no documents in SCIP file"); + return; + } + + const references = extractReferences(sampleDoc); + expect(Array.isArray(references)).toBe(true); + + if (references.length > 0) { + const ref = references[0]; + expect(ref).toHaveProperty("symbol"); + expect(ref).toHaveProperty("range"); + expect(ref).toHaveProperty("line"); + expect(ref.range.length).toBe(4); + expect(typeof ref.symbol).toBe("string"); + expect(typeof ref.line).toBe("number"); + } + }); + + test("should not have overlap between definitions and references", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + const sampleDoc = scipData.documents[0]; + + if (!sampleDoc) { + console.log("Skipping test: no documents in SCIP file"); + return; + } + + const definitions = extractDefinitions(sampleDoc); + const references = extractReferences(sampleDoc); + + // Build sets of occurrence indices + const defIndices = new Set( + sampleDoc.occurrences + .map((occ, idx) => (occ.symbolRoles & 0x1 ? idx : -1)) + .filter((idx) => idx !== -1) + ); + + const refIndices = new Set( + sampleDoc.occurrences + .map((occ, idx) => (!(occ.symbolRoles & 0x1) ? idx : -1)) + .filter((idx) => idx !== -1) + ); + + // Should have no overlap + for (const defIdx of defIndices) { + expect(refIndices.has(defIdx)).toBe(false); + } + }); + + test("should build lookup maps correctly", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + const maps = buildLookupMaps(scipData); + + expect(maps).toHaveProperty("documentsByPath"); + expect(maps).toHaveProperty("symbolsById"); + expect(maps).toHaveProperty("definitionsBySymbol"); + + expect(maps.documentsByPath instanceof Map).toBe(true); + expect(maps.symbolsById instanceof Map).toBe(true); + expect(maps.definitionsBySymbol instanceof Map).toBe(true); + + // Documents map should have all documents + expect(maps.documentsByPath.size).toBe(scipData.documents.length); + + // Symbol map should have at least external symbols + expect(maps.symbolsById.size).toBeGreaterThanOrEqual( + scipData.externalSymbols.length + ); + }); + + test("should find definition file for a symbol", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + const sampleDoc = scipData.documents[0]; + + if (!sampleDoc) { + console.log("Skipping test: no documents in SCIP file"); + return; + } + + const maps = buildLookupMaps(scipData); + const definitions = extractDefinitions(sampleDoc); + + if (definitions.length > 0) { + const def = definitions[0]; + const defFile = findDefinitionFile({ + symbol: def.symbol, + documents: maps.documentsByPath, + symbols: maps.symbolsById, + }); + + // Should find the file (or null for external symbols) + expect(defFile === null || typeof defFile === "string").toBe(true); + } + }); + + test("should get document symbols with metadata", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + const sampleDoc = scipData.documents[0]; + + if (!sampleDoc) { + console.log("Skipping test: no documents in SCIP file"); + return; + } + + const maps = buildLookupMaps(scipData); + const docSymbols = getDocumentSymbols({ + doc: sampleDoc, + symbolsById: maps.symbolsById, + }); + + expect(Array.isArray(docSymbols)).toBe(true); + + if (docSymbols.length > 0) { + const sym = docSymbols[0]; + expect(sym).toHaveProperty("symbol"); + expect(sym).toHaveProperty("kind"); + expect(sym).toHaveProperty("range"); + expect(typeof sym.symbol).toBe("string"); + expect(typeof sym.kind).toBe("number"); + expect(sym.range.length).toBe(4); + } + }); + + test("should get file dependencies excluding self-references", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + const sampleDoc = scipData.documents[0]; + + if (!sampleDoc) { + console.log("Skipping test: no documents in SCIP file"); + return; + } + + const maps = buildLookupMaps(scipData); + const deps = getFileDependencies({ + doc: sampleDoc, + documentsByPath: maps.documentsByPath, + symbolsById: maps.symbolsById, + definitionsBySymbol: maps.definitionsBySymbol, + }); + + expect(deps instanceof Map).toBe(true); + + // Verify no self-references + expect(deps.has(sampleDoc.relativePath)).toBe(false); + + // Dependencies should map file path -> set of symbols + for (const [depPath, symbols] of deps) { + expect(typeof depPath).toBe("string"); + expect(symbols instanceof Set).toBe(true); + expect(depPath).not.toBe(sampleDoc.relativePath); + + // All symbols should be non-empty strings + for (const sym of symbols) { + expect(typeof sym).toBe("string"); + expect(sym.length).toBeGreaterThan(0); + } + } + }); + + test("should handle SCIP metadata", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + + if (scipData.metadata) { + expect(scipData.metadata).toHaveProperty("toolName"); + expect(scipData.metadata).toHaveProperty("toolVersion"); + + if (scipData.metadata.toolName) { + expect(typeof scipData.metadata.toolName).toBe("string"); + } + } + }); + + test("should parse external symbols", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + + expect(Array.isArray(scipData.externalSymbols)).toBe(true); + + if (scipData.externalSymbols.length > 0) { + const extSym = scipData.externalSymbols[0]; + expect(extSym).toHaveProperty("symbol"); + expect(extSym).toHaveProperty("kind"); + expect(typeof extSym.symbol).toBe("string"); + expect(typeof extSym.kind).toBe("number"); + } + }); + + test("should correctly identify local symbols", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + const maps = buildLookupMaps(scipData); + + // Find a local symbol (contains 'local' in symbol identifier) + let foundLocalSymbol = false; + for (const doc of scipData.documents) { + const definitions = extractDefinitions(doc); + for (const def of definitions) { + if (def.symbol.includes("local")) { + foundLocalSymbol = true; + + // Local symbols should be defined in the same file + const defFile = findDefinitionFile({ + symbol: def.symbol, + documents: maps.documentsByPath, + symbols: maps.symbolsById, + }); + + // Should find in current document or return null + expect(defFile === null || defFile === doc.relativePath).toBe(true); + break; + } + } + if (foundLocalSymbol) break; + } + }); + + test("should handle documents with no symbols", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + + // Find or create a document with no symbols + const emptyDoc: ParsedDocument = { + relativePath: "test/empty.ts", + language: "TypeScript", + occurrences: [], + symbols: [], + }; + + const definitions = extractDefinitions(emptyDoc); + const references = extractReferences(emptyDoc); + + expect(definitions.length).toBe(0); + expect(references.length).toBe(0); + + const maps = buildLookupMaps(scipData); + const docSymbols = getDocumentSymbols({ + doc: emptyDoc, + symbolsById: maps.symbolsById, + }); + expect(docSymbols.length).toBe(0); + + const deps = getFileDependencies({ + doc: emptyDoc, + documentsByPath: maps.documentsByPath, + symbolsById: maps.symbolsById, + definitionsBySymbol: maps.definitionsBySymbol, + }); + expect(deps.size).toBe(0); + }); + + test("should handle 3-element ranges (same line)", async () => { + if (skipTests) { + console.log("Skipping test: example SCIP file not found"); + return; + } + + const scipData = await parseScipFile(exampleScipPath); + + // This is tested implicitly in the parsing - if we got here without errors, + // the parser handled both 3 and 4 element ranges correctly + expect(scipData.documents.length).toBeGreaterThan(0); + }); }); diff --git a/test/db/queries.test.ts b/test/db/queries.test.ts index e40f70d..611165d 100644 --- a/test/db/queries.test.ts +++ b/test/db/queries.test.ts @@ -3,23 +3,23 @@ import { Database } from "bun:sqlite"; import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import { - getDependencies, - getFileDependencies, - getFileDependents, - getFileSymbols, - getReverseDependencies, - searchSymbols, + getDependencies, + getFileDependencies, + getFileDependents, + getFileSymbols, + getReverseDependencies, + searchSymbols, } from "../../src/db/queries.ts"; describe("Database Queries", () => { - let db: Database; + let db: Database; - beforeAll(() => { - // Create in-memory test database - db = new Database(":memory:"); + beforeAll(() => { + // Create in-memory test database + db = new Database(":memory:"); - // Create schema - db.exec(` + // Create schema + db.exec(` CREATE TABLE files ( id INTEGER PRIMARY KEY, path TEXT UNIQUE NOT NULL, @@ -75,181 +75,181 @@ describe("Database Queries", () => { ); `); - // Insert test files - db.run( - "INSERT INTO files (id, path, language, mtime, indexed_at) VALUES (1, 'packages/app-utils/src/index.ts', 'typescript', 1000, 1000)", - ); - db.run( - "INSERT INTO files (id, path, language, mtime, indexed_at) VALUES (2, 'packages/app-utils/src/logger.ts', 'typescript', 1000, 1000)", - ); - db.run( - "INSERT INTO files (id, path, language, mtime, indexed_at) VALUES (3, 'apps/api-worker/src/index.ts', 'typescript', 1000, 1000)", - ); - db.run( - "INSERT INTO files (id, path, language, mtime, indexed_at) VALUES (4, 'packages/app-auth/src/index.ts', 'typescript', 1000, 1000)", - ); - - // Insert test symbols (all non-local symbols, so is_local = 0) - db.run(`INSERT INTO symbols (id, file_id, name, scip_symbol, kind, start_line, end_line, start_char, end_char, package, is_local) + // Insert test files + db.run( + "INSERT INTO files (id, path, language, mtime, indexed_at) VALUES (1, 'packages/app-utils/src/index.ts', 'typescript', 1000, 1000)" + ); + db.run( + "INSERT INTO files (id, path, language, mtime, indexed_at) VALUES (2, 'packages/app-utils/src/logger.ts', 'typescript', 1000, 1000)" + ); + db.run( + "INSERT INTO files (id, path, language, mtime, indexed_at) VALUES (3, 'apps/api-worker/src/index.ts', 'typescript', 1000, 1000)" + ); + db.run( + "INSERT INTO files (id, path, language, mtime, indexed_at) VALUES (4, 'packages/app-auth/src/index.ts', 'typescript', 1000, 1000)" + ); + + // Insert test symbols (all non-local symbols, so is_local = 0) + db.run(`INSERT INTO symbols (id, file_id, name, scip_symbol, kind, start_line, end_line, start_char, end_char, package, is_local) VALUES (1, 1, 'exportUtils', 'scip-typescript npm @pkg/app-utils 1.0.0 src/index.ts/exportUtils.', 'function', 5, 10, 0, 1, '@pkg/app-utils', 0)`); - db.run(`INSERT INTO symbols (id, file_id, name, scip_symbol, kind, start_line, end_line, start_char, end_char, package, is_local) + db.run(`INSERT INTO symbols (id, file_id, name, scip_symbol, kind, start_line, end_line, start_char, end_char, package, is_local) VALUES (2, 2, 'Logger', 'scip-typescript npm @pkg/app-utils 1.0.0 src/logger.ts/Logger#', 'interface', 3, 8, 0, 1, '@pkg/app-utils', 0)`); - db.run(`INSERT INTO symbols (id, file_id, name, scip_symbol, kind, start_line, end_line, start_char, end_char, package, is_local) + db.run(`INSERT INTO symbols (id, file_id, name, scip_symbol, kind, start_line, end_line, start_char, end_char, package, is_local) VALUES (3, 3, 'main', 'scip-typescript npm @pkg/api-worker 1.0.0 src/index.ts/main.', 'function', 1, 5, 0, 1, '@pkg/api-worker', 0)`); - db.run(`INSERT INTO symbols (id, file_id, name, scip_symbol, kind, start_line, end_line, start_char, end_char, package, is_local) + db.run(`INSERT INTO symbols (id, file_id, name, scip_symbol, kind, start_line, end_line, start_char, end_char, package, is_local) VALUES (4, 4, 'AuthSession', 'scip-typescript npm @pkg/app-auth 1.0.0 src/index.ts/AuthSession#', 'type', 2, 6, 0, 1, '@pkg/app-auth', 0)`); - // Insert dependencies - db.run(`INSERT INTO dependencies (from_file_id, to_file_id, symbol_count, symbols) + // Insert dependencies + db.run(`INSERT INTO dependencies (from_file_id, to_file_id, symbol_count, symbols) VALUES (1, 2, 1, '["Logger"]')`); - db.run(`INSERT INTO dependencies (from_file_id, to_file_id, symbol_count, symbols) + db.run(`INSERT INTO dependencies (from_file_id, to_file_id, symbol_count, symbols) VALUES (3, 1, 1, '["exportUtils"]')`); - db.run(`INSERT INTO dependencies (from_file_id, to_file_id, symbol_count, symbols) + db.run(`INSERT INTO dependencies (from_file_id, to_file_id, symbol_count, symbols) VALUES (3, 2, 1, '["Logger"]')`); - db.run(`INSERT INTO dependencies (from_file_id, to_file_id, symbol_count, symbols) + db.run(`INSERT INTO dependencies (from_file_id, to_file_id, symbol_count, symbols) VALUES (3, 4, 1, '["AuthSession"]')`); - // Insert symbol references - db.run( - "INSERT INTO symbol_references (symbol_id, file_id, line) VALUES (2, 3, 10)", - ); - db.run( - "INSERT INTO symbol_references (symbol_id, file_id, line) VALUES (1, 3, 15)", - ); - - // Insert packages - db.run( - "INSERT INTO packages (name, manager, symbol_count) VALUES ('@pkg/app-utils', 'npm', 2)", - ); - db.run( - "INSERT INTO packages (name, manager, symbol_count) VALUES ('@pkg/api-worker', 'npm', 1)", - ); - db.run( - "INSERT INTO packages (name, manager, symbol_count) VALUES ('@pkg/app-auth', 'npm', 1)", - ); - }); - - afterAll(() => { - db.close(); - }); - - // Note: Trivial queries (getFileCount, getSymbolCount, getPackages) are not tested - // They're simple SELECT COUNT(*) or SELECT name queries with no complex logic - - describe("File Queries", () => { - test("getFileSymbols should return symbols for a file", () => { - const symbols = getFileSymbols(db, "packages/app-utils/src/index.ts"); - - expect(Array.isArray(symbols)).toBe(true); - // File should have at least one symbol (module) - expect(symbols.length).toBeGreaterThanOrEqual(1); - - if (symbols.length > 0) { - expect(symbols[0]).toHaveProperty("name"); - expect(symbols[0]).toHaveProperty("kind"); - expect(symbols[0]).toHaveProperty("lines"); - } - }); - - test("getFileDependencies should return dependencies", () => { - const deps = getFileDependencies(db, "packages/app-utils/src/index.ts"); - - expect(Array.isArray(deps)).toBe(true); - - if (deps.length > 0) { - expect(deps[0]).toHaveProperty("path"); - } - }); - - test("getFileDependents should return dependents", () => { - const dependents = getFileDependents( - db, - "packages/app-utils/src/logger.ts", - ); - - expect(Array.isArray(dependents)).toBe(true); - // logger.ts is used by other files, so should have dependents - expect(dependents.length).toBeGreaterThan(0); - - expect(dependents[0]).toHaveProperty("path"); - expect(dependents[0]).toHaveProperty("refs"); - expect(typeof dependents[0].refs).toBe("number"); - }); - }); - - describe("Symbol Queries", () => { - test("searchSymbols should find symbols by name", () => { - const results = searchSymbols(db, "Logger", { limit: 10 }); - - expect(Array.isArray(results)).toBe(true); - - if (results.length > 0) { - expect(results[0]).toHaveProperty("name"); - expect(results[0]).toHaveProperty("kind"); - expect(results[0]).toHaveProperty("path"); - } - }); - - test("searchSymbols should respect limit option", () => { - const results = searchSymbols(db, "index", { limit: 5 }); - - expect(results.length).toBeLessThanOrEqual(5); - }); - - test("searchSymbols should return empty array for non-existent symbol", () => { - const results = searchSymbols(db, "NonExistentSymbolXYZ123", { - limit: 10, - }); - - expect(Array.isArray(results)).toBe(true); - expect(results.length).toBe(0); - }); - }); - - describe("Dependency Graph Queries", () => { - test("getDependencies should return dependencies at depth 1", () => { - const deps = getDependencies(db, "packages/app-utils/src/index.ts", 1); - - expect(Array.isArray(deps)).toBe(true); - // Should have logger.ts as dependency - expect(deps.length).toBeGreaterThan(0); - - if (deps.length > 0) { - expect(deps[0]).toHaveProperty("path"); - expect(deps[0]).toHaveProperty("depth"); - expect(deps[0].depth).toBe(1); - } - }); - - test("getDependencies should handle depth 2", () => { - const deps = getDependencies(db, "apps/api-worker/src/index.ts", 2); - - expect(Array.isArray(deps)).toBe(true); - // api-worker depends on multiple files at different depths - expect(deps.length).toBeGreaterThan(0); - }); - - test("getReverseDependencies should return dependents", () => { - const rdeps = getReverseDependencies( - db, - "packages/app-utils/src/logger.ts", - 1, - ); - - expect(Array.isArray(rdeps)).toBe(true); - expect(rdeps.length).toBeGreaterThan(0); - - expect(rdeps[0]).toHaveProperty("path"); - expect(rdeps[0]).toHaveProperty("depth"); - expect(rdeps[0].depth).toBe(1); - }); - - test("getDependencies should return empty array for file with no deps", () => { - const deps = getDependencies(db, "packages/app-utils/src/logger.ts", 1); - - expect(Array.isArray(deps)).toBe(true); - // logger.ts has no dependencies in our test data - expect(deps.length).toBe(0); - }); - }); + // Insert symbol references + db.run( + "INSERT INTO symbol_references (symbol_id, file_id, line) VALUES (2, 3, 10)" + ); + db.run( + "INSERT INTO symbol_references (symbol_id, file_id, line) VALUES (1, 3, 15)" + ); + + // Insert packages + db.run( + "INSERT INTO packages (name, manager, symbol_count) VALUES ('@pkg/app-utils', 'npm', 2)" + ); + db.run( + "INSERT INTO packages (name, manager, symbol_count) VALUES ('@pkg/api-worker', 'npm', 1)" + ); + db.run( + "INSERT INTO packages (name, manager, symbol_count) VALUES ('@pkg/app-auth', 'npm', 1)" + ); + }); + + afterAll(() => { + db.close(); + }); + + // Note: Trivial queries (getFileCount, getSymbolCount, getPackages) are not tested + // They're simple SELECT COUNT(*) or SELECT name queries with no complex logic + + describe("File Queries", () => { + test("getFileSymbols should return symbols for a file", () => { + const symbols = getFileSymbols(db, "packages/app-utils/src/index.ts"); + + expect(Array.isArray(symbols)).toBe(true); + // File should have at least one symbol (module) + expect(symbols.length).toBeGreaterThanOrEqual(1); + + if (symbols.length > 0) { + expect(symbols[0]).toHaveProperty("name"); + expect(symbols[0]).toHaveProperty("kind"); + expect(symbols[0]).toHaveProperty("lines"); + } + }); + + test("getFileDependencies should return dependencies", () => { + const deps = getFileDependencies(db, "packages/app-utils/src/index.ts"); + + expect(Array.isArray(deps)).toBe(true); + + if (deps.length > 0) { + expect(deps[0]).toHaveProperty("path"); + } + }); + + test("getFileDependents should return dependents", () => { + const dependents = getFileDependents( + db, + "packages/app-utils/src/logger.ts" + ); + + expect(Array.isArray(dependents)).toBe(true); + // logger.ts is used by other files, so should have dependents + expect(dependents.length).toBeGreaterThan(0); + + expect(dependents[0]).toHaveProperty("path"); + expect(dependents[0]).toHaveProperty("refs"); + expect(typeof dependents[0].refs).toBe("number"); + }); + }); + + describe("Symbol Queries", () => { + test("searchSymbols should find symbols by name", () => { + const results = searchSymbols(db, "Logger", { limit: 10 }); + + expect(Array.isArray(results)).toBe(true); + + if (results.length > 0) { + expect(results[0]).toHaveProperty("name"); + expect(results[0]).toHaveProperty("kind"); + expect(results[0]).toHaveProperty("path"); + } + }); + + test("searchSymbols should respect limit option", () => { + const results = searchSymbols(db, "index", { limit: 5 }); + + expect(results.length).toBeLessThanOrEqual(5); + }); + + test("searchSymbols should return empty array for non-existent symbol", () => { + const results = searchSymbols(db, "NonExistentSymbolXYZ123", { + limit: 10, + }); + + expect(Array.isArray(results)).toBe(true); + expect(results.length).toBe(0); + }); + }); + + describe("Dependency Graph Queries", () => { + test("getDependencies should return dependencies at depth 1", () => { + const deps = getDependencies(db, "packages/app-utils/src/index.ts", 1); + + expect(Array.isArray(deps)).toBe(true); + // Should have logger.ts as dependency + expect(deps.length).toBeGreaterThan(0); + + if (deps.length > 0) { + expect(deps[0]).toHaveProperty("path"); + expect(deps[0]).toHaveProperty("depth"); + expect(deps[0].depth).toBe(1); + } + }); + + test("getDependencies should handle depth 2", () => { + const deps = getDependencies(db, "apps/api-worker/src/index.ts", 2); + + expect(Array.isArray(deps)).toBe(true); + // api-worker depends on multiple files at different depths + expect(deps.length).toBeGreaterThan(0); + }); + + test("getReverseDependencies should return dependents", () => { + const rdeps = getReverseDependencies( + db, + "packages/app-utils/src/logger.ts", + 1 + ); + + expect(Array.isArray(rdeps)).toBe(true); + expect(rdeps.length).toBeGreaterThan(0); + + expect(rdeps[0]).toHaveProperty("path"); + expect(rdeps[0]).toHaveProperty("depth"); + expect(rdeps[0].depth).toBe(1); + }); + + test("getDependencies should return empty array for file with no deps", () => { + const deps = getDependencies(db, "packages/app-utils/src/logger.ts", 1); + + expect(Array.isArray(deps)).toBe(true); + // logger.ts has no dependencies in our test data + expect(deps.length).toBe(0); + }); + }); }); diff --git a/test/utils/fileScanner.test.ts b/test/utils/fileScanner.test.ts index cc00fc2..8e0f7db 100644 --- a/test/utils/fileScanner.test.ts +++ b/test/utils/fileScanner.test.ts @@ -4,102 +4,105 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import { mkdir, rm, writeFile } from "fs/promises"; import { join } from "path"; import { - filterChangedDocuments, - scanDocumentFiles, + filterChangedDocuments, + scanDocumentFiles, } from "../../src/utils/fileScanner.ts"; describe("File Scanner", () => { - const testDir = join(process.cwd(), "test", "fixtures", "test-repo"); - - beforeAll(async () => { - // Create test directory structure - await mkdir(testDir, { recursive: true }); - await mkdir(join(testDir, "docs"), { recursive: true }); - await mkdir(join(testDir, "node_modules"), { recursive: true }); - await mkdir(join(testDir, "src"), { recursive: true }); - - // Create test files - await writeFile(join(testDir, "README.md"), "# Test"); - await writeFile(join(testDir, "docs", "guide.md"), "# Guide"); - await writeFile(join(testDir, "node_modules", "foo.md"), "# Foo"); - await writeFile(join(testDir, "notes.txt"), "Project notes"); - - // Create .gitignore - await writeFile(join(testDir, ".gitignore"), "node_modules/\n*.log\n"); - }); - - afterAll(async () => { - // Clean up test directory - await rm(testDir, { recursive: true, force: true }); - }); - - test("should scan and find document files", async () => { - const docs = await scanDocumentFiles(testDir); - - expect(docs.length).toBeGreaterThan(0); - expect(docs.some((d) => d.path.endsWith("README.md"))).toBe(true); - expect(docs.some((d) => d.path.endsWith("guide.md"))).toBe(true); - expect(docs.some((d) => d.path.endsWith("notes.txt"))).toBe(true); - }); - - test("should respect .gitignore rules", async () => { - const docs = await scanDocumentFiles(testDir); - - // Should not include files in node_modules - expect(docs.some((d) => d.path.includes("node_modules"))).toBe(false); - }); - - test("should include file metadata", async () => { - const docs = await scanDocumentFiles(testDir); - const readme = docs.find((d) => d.path.endsWith("README.md")); - - expect(readme).toBeDefined(); - expect(readme?.mtime).toBeGreaterThan(0); - expect(readme?.type).toBe("md"); - }); - - test("should filter by custom extensions", async () => { - const docs = await scanDocumentFiles(testDir, [".md"]); - - expect(docs.every((d) => d.type === "md")).toBe(true); - expect(docs.some((d) => d.type === "txt")).toBe(false); - }); - - test("should filter changed documents", () => { - const existingDocs = new Map([ - ["README.md", 1000], - ["docs/guide.md", 2000], - ]); - - const scannedDocs = [ - { path: "README.md", mtime: 1000, type: "md" }, // unchanged - { path: "docs/guide.md", mtime: 3000, type: "md" }, // modified - { path: "NEW.md", mtime: 4000, type: "md" }, // new - ]; - - const changed = filterChangedDocuments(existingDocs, scannedDocs); - - expect(changed.length).toBe(2); - expect(changed.some((d) => d.path === "docs/guide.md")).toBe(true); - expect(changed.some((d) => d.path === "NEW.md")).toBe(true); - expect(changed.some((d) => d.path === "README.md")).toBe(false); - }); - - test("should scan and find .txt files", async () => { - const docs = await scanDocumentFiles(testDir); - - // Should include .txt files - expect(docs.some((d) => d.path.endsWith("notes.txt"))).toBe(true); - - const txtFile = docs.find((d) => d.path.endsWith("notes.txt")); - expect(txtFile?.type).toBe("txt"); - }); - - test("should include .txt in default extensions", async () => { - const docs = await scanDocumentFiles(testDir); - - // Verify that .txt files are scanned by default - const txtDocs = docs.filter((d) => d.type === "txt"); - expect(txtDocs.length).toBeGreaterThan(0); - }); + const testDir = join(process.cwd(), "test", "fixtures", "test-repo"); + + beforeAll(async () => { + // Create test directory structure + await mkdir(testDir, { recursive: true }); + await mkdir(join(testDir, "docs"), { recursive: true }); + await mkdir(join(testDir, "node_modules"), { recursive: true }); + await mkdir(join(testDir, "src"), { recursive: true }); + + // Create test files + await writeFile(join(testDir, "README.md"), "# Test"); + await writeFile(join(testDir, "docs", "guide.md"), "# Guide"); + await writeFile(join(testDir, "node_modules", "foo.md"), "# Foo"); + await writeFile(join(testDir, "notes.txt"), "Project notes"); + + // Create .gitignore + await writeFile(join(testDir, ".gitignore"), "node_modules/\n*.log\n"); + }); + + afterAll(async () => { + // Clean up test directory + await rm(testDir, { recursive: true, force: true }); + }); + + test("should scan and find document files", async () => { + const docs = await scanDocumentFiles({ repoRoot: testDir }); + + expect(docs.length).toBeGreaterThan(0); + expect(docs.some((d) => d.path.endsWith("README.md"))).toBe(true); + expect(docs.some((d) => d.path.endsWith("guide.md"))).toBe(true); + expect(docs.some((d) => d.path.endsWith("notes.txt"))).toBe(true); + }); + + test("should respect .gitignore rules", async () => { + const docs = await scanDocumentFiles({ repoRoot: testDir }); + + // Should not include files in node_modules + expect(docs.some((d) => d.path.includes("node_modules"))).toBe(false); + }); + + test("should include file metadata", async () => { + const docs = await scanDocumentFiles({ repoRoot: testDir }); + const readme = docs.find((d) => d.path.endsWith("README.md")); + + expect(readme).toBeDefined(); + expect(readme?.mtime).toBeGreaterThan(0); + expect(readme?.type).toBe("md"); + }); + + test("should filter by custom extensions", async () => { + const docs = await scanDocumentFiles({ + repoRoot: testDir, + extensions: [".md"], + }); + + expect(docs.every((d) => d.type === "md")).toBe(true); + expect(docs.some((d) => d.type === "txt")).toBe(false); + }); + + test("should filter changed documents", () => { + const existingDocs = new Map([ + ["README.md", 1000], + ["docs/guide.md", 2000], + ]); + + const scannedDocs = [ + { path: "README.md", mtime: 1000, type: "md" }, // unchanged + { path: "docs/guide.md", mtime: 3000, type: "md" }, // modified + { path: "NEW.md", mtime: 4000, type: "md" }, // new + ]; + + const changed = filterChangedDocuments({ existingDocs, scannedDocs }); + + expect(changed.length).toBe(2); + expect(changed.some((d) => d.path === "docs/guide.md")).toBe(true); + expect(changed.some((d) => d.path === "NEW.md")).toBe(true); + expect(changed.some((d) => d.path === "README.md")).toBe(false); + }); + + test("should scan and find .txt files", async () => { + const docs = await scanDocumentFiles({ repoRoot: testDir }); + + // Should include .txt files + expect(docs.some((d) => d.path.endsWith("notes.txt"))).toBe(true); + + const txtFile = docs.find((d) => d.path.endsWith("notes.txt")); + expect(txtFile?.type).toBe("txt"); + }); + + test("should include .txt in default extensions", async () => { + const docs = await scanDocumentFiles({ repoRoot: testDir }); + + // Verify that .txt files are scanned by default + const txtDocs = docs.filter((d) => d.type === "txt"); + expect(txtDocs.length).toBeGreaterThan(0); + }); }); From b46d898431e71e5d64dbd54f449a690f9efb4c41 Mon Sep 17 00:00:00 2001 From: Yash Date: Sat, 24 Jan 2026 13:39:44 +0700 Subject: [PATCH 3/4] feat(init): add --language flag to skip project detection Use explicit language flag to init in any directory without requiring project files. Co-Authored-By: Claude Sonnet 4.5 --- src/commands/init.ts | 27 ++++--- src/index.ts | 6 +- src/utils/config.ts | 91 ++++++++++++++++----- test/commands/commands.test.ts | 87 ++++++++++++++++++++- test/utils/config.test.ts | 139 +++++++++++++++++++++++++++++---- 5 files changed, 302 insertions(+), 48 deletions(-) diff --git a/src/commands/init.ts b/src/commands/init.ts index 99aa8cc..6554095 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -2,6 +2,7 @@ import { existsSync } from "fs"; import { join } from "path"; import type { InitResult } from "../types.ts"; import { + LanguageSchema, createDefaultConfig, isInitialized, saveConfig, @@ -11,29 +12,35 @@ import { outputJson } from "../utils/output.ts"; import { findRepoRoot, getConfigPath, getDoraDir } from "../utils/paths.ts"; import { copyTemplates } from "../utils/templates.ts"; -export async function init(): Promise { - // Find repository root - const root = await findRepoRoot(); +export async function init(params?: { language?: string }): Promise { + if (params?.language) { + const result = LanguageSchema.safeParse(params.language); + if (!result.success) { + throw new CtxError( + `Invalid language: ${params.language}. Valid options are: ${LanguageSchema.options.join(", ")}`, + ); + } + } + + const root = params?.language ? process.cwd() : await findRepoRoot(); - // Check if already initialized if (isInitialized(root)) { throw new CtxError( `Repository already initialized. Config exists at ${getConfigPath(root)}`, ); } - // Create .dora directory const doraDir = getDoraDir(root); await Bun.write(join(doraDir, ".gitkeep"), ""); - // Copy template files (docs) await copyTemplates(doraDir); - // Add .dora to .gitignore await addToGitignore(root); - // Create and save initial config - const config = createDefaultConfig(root); + const config = createDefaultConfig({ + root, + language: params?.language, + }); await saveConfig(config); const result: InitResult = { @@ -56,12 +63,10 @@ async function addToGitignore(root: string): Promise { content = await Bun.file(gitignorePath).text(); } - // Check if .dora is already in .gitignore if (content.includes(".dora")) { return; } - // Add .dora to .gitignore const newContent = content.trim() ? `${content.trim()}\n\n# dora code context index\n.dora/\n` : `# dora code context index\n.dora/\n`; diff --git a/src/index.ts b/src/index.ts index bcbe297..d03468e 100755 --- a/src/index.ts +++ b/src/index.ts @@ -42,7 +42,11 @@ program program .command("init") .description("Initialize dora in the current repository") - .action(wrapCommand(init)); + .option( + "-l, --language ", + "Project language (typescript, javascript, python, rust, go, java)", + ) + .action(wrapCommand((options) => init({ language: options.language }))); program .command("index") diff --git a/src/utils/config.ts b/src/utils/config.ts index dba197b..71d5e7c 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -22,10 +22,20 @@ const IndexStateSchema = z.object({ databaseMtime: z.number(), }); +export const LanguageSchema = z.enum([ + "typescript", + "javascript", + "python", + "rust", + "go", + "java", +]); + const ConfigSchema = z.object({ root: z.string().min(1), scip: z.string().min(1), db: z.string().min(1), + language: LanguageSchema.optional(), commands: z .object({ index: z.string().optional(), @@ -136,7 +146,6 @@ function detectWorkspaceType(root: string): "bun" | "pnpm" | "yarn" | null { return "yarn"; } } catch { - // Ignore JSON parse errors } } @@ -146,38 +155,79 @@ function detectWorkspaceType(root: string): "bun" | "pnpm" | "yarn" | null { /** * Detect project type and return appropriate SCIP indexer command */ -function detectIndexerCommand(root: string): string { +function detectIndexerCommand(params: { + root: string; + language?: string; +}): string { + const { root, language } = params; + + if (language) { + switch (language) { + case "typescript": + case "javascript": { + const workspaceType = detectWorkspaceType(root); + const needsInferTsConfig = language === "javascript"; + + if (workspaceType === "bun") { + return needsInferTsConfig + ? "scip-typescript index --infer-tsconfig --output .dora/index.scip" + : "scip-typescript index --output .dora/index.scip"; + } + if (workspaceType === "pnpm") { + return needsInferTsConfig + ? "scip-typescript index --infer-tsconfig --pnpm-workspaces --output .dora/index.scip" + : "scip-typescript index --pnpm-workspaces --output .dora/index.scip"; + } + if (workspaceType === "yarn") { + return needsInferTsConfig + ? "scip-typescript index --infer-tsconfig --yarn-workspaces --output .dora/index.scip" + : "scip-typescript index --yarn-workspaces --output .dora/index.scip"; + } + return needsInferTsConfig + ? "scip-typescript index --infer-tsconfig --output .dora/index.scip" + : "scip-typescript index --output .dora/index.scip"; + } + case "python": + return "scip-python index --output .dora/index.scip"; + case "rust": + return "rust-analyzer scip . --output .dora/index.scip"; + case "go": + return "scip-go --output .dora/index.scip"; + case "java": + return "scip-java index --output .dora/index.scip"; + default: + return "scip-typescript index --output .dora/index.scip"; + } + } + const hasTsConfig = existsSync(join(root, "tsconfig.json")); const hasPackageJson = existsSync(join(root, "package.json")); - // TypeScript/JavaScript projects if (hasTsConfig || hasPackageJson) { const workspaceType = detectWorkspaceType(root); - // For JavaScript projects (no tsconfig.json), add --infer-tsconfig flag const needsInferTsConfig = !hasTsConfig && hasPackageJson; - // Build command based on workspace type if (workspaceType === "bun") { return needsInferTsConfig ? "scip-typescript index --infer-tsconfig --output .dora/index.scip" : "scip-typescript index --output .dora/index.scip"; - } else if (workspaceType === "pnpm") { + } + if (workspaceType === "pnpm") { return needsInferTsConfig ? "scip-typescript index --infer-tsconfig --pnpm-workspaces --output .dora/index.scip" : "scip-typescript index --pnpm-workspaces --output .dora/index.scip"; - } else if (workspaceType === "yarn") { + } + if (workspaceType === "yarn") { return needsInferTsConfig ? "scip-typescript index --infer-tsconfig --yarn-workspaces --output .dora/index.scip" : "scip-typescript index --yarn-workspaces --output .dora/index.scip"; - } else { - return needsInferTsConfig - ? "scip-typescript index --infer-tsconfig --output .dora/index.scip" - : "scip-typescript index --output .dora/index.scip"; } + return needsInferTsConfig + ? "scip-typescript index --infer-tsconfig --output .dora/index.scip" + : "scip-typescript index --output .dora/index.scip"; } - // Python - check for Python project files if ( existsSync(join(root, "setup.py")) || existsSync(join(root, "pyproject.toml")) || @@ -186,17 +236,14 @@ function detectIndexerCommand(root: string): string { return "scip-python index --output .dora/index.scip"; } - // Rust - check for Cargo.toml if (existsSync(join(root, "Cargo.toml"))) { return "rust-analyzer scip . --output .dora/index.scip"; } - // Go - check for go.mod if (existsSync(join(root, "go.mod"))) { return "scip-go --output .dora/index.scip"; } - // Java - check for Maven or Gradle if ( existsSync(join(root, "pom.xml")) || existsSync(join(root, "build.gradle")) || @@ -205,20 +252,26 @@ function detectIndexerCommand(root: string): string { return "scip-java index --output .dora/index.scip"; } - // Default to TypeScript (most common) return "scip-typescript index --output .dora/index.scip"; } /** * Create default configuration */ -export function createDefaultConfig(root: string): Config { - const indexCommand = detectIndexerCommand(root); +export function createDefaultConfig(params: { + root: string; + language?: string; +}): Config { + const indexCommand = detectIndexerCommand({ + root: params.root, + language: params.language, + }); return { - root, + root: params.root, scip: ".dora/index.scip", db: ".dora/dora.db", + language: params.language, commands: { index: indexCommand, }, diff --git a/test/commands/commands.test.ts b/test/commands/commands.test.ts index 22051ef..7d147ca 100644 --- a/test/commands/commands.test.ts +++ b/test/commands/commands.test.ts @@ -1,6 +1,13 @@ // Integration tests for commands -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + test, +} from "bun:test"; import { existsSync, mkdirSync, rmSync, writeFileSync } from "fs"; import { join } from "path"; import { init } from "../../src/commands/init.ts"; @@ -93,6 +100,84 @@ describe("Commands Integration Tests", () => { }); }); + describe("init command with language flag", () => { + const langTestDir = "/tmp/ctx-lang-test-" + Date.now(); + let savedCwd: string; + + beforeAll(() => { + savedCwd = process.cwd(); + mkdirSync(langTestDir, { recursive: true }); + }); + + afterEach(() => { + process.chdir(savedCwd); + if (existsSync(join(langTestDir, ".dora"))) { + rmSync(join(langTestDir, ".dora"), { recursive: true, force: true }); + } + }); + + afterAll(() => { + process.chdir(savedCwd); + if (existsSync(langTestDir)) { + rmSync(langTestDir, { recursive: true, force: true }); + } + }); + + test("should initialize with explicit python language", async () => { + process.chdir(langTestDir); + + captureOutput(); + await init({ language: "python" }); + restoreOutput(); + + expect(capturedOutput).toHaveProperty("success", true); + + const configPath = join(langTestDir, ".dora", "config.json"); + const config = JSON.parse(await Bun.file(configPath).text()); + + expect(config.language).toBe("python"); + expect(config.commands.index).toBe( + "scip-python index --output .dora/index.scip", + ); + }); + + test("should initialize with explicit rust language", async () => { + process.chdir(langTestDir); + + captureOutput(); + await init({ language: "rust" }); + restoreOutput(); + + expect(capturedOutput).toHaveProperty("success", true); + + const configPath = join(langTestDir, ".dora", "config.json"); + const config = JSON.parse(await Bun.file(configPath).text()); + + expect(config.language).toBe("rust"); + expect(config.commands.index).toBe( + "rust-analyzer scip . --output .dora/index.scip", + ); + }); + + test("should fail with invalid language", async () => { + process.chdir(langTestDir); + + captureOutput(); + + let error: Error | null = null; + try { + await init({ language: "invalid" as any }); + } catch (e) { + error = e as Error; + } + restoreOutput(); + + expect(error).not.toBeNull(); + expect(error?.message).toContain("Invalid language"); + expect(error?.message).toContain("invalid"); + }); + }); + describe("status command", () => { test("should show initialized but not indexed", async () => { captureOutput(); diff --git a/test/utils/config.test.ts b/test/utils/config.test.ts index 4451024..0756952 100644 --- a/test/utils/config.test.ts +++ b/test/utils/config.test.ts @@ -101,7 +101,7 @@ describe("Config Management", () => { describe("createDefaultConfig", () => { test("should create default config with correct structure", () => { const root = "/Users/test/repo"; - const config = createDefaultConfig(root); + const config = createDefaultConfig({ root }); expect(config.root).toBe(root); expect(config.db).toBe(".dora/dora.db"); @@ -113,7 +113,7 @@ describe("Config Management", () => { test("should create config with absolute root path", () => { const root = "/absolute/path/to/repo"; - const config = createDefaultConfig(root); + const config = createDefaultConfig({ root }); expect(config.root).toBe(root); }); @@ -138,13 +138,12 @@ describe("Config Management", () => { }); test("should add --infer-tsconfig for JavaScript-only project", async () => { - // Create package.json only (no tsconfig.json) await Bun.write( `${tempDir}/package.json`, JSON.stringify({ name: "test" }), ); - const config = createDefaultConfig(tempDir); + const config = createDefaultConfig({ root: tempDir }); expect(config.commands?.index).toBe( "scip-typescript index --infer-tsconfig --output .dora/index.scip", @@ -152,7 +151,6 @@ describe("Config Management", () => { }); test("should NOT add --infer-tsconfig for TypeScript project", async () => { - // Create both package.json and tsconfig.json await Bun.write( `${tempDir}/package.json`, JSON.stringify({ name: "test" }), @@ -162,7 +160,7 @@ describe("Config Management", () => { JSON.stringify({ compilerOptions: {} }), ); - const config = createDefaultConfig(tempDir); + const config = createDefaultConfig({ root: tempDir }); expect(config.commands?.index).toBe( "scip-typescript index --output .dora/index.scip", @@ -170,7 +168,6 @@ describe("Config Management", () => { }); test("should add --infer-tsconfig for JavaScript + pnpm workspace", async () => { - // Create package.json and pnpm-workspace.yaml (no tsconfig.json) await Bun.write( `${tempDir}/package.json`, JSON.stringify({ name: "test" }), @@ -180,7 +177,7 @@ describe("Config Management", () => { "packages:\n - packages/*", ); - const config = createDefaultConfig(tempDir); + const config = createDefaultConfig({ root: tempDir }); expect(config.commands?.index).toBe( "scip-typescript index --infer-tsconfig --pnpm-workspaces --output .dora/index.scip", @@ -188,7 +185,6 @@ describe("Config Management", () => { }); test("should NOT add --infer-tsconfig for TypeScript + pnpm workspace", async () => { - // Create package.json, tsconfig.json, and pnpm-workspace.yaml await Bun.write( `${tempDir}/package.json`, JSON.stringify({ name: "test" }), @@ -202,7 +198,7 @@ describe("Config Management", () => { "packages:\n - packages/*", ); - const config = createDefaultConfig(tempDir); + const config = createDefaultConfig({ root: tempDir }); expect(config.commands?.index).toBe( "scip-typescript index --pnpm-workspaces --output .dora/index.scip", @@ -210,13 +206,12 @@ describe("Config Management", () => { }); test("should add --infer-tsconfig for JavaScript + yarn workspace", async () => { - // Create package.json with workspaces (no tsconfig.json) await Bun.write( `${tempDir}/package.json`, JSON.stringify({ name: "test", workspaces: ["packages/*"] }), ); - const config = createDefaultConfig(tempDir); + const config = createDefaultConfig({ root: tempDir }); expect(config.commands?.index).toBe( "scip-typescript index --infer-tsconfig --yarn-workspaces --output .dora/index.scip", @@ -224,7 +219,6 @@ describe("Config Management", () => { }); test("should NOT add --infer-tsconfig for TypeScript + yarn workspace", async () => { - // Create package.json with workspaces and tsconfig.json await Bun.write( `${tempDir}/package.json`, JSON.stringify({ name: "test", workspaces: ["packages/*"] }), @@ -234,7 +228,7 @@ describe("Config Management", () => { JSON.stringify({ compilerOptions: {} }), ); - const config = createDefaultConfig(tempDir); + const config = createDefaultConfig({ root: tempDir }); expect(config.commands?.index).toBe( "scip-typescript index --yarn-workspaces --output .dora/index.scip", @@ -242,17 +236,130 @@ describe("Config Management", () => { }); test("should handle tsconfig.json only (no package.json)", async () => { - // Create tsconfig.json only await Bun.write( `${tempDir}/tsconfig.json`, JSON.stringify({ compilerOptions: {} }), ); - const config = createDefaultConfig(tempDir); + const config = createDefaultConfig({ root: tempDir }); expect(config.commands?.index).toBe( "scip-typescript index --output .dora/index.scip", ); }); }); + + describe("Language flag", () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = `/tmp/dora-test-${Date.now()}`; + await Bun.write(`${tempDir}/.keep`, ""); + }); + + afterEach(async () => { + try { + await Bun.$`rm -rf ${tempDir}`; + } catch { + } + }); + + test("should use explicit language when provided", async () => { + const config = createDefaultConfig({ + root: tempDir, + language: "python", + }); + + expect(config.language).toBe("python"); + expect(config.commands?.index).toBe( + "scip-python index --output .dora/index.scip", + ); + }); + + test("should use rust indexer when language is rust", async () => { + const config = createDefaultConfig({ + root: tempDir, + language: "rust", + }); + + expect(config.language).toBe("rust"); + expect(config.commands?.index).toBe( + "rust-analyzer scip . --output .dora/index.scip", + ); + }); + + test("should use go indexer when language is go", async () => { + const config = createDefaultConfig({ + root: tempDir, + language: "go", + }); + + expect(config.language).toBe("go"); + expect(config.commands?.index).toBe("scip-go --output .dora/index.scip"); + }); + + test("should use java indexer when language is java", async () => { + const config = createDefaultConfig({ + root: tempDir, + language: "java", + }); + + expect(config.language).toBe("java"); + expect(config.commands?.index).toBe( + "scip-java index --output .dora/index.scip", + ); + }); + + test("should use typescript indexer when language is typescript", async () => { + const config = createDefaultConfig({ + root: tempDir, + language: "typescript", + }); + + expect(config.language).toBe("typescript"); + expect(config.commands?.index).toBe( + "scip-typescript index --output .dora/index.scip", + ); + }); + + test("should use --infer-tsconfig when language is javascript", async () => { + const config = createDefaultConfig({ + root: tempDir, + language: "javascript", + }); + + expect(config.language).toBe("javascript"); + expect(config.commands?.index).toBe( + "scip-typescript index --infer-tsconfig --output .dora/index.scip", + ); + }); + + test("should not have language field when no language provided", async () => { + await Bun.write( + `${tempDir}/package.json`, + JSON.stringify({ name: "test" }), + ); + + const config = createDefaultConfig({ root: tempDir }); + + expect(config.language).toBeUndefined(); + }); + + test("should respect workspace type with explicit language", async () => { + await Bun.write( + `${tempDir}/pnpm-workspace.yaml`, + "packages:\n - packages/*", + ); + + const config = createDefaultConfig({ + root: tempDir, + language: "typescript", + }); + + expect(config.language).toBe("typescript"); + expect(config.commands?.index).toBe( + "scip-typescript index --pnpm-workspaces --output .dora/index.scip", + ); + }); + }); }); From 1054136ba96d19f700731983794cb662c5128b27 Mon Sep 17 00:00:00 2001 From: Yash Date: Sat, 24 Jan 2026 13:42:37 +0700 Subject: [PATCH 4/4] chore: update CHANGELOG for v1.4.5 Co-Authored-By: Claude Sonnet 4.5 --- CHANGELOG.md | 8 ++++++++ package.json | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b84824..c520b68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # @butttons/dora +## 1.4.5 + +### Patch Changes + +- Add `--language` flag to `dora init` for explicit language specification (typescript, javascript, python, rust, go, java) +- Optimize document processing performance and fix `--ignore` flag handling +- Refactor multi-parameter functions to use object parameters for better readability and maintainability + ## 1.4.4 ### Patch Changes diff --git a/package.json b/package.json index 2b33262..7a9dcc6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@butttons/dora", - "version": "1.4.4", + "version": "1.4.5", "module": "src/index.ts", "type": "module", "private": true,