From 7b78198154fcc75e3a002db91c6415d625989dad Mon Sep 17 00:00:00 2001
From: minseol2 <blessoms2017@veluga.io>
Date: Tue, 20 Jan 2026 22:56:43 +0900
Subject: [PATCH] fix(debug): use streaming to handle large JSONL files

The debug module was still using `readFile` + `split('\n')` pattern which can cause
`RangeError: Invalid string length` when processing large JSONL files (>512MB).

This change:
- Exports `processJSONLFileByLine` from data-loader.ts for reuse
- Refactors `detectMismatches` in debug.ts to use streaming
- Ensures consistent memory-safe file processing across the codebase

Fixes memory issues when running debug commands on large usage datasets.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
---
 apps/ccusage/src/data-loader.ts |  2 +-
 apps/ccusage/src/debug.ts       | 18 ++++++------------
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/apps/ccusage/src/data-loader.ts b/apps/ccusage/src/data-loader.ts
index 8e22aae2..2ccf1c2c 100644
--- a/apps/ccusage/src/data-loader.ts
+++ b/apps/ccusage/src/data-loader.ts
@@ -535,7 +535,7 @@ export function createUniqueHash(data: UsageData): string | null {
  * @param filePath - Path to the JSONL file
  * @param processLine - Callback function to process each line
  */
-async function processJSONLFileByLine(
+export async function processJSONLFileByLine(
 	filePath: string,
 	processLine: (line: string, lineNumber: number) => void | Promise<void>,
 ): Promise<void> {
diff --git a/apps/ccusage/src/debug.ts b/apps/ccusage/src/debug.ts
index 39af5e28..2ed0c629 100644
--- a/apps/ccusage/src/debug.ts
+++ b/apps/ccusage/src/debug.ts
@@ -7,7 +7,6 @@
  * @module debug
  */
 
-import { readFile } from 'node:fs/promises';
 import path from 'node:path';
 import { Result } from '@praha/byethrow';
 import { createFixture } from 'fs-fixture';
@@ -19,7 +18,7 @@ import {
 	USAGE_DATA_GLOB_PATTERN,
 } from './_consts.ts';
 import { PricingFetcher } from './_pricing-fetcher.ts';
-import { getClaudePaths, usageDataSchema } from './data-loader.ts';
+import { getClaudePaths, processJSONLFileByLine, usageDataSchema } from './data-loader.ts';
 import { logger } from './logger.ts';
 
 /**
@@ -106,13 +105,8 @@ export async function detectMismatches(claudePath?: string): Promise<MismatchSta
 	};
 
 	for (const file of files) {
-		const content = await readFile(file, 'utf-8');
-		const lines = content
-			.trim()
-			.split('\n')
-			.filter((line) => line.length > 0);
-
-		for (const line of lines) {
+		// Use streaming to avoid memory issues with large files
+		await processJSONLFileByLine(file, async (line) => {
 			const parseParser = Result.try({
 				try: () => JSON.parse(line) as unknown,
 				catch: () => new Error('Invalid JSON'),
@@ -120,13 +114,13 @@ export async function detectMismatches(claudePath?: string): Promise<MismatchSta
 
 			const parseResult = parseParser();
 			if (Result.isFailure(parseResult)) {
-				continue;
+				return;
 			}
 
 			const schemaResult = v.safeParse(usageDataSchema, parseResult.value);
 
 			if (!schemaResult.success) {
-				continue;
+				return;
 			}
 
 			const data = schemaResult.output;
@@ -206,7 +200,7 @@ export async function detectMismatches(claudePath?: string): Promise<MismatchSta
 					(modelStat.avgPercentDiff * (modelStat.total - 1) + percentDiff) / modelStat.total;
 				stats.modelStats.set(model, modelStat);
 			}
-		}
+		});
 	}
 
 	return stats;