diff --git a/app/api/users/create/route.ts b/app/api/users/create/route.ts index b52add7..ddac94a 100644 --- a/app/api/users/create/route.ts +++ b/app/api/users/create/route.ts @@ -2,15 +2,18 @@ import { getAdminDB } from '@/lib/firebase-admin'; import { NextResponse } from 'next/server'; import { encryptToken } from '@/lib/encryption'; + export async function POST(request: Request) { try { - console.log('🔔 Starting user creation/update process'); + console.log('[Create User] Starting user creation/update process'); const adminDb = getAdminDB(); const { uid, email, displayName, accessToken, photoURL } = await request.json(); - console.log('📝 User data received:', { uid, email, displayName }); + console.log('[Create User] User data received:', { uid, email, displayName, hasToken: !!accessToken }); // Encrypt the access token before storing + console.log('[Create User] Encrypting access token...'); const encryptedToken = await encryptToken(accessToken); + console.log('[Create User] Token encrypted, length:', encryptedToken?.length); // First check if users collection exists and if this user exists const usersRef = adminDb.collection('users'); @@ -18,7 +21,7 @@ export async function POST(request: Request) { const now = new Date(); if (!userDoc.exists) { - console.log('👤 Creating new user document'); + console.log('[Create User] Creating new user document'); // User doesn't exist, create new user document await usersRef.doc(uid).set({ uid, @@ -30,11 +33,11 @@ export async function POST(request: Request) { lastLogin: now, hasWatchHistory: false }); - console.log('✅ New user created successfully'); + console.log('[Create User] New user created successfully'); return NextResponse.json({ success: true, isNewUser: true }); } - console.log('🔄 Updating existing user'); + console.log('[Create User] Updating existing user'); // Update existing user's last login await usersRef.doc(uid).update({ lastLogin: now, @@ -43,7 +46,7 @@ export async function POST(request: Request) { displayName, photoURL, }); - console.log('✅ User updated successfully'); + console.log('[Create User] User updated successfully'); return NextResponse.json({ success: true, @@ -51,7 +54,7 @@ export async function POST(request: Request) { hasWatchHistory: userDoc.data()?.hasWatchHistory || false }); } catch (error) { - console.error('❌ Error in user creation/update:', error); + console.error('[Create User] ERROR:', error); return NextResponse.json({ error: 'Failed to create/update user', details: error instanceof Error ? error.message : 'Unknown error' diff --git a/app/api/users/get-history/route.ts b/app/api/users/get-history/route.ts index ea0264a..263e696 100644 --- a/app/api/users/get-history/route.ts +++ b/app/api/users/get-history/route.ts @@ -6,61 +6,114 @@ import { fetchAndExtractZip } from '@/lib/google-drive-backend'; export async function GET(request: Request) { try { - console.log('🔔 Starting watch history fetch process'); + console.log('\n========== GET /api/users/get-history =========='); + console.log('[Step 1] Starting watch history fetch process'); // Initialize Firebase Admin first + console.log('[Step 2] Initializing Firebase Admin...'); const adminDb = getAdminDB(); + console.log('[Step 2] Firebase Admin initialized successfully'); // Get the authorization header + console.log('[Step 3] Checking authorization header...'); const authHeader = request.headers.get('Authorization'); if (!authHeader?.startsWith('Bearer ')) { - console.log('❌ No authorization header found'); + console.log('[Step 3] FAILED: No authorization header found'); return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); } + console.log('[Step 3] Authorization header present'); const idToken = authHeader.split('Bearer ')[1]; + console.log('[Step 4] Verifying ID token (length:', idToken?.length, ')'); // Verify the ID token const auth = getAuth(); const decodedToken = await auth.verifyIdToken(idToken); const uid = decodedToken.uid; - - console.log('📝 Fetching watch history for user:', uid); + console.log('[Step 4] Token verified, uid:', uid); // Get the user's document from Firestore + console.log('[Step 5] Fetching user document from Firestore...'); const userDoc = await adminDb.collection('users').doc(uid).get(); + console.log('[Step 5] User document fetched, exists:', userDoc.exists); if (!userDoc.exists) { - console.log('❌ User document not found'); + console.log('[Step 5] FAILED: User document not found'); return NextResponse.json({ error: 'User not found' }, { status: 404 }); } const userData = userDoc.data(); const encryptedToken = userData?.accessToken; + console.log('[Step 6] Checking encrypted token, present:', !!encryptedToken, ', length:', encryptedToken?.length); if (!encryptedToken) { - console.log('❌ No access token found for user'); + console.log('[Step 6] FAILED: No access token found for user'); return NextResponse.json({ error: 'No access token found' }, { status: 404 }); } + console.log('[Step 6] Encrypted token found'); // Decrypt the token + console.log('[Step 7] Decrypting access token...'); const accessToken = await decryptToken(encryptedToken); + console.log('[Step 7] Token decrypted successfully, length:', accessToken?.length); // Use the token to fetch watch history data + console.log('[Step 8] Fetching watch history from Google Drive...'); const watchHistoryData = await fetchAndExtractZip(accessToken); + const dataInfo = Array.isArray(watchHistoryData) + ? `array with ${watchHistoryData.length} entries` + : `object with keys: ${Object.keys(watchHistoryData || {}).slice(0, 5).join(', ')}`; + console.log('[Step 8] Watch history fetched:', dataInfo); // Update the user's last data update timestamp + console.log('[Step 9] Updating lastDataUpdate timestamp...'); await adminDb.collection('users').doc(uid).update({ lastDataUpdate: new Date(), }); + console.log('[Step 9] Timestamp updated'); - console.log('✅ Successfully retrieved and processed watch history data'); + console.log('[SUCCESS] Watch history fetch complete'); + console.log('=================================================\n'); return NextResponse.json({ data: watchHistoryData }); } catch (error) { - console.error('❌ Error fetching watch history:', error); + console.error('\n[ERROR] /api/users/get-history failed'); + console.error('Error type:', error instanceof Error ? error.constructor.name : typeof error); + console.error('Error message:', error instanceof Error ? error.message : String(error)); + console.error('Error stack:', error instanceof Error ? error.stack : 'No stack trace'); + console.error('=================================================\n'); + + // Return more specific error info for debugging + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + + // Categorize errors for better client-side handling + if (errorMessage.includes('No Takeout zip files found')) { + return NextResponse.json({ + error: 'NO_TAKEOUT_FOLDER', + message: 'No Google Takeout export found in your Drive. Please create one first.', + details: errorMessage + }, { status: 404 }); + } + + if (errorMessage.includes('decrypt') || errorMessage.includes('Unsupported state')) { + return NextResponse.json({ + error: 'TOKEN_ERROR', + message: 'Session expired. Please log out and log in again.', + details: errorMessage + }, { status: 401 }); + } + + if (errorMessage.includes('Watch history file not found')) { + return NextResponse.json({ + error: 'INVALID_TAKEOUT', + message: 'Your Takeout export does not contain YouTube watch history. Please export with history included.', + details: errorMessage + }, { status: 404 }); + } + return NextResponse.json({ - error: 'Failed to fetch watch history', - details: error instanceof Error ? error.message : 'Unknown error' + error: 'FETCH_FAILED', + message: 'Failed to fetch watch history', + details: errorMessage }, { status: 500 }); } } \ No newline at end of file diff --git a/app/api/videos/metadata/route.ts b/app/api/videos/metadata/route.ts new file mode 100644 index 0000000..bb885bc --- /dev/null +++ b/app/api/videos/metadata/route.ts @@ -0,0 +1,266 @@ +import { NextResponse } from 'next/server'; +import { Redis } from '@upstash/redis'; +import { deflateSync, unzipSync } from 'zlib'; +import type { YouTubeVideoMetadata, MetadataResponse } from '@/types/youtube'; + +// Initialize Redis (server-side only) +// Uses non-public vars if available, falls back to NEXT_PUBLIC_ vars +let redis: Redis | null = null; +try { + const redisUrl = process.env.UPSTASH_REDIS_REST_URL || process.env.NEXT_PUBLIC_UPSTASH_REDIS_REST_URL; + const redisToken = process.env.UPSTASH_REDIS_REST_TOKEN || process.env.NEXT_PUBLIC_UPSTASH_REDIS_REST_TOKEN; + + if (redisUrl && redisToken) { + redis = new Redis({ + url: redisUrl, + token: redisToken, + }); + console.log('[Metadata API] Redis initialized'); + } else { + console.log('[Metadata API] Redis not configured, caching disabled'); + } +} catch (e) { + console.warn('[Metadata API] Failed to initialize Redis:', e); +} + +// Constants - use non-public vars if available, fall back to NEXT_PUBLIC_ +const YOUTUBE_API_KEY = process.env.YOUTUBE_API_KEY || process.env.NEXT_PUBLIC_YOUTUBE_API_KEY; +const BATCH_SIZE = 50; // YouTube API limit +const REDIS_CHUNK_SIZE = 100; + +/** + * Compress and encode metadata for Redis storage + */ +function compressMetadata(metadata: YouTubeVideoMetadata): string { + const compressed = deflateSync(JSON.stringify(metadata)); + return Buffer.from(compressed).toString('base64'); +} + +/** + * Decompress and decode metadata from Redis storage + */ +function decompressMetadata(data: string): YouTubeVideoMetadata | null { + try { + const buffer = Buffer.from(data, 'base64'); + const decompressed = unzipSync(buffer); + return JSON.parse(decompressed.toString('utf8')); + } catch { + return null; + } +} + +/** + * Fetch metadata from Redis cache + */ +async function getFromCache(videoIds: string[]): Promise> { + const result = new Map(); + + if (!redis || videoIds.length === 0) { + return result; + } + + try { + // Process in chunks to avoid overwhelming Redis + for (let i = 0; i < videoIds.length; i += REDIS_CHUNK_SIZE) { + const chunk = videoIds.slice(i, i + REDIS_CHUNK_SIZE); + const keys = chunk.map(id => `yt:${id}`); + + const cached = await redis.mget(...keys); + + cached.forEach((data, index) => { + if (data) { + const metadata = decompressMetadata(data); + if (metadata) { + result.set(chunk[index], metadata); + } + } + }); + } + } catch (error) { + console.warn('[Metadata API] Cache read error:', error); + } + + return result; +} + +/** + * Save metadata to Redis cache + */ +async function saveToCache(entries: Map): Promise { + if (!redis || entries.size === 0) { + return; + } + + try { + const entriesArray = Array.from(entries.entries()); + + for (let i = 0; i < entriesArray.length; i += REDIS_CHUNK_SIZE) { + const chunk = entriesArray.slice(i, i + REDIS_CHUNK_SIZE); + const pipeline = redis.pipeline(); + + for (const [id, metadata] of chunk) { + const compressed = compressMetadata(metadata); + pipeline.set(`yt:${id}`, compressed); + } + + await pipeline.exec(); + } + + console.log(`[Metadata API] Cached ${entries.size} videos`); + } catch (error) { + console.warn('[Metadata API] Cache write error:', error); + } +} + +/** + * Fetch metadata from YouTube API + */ +async function fetchFromYouTube(videoIds: string[]): Promise> { + const result = new Map(); + + if (!YOUTUBE_API_KEY || videoIds.length === 0) { + if (!YOUTUBE_API_KEY) { + console.warn('[Metadata API] YOUTUBE_API_KEY not configured'); + } + return result; + } + + // Process in batches of 50 (YouTube API limit) + for (let i = 0; i < videoIds.length; i += BATCH_SIZE) { + const batch = videoIds.slice(i, i + BATCH_SIZE); + + try { + const url = new URL('https://www.googleapis.com/youtube/v3/videos'); + url.searchParams.set('part', 'snippet,statistics,contentDetails'); + url.searchParams.set('id', batch.join(',')); + url.searchParams.set('key', YOUTUBE_API_KEY); + url.searchParams.set('fields', + 'items(id,snippet(title,channelTitle,categoryId,publishedAt,tags),' + + 'statistics(viewCount,likeCount,commentCount),contentDetails/duration)' + ); + + const response = await fetch(url.toString(), { + headers: { 'Content-Type': 'application/json' }, + }); + + if (!response.ok) { + const errorText = await response.text(); + console.error(`[Metadata API] YouTube API error ${response.status}:`, errorText); + continue; + } + + const data = await response.json(); + + if (!data.items) { + continue; + } + + for (const video of data.items) { + const metadata: YouTubeVideoMetadata = { + video_id: video.id, + title: video.snippet?.title || '', + channel: video.snippet?.channelTitle || '', + category_id: video.snippet?.categoryId || '', + published_at: video.snippet?.publishedAt || '', + tags: video.snippet?.tags || [], + view_count: parseInt(video.statistics?.viewCount || '0', 10), + like_count: parseInt(video.statistics?.likeCount || '0', 10), + comment_count: parseInt(video.statistics?.commentCount || '0', 10), + made_for_kids: video.contentDetails?.contentRating?.ytRating === 'ytAgeRestricted', + duration: video.contentDetails?.duration || '', + }; + result.set(video.id, metadata); + } + + console.log(`[Metadata API] Fetched batch ${Math.floor(i / BATCH_SIZE) + 1}: ${data.items.length} videos`); + + // Small delay between batches to be nice to YouTube API + if (i + BATCH_SIZE < videoIds.length) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + } catch (error) { + console.error(`[Metadata API] Batch error:`, error); + } + } + + return result; +} + +/** + * POST /api/videos/metadata + * + * Request body: { videoIds: string[] } + * Response: { metadata: Record, cached: number, fetched: number, failed: number } + */ +export async function POST(request: Request) { + try { + const body = await request.json(); + const { videoIds } = body; + + if (!Array.isArray(videoIds) || videoIds.length === 0) { + return NextResponse.json( + { error: 'videoIds must be a non-empty array' }, + { status: 400 } + ); + } + + // Deduplicate and filter valid IDs + const uniqueIds = [...new Set(videoIds)].filter(id => + typeof id === 'string' && id.length > 0 + ); + + console.log(`[Metadata API] Request for ${uniqueIds.length} videos`); + + const metadata: Record = {}; + let cachedCount = 0; + let fetchedCount = 0; + + // Step 1: Check Redis cache + const cachedMetadata = await getFromCache(uniqueIds); + cachedCount = cachedMetadata.size; + + for (const [id, data] of cachedMetadata) { + metadata[id] = data; + } + + // Step 2: Fetch missing from YouTube API + const missingIds = uniqueIds.filter(id => !metadata[id]); + + if (missingIds.length > 0) { + console.log(`[Metadata API] Cache hit: ${cachedCount}, fetching ${missingIds.length} from YouTube`); + + const fetchedMetadata = await fetchFromYouTube(missingIds); + fetchedCount = fetchedMetadata.size; + + for (const [id, data] of fetchedMetadata) { + metadata[id] = data; + } + + // Step 3: Cache newly fetched metadata (fire and forget) + if (fetchedMetadata.size > 0) { + saveToCache(fetchedMetadata).catch(err => + console.warn('[Metadata API] Background cache save failed:', err) + ); + } + } + + const failedCount = uniqueIds.length - cachedCount - fetchedCount; + + const response: MetadataResponse = { + metadata, + cached: cachedCount, + fetched: fetchedCount, + failed: failedCount, + }; + + console.log(`[Metadata API] Response: ${cachedCount} cached, ${fetchedCount} fetched, ${failedCount} failed`); + + return NextResponse.json(response); + } catch (error) { + console.error('[Metadata API] Error:', error); + return NextResponse.json( + { error: 'Failed to fetch metadata', details: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ); + } +} diff --git a/app/dashboard/categories/page.tsx b/app/dashboard/categories/page.tsx index eb4c4f0..f31e464 100644 --- a/app/dashboard/categories/page.tsx +++ b/app/dashboard/categories/page.tsx @@ -13,6 +13,7 @@ import { DashboardHeader } from "@/components/dashboard-header" import { fetchCategoryData } from "@/lib/fetch-categories-data" import { getCategoryName } from "@/lib/youtube-categories" + interface CategoryData { year: number totalWatchTime: number diff --git a/app/dashboard/creators/page.tsx b/app/dashboard/creators/page.tsx index ee3a9b6..0bb9605 100644 --- a/app/dashboard/creators/page.tsx +++ b/app/dashboard/creators/page.tsx @@ -13,6 +13,8 @@ import { fetchDefaultComparison, DashboardStats } from "@/lib/fetch-dashboard-da import { getChannelThumbnailCached } from "@/lib/youtube-api" import { CreatorCard } from "@/components/creator-card" + + // Mock data for fallback const mockCreatorStats = { primaryYear: { diff --git a/app/dashboard/page.tsx b/app/dashboard/page.tsx index cff78d9..ff66efa 100644 --- a/app/dashboard/page.tsx +++ b/app/dashboard/page.tsx @@ -36,6 +36,8 @@ import { getChannelThumbnailCached } from "@/lib/youtube-api" import { WordCloudComponent } from "@/components/word-cloud" import { CreatorCard } from "@/components/creator-card" + + // Mock stats for sample user const mockStats = { primaryYear: { diff --git a/app/login/page.tsx b/app/login/page.tsx index 344acf2..f7885ca 100644 --- a/app/login/page.tsx +++ b/app/login/page.tsx @@ -8,6 +8,7 @@ import { Card, CardContent, CardDescription, CardFooter, CardHeader, CardTitle } import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" import { useAuth } from "@/contexts/auth-context" import { GoogleLogin } from "@/components/google-login" +import { logger } from "@/lib/logger" export default function LoginPage() { const [isLoading, setIsLoading] = useState(false) diff --git a/app/profile/page.tsx b/app/profile/page.tsx index 62d78d8..ab2ce7d 100644 --- a/app/profile/page.tsx +++ b/app/profile/page.tsx @@ -4,7 +4,7 @@ import { useEffect, useState } from "react" import { useRouter } from "next/navigation" import Link from "next/link" import Image from "next/image" -import { ArrowLeft, Calendar, Clock, Film, Mail, MapPin, RefreshCw } from "lucide-react" +import { ArrowLeft, Calendar, Clock, Film, Mail, MapPin, RefreshCw, Bug } from "lucide-react" import { Button } from "@/components/ui/button" import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card" import { Separator } from "@/components/ui/separator" @@ -67,7 +67,7 @@ export default function ProfilePage() { description: "Your YouTube watch history has been successfully updated and processed.", }); } catch (error: any) { - console.error("❌ Error:", error.message); + console.error("Error refreshing data:", error.message); toast({ title: "Error", description: "Could not refresh watch history. Please try again later.", diff --git a/components/google-login.tsx b/components/google-login.tsx index e39bc06..0631ee3 100644 --- a/components/google-login.tsx +++ b/components/google-login.tsx @@ -67,11 +67,14 @@ export function GoogleLogin({ variant }: GoogleLoginProps) { if (!hasCompleteData) { setDataLoadingStatus("Fetching your YouTube watch history...") - console.log("📦 Watch history data not found or incomplete, fetching..."); + console.log("[Login] Watch history data not found or incomplete, fetching..."); try { + console.log("[Login] Getting ID token for API call..."); const idToken = await user.getIdToken(); + console.log("[Login] ID token obtained, length:", idToken?.length); + console.log("[Login] Calling /api/users/get-history..."); const response = await fetch('/api/users/get-history', { method: 'GET', headers: { @@ -80,29 +83,74 @@ export function GoogleLogin({ variant }: GoogleLoginProps) { }, }); + console.log("[Login] Response status:", response.status); + if (!response.ok) { - throw new Error(`Failed to fetch watch history: ${response.status}`); + const errorData = await response.json().catch(() => ({})); + console.error("[Login] Error response:", errorData); + + // Handle specific error types + if (errorData.error === 'NO_TAKEOUT_FOLDER') { + throw new Error('NO_TAKEOUT: ' + (errorData.message || 'No Takeout export found')); + } + if (errorData.error === 'TOKEN_ERROR') { + throw new Error('TOKEN_ERROR: ' + (errorData.message || 'Please re-login')); + } + if (errorData.error === 'INVALID_TAKEOUT') { + throw new Error('INVALID_TAKEOUT: ' + (errorData.message || 'Takeout missing history')); + } + + throw new Error(`Failed to fetch watch history: ${response.status} - ${errorData.message || errorData.details || 'Unknown error'}`); } - const { data } = await response.json(); + const responseData = await response.json(); + console.log("[Login] Response received, has data:", !!responseData.data); + + const { data } = responseData; if (!data) { - throw new Error("No watch history data available"); + throw new Error("No watch history data in response"); } + console.log("[Login] Processing and storing watch history..."); // Process and store the watch history data await processAndStoreWatchHistoryByYear(data); + console.log("[Login] Watch history stored successfully!"); toast({ title: "Data imported", description: "Your YouTube watch history has been successfully imported.", }); } catch (error: any) { - console.error("❌ Error:", error.message); - toast({ - title: "Using Sample Data", - description: "Could not fetch your watch history. You'll see sample data on the dashboard. Click the refresh button to try again later.", - }); + console.error("[Login] Watch history fetch error:"); + console.error("[Login] Error message:", error.message); + console.error("[Login] Full error:", error); + + // Show appropriate message based on error type + if (error.message.includes('NO_TAKEOUT')) { + toast({ + title: "No Takeout Export Found", + description: "Please create a Google Takeout export with your YouTube history first.", + variant: "destructive", + }); + } else if (error.message.includes('TOKEN_ERROR')) { + toast({ + title: "Session Error", + description: "Please log out and log in again to refresh your session.", + variant: "destructive", + }); + } else if (error.message.includes('INVALID_TAKEOUT')) { + toast({ + title: "Invalid Takeout Export", + description: "Your Takeout export doesn't contain YouTube history. Please export again with history included.", + variant: "destructive", + }); + } else { + toast({ + title: "Using Sample Data", + description: "Could not fetch your watch history. You'll see sample data on the dashboard.", + }); + } isSampleUser = true; } } else { diff --git a/lib/encryption.ts b/lib/encryption.ts index 7237ec2..995a6f2 100644 --- a/lib/encryption.ts +++ b/lib/encryption.ts @@ -1,17 +1,33 @@ import crypto from 'crypto'; if (!process.env.ENCRYPTION_KEY) { + console.error('[Encryption] ENCRYPTION_KEY environment variable is missing!'); throw new Error('ENCRYPTION_KEY environment variable is required'); } +// Validate encryption key format +if (!/^[a-f0-9]{64}$/i.test(process.env.ENCRYPTION_KEY)) { + console.error('[Encryption] ENCRYPTION_KEY must be 64 hexadecimal characters (32 bytes)'); + console.error('[Encryption] Current length:', process.env.ENCRYPTION_KEY.length); + throw new Error('ENCRYPTION_KEY must be 64 hexadecimal characters'); +} + // Use the encryption key from environment variable const ENCRYPTION_KEY = Buffer.from(process.env.ENCRYPTION_KEY, 'hex'); const ALGORITHM = 'aes-256-gcm'; const IV_LENGTH = 12; const AUTH_TAG_LENGTH = 16; +console.log('[Encryption] Module initialized, key length:', ENCRYPTION_KEY.length, 'bytes'); + export async function encryptToken(token: string): Promise { + console.log('[Encryption] Starting, token length:', token?.length); + try { + if (!token) { + throw new Error('Cannot encrypt empty token'); + } + // Generate a random IV const iv = crypto.randomBytes(IV_LENGTH); @@ -34,22 +50,35 @@ export async function encryptToken(token: string): Promise { authTag ]); - return result.toString('base64'); + const base64Result = result.toString('base64'); + console.log('[Encryption] SUCCESS, result length:', base64Result.length); + return base64Result; } catch (error) { - console.error('Error encrypting token:', error); - throw new Error('Failed to encrypt token'); + console.error('[Encryption] FAILED:', error instanceof Error ? error.message : error); + throw new Error(`Failed to encrypt token: ${error instanceof Error ? error.message : 'Unknown error'}`); } } export async function decryptToken(encryptedToken: string): Promise { + console.log('[Decryption] Starting...'); + console.log('[Decryption] Input length:', encryptedToken?.length); + console.log('[Decryption] ENCRYPTION_KEY length:', ENCRYPTION_KEY?.length, 'bytes'); + try { // Convert from base64 const buffer = Buffer.from(encryptedToken, 'base64'); + console.log('[Decryption] Buffer length after base64 decode:', buffer.length); + + // Validate buffer length + if (buffer.length < IV_LENGTH + AUTH_TAG_LENGTH + 1) { + throw new Error(`Invalid encrypted token: buffer too short (${buffer.length} bytes, need at least ${IV_LENGTH + AUTH_TAG_LENGTH + 1})`); + } // Extract IV, encrypted data, and auth tag const iv = buffer.slice(0, IV_LENGTH); const encrypted = buffer.slice(IV_LENGTH, -AUTH_TAG_LENGTH); const authTag = buffer.slice(-AUTH_TAG_LENGTH); + console.log('[Decryption] IV length:', iv.length, ', encrypted length:', encrypted.length, ', authTag length:', authTag.length); // Create decipher const decipher = crypto.createDecipheriv( @@ -65,9 +94,13 @@ export async function decryptToken(encryptedToken: string): Promise { let decrypted = decipher.update(encrypted); decrypted = Buffer.concat([decrypted, decipher.final()]); - return decrypted.toString('utf8'); + const result = decrypted.toString('utf8'); + console.log('[Decryption] SUCCESS, decrypted token length:', result.length); + return result; } catch (error) { - console.error('Error decrypting token:', error); - throw new Error('Failed to decrypt token'); + console.error('[Decryption] FAILED'); + console.error('[Decryption] Error type:', error instanceof Error ? error.constructor.name : typeof error); + console.error('[Decryption] Error message:', error instanceof Error ? error.message : String(error)); + throw new Error(`Failed to decrypt token: ${error instanceof Error ? error.message : 'Unknown error'}`); } } \ No newline at end of file diff --git a/lib/fetch-categories-data.ts b/lib/fetch-categories-data.ts index c2b3873..9cfdfb2 100644 --- a/lib/fetch-categories-data.ts +++ b/lib/fetch-categories-data.ts @@ -1,6 +1,7 @@ import { openDB } from "idb" import { DB_NAME, FILES_STORE } from './constants' import { youtubeCategories } from './youtube-categories' +import { logger } from "./logger" interface VideoStats { videoId: string diff --git a/lib/fetch-dashboard-data.ts b/lib/fetch-dashboard-data.ts index 94de6d1..42341e0 100644 --- a/lib/fetch-dashboard-data.ts +++ b/lib/fetch-dashboard-data.ts @@ -1,6 +1,7 @@ import { openDB } from "idb" import { DB_NAME, FILES_STORE } from './constants' import { getCategoryName } from './youtube-categories' +import { logger } from './logger' interface WatchHistoryEntry { title: string diff --git a/lib/fetch-watch-history.ts b/lib/fetch-watch-history.ts index 3477894..d183520 100644 --- a/lib/fetch-watch-history.ts +++ b/lib/fetch-watch-history.ts @@ -1,8 +1,9 @@ import { processAndStoreWatchHistoryByYear} from "./indexeddb" +import { logger } from "./logger" async function fetchWatchHistory(accessToken: string) { try { - console.log("🔑 Using access token:", accessToken ? "Token present" : "No token"); + console.log("Using access token:", accessToken ? "Token present" : "No token"); const response = await fetch('/api/fetch-watch-history', { method: 'POST', @@ -14,7 +15,7 @@ async function fetchWatchHistory(accessToken: string) { if (!response.ok) { const errorText = await response.text(); - console.error("❌ API Response error:", { + logger.error("API Response error:", { status: response.status, statusText: response.statusText, body: errorText @@ -31,7 +32,7 @@ async function fetchWatchHistory(accessToken: string) { const { data } = await response.json(); return data; } catch (error: any) { - console.error("❌ Error in fetchWatchHistory:", { + logger.error("Error in fetchWatchHistory:", { message: error.message, stack: error.stack, name: error.name @@ -45,7 +46,7 @@ export async function fetchAndProcessWatchHistory(accessToken: string, userId: s // Fetch watch history from YouTube API try { const watchHistory = await fetchWatchHistory(accessToken); - console.log("🔄 Processing and storing watch history data by year"); + console.log("Processing and storing watch history data by year"); await processAndStoreWatchHistoryByYear(watchHistory); } catch (error: any) { if (error.message === 'NO_TAKEOUT_FOLDER') { @@ -57,7 +58,7 @@ export async function fetchAndProcessWatchHistory(accessToken: string, userId: s } // Update watch history status - console.log("🔄 Updating watch history status..."); + console.log("Updating watch history status..."); const statusResponse = await fetch('/api/users/update-history-status', { method: 'POST', headers: { @@ -71,7 +72,7 @@ export async function fetchAndProcessWatchHistory(accessToken: string, userId: s if (!statusResponse.ok) { const errorText = await statusResponse.text(); - console.error("❌ Failed to update watch history status:", { + logger.error("Failed to update watch history status:", { status: statusResponse.status, statusText: statusResponse.statusText, body: errorText @@ -79,10 +80,10 @@ export async function fetchAndProcessWatchHistory(accessToken: string, userId: s throw new Error(`Failed to update watch history status: ${statusResponse.status}`); } - console.log("✅ Watch history data processed and stored successfully"); + console.log("Watch history data processed and stored successfully"); return true; } catch (error) { - console.error("❌ Error in fetchAndProcessWatchHistory:", error); + logger.error("Error in fetchAndProcessWatchHistory:", error); throw error; } } \ No newline at end of file diff --git a/lib/fetch-watch-time-data.ts b/lib/fetch-watch-time-data.ts index 8f5bee0..0604407 100644 --- a/lib/fetch-watch-time-data.ts +++ b/lib/fetch-watch-time-data.ts @@ -1,5 +1,6 @@ import { openDB } from "idb" import { DB_NAME, FILES_STORE } from './constants' +import { logger } from "./logger" interface WatchHistoryEntry { channel_name: string diff --git a/lib/firebase-admin.ts b/lib/firebase-admin.ts index 4cd9f52..df30ecc 100644 --- a/lib/firebase-admin.ts +++ b/lib/firebase-admin.ts @@ -3,13 +3,27 @@ import { getFirestore } from 'firebase-admin/firestore'; export function getAdminDB() { if (getApps().length === 0) { - initializeApp({ - credential: cert({ - projectId: process.env.FIREBASE_PROJECT_ID, - clientEmail: process.env.FIREBASE_CLIENT_EMAIL, - privateKey: process.env.FIREBASE_PRIVATE_KEY?.replace(/\\n/g, '\n'), - }), - }); + console.log('[Firebase Admin] Initializing new app...'); + console.log('[Firebase Admin] projectId present:', !!process.env.FIREBASE_PROJECT_ID); + console.log('[Firebase Admin] clientEmail present:', !!process.env.FIREBASE_CLIENT_EMAIL); + console.log('[Firebase Admin] privateKey present:', !!process.env.FIREBASE_PRIVATE_KEY); + console.log('[Firebase Admin] privateKey length:', process.env.FIREBASE_PRIVATE_KEY?.length); + + try { + initializeApp({ + credential: cert({ + projectId: process.env.FIREBASE_PROJECT_ID, + clientEmail: process.env.FIREBASE_CLIENT_EMAIL, + privateKey: process.env.FIREBASE_PRIVATE_KEY?.replace(/\\n/g, '\n'), + }), + }); + console.log('[Firebase Admin] App initialized successfully'); + } catch (initError) { + console.error('[Firebase Admin] FAILED to initialize:', initError); + throw initError; + } + } else { + console.log('[Firebase Admin] Using existing app instance'); } return getFirestore(); } diff --git a/lib/google-drive-backend.ts b/lib/google-drive-backend.ts index 9d48bc9..88480b6 100644 --- a/lib/google-drive-backend.ts +++ b/lib/google-drive-backend.ts @@ -4,72 +4,175 @@ import JSZip from "jszip"; // Initialize Google Drive API client const drive = google.drive("v3"); +// Target path for watch history in the ZIP +const TARGET_FILE_PATH = "Takeout/YouTube and YouTube Music/history/watch-history.json"; + +/** + * Extract the base timestamp from a Takeout filename. + * e.g., "takeout-20260103T224344Z-001.zip" -> "20260103T224344Z" + * e.g., "takeout-20260103T224344Z-3-001.zip" -> "20260103T224344Z" + */ +function extractTakeoutTimestamp(filename: string): string | null { + // Match pattern: takeout-TIMESTAMP-... or takeout-TIMESTAMP-N-... + const match = filename.match(/takeout-(\d{8}T\d{6}Z)/i); + return match ? match[1] : null; +} + /** - * Fetches the latest zip file from the Takeout folder in Google Drive, extracts the `watch-history.json` file, - * and returns its contents. + * Group Takeout files by their export session (same timestamp = same export) + */ +function groupTakeoutFiles(files: { id?: string | null; name?: string | null; createdTime?: string | null }[]): Map { + const groups = new Map(); + + for (const file of files) { + if (!file.name) continue; + const timestamp = extractTakeoutTimestamp(file.name); + if (timestamp) { + if (!groups.has(timestamp)) { + groups.set(timestamp, []); + } + groups.get(timestamp)!.push(file); + } + } + + return groups; +} + +/** + * Fetches Takeout zip files from Google Drive, searches through multi-part exports, + * extracts the `watch-history.json` file, and returns its contents. * @param {string} accessToken - The Google OAuth access token. * @returns {Promise} - The parsed contents of `watch-history.json`. */ export async function fetchAndExtractZip(accessToken: string): Promise { + console.log('\n---------- fetchAndExtractZip ----------'); + console.log('[Drive] Starting fetch, token length:', accessToken?.length); + try { // Set up and authenticate + console.log('[Drive Step 1] Setting up OAuth client...'); const authClient = new google.auth.OAuth2(); authClient.setCredentials({ access_token: accessToken }); + console.log('[Drive Step 1] OAuth client ready'); - // Search for Takeout files + // Search for Takeout files - get more to handle multi-part exports + console.log('[Drive Step 2] Searching for Takeout ZIP files...'); + const fileList = await drive.files.list({ auth: authClient, q: "name contains 'takeout' and mimeType='application/x-zip'", orderBy: 'createdTime desc', - pageSize: 1, + pageSize: 50, // Get more files to find all parts fields: 'files(id, name, mimeType, createdTime)', }); const files = fileList.data.files; + console.log('[Drive Step 2] Total ZIP files found:', files?.length ?? 0); + if (!files?.length) { - console.log("❌ No Takeout files found"); - throw new Error("No Takeout zip files found in Google Drive"); + console.log('[Drive Step 2] FAILED: No Takeout files found'); + throw new Error("No Takeout zip files found in Google Drive. Make sure you exported to 'Add to Drive' in Google Takeout."); } + + // Log all files found + files.forEach((f, i) => { + console.log(`[Drive Step 2] [${i}] ${f.name} - ${f.createdTime}`); + }); - const latestFile = files[0]; - console.log("📁 Found Takeout file:", latestFile.name); + // Group files by export session + const groups = groupTakeoutFiles(files); + console.log('[Drive Step 3] Found', groups.size, 'distinct Takeout export session(s)'); + + // Get the most recent export session (sorted by timestamp) + const sortedTimestamps = Array.from(groups.keys()).sort().reverse(); + + for (const timestamp of sortedTimestamps) { + const sessionFiles = groups.get(timestamp)!; + console.log(`[Drive Step 3] Checking export session: ${timestamp} (${sessionFiles.length} part(s))`); + + // Search through each part of this export session + for (const file of sessionFiles) { + console.log(`[Drive Step 4] Downloading: ${file.name} (id: ${file.id})`); + + try { + const response = await drive.files.get({ + auth: authClient, + fileId: file.id!, + alt: "media", + }, { responseType: "arraybuffer" }); - // Download the zip file - const response = await drive.files.get({ - auth: authClient, - fileId: latestFile.id, - alt: "media", - }, { responseType: "arraybuffer" }); + if (!response.data) { + console.log('[Drive Step 4] No data received, skipping...'); + continue; + } + + const size = (response.data as ArrayBuffer).byteLength; + console.log(`[Drive Step 4] Downloaded ${size} bytes`); - if (!response.data) { - throw new Error("No data received from file download"); - } + // Process the zip file + const zipBuffer = Buffer.from(response.data as ArrayBuffer); + const zip = await JSZip.loadAsync(zipBuffer); + const zipFileNames = Object.keys(zip.files); + + console.log(`[Drive Step 5] ZIP contains ${zipFileNames.length} file(s)`); + + // Log a few files from the ZIP for debugging + const sampleFiles = zipFileNames.slice(0, 5); + sampleFiles.forEach(name => console.log(`[Drive] - ${name}`)); + if (zipFileNames.length > 5) { + console.log(`[Drive] ... and ${zipFileNames.length - 5} more`); + } - // Process the zip file - const zipBuffer = Buffer.from(response.data as ArrayBuffer); - const zip = await JSZip.loadAsync(zipBuffer); - - // Log available files to help with debugging - console.log("📑 Files in zip:", Object.keys(zip.files).length); - - // Try to find the watch history file - const targetFilePath = "Takeout/YouTube and YouTube Music/history/watch-history.json"; - console.log("🔍 Searching for watch history at path:", targetFilePath); + // Check if this ZIP contains watch history + const targetFile = zip.file(TARGET_FILE_PATH); + + if (targetFile) { + console.log('[Drive Step 6] FOUND watch-history.json in:', file.name); + + // Extract and parse the data + const fileContent = await targetFile.async("string"); + console.log('[Drive Step 6] File content length:', fileContent.length, 'chars'); + + const parsedData = JSON.parse(fileContent); + const entryCount = Array.isArray(parsedData) ? parsedData.length : 'N/A (not array)'; + console.log('[Drive Step 6] Parsed data, entries:', entryCount); + console.log('[Drive] SUCCESS: fetchAndExtractZip complete'); + console.log('----------------------------------------\n'); - const targetFile = zip.file(targetFilePath); - if (!targetFile) { - console.log(` ❌ Not found at path: ${targetFilePath}`); - throw new Error("Watch history file not found in zip"); + return parsedData; + } else { + console.log('[Drive Step 5] watch-history.json NOT in this part, checking next...'); + + // Check if there's any YouTube-related content + const ytContent = zipFileNames.filter(n => n.toLowerCase().includes('youtube')); + if (ytContent.length > 0) { + console.log('[Drive] YouTube content found but no watch-history:'); + ytContent.slice(0, 5).forEach(f => console.log(`[Drive] - ${f}`)); + } + } + } catch (downloadError: any) { + console.log(`[Drive Step 4] Error downloading ${file.name}:`, downloadError.message); + // Continue to next file + } + } } + + // If we get here, we searched all files and didn't find watch history + console.log('[Drive] FAILED: watch-history.json not found in any Takeout ZIP'); + throw new Error( + "Watch history file not found in any Takeout ZIP. " + + "Make sure you selected 'history' under 'YouTube and YouTube Music' data options when creating the export." + ); - // Extract and parse the data - const fileContent = await targetFile.async("string"); - const parsedData = JSON.parse(fileContent); - console.log("✅ Successfully extracted watch history data"); - - return parsedData; } catch (error: any) { - console.error("❌ Error:", error.message); + console.error('\n[Drive] FAILED: fetchAndExtractZip error'); + console.error('Error type:', error?.constructor?.name); + console.error('Error message:', error?.message); + if (error?.response?.status) { + console.error('HTTP Status:', error.response.status); + console.error('HTTP Data:', error.response.data); + } + console.error('----------------------------------------\n'); throw error; } } \ No newline at end of file diff --git a/lib/indexeddb.ts b/lib/indexeddb.ts index 97dbc22..9a63912 100644 --- a/lib/indexeddb.ts +++ b/lib/indexeddb.ts @@ -1,6 +1,7 @@ import { openDB, IDBPDatabase } from "idb" import { getVideosMetadata } from "./youtube-metadata" import { DB_NAME, FILES_STORE, WATCH_HISTORY_FILE, AVAILABLE_YEARS_FILE } from './constants' +import { logger } from "./logger" // Current database version - increment this when schema changes const DB_VERSION = 1 @@ -15,7 +16,7 @@ async function initDB(): Promise { upgrade(db) { // Create object store if it doesn't exist if (!db.objectStoreNames.contains(FILES_STORE)) { - console.log(`🔄 Creating object store: ${FILES_STORE}`) + console.log(`Creating object store: ${FILES_STORE}`) db.createObjectStore(FILES_STORE, { keyPath: "fileName" }) } }, @@ -28,7 +29,7 @@ async function initDB(): Promise { return db } catch (error) { - console.error("❌ Database initialization failed:", error) + logger.error("Database initialization failed:", error) throw error } } @@ -41,7 +42,7 @@ export async function isDataInIndexedDB(fileId: string): Promise { const existingFile = await store.get(fileId) return !!existingFile } catch (error) { - console.error("❌ Error checking IndexedDB:", error) + logger.error("Error checking IndexedDB:", error) return false } } @@ -74,9 +75,9 @@ export async function storeDataInIndexedDB(data: { [key: string]: string }) { await Promise.all(putPromises) await tx.done - console.log(`✅ Successfully stored ${Object.keys(data).length} items in IndexedDB`) + console.log(`Successfully stored ${Object.keys(data).length} items in IndexedDB`) } catch (error: unknown) { - console.error("❌ Error storing data in IndexedDB:", error) + logger.error("Error storing data in IndexedDB:", error) // Add more context to the error const errorMessage = error instanceof Error ? error.message : 'Unknown error' const enhancedError = new Error(`Failed to store data in IndexedDB: ${errorMessage}`) @@ -102,20 +103,20 @@ export async function storeDataInIndexedDB(data: { [key: string]: string }) { export async function processAndStoreWatchHistoryByYear(watchHistoryData: any[]) { // First store the raw data as backup - console.log('💾 Storing raw watch history data...') + console.log('Storing raw watch history data...') await storeDataInIndexedDB({ [WATCH_HISTORY_FILE]: JSON.stringify(watchHistoryData) }) - console.log('✅ Raw watch history data stored successfully') + console.log('Raw watch history data stored successfully') // Group data by year const dataByYear: { [year: string]: any[] } = {} const videoIdsByYear: { [year: string]: Set } = {} - console.log('📅 Grouping entries by year...') + console.log('Grouping entries by year...') watchHistoryData.forEach(entry => { if (!entry.time) { - console.warn('⚠️ Entry missing time:', entry) + logger.warn('Entry missing time:', entry) return } @@ -130,7 +131,7 @@ export async function processAndStoreWatchHistoryByYear(watchHistoryData: any[]) const videoId = videoIdMatch ? videoIdMatch[1] : null if (!videoId) { - console.warn('⚠️ Could not extract video ID from URL:', entry.titleUrl) + logger.warn('Could not extract video ID from URL:', entry.titleUrl) } // Create processed entry @@ -148,11 +149,11 @@ export async function processAndStoreWatchHistoryByYear(watchHistoryData: any[]) } }) - console.log('📊 Data grouped by year:', Object.keys(dataByYear)) + console.log('Data grouped by year:', Object.keys(dataByYear)) // Store available years const availableYears = Object.keys(dataByYear).map(Number).sort((a, b) => b - a) - console.log('📅 Storing available years:', availableYears) + console.log('Storing available years:', availableYears) await storeDataInIndexedDB({ [AVAILABLE_YEARS_FILE]: JSON.stringify(availableYears) }) @@ -166,17 +167,17 @@ export async function processAndStoreWatchHistoryByYear(watchHistoryData: any[]) .slice(0, 2) // Take the two most recent .map(String) // Convert back to strings - console.log('🎯 Processing years:', years) + console.log('Processing years:', years) // Fetch metadata for videos from the most recent two complete years for (const year of years) { const videoIds = Array.from(videoIdsByYear[year]) - console.log(`📺 Year ${year}: Processing ${videoIds.length} videos`) + console.log(`Year ${year}: Processing ${videoIds.length} videos`) if (videoIds.length > 0) { - console.log(`🔄 Fetching metadata for ${videoIds.length} videos in ${year}...`) + console.log(`Fetching metadata for ${videoIds.length} videos in ${year}...`) const metadataMap = await getVideosMetadata(videoIds) - console.log(`✅ Retrieved metadata for ${metadataMap.size} videos in ${year}`) + console.log(`Retrieved metadata for ${metadataMap.size} videos in ${year}`) // Update entries with metadata let updatedCount = 0 @@ -198,12 +199,12 @@ export async function processAndStoreWatchHistoryByYear(watchHistoryData: any[]) } return entry }) - console.log(`📊 Year ${year}: Updated ${updatedCount} entries, skipped ${skippedCount}`) + console.log(`Year ${year}: Updated ${updatedCount} entries, skipped ${skippedCount}`) } } // Store all year data in a single transaction - console.log('💾 Storing processed data...') + console.log('Storing processed data...') await storeDataInIndexedDB( Object.fromEntries( Object.entries(dataByYear).map(([year, data]) => [ @@ -213,7 +214,7 @@ export async function processAndStoreWatchHistoryByYear(watchHistoryData: any[]) ) ) - console.log('✅ Watch history processing complete!') + console.log('Watch history processing complete!') } export async function isWatchHistoryDataComplete(): Promise { @@ -221,14 +222,14 @@ export async function isWatchHistoryDataComplete(): Promise { // Check for main watch history file const hasMainData = await isDataInIndexedDB(WATCH_HISTORY_FILE); if (!hasMainData) { - console.log("❌ Main watch history file not found"); + console.log("Main watch history file not found"); return false; } // Get available years from IndexedDB const availableYearsData = await isDataInIndexedDB(AVAILABLE_YEARS_FILE); if (!availableYearsData) { - console.log("❌ Available years data not found"); + console.log("Available years data not found"); return false; } @@ -238,13 +239,13 @@ export async function isWatchHistoryDataComplete(): Promise { const data = await store.get(AVAILABLE_YEARS_FILE); if (!data) { - console.log("❌ Could not read available years data"); + console.log("Could not read available years data"); return false; } const years = JSON.parse(data.content); if (!Array.isArray(years) || years.length === 0) { - console.log("❌ No available years found"); + console.log("No available years found"); return false; } @@ -255,14 +256,14 @@ export async function isWatchHistoryDataComplete(): Promise { const allYearsExist = yearFilesExist.every(exists => exists); if (!allYearsExist) { - console.log("❌ Not all year files exist"); + console.log("Not all year files exist"); return false; } - console.log("✅ All watch history data is complete"); + console.log("All watch history data is complete"); return true; } catch (error) { - console.error("❌ Error checking watch history data completeness:", error); + logger.error("Error checking watch history data completeness:", error); return false; } } \ No newline at end of file diff --git a/lib/logger.ts b/lib/logger.ts new file mode 100644 index 0000000..40228c4 --- /dev/null +++ b/lib/logger.ts @@ -0,0 +1,63 @@ +/** + * Simple logger utility for consistent logging across the app. + * Can be configured to enable/disable logging based on environment. + */ + +const isDev = process.env.NODE_ENV === 'development'; +const DEBUG = process.env.DEBUG === 'true' || isDev; + +type LogLevel = 'log' | 'warn' | 'error' | 'debug'; + +interface LoggerOptions { + prefix?: string; + enabled?: boolean; +} + +function formatMessage(prefix: string, ...args: unknown[]): unknown[] { + const timestamp = new Date().toISOString().split('T')[1].slice(0, 12); + return [`[${timestamp}][${prefix}]`, ...args]; +} + +function createLogger(options: LoggerOptions = {}) { + const { prefix = 'App', enabled = DEBUG } = options; + + return { + log: (...args: unknown[]) => { + if (enabled) { + console.log(...formatMessage(prefix, ...args)); + } + }, + + warn: (...args: unknown[]) => { + if (enabled) { + console.warn(...formatMessage(prefix, ...args)); + } + }, + + error: (...args: unknown[]) => { + // Errors always log + console.error(...formatMessage(prefix, ...args)); + }, + + debug: (...args: unknown[]) => { + if (enabled && DEBUG) { + console.debug(...formatMessage(prefix, ...args)); + } + }, + + // Create a child logger with a sub-prefix + child: (subPrefix: string) => { + return createLogger({ prefix: `${prefix}:${subPrefix}`, enabled }); + }, + }; +} + +// Default logger instance +export const logger = createLogger({ prefix: 'YTW' }); + +// Named loggers for different modules +export const dbLogger = createLogger({ prefix: 'IndexedDB' }); +export const apiLogger = createLogger({ prefix: 'API' }); +export const driveLogger = createLogger({ prefix: 'Drive' }); + +export default logger; diff --git a/lib/process-watch-history.ts b/lib/process-watch-history.ts index 47d9bdd..30d5076 100644 --- a/lib/process-watch-history.ts +++ b/lib/process-watch-history.ts @@ -1,15 +1,16 @@ import { processAndStoreWatchHistoryByYear as processWatchHistory } from './indexeddb'; +import { logger } from "./logger" export async function processAndStoreWatchHistoryByYear(watchHistory: any[]): Promise { try { - console.log('📊 Starting watch history processing...'); - console.log(`📥 Total entries to process: ${watchHistory.length}`); + console.log('Starting watch history processing...'); + console.log(`Total entries to process: ${watchHistory.length}`); await processWatchHistory(watchHistory); - console.log('✅ Watch history data processed and stored successfully'); + console.log('Watch history data processed and stored successfully'); } catch (error) { - console.error('❌ Error processing watch history:', error); + logger.error('Error processing watch history:', error); throw error; } } \ No newline at end of file diff --git a/lib/youtube-metadata.ts b/lib/youtube-metadata.ts index e614065..bdb3071 100644 --- a/lib/youtube-metadata.ts +++ b/lib/youtube-metadata.ts @@ -1,334 +1,199 @@ -import { deflateSync, unzipSync } from "zlib"; -import { Buffer } from "buffer"; -import { Redis } from "@upstash/redis"; - -const redis = new Redis({ - url: process.env.NEXT_PUBLIC_UPSTASH_REDIS_REST_URL!, - token: process.env.NEXT_PUBLIC_UPSTASH_REDIS_REST_TOKEN!, -}); - -interface YouTubeVideoMetadata { - video_id: string; - title: string; - channel: string; - category_id: string; - published_at: string; - tags: string[]; - view_count: number; - like_count: number; - comment_count: number; - made_for_kids: boolean; - duration: string; +/** + * Client-side metadata fetching with local cache + server API + * + * Flow: + * 1. Check local IndexedDB cache first (instant) + * 2. Call server API for missing videos (API handles Redis cache + YouTube API) + * 3. Store results in local IndexedDB for next time + */ + +import { openDB, type IDBPDatabase } from 'idb'; +import type { YouTubeVideoMetadata, MetadataResponse } from '@/types/youtube'; + +// Re-export type for convenience +export type { YouTubeVideoMetadata }; + +// IndexedDB configuration +const DB_NAME = 'youtube-metadata-cache'; +const STORE_NAME = 'metadata'; +const DB_VERSION = 1; + +/** + * Get or create the IndexedDB database + */ +async function getDB(): Promise { + return openDB(DB_NAME, DB_VERSION, { + upgrade(db) { + if (!db.objectStoreNames.contains(STORE_NAME)) { + db.createObjectStore(STORE_NAME); + console.log('[Metadata] Created local cache store'); + } + }, + }); } -// TODO: not needed any more, remove when cleaning up -export async function getVideoMetadata( - videoId: string -): Promise { - try { - const cachedData = await redis.get(videoId); - if (cachedData) { - const decodedBytes = Buffer.from(cachedData as string, "base64"); - const decompressedData = unzipSync(decodedBytes); - return JSON.parse(decompressedData.toString("utf-8")); - } +/** + * Read metadata from local IndexedDB cache + */ +async function getFromLocalCache(videoIds: string[]): Promise> { + const result = new Map(); - // If not in cache, fetch from YouTube API - const response = await fetch( - `https://www.googleapis.com/youtube/v3/videos?` + - `part=snippet,statistics,contentDetails&` + - `id=${videoId}&` + - `key=${process.env.YOUTUBE_API_KEY}&` + - `fields=items(` + - `id,snippet(` + - `title,channelTitle,categoryId,publishedAt,tags` + - `),` + - `statistics(` + - `viewCount,likeCount,commentCount` + - `),` + - `contentDetails/duration` + - `)`, - { - headers: { - "Content-Type": "application/json", - }, - } + try { + const db = await getDB(); + const tx = db.transaction(STORE_NAME, 'readonly'); + const store = tx.objectStore(STORE_NAME); + + await Promise.all( + videoIds.map(async (id) => { + const cached = await store.get(id) as YouTubeVideoMetadata | undefined; + if (cached) { + result.set(id, cached); + } + }) ); - if (!response.ok) { - throw new Error("Failed to fetch video metadata"); - } + await tx.done; + } catch (error) { + console.warn('[Metadata] Local cache read error:', error); + } - const data = await response.json(); - if (!data.items || data.items.length === 0) { - return null; - } + return result; +} - const video = data.items[0]; - const metadata: YouTubeVideoMetadata = { - video_id: videoId, - title: video.snippet?.title || "", - channel: video.snippet?.channelTitle || "", - category_id: video.snippet?.categoryId || "", - published_at: video.snippet?.publishedAt || "", - tags: video.snippet?.tags || [], - view_count: parseInt(video.statistics?.viewCount || "0"), - like_count: parseInt(video.statistics?.likeCount || "0"), - comment_count: parseInt(video.statistics?.commentCount || "0"), - made_for_kids: - video.contentDetails?.contentRating?.ytRating === "ytAgeRestricted" || - false, - duration: video.contentDetails?.duration || "", - }; +/** + * Save metadata to local IndexedDB cache + */ +async function saveToLocalCache(entries: Map): Promise { + if (entries.size === 0) return; - const compressedEntry = deflateSync(JSON.stringify(metadata)); - const encodedEntry = Buffer.from(compressedEntry).toString("base64"); - await redis.set(videoId, encodedEntry); + try { + const db = await getDB(); + const tx = db.transaction(STORE_NAME, 'readwrite'); + const store = tx.objectStore(STORE_NAME); - return metadata; + for (const [id, metadata] of entries) { + await store.put(metadata, id); + } + + await tx.done; + console.log(`[Metadata] Saved ${entries.size} to local cache`); } catch (error) { - console.error("Error fetching video metadata:", error); - return null; + console.warn('[Metadata] Local cache write error:', error); } } -export async function getVideosMetadata( - videoIds: string[] -): Promise> { - console.log(`📥 Fetching metadata for ${videoIds.length} videos...`); +/** + * Fetch metadata from server API (which handles Redis + YouTube API) + */ +async function fetchFromServer(videoIds: string[]): Promise> { + const result = new Map(); - const metadataMap = new Map(); - const missingIds: string[] = []; - let errorCount = 0; - const MAX_ERRORS = 5; // Maximum number of errors before stopping + if (videoIds.length === 0) return result; try { - // Fetch all video metadata at once using MGET - console.log(`🔄 Fetching metadata for ${videoIds.length} videos from Redis...`); + // Split into chunks to avoid huge requests const CHUNK_SIZE = 500; - const CONCURRENT_CHUNKS = 5; // Number of chunks to process at once - const chunks = []; - const allCachedData: (string | null)[] = []; - - // Split videoIds into chunks of 1000 + for (let i = 0; i < videoIds.length; i += CHUNK_SIZE) { - chunks.push(videoIds.slice(i, i + CHUNK_SIZE)); - } - - console.log(`📦 Processing ${chunks.length} chunks of ${CHUNK_SIZE} videos each (${CONCURRENT_CHUNKS} concurrent)...`); - - // Process chunks in groups with controlled concurrency - for (let i = 0; i < chunks.length; i += CONCURRENT_CHUNKS) { - const currentChunks = chunks.slice(i, i + CONCURRENT_CHUNKS); - console.log(`🔄 Processing chunks ${i + 1}-${Math.min(i + CONCURRENT_CHUNKS, chunks.length)}/${chunks.length}...`); - - try { - // Execute multiple MGET operations concurrently - const chunkResults = await Promise.all( - currentChunks.map(async (chunk, chunkIndex) => { - const chunkData = await redis.mget(...chunk) as (string | null)[]; - if (!chunkData) { - throw new Error(`Redis MGET returned no data for chunk ${i + chunkIndex + 1}`); - } - return chunkData; - }) - ); - - // Combine results - chunkResults.forEach(result => { - allCachedData.push(...result); - }); - - // Add delay between groups of chunks except for the last group - if (i + CONCURRENT_CHUNKS < chunks.length) { - console.log(`⏳ Waiting 1 second before next group of chunks...`); - await new Promise(resolve => setTimeout(resolve, 1000)); - } - } catch (error: any) { - console.error("❌ Redis MGET Error:", { - error: error.message, - stack: error.stack, - chunkGroup: `${i + 1}-${Math.min(i + CONCURRENT_CHUNKS, chunks.length)}`, - totalChunks: chunks.length, - videoCount: currentChunks.reduce((sum, chunk) => sum + chunk.length, 0) - }); - throw new Error("Failed to retrieve data from Redis: " + error.message); - } - } - - // Process the results - allCachedData.forEach((data, index) => { - const videoId = videoIds[index]; + const chunk = videoIds.slice(i, i + CHUNK_SIZE); - if (!data) { - missingIds.push(videoId); - return; - } - - try { - const decodedBytes = Buffer.from(data as string, "base64"); - const decompressedData = unzipSync(decodedBytes); - const jsonString = decompressedData.toString('utf8'); - const metadata = JSON.parse(jsonString); - metadataMap.set(videoId, metadata); - } catch (parseError) { - errorCount++; - if (errorCount <= MAX_ERRORS) { - console.error(`❌ Error processing video ${videoId}:`, parseError); - } - if (errorCount >= MAX_ERRORS) { - throw new Error(`Too many errors (${errorCount}) while processing videos. Stopping...`); - } - missingIds.push(videoId); + const response = await fetch('/api/videos/metadata', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ videoIds: chunk }), + }); + + if (!response.ok) { + const error = await response.json().catch(() => ({})); + console.error('[Metadata] Server API error:', response.status, error); + continue; } - }); - - console.log(`📊 Found ${metadataMap.size} videos in cache, ${missingIds.length} to fetch from YouTube API`); - if (errorCount > 0) { - console.log(`⚠️ Encountered ${errorCount} errors while processing videos`); - } - // If we have videos to fetch from YouTube API - if (missingIds.length > 0) { - // YouTube API has a limit of 50 videos per request - const BATCH_SIZE = 50; - const CONCURRENCY_LIMIT = 5; - const batches = []; - const cacheEntries: Record = {}; + const data: MetadataResponse = await response.json(); - for (let i = 0; i < missingIds.length; i += BATCH_SIZE) { - batches.push(missingIds.slice(i, i + BATCH_SIZE)); + for (const [id, metadata] of Object.entries(data.metadata)) { + result.set(id, metadata); } - console.log(`🔄 Fetching ${batches.length} batches of videos from YouTube API (${CONCURRENCY_LIMIT} concurrent)...`); - - // Process batches with controlled concurrency - const processBatch = async (batch: string[], index: number) => { - console.log(`📦 Processing batch ${index + 1}/${batches.length} (${batch.length} videos)`); - - try { - const response = await fetch( - `https://www.googleapis.com/youtube/v3/videos?` + - `part=snippet,statistics,contentDetails&` + - `id=${batch.join(",")}&` + - `key=${process.env.NEXT_PUBLIC_YOUTUBE_API_KEY}&` + - `fields=items(` + - `id,snippet(` + - `title,channelTitle,categoryId,publishedAt,tags` + - `),` + - `statistics(` + - `viewCount,likeCount,commentCount` + - `),` + - `contentDetails/duration` + - `)`, - { - headers: { - "Content-Type": "application/json", - }, - } - ); - - if (!response.ok) { - const errorText = await response.text(); - const errorDetails = { - status: response.status, - statusText: response.statusText, - body: errorText, - batchIndex: index + 1, - totalBatches: batches.length, - videoIds: batch - }; - console.error("❌ YouTube API Error:", errorDetails); - throw new Error(`YouTube API error: ${response.status} - ${response.statusText}\nResponse: ${errorText}`); - } + console.log(`[Metadata] Server chunk ${Math.floor(i / CHUNK_SIZE) + 1}: ${data.cached} cached, ${data.fetched} fetched`); + } + } catch (error) { + console.error('[Metadata] Server API error:', error); + } - const data = await response.json(); - - if (!data.items) { - console.warn("⚠️ No items returned from YouTube API for batch:", { - batchIndex: index + 1, - totalBatches: batches.length, - videoIds: batch - }); - return; - } + return result; +} - // Process each video in the batch - for (const video of data.items) { - const metadata: YouTubeVideoMetadata = { - video_id: video.id, - title: video.snippet?.title || "", - channel: video.snippet?.channelTitle || "", - category_id: video.snippet?.categoryId || "", - published_at: video.snippet?.publishedAt || "", - tags: video.snippet?.tags || [], - view_count: parseInt(video.statistics?.viewCount || "0"), - like_count: parseInt(video.statistics?.likeCount || "0"), - comment_count: parseInt(video.statistics?.commentCount || "0"), - made_for_kids: video.contentDetails?.contentRating?.ytRating === "ytAgeRestricted" || false, - duration: video.contentDetails?.duration || "", - }; +/** + * Get metadata for multiple videos + * + * Uses multi-tier caching: + * 1. Local IndexedDB (fastest, per-user) + * 2. Server API which uses Redis (shared across users) + YouTube API + * + * Never throws - returns partial results on error + */ +export async function getVideosMetadata( + videoIds: string[] +): Promise> { + const startTime = Date.now(); + console.log(`[Metadata] Starting fetch for ${videoIds.length} videos`); - // Store in memory - metadataMap.set(video.id, metadata); + // Deduplicate + const uniqueIds = [...new Set(videoIds)]; + const metadataMap = new Map(); - // Prepare for Redis caching - const compressedEntry = deflateSync(JSON.stringify(metadata)); - const encodedEntry = Buffer.from(compressedEntry).toString("base64"); - cacheEntries[video.id] = encodedEntry; - } - } catch (error: any) { - console.error(`❌ Error processing batch ${index + 1}:`, error.message); - } - }; + try { + // Step 1: Check local cache + const localCached = await getFromLocalCache(uniqueIds); + console.log(`[Metadata] Local cache: ${localCached.size}/${uniqueIds.length}`); + + for (const [id, metadata] of localCached) { + metadataMap.set(id, metadata); + } - // Process batches with controlled concurrency - for (let i = 0; i < batches.length; i += CONCURRENCY_LIMIT) { - const currentBatches = batches.slice(i, i + CONCURRENCY_LIMIT); - await Promise.all(currentBatches.map((batch, index) => processBatch(batch, i + index))); - - // Add a small delay between groups of concurrent requests - if (i + CONCURRENCY_LIMIT < batches.length) { - await new Promise(resolve => setTimeout(resolve, 500)); // 0.5 second delay between groups - } + // Step 2: Fetch missing from server + const missingIds = uniqueIds.filter(id => !metadataMap.has(id)); + + if (missingIds.length > 0) { + console.log(`[Metadata] Fetching ${missingIds.length} from server...`); + + const serverData = await fetchFromServer(missingIds); + + for (const [id, metadata] of serverData) { + metadataMap.set(id, metadata); } - // Cache all entries in Redis at once using MSET - if (Object.keys(cacheEntries).length > 0) { - console.log(`🔄 Caching ${Object.keys(cacheEntries).length} videos in Redis...`); - - const CACHE_CHUNK_SIZE = 500; - const entries = Object.entries(cacheEntries); - const chunks = []; - - // Split entries into chunks of 10,000 - for (let i = 0; i < entries.length; i += CACHE_CHUNK_SIZE) { - chunks.push(entries.slice(i, i + CACHE_CHUNK_SIZE)); - } - - console.log(`📦 Caching in ${chunks.length} chunks of up to ${CACHE_CHUNK_SIZE} entries each...`); - - // Process each chunk with a delay - for (let i = 0; i < chunks.length; i++) { - const chunk = chunks[i]; - const chunkObject = Object.fromEntries(chunk); - - console.log(`🔄 Caching chunk ${i + 1}/${chunks.length} (${chunk.length} entries)...`); - await redis.mset(chunkObject); - - // Add delay between chunks except for the last one - if (i < chunks.length - 1) { - console.log(`⏳ Waiting 2 seconds before next cache chunk...`); - await new Promise(resolve => setTimeout(resolve, 2000)); - } - } + // Step 3: Save newly fetched to local cache (background) + if (serverData.size > 0) { + saveToLocalCache(serverData).catch(err => + console.warn('[Metadata] Background cache save failed:', err) + ); } } - console.log(`✅ Successfully processed ${metadataMap.size} videos`); + const duration = Date.now() - startTime; + console.log(`[Metadata] Complete: ${metadataMap.size}/${uniqueIds.length} in ${duration}ms`); + + return metadataMap; + } catch (error) { + console.error('[Metadata] Error:', error); + // Return what we have return metadataMap; - } catch (error: any) { - console.error("❌ Error in getVideosMetadata:", error.message); - throw error; + } +} + +/** + * Clear the local metadata cache + */ +export async function clearLocalMetadataCache(): Promise { + try { + const db = await getDB(); + const tx = db.transaction(STORE_NAME, 'readwrite'); + await tx.objectStore(STORE_NAME).clear(); + await tx.done; + console.log('[Metadata] Local cache cleared'); + } catch (error) { + console.warn('[Metadata] Failed to clear cache:', error); } } diff --git a/package.json b/package.json index 58612b7..d92a972 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "@radix-ui/react-select": "^2.2.2", "@radix-ui/react-separator": "1.1.1", "@radix-ui/react-slot": "1.1.1", + "@radix-ui/react-switch": "^1.2.6", "@radix-ui/react-tabs": "1.1.2", "@radix-ui/react-toast": "1.2.4", "@types/dexie": "^1.3.32", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fa1af79..4a6f022 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -41,6 +41,9 @@ importers: '@radix-ui/react-slot': specifier: 1.1.1 version: 1.1.1(@types/react@19.2.7)(react@18.3.1) + '@radix-ui/react-switch': + specifier: ^1.2.6 + version: 1.2.6(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) '@radix-ui/react-tabs': specifier: 1.1.2 version: 1.1.2(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -1168,6 +1171,19 @@ packages: '@types/react': optional: true + '@radix-ui/react-switch@1.2.6': + resolution: {integrity: sha512-bByzr1+ep1zk4VubeEVViV592vu2lHE2BZY5OnzehZqOOgogN80+mNtCqPkhn2gklJqOpxWgPoYTSnhBCqpOXQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-tabs@1.1.2': resolution: {integrity: sha512-9u/tQJMcC2aGq7KXpGivMm1mgq7oRJKXphDwdypPd/j21j/2znamPU8WkXgnhUaTrSFNIt8XhOyCAupg8/GbwQ==} peerDependencies: @@ -4166,6 +4182,21 @@ snapshots: optionalDependencies: '@types/react': 19.2.7 + '@radix-ui/react-switch@1.2.6(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.7)(react@18.3.1) + '@radix-ui/react-context': 1.1.2(@types/react@19.2.7)(react@18.3.1) + '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.7)(react@18.3.1) + '@radix-ui/react-use-previous': 1.1.1(@types/react@19.2.7)(react@18.3.1) + '@radix-ui/react-use-size': 1.1.1(@types/react@19.2.7)(react@18.3.1) + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + optionalDependencies: + '@types/react': 19.2.7 + '@types/react-dom': 19.2.3(@types/react@19.2.7) + '@radix-ui/react-tabs@1.1.2(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': dependencies: '@radix-ui/primitive': 1.1.1 diff --git a/types/youtube.ts b/types/youtube.ts new file mode 100644 index 0000000..970d97f --- /dev/null +++ b/types/youtube.ts @@ -0,0 +1,43 @@ +// YouTube-related types shared across the application + +export interface YouTubeVideoMetadata { + video_id: string; + title: string; + channel: string; + category_id: string; + published_at: string; + tags: string[]; + view_count: number; + like_count: number; + comment_count: number; + made_for_kids: boolean; + duration: string; +} + +export interface WatchHistoryEntry { + title: string; + titleUrl?: string; + time: string; + video_id?: string; + // Metadata fields (added after processing) + channel?: string; + category_id?: string; + published_at?: string; + tags?: string[]; + view_count?: number; + like_count?: number; + comment_count?: number; + made_for_kids?: boolean; + duration?: string; +} + +export interface MetadataRequest { + videoIds: string[]; +} + +export interface MetadataResponse { + metadata: Record; + cached: number; + fetched: number; + failed: number; +}