logger_config.py

import os
import logging
from datetime import datetime

class LoggerConfig:
    """Centralized logging configuration for the entire application."""
    
    def __init__(self):
        # Create logs directory if it doesn't exist
        self.logs_dir = "./logs"
        self.debug_dir = os.path.join(self.logs_dir, "debug")
        os.makedirs(self.logs_dir, exist_ok=True)
        os.makedirs(self.debug_dir, exist_ok=True)
        
        # Get debug mode from environment variable
        self.debug_mode = os.getenv('GRAPHRAG_DEBUG', '').lower() == 'true'
        
        # Create log filenames with timestamp
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        self.main_log = os.path.join(self.logs_dir, f"graphrag_{timestamp}.log")
        self.scraping_log = os.path.join(self.debug_dir, f"scraping_debug_{timestamp}.log")
        
        # Set up handlers
        self._setup_main_logger()
        self._setup_scraping_logger()
        self._setup_llm_logger()
        
        # Log initial messages
        logging.info(f"Starting new session, logging to: {self.main_log}")
        if self.debug_mode:
            logging.info("Debug mode is ON - detailed logging enabled")
            logging.info(f"Web scraping debug logs will be written to: {self.scraping_log}")
    
    def _setup_main_logger(self):
        """Set up the main application logger."""
        # Main file handler
        main_handler = logging.FileHandler(self.main_log, mode='w')
        main_handler.setLevel(logging.DEBUG if self.debug_mode else logging.INFO)
        main_handler.setFormatter(logging.Formatter(
            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
        ))
        
        # Console handler
        console_handler = logging.StreamHandler()
        console_handler.setLevel(logging.INFO)
        console_handler.setFormatter(logging.Formatter(
            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
        ))
        
        # Configure root logger
        logging.basicConfig(
            level=logging.DEBUG if self.debug_mode else logging.INFO,
            handlers=[main_handler, console_handler],
            force=True
        )
    
    def _setup_scraping_logger(self):
        """Set up the web scraping logger."""
        scraping_logger = logging.getLogger('web-scraping')
        scraping_logger.setLevel(logging.DEBUG if self.debug_mode else logging.INFO)
        scraping_logger.propagate = False
        
        # Create handler for scraping logs
        scraping_handler = logging.FileHandler(self.scraping_log, mode='w', encoding='utf-8')
        scraping_handler.setLevel(logging.DEBUG if self.debug_mode else logging.INFO)
        scraping_handler.setFormatter(logging.Formatter(
            '%(asctime)s - %(levelname)s - %(message)s'
        ))
        
        scraping_logger.addHandler(scraping_handler)
    
    def _setup_llm_logger(self):
        """Set up the LLM interactions logger."""
        llm_logger = logging.getLogger('llm-interaction')
        llm_logger.setLevel(logging.DEBUG if self.debug_mode else logging.INFO)
        
        # Use the main log file for LLM interactions
        llm_handler = logging.FileHandler(self.main_log, mode='a')
        llm_handler.setLevel(logging.DEBUG if self.debug_mode else logging.INFO)
        llm_handler.setFormatter(logging.Formatter(
            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
        ))
        
        llm_logger.addHandler(llm_handler)
    
    def get_scraping_logger(self):
        """Get the web scraping logger."""
        return logging.getLogger('web-scraping')
    
    def get_llm_logger(self):
        """Get the LLM interactions logger."""
        return logging.getLogger('llm-interaction')
    
    def log_scraped_content(self, url, title, content):
        """Log scraped content to a debug file."""
        if not self.debug_mode:
            return None
            
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        url_hash = abs(hash(url))
        debug_file = os.path.join(self.debug_dir, f"content_{timestamp}_{url_hash}.txt")
        
        try:
            with open(debug_file, 'w', encoding='utf-8') as f:
                f.write("=" * 80 + "\n")
                f.write(f"Timestamp: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
                f.write(f"URL: {url}\n")
                f.write(f"Title: {title}\n")
                f.write(f"Word Count: {len(content.split())}\n")
                f.write("=" * 80 + "\n")
                f.write("\nCONTENT:\n")
                f.write("=" * 80 + "\n\n")
                f.write(content)
                f.write("\n\n" + "=" * 80 + "\n")
                f.flush()
            
            scraping_logger = self.get_scraping_logger()
            scraping_logger.debug(f"Scraped content saved to: {debug_file}")
            scraping_logger.debug(f"Content summary - URL: {url}, Title: {title}, Words: {len(content.split())}")
            
            return debug_file
        except Exception as e:
            logging.error(f"Error saving debug content for {url}: {str(e)}")
            return None

# Create a global instance
logger_config = LoggerConfig()