diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..a86ec7d --- /dev/null +++ b/.env.example @@ -0,0 +1,64 @@ +# .env.example - Copy to .env for local development and fill in your values +# Server Port +# PORT=3000 + +# Logging Level (e.g., 'debug', 'info', 'warn', 'error') +# LOG_LEVEL=info + +# Redis Configuration +# REDIS_HOST=127.0.0.1 +# REDIS_PORT=6379 +# REDIS_PASSWORD=your_redis_password + +# API Keys +GROQ_API_KEY=your_groq_api_key +ELEVENLABS_API_KEY=your_elevenlabs_api_key +# Add other API keys for services that require them (e.g. Runway, Claude if they have direct key auth) + +# Google Cloud Credentials +# Option 1: Path to service account JSON file (recommended for local dev, CI/CD, some cloud environments) +# GOOGLE_APPLICATION_CREDENTIALS=/path/to/your/gcp-credentials.json + +# Option 2: JSON string content of the service account key (useful for some PaaS environments) +# Ensure the JSON string is correctly escaped if necessary when setting the environment variable. +# GOOGLE_CREDENTIALS_JSON='{"type": "service_account", "project_id": "your-project-id", ...}' + +# External Service URLs (override defaults if necessary, though many services are SDK/Playwright based) +# CLAUDE_SERVICE_URL=https://claude.ai +# GEMINI_SERVICE_URL=https://gemini.google.com +# ELEVENLABS_SERVICE_URL=https://elevenlabs.io +# RUNWAY_SERVICE_URL=https://runway.ml +# CANVA_SERVICE_URL=https://canva.com + +# Worker and Queue Settings +# WORKER_CONCURRENCY=5 +# JOB_DEFAULT_ATTEMPTS=3 +# JOB_DEFAULT_BACKOFF_DELAY=5000 # milliseconds + +# API Rate Limiting Settings +# API_RATE_LIMIT_WINDOW_MS=900000 # 15 minutes in milliseconds +# API_RATE_LIMIT_MAX=100 + +# Worker Rate Limiter Settings (for BullMQ worker) +# WORKER_RATE_LIMIT_MAX=10 +# WORKER_RATE_LIMIT_DURATION_MS=60000 # milliseconds + +# Max characters for text input to AI services (e.g., Groq, Claude) +# AI_INPUT_MAX_CHARS=80000 + +# Timeouts for External Interactions (in milliseconds) +# DEFAULT_EXTERNAL_API_TIMEOUT_MS=30000 +# GROQ_TIMEOUT_MS=30000 +# WEB_EXTRACTOR_NAVIGATION_TIMEOUT_MS=60000 +# NEW_SERVICE_AI_REQUEST_TIMEOUT_MS=60000 +# RUNWAY_VIDEO_GENERATION_TIMEOUT_MS=180000 +# RUNWAY_DOWNLOAD_TIMEOUT_MS=60000 + +# Debugging - Playwright Failure Artifacts +# If true, saves a screenshot and HTML dump to tempDir on Playwright errors in services like youtube.js +# DEBUG_SAVE_PLAYWRIGHT_FAILURE_ARTIFACTS=false + +# Temporary Directory +# TEMP_DIR=/tmp/viral_content_temp # Example absolute path for overriding default (project_root/temp) +# If using a relative path, ensure it's handled correctly by the application. +# The default in config/index.js is path.join(__dirname, '..', 'temp') which resolves to /temp diff --git a/config/index.js b/config/index.js new file mode 100644 index 0000000..e91bc32 --- /dev/null +++ b/config/index.js @@ -0,0 +1,79 @@ +// config/index.js +require('dotenv').config(); // Load .env file if present (primarily for development) + +module.exports = { + // Server Port + port: parseInt(process.env.PORT, 10) || 3000, + + // Logging Level + logLevel: process.env.LOG_LEVEL || 'info', + + // Redis Configuration + redis: { + host: process.env.REDIS_HOST || '127.0.0.1', + port: parseInt(process.env.REDIS_PORT, 10) || 6379, + password: process.env.REDIS_PASSWORD || undefined, + }, + + // API Keys + groqApiKey: process.env.GROQ_API_KEY, + // Add other API keys here as they are identified, e.g., ELEVENLABS_API_KEY + elevenlabsApiKey: process.env.ELEVENLABS_API_KEY, + + + // Google Cloud Credentials + googleApplicationCredentials: process.env.GOOGLE_APPLICATION_CREDENTIALS, // Path to JSON file + googleCredentialsJson: process.env.GOOGLE_CREDENTIALS_JSON, // JSON string + + // External Service URLs (defaults provided) + // These will be used to populate serviceRegistry dynamically + serviceUrls: { + claude: process.env.CLAUDE_SERVICE_URL || 'https://claude.ai', + gemini: process.env.GEMINI_SERVICE_URL || 'https://gemini.google.com', + elevenlabs: process.env.ELEVENLABS_SERVICE_URL || 'https://elevenlabs.io', + runway: process.env.RUNWAY_SERVICE_URL || 'https://runway.ml', + canva: process.env.CANVA_SERVICE_URL || 'https://canva.com', + // YouTube, TikTok, Instagram URLs are more for reference, + // as their services might use Playwright or SDKs directly. + // But can be included for consistency if needed. + youtube: process.env.YOUTUBE_SERVICE_URL || 'https://youtube.com', + tiktok: process.env.TIKTOK_SERVICE_URL || 'https://tiktok.com', + instagram: process.env.INSTAGRAM_SERVICE_URL || 'https://instagram.com', + }, + + // Worker and Queue Settings + workerConcurrency: parseInt(process.env.WORKER_CONCURRENCY, 10) || 5, + jobDefaultAttempts: parseInt(process.env.JOB_DEFAULT_ATTEMPTS, 10) || 3, + jobDefaultBackoffDelay: parseInt(process.env.JOB_DEFAULT_BACKOFF_DELAY, 10) || 5000, // ms + + // API Rate Limiting Settings + apiRateLimitWindowMs: parseInt(process.env.API_RATE_LIMIT_WINDOW_MS, 10) || 15 * 60 * 1000, // 15 minutes + apiRateLimitMax: parseInt(process.env.API_RATE_LIMIT_MAX, 10) || 100, + + // Worker Rate Limiter Settings + workerRateLimit: { + max: parseInt(process.env.WORKER_RATE_LIMIT_MAX, 10) || 10, // Max jobs per duration + duration: parseInt(process.env.WORKER_RATE_LIMIT_DURATION_MS, 10) || 60000, // Duration in milliseconds + }, + + // Add other configurations here as needed + // Example: TEMP_DIR + tempDir: process.env.TEMP_DIR || require('path').join(__dirname, '..', 'temp'), // Relative to project root + + // AI Input Settings + aiInputMaxChars: parseInt(process.env.AI_INPUT_MAX_CHARS, 10) || 80000, // Max chars for AI input + + // Timeouts for External Interactions (in milliseconds) + timeouts: { + defaultExternalApiMs: parseInt(process.env.DEFAULT_EXTERNAL_API_TIMEOUT_MS, 10) || 30000, + groqMs: parseInt(process.env.GROQ_TIMEOUT_MS, 10) || 30000, + webExtractorNavigationMs: parseInt(process.env.WEB_EXTRACTOR_NAVIGATION_TIMEOUT_MS, 10) || 60000, + newServiceAiRequestMs: parseInt(process.env.NEW_SERVICE_AI_REQUEST_TIMEOUT_MS, 10) || 60000, + runwayVideoGenerationMs: parseInt(process.env.RUNWAY_VIDEO_GENERATION_TIMEOUT_MS, 10) || 180000, // 3 minutes + runwayDownloadMs: parseInt(process.env.RUNWAY_DOWNLOAD_TIMEOUT_MS, 10) || 60000, // 1 minute + }, + + debug: { + savePlaywrightFailureArtifacts: (process.env.DEBUG_SAVE_PLAYWRIGHT_FAILURE_ARTIFACTS === 'true') || false, + }, +}; diff --git a/core/viralSystem.js b/core/viralSystem.js new file mode 100644 index 0000000..c6c5ec9 --- /dev/null +++ b/core/viralSystem.js @@ -0,0 +1,379 @@ +// core/viralSystem.js +const { google } = require('googleapis'); +const fs = require('fs').promises; +const path = require('path'); +const { v4: uuidv4 } = require('uuid'); +const stream = require('stream'); // Required for fs.createReadStream +const config = require('../config'); // Added config require +const logger = require('../lib/logger'); // Added logger require + +// TEMP_DIR is now sourced from config.tempDir +// CREDENTIALS_PATH is removed as we are using environment variables. + +// Helper function for text truncation +function truncateText(text, maxLength) { + if (typeof text !== 'string') { + logger.warn({ inputTextType: typeof text }, 'truncateText received non-string input. Returning as is.'); + return text; + } + if (text.length <= maxLength) { + return text; + } + logger.debug({ originalLength: text.length, maxLength }, `Truncating text to ${maxLength} characters.`); + return text.substring(0, maxLength); +} + +// Define serviceRegistry here +// Paths are relative to this file (core/viralSystem.js) + +// List of services that are expected but their module files are missing +const missingServiceFiles = [ + 'claude.js', + 'gemini.js', + 'elevenlabs.js', + 'canva.js', + 'tiktok.js', + 'instagram.js' +]; + +if (missingServiceFiles.length > 0) { + logger.warn({ + disabledServices: missingServiceFiles.map(f => f.replace('.js', '')), + missingFiles: missingServiceFiles.map(f => `services/${f}`) + }, 'Some services are disabled due to missing module files. Corresponding entries in serviceRegistry will be commented out.'); +} + +const serviceRegistry = { + // Assuming 'services' directory is at project root, sibling to 'core' + webExtractor: { module: '../services/webExtractor', type: 'local' }, // Does not use a URL from config.serviceUrls + groq: { module: '../services/groq', type: 'api' }, // Does not use a URL from config.serviceUrls for constructor + // claude: { module: '../services/claude', url: config.serviceUrls.claude }, // File not found: services/claude.js + // gemini: { module: '../services/gemini', url: config.serviceUrls.gemini }, // File not found: services/gemini.js + // elevenlabs: { module: '../services/elevenlabs', url: config.serviceUrls.elevenlabs }, // File not found: services/elevenlabs.js + runway: { module: '../services/runway', url: config.serviceUrls.runway }, + // canva: { module: '../services/canva', url: config.serviceUrls.canva }, // File not found: services/canva.js + youtube: { module: '../services/youtube', url: config.serviceUrls.youtube }, // Assuming constructor might take a base URL + // tiktok: { module: '../services/tiktok', url: config.serviceUrls.tiktok }, // File not found: services/tiktok.js + // instagram: { module: '../services/instagram', url: config.serviceUrls.instagram } // File not found: services/instagram.js +}; + +class ViralContentSystem { + constructor() { + this.services = {}; + this.driveClient = null; + // Service registry can be passed in or attached like this + // If it's global to the module, this.serviceRegistry isn't strictly needed + // but can be useful if registry could vary per instance (though not the case here). + this.serviceRegistry = serviceRegistry; + } + + async initialize() { // For base system resources (Drive, temp dirs) + try { + this.driveClient = await this.authenticateGoogleDrive(); + await fs.mkdir(config.tempDir, { recursive: true }); + logger.info('ViralContentSystem base initialized (Drive client, TempDir).'); + } catch (error) { + logger.error({ err: error }, 'Error during ViralContentSystem base initialization'); + throw error; + } + } + + async _loadService(name) { + if (this.services[name]) return this.services[name]; + + const serviceConfig = this.serviceRegistry[name]; // Renamed to avoid conflict with global config + if (!serviceConfig) { + logger.error({ serviceName: name }, 'Service config not found in registry.'); + throw new Error(`Unsupported service in VCS: ${name}`); + } + + const modulePath = serviceConfig.module; + + try { + const ServiceModule = require(modulePath); + const serviceInstance = serviceConfig.url ? + new ServiceModule(name, serviceConfig.url) : + new ServiceModule(); + + if (serviceInstance.initialize) { + await serviceInstance.initialize(); + } + this.services[name] = serviceInstance; + logger.info({ serviceName: name }, 'Service loaded for ViralContentSystem.'); + return serviceInstance; + } catch (error) { + logger.error({ err: error, serviceName: name, modulePath }, `Error loading service module`); + throw error; + } + } + + async initialize_dependent_services() { + logger.info('ViralContentSystem initializing dependent services...'); + this.services = {}; + for (const name of Object.keys(this.serviceRegistry)) { + try { + await this._loadService(name); + } catch (error) { + // Error is already logged in _loadService + logger.error({ err: error, serviceName: name }, `Failed to initialize service in ViralContentSystem. Error: ${error.message}`); + } + } + logger.info('ViralContentSystem dependent services initialization attempt complete.'); + } + + async authenticateGoogleDrive() { + let auth; + const scopes = ['https://www.googleapis.com/auth/drive']; + + if (config.googleCredentialsJson) { + logger.info('Attempting to use Google Drive credentials from GOOGLE_CREDENTIALS_JSON (via config).'); + try { + const credentials = JSON.parse(config.googleCredentialsJson); + auth = new google.auth.GoogleAuth({ credentials, scopes }); + } catch (error) { + logger.error({ err: error }, 'Failed to parse GOOGLE_CREDENTIALS_JSON (from config)'); + throw new Error('Malformed GOOGLE_CREDENTIALS_JSON (from config). Please check the environment variable or config setup.'); + } + } else if (config.googleApplicationCredentials) { + logger.info('Using Google Drive credentials from GOOGLE_APPLICATION_CREDENTIALS (via config).'); + auth = new google.auth.GoogleAuth({ scopes }); // Relies on GOOGLE_APPLICATION_CREDENTIALS env var being set + } else { + logger.error('Google Drive credentials not configured. Set GOOGLE_APPLICATION_CREDENTIALS or GOOGLE_CREDENTIALS_JSON.'); + throw new Error('Google Drive credentials not configured. Unable to initialize Drive client.'); + } + + return google.drive({ version: 'v3', auth }); + } + + async uploadToDrive(filePath, fileName) { + if (!this.driveClient) { + logger.error("Google Drive client not initialized. Cannot upload file."); + throw new Error("Google Drive client not initialized. Ensure credentials are set and valid."); + } + + const absoluteFilePath = path.isAbsolute(filePath) ? filePath : path.join(config.tempDir, filePath); + + try { + await fs.access(absoluteFilePath); + } catch (e) { + logger.error({ err: e, filePath: absoluteFilePath }, 'File not found for upload'); + throw new Error(`File not found for upload: ${absoluteFilePath}`); + } + + const media = { + mimeType: 'application/octet-stream', // Or determine dynamically + body: fs.createReadStream(absoluteFilePath) + }; + const res = await this.driveClient.files.create({ + requestBody: { name: fileName, parents: ['root'] }, // Consider making parent configurable + media, + fields: 'id, webViewLink' + }); + return res.data; + } + + async createViralContent(topic) { + const contentId = uuidv4(); + let strategy, assets = {}, finalVideo, driveResult, posts = {}; + + // Step 1: Content strategy with Groq + try { + if (!this.services.groq) throw new Error("Groq service not available/initialized."); + strategy = await this.services.groq.generateStrategy(topic); + logger.debug({ topic, strategyTitle: strategy.title }, `Groq strategy generated`); + } catch (error) { + logger.error({ err: error, topic, step: 'GroqStrategy' }, 'Error during Groq strategy generation'); + throw error; + } + + // Step 2: Media creation + const mediaCreationSteps = [ + { name: 'ClaudeScript', service: 'claude', func: 'generateScript', input: strategy, outputField: 'script' }, + { name: 'RunwayImage', service: 'runway', func: 'generateImage', input: strategy.visualPrompt, outputField: 'image' }, + { name: 'ElevenLabsAudio', service: 'elevenlabs', func: 'generateAudio', input: strategy.scriptSegment, outputField: 'audio' }, + { name: 'RunwayVideo', service: 'runway', func: 'generateVideo', input: strategy, outputField: 'video' } + ]; + + for (const step of mediaCreationSteps) { + try { + if (!this.services[step.service]) throw new Error(`${step.service} service not available/initialized.`); + assets[step.outputField] = await this.services[step.service][step.func](step.input); + logger.debug({ strategyTitle: strategy.title, step: step.name }, `${step.name} generated`); + } catch (error) { + logger.error({ err: error, strategyTitle: strategy.title, step: step.name }, `Error during ${step.name}`); + throw error; + } + } + + // Step 3: Compile final content + try { + if (!this.services.canva) throw new Error("Canva service not available/initialized."); + finalVideo = await this.services.canva.compileVideo({ + ...assets, + music: strategy.viralMusicPrompt, + title: strategy.title, + caption: strategy.caption, + }); + logger.debug({ strategyTitle: strategy.title }, `Video compiled with Canva`); + } catch (error) { + logger.error({ err: error, strategyTitle: strategy.title, step: 'CanvaCompilation' }, 'Error compiling video with Canva'); + throw error; + } + + // Step 4: Save to Drive + try { + const sanitizedTitle = strategy.title.replace(/[^a-zA-Z0-9]/g, '_'); + driveResult = await this.uploadToDrive( + finalVideo.path, + `${sanitizedTitle}-${contentId}.mp4` + ); + logger.info({ strategyTitle: strategy.title, driveLink: driveResult.webViewLink }, `Content uploaded to Drive`); + } catch (error) { + logger.error({ err: error, strategyTitle: strategy.title, step: 'DriveUpload' }, 'Error uploading to Drive'); + throw error; + } + + // Step 5: Social distribution + const socialServices = ['youtube', 'tiktok', 'instagram']; + for(const serviceName of socialServices) { + try { + if (!this.services[serviceName]) throw new Error(`${serviceName} service not available/initialized.`); + posts[serviceName] = await this.services[serviceName].postContent({ + videoPath: finalVideo.path, + title: strategy.title, + description: strategy.description, + caption: strategy.caption, + tags: strategy.hashtags + }); + logger.info({ strategyTitle: strategy.title, service: serviceName }, `Content posted to ${serviceName}`); + } catch (error) { + logger.error({ err: error, strategyTitle: strategy.title, service: serviceName, step: 'SocialDistribution'}, `Error posting to ${serviceName}`); + throw error; + } + } + + return { + contentId, + strategy, + driveLink: driveResult.webViewLink, + posts + }; + } + + async createViralContentFromUrl(url, userId) { + const contentId = uuidv4(); + let extractedText, strategy, assets = {}, finalVideoPath, driveResult, posts = {}; + + // Step 1: Extract text from URL + try { + if (!this.services.webExtractor) throw new Error("WebExtractorService not loaded or available."); + extractedText = await this.services.webExtractor.extractText(url); + if (!extractedText) { + logger.warn({ url }, 'No content extracted from URL (extractor returned null/empty)'); + throw new Error(`No content could be extracted from URL: ${url}`); + } + logger.debug({ url, originalTextLength: extractedText.length }, `Text extracted from URL`); + } catch (error) { + logger.error({ err: error, url, step: 'WebExtraction' }, `Error during web extraction from URL`); + throw error; + } + + // Process text for AI: Truncate if necessary + let processedTextForAI = extractedText; + if (extractedText && extractedText.length > config.aiInputMaxChars) { + processedTextForAI = truncateText(extractedText, config.aiInputMaxChars); + logger.warn({ + originalLength: extractedText.length, + truncatedLength: processedTextForAI.length, + maxLength: config.aiInputMaxChars, + url: url + }, 'Extracted text from URL was truncated before sending to AI strategy generator.'); + } + + // Step 2: Content strategy with Groq using processed (potentially truncated) text + try { + if (!this.services.groq) throw new Error("Groq service not available/initialized."); + const topicForGroq = `Content strategy for URL: ${url}`; // Topic can still be the full URL for context + strategy = await this.services.groq.generateStrategy(topicForGroq, processedTextForAI); + logger.debug({ url, strategyTitle: strategy.title, processedTextLength: processedTextForAI.length }, `Groq strategy generated for URL content`); + } catch (error) { + logger.error({ err: error, url, step: 'GroqStrategyForURL' }, 'Error during Groq strategy generation for URL'); + throw error; + } + + // Step 3: Media creation (same loop as createViralContent, context varies) + const mediaCreationStepsUrl = [ + { name: 'ClaudeScript', service: 'claude', func: 'generateScript', input: strategy, outputField: 'script' }, + { name: 'RunwayImage', service: 'runway', func: 'generateImage', input: strategy.visualPrompt, outputField: 'image' }, + { name: 'ElevenLabsAudio', service: 'elevenlabs', func: 'generateAudio', input: strategy.scriptSegment, outputField: 'audio' }, + { name: 'RunwayVideo', service: 'runway', func: 'generateVideo', input: strategy, outputField: 'video' } + ]; + + for (const step of mediaCreationStepsUrl) { + try { + if (!this.services[step.service]) throw new Error(`${step.service} service not available/initialized.`); + assets[step.outputField] = await this.services[step.service][step.func](step.input); + logger.debug({ strategyTitle: strategy.title, step: step.name, context: 'URL_Based' }, `${step.name} generated for URL content`); + } catch (error) { + logger.error({ err: error, strategyTitle: strategy.title, step: step.name, context: 'URL_Based' }, `Error during ${step.name} for URL content`); + throw error; + } + } + + // Step 4: Compile final content + try { + if (!this.services.canva) throw new Error("Canva service not available/initialized."); + finalVideoPath = await this.services.canva.compileVideo({ + ...assets, + music: strategy.viralMusicPrompt, + title: strategy.title, + caption: strategy.caption + }); + logger.debug({ strategyTitle: strategy.title, context: 'URL_Based' }, `Video compiled with Canva for URL content`); + } catch (error) { + logger.error({ err: error, strategyTitle: strategy.title, step: 'CanvaCompilationURL', context: 'URL_Based' }, 'Error compiling video with Canva for URL content'); + throw error; + } + + // Step 5: Save to Drive + try { + const sanitizedTitle = strategy.title.replace(/[^a-zA-Z0-9]/g, '_'); + driveResult = await this.uploadToDrive( + finalVideoPath, + `${sanitizedTitle}-${contentId}.mp4` + ); + logger.info({ strategyTitle: strategy.title, driveLink: driveResult.webViewLink, context: 'URL_Based' }, `Content from URL uploaded to Drive`); + } catch (error) { + logger.error({ err: error, strategyTitle: strategy.title, step: 'DriveUploadURL', context: 'URL_Based' }, 'Error uploading to Drive for URL content'); + throw error; + } + + // Step 6: Social distribution + const socialServices = ['youtube', 'tiktok', 'instagram']; + for(const serviceName of socialServices) { + try { + if (!this.services[serviceName]) throw new Error(`${serviceName} service not available/initialized.`); + posts[serviceName] = await this.services[serviceName].postContent({ + videoPath: finalVideoPath, + title: strategy.title, + description: strategy.description, + caption: strategy.caption, + tags: strategy.hashtags + }); + logger.info({ strategyTitle: strategy.title, service: serviceName, context: 'URL_Based' }, `Content from URL posted to ${serviceName}`); + } catch (error) { + logger.error({ err: error, strategyTitle: strategy.title, service: serviceName, step: 'SocialDistributionURL', context: 'URL_Based'}, `Error posting content from URL to ${serviceName}`); + throw error; + } + } + + return { + contentId, + strategy, + driveLink: driveResult.webViewLink, + posts + }; + } +} + +module.exports = { ViralContentSystem }; diff --git a/lib/logger.js b/lib/logger.js new file mode 100644 index 0000000..bcc250d --- /dev/null +++ b/lib/logger.js @@ -0,0 +1,37 @@ +// lib/logger.js +const pino = require('pino'); +const config = require('../config'); // Assuming config/index.js exists + +const isProduction = process.env.NODE_ENV === 'production'; + +const loggerOptions = { + level: config.logLevel || 'info', // Default to 'info' if not in config + // Pino includes timestamp, pid, hostname by default. + // In JSON output for production, these are good. + // For development, pino-pretty can simplify the output. +}; + +// Conditional transport for pretty printing in development +if (!isProduction) { + loggerOptions.transport = { + target: 'pino-pretty', + options: { + colorize: true, + translateTime: 'SYS:yyyy-mm-dd HH:MM:ss.l', // More readable timestamp format + ignore: 'pid,hostname', // Fields to ignore in pretty print + levelFirst: true, // Show level first + messageFormat: '{levelLabel} - {msg}', // Custom message format + }, + }; +} + +const logger = pino(loggerOptions); + +// Log that the logger is initialized (using itself) +// Using an object for the first argument to pino allows structured logging +logger.info({ + logLevel: logger.level, // Use logger.level to get the actual level pino is using + prettyPrint: !isProduction, +}, 'Pino logger initialized.'); + +module.exports = logger; diff --git a/lib/queue.js b/lib/queue.js new file mode 100644 index 0000000..91f78c3 --- /dev/null +++ b/lib/queue.js @@ -0,0 +1,69 @@ +// lib/queue.js +const { Queue } = require('bullmq'); +const config = require('../config'); // Added config require +const logger = require('../lib/logger'); // Added logger require + +const QUEUE_NAME = 'contentCreationQueue'; + +// Create a connection object for ioredis +const connection = { + host: config.redis.host, + port: config.redis.port, + password: config.redis.password, // Will be undefined if not set, which is fine for ioredis + maxRetriesPerRequest: null // Recommended by BullMQ docs for some environments +}; + +const contentCreationQueue = new Queue(QUEUE_NAME, { + connection, + defaultJobOptions: { // Default options for jobs added to this queue + attempts: config.jobDefaultAttempts, // Retry failed jobs up to X times + backoff: { + type: 'exponential', + delay: config.jobDefaultBackoffDelay, // Initial delay + }, + removeOnComplete: { // Keep completed jobs for a limited time or count + count: 1000, // Keep the last 1000 completed jobs + age: 24 * 60 * 60 // Keep for 24 hours (in seconds) + }, + removeOnFail: { // Keep failed jobs for a longer period or more count + count: 5000, // Keep the last 5000 failed jobs + age: 7 * 24 * 60 * 60 // Keep for 7 days (in seconds) + } + } +}); + +contentCreationQueue.on('error', (error) => { + logger.error({ err: error, queueName: QUEUE_NAME }, `BullMQ Queue Error`); +}); + +// Simple check to see if connection is established (optional) +// BullMQ doesn't have a direct 'connect' event on the Queue object itself for the initial connection in the same way ioredis client does. +// The queue will attempt to connect when operations are performed or workers are attached. +// We can, however, try a benign command or check client status if we had direct access to the ioredis instance BullMQ uses. +// For now, the 'error' listener and successful instantiation are primary indicators. + +// A more robust check could involve trying to add a dummy job or querying queue status, +// but that's more involved than typical initialization logging. +// Alternatively, BullMQ's Worker class has more explicit connection events. + +logger.info({ + queueName: QUEUE_NAME, + redisHost: config.redis.host, + redisPort: config.redis.port +}, `BullMQ Queue initialized. Waiting for connection to Redis.`); + +// To confirm connection, you might ping Redis using the client BullMQ creates: +// (async () => { +// try { +// const redisClient = await contentCreationQueue.client; // Gets the ioredis client instance +// const pong = await redisClient.ping(); +// if (pong === 'PONG') { +// console.log(`Successfully connected to Redis and received PONG for queue ${QUEUE_NAME}.`); +// } +// } catch (err) { +// console.error(`Failed to connect to Redis for queue ${QUEUE_NAME}:`, err); +// } +// })(); + + +module.exports = contentCreationQueue; diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..c8ead77 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,1365 @@ +{ + "name": "app", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "async-retry": "^1.3.3", + "bullmq": "^5.53.2", + "express-rate-limit": "^7.5.0", + "helmet": "^8.1.0", + "ioredis": "^5.6.1", + "pino": "^9.7.0" + }, + "devDependencies": { + "dotenv": "^16.5.0", + "pino-pretty": "^13.0.0" + } + }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "peer": true, + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/async-retry": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", + "dependencies": { + "retry": "0.13.1" + } + }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/body-parser": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.0.tgz", + "integrity": "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==", + "peer": true, + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.0", + "http-errors": "^2.0.0", + "iconv-lite": "^0.6.3", + "on-finished": "^2.4.1", + "qs": "^6.14.0", + "raw-body": "^3.0.0", + "type-is": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/bullmq": { + "version": "5.53.2", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.53.2.tgz", + "integrity": "sha512-xHgxrP/yNJHD7VCw1h+eRBh+2TCPBCM39uC9gCyksYc6ufcJP+HTZ/A2lzB2x7qMFWrvsX7tM40AT2BmdkYL/Q==", + "dependencies": { + "cron-parser": "^4.9.0", + "ioredis": "^5.4.1", + "msgpackr": "^1.11.2", + "node-abort-controller": "^3.1.1", + "semver": "^7.5.4", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "peer": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "peer": true, + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "peer": true, + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "dev": true + }, + "node_modules/content-disposition": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.0.tgz", + "integrity": "sha512-Au9nRL8VNUut/XSzbQA38+M78dzP4D+eqg3gfJHMIHHYa3bg067xj1KxMUWj+VULbiZMowKngFFbKczUrNJ1mg==", + "peer": true, + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "peer": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "peer": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "peer": true, + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/dateformat": { + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", + "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "peer": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/detect-libc": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz", + "integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/dotenv": { + "version": "16.5.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz", + "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "peer": true, + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "peer": true + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "peer": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dev": true, + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "peer": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "peer": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "peer": true, + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "peer": true + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "peer": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/express/-/express-5.1.0.tgz", + "integrity": "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA==", + "peer": true, + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.0", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-7.5.0.tgz", + "integrity": "sha512-eB5zbQh5h+VenMPM3fh+nw1YExi5nMr6HUCR62ELSP11huvxm/Uir1H1QEyTkk5QX6A58pX6NmaTMceKZ0Eodg==", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": "^4.11 || 5 || ^5.0.0-beta.1" + } + }, + "node_modules/fast-copy": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-3.0.2.tgz", + "integrity": "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ==", + "dev": true + }, + "node_modules/fast-redact": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.5.0.tgz", + "integrity": "sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A==", + "engines": { + "node": ">=6" + } + }, + "node_modules/fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", + "dev": true + }, + "node_modules/finalhandler": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.0.tgz", + "integrity": "sha512-/t88Ty3d5JWQbWYgaOGCCYfXRwV1+be02WqYYlL6h0lEiUAMPM8o8qKGO01YIkOHzka2up08wvgYD0mDiI+q3Q==", + "peer": true, + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "peer": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "peer": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "peer": true, + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "peer": true, + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "peer": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "peer": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "peer": true, + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/helmet": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/helmet/-/helmet-8.1.0.tgz", + "integrity": "sha512-jOiHyAZsmnr8LqoPGmCjYAaiuWwjAPLgY8ZX2XrmHawt99/u1y6RgrZMTeoPfpUbV96HOalYgz1qzkRbw54Pmg==", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/help-me": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz", + "integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==", + "dev": true + }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "peer": true, + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "peer": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "peer": true + }, + "node_modules/ioredis": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz", + "integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "peer": true, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "peer": true + }, + "node_modules/joycon": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", + "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, + "node_modules/luxon": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.6.1.tgz", + "integrity": "sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ==", + "engines": { + "node": ">=12" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "peer": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "peer": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "peer": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "peer": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.1.tgz", + "integrity": "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==", + "peer": true, + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/msgpackr": { + "version": "1.11.4", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.4.tgz", + "integrity": "sha512-uaff7RG9VIC4jacFW9xzL3jc0iM32DNHe4jYVycBcjUePT/Klnfj7pqtWJt9khvDFizmjN2TlYniYmSS2LIaZg==", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "peer": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==" + }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "peer": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "peer": true, + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "peer": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.2.0.tgz", + "integrity": "sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ==", + "peer": true, + "engines": { + "node": ">=16" + } + }, + "node_modules/pino": { + "version": "9.7.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-9.7.0.tgz", + "integrity": "sha512-vnMCM6xZTb1WDmLvtG2lE/2p+t9hDEIvTWJsu6FejkE62vB7gDhvzrpFR4Cw2to+9JNQxVnkAKVPA1KPB98vWg==", + "dependencies": { + "atomic-sleep": "^1.0.0", + "fast-redact": "^3.1.1", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^2.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^3.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-2.0.0.tgz", + "integrity": "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-pretty": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-13.0.0.tgz", + "integrity": "sha512-cQBBIVG3YajgoUjo1FdKVRX6t9XPxwB9lcNJVD5GCnNM4Y6T12YYx8c6zEejxQsU0wrg9TwmDulcE9LR7qcJqA==", + "dev": true, + "dependencies": { + "colorette": "^2.0.7", + "dateformat": "^4.6.3", + "fast-copy": "^3.0.2", + "fast-safe-stringify": "^2.1.1", + "help-me": "^5.0.0", + "joycon": "^3.1.1", + "minimist": "^1.2.6", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^2.0.0", + "pump": "^3.0.0", + "secure-json-parse": "^2.4.0", + "sonic-boom": "^4.0.1", + "strip-json-comments": "^3.1.1" + }, + "bin": { + "pino-pretty": "bin.js" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.0.0.tgz", + "integrity": "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA==" + }, + "node_modules/process-warning": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", + "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ] + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "peer": true, + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/pump": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "dev": true, + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "peer": true, + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "peer": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.0.tgz", + "integrity": "sha512-RmkhL8CAyCRPXCE28MMH0z2PNWQBNk2Q09ZdxM9IOOXwxwZbN+qbWaatPkdkWIKL2ZVDImrN/pK5HTRz2PcS4g==", + "peer": true, + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.6.3", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "engines": { + "node": ">= 12.13.0" + } + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "engines": { + "node": ">= 4" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "peer": true, + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "peer": true + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "engines": { + "node": ">=10" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "peer": true + }, + "node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", + "dev": true + }, + "node_modules/semver": { + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.0.tgz", + "integrity": "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw==", + "peer": true, + "dependencies": { + "debug": "^4.3.5", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "mime-types": "^3.0.1", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/serve-static": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ==", + "peer": true, + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "peer": true + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "peer": true, + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "peer": true, + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "peer": true, + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "peer": true, + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/sonic-boom": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.0.tgz", + "integrity": "sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww==", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "peer": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/thread-stream": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-3.1.0.tgz", + "integrity": "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==", + "dependencies": { + "real-require": "^0.2.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "peer": true, + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" + }, + "node_modules/type-is": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", + "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", + "peer": true, + "dependencies": { + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "peer": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "peer": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..4bae2c4 --- /dev/null +++ b/package.json @@ -0,0 +1,14 @@ +{ + "dependencies": { + "async-retry": "^1.3.3", + "bullmq": "^5.53.2", + "express-rate-limit": "^7.5.0", + "helmet": "^8.1.0", + "ioredis": "^5.6.1", + "pino": "^9.7.0" + }, + "devDependencies": { + "dotenv": "^16.5.0", + "pino-pretty": "^13.0.0" + } +} diff --git a/server.js b/server.js index 0d34d6b..f47d3e7 100644 --- a/server.js +++ b/server.js @@ -4,202 +4,209 @@ const { chromium } = require('playwright'); const fs = require('fs').promises; const path = require('path'); const axios = require('axios'); -const { google } = require('googleapis'); -const { v4: uuidv4 } = require('uuid'); +// server.js - Enhanced Viral Content MCP System +const express = require('express'); +// const { chromium } = require('playwright'); // Playwright might not be needed directly in server.js anymore +const fs = require('fs').promises; +const path = require('path'); +// const axios = require('axios'); // If not used by other parts of server.js, can be removed +// const { google } = require('googleapis'); // Moved to core/viralSystem.js +// const { v4: uuidv4 } = require('uuid'); // Moved to core/viralSystem.js +const contentCreationQueue = require('./lib/queue'); +const { ViralContentSystem } = require('./core/viralSystem'); // Import from new location +const helmet = require('helmet'); // Added helmet require +const rateLimit = require('express-rate-limit'); // Added express-rate-limit require +const config = require('./config'); // Added config require +const logger = require('./lib/logger'); // Added logger require const app = express(); -const port = 3000; +app.use(helmet()); // Use helmet for enhanced security +// const port = 3000; // Port now from config const SESSION_DIR = path.join(__dirname, 'sessions'); -const TEMP_DIR = path.join(__dirname, 'temp'); - -// Service registry with enhanced capabilities -const serviceRegistry = { - groq: { module: './services/groq', type: 'api' }, - claude: { module: './services/claude', url: 'https://claude.ai' }, - gemini: { module: './services/gemini', url: 'https://gemini.google.com' }, - elevenlabs: { module: './services/elevenlabs', url: 'https://elevenlabs.io' }, - runway: { module: './services/runway', url: 'https://runway.ml' }, - canva: { module: './services/canva', url: 'https://canva.com' }, - youtube: { module: './services/youtube', url: 'https://youtube.com' }, - tiktok: { module: './services/tiktok', url: 'https://tiktok.com' }, - instagram: { module: './services/instagram', url: 'https://instagram.com' } -}; +// const TEMP_DIR = path.join(__dirname, 'temp'); // TEMP_DIR is now managed by ViralContentSystem via config + +// Service registry, loadService, initializeServices are removed as they are now in ViralContentSystem. // Middleware app.use(express.json()); -class ViralContentSystem { - constructor() { - this.services = {}; - this.driveClient = null; - } - - async initialize() { - // Initialize Google Drive - this.driveClient = await this.authenticateGoogleDrive(); - - // Create temp directory - await fs.mkdir(TEMP_DIR, { recursive: true }); - } - - async authenticateGoogleDrive() { - const auth = new google.auth.GoogleAuth({ - keyFile: 'credentials.json', - scopes: ['https://www.googleapis.com/auth/drive'] - }); - return google.drive({ version: 'v3', auth }); - } - - async uploadToDrive(filePath, fileName) { - const media = { mimeType: 'application/octet-stream', body: fs.createReadStream(filePath) }; - const res = await this.driveClient.files.create({ - requestBody: { name: fileName, parents: ['root'] }, - media, - fields: 'id, webViewLink' - }); - return res.data; - } - - async createViralContent(topic) { - const contentId = uuidv4(); - - // Step 1: Content strategy with Groq - const strategy = await this.services.groq.generateStrategy(topic); - - // Step 2: Media creation - const assets = { - script: await this.services.claude.generateScript(strategy), - image: await this.services.runway.generateImage(strategy.visualPrompt), - audio: await this.services.elevenlabs.generateAudio(strategy.scriptSegment), - video: await this.services.runway.generateVideo(strategy) - }; - - // Step 3: Compile final content - const finalVideo = await this.services.canva.compileVideo({ - ...assets, - music: strategy.viralMusicPrompt - }); - - // Step 4: Save to Drive - const driveResult = await this.uploadToDrive( - finalVideo.path, - `${strategy.title}-${contentId}.mp4` - ); - - // Step 5: Social distribution - const posts = { - youtube: await this.services.youtube.postContent({ - video: finalVideo.path, - title: strategy.title, - description: strategy.description, - tags: strategy.hashtags - }), - tiktok: await this.services.tiktok.postContent({ - video: finalVideo.path, - caption: strategy.caption, - tags: strategy.hashtags - }), - instagram: await this.services.instagram.postContent({ - video: finalVideo.path, - caption: strategy.caption, - tags: strategy.hashtags - }) - }; - - return { - contentId, - strategy, - driveLink: driveResult.webViewLink, - posts - }; - } -} +// ViralContentSystem class definition is removed from here. + +// Initialize system (ViralContentSystem instance for the server, if needed for other routes or direct use) +// For a setup where all work is done by workers, this server-side instance might be minimal +// or not used for createViralContent/createViralContentFromUrl. +// However, the existing `start` function initializes and uses it for service cleanup. +// So, we still need an instance, but its services are loaded differently. +let viralSystem; // Declare to be initialized in start() -// Initialize system -const viralSystem = new ViralContentSystem(); -viralSystem.initialize(); +// Health Check Endpoint +app.get('/health', (req, res) => { + const healthStatus = { + status: 'ok', + timestamp: new Date().toISOString(), + uptime: process.uptime(), // Optional: include process uptime in seconds + }; + res.status(200).json(healthStatus); +}); + +// Rate limiter configuration +const apiLimiter = rateLimit({ + windowMs: config.apiRateLimitWindowMs, + max: config.apiRateLimitMax, + standardHeaders: true, + legacyHeaders: false, + message: { + jsonrpc: '2.0', + error: { + code: -32005, // Custom error code for rate limiting + message: 'Too many requests created from this IP, please try again after 15 minutes.' + }, + id: null // Typically, no specific request id for a rate limit global error + // If req.body.id is needed, a custom handler would be more appropriate + }, + // Note: For the 'id' in the rate limit message, if it's crucial to reflect the specific request's id, + // a custom 'handler' function for rateLimit would be needed to access `req.body.id`. + // The default 'message' option doesn't have direct access to 'req'. + // For now, 'id: null' is kept as per the simpler setup. +}); // MCP Endpoint for viral content creation -app.post('/mcp/viral-content', async (req, res) => { - const { id, method, params } = req.body; - +app.post('/mcp/viral-content', apiLimiter, async (req, res) => { // Added apiLimiter to the route + const { id: requestId, method, params } = req.body; // Renamed id to requestId for clarity + + // Ensure params is an object if it's undefined, for safer access later + const safeParams = params || {}; + try { - if (method !== 'create_viral_content') { + // Validate method and parameters first + if (method === 'create_viral_content') { + if (!safeParams.topic || typeof safeParams.topic !== 'string' || safeParams.topic.trim() === '') { + return res.status(400).json({ + jsonrpc: '2.0', + error: { code: -32602, message: 'Invalid params: Missing or empty topic' }, + id: requestId + }); + } + } else if (method === 'create_viral_content_from_url') { + if (!safeParams.url || typeof safeParams.url !== 'string' || safeParams.url.trim() === '') { + return res.status(400).json({ + jsonrpc: '2.0', + error: { code: -32602, message: 'Invalid params: Missing or empty url' }, + id: requestId + }); + } + } else { return res.status(400).json({ jsonrpc: '2.0', error: { code: -32601, message: 'Method not found' }, - id + id: requestId }); } - - const { topic } = params; - if (!topic) { - return res.status(400).json({ + + // Prepare job data + const jobData = { ...safeParams, userId: safeParams.userId || null }; // Pass necessary params to the job + // 'method' will be the job name + + // Add job to queue + try { + const job = await contentCreationQueue.add(method, jobData); + logger.info({ jobId: job.id, jobName: method, jobData }, 'Job added to queue'); + + // Respond with 202 Accepted + return res.status(202).json({ jsonrpc: '2.0', - error: { code: -32602, message: 'Missing topic parameter' }, - id + result: { + status: 'pending', + jobId: job.id, + message: 'Content creation request accepted and queued.' + }, + id: requestId + }); + + } catch (queueError) { + logger.error({ err: queueError, jobName: method, jobData, requestId }, 'Failed to add job to queue'); + return res.status(503).json({ // Service Unavailable + jsonrpc: '2.0', + error: { + code: -32001, // Custom server error code for queue failure + message: 'Failed to queue content creation request. Please try again later.' + }, + id: requestId }); } - - const result = await viralSystem.createViralContent(topic); - - res.json({ - jsonrpc: '2.0', - result, - id - }); + } catch (error) { - console.error(`Viral content error: ${error.message}`); + // This main catch block now primarily handles unexpected errors + // or errors from the validation logic if any were missed (though they should return directly). + // Log the full error server-side for debugging + logger.error({ err: error, requestId }, 'API Endpoint Unhandled Error'); + + // Send a generic error message to the client res.status(500).json({ jsonrpc: '2.0', - error: { code: -32000, message: error.message }, - id + error: { + code: -32000, // Standard JSON-RPC server error code + message: 'An internal server error occurred. The issue has been logged. Please try again later.' + }, + id: requestId // Ensure 'id' is correctly sourced from the request body (aliased as requestId) }); } }); -// Service loader -async function loadService(name) { - if (viralSystem.services[name]) return viralSystem.services[name]; - - const config = serviceRegistry[name]; - if (!config) throw new Error(`Unsupported service: ${name}`); - - const Service = require(config.module); - const service = config.url ? - new Service(name, config.url) : - new Service(); - - if (service.initialize) await service.initialize(); - viralSystem.services[name] = service; - return service; -} - -// Initialize services -async function initializeServices() { - for (const name of Object.keys(serviceRegistry)) { - await loadService(name); - } -} +// Service loader and initializeServices are removed. // Start server async function start() { - await fs.mkdir(SESSION_DIR, { recursive: true }); - await initializeServices(); + try { + await fs.mkdir(SESSION_DIR, { recursive: true }); + + logger.info('Initializing ViralContentSystem for server...'); + viralSystem = new ViralContentSystem(); + await viralSystem.initialize(); // Base initialization (Drive, TempDir) + await viralSystem.initialize_dependent_services(); // Initialize all dependent services + logger.info('ViralContentSystem for server initialized successfully.'); + logger.info('Helmet middleware enabled for enhanced security.'); + logger.info('API rate limiting enabled for /mcp/viral-content.'); + logger.info('Health check endpoint /health configured.'); // Log health check endpoint - app.listen(port, () => { - console.log(`Viral Content MCP running on port ${port}`); - console.log(`Supported services: ${Object.keys(serviceRegistry).join(', ')}`); - }); + app.listen(config.port, () => { + logger.info(`Viral Content MCP running on port ${config.port}`); + }); + } catch (error) { + logger.fatal({ err: error }, 'Failed to start server'); + process.exit(1); + } } // Cleanup process.on('SIGINT', async () => { - console.log('Shutting down...'); - for (const service of Object.values(viralSystem.services)) { - if (service.close) await service.close(); + logger.info('Shutting down server gracefully...'); + if (viralSystem && viralSystem.services) { + for (const serviceName in viralSystem.services) { + const service = viralSystem.services[serviceName]; + if (service && typeof service.close === 'function') { + try { + await service.close(); + logger.info({ serviceName }, `Service closed.`); + } catch (err) { + logger.error({ err, serviceName }, `Error closing service.`); + } + } + } } - process.exit(); + // Also close the queue connection + if (contentCreationQueue && typeof contentCreationQueue.close === 'function') { + try { + await contentCreationQueue.close(); + logger.info('BullMQ contentCreationQueue closed.'); + } catch (err) { + logger.error({ err }, 'Error closing BullMQ queue.'); + } + } + logger.info('Server shutdown complete.'); + process.exit(0); }); -start(); \ No newline at end of file +start(); + +// module.exports = { ViralContentSystem }; // This line is removed. \ No newline at end of file diff --git a/services/groq.js b/services/groq.js index 4b08b71..31be7f9 100644 --- a/services/groq.js +++ b/services/groq.js @@ -1,39 +1,131 @@ const { Groq } = require("groq-sdk"); +const retry = require('async-retry'); +const config = require('../../config'); // Added config require +const logger = require('../../lib/logger'); // Added logger require class GroqService { constructor() { - this.groq = new Groq({ apiKey: process.env.GROQ_API_KEY }); + if (!config.groqApiKey) { + logger.warn("Groq API key is not set in config. GroqService will not be able to function."); + } + this.groq = new Groq({ + apiKey: config.groqApiKey, + timeout: config.timeouts.groqMs, // Added timeout from config + }); } - async generateStrategy(topic) { - const prompt = `Develop a viral content strategy about "${topic}" including: - - Psychological hooks for maximum engagement - - Trending music style recommendations - - Visual style (cinematic, meme, documentary, etc.) - - Target audience personas - - Viral hashtags (5-7) - - Attention-grabbing title - - 2-sentence captivating description - - Short platform-specific captions - - Respond in JSON format: { - title: "", - description: "", - hashtags: [], - visualPrompt: "", - viralMusicPrompt: "", - scriptSegment: "", - caption: "", - audience: "" - }`; - - const response = await this.groq.chat.completions.create({ - messages: [{ role: "user", content: prompt }], - model: "mixtral-8x7b-32768", - response_format: { type: "json_object" } + async generateStrategy(topic, urlContent) { + let prompt; + if (urlContent) { + prompt = `Analyze the following text and develop a viral content strategy based on it: + "${urlContent}" + + The strategy should include: + - Psychological hooks for maximum engagement + - Trending music style recommendations + - Visual style (cinematic, meme, documentary, etc.) + - Target audience personas + - Viral hashtags (5-7) + - Attention-grabbing title + - 2-sentence captivating description + - Short platform-specific captions + + Respond in JSON format: { + title: "", + description: "", + hashtags: [], + visualPrompt: "", + viralMusicPrompt: "", + scriptSegment: "", + caption: "", + audience: "" + }`; + } else { + prompt = `Develop a viral content strategy about "${topic}" including: + - Psychological hooks for maximum engagement + - Trending music style recommendations + - Visual style (cinematic, meme, documentary, etc.) + - Target audience personas + - Viral hashtags (5-7) + - Attention-grabbing title + - 2-sentence captivating description + - Short platform-specific captions + + Respond in JSON format: { + title: "", + description: "", + hashtags: [], + visualPrompt: "", + viralMusicPrompt: "", + scriptSegment: "", + caption: "", + audience: "" + }`; + } + + const contextIdentifier = urlContent ? `URL content (snippet: ${urlContent.substring(0, 50)}...)` : `topic: "${topic}"`; + + return retry(async (bail, attemptNumber) => { + try { + if (attemptNumber > 1) { + logger.info({ contextIdentifier, attemptNumber }, `Retrying Groq strategy generation`); + } + + const response = await this.groq.chat.completions.create( + { + messages: [{ role: "user", content: prompt }], + model: "mixtral-8x7b-32768", + response_format: { type: "json_object" } + } + // Per-request timeout can also be set here using an AbortSignal, + // but client-level timeout is generally cleaner if all requests should use it. + // Example: { signal: AbortSignal.timeout(config.timeouts.groqMs) } + ); + + if (!response || !response.choices || !response.choices[0] || !response.choices[0].message || !response.choices[0].message.content) { + logger.error({ contextIdentifier, attemptNumber, responseContent: response }, `Malformed response from Groq`); + throw new Error("Malformed response from Groq API"); + } + + const messageContent = response.choices[0].message.content; + + try { + return JSON.parse(messageContent); + } catch (jsonError) { + logger.error({ err: jsonError, contextIdentifier, messageContent }, `Failed to parse JSON response from Groq`); + bail(new Error(`Failed to parse JSON response from Groq: ${jsonError.message}`)); + return null; + } + + } catch (error) { + // Log the error with context + logger.warn({ err: error, contextIdentifier, attemptNumber, isBail: error.bail }, `Groq API call attempt failed`); + + if (error.status === 401 || error.status === 403) { + logger.error({ err: error, contextIdentifier, status: error.status }, `Authentication/Authorization error with Groq API. Bailing out.`); + bail(error); + return null; + } else if (error.message === "Malformed response from Groq API" && attemptNumber === (config.jobDefaultAttempts || 3)) { + // If it's the last attempt for a malformed response, bail. + logger.error({ err: error, contextIdentifier, attemptNumber }, `Malformed response from Groq on final attempt. Bailing out.`); + bail(error); + return null; + } + // For other errors (429, 5xx, network issues, or malformed not on last attempt), re-throw to allow retries + throw error; + } + }, { + retries: config.jobDefaultAttempts || 3, // Use config or default + factor: 2, + minTimeout: config.jobDefaultBackoffDelay || 1000, // Use config or default + maxTimeout: 10000, // Keep a max timeout + onRetry: (error, attemptNumber) => { + logger.warn({ err: error, contextIdentifier, attemptNumber }, `Preparing for Groq retry attempt`); + } + }).catch(finalError => { + logger.error({ err: finalError, contextIdentifier }, `Failed to generate Groq strategy after multiple retries`); + throw new Error(`Failed to generate Groq strategy for ${contextIdentifier} after multiple retries: ${finalError.message}`); }); - - return JSON.parse(response.choices[0].message.content); } } diff --git a/services/new_service.js b/services/new_service.js index a184925..668e171 100644 --- a/services/new_service.js +++ b/services/new_service.js @@ -1,28 +1,87 @@ const { BaseAIService } = require('../base'); +const retry = require('async-retry'); +const logger = require('../../lib/logger'); // Assuming logger is in lib at root +const config = require('../../config'); // Assuming config is at root class NewService extends BaseAIService { + constructor(name, url) { // Added constructor to accept name/url like other services + super(name, url); // Pass to BaseAIService + // this.page is initialized in BaseAIService.initialize() + } + async isLoginRequired() { - // Implement login check logic + // Implement login check logic if this service requires it + // For now, assuming false or handled by BaseAIService if page is available + logger.debug({ serviceName: this.name }, 'isLoginRequired check'); return false; } async login() { // Implement login logic if needed + logger.info({ serviceName: this.name }, 'Attempting login (if required)'); + // This would involve Playwright actions similar to generateContent } async generateContent(prompt) { - await this.page.goto('https://new-ai-service.com'); - - // Enter prompt - await this.page.fill('textarea#prompt-input', prompt); - await this.page.click('button#submit-btn'); + const serviceName = this.name || 'NewService'; // Use instance name or default + const methodName = 'generateContent'; + + if (!this.page) { + logger.error({ serviceName, methodName }, 'Playwright page not initialized for NewService.'); + throw new Error('Playwright page not initialized for NewService. Call initialize() first.'); + } - // Wait for response - await this.page.waitForSelector('.ai-response'); - return await this.page.$eval('.ai-response', el => el.textContent); + return retry(async (bail, attemptNumber) => { + logger.debug({ attemptNumber, serviceName, methodName, promptLength: prompt ? prompt.length : 0 }, 'Attempting content generation'); + try { + // Ensure page is navigated to the correct URL if not already there or if state is uncertain + // This could be part of a 'ensurePageReady' method if complex + // For now, goto is included in the retry block. + await this.page.goto(this.url || 'https://new-ai-service.com'); // Use this.url if provided + + logger.debug({ serviceName, methodName, attemptNumber }, 'Filling prompt and submitting'); + await this.page.fill('textarea#prompt-input', prompt); + await this.page.click('button#submit-btn'); + + logger.debug({ serviceName, methodName, attemptNumber }, 'Waiting for response selector'); + await this.page.waitForSelector('.ai-response', { timeout: config.timeouts.newServiceAiRequestMs }); + + const responseText = await this.page.$eval('.ai-response', el => el.textContent); + + if (!responseText || responseText.trim() === '') { + logger.warn({ serviceName, methodName, attemptNumber }, 'Empty response from AI service.'); + throw new Error('Empty response from AI service'); // Trigger retry for empty response + } + + logger.info({ serviceName, methodName, attemptNumber, responseLength: responseText.length }, 'Content generated successfully'); + return responseText; + } catch (error) { + logger.warn({ err: error, attemptNumber, serviceName, methodName }, 'Content generation attempt failed'); + + // Example: Bailing on a specific type of error if needed (e.g., Playwright's TargetClosedError) + // if (error.name === 'TargetClosedError') { + // logger.error({ err: error, serviceName, methodName }, 'Target closed, navigation failed. Bailing.'); + // bail(error); + // return; + // } + + // For most Playwright errors (timeout, navigation, element not found), retrying might help + throw error; // Re-throw to trigger retry + } + }, { + retries: config.jobDefaultAttempts || 3, + factor: 2, + minTimeout: config.jobDefaultBackoffDelay || 2000, // Slightly longer for UI interactions + maxTimeout: 15000, + onRetry: (err, attempt) => { + logger.warn({ err, attempt, serviceName, methodName }, 'Retrying content generation call...'); + } + }); } async uploadToPlatform(content) { + // Implement platform-specific upload logic, potentially with its own retry logic + logger.info({ serviceName: this.name, contentLehgth: content ? content.length : 0 }, 'Uploading to platform (stub)'); // Implement platform-specific upload logic } } diff --git a/services/runaway.js b/services/runaway.js index fb01977..761d9b7 100644 --- a/services/runaway.js +++ b/services/runaway.js @@ -1,27 +1,74 @@ const { BaseAIService } = require('../base'); +const retry = require('async-retry'); +const logger = require('../../lib/logger'); +const config = require('../../config'); +const path = require('path'); // Added path require class RunwayService extends BaseAIService { + constructor(name, url) { // Added constructor + super(name, url); // Pass to BaseAIService + // this.page is initialized in BaseAIService.initialize() + } + async generateVideo(strategy) { - await this.page.goto('https://app.runwayml.com/video-tools', { waitUntil: 'networkidle' }); - - // Enter text-to-video prompt - await this.page.fill('textarea.prompt-input', strategy.visualPrompt); - await this.page.click('button.generate-video'); - - // Wait for generation - await this.page.waitForSelector('.generated-video', { timeout: 180000 }); - - // Download video - const [download] = await Promise.all([ - this.page.waitForEvent('download'), - this.page.click('button.download-video') - ]); - - const fileName = `video-${Date.now()}.mp4`; - const savePath = path.join(TEMP_DIR, fileName); - await download.saveAs(savePath); - - return { path: savePath, fileName }; + const serviceName = this.name || 'RunwayService'; + const methodName = 'generateVideo'; + + if (!this.page) { + logger.error({ serviceName, methodName }, 'Playwright page not initialized for RunwayService.'); + throw new Error('Playwright page not initialized for RunwayService. Call initialize() first.'); + } + + return retry(async (bail, attemptNumber) => { + logger.debug({ attemptNumber, serviceName, methodName, visualPrompt: strategy.visualPrompt }, 'Attempting video generation'); + try { + // Navigate to the page + // Using this.url if provided by serviceRegistry, otherwise default. + const targetUrl = this.url || 'https://app.runwayml.com/video-tools'; + await this.page.goto(targetUrl, { waitUntil: 'networkidle' }); + + logger.debug({ serviceName, methodName, attemptNumber }, 'Filling prompt and submitting for video generation'); + await this.page.fill('textarea.prompt-input', strategy.visualPrompt); + await this.page.click('button.generate-video'); + + logger.debug({ serviceName, methodName, attemptNumber }, 'Waiting for video generation selector'); + await this.page.waitForSelector('.generated-video', { timeout: config.timeouts.runwayVideoGenerationMs }); + + logger.debug({ serviceName, methodName, attemptNumber }, 'Attempting to download video'); + const [download] = await Promise.all([ + this.page.waitForEvent('download', {timeout: config.timeouts.runwayDownloadMs }), + this.page.click('button.download-video') + ]); + + const fileName = `video-${Date.now()}.mp4`; + // Use config.tempDir for save path + const savePath = path.join(config.tempDir, fileName); + await download.saveAs(savePath); + logger.info({ serviceName, methodName, attemptNumber, savePath, fileName }, 'Video downloaded successfully'); + + return { path: savePath, fileName }; + } catch (error) { + logger.warn({ err: error, attemptNumber, serviceName, methodName }, 'Video generation attempt failed'); + + // Example: Bailing on a specific type of error if needed + // if (error.name === 'TargetClosedError' || error.message.includes('Download failed')) { + // logger.error({ err: error, serviceName, methodName }, 'Critical error during video generation/download. Bailing.'); + // bail(error); + // return; + // } + throw error; // Re-throw to trigger retry + } + }, { + retries: config.jobDefaultAttempts || 2, // Videos can be long, maybe fewer retries than text + factor: 2, + minTimeout: config.jobDefaultBackoffDelay || 5000, // Longer min timeout + maxTimeout: 30000, + onRetry: (err, attempt) => { + logger.warn({ err, attempt, serviceName, methodName }, 'Retrying video generation call...'); + // Optional: Add logic here to reset page state if necessary, e.g., this.page.reload() + // However, this might be complex. The `goto` at the start of the try block often handles this. + } + }); } } diff --git a/services/webExtractor.js b/services/webExtractor.js new file mode 100644 index 0000000..6f31323 --- /dev/null +++ b/services/webExtractor.js @@ -0,0 +1,76 @@ +const playwright = require('playwright'); +const retry = require('async-retry'); +const logger = require('../../lib/logger'); // Added logger require +const config = require('../../config'); // Added config require for retry options + +class WebExtractorService { + constructor() { + this.browser = null; + this.context = null; + this.page = null; + } + + async initialize() { + try { + this.browser = await playwright.chromium.launch(); + this.context = await this.browser.newContext(); + this.page = await this.context.newPage(); + logger.info('Playwright initialized successfully for WebExtractorService.'); + } catch (error) { + logger.error({ err: error }, 'Error initializing Playwright for WebExtractorService'); + throw error; + } + } + + async extractText(url) { + if (!this.page) { + logger.error({ url }, 'Playwright page is not initialized in WebExtractorService. Call initialize() first.'); + throw new Error('Playwright page not initialized.'); + } + + return retry(async (bail, attemptNumber) => { + try { + if (attemptNumber > 1) { + logger.info({ url, attemptNumber }, `Retrying text extraction from URL`); + } + await this.page.goto(url, { waitUntil: 'domcontentloaded', timeout: config.timeouts.webExtractorNavigationMs }); + const bodyText = await this.page.locator('body').innerText(); + if (!bodyText || bodyText.trim() === '') { + logger.warn({ url, attemptNumber, textLength: bodyText.length }, `Extracted empty text. Retrying if attempts remain.`); + throw new Error(`Extracted empty text from ${url}`); + } + return bodyText; + } catch (error) { + logger.warn({ err: error, url, attemptNumber }, `Text extraction attempt failed`); + throw error; + } + }, { + retries: config.jobDefaultAttempts || 3, // Using general job attempts, could be specific + factor: 2, + minTimeout: config.jobDefaultBackoffDelay || 1000, // Using general backoff, could be specific + maxTimeout: 5000, + onRetry: (error, attemptNumber) => { + logger.warn({ err: error, url, attemptNumber }, `Preparing for text extraction retry attempt`); + } + }).catch(error => { + logger.error({ err: error, url }, `Failed to extract text from URL after multiple retries`); + return null; + }); + } + + async close() { + try { + if (this.browser) { + await this.browser.close(); + logger.info('Playwright browser closed for WebExtractorService.'); + this.browser = null; + this.context = null; + this.page = null; + } + } catch (error) { + logger.error({ err: error }, 'Error closing Playwright browser for WebExtractorService'); + } + } +} + +module.exports = WebExtractorService; diff --git a/services/youtube.js b/services/youtube.js index d95892a..7a6fae3 100644 --- a/services/youtube.js +++ b/services/youtube.js @@ -1,34 +1,127 @@ const { BaseAIService } = require('../base'); +const retry = require('async-retry'); +const path = require('path'); +const fs = require('fs').promises; // Not strictly needed for screenshot, but good for HTML dump if added +const logger = require('../../lib/logger'); +const config = require('../../config'); class YouTubeService extends BaseAIService { - async postContent({ video, title, description, tags }) { - await this.page.goto('https://studio.youtube.com', { waitUntil: 'networkidle' }); - - // Click upload button - await this.page.click('button[aria-label="Create"]'); - await this.page.click('text="Upload video"'); - - // Upload file - const [fileChooser] = await Promise.all([ - this.page.waitForEvent('filechooser'), - this.page.click('div#upload-prompt-box') - ]); - await fileChooser.setFiles(video); - - // Fill details - await this.page.fill('input#textbox', title); - await this.page.fill('textarea#description', description); - await this.page.fill('input#tags', tags.join(',')); - - // Set as public - await this.page.click('button[name="PUBLIC"]'); - - // Publish - await this.page.click('button#done-button'); + constructor(name, url) { // BaseAIService might pass page, or initialize it + super(name, url); + // this.page should be initialized by BaseAIService's initialize method + } + + async postContent({ videoPath, title, description, tags }) { // Renamed 'video' to 'videoPath' for clarity + const serviceName = this.name || 'YouTubeService'; + const methodName = 'postContent'; + + if (!this.page || this.page.isClosed()) { + logger.error({ serviceName, methodName }, 'Playwright page is not available or closed. Attempting to re-initialize.'); + // Attempt to re-initialize the page if BaseAIService provides such a method, + // or if this service's own initialize() can be safely called. + // This depends on BaseAIService structure. For now, we'll assume initialize sets up a page. + try { + await this.initialize(); // This should set up this.page from BaseAIService + if (!this.page || this.page.isClosed()) { + throw new Error('Failed to re-initialize Playwright page.'); + } + logger.info({serviceName, methodName}, 'Playwright page re-initialized successfully.'); + } catch (initError) { + logger.error({ err: initError, serviceName, methodName }, 'Failed to re-initialize Playwright page during postContent.'); + throw initError; // Propagate error if re-initialization fails + } + } - // Get video URL - await this.page.waitForSelector('a.ytcp-video-info'); - return await this.page.$eval('a.ytcp-video-info', a => a.href); + return retry(async (bail, attemptNumber) => { + logger.debug({ attemptNumber, serviceName, methodName, videoPath, title }, 'Attempting to post content to YouTube'); + try { + // Ensure page is not closed at the start of an attempt + if (!this.page || this.page.isClosed()) { + logger.warn({ serviceName, attemptNumber, methodName }, 'Page was closed at start of attempt. This should ideally be handled by re-initialization before retry.'); + throw new Error('Playwright page is closed at start of retry attempt.'); + } + + await this.page.goto(this.url || 'https://studio.youtube.com', { waitUntil: 'networkidle', timeout: config.timeouts.youtubeNavigationMs || 60000 }); + + logger.debug({ serviceName, methodName, attemptNumber }, 'Clicking create button'); + await this.page.click('button[aria-label="Create"]'); + logger.debug({ serviceName, methodName, attemptNumber }, 'Clicking upload video text'); + await this.page.click('text="Upload video"'); + + logger.debug({ serviceName, methodName, attemptNumber, videoPath }, 'Setting files for upload'); + const [fileChooser] = await Promise.all([ + this.page.waitForEvent('filechooser', { timeout: config.timeouts.youtubeFileChooserMs || 15000 }), + this.page.click('div#upload-prompt-box') + ]); + await fileChooser.setFiles(videoPath); // Use videoPath + logger.info({ serviceName, methodName, attemptNumber, videoPath }, 'Video file selected for upload.'); + + logger.debug({ serviceName, methodName, attemptNumber }, 'Filling video details'); + await this.page.waitForSelector('input#textbox', { timeout: config.timeouts.youtubeElementWaitMs || 30000 }); + await this.page.fill('input#textbox', title); + await this.page.fill('textarea#description', description); + await this.page.fill('input#tags', tags.join(',')); + + // "Not for kids" selection - this might be needed to unblock publishing + // Selector might vary based on UI language + // logger.debug({ serviceName, methodName, attemptNumber }, 'Selecting "Not for kids"'); + // await this.page.click('input[name="VIDEO_MADE_FOR_KIDS_NOT_MFK"]'); // Example selector + + logger.debug({ serviceName, methodName, attemptNumber }, 'Clicking "Next" multiple times to reach visibility'); + for (let i = 0; i < 3; i++) { // Typically 3 "Next" clicks: Details -> Checks -> Visibility + await this.page.waitForSelector('button#next-button', { timeout: config.timeouts.youtubeElementWaitMs || 10000 }); + await this.page.click('button#next-button'); + logger.debug({ serviceName, methodName, attemptNumber, clickCount: i + 1 }, 'Clicked "Next" button'); + await this.page.waitForTimeout(1000); // Small delay for UI to update + } + + logger.debug({ serviceName, methodName, attemptNumber }, 'Setting visibility to Public'); + await this.page.waitForSelector('button[name="PUBLIC"]', { timeout: config.timeouts.youtubeElementWaitMs || 10000 }); + await this.page.click('button[name="PUBLIC"]'); + + logger.debug({ serviceName, methodName, attemptNumber }, 'Clicking done/publish button'); + await this.page.waitForSelector('button#done-button', { timeout: config.timeouts.youtubeElementWaitMs || 10000 }); + await this.page.click('button#done-button'); + + logger.debug({ serviceName, methodName, attemptNumber }, 'Waiting for video link selector'); + // Increased timeout for this selector as video processing can take time + await this.page.waitForSelector('a.ytcp-video-info', { timeout: config.timeouts.youtubeVideoLinkMs || 300000 }); // 5 minutes + const videoUrl = await this.page.$eval('a.ytcp-video-info', a => a.href); + + logger.info({ serviceName, methodName, attemptNumber, videoUrl }, 'Content posted successfully to YouTube'); + return videoUrl; + + } catch (error) { + logger.warn({ err: error, serviceName, attemptNumber, methodName }, 'YouTube postContent attempt failed'); + if (config.debug.savePlaywrightFailureArtifacts && this.page && !this.page.isClosed()) { + try { + const timestamp = Date.now(); + const screenshotPath = path.join(config.tempDir, `youtube_failure_attempt${attemptNumber}_${timestamp}.png`); + await this.page.screenshot({ path: screenshotPath, fullPage: true }); + logger.info({ screenshotPath }, 'Saved screenshot on Playwright failure (YouTube).'); + } catch (artifactError) { + logger.error({ err: artifactError, serviceName }, 'Failed to save Playwright failure artifacts (YouTube).'); + } + } + // Example: Bailing on specific errors like invalid credentials or account issues + // if (error.message.includes('Authentication failed') || error.message.includes('Account issue')) { + // logger.error({ err: error, serviceName, methodName }, 'Non-retriable error from YouTube. Bailing.'); + // bail(error); + // return; + // } + throw error; + } + }, { + retries: config.jobDefaultAttempts || 2, // YouTube uploads can be sensitive; adjust retries + factor: 2, + minTimeout: config.jobDefaultBackoffDelay || 10000, // Longer min timeout for UI operations + maxTimeout: 60000, // Max 1 minute between retries + onRetry: (err, attempt) => { + logger.warn({ err, attempt, serviceName, methodName }, 'Retrying YouTube postContent call...'); + // Consider if page needs to be reloaded or re-navigated on retry. + // The current logic re-navigates at the start of each try block. + } + }); } } diff --git a/worker.js b/worker.js new file mode 100644 index 0000000..e0d40d4 --- /dev/null +++ b/worker.js @@ -0,0 +1,131 @@ +// worker.js +const { Worker } = require('bullmq'); +const config = require('./config'); // Added config require +const { ViralContentSystem } = require('./core/viralSystem'); // Updated import +const logger = require('./lib/logger'); // Added logger require + +const QUEUE_NAME = 'contentCreationQueue'; + +// Create a reusable Redis connection object for the Worker +const workerConnection = { + host: config.redis.host, + port: config.redis.port, + password: config.redis.password, // Will be undefined if not set, which is fine for ioredis + // BullMQ recommends setting maxRetriesPerRequest to null for worker connections + // to prevent ioredis from retrying commands internally, allowing BullMQ to handle retries. + maxRetriesPerRequest: null, +}; + +let viralSystem; // To hold the ViralContentSystem instance + +// Define the job processor function +const processor = async (job) => { + logger.info({ jobId: job.id, jobName: job.name, jobData: job.data }, 'Processing job'); + + if (!viralSystem) { + logger.error({ jobId: job.id, jobName: job.name }, 'ViralContentSystem not initialized. Worker might be starting up or encountered an issue.'); + throw new Error('ViralContentSystem not available at job processing time'); + } + + try { + let result; + if (job.name === 'create_viral_content') { + if (!job.data || typeof job.data.topic !== 'string' || job.data.topic.trim() === '') { + logger.warn({ jobId: job.id, jobData: job.data }, 'Invalid or missing topic for create_viral_content job'); + throw new Error('Invalid or missing topic for create_viral_content job'); + } + result = await viralSystem.createViralContent(job.data.topic); + } else if (job.name === 'create_viral_content_from_url') { + if (!job.data || typeof job.data.url !== 'string' || job.data.url.trim() === '') { + logger.warn({ jobId: job.id, jobData: job.data }, 'Invalid or missing url for create_viral_content_from_url job'); + throw new Error('Invalid or missing url for create_viral_content_from_url job'); + } + result = await viralSystem.createViralContentFromUrl(job.data.url, job.data.userId); + } else { + logger.error({ jobId: job.id, jobName: job.name }, 'Unknown job name'); + throw new Error(`Unknown job name: ${job.name}`); + } + logger.info({ jobId: job.id }, 'Job completed successfully by processor.'); + return result; + } catch (error) { + logger.error({ err: error, jobId: job.id, jobName: job.name }, 'Failed to process job in processor'); + throw error; + } +}; + +// Initialize ViralContentSystem and then start the worker +async function main() { + logger.info('Initializing ViralContentSystem for worker...'); + try { + viralSystem = new ViralContentSystem(); + await viralSystem.initialize(); + await viralSystem.initialize_dependent_services(); + logger.info('ViralContentSystem and its dependent services initialized successfully for worker.'); + + const worker = new Worker(QUEUE_NAME, processor, { + connection: workerConnection, + concurrency: config.workerConcurrency, + limiter: { + max: config.workerRateLimit.max, + duration: config.workerRateLimit.duration, + }, + }); + + worker.on('completed', (job, result) => { + const driveLink = (result && result.driveLink) || (result && result.posts && result.posts.youtube && result.posts.youtube.webViewLink) || 'N/A'; + logger.info({ jobId: job.id, jobName: job.name, driveLink }, 'Job completed.'); + }); + + worker.on('failed', (job, err) => { + const jobId = job ? job.id : 'N/A'; + const jobName = job ? job.name : 'N/A'; + const attemptsMade = job ? job.attemptsMade : 'N/A'; + logger.error({ err, jobId, jobName, attemptsMade }, 'Job marked as Failed.'); + }); + + worker.on('error', err => { + logger.error({ err }, 'BullMQ Worker Error'); + }); + + logger.info({ queueName: QUEUE_NAME, concurrency: config.workerConcurrency }, 'Worker started. Waiting for jobs...'); + + // Graceful shutdown + const signals = ['SIGINT', 'SIGTERM', 'SIGQUIT']; + signals.forEach(signal => { + process.on(signal, async () => { + logger.info({ signal }, `Received signal. Shutting down worker gracefully...`); + try { + await worker.close(); + logger.info('BullMQ Worker closed.'); + + if (viralSystem && viralSystem.services) { + logger.info('Closing services managed by ViralContentSystem...'); + for (const serviceName in viralSystem.services) { + const service = viralSystem.services[serviceName]; + if (service && typeof service.close === 'function') { + try { + await service.close(); + logger.info({ serviceName }, `Service closed.`); + } catch (closeError) { + logger.error({ err: closeError, serviceName }, `Error closing service.`); + } + } + } + logger.info('All manageable services closed.'); + } + } catch (err) { + logger.error({ err }, 'Error during graceful shutdown.'); + } finally { + logger.info('Worker process exiting.'); + process.exit(0); + } + }); + }); + + } catch (error) { + logger.fatal({ err: error }, 'Failed to initialize ViralContentSystem or Worker'); + process.exit(1); + } +} + +main();