🎯 Orchestration Module Implementation

apps/aggregator/.env.example

@@ -1,12 +1,274 @@
-# Server Configuration
+# ==============================================================================
+# SERVER CONFIGURATION
+# ==============================================================================
 PORT=3001
+NODE_ENV=development
 
-# Ingestor Service Configuration
+# ==============================================================================
+# INGESTOR SERVICE INTEGRATION
+# ==============================================================================
 INGESTOR_WS_URL=ws://localhost:3000
 INGESTOR_HTTP_URL=http://localhost:3000
 
-# Redis (optional; used for caching/session when set; health check verifies connectivity)
-# REDIS_URL=redis://localhost:6379
-
-# Signer Service (for publishing aggregated data)
+# ==============================================================================
+# SIGNER SERVICE INTEGRATION
+# ==============================================================================
 SIGNER_URL=http://localhost:3002
+
+# ==============================================================================
+# AGGREGATION PIPELINE - PROCESSING MODES
+# ==============================================================================
+
+# Enable real-time price processing (process immediately)
+AGGREGATION_REALTIME_ENABLED=true
+
+# Enable batch processing (accumulate prices and process periodically)
+AGGREGATION_BATCH_ENABLED=false
+
+# Batch processing interval in milliseconds
+AGGREGATION_BATCH_INTERVAL_MS=5000
+
+# Minimum number of prices to trigger batch processing
+AGGREGATION_BATCH_MIN_PRICES=3
+
+# ==============================================================================
+# AGGREGATION PIPELINE - AGGREGATION STRATEGY
+# ==============================================================================
+
+# Aggregation method: WEIGHTED_MEDIAN, WEIGHTED_MEAN, TRIMMED_MEAN
+AGGREGATION_METHOD=WEIGHTED_MEDIAN
+
+# Minimum number of sources required for aggregation
+AGGREGATION_MIN_SOURCES=2
+
+# Maximum number of sources to consider
+AGGREGATION_MAX_SOURCES=50
+
+# ==============================================================================
+# AGGREGATION PIPELINE - OUTLIER DETECTION
+# ==============================================================================
+
+# Outlier detection method: MODIFIED_Z_SCORE, IQR, MEDIAN_ABSOLUTE_DEVIATION
+OUTLIER_METHOD=MODIFIED_Z_SCORE
+
+# Outlier threshold (0-1): Higher = more lenient, lower = stricter
+OUTLIER_THRESHOLD=0.03
+
+# ==============================================================================
+# AGGREGATION PIPELINE - NORMALIZATION
+# ==============================================================================
+
+# Enable price normalization (convert to standard format)
+NORMALIZATION_ENABLED=true
+
+# ==============================================================================
+# AGGREGATION PIPELINE - QUALITY THRESHOLDS
+# ==============================================================================
+
+# Minimum confidence score (0-1) required for aggregated prices
+MIN_CONFIDENCE=0.7
+
+# Maximum coefficient of variation allowed (0-1)
+MAX_CV=0.05
+
+# ==============================================================================
+# AGGREGATION PIPELINE - STAGE TIMEOUTS (milliseconds)
+# ==============================================================================
+
+# Timeout for normalization stage
+NORMALIZATION_TIMEOUT_MS=5000
+
+# Timeout for outlier detection stage
+OUTLIER_DETECTION_TIMEOUT_MS=3000
+
+# Timeout for aggregation stage
+AGGREGATION_TIMEOUT_MS=5000
+
+# Timeout for storage stage
+STORAGE_TIMEOUT_MS=10000
+
+# ==============================================================================
+# CIRCUIT BREAKER CONFIGURATION
+# ==============================================================================
+
+# Number of failures before opening circuit
+CB_FAILURE_THRESHOLD=5
+
+# Number of successes in HALF_OPEN state to close circuit
+CB_SUCCESS_THRESHOLD=2
+
+# Timeout in milliseconds before attempting recovery
+CB_TIMEOUT_MS=30000
+
+# Maximum concurrent requests allowed in HALF_OPEN state
+CB_HALF_OPEN_MAX_REQUESTS=3
+
+# Enable detailed circuit breaker logging
+CB_ENABLE_LOGGING=true
+
+# ==============================================================================
+# STORAGE CONFIGURATION
+# ==============================================================================
+
+# Enable storing aggregated prices to database
+STORAGE_ENABLED=true
+
+# Number of days to retain stored prices
+STORAGE_RETENTION_DAYS=30
+
+# ==============================================================================
+# METRICS & MONITORING
+# ==============================================================================
+
+# Enable metrics collection
+METRICS_ENABLED=true
+
+# Metrics collection interval in milliseconds
+METRICS_INTERVAL_MS=60000
+
+# Enable verbose logging (more detailed logs)
+VERBOSE_LOGGING=false
+
+# ==============================================================================
+# PRECISION CONFIGURATION BY SYMBOL
+# ==============================================================================
+# Note: These can be configured in code at src/config/source-weights.config.ts
+# Format is JSON object mapping symbols to decimal places
+# Example: {"EUR/USD": 6, "BTC/USD": 2, "AAPL": 2}
+
+PRECISION_BY_SYMBOL='{"EUR/USD":6,"GBP/USD":6,"BTC/USD":2,"ETH/USD":4}'
+
+# ==============================================================================
+# SOURCE WEIGHTS CONFIGURATION
+# ==============================================================================
+# Note: Source weights are configured in src/config/source-weights.config.ts
+#
+# Weight Guidelines:
+# - 2.0: Premium sources (Bloomberg, Reuters)
+# - 1.5: High reliability (AlphaVantage, IEX)
+# - 1.2: Standard reliability (YahooFinance, Finnhub)
+# - 1.0: Baseline/unknown sources
+# - 0.5-0.8: Lower priority sources
+# - 0.0: Disabled/excluded sources
+
+# ==============================================================================
+# FEATURE FLAGS
+# ==============================================================================
+
+# Enable real-time price streaming (WebSocket)
+FEATURE_REALTIME_STREAMING=true
+
+# Enable batch aggregation
+FEATURE_BATCH_AGGREGATION=false
+
+# Enable adaptive source weighting based on accuracy
+FEATURE_ADAPTIVE_WEIGHTING=false
+
+# ==============================================================================
+# DEVELOPMENT & DEBUGGING
+# ==============================================================================
+
+# Log all price events (may be verbose in production)
+DEBUG_LOG_PRICES=false
+
+# Log all circuit breaker state changes
+DEBUG_LOG_CIRCUIT_BREAKER=false
+
+# Enable source weight debugging
+DEBUG_LOG_WEIGHTS=false
+
+# ==============================================================================
+# DATABASE CONFIGURATION (if using storage)
+# ==============================================================================
+
+# Database type: postgres, mysql, sqlite, mongodb
+DB_TYPE=postgres
+
+# Database host
+DB_HOST=localhost
+
+# Database port
+DB_PORT=5432
+
+# Database name
+DB_DATABASE=neko_oracle
+
+# Database user
+DB_USER=oracle_user
+
+# Database password
+DB_PASSWORD=change_me_in_production
+
+# ==============================================================================
+# LOGGING CONFIGURATION
+# ==============================================================================
+
+# Log level: error, warn, log, debug, verbose
+LOG_LEVEL=log
+
+# Enable file logging
+LOG_FILE_ENABLED=false
+
+# Log file path
+LOG_FILE_PATH=./logs/aggregator.log
+
+# ==============================================================================
+# SECURITY
+# ==============================================================================
+
+# API Key for authentication (if required)
+API_KEY=your_api_key_here
+
+# CORS allowed origins
+CORS_ORIGINS=http://localhost:3000,http://localhost:3001
+
+# ==============================================================================
+# PERFORMANCE TUNING
+# ==============================================================================
+
+# Maximum number of concurrent price processing tasks
+MAX_CONCURRENT_TASKS=10
+
+# Enable request caching
+CACHE_ENABLED=true
+
+# Cache TTL in seconds
+CACHE_TTL=60
+
+# ==============================================================================
+# ALERTS & NOTIFICATIONS
+# ==============================================================================
+
+# Enable alerts when confidence drops below threshold
+ALERT_LOW_CONFIDENCE=true
+
+# Enable alerts when circuit breaker opens
+ALERT_CIRCUIT_BREAKER_OPEN=true
+
+# Alert notification endpoint (e.g., Slack, PagerDuty)
+ALERT_WEBHOOK_URL=https://hooks.slack.com/services/YOUR/WEBHOOK/URL
+
+# ==============================================================================
+# EXPORT & INTEGRATIONS
+# ==============================================================================
+
+# Enable exporting aggregated prices to external systems
+EXPORT_ENABLED=false
+
+# Export interval in milliseconds
+EXPORT_INTERVAL_MS=10000
+
+# Export destination URL
+EXPORT_URL=http://localhost:3002/api/prices
+
+# ==============================================================================
+# DEFAULTS
+# ==============================================================================
+# These are fallback values used if environment variables are not set
+#
+# Default aggregation method: WEIGHTED_MEDIAN
+# Default min sources: 2
+# Default max sources: 50
+# Default batch interval: 5000ms
+# Default storage enabled: true
+# ==============================================================================