Universe files generation for multiple markets

.github/workflows/gh-actions.yml

@@ -3,6 +3,8 @@ name: Build & Test
 on:
   push:
     branches: ['*']
+  pull_request:
+    branches: ['*']
 
 jobs:
   build:

Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs

@@ -20,13 +20,10 @@
 using System.Threading;
 using System.Threading.Tasks;
 using NodaTime;
-using QuantConnect.Configuration;
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
 using QuantConnect.Interfaces;
-using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
-using QuantConnect.Lean.Engine.HistoricalData;
 using QuantConnect.Logging;
 using QuantConnect.Securities;
 using QuantConnect.Util;
@@ -46,7 +43,7 @@ public abstract class DerivativeUniverseGenerator
 
         protected readonly IDataProvider _dataProvider;
         protected readonly IHistoryProvider _historyProvider;
-        protected readonly ZipDataCacheProvider _dataCacheProvider;
+        protected readonly IDataCacheProvider _dataCacheProvider;
 
         protected readonly MarketHoursDatabase _marketHoursDatabase;
 
@@ -67,33 +64,20 @@ public abstract class DerivativeUniverseGenerator
         /// <param name="market">Market of data to process</param>
         /// <param name="dataFolderRoot">Path to the data folder</param>
         /// <param name="outputFolderRoot">Path to the output folder</param>
+        /// <param name="dataProvider">The data provider to use</param>
+        /// <param name="dataCacheProvider">The data cache provider to use</param>
+        /// <param name="historyProvider">The history provider to use</param>
         public DerivativeUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string dataFolderRoot,
-            string outputFolderRoot)
+            string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider)
         {
             _processingDate = processingDate;
             _securityType = securityType;
             _market = market;
             _dataFolderRoot = dataFolderRoot;
             _outputFolderRoot = outputFolderRoot;
-
-            _dataProvider = Composer.Instance.GetExportedValueByTypeName<IDataProvider>(Config.Get("data-provider", "DefaultDataProvider"));
-
-            var mapFileProvider = Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider", "LocalZipMapFileProvider"));
-            mapFileProvider.Initialize(_dataProvider);
-
-            var factorFileProvider = Composer.Instance.GetExportedValueByTypeName<IFactorFileProvider>(Config.Get("factor-file-provider", "LocalZipFactorFileProvider"));
-            factorFileProvider.Initialize(mapFileProvider, _dataProvider);
-
-            var api = new Api.Api();
-            api.Initialize(Globals.UserId, Globals.UserToken, Globals.DataFolder);
-
-            _dataCacheProvider = new ZipDataCacheProvider(_dataProvider);
-            _historyProvider = new HistoryProviderManager();
-            var parameters = new HistoryProviderInitializeParameters(null, api, _dataProvider, _dataCacheProvider, mapFileProvider,
-                factorFileProvider, (_) => { }, true, new DataPermissionManager(), null,
-                new AlgorithmSettings() { DailyPreciseEndTime = securityType == SecurityType.IndexOption });
-            _historyProvider.Initialize(parameters);
-
+            _dataProvider = dataProvider;
+            _dataCacheProvider = dataCacheProvider;
+            _historyProvider = historyProvider;
             _marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
         }
 

Lean.DataSource.DerivativeUniverseGenerator/Program.cs

@@ -21,6 +21,10 @@
 using QuantConnect.Configuration;
 using QuantConnect.Logging;
 using QuantConnect.Util;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Lean.Engine.HistoricalData;
 
 namespace QuantConnect.DataSource.DerivativeUniverseGenerator
 {
@@ -46,47 +50,69 @@ public abstract class Program
 
         protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null)
         {
-            Initialize(args, out var securityType, out var market, out var dataFolderRoot, out var outputFolderRoot,
+            Initialize(args, out var securityType, out var markets, out var dataFolderRoot, out var outputFolderRoot,
                 argNamesToIgnore ?? Array.Empty<string>());
 
             Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): " +
-                $"Security type: {securityType}. Market: {market}. Data folder: {dataFolderRoot}. Output folder: {outputFolderRoot}");
+                $"Security type: {securityType}. Markets: {string.Join(", ", markets)}. Data folder: {dataFolderRoot}. Output folder: {outputFolderRoot}");
             Log.DebuggingEnabled = Config.GetBool("debug-mode");
 
             var dateStr = Environment.GetEnvironmentVariable(DataFleetDeploymentDateEnvVariable) ?? $"{DateTime.UtcNow.Date:yyyyMMdd}";
             var processingDate = DateTime.ParseExact(dateStr, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
 
+            var dataProvider = Composer.Instance.GetExportedValueByTypeName<IDataProvider>(Config.Get("data-provider", "DefaultDataProvider"));
+
+            var mapFileProvider = Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider", "LocalZipMapFileProvider"));
+            mapFileProvider.Initialize(dataProvider);
+
+            var factorFileProvider = Composer.Instance.GetExportedValueByTypeName<IFactorFileProvider>(Config.Get("factor-file-provider", "LocalZipFactorFileProvider"));
+            factorFileProvider.Initialize(mapFileProvider, dataProvider);
+            var api = new Api.Api();
+            api.Initialize(Globals.UserId, Globals.UserToken, Globals.DataFolder);
+
+            var dataCacheProvider = new ZipDataCacheProvider(dataProvider);
+            var historyProvider = new HistoryProviderManager();
+            var parameters = new HistoryProviderInitializeParameters(null, api, dataProvider, dataCacheProvider, mapFileProvider,
+                factorFileProvider, (_) => { }, true, new DataPermissionManager(), null,
+                new AlgorithmSettings() { DailyPreciseEndTime = securityType == SecurityType.IndexOption });
+            historyProvider.Initialize(parameters);
+
             var timer = new Stopwatch();
             timer.Start();
 
-            var optionsUniverseGenerator = GetUniverseGenerator(securityType, market, dataFolderRoot, outputFolderRoot, processingDate);
-
-            try
+            foreach (var market in markets)
             {
-                if (!optionsUniverseGenerator.Run())
+                var optionsUniverseGenerator = GetUniverseGenerator(securityType, market, dataFolderRoot, outputFolderRoot, processingDate,
+                    dataProvider, dataCacheProvider, historyProvider);
+
+                try
                 {
-                    Log.Error($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Failed to generate options universe.");
+                    if (!optionsUniverseGenerator.Run())
+                    {
+                        Log.Error($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Failed to generate universe.");
+                        Environment.Exit(1);
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating universe.");
                     Environment.Exit(1);
                 }
             }
-            catch (Exception ex)
-            {
-                Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating options universe.");
-                Environment.Exit(1);
-            }
 
             Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): DONE in {timer.Elapsed:g}");
 
             Environment.Exit(0);
         }
 
         protected abstract DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, string dataFolderRoot,
-            string outputFolderRoot, DateTime processingDate);
+            string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider,
+            HistoryProviderManager historyProvider);
 
         /// <summary>
         /// Validate and extract command line args and configuration options.
         /// </summary>
-        protected virtual void Initialize(string[] args, out SecurityType securityType, out string market, out string dataFolderRoot,
+        protected virtual void Initialize(string[] args, out SecurityType securityType, out string[] markets, out string dataFolderRoot,
             out string outputFolderRoot, string[] argNamesToIgnore)
         {
             var argsData = args.Select(x => x.Split('=')).ToDictionary(x => x[0], x => x.Length > 1 ? x[1] : null);
@@ -108,10 +134,15 @@ protected virtual void Initialize(string[] args, out SecurityType securityType,
                 securityType = default;
             }
 
-            if (!argsData.TryGetValue("--market", out market) && !Config.TryGetValue("market", out market) || string.IsNullOrEmpty(market))
+            if (!argsData.TryGetValue("--market", out var marketsStr) &&
+                !Config.TryGetValue("market", out marketsStr) || string.IsNullOrEmpty(marketsStr))
+            {
+                markets = [Market.USA];
+                Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): no market given, defaulting to '{Market.USA}'");
+            }
+            else
             {
-                market = Market.USA;
-                Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): no market given, defaulting to '{market}'");
+                markets = marketsStr.Split(",").Select(x => x.Trim()).ToArray();
             }
 
             // TODO: Should we set the "data-folder" config to "processed-data-directory"?

Lean.DataSource.FuturesUniverseGenerator/FuturesUniverseGenerator.cs

@@ -16,6 +16,7 @@
 using System;
 using System.Collections.Generic;
 using QuantConnect.DataSource.DerivativeUniverseGenerator;
+using QuantConnect.Interfaces;
 
 namespace QuantConnect.DataSource.FuturesUniverseGenerator
 {
@@ -31,8 +32,13 @@ public class FuturesUniverseGenerator : DerivativeUniverseGenerator.DerivativeUn
         /// <param name="market">Market of data to process</param>
         /// <param name="dataFolderRoot">Path to the data folder</param>
         /// <param name="outputFolderRoot">Path to the output folder</param>
-        public FuturesUniverseGenerator(DateTime processingDate, string market, string dataFolderRoot, string outputFolderRoot)
-            : base(processingDate, SecurityType.Future, market, dataFolderRoot, outputFolderRoot)
+        /// <param name="dataProvider">The data provider to use</param>
+        /// <param name="dataCacheProvider">The data cache provider to use</param>
+        /// <param name="historyProvider">The history provider to use</param>
+        public FuturesUniverseGenerator(DateTime processingDate, string market, string dataFolderRoot, string outputFolderRoot,
+            IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider)
+            : base(processingDate, SecurityType.Future, market, dataFolderRoot, outputFolderRoot, dataProvider,
+                  dataCacheProvider, historyProvider)
         {
         }
 

Lean.DataSource.FuturesUniverseGenerator/Program.cs

@@ -13,6 +13,9 @@
  * limitations under the License.
 */
 
+using QuantConnect.Interfaces;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Lean.Engine.HistoricalData;
 using System;
 
 namespace QuantConnect.DataSource.FuturesUniverseGenerator
@@ -34,9 +37,11 @@ public static void Main(string[] args)
         }
 
         protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market,
-            string dataFolderRoot, string outputFolderRoot, DateTime processingDate)
+            string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider,
+            HistoryProviderManager historyProvider)
         {
-            return new FuturesUniverseGenerator(processingDate, market, dataFolderRoot, outputFolderRoot);
+            return new FuturesUniverseGenerator(processingDate, market, dataFolderRoot, outputFolderRoot, dataProvider,
+                dataCacheProvider, historyProvider);
         }
     }
 }

Lean.DataSource.OptionsUniverseGenerator/OptionsUniverseGenerator.cs

@@ -21,6 +21,7 @@
 using QuantConnect.DataSource.DerivativeUniverseGenerator;
 using System.Collections.Generic;
 using QuantConnect.Logging;
+using QuantConnect.Interfaces;
 
 namespace QuantConnect.DataSource.OptionsUniverseGenerator
 {
@@ -39,9 +40,12 @@ public class OptionsUniverseGenerator : DerivativeUniverseGenerator.DerivativeUn
         /// <param name="market">Market of data to process</param>
         /// <param name="dataFolderRoot">Path to the data folder</param>
         /// <param name="outputFolderRoot">Path to the output folder</param>
+        /// <param name="dataProvider">The data provider to use</param>
+        /// <param name="dataCacheProvider">The data cache provider to use</param>
+        /// <param name="historyProvider">The history provider to use</param>
         public OptionsUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string dataFolderRoot,
-            string outputFolderRoot)
-            : base(processingDate, securityType, market, dataFolderRoot, outputFolderRoot)
+            string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider)
+            : base(processingDate, securityType, market, dataFolderRoot, outputFolderRoot, dataProvider, dataCacheProvider, historyProvider)
         {
             if (!_supportedSecurityTypes.Contains(securityType))
             {

Lean.DataSource.OptionsUniverseGenerator/Program.cs

@@ -13,6 +13,9 @@
  * limitations under the License.
 */
 
+using QuantConnect.Interfaces;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Lean.Engine.HistoricalData;
 using System;
 
 namespace QuantConnect.DataSource.OptionsUniverseGenerator
@@ -35,9 +38,11 @@ public static void Main(string[] args)
         }
 
         protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market,
-            string dataFolderRoot, string outputFolderRoot, DateTime processingDate)
+            string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider,
+            HistoryProviderManager historyProvider)
         {
-            return new OptionsUniverseGenerator(processingDate, securityType, market, dataFolderRoot, outputFolderRoot);
+            return new OptionsUniverseGenerator(processingDate, securityType, market, dataFolderRoot, outputFolderRoot,
+                dataProvider, dataCacheProvider, historyProvider);
         }
     }
 }