From ad22d222b68c4c8b793e356b1bc36eb0f41ca96a Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 26 Dec 2024 12:55:22 -0400 Subject: [PATCH 1/4] Allow universe files generation for multiple markets at a time --- .../Program.cs | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs index c9317a8..89e37b1 100644 --- a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs +++ b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs @@ -46,11 +46,11 @@ public abstract class Program protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null) { - Initialize(args, out var securityType, out var market, out var dataFolderRoot, out var outputFolderRoot, + Initialize(args, out var securityType, out var markets, out var dataFolderRoot, out var outputFolderRoot, argNamesToIgnore ?? Array.Empty()); Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): " + - $"Security type: {securityType}. Market: {market}. Data folder: {dataFolderRoot}. Output folder: {outputFolderRoot}"); + $"Security type: {securityType}. Markets: {string.Join(", ", markets)}. Data folder: {dataFolderRoot}. Output folder: {outputFolderRoot}"); Log.DebuggingEnabled = Config.GetBool("debug-mode"); var dateStr = Environment.GetEnvironmentVariable(DataFleetDeploymentDateEnvVariable) ?? $"{DateTime.UtcNow.Date:yyyyMMdd}"; @@ -59,20 +59,25 @@ protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null) var timer = new Stopwatch(); timer.Start(); - var optionsUniverseGenerator = GetUniverseGenerator(securityType, market, dataFolderRoot, outputFolderRoot, processingDate); - - try + foreach (var market in markets) { - if (!optionsUniverseGenerator.Run()) + var optionsUniverseGenerator = GetUniverseGenerator(securityType, market, dataFolderRoot, outputFolderRoot, processingDate); + + try + { + if (!optionsUniverseGenerator.Run()) + { + Log.Error($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Failed to generate universe."); + Environment.Exit(1); + } + } + catch (Exception ex) { - Log.Error($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Failed to generate options universe."); + Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating universe."); Environment.Exit(1); } - } - catch (Exception ex) - { - Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating options universe."); - Environment.Exit(1); + + Composer.Instance.Reset(); } Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): DONE in {timer.Elapsed:g}"); @@ -86,7 +91,7 @@ protected abstract DerivativeUniverseGenerator GetUniverseGenerator(SecurityType /// /// Validate and extract command line args and configuration options. /// - protected virtual void Initialize(string[] args, out SecurityType securityType, out string market, out string dataFolderRoot, + protected virtual void Initialize(string[] args, out SecurityType securityType, out string[] markets, out string dataFolderRoot, out string outputFolderRoot, string[] argNamesToIgnore) { var argsData = args.Select(x => x.Split('=')).ToDictionary(x => x[0], x => x.Length > 1 ? x[1] : null); @@ -108,10 +113,15 @@ protected virtual void Initialize(string[] args, out SecurityType securityType, securityType = default; } - if (!argsData.TryGetValue("--market", out market) && !Config.TryGetValue("market", out market) || string.IsNullOrEmpty(market)) + if (!argsData.TryGetValue("--market", out var marketsStr) && + !Config.TryGetValue("market", out marketsStr) || string.IsNullOrEmpty(marketsStr)) + { + markets = [Market.USA]; + Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): no market given, defaulting to '{Market.USA}'"); + } + else { - market = Market.USA; - Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): no market given, defaulting to '{market}'"); + markets = marketsStr.Split(",").Select(x => x.Trim()).ToArray(); } // TODO: Should we set the "data-folder" config to "processed-data-directory"? From 721442edbf654bd999a223ed2d0588639e3f0b07 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 26 Dec 2024 12:56:03 -0400 Subject: [PATCH 2/4] Trigger GH actions on pull requests --- .github/workflows/gh-actions.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/gh-actions.yml b/.github/workflows/gh-actions.yml index accee73..038cb5a 100644 --- a/.github/workflows/gh-actions.yml +++ b/.github/workflows/gh-actions.yml @@ -3,6 +3,8 @@ name: Build & Test on: push: branches: ['*'] + pull_request: + branches: ['*'] jobs: build: From f1f0f19b4ae33a547d85f964cd716e0f8db6eed3 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 26 Dec 2024 15:26:25 -0400 Subject: [PATCH 3/4] Cleanup --- .../DerivativeUniverseGenerator.cs | 32 +++++-------------- .../Program.cs | 29 ++++++++++++++--- .../FuturesUniverseGenerator.cs | 10 ++++-- .../Program.cs | 9 ++++-- .../OptionsUniverseGenerator.cs | 8 +++-- .../Program.cs | 9 ++++-- 6 files changed, 61 insertions(+), 36 deletions(-) diff --git a/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs b/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs index be55e4d..20b146d 100644 --- a/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs +++ b/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs @@ -20,13 +20,10 @@ using System.Threading; using System.Threading.Tasks; using NodaTime; -using QuantConnect.Configuration; using QuantConnect.Data; using QuantConnect.Data.Market; using QuantConnect.Interfaces; -using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Lean.Engine.DataFeeds.Enumerators; -using QuantConnect.Lean.Engine.HistoricalData; using QuantConnect.Logging; using QuantConnect.Securities; using QuantConnect.Util; @@ -46,7 +43,7 @@ public abstract class DerivativeUniverseGenerator protected readonly IDataProvider _dataProvider; protected readonly IHistoryProvider _historyProvider; - protected readonly ZipDataCacheProvider _dataCacheProvider; + protected readonly IDataCacheProvider _dataCacheProvider; protected readonly MarketHoursDatabase _marketHoursDatabase; @@ -67,33 +64,20 @@ public abstract class DerivativeUniverseGenerator /// Market of data to process /// Path to the data folder /// Path to the output folder + /// The data provider to use + /// The data cache provider to use + /// The history provider to use public DerivativeUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string dataFolderRoot, - string outputFolderRoot) + string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider) { _processingDate = processingDate; _securityType = securityType; _market = market; _dataFolderRoot = dataFolderRoot; _outputFolderRoot = outputFolderRoot; - - _dataProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("data-provider", "DefaultDataProvider")); - - var mapFileProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("map-file-provider", "LocalZipMapFileProvider")); - mapFileProvider.Initialize(_dataProvider); - - var factorFileProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("factor-file-provider", "LocalZipFactorFileProvider")); - factorFileProvider.Initialize(mapFileProvider, _dataProvider); - - var api = new Api.Api(); - api.Initialize(Globals.UserId, Globals.UserToken, Globals.DataFolder); - - _dataCacheProvider = new ZipDataCacheProvider(_dataProvider); - _historyProvider = new HistoryProviderManager(); - var parameters = new HistoryProviderInitializeParameters(null, api, _dataProvider, _dataCacheProvider, mapFileProvider, - factorFileProvider, (_) => { }, true, new DataPermissionManager(), null, - new AlgorithmSettings() { DailyPreciseEndTime = securityType == SecurityType.IndexOption }); - _historyProvider.Initialize(parameters); - + _dataProvider = dataProvider; + _dataCacheProvider = dataCacheProvider; + _historyProvider = historyProvider; _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); } diff --git a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs index 89e37b1..27a77f4 100644 --- a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs +++ b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs @@ -21,6 +21,10 @@ using QuantConnect.Configuration; using QuantConnect.Logging; using QuantConnect.Util; +using QuantConnect.Data; +using QuantConnect.Interfaces; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Lean.Engine.HistoricalData; namespace QuantConnect.DataSource.DerivativeUniverseGenerator { @@ -56,12 +60,30 @@ protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null) var dateStr = Environment.GetEnvironmentVariable(DataFleetDeploymentDateEnvVariable) ?? $"{DateTime.UtcNow.Date:yyyyMMdd}"; var processingDate = DateTime.ParseExact(dateStr, DateFormat.EightCharacter, CultureInfo.InvariantCulture); + var dataProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("data-provider", "DefaultDataProvider")); + + var mapFileProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("map-file-provider", "LocalZipMapFileProvider")); + mapFileProvider.Initialize(dataProvider); + + var factorFileProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("factor-file-provider", "LocalZipFactorFileProvider")); + factorFileProvider.Initialize(mapFileProvider, dataProvider); + var api = new Api.Api(); + api.Initialize(Globals.UserId, Globals.UserToken, Globals.DataFolder); + + var dataCacheProvider = new ZipDataCacheProvider(dataProvider); + var historyProvider = new HistoryProviderManager(); + var parameters = new HistoryProviderInitializeParameters(null, api, dataProvider, dataCacheProvider, mapFileProvider, + factorFileProvider, (_) => { }, true, new DataPermissionManager(), null, + new AlgorithmSettings() { DailyPreciseEndTime = securityType == SecurityType.IndexOption }); + historyProvider.Initialize(parameters); + var timer = new Stopwatch(); timer.Start(); foreach (var market in markets) { - var optionsUniverseGenerator = GetUniverseGenerator(securityType, market, dataFolderRoot, outputFolderRoot, processingDate); + var optionsUniverseGenerator = GetUniverseGenerator(securityType, market, dataFolderRoot, outputFolderRoot, processingDate, + dataProvider, dataCacheProvider, historyProvider); try { @@ -76,8 +98,6 @@ protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null) Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating universe."); Environment.Exit(1); } - - Composer.Instance.Reset(); } Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): DONE in {timer.Elapsed:g}"); @@ -86,7 +106,8 @@ protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null) } protected abstract DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, string dataFolderRoot, - string outputFolderRoot, DateTime processingDate); + string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, ZipDataCacheProvider dataCacheProvider, + HistoryProviderManager historyProvider); /// /// Validate and extract command line args and configuration options. diff --git a/Lean.DataSource.FuturesUniverseGenerator/FuturesUniverseGenerator.cs b/Lean.DataSource.FuturesUniverseGenerator/FuturesUniverseGenerator.cs index e18293a..950722f 100644 --- a/Lean.DataSource.FuturesUniverseGenerator/FuturesUniverseGenerator.cs +++ b/Lean.DataSource.FuturesUniverseGenerator/FuturesUniverseGenerator.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; using QuantConnect.DataSource.DerivativeUniverseGenerator; +using QuantConnect.Interfaces; namespace QuantConnect.DataSource.FuturesUniverseGenerator { @@ -31,8 +32,13 @@ public class FuturesUniverseGenerator : DerivativeUniverseGenerator.DerivativeUn /// Market of data to process /// Path to the data folder /// Path to the output folder - public FuturesUniverseGenerator(DateTime processingDate, string market, string dataFolderRoot, string outputFolderRoot) - : base(processingDate, SecurityType.Future, market, dataFolderRoot, outputFolderRoot) + /// The data provider to use + /// The data cache provider to use + /// The history provider to use + public FuturesUniverseGenerator(DateTime processingDate, string market, string dataFolderRoot, string outputFolderRoot, + IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider) + : base(processingDate, SecurityType.Future, market, dataFolderRoot, outputFolderRoot, dataProvider, + dataCacheProvider, historyProvider) { } diff --git a/Lean.DataSource.FuturesUniverseGenerator/Program.cs b/Lean.DataSource.FuturesUniverseGenerator/Program.cs index 66055a4..44b1638 100644 --- a/Lean.DataSource.FuturesUniverseGenerator/Program.cs +++ b/Lean.DataSource.FuturesUniverseGenerator/Program.cs @@ -13,6 +13,9 @@ * limitations under the License. */ +using QuantConnect.Interfaces; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Lean.Engine.HistoricalData; using System; namespace QuantConnect.DataSource.FuturesUniverseGenerator @@ -34,9 +37,11 @@ public static void Main(string[] args) } protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, - string dataFolderRoot, string outputFolderRoot, DateTime processingDate) + string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, ZipDataCacheProvider dataCacheProvider, + HistoryProviderManager historyProvider) { - return new FuturesUniverseGenerator(processingDate, market, dataFolderRoot, outputFolderRoot); + return new FuturesUniverseGenerator(processingDate, market, dataFolderRoot, outputFolderRoot, dataProvider, + dataCacheProvider, historyProvider); } } } diff --git a/Lean.DataSource.OptionsUniverseGenerator/OptionsUniverseGenerator.cs b/Lean.DataSource.OptionsUniverseGenerator/OptionsUniverseGenerator.cs index 94e6bdb..13bc70c 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/OptionsUniverseGenerator.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/OptionsUniverseGenerator.cs @@ -21,6 +21,7 @@ using QuantConnect.DataSource.DerivativeUniverseGenerator; using System.Collections.Generic; using QuantConnect.Logging; +using QuantConnect.Interfaces; namespace QuantConnect.DataSource.OptionsUniverseGenerator { @@ -39,9 +40,12 @@ public class OptionsUniverseGenerator : DerivativeUniverseGenerator.DerivativeUn /// Market of data to process /// Path to the data folder /// Path to the output folder + /// The data provider to use + /// The data cache provider to use + /// The history provider to use public OptionsUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string dataFolderRoot, - string outputFolderRoot) - : base(processingDate, securityType, market, dataFolderRoot, outputFolderRoot) + string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider) + : base(processingDate, securityType, market, dataFolderRoot, outputFolderRoot, dataProvider, dataCacheProvider, historyProvider) { if (!_supportedSecurityTypes.Contains(securityType)) { diff --git a/Lean.DataSource.OptionsUniverseGenerator/Program.cs b/Lean.DataSource.OptionsUniverseGenerator/Program.cs index fbd34df..e783e1d 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/Program.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/Program.cs @@ -13,6 +13,9 @@ * limitations under the License. */ +using QuantConnect.Interfaces; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Lean.Engine.HistoricalData; using System; namespace QuantConnect.DataSource.OptionsUniverseGenerator @@ -35,9 +38,11 @@ public static void Main(string[] args) } protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, - string dataFolderRoot, string outputFolderRoot, DateTime processingDate) + string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, ZipDataCacheProvider dataCacheProvider, + HistoryProviderManager historyProvider) { - return new OptionsUniverseGenerator(processingDate, securityType, market, dataFolderRoot, outputFolderRoot); + return new OptionsUniverseGenerator(processingDate, securityType, market, dataFolderRoot, outputFolderRoot, + dataProvider, dataCacheProvider, historyProvider); } } } From 3fabd094f2a44694b0bf4aa00b4879311bec5cce Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 26 Dec 2024 17:43:32 -0400 Subject: [PATCH 4/4] Refactor GetUniverseGenerator method signatures Updated GetUniverseGenerator method signatures in Program.cs to use IDataCacheProvider instead of ZipDataCacheProvider --- Lean.DataSource.DerivativeUniverseGenerator/Program.cs | 2 +- Lean.DataSource.FuturesUniverseGenerator/Program.cs | 2 +- Lean.DataSource.OptionsUniverseGenerator/Program.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs index 27a77f4..ebc7551 100644 --- a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs +++ b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs @@ -106,7 +106,7 @@ protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null) } protected abstract DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, string dataFolderRoot, - string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, ZipDataCacheProvider dataCacheProvider, + string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, HistoryProviderManager historyProvider); /// diff --git a/Lean.DataSource.FuturesUniverseGenerator/Program.cs b/Lean.DataSource.FuturesUniverseGenerator/Program.cs index 44b1638..320baa6 100644 --- a/Lean.DataSource.FuturesUniverseGenerator/Program.cs +++ b/Lean.DataSource.FuturesUniverseGenerator/Program.cs @@ -37,7 +37,7 @@ public static void Main(string[] args) } protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, - string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, ZipDataCacheProvider dataCacheProvider, + string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, HistoryProviderManager historyProvider) { return new FuturesUniverseGenerator(processingDate, market, dataFolderRoot, outputFolderRoot, dataProvider, diff --git a/Lean.DataSource.OptionsUniverseGenerator/Program.cs b/Lean.DataSource.OptionsUniverseGenerator/Program.cs index e783e1d..b40ea10 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/Program.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/Program.cs @@ -38,7 +38,7 @@ public static void Main(string[] args) } protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, - string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, ZipDataCacheProvider dataCacheProvider, + string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, HistoryProviderManager historyProvider) { return new OptionsUniverseGenerator(processingDate, securityType, market, dataFolderRoot, outputFolderRoot,