diff --git a/.github/workflows/gh-actions.yml b/.github/workflows/gh-actions.yml index accee73..038cb5a 100644 --- a/.github/workflows/gh-actions.yml +++ b/.github/workflows/gh-actions.yml @@ -3,6 +3,8 @@ name: Build & Test on: push: branches: ['*'] + pull_request: + branches: ['*'] jobs: build: diff --git a/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs b/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs index be55e4d..20b146d 100644 --- a/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs +++ b/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs @@ -20,13 +20,10 @@ using System.Threading; using System.Threading.Tasks; using NodaTime; -using QuantConnect.Configuration; using QuantConnect.Data; using QuantConnect.Data.Market; using QuantConnect.Interfaces; -using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Lean.Engine.DataFeeds.Enumerators; -using QuantConnect.Lean.Engine.HistoricalData; using QuantConnect.Logging; using QuantConnect.Securities; using QuantConnect.Util; @@ -46,7 +43,7 @@ public abstract class DerivativeUniverseGenerator protected readonly IDataProvider _dataProvider; protected readonly IHistoryProvider _historyProvider; - protected readonly ZipDataCacheProvider _dataCacheProvider; + protected readonly IDataCacheProvider _dataCacheProvider; protected readonly MarketHoursDatabase _marketHoursDatabase; @@ -67,33 +64,20 @@ public abstract class DerivativeUniverseGenerator /// Market of data to process /// Path to the data folder /// Path to the output folder + /// The data provider to use + /// The data cache provider to use + /// The history provider to use public DerivativeUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string dataFolderRoot, - string outputFolderRoot) + string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider) { _processingDate = processingDate; _securityType = securityType; _market = market; _dataFolderRoot = dataFolderRoot; _outputFolderRoot = outputFolderRoot; - - _dataProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("data-provider", "DefaultDataProvider")); - - var mapFileProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("map-file-provider", "LocalZipMapFileProvider")); - mapFileProvider.Initialize(_dataProvider); - - var factorFileProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("factor-file-provider", "LocalZipFactorFileProvider")); - factorFileProvider.Initialize(mapFileProvider, _dataProvider); - - var api = new Api.Api(); - api.Initialize(Globals.UserId, Globals.UserToken, Globals.DataFolder); - - _dataCacheProvider = new ZipDataCacheProvider(_dataProvider); - _historyProvider = new HistoryProviderManager(); - var parameters = new HistoryProviderInitializeParameters(null, api, _dataProvider, _dataCacheProvider, mapFileProvider, - factorFileProvider, (_) => { }, true, new DataPermissionManager(), null, - new AlgorithmSettings() { DailyPreciseEndTime = securityType == SecurityType.IndexOption }); - _historyProvider.Initialize(parameters); - + _dataProvider = dataProvider; + _dataCacheProvider = dataCacheProvider; + _historyProvider = historyProvider; _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); } diff --git a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs index c9317a8..ebc7551 100644 --- a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs +++ b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs @@ -21,6 +21,10 @@ using QuantConnect.Configuration; using QuantConnect.Logging; using QuantConnect.Util; +using QuantConnect.Data; +using QuantConnect.Interfaces; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Lean.Engine.HistoricalData; namespace QuantConnect.DataSource.DerivativeUniverseGenerator { @@ -46,34 +50,55 @@ public abstract class Program protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null) { - Initialize(args, out var securityType, out var market, out var dataFolderRoot, out var outputFolderRoot, + Initialize(args, out var securityType, out var markets, out var dataFolderRoot, out var outputFolderRoot, argNamesToIgnore ?? Array.Empty()); Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): " + - $"Security type: {securityType}. Market: {market}. Data folder: {dataFolderRoot}. Output folder: {outputFolderRoot}"); + $"Security type: {securityType}. Markets: {string.Join(", ", markets)}. Data folder: {dataFolderRoot}. Output folder: {outputFolderRoot}"); Log.DebuggingEnabled = Config.GetBool("debug-mode"); var dateStr = Environment.GetEnvironmentVariable(DataFleetDeploymentDateEnvVariable) ?? $"{DateTime.UtcNow.Date:yyyyMMdd}"; var processingDate = DateTime.ParseExact(dateStr, DateFormat.EightCharacter, CultureInfo.InvariantCulture); + var dataProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("data-provider", "DefaultDataProvider")); + + var mapFileProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("map-file-provider", "LocalZipMapFileProvider")); + mapFileProvider.Initialize(dataProvider); + + var factorFileProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("factor-file-provider", "LocalZipFactorFileProvider")); + factorFileProvider.Initialize(mapFileProvider, dataProvider); + var api = new Api.Api(); + api.Initialize(Globals.UserId, Globals.UserToken, Globals.DataFolder); + + var dataCacheProvider = new ZipDataCacheProvider(dataProvider); + var historyProvider = new HistoryProviderManager(); + var parameters = new HistoryProviderInitializeParameters(null, api, dataProvider, dataCacheProvider, mapFileProvider, + factorFileProvider, (_) => { }, true, new DataPermissionManager(), null, + new AlgorithmSettings() { DailyPreciseEndTime = securityType == SecurityType.IndexOption }); + historyProvider.Initialize(parameters); + var timer = new Stopwatch(); timer.Start(); - var optionsUniverseGenerator = GetUniverseGenerator(securityType, market, dataFolderRoot, outputFolderRoot, processingDate); - - try + foreach (var market in markets) { - if (!optionsUniverseGenerator.Run()) + var optionsUniverseGenerator = GetUniverseGenerator(securityType, market, dataFolderRoot, outputFolderRoot, processingDate, + dataProvider, dataCacheProvider, historyProvider); + + try { - Log.Error($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Failed to generate options universe."); + if (!optionsUniverseGenerator.Run()) + { + Log.Error($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Failed to generate universe."); + Environment.Exit(1); + } + } + catch (Exception ex) + { + Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating universe."); Environment.Exit(1); } } - catch (Exception ex) - { - Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating options universe."); - Environment.Exit(1); - } Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): DONE in {timer.Elapsed:g}"); @@ -81,12 +106,13 @@ protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null) } protected abstract DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, string dataFolderRoot, - string outputFolderRoot, DateTime processingDate); + string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, + HistoryProviderManager historyProvider); /// /// Validate and extract command line args and configuration options. /// - protected virtual void Initialize(string[] args, out SecurityType securityType, out string market, out string dataFolderRoot, + protected virtual void Initialize(string[] args, out SecurityType securityType, out string[] markets, out string dataFolderRoot, out string outputFolderRoot, string[] argNamesToIgnore) { var argsData = args.Select(x => x.Split('=')).ToDictionary(x => x[0], x => x.Length > 1 ? x[1] : null); @@ -108,10 +134,15 @@ protected virtual void Initialize(string[] args, out SecurityType securityType, securityType = default; } - if (!argsData.TryGetValue("--market", out market) && !Config.TryGetValue("market", out market) || string.IsNullOrEmpty(market)) + if (!argsData.TryGetValue("--market", out var marketsStr) && + !Config.TryGetValue("market", out marketsStr) || string.IsNullOrEmpty(marketsStr)) + { + markets = [Market.USA]; + Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): no market given, defaulting to '{Market.USA}'"); + } + else { - market = Market.USA; - Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): no market given, defaulting to '{market}'"); + markets = marketsStr.Split(",").Select(x => x.Trim()).ToArray(); } // TODO: Should we set the "data-folder" config to "processed-data-directory"? diff --git a/Lean.DataSource.FuturesUniverseGenerator/FuturesUniverseGenerator.cs b/Lean.DataSource.FuturesUniverseGenerator/FuturesUniverseGenerator.cs index e18293a..950722f 100644 --- a/Lean.DataSource.FuturesUniverseGenerator/FuturesUniverseGenerator.cs +++ b/Lean.DataSource.FuturesUniverseGenerator/FuturesUniverseGenerator.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; using QuantConnect.DataSource.DerivativeUniverseGenerator; +using QuantConnect.Interfaces; namespace QuantConnect.DataSource.FuturesUniverseGenerator { @@ -31,8 +32,13 @@ public class FuturesUniverseGenerator : DerivativeUniverseGenerator.DerivativeUn /// Market of data to process /// Path to the data folder /// Path to the output folder - public FuturesUniverseGenerator(DateTime processingDate, string market, string dataFolderRoot, string outputFolderRoot) - : base(processingDate, SecurityType.Future, market, dataFolderRoot, outputFolderRoot) + /// The data provider to use + /// The data cache provider to use + /// The history provider to use + public FuturesUniverseGenerator(DateTime processingDate, string market, string dataFolderRoot, string outputFolderRoot, + IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider) + : base(processingDate, SecurityType.Future, market, dataFolderRoot, outputFolderRoot, dataProvider, + dataCacheProvider, historyProvider) { } diff --git a/Lean.DataSource.FuturesUniverseGenerator/Program.cs b/Lean.DataSource.FuturesUniverseGenerator/Program.cs index 66055a4..320baa6 100644 --- a/Lean.DataSource.FuturesUniverseGenerator/Program.cs +++ b/Lean.DataSource.FuturesUniverseGenerator/Program.cs @@ -13,6 +13,9 @@ * limitations under the License. */ +using QuantConnect.Interfaces; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Lean.Engine.HistoricalData; using System; namespace QuantConnect.DataSource.FuturesUniverseGenerator @@ -34,9 +37,11 @@ public static void Main(string[] args) } protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, - string dataFolderRoot, string outputFolderRoot, DateTime processingDate) + string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, + HistoryProviderManager historyProvider) { - return new FuturesUniverseGenerator(processingDate, market, dataFolderRoot, outputFolderRoot); + return new FuturesUniverseGenerator(processingDate, market, dataFolderRoot, outputFolderRoot, dataProvider, + dataCacheProvider, historyProvider); } } } diff --git a/Lean.DataSource.OptionsUniverseGenerator/OptionsUniverseGenerator.cs b/Lean.DataSource.OptionsUniverseGenerator/OptionsUniverseGenerator.cs index 94e6bdb..13bc70c 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/OptionsUniverseGenerator.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/OptionsUniverseGenerator.cs @@ -21,6 +21,7 @@ using QuantConnect.DataSource.DerivativeUniverseGenerator; using System.Collections.Generic; using QuantConnect.Logging; +using QuantConnect.Interfaces; namespace QuantConnect.DataSource.OptionsUniverseGenerator { @@ -39,9 +40,12 @@ public class OptionsUniverseGenerator : DerivativeUniverseGenerator.DerivativeUn /// Market of data to process /// Path to the data folder /// Path to the output folder + /// The data provider to use + /// The data cache provider to use + /// The history provider to use public OptionsUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string dataFolderRoot, - string outputFolderRoot) - : base(processingDate, securityType, market, dataFolderRoot, outputFolderRoot) + string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider) + : base(processingDate, securityType, market, dataFolderRoot, outputFolderRoot, dataProvider, dataCacheProvider, historyProvider) { if (!_supportedSecurityTypes.Contains(securityType)) { diff --git a/Lean.DataSource.OptionsUniverseGenerator/Program.cs b/Lean.DataSource.OptionsUniverseGenerator/Program.cs index fbd34df..b40ea10 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/Program.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/Program.cs @@ -13,6 +13,9 @@ * limitations under the License. */ +using QuantConnect.Interfaces; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Lean.Engine.HistoricalData; using System; namespace QuantConnect.DataSource.OptionsUniverseGenerator @@ -35,9 +38,11 @@ public static void Main(string[] args) } protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, - string dataFolderRoot, string outputFolderRoot, DateTime processingDate) + string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, + HistoryProviderManager historyProvider) { - return new OptionsUniverseGenerator(processingDate, securityType, market, dataFolderRoot, outputFolderRoot); + return new OptionsUniverseGenerator(processingDate, securityType, market, dataFolderRoot, outputFolderRoot, + dataProvider, dataCacheProvider, historyProvider); } } }