From ebc6d5b448a4b14120b0a1dcce31ad5a0b894fc6 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Wed, 17 Sep 2025 13:38:54 -0300 Subject: [PATCH] Handle missing daily index data - On missing daily index data, we will build it from minute --- .../IndexHistoryProvider.cs | 113 +++++++++++------- .../IndexHistoryProviderTests.cs | 4 +- 2 files changed, 69 insertions(+), 48 deletions(-) diff --git a/Lean.DataSource.OptionsUniverseGenerator/IndexHistoryProvider.cs b/Lean.DataSource.OptionsUniverseGenerator/IndexHistoryProvider.cs index 43b9bdd..1e487ea 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/IndexHistoryProvider.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/IndexHistoryProvider.cs @@ -13,19 +13,20 @@ * limitations under the License. */ -using System; -using NodaTime; -using RestSharp; using Newtonsoft.Json; -using System.Threading; +using NodaTime; using QuantConnect.Data; -using QuantConnect.Logging; using QuantConnect.Data.Market; -using System.Collections.Generic; using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Lean.Engine.HistoricalData; -using QuantConnect.Util; +using QuantConnect.Logging; using QuantConnect.Securities; +using QuantConnect.Util; +using RestSharp; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; namespace QuantConnect.DataSource.OptionsUniverseGenerator { @@ -127,68 +128,88 @@ public IEnumerable GetHistory(HistoryRequest request) var start = Time.DateTimeToUnixTimeStamp(request.StartTimeUtc); var end = Time.DateTimeToUnixTimeStamp(request.EndTimeUtc); - // let's retry on failure - const int maxRetries = 10; - for (var retryCount = 0; retryCount <= maxRetries; retryCount++) + foreach (var interval in new[] { "1d", "1m" }) { - if (retryCount > 0) - { - Thread.Sleep(2 * Time.OneSecond); - Log.Trace($"IndexHistoryProvider.GetHistory(): Retry attempt {retryCount}/{maxRetries} for " + - $"{request.Symbol}-{request.Resolution}-{request.TickType}."); - } - - var restRequest = new RestRequest($"chart/{symbol}"); - restRequest.AddQueryParameter("period1", start.ToString()); - restRequest.AddQueryParameter("period2", end.ToString()); - restRequest.AddQueryParameter("interval", "1d"); - restRequest.AddQueryParameter("includePrePost", request.IncludeExtendedMarketHours.ToString()); - - try + // let's retry on failure + const int maxRetries = 5; + for (var retryCount = 0; retryCount <= maxRetries; retryCount++) { - var response = _restClient.Get(restRequest); - if (response.StatusCode != System.Net.HttpStatusCode.OK) + if (retryCount > 0) { - Log.Error($"IndexHistoryProvider.GetHistory(): Failed to get history for {symbol}. Status code: {response.StatusCode}."); - continue; + Thread.Sleep((2 + retryCount) * Time.OneSecond); + Log.Trace($"IndexHistoryProvider.GetHistory(): Retry attempt {retryCount}/{maxRetries} for " + + $"{request.Symbol}-{request.Resolution}-{request.TickType}."); } - var content = response.Content; + var restRequest = new RestRequest($"chart/{symbol}"); + restRequest.AddQueryParameter("period1", start.ToString()); + restRequest.AddQueryParameter("period2", end.ToString()); + restRequest.AddQueryParameter("interval", interval); + restRequest.AddQueryParameter("includePrePost", request.IncludeExtendedMarketHours.ToString()); - // Log the response content for debugging purposes - Log.Trace($"IndexHistoryProvider.GetHistory(): Response content for {request.Symbol}-{request.Resolution}-{request.TickType}: {content}"); - - var indexPrices = JsonConvert.DeserializeObject(content); - if (indexPrices == null) + try + { + var response = _restClient.Get(restRequest); + if (response.StatusCode != System.Net.HttpStatusCode.OK) + { + Log.Error($"IndexHistoryProvider.GetHistory(): Failed to get history for {symbol}. Status code: {response.StatusCode}."); + continue; + } + + var content = response.Content; + + // Log the response content for debugging purposes + Log.Trace($"IndexHistoryProvider.GetHistory(): Response content for {request.Symbol}-{request.Resolution}-{request.TickType}: {content}"); + + var indexPrices = JsonConvert.DeserializeObject(content); + if (indexPrices == null) + { + Log.Error($"IndexHistoryProvider.GetHistory(): Failed to deserialize response for {symbol}."); + continue; + } + // to catch any failure in the data and retry we need to yield it + var result = ParseHistory(request.Symbol, indexPrices, request.ExchangeHours, interval); + if (interval == "1m") + { + result = result.Where(x => request.ExchangeHours.IsOpen(x.Time, x.EndTime, extendedMarketHours: false)); + result = LeanData.AggregateTradeBars(result, request.Symbol, Time.OneDay); + } + + foreach (var point in result) + { + if (_useDailyPreciseEndTime) + { + var calendar = LeanData.GetDailyCalendar(point.EndTime, request.ExchangeHours, false); + point.Time = calendar.Start; + point.EndTime = calendar.End; + } + } + return result.ToList(); + } + catch (Exception exception) { - Log.Error($"IndexHistoryProvider.GetHistory(): Failed to deserialize response for {symbol}."); + Log.Error($"IndexHistoryProvider.GetHistory(): Failed to parse response for {symbol}. Exception: {exception}"); continue; } - return ParseHistory(request.Symbol, indexPrices, request.ExchangeHours); - } - catch (Exception exception) - { - Log.Error($"IndexHistoryProvider.GetHistory(): Failed to parse response for {symbol}. Exception: {exception}"); - continue; } } return null; } - private IEnumerable ParseHistory(Symbol symbol, YahooFinanceIndexPrices indexPrices, SecurityExchangeHours exchange) + private IEnumerable ParseHistory(Symbol symbol, YahooFinanceIndexPrices indexPrices, SecurityExchangeHours exchange, string interval) { - for (int i = 0; i < indexPrices.Timestamps.Count; i++) + for (var i = 0; i < indexPrices.Timestamps.Count; i++) { + DateTime endTime; var time = Time.UnixTimeStampToDateTime(indexPrices.Timestamps[i]).ConvertFromUtc(exchange.TimeZone); - var endTime = DateTime.MaxValue; - if (!_useDailyPreciseEndTime) + if (interval == "1d") { time = time.Date; endTime = time.AddDays(1); } else { - endTime = LeanData.GetNextDailyEndTime(symbol, time, exchange); + endTime = time.AddMinutes(1); } var open = indexPrices.OpenPrices[i]; diff --git a/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/IndexHistoryProviderTests.cs b/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/IndexHistoryProviderTests.cs index 32ca9e5..0452d92 100644 --- a/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/IndexHistoryProviderTests.cs +++ b/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/IndexHistoryProviderTests.cs @@ -42,8 +42,8 @@ public void GetsDailyHistory(string ticker) var marketHoursEntry = MarketHoursDatabase.FromDataFolder().GetEntry(symbol.ID.Market, symbol, symbol.SecurityType); var request = new HistoryRequest( - new DateTime(2024, 01, 08, 13, 30, 0), - new DateTime(2024, 02, 08, 13, 30, 0), + new DateTime(2025, 9, 16, 0, 0, 0), + new DateTime(2025, 9, 17, 0, 0, 0), typeof(TradeBar), symbol, Resolution.Daily,