diff --git a/Lean.DataSource.DerivativeUniverseGenerator/ChainSymbolProvider.cs b/Lean.DataSource.DerivativeUniverseGenerator/ChainSymbolProvider.cs index 450e174..93036d9 100644 --- a/Lean.DataSource.DerivativeUniverseGenerator/ChainSymbolProvider.cs +++ b/Lean.DataSource.DerivativeUniverseGenerator/ChainSymbolProvider.cs @@ -93,13 +93,13 @@ public Dictionary> GetSymbols() return result; } - - /// + + /// /// Gets the zip file names for the canonical symbols where the contracts or universe constituents will be read from. /// - protected virtual IEnumerable GetZipFileNames(DateTime date, Resolution resolution, TickType tickType) + protected virtual IEnumerable GetZipFileNames(DateTime date, Resolution resolution) { - var tickTypeLower = tickType.TickTypeToLower(); + var tickTypesLower = _symbolsDataTickTypes.Select(tickType => tickType.TickTypeToLower()).ToArray(); if (resolution == Resolution.Minute) { @@ -112,8 +112,11 @@ protected virtual IEnumerable GetZipFileNames(DateTime date, Resolution var optionStyleLower = _securityType.DefaultOptionStyle().OptionStyleToLower(); return directories - .Select(directory => Path.Combine(directory, $"{dateStr}_{tickTypeLower}_{optionStyleLower}.zip")) - .Where(fileName => File.Exists(fileName)); + .Select(directory => tickTypesLower + .Select(tickTypeLower => Path.Combine(directory, $"{dateStr}_{tickTypeLower}_{optionStyleLower}.zip")) + .Where(fileName => File.Exists(fileName)) + .FirstOrDefault()) + .Where(fileName => fileName != null); } // Support for resolutions higher than minute, just for Lean local repo data generation else @@ -125,12 +128,20 @@ protected virtual IEnumerable GetZipFileNames(DateTime date, Resolution Path.Combine(_dataSourceFolder, resolution.ResolutionToLower()), $"*{dateStr}*.zip", SearchOption.AllDirectories) - .Where(fileName => + .Select(fileName => { var fileInfo = new FileInfo(fileName); var fileNameParts = fileInfo.Name.Split('_'); - return fileNameParts.Length == 4 && fileNameParts[1] == dateStr && fileNameParts[2] == tickTypeLower; - }); + var tickTypeIndex = Array.IndexOf(tickTypesLower, fileNameParts[2]); + + return (fileName, directoryName: fileInfo.DirectoryName, tickTypeIndex); + }) + // Get only supported tick type data + .Where(tuple => tuple.tickTypeIndex > -1) + // For each contract get the first matching tick type file + .OrderBy(tuple => tuple.tickTypeIndex) + .GroupBy(tuple => tuple.directoryName) + .Select(group => group.First().fileName); } catch (DirectoryNotFoundException) { @@ -139,23 +150,6 @@ protected virtual IEnumerable GetZipFileNames(DateTime date, Resolution } } - /// - /// Gets the zip file names for the canonical symbols where the contracts or universe constituents will be read from. - /// - private IEnumerable GetZipFileNames(DateTime date, Resolution resolution) - { - foreach (var tickType in _symbolsDataTickTypes) - { - var fileNames = GetZipFileNames(date, resolution, tickType).ToList(); - if (fileNames.Count > 0) - { - return fileNames; - } - } - - return Enumerable.Empty(); - } - /// /// Reads the symbols from the zip entry names for the given canonical symbol. /// diff --git a/Lean.DataSource.FuturesUniverseGenerator/FutureChainSymbolProvider.cs b/Lean.DataSource.FuturesUniverseGenerator/FutureChainSymbolProvider.cs index 64a29a2..e620e03 100644 --- a/Lean.DataSource.FuturesUniverseGenerator/FutureChainSymbolProvider.cs +++ b/Lean.DataSource.FuturesUniverseGenerator/FutureChainSymbolProvider.cs @@ -36,31 +36,42 @@ public FutureChainSymbolProvider(IDataCacheProvider dataCacheProvider, DateTime { } - protected override IEnumerable GetZipFileNames(DateTime date, Resolution resolution, TickType tickType) + protected override IEnumerable GetZipFileNames(DateTime date, Resolution resolution) { - var tickTypeLower = tickType.TickTypeToLower(); + var tickTypesLower = _symbolsDataTickTypes.Select(tickType => tickType.TickTypeToLower()).ToArray(); if (resolution == Resolution.Minute) { var basePath = Path.Combine(_dataSourceFolder, resolution.ResolutionToLower()); var dateStr = date.ToString("yyyyMMdd"); - return Directory.EnumerateDirectories(basePath, "*", new EnumerationOptions() { RecurseSubdirectories = true, MaxRecursionDepth = 1 }) - .Select(directory => Path.Combine(directory, $"{dateStr}_{tickTypeLower}.zip")) - .Where(fileName => File.Exists(fileName)); + return Directory.EnumerateDirectories(basePath) + .Select(directory => tickTypesLower + .Select(tickTypeLower => Path.Combine(directory, $"{dateStr}_{tickTypeLower}.zip")) + .Where(fileName => File.Exists(fileName)) + .FirstOrDefault()) + .Where(fileName => fileName != null); } // Support for resolutions higher than minute, just for Lean local repo data generation else { try { - return Directory.EnumerateFiles(Path.Combine(_dataSourceFolder, resolution.ResolutionToLower()), $"*_{tickTypeLower}.zip") - .Where(fileName => + return Directory.EnumerateFiles(Path.Combine(_dataSourceFolder, resolution.ResolutionToLower()), $"*.zip") + .Select(fileName => { var fileInfo = new FileInfo(fileName); var fileNameParts = Path.GetFileNameWithoutExtension(fileInfo.Name).Split('_'); - return fileNameParts.Length == 2 && fileNameParts[1] == tickTypeLower; - }); + var tickTypeIndex = Array.IndexOf(tickTypesLower, fileNameParts[1]); + + return (fileName, ticker: fileNameParts[0], tickTypeIndex); + }) + // Get only supported tick type data + .Where(tuple => tuple.tickTypeIndex > -1) + // For each ticker get the first matching tick type file + .OrderBy(tuple => tuple.tickTypeIndex) + .GroupBy(tuple => tuple.ticker) + .Select(group => group.First().fileName); } catch (DirectoryNotFoundException) {