Skip to content

Improve symbols chain fetching from files #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jhonabreul merged 2 commits into from
Jan 3, 2025
Merged

Improve symbols chain fetching from files #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7ac8cc9
Improve symbols chain fetching from files
jhonabreul Jan 3, 2025
efbfe17
Minor change
jhonabreul Jan 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 20 additions & 26 deletions Lean.DataSource.DerivativeUniverseGenerator/ChainSymbolProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,13 +93,13 @@ public Dictionary<Symbol, List<Symbol>> GetSymbols()

return result;
}
/// <summary>

/// <summary>
/// Gets the zip file names for the canonical symbols where the contracts or universe constituents will be read from.
/// </summary>
protected virtual IEnumerable<string> GetZipFileNames(DateTime date, Resolution resolution, TickType tickType)
protected virtual IEnumerable<string> GetZipFileNames(DateTime date, Resolution resolution)
{
var tickTypeLower = tickType.TickTypeToLower();
var tickTypesLower = _symbolsDataTickTypes.Select(tickType => tickType.TickTypeToLower()).ToArray();

if (resolution == Resolution.Minute)
{
Expand All @@ -112,8 +112,11 @@ protected virtual IEnumerable<string> GetZipFileNames(DateTime date, Resolution
var optionStyleLower = _securityType.DefaultOptionStyle().OptionStyleToLower();

return directories
.Select(directory => Path.Combine(directory, $"{dateStr}_{tickTypeLower}_{optionStyleLower}.zip"))
.Where(fileName => File.Exists(fileName));
.Select(directory => tickTypesLower
.Select(tickTypeLower => Path.Combine(directory, $"{dateStr}_{tickTypeLower}_{optionStyleLower}.zip"))
.Where(fileName => File.Exists(fileName))
.FirstOrDefault())
.Where(fileName => fileName != null);
}
// Support for resolutions higher than minute, just for Lean local repo data generation
else
Expand All @@ -125,12 +128,20 @@ protected virtual IEnumerable<string> GetZipFileNames(DateTime date, Resolution
Path.Combine(_dataSourceFolder, resolution.ResolutionToLower()),
$"*{dateStr}*.zip",
SearchOption.AllDirectories)
.Where(fileName =>
.Select(fileName =>
{
var fileInfo = new FileInfo(fileName);
var fileNameParts = fileInfo.Name.Split('_');
return fileNameParts.Length == 4 && fileNameParts[1] == dateStr && fileNameParts[2] == tickTypeLower;
});
var tickTypeIndex = Array.IndexOf(tickTypesLower, fileNameParts[2]);

return (fileName, directoryName: fileInfo.DirectoryName, tickTypeIndex);
})
// Get only supported tick type data
.Where(tuple => tuple.tickTypeIndex > -1)
// For each contract get the first matching tick type file
.OrderBy(tuple => tuple.tickTypeIndex)
.GroupBy(tuple => tuple.directoryName)
.Select(group => group.First().fileName);
}
catch (DirectoryNotFoundException)
{
Expand All @@ -139,23 +150,6 @@ protected virtual IEnumerable<string> GetZipFileNames(DateTime date, Resolution
}
}

/// <summary>
/// Gets the zip file names for the canonical symbols where the contracts or universe constituents will be read from.
/// </summary>
private IEnumerable<string> GetZipFileNames(DateTime date, Resolution resolution)
{
foreach (var tickType in _symbolsDataTickTypes)
{
var fileNames = GetZipFileNames(date, resolution, tickType).ToList();
if (fileNames.Count > 0)
{
return fileNames;
}
}

return Enumerable.Empty<string>();
}

/// <summary>
/// Reads the symbols from the zip entry names for the given canonical symbol.
/// </summary>
Expand Down
29 changes: 20 additions & 9 deletions Lean.DataSource.FuturesUniverseGenerator/FutureChainSymbolProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,31 +36,42 @@ public FutureChainSymbolProvider(IDataCacheProvider dataCacheProvider, DateTime
{
}

protected override IEnumerable<string> GetZipFileNames(DateTime date, Resolution resolution, TickType tickType)
protected override IEnumerable<string> GetZipFileNames(DateTime date, Resolution resolution)
{
var tickTypeLower = tickType.TickTypeToLower();
var tickTypesLower = _symbolsDataTickTypes.Select(tickType => tickType.TickTypeToLower()).ToArray();

if (resolution == Resolution.Minute)
{
var basePath = Path.Combine(_dataSourceFolder, resolution.ResolutionToLower());
var dateStr = date.ToString("yyyyMMdd");

return Directory.EnumerateDirectories(basePath, "*", new EnumerationOptions() { RecurseSubdirectories = true, MaxRecursionDepth = 1 })
.Select(directory => Path.Combine(directory, $"{dateStr}_{tickTypeLower}.zip"))
.Where(fileName => File.Exists(fileName));
return Directory.EnumerateDirectories(basePath)
.Select(directory => tickTypesLower
.Select(tickTypeLower => Path.Combine(directory, $"{dateStr}_{tickTypeLower}.zip"))
.Where(fileName => File.Exists(fileName))
.FirstOrDefault())
.Where(fileName => fileName != null);
}
// Support for resolutions higher than minute, just for Lean local repo data generation
else
{
try
{
return Directory.EnumerateFiles(Path.Combine(_dataSourceFolder, resolution.ResolutionToLower()), $"*_{tickTypeLower}.zip")
.Where(fileName =>
return Directory.EnumerateFiles(Path.Combine(_dataSourceFolder, resolution.ResolutionToLower()), $"*.zip")
.Select(fileName =>
{
var fileInfo = new FileInfo(fileName);
var fileNameParts = Path.GetFileNameWithoutExtension(fileInfo.Name).Split('_');
return fileNameParts.Length == 2 && fileNameParts[1] == tickTypeLower;
});
var tickTypeIndex = Array.IndexOf(tickTypesLower, fileNameParts[1]);

return (fileName, ticker: fileNameParts[0], tickTypeIndex);
})
// Get only supported tick type data
.Where(tuple => tuple.tickTypeIndex > -1)
// For each ticker get the first matching tick type file
.OrderBy(tuple => tuple.tickTypeIndex)
.GroupBy(tuple => tuple.ticker)
.Select(group => group.First().fileName);
}
catch (DirectoryNotFoundException)
{
Expand Down
Loading