From f85ccac218e0a5a8a97d6175191b606a93b91d87 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Fri, 19 Jul 2024 13:11:45 +0800 Subject: [PATCH 1/9] Add addional field interface and generation --- .../AdditionalFieldGenerator.cs | 82 +++++++++++++++++++ .../IAdditionalFields.cs | 33 ++++++++ 2 files changed, 115 insertions(+) create mode 100644 Lean.DataSource.DerivativeUniverseGenerator/AdditionalFieldGenerator.cs create mode 100644 Lean.DataSource.DerivativeUniverseGenerator/IAdditionalFields.cs diff --git a/Lean.DataSource.DerivativeUniverseGenerator/AdditionalFieldGenerator.cs b/Lean.DataSource.DerivativeUniverseGenerator/AdditionalFieldGenerator.cs new file mode 100644 index 0000000..92eb6ef --- /dev/null +++ b/Lean.DataSource.DerivativeUniverseGenerator/AdditionalFieldGenerator.cs @@ -0,0 +1,82 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2024 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Logging; +using System; +using System.IO; +using System.Linq; + +namespace QuantConnect.DataSource.DerivativeUniverseGenerator +{ + /// + /// Generate additional fields that needed to calculate from the whoel derivative chain + /// + public class AdditionalFieldGenerator + { + protected readonly DateTime _processingDate; + protected readonly string _rootPath; + + /// + /// Instantiate a new instance of + /// + /// + /// + public AdditionalFieldGenerator(DateTime processingDate, string rootPath) + { + _processingDate = processingDate; + _rootPath = rootPath; + } + + /// + /// Run the additional fields generation + /// + /// If the generator run successfully + public virtual bool Run() + { + throw new NotImplementedException("AdditionalFieldGenerator.Run(): Run method must be implemented."); + } + + /// + /// Write the additional fields to the Csv file being generated + /// + /// Target csv file path + /// The addtional field content + protected virtual void WriteToCsv(string csvPath, IAdditionalFields additionalFields) + { + if (string.IsNullOrWhiteSpace(csvPath)) + { + Log.Error("AdditionalFieldGenerator.WriteToCsv(): invalid file path provided"); + return; + } + + var csv = File.ReadAllLines(csvPath) + .Where(s => !string.IsNullOrWhiteSpace(s)) + .ToList(); + for (int i = 0; i < csv.Count; i++) + { + if (i == 0) + { + csv[i] += $",{additionalFields.GetHeader()}"; + } + else + { + csv[i] += $",{additionalFields.ToCsv()}"; + } + } + + File.WriteAllLines(csvPath, csv); + } + } +} diff --git a/Lean.DataSource.DerivativeUniverseGenerator/IAdditionalFields.cs b/Lean.DataSource.DerivativeUniverseGenerator/IAdditionalFields.cs new file mode 100644 index 0000000..5fe23b7 --- /dev/null +++ b/Lean.DataSource.DerivativeUniverseGenerator/IAdditionalFields.cs @@ -0,0 +1,33 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2024 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.DataSource.DerivativeUniverseGenerator +{ + /// + /// Interface of additional fields which requires the whole derivative chain to calculate from + /// + public interface IAdditionalFields + { + /// + /// Convert the entry to a CSV string. + /// + string ToCsv(); + + /// + /// Gets the header of the CSV file + /// + string GetHeader(); + } +} From f50c935efbf5195615e7b54d711182ffd0a8a9e9 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Fri, 19 Jul 2024 13:12:02 +0800 Subject: [PATCH 2/9] add option iv rank and percentile as additional field --- .../OptionAdditionalFieldGenerator.cs | 125 ++++++++++++++++++ .../OptionAdditionalFields.cs | 92 +++++++++++++ 2 files changed, 217 insertions(+) create mode 100644 Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs create mode 100644 Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFields.cs diff --git a/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs new file mode 100644 index 0000000..72b466a --- /dev/null +++ b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs @@ -0,0 +1,125 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2024 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Logging; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; + +namespace QuantConnect.DataSource.OptionsUniverseGenerator +{ + public class OptionAdditionalFieldGenerator : DerivativeUniverseGenerator.AdditionalFieldGenerator + { + private const string _impliedVolHeader = "implied_volatility"; + private const string _deltaHeader = "delta"; + private const string _sidHeader = "#symbol_id"; + private const string _tickerHeader = "symbol_value"; + + public OptionAdditionalFieldGenerator(DateTime processingDate, string rootPath) + : base(processingDate, rootPath) + { + } + + public override bool Run() + { + Log.Trace($"OptionAdditionalFieldGenerator.Run(): Processing additional fields for date {_processingDate:yyyy-MM-dd}"); + + // per symbol + try + { + foreach (var subFolder in Directory.GetDirectories(_rootPath)) + { + var dateFile = Path.Combine(subFolder, $"{_processingDate:yyyyMMdd}.csv"); + var symbol = subFolder.Split(Path.DirectorySeparatorChar)[^1].ToUpper(); + if (!File.Exists(dateFile)) + { + Log.Error($"OptionAdditionalFieldGenerator.Run(): no universe file found for {symbol} in {_processingDate:yyyy-MM-dd}"); + return false; + } + + var ivs = GetIvs(_processingDate, subFolder); + var additionalFields = new OptionAdditionalFields(); + additionalFields.Update(ivs); + + WriteToCsv(dateFile, additionalFields); + } + } + catch (Exception ex) + { + Log.Error(ex, + $"OptionAdditionalFieldGenerator.Run(): Error processing addtional fields for date {_processingDate:yyyy-MM-dd}"); + return false; + } + return true; + } + + private List GetIvs(DateTime currentDateTime, string path) + { + // get i-year ATM IVs to calculate IV rank and percentile + var lastYearFiles = Directory.EnumerateFiles(path, "*.csv") + .AsParallel() + .Where(file => DateTime.TryParseExact(Path.GetFileNameWithoutExtension(file), "yyyyMMdd", + CultureInfo.InvariantCulture, DateTimeStyles.None, out var fileDate) + && fileDate > currentDateTime.AddYears(-1) + && fileDate <= currentDateTime) + .OrderBy(file => file) + .ToList(); + + return lastYearFiles.Select(csvFile => GetAtmIv(currentDateTime, csvFile)) + .ToList(); + } + + private decimal GetAtmIv(DateTime currentDateTime, string csvPath) + { + var lines = File.ReadAllLines(csvPath) + .Where(s => !string.IsNullOrWhiteSpace(s)) + .ToList(); + var headers = lines[0].Split(','); + var deltaIndex = Array.IndexOf(headers, _deltaHeader); + var ivIndex = Array.IndexOf(headers, _impliedVolHeader); + var sidIndex = Array.IndexOf(headers, _sidHeader); + var tickerIndex = Array.IndexOf(headers, _tickerHeader); + + if (deltaIndex == -1 || ivIndex == -1 || sidIndex == -1 || tickerIndex == -1) + { + return -1m; + } + + var filtered = lines.Skip(2) + .Select(line => + { + var values = line.Split(','); + var symbol = new Symbol(SecurityIdentifier.Parse(values[sidIndex]), values[tickerIndex]); + var delta = decimal.Parse(values[deltaIndex]); + var iv = decimal.Parse(values[ivIndex]); + return (Expiry: symbol.ID.Date, Delta: delta, ImpliedVolatility: iv); + }) + .Where(x => x.Expiry >= currentDateTime.AddDays(30)) + .ToList(); + if (filtered.Count == 0) + { + return -1m; + } + + // Skip underlying row + return filtered.OrderBy(x => x.Expiry) + .ThenBy(x => Math.Abs(x.Delta - 0.5m)) + .First() + .ImpliedVolatility; + } + } +} diff --git a/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFields.cs b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFields.cs new file mode 100644 index 0000000..75bf165 --- /dev/null +++ b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFields.cs @@ -0,0 +1,92 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2024 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace QuantConnect.DataSource.OptionsUniverseGenerator +{ + /// + /// Option additional fields from the daily option universe file + /// + public class OptionAdditionalFields : DerivativeUniverseGenerator.IAdditionalFields + { + /// + /// Implied Volatility Rank + /// + /// The relative volatility over the past year + public decimal? IvRank { get; set; } = null; + + /// + /// Implied Volatility Percentile + /// + /// The ratio of the current implied volatility baing higher than that over the past year + public decimal? IvPercentile { get; set; } = null; + + /// + /// Update the additional fields + /// + /// List of past year's ATM implied volatilities + public void Update(List ivs) + { + IvRank = CalculateIvRank(ivs); + IvPercentile = CalculateIvPercentile(ivs); + } + + /// + /// Convert the entry to a CSV string. + /// + public string GetHeader() + { + return "iv_rank,iv_percentile"; + } + + /// + /// Gets the header of the CSV file + /// + public string ToCsv() + { + return $"{WriteNullableField(IvRank)},{WriteNullableField(IvPercentile)}"; + } + + // source: https://www.tastylive.com/concepts-strategies/implied-volatility-rank-percentile + private decimal? CalculateIvRank(List ivs) + { + if (ivs.Count < 2) + { + return null; + } + var oneYearLow = ivs.Min(); + return (ivs[^1] - oneYearLow) / (ivs.Max() - oneYearLow); + } + + // source: https://www.tastylive.com/concepts-strategies/implied-volatility-rank-percentile + private decimal? CalculateIvPercentile(List ivs) + { + if (ivs.Count < 2) + { + return null; + } + var daysBelowCurrentIv = ivs.Count(x => x < ivs[^1]); + return Convert.ToDecimal(daysBelowCurrentIv) / Convert.ToDecimal(ivs.Count); + } + + private string WriteNullableField(decimal? field) + { + return field.HasValue ? field.Value.ToString() : string.Empty; + } + } +} From 82aa0461c857f3b8c9c3c5511f7c4705cca55b0c Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Fri, 19 Jul 2024 13:33:14 +0800 Subject: [PATCH 3/9] Fix bugs --- .../ChainSymbolProvider.cs | 9 +-------- .../DerivativeUniverseGenerator.cs | 4 ++++ .../Program.cs | 20 +++++++++++++++++++ .../Program.cs | 6 ++++++ 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/Lean.DataSource.DerivativeUniverseGenerator/ChainSymbolProvider.cs b/Lean.DataSource.DerivativeUniverseGenerator/ChainSymbolProvider.cs index 3ad993f..9c873de 100644 --- a/Lean.DataSource.DerivativeUniverseGenerator/ChainSymbolProvider.cs +++ b/Lean.DataSource.DerivativeUniverseGenerator/ChainSymbolProvider.cs @@ -112,19 +112,12 @@ private IEnumerable GetZipFileNames(DateTime date, Resolution resolution else { var dateStr = date.ToString("yyyy"); - return Directory.EnumerateFiles(Path.Combine(_dataSourceFolder, resolution.ResolutionToLower()), $"{dateStr}*.zip", SearchOption.AllDirectories) + return Directory.EnumerateFiles(Path.Combine(_dataSourceFolder, resolution.ResolutionToLower()), $"*{dateStr}*.zip", SearchOption.AllDirectories) .Where(fileName => { var fileInfo = new FileInfo(fileName); var fileNameParts = fileInfo.Name.Split('_'); - if (resolution == Resolution.Minute) - { - return fileNameParts.Length == 3 && - fileNameParts[0] == dateStr && - fileNameParts[1] == tickTypeLower; - } - return fileNameParts.Length == 4 && fileNameParts[1] == dateStr && fileNameParts[2] == tickTypeLower; diff --git a/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs b/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs index 0d28637..04bc0ac 100644 --- a/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs +++ b/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs @@ -77,6 +77,10 @@ public DerivativeUniverseGenerator(DateTime processingDate, SecurityType securit _outputFolderRoot = outputFolderRoot; _universesOutputFolderRoot = Path.Combine(_outputFolderRoot, _securityType.SecurityTypeToLower(), _market, "universes"); + if (!Directory.Exists(_universesOutputFolderRoot)) + { + Directory.CreateDirectory(_universesOutputFolderRoot); + } _dataProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("data-provider", "DefaultDataProvider")); diff --git a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs index 9db336c..9b7828b 100644 --- a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs +++ b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs @@ -20,6 +20,7 @@ using QuantConnect.Configuration; using QuantConnect.Logging; +using System.IO; namespace QuantConnect.DataSource.DerivativeUniverseGenerator { @@ -73,6 +74,23 @@ protected virtual void MainImpl(string[] args) Environment.Exit(1); } + var universeOutputPath = Path.Combine(outputFolderRoot, securityType.SecurityTypeToLower(), market, "universes"); + var optionsAdditionalFieldGenerator = GetAdditionalFieldGenerator(processingDate, universeOutputPath); + + try + { + if (!optionsAdditionalFieldGenerator.Run()) + { + Log.Error($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Failed to generate additional fields."); + Environment.Exit(1); + } + } + catch (Exception ex) + { + Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating additional fields."); + Environment.Exit(1); + } + Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): DONE in {timer.Elapsed:g}"); Environment.Exit(0); @@ -81,6 +99,8 @@ protected virtual void MainImpl(string[] args) protected abstract DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, string dataFolderRoot, string outputFolderRoot, DateTime processingDate); + protected abstract AdditionalFieldGenerator GetAdditionalFieldGenerator(DateTime processingDate, string outputFolderRoot); + /// /// Validate and extract command line args and configuration options. /// diff --git a/Lean.DataSource.OptionsUniverseGenerator/Program.cs b/Lean.DataSource.OptionsUniverseGenerator/Program.cs index fbd34df..b565f63 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/Program.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/Program.cs @@ -39,5 +39,11 @@ protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUn { return new OptionsUniverseGenerator(processingDate, securityType, market, dataFolderRoot, outputFolderRoot); } + + protected override DerivativeUniverseGenerator.AdditionalFieldGenerator GetAdditionalFieldGenerator(DateTime processingDate, + string outputFolderRoot) + { + return new OptionAdditionalFieldGenerator(processingDate, outputFolderRoot); + } } } From 7a3a77c3f9fe6858d0748ae5a29978d66e21d334 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Sun, 21 Jul 2024 22:12:59 +0800 Subject: [PATCH 4/9] iv30 --- .../OptionAdditionalFieldGenerator.cs | 24 +++++++++++++++---- .../OptionAdditionalFields.cs | 11 +++++++-- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs index 72b466a..9e3c394 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs @@ -99,6 +99,7 @@ private decimal GetAtmIv(DateTime currentDateTime, string csvPath) return -1m; } + // Skip underlying row var filtered = lines.Skip(2) .Select(line => { @@ -108,18 +109,33 @@ private decimal GetAtmIv(DateTime currentDateTime, string csvPath) var iv = decimal.Parse(values[ivIndex]); return (Expiry: symbol.ID.Date, Delta: delta, ImpliedVolatility: iv); }) - .Where(x => x.Expiry >= currentDateTime.AddDays(30)) + .Where(x => x.ImpliedVolatility != 0m) .ToList(); if (filtered.Count == 0) { return -1m; } - // Skip underlying row - return filtered.OrderBy(x => x.Expiry) - .ThenBy(x => Math.Abs(x.Delta - 0.5m)) + var expiries = filtered.Select(x => x.Expiry).ToList().Distinct(); + var day30 = currentDateTime.AddDays(30); + var nearExpiry = expiries.Where(x => x <= day30).Max(); + var farExpiry = expiries.Where(x => x >= day30).Min(); + + var nearIv = filtered.Where(x => x.Expiry == nearExpiry) + .OrderBy(x => Math.Abs(x.Delta - 0.5m)) + .First() + .ImpliedVolatility; + if (nearExpiry == farExpiry) + { + return nearIv; + } + var farIv = filtered.Where(x => x.Expiry == farExpiry) + .OrderBy(x => Math.Abs(x.Delta - 0.5m)) .First() .ImpliedVolatility; + // Linear interpolation + return (nearIv * Convert.ToDecimal((farExpiry - day30).TotalDays) + farIv * Convert.ToDecimal((day30 - nearExpiry).TotalDays)) + / Convert.ToDecimal((farExpiry - nearExpiry).TotalDays); } } } diff --git a/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFields.cs b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFields.cs index 75bf165..30fb4fb 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFields.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFields.cs @@ -24,6 +24,12 @@ namespace QuantConnect.DataSource.OptionsUniverseGenerator /// public class OptionAdditionalFields : DerivativeUniverseGenerator.IAdditionalFields { + /// + /// Expected 30 Days Implied Volatility + /// + /// Linearly interpolated by bracket method + public decimal? Iv30 { get; set; } = null; + /// /// Implied Volatility Rank /// @@ -42,6 +48,7 @@ public class OptionAdditionalFields : DerivativeUniverseGenerator.IAdditionalFie /// List of past year's ATM implied volatilities public void Update(List ivs) { + Iv30 = ivs[^1]; IvRank = CalculateIvRank(ivs); IvPercentile = CalculateIvPercentile(ivs); } @@ -51,7 +58,7 @@ public void Update(List ivs) /// public string GetHeader() { - return "iv_rank,iv_percentile"; + return "iv_30,iv_rank,iv_percentile"; } /// @@ -59,7 +66,7 @@ public string GetHeader() /// public string ToCsv() { - return $"{WriteNullableField(IvRank)},{WriteNullableField(IvPercentile)}"; + return $"{WriteNullableField(Iv30)},{WriteNullableField(IvRank)},{WriteNullableField(IvPercentile)}"; } // source: https://www.tastylive.com/concepts-strategies/implied-volatility-rank-percentile From e5968cf5ce756225256c1280a7919f9576801a24 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Mon, 22 Jul 2024 20:37:00 +0800 Subject: [PATCH 5/9] fix bug on date logic --- .../OptionAdditionalFieldGenerator.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs index 9e3c394..36abe81 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs @@ -79,11 +79,11 @@ private List GetIvs(DateTime currentDateTime, string path) .OrderBy(file => file) .ToList(); - return lastYearFiles.Select(csvFile => GetAtmIv(currentDateTime, csvFile)) + return lastYearFiles.Select(csvFile => GetAtmIv(csvFile)) .ToList(); } - private decimal GetAtmIv(DateTime currentDateTime, string csvPath) + private decimal GetAtmIv(string csvPath) { var lines = File.ReadAllLines(csvPath) .Where(s => !string.IsNullOrWhiteSpace(s)) @@ -117,6 +117,8 @@ private decimal GetAtmIv(DateTime currentDateTime, string csvPath) } var expiries = filtered.Select(x => x.Expiry).ToList().Distinct(); + var currentDateTime = DateTime.ParseExact(Path.GetFileNameWithoutExtension(csvPath), "yyyyMMdd", + CultureInfo.InvariantCulture, DateTimeStyles.None); var day30 = currentDateTime.AddDays(30); var nearExpiry = expiries.Where(x => x <= day30).Max(); var farExpiry = expiries.Where(x => x >= day30).Min(); From 4479660d386869d2209ef3ffadbad1133e7e5a92 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Mon, 22 Jul 2024 21:18:25 +0800 Subject: [PATCH 6/9] add additional field test comparisons --- .../TestData/generated_samples.csv | 21 ++++++++ .../TestData/test_comparison.py | 48 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestData/generated_samples.csv create mode 100644 QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestData/test_comparison.py diff --git a/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestData/generated_samples.csv b/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestData/generated_samples.csv new file mode 100644 index 0000000..9009f85 --- /dev/null +++ b/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestData/generated_samples.csv @@ -0,0 +1,21 @@ +,iv_30,iv_rank,iv_percentile,test_iv_rank,test_iv_percentile +2019-01-05,0.4033288714285714,0.7602486820250108,0.9641434262948207,76.02486820250108,96.42857142857143 +2019-01-08,0.3413325714285714,0.5940866764233323,0.8764940239043824,59.408667642333235,87.6984126984127 +2019-01-09,0.4053031714285714,0.7655401856737672,0.9641434262948207,76.55401856737673,96.42857142857143 +2019-01-10,0.3356759,0.5789257092430145,0.8605577689243027,57.89257092430147,86.11111111111111 +2019-01-11,0.3325976714285714,0.5706754647424723,0.8565737051792828,57.06754647424723,85.71428571428571 +2019-01-12,0.3398087,0.590002407961335,0.8645418326693227,59.00024079613352,86.5079365079365 +2019-01-15,0.3296392428571428,0.562746307278916,0.848605577689243,56.2746307278916,84.92063492063492 +2019-01-16,0.3535012571428571,0.6267010941195853,0.8928571428571428,62.67010941195852,89.28571428571429 +2019-01-17,0.3306122,0.5653540195169305,0.8452380952380952,56.535401951693075,84.52380952380952 +2019-01-18,0.3346532142857142,0.5761847148709971,0.8492063492063492,57.61847148709971,84.92063492063492 +2019-01-19,0.2940956428571428,0.4674826244090702,0.7579365079365079,46.74826244090702,75.79365079365078 +2019-01-23,0.2845410428571428,0.4418744591036923,0.7091633466135459,44.18744591036924,71.03174603174604 +2019-01-24,0.3095117,0.5088006219883972,0.7928286852589641,50.880062198839724,79.36507936507937 +2019-01-25,0.3287171142857142,0.5602748253864291,0.8286852589641434,56.0274825386429,82.93650793650794 +2019-01-26,0.3125058285714285,0.5168254623168983,0.8047808764940239,51.68254623168983,80.55555555555556 +2019-01-29,0.2870498428571428,0.448598525534326,0.7131474103585657,44.859852553432596,71.42857142857143 +2019-01-30,0.3180328,0.5316388085565062,0.8127490039840637,53.16388085565064,81.34920634920636 +2019-01-31,0.3599641,0.6440227557515107,0.904382470119522,64.4022755751511,90.47619047619048 +2019-02-01,0.2263559428571428,0.2859272023781422,0.5059760956175298,28.592720237814223,50.39682539682539 +2019-02-02,0.2361797857142857,0.3122569903148188,0.545816733067729,31.225699031481884,54.36507936507936 diff --git a/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestData/test_comparison.py b/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestData/test_comparison.py new file mode 100644 index 0000000..4f0b07a --- /dev/null +++ b/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestData/test_comparison.py @@ -0,0 +1,48 @@ +from datetime import datetime, timedelta +import pandas as pd +from pathlib import Path + +OUTPUT_PATH = "/temp-output-directory/option/usa/universes/aapl" + +df = pd.DataFrame() +for i, file in enumerate(Path.glob(Path.cwd(), "*.csv")): + if i == 0: + continue + try: + df_daily = pd.read_csv(file, usecols=[14, 15, 16]) + series = df_daily.iloc[-1] + daily_entry = series.to_frame().T + daily_entry.index = [datetime.strptime(file.stem, "%Y%m%d") + timedelta(1)] + df = pd.concat([df, daily_entry]) + except: + pass + +### Test comparison generation +### Adapted from https://s3.amazonaws.com/tastytradepublicmedia/website/cms/SKINNY_ivr_and_ivp.txt/original/SKINNY_ivr_and_ivp.txt?_sp=50d826c5-0948-4c48-9851-02767cd310a9.1721559732707 +### Michael Rechenthin, Ph.D., tastytrade/dough Research Team +# Extract IV column +history = df[["iv_30"]] + +# calculate the IV rank +# --------------------------- +# calculate the IV rank +low_over_timespan = history.rolling(252).min() +high_over_timespan = history.rolling(252).max() +iv_rank = (history - low_over_timespan) / (high_over_timespan - low_over_timespan) * 100 + +# calculate the IV percentile +# --------------------------- +# how many times over the past year, has IV been below the current IV +def below(df1): + count = 0 + for i in range(df1.shape[0]): + if df1[-1] > df1[i]: + count += 1 + return count +count_below = history.rolling(252).apply(below) +iv_percentile = count_below / 252 * 100 + +df = pd.concat([df, iv_rank, iv_percentile], axis=1) +df.columns = list(df.columns)[:3] + ["test_iv_rank", "test_iv_percentile"] + +df.dropna().to_csv("generated_samples.csv") \ No newline at end of file From 6734f6599da81fdc8db93fc1fb00a5c1696d0689 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Mon, 22 Jul 2024 21:32:54 +0800 Subject: [PATCH 7/9] optimization --- .../OptionAdditionalFieldGenerator.cs | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs index 36abe81..dd1d729 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs @@ -28,6 +28,7 @@ public class OptionAdditionalFieldGenerator : DerivativeUniverseGenerator.Additi private const string _deltaHeader = "delta"; private const string _sidHeader = "#symbol_id"; private const string _tickerHeader = "symbol_value"; + private Dictionary> _iv30s = new(); public OptionAdditionalFieldGenerator(DateTime processingDate, string rootPath) : base(processingDate, rootPath) @@ -43,6 +44,7 @@ public override bool Run() { foreach (var subFolder in Directory.GetDirectories(_rootPath)) { + _iv30s[subFolder] = new(); var dateFile = Path.Combine(subFolder, $"{_processingDate:yyyyMMdd}.csv"); var symbol = subFolder.Split(Path.DirectorySeparatorChar)[^1].ToUpper(); if (!File.Exists(dateFile)) @@ -72,14 +74,22 @@ private List GetIvs(DateTime currentDateTime, string path) // get i-year ATM IVs to calculate IV rank and percentile var lastYearFiles = Directory.EnumerateFiles(path, "*.csv") .AsParallel() - .Where(file => DateTime.TryParseExact(Path.GetFileNameWithoutExtension(file), "yyyyMMdd", + .Where(file => DateTime.TryParseExact(Path.GetFileNameWithoutExtension(file), "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var fileDate) && fileDate > currentDateTime.AddYears(-1) - && fileDate <= currentDateTime) - .OrderBy(file => file) - .ToList(); + && fileDate <= currentDateTime + && !_iv30s[path].ContainsKey(fileDate)) + .ToDictionary( + file => DateTime.ParseExact(Path.GetFileNameWithoutExtension(file), "yyyyMMdd", + CultureInfo.InvariantCulture, DateTimeStyles.None), + file => GetAtmIv(file) + ); + _iv30s[path] = _iv30s[path].Concat(lastYearFiles) + .ToDictionary(x => x.Key, x => x.Value); - return lastYearFiles.Select(csvFile => GetAtmIv(csvFile)) + return _iv30s[path].Where(x => x.Key > currentDateTime.AddYears(-1) && x.Key <= currentDateTime) + .OrderBy(x => x.Key) + .Select(x => x.Value) .ToList(); } From 45520095c796b8d5773a38c55aa579385f9d5aeb Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Tue, 24 Jun 2025 20:19:06 +0800 Subject: [PATCH 8/9] fix builds --- .../Program.cs | 25 ++++++++++--------- .../Program.cs | 5 ++++ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs index ced7487..55310d3 100644 --- a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs +++ b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs @@ -17,6 +17,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Globalization; +using System.IO; using System.Linq; using Newtonsoft.Json; using QuantConnect.Configuration; @@ -102,24 +103,24 @@ protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null) Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating universe."); Environment.Exit(1); } - } - var universeOutputPath = Path.Combine(outputFolderRoot, securityType.SecurityTypeToLower(), market, "universes"); - var optionsAdditionalFieldGenerator = GetAdditionalFieldGenerator(processingDate, universeOutputPath); + var universeOutputPath = Path.Combine(outputFolderRoot, securityType.SecurityTypeToLower(), market, "universes"); + var optionsAdditionalFieldGenerator = GetAdditionalFieldGenerator(processingDate, universeOutputPath); - try - { - if (!optionsAdditionalFieldGenerator.Run()) + try { - Log.Error($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Failed to generate additional fields."); + if (!optionsAdditionalFieldGenerator.Run()) + { + Log.Error($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Failed to generate additional fields."); + Environment.Exit(1); + } + } + catch (Exception ex) + { + Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating additional fields."); Environment.Exit(1); } } - catch (Exception ex) - { - Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating additional fields."); - Environment.Exit(1); - } Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): DONE in {timer.Elapsed:g}"); diff --git a/Lean.DataSource.FuturesUniverseGenerator/Program.cs b/Lean.DataSource.FuturesUniverseGenerator/Program.cs index 320baa6..6516e73 100644 --- a/Lean.DataSource.FuturesUniverseGenerator/Program.cs +++ b/Lean.DataSource.FuturesUniverseGenerator/Program.cs @@ -43,5 +43,10 @@ protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUn return new FuturesUniverseGenerator(processingDate, market, dataFolderRoot, outputFolderRoot, dataProvider, dataCacheProvider, historyProvider); } + + protected override DerivativeUniverseGenerator.AdditionalFieldGenerator GetAdditionalFieldGenerator(DateTime dateTime, string _) + { + return null; + } } } From 8502b3d7ffb7e04832813335ac4324ece353417d Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Thu, 26 Jun 2025 15:26:56 +0800 Subject: [PATCH 9/9] To make independent output for IVRanks --- .../OptionAdditionalFieldGenerator.cs | 54 +++++++++++++++---- .../Program.cs | 16 +++++- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs index dd1d729..c0fe3b2 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs @@ -13,12 +13,15 @@ * limitations under the License. */ -using QuantConnect.Logging; using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; +using QuantConnect.Data.Auxiliary; +using QuantConnect.Interfaces; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Logging; namespace QuantConnect.DataSource.OptionsUniverseGenerator { @@ -28,21 +31,28 @@ public class OptionAdditionalFieldGenerator : DerivativeUniverseGenerator.Additi private const string _deltaHeader = "delta"; private const string _sidHeader = "#symbol_id"; private const string _tickerHeader = "symbol_value"; + private readonly string _outputPath; + private readonly string _addtionalFieldDataPath; private Dictionary> _iv30s = new(); + private static readonly IMapFileProvider _mapFileProvider = new LocalZipMapFileProvider(); - public OptionAdditionalFieldGenerator(DateTime processingDate, string rootPath) + public OptionAdditionalFieldGenerator(DateTime processingDate, string rootPath, string newOutputPath, string addtionalFieldDataPath) : base(processingDate, rootPath) { + _outputPath = newOutputPath; + _addtionalFieldDataPath = addtionalFieldDataPath; + _mapFileProvider.Initialize(new DefaultDataProvider()); } public override bool Run() { Log.Trace($"OptionAdditionalFieldGenerator.Run(): Processing additional fields for date {_processingDate:yyyy-MM-dd}"); + var dataByTicker = new List(); // per symbol - try + foreach (var subFolder in Directory.GetDirectories(_rootPath)) { - foreach (var subFolder in Directory.GetDirectories(_rootPath)) + try { _iv30s[subFolder] = new(); var dateFile = Path.Combine(subFolder, $"{_processingDate:yyyyMMdd}.csv"); @@ -57,14 +67,18 @@ public override bool Run() var additionalFields = new OptionAdditionalFields(); additionalFields.Update(ivs); + var sid = SecurityIdentifier.GenerateEquity(symbol, Market.USA, true, _mapFileProvider, _processingDate); + dataByTicker.Add($"{sid},{symbol},{additionalFields.ToCsv()}"); + WriteToCsv(dateFile, additionalFields); + + SaveContentToFile(_outputPath, _addtionalFieldDataPath, $"{_processingDate:yyyyMMdd}", dataByTicker); + } + catch (Exception ex) + { + Log.Error(ex, + $"OptionAdditionalFieldGenerator.Run(): Error processing addtional fields for date {_processingDate:yyyy-MM-dd}"); } - } - catch (Exception ex) - { - Log.Error(ex, - $"OptionAdditionalFieldGenerator.Run(): Error processing addtional fields for date {_processingDate:yyyy-MM-dd}"); - return false; } return true; } @@ -149,5 +163,25 @@ private decimal GetAtmIv(string csvPath) return (nearIv * Convert.ToDecimal((farExpiry - day30).TotalDays) + farIv * Convert.ToDecimal((day30 - nearExpiry).TotalDays)) / Convert.ToDecimal((farExpiry - nearExpiry).TotalDays); } + + private static void SaveContentToFile(string destinationFolder, string processedFolder, string name, IEnumerable contents) + { + Directory.CreateDirectory(destinationFolder); + name = $"{name.ToLowerInvariant()}.csv"; + var filePath = Path.Combine(processedFolder, name); + + var lines = new HashSet(contents); + if (File.Exists(filePath)) + { + foreach (var line in File.ReadAllLines(filePath)) + { + lines.Add(line); + } + } + + var finalLines = lines.OrderBy(x => x.Split(',').First()).ToList(); + var finalPath = Path.Combine(destinationFolder, name); + File.WriteAllLines(finalPath, finalLines); + } } } diff --git a/Lean.DataSource.OptionsUniverseGenerator/Program.cs b/Lean.DataSource.OptionsUniverseGenerator/Program.cs index 533f994..c3b98c5 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/Program.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/Program.cs @@ -13,10 +13,12 @@ * limitations under the License. */ +using QuantConnect.Configuration; using QuantConnect.Interfaces; using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Lean.Engine.HistoricalData; using System; +using System.IO; namespace QuantConnect.DataSource.OptionsUniverseGenerator { @@ -48,7 +50,19 @@ protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUn protected override DerivativeUniverseGenerator.AdditionalFieldGenerator GetAdditionalFieldGenerator(DateTime processingDate, string outputFolderRoot) { - return new OptionAdditionalFieldGenerator(processingDate, outputFolderRoot); + var addtionalFieldOutputDirectory = Path.Combine( + Config.Get("temp-output-directory", "/temp-output-directory"), + "alternative", + "quantconnect", + "ivrank" + ); + var processedDirectory = Path.Combine( + Config.Get("processed-output-directory", Globals.DataFolder), + "alternative", + "quantconnect", + "ivrank" + ); + return new OptionAdditionalFieldGenerator(processingDate, outputFolderRoot, addtionalFieldOutputDirectory, processedDirectory); } } }