diff --git a/Lean.DataSource.DerivativeUniverseGenerator/AdditionalFieldGenerator.cs b/Lean.DataSource.DerivativeUniverseGenerator/AdditionalFieldGenerator.cs
new file mode 100644
index 0000000..92eb6ef
--- /dev/null
+++ b/Lean.DataSource.DerivativeUniverseGenerator/AdditionalFieldGenerator.cs
@@ -0,0 +1,82 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2024 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Logging;
+using System;
+using System.IO;
+using System.Linq;
+
+namespace QuantConnect.DataSource.DerivativeUniverseGenerator
+{
+ ///
+ /// Generate additional fields that needed to calculate from the whoel derivative chain
+ ///
+ public class AdditionalFieldGenerator
+ {
+ protected readonly DateTime _processingDate;
+ protected readonly string _rootPath;
+
+ ///
+ /// Instantiate a new instance of
+ ///
+ ///
+ ///
+ public AdditionalFieldGenerator(DateTime processingDate, string rootPath)
+ {
+ _processingDate = processingDate;
+ _rootPath = rootPath;
+ }
+
+ ///
+ /// Run the additional fields generation
+ ///
+ /// If the generator run successfully
+ public virtual bool Run()
+ {
+ throw new NotImplementedException("AdditionalFieldGenerator.Run(): Run method must be implemented.");
+ }
+
+ ///
+ /// Write the additional fields to the Csv file being generated
+ ///
+ /// Target csv file path
+ /// The addtional field content
+ protected virtual void WriteToCsv(string csvPath, IAdditionalFields additionalFields)
+ {
+ if (string.IsNullOrWhiteSpace(csvPath))
+ {
+ Log.Error("AdditionalFieldGenerator.WriteToCsv(): invalid file path provided");
+ return;
+ }
+
+ var csv = File.ReadAllLines(csvPath)
+ .Where(s => !string.IsNullOrWhiteSpace(s))
+ .ToList();
+ for (int i = 0; i < csv.Count; i++)
+ {
+ if (i == 0)
+ {
+ csv[i] += $",{additionalFields.GetHeader()}";
+ }
+ else
+ {
+ csv[i] += $",{additionalFields.ToCsv()}";
+ }
+ }
+
+ File.WriteAllLines(csvPath, csv);
+ }
+ }
+}
diff --git a/Lean.DataSource.DerivativeUniverseGenerator/IAdditionalFields.cs b/Lean.DataSource.DerivativeUniverseGenerator/IAdditionalFields.cs
new file mode 100644
index 0000000..5fe23b7
--- /dev/null
+++ b/Lean.DataSource.DerivativeUniverseGenerator/IAdditionalFields.cs
@@ -0,0 +1,33 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2024 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+namespace QuantConnect.DataSource.DerivativeUniverseGenerator
+{
+ ///
+ /// Interface of additional fields which requires the whole derivative chain to calculate from
+ ///
+ public interface IAdditionalFields
+ {
+ ///
+ /// Convert the entry to a CSV string.
+ ///
+ string ToCsv();
+
+ ///
+ /// Gets the header of the CSV file
+ ///
+ string GetHeader();
+ }
+}
diff --git a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs
index 80e3916..55310d3 100644
--- a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs
+++ b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs
@@ -13,11 +13,13 @@
* limitations under the License.
*/
+using System;
+using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
+using System.IO;
using System.Linq;
-using System;
-
+using Newtonsoft.Json;
using QuantConnect.Configuration;
using QuantConnect.Logging;
using QuantConnect.Util;
@@ -25,8 +27,6 @@
using QuantConnect.Interfaces;
using QuantConnect.Lean.Engine.DataFeeds;
using QuantConnect.Lean.Engine.HistoricalData;
-using Newtonsoft.Json;
-using System.Collections.Generic;
namespace QuantConnect.DataSource.DerivativeUniverseGenerator
{
@@ -103,6 +103,23 @@ protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null)
Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating universe.");
Environment.Exit(1);
}
+
+ var universeOutputPath = Path.Combine(outputFolderRoot, securityType.SecurityTypeToLower(), market, "universes");
+ var optionsAdditionalFieldGenerator = GetAdditionalFieldGenerator(processingDate, universeOutputPath);
+
+ try
+ {
+ if (!optionsAdditionalFieldGenerator.Run())
+ {
+ Log.Error($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Failed to generate additional fields.");
+ Environment.Exit(1);
+ }
+ }
+ catch (Exception ex)
+ {
+ Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating additional fields.");
+ Environment.Exit(1);
+ }
}
Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): DONE in {timer.Elapsed:g}");
@@ -114,6 +131,8 @@ protected abstract DerivativeUniverseGenerator GetUniverseGenerator(SecurityType
string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider,
HistoryProviderManager historyProvider);
+ protected abstract AdditionalFieldGenerator GetAdditionalFieldGenerator(DateTime processingDate, string outputFolderRoot);
+
///
/// Validate and extract command line args and configuration options.
///
diff --git a/Lean.DataSource.FuturesUniverseGenerator/Program.cs b/Lean.DataSource.FuturesUniverseGenerator/Program.cs
index 320baa6..6516e73 100644
--- a/Lean.DataSource.FuturesUniverseGenerator/Program.cs
+++ b/Lean.DataSource.FuturesUniverseGenerator/Program.cs
@@ -43,5 +43,10 @@ protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUn
return new FuturesUniverseGenerator(processingDate, market, dataFolderRoot, outputFolderRoot, dataProvider,
dataCacheProvider, historyProvider);
}
+
+ protected override DerivativeUniverseGenerator.AdditionalFieldGenerator GetAdditionalFieldGenerator(DateTime dateTime, string _)
+ {
+ return null;
+ }
}
}
diff --git a/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs
new file mode 100644
index 0000000..c0fe3b2
--- /dev/null
+++ b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs
@@ -0,0 +1,187 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2024 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Interfaces;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Logging;
+
+namespace QuantConnect.DataSource.OptionsUniverseGenerator
+{
+ public class OptionAdditionalFieldGenerator : DerivativeUniverseGenerator.AdditionalFieldGenerator
+ {
+ private const string _impliedVolHeader = "implied_volatility";
+ private const string _deltaHeader = "delta";
+ private const string _sidHeader = "#symbol_id";
+ private const string _tickerHeader = "symbol_value";
+ private readonly string _outputPath;
+ private readonly string _addtionalFieldDataPath;
+ private Dictionary> _iv30s = new();
+ private static readonly IMapFileProvider _mapFileProvider = new LocalZipMapFileProvider();
+
+ public OptionAdditionalFieldGenerator(DateTime processingDate, string rootPath, string newOutputPath, string addtionalFieldDataPath)
+ : base(processingDate, rootPath)
+ {
+ _outputPath = newOutputPath;
+ _addtionalFieldDataPath = addtionalFieldDataPath;
+ _mapFileProvider.Initialize(new DefaultDataProvider());
+ }
+
+ public override bool Run()
+ {
+ Log.Trace($"OptionAdditionalFieldGenerator.Run(): Processing additional fields for date {_processingDate:yyyy-MM-dd}");
+ var dataByTicker = new List();
+
+ // per symbol
+ foreach (var subFolder in Directory.GetDirectories(_rootPath))
+ {
+ try
+ {
+ _iv30s[subFolder] = new();
+ var dateFile = Path.Combine(subFolder, $"{_processingDate:yyyyMMdd}.csv");
+ var symbol = subFolder.Split(Path.DirectorySeparatorChar)[^1].ToUpper();
+ if (!File.Exists(dateFile))
+ {
+ Log.Error($"OptionAdditionalFieldGenerator.Run(): no universe file found for {symbol} in {_processingDate:yyyy-MM-dd}");
+ return false;
+ }
+
+ var ivs = GetIvs(_processingDate, subFolder);
+ var additionalFields = new OptionAdditionalFields();
+ additionalFields.Update(ivs);
+
+ var sid = SecurityIdentifier.GenerateEquity(symbol, Market.USA, true, _mapFileProvider, _processingDate);
+ dataByTicker.Add($"{sid},{symbol},{additionalFields.ToCsv()}");
+
+ WriteToCsv(dateFile, additionalFields);
+
+ SaveContentToFile(_outputPath, _addtionalFieldDataPath, $"{_processingDate:yyyyMMdd}", dataByTicker);
+ }
+ catch (Exception ex)
+ {
+ Log.Error(ex,
+ $"OptionAdditionalFieldGenerator.Run(): Error processing addtional fields for date {_processingDate:yyyy-MM-dd}");
+ }
+ }
+ return true;
+ }
+
+ private List GetIvs(DateTime currentDateTime, string path)
+ {
+ // get i-year ATM IVs to calculate IV rank and percentile
+ var lastYearFiles = Directory.EnumerateFiles(path, "*.csv")
+ .AsParallel()
+ .Where(file => DateTime.TryParseExact(Path.GetFileNameWithoutExtension(file), "yyyyMMdd",
+ CultureInfo.InvariantCulture, DateTimeStyles.None, out var fileDate)
+ && fileDate > currentDateTime.AddYears(-1)
+ && fileDate <= currentDateTime
+ && !_iv30s[path].ContainsKey(fileDate))
+ .ToDictionary(
+ file => DateTime.ParseExact(Path.GetFileNameWithoutExtension(file), "yyyyMMdd",
+ CultureInfo.InvariantCulture, DateTimeStyles.None),
+ file => GetAtmIv(file)
+ );
+ _iv30s[path] = _iv30s[path].Concat(lastYearFiles)
+ .ToDictionary(x => x.Key, x => x.Value);
+
+ return _iv30s[path].Where(x => x.Key > currentDateTime.AddYears(-1) && x.Key <= currentDateTime)
+ .OrderBy(x => x.Key)
+ .Select(x => x.Value)
+ .ToList();
+ }
+
+ private decimal GetAtmIv(string csvPath)
+ {
+ var lines = File.ReadAllLines(csvPath)
+ .Where(s => !string.IsNullOrWhiteSpace(s))
+ .ToList();
+ var headers = lines[0].Split(',');
+ var deltaIndex = Array.IndexOf(headers, _deltaHeader);
+ var ivIndex = Array.IndexOf(headers, _impliedVolHeader);
+ var sidIndex = Array.IndexOf(headers, _sidHeader);
+ var tickerIndex = Array.IndexOf(headers, _tickerHeader);
+
+ if (deltaIndex == -1 || ivIndex == -1 || sidIndex == -1 || tickerIndex == -1)
+ {
+ return -1m;
+ }
+
+ // Skip underlying row
+ var filtered = lines.Skip(2)
+ .Select(line =>
+ {
+ var values = line.Split(',');
+ var symbol = new Symbol(SecurityIdentifier.Parse(values[sidIndex]), values[tickerIndex]);
+ var delta = decimal.Parse(values[deltaIndex]);
+ var iv = decimal.Parse(values[ivIndex]);
+ return (Expiry: symbol.ID.Date, Delta: delta, ImpliedVolatility: iv);
+ })
+ .Where(x => x.ImpliedVolatility != 0m)
+ .ToList();
+ if (filtered.Count == 0)
+ {
+ return -1m;
+ }
+
+ var expiries = filtered.Select(x => x.Expiry).ToList().Distinct();
+ var currentDateTime = DateTime.ParseExact(Path.GetFileNameWithoutExtension(csvPath), "yyyyMMdd",
+ CultureInfo.InvariantCulture, DateTimeStyles.None);
+ var day30 = currentDateTime.AddDays(30);
+ var nearExpiry = expiries.Where(x => x <= day30).Max();
+ var farExpiry = expiries.Where(x => x >= day30).Min();
+
+ var nearIv = filtered.Where(x => x.Expiry == nearExpiry)
+ .OrderBy(x => Math.Abs(x.Delta - 0.5m))
+ .First()
+ .ImpliedVolatility;
+ if (nearExpiry == farExpiry)
+ {
+ return nearIv;
+ }
+ var farIv = filtered.Where(x => x.Expiry == farExpiry)
+ .OrderBy(x => Math.Abs(x.Delta - 0.5m))
+ .First()
+ .ImpliedVolatility;
+ // Linear interpolation
+ return (nearIv * Convert.ToDecimal((farExpiry - day30).TotalDays) + farIv * Convert.ToDecimal((day30 - nearExpiry).TotalDays))
+ / Convert.ToDecimal((farExpiry - nearExpiry).TotalDays);
+ }
+
+ private static void SaveContentToFile(string destinationFolder, string processedFolder, string name, IEnumerable contents)
+ {
+ Directory.CreateDirectory(destinationFolder);
+ name = $"{name.ToLowerInvariant()}.csv";
+ var filePath = Path.Combine(processedFolder, name);
+
+ var lines = new HashSet(contents);
+ if (File.Exists(filePath))
+ {
+ foreach (var line in File.ReadAllLines(filePath))
+ {
+ lines.Add(line);
+ }
+ }
+
+ var finalLines = lines.OrderBy(x => x.Split(',').First()).ToList();
+ var finalPath = Path.Combine(destinationFolder, name);
+ File.WriteAllLines(finalPath, finalLines);
+ }
+ }
+}
diff --git a/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFields.cs b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFields.cs
new file mode 100644
index 0000000..30fb4fb
--- /dev/null
+++ b/Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFields.cs
@@ -0,0 +1,99 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2024 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace QuantConnect.DataSource.OptionsUniverseGenerator
+{
+ ///
+ /// Option additional fields from the daily option universe file
+ ///
+ public class OptionAdditionalFields : DerivativeUniverseGenerator.IAdditionalFields
+ {
+ ///
+ /// Expected 30 Days Implied Volatility
+ ///
+ /// Linearly interpolated by bracket method
+ public decimal? Iv30 { get; set; } = null;
+
+ ///
+ /// Implied Volatility Rank
+ ///
+ /// The relative volatility over the past year
+ public decimal? IvRank { get; set; } = null;
+
+ ///
+ /// Implied Volatility Percentile
+ ///
+ /// The ratio of the current implied volatility baing higher than that over the past year
+ public decimal? IvPercentile { get; set; } = null;
+
+ ///
+ /// Update the additional fields
+ ///
+ /// List of past year's ATM implied volatilities
+ public void Update(List ivs)
+ {
+ Iv30 = ivs[^1];
+ IvRank = CalculateIvRank(ivs);
+ IvPercentile = CalculateIvPercentile(ivs);
+ }
+
+ ///
+ /// Convert the entry to a CSV string.
+ ///
+ public string GetHeader()
+ {
+ return "iv_30,iv_rank,iv_percentile";
+ }
+
+ ///
+ /// Gets the header of the CSV file
+ ///
+ public string ToCsv()
+ {
+ return $"{WriteNullableField(Iv30)},{WriteNullableField(IvRank)},{WriteNullableField(IvPercentile)}";
+ }
+
+ // source: https://www.tastylive.com/concepts-strategies/implied-volatility-rank-percentile
+ private decimal? CalculateIvRank(List ivs)
+ {
+ if (ivs.Count < 2)
+ {
+ return null;
+ }
+ var oneYearLow = ivs.Min();
+ return (ivs[^1] - oneYearLow) / (ivs.Max() - oneYearLow);
+ }
+
+ // source: https://www.tastylive.com/concepts-strategies/implied-volatility-rank-percentile
+ private decimal? CalculateIvPercentile(List ivs)
+ {
+ if (ivs.Count < 2)
+ {
+ return null;
+ }
+ var daysBelowCurrentIv = ivs.Count(x => x < ivs[^1]);
+ return Convert.ToDecimal(daysBelowCurrentIv) / Convert.ToDecimal(ivs.Count);
+ }
+
+ private string WriteNullableField(decimal? field)
+ {
+ return field.HasValue ? field.Value.ToString() : string.Empty;
+ }
+ }
+}
diff --git a/Lean.DataSource.OptionsUniverseGenerator/Program.cs b/Lean.DataSource.OptionsUniverseGenerator/Program.cs
index b40ea10..c3b98c5 100644
--- a/Lean.DataSource.OptionsUniverseGenerator/Program.cs
+++ b/Lean.DataSource.OptionsUniverseGenerator/Program.cs
@@ -13,10 +13,12 @@
* limitations under the License.
*/
+using QuantConnect.Configuration;
using QuantConnect.Interfaces;
using QuantConnect.Lean.Engine.DataFeeds;
using QuantConnect.Lean.Engine.HistoricalData;
using System;
+using System.IO;
namespace QuantConnect.DataSource.OptionsUniverseGenerator
{
@@ -44,5 +46,23 @@ protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUn
return new OptionsUniverseGenerator(processingDate, securityType, market, dataFolderRoot, outputFolderRoot,
dataProvider, dataCacheProvider, historyProvider);
}
+
+ protected override DerivativeUniverseGenerator.AdditionalFieldGenerator GetAdditionalFieldGenerator(DateTime processingDate,
+ string outputFolderRoot)
+ {
+ var addtionalFieldOutputDirectory = Path.Combine(
+ Config.Get("temp-output-directory", "/temp-output-directory"),
+ "alternative",
+ "quantconnect",
+ "ivrank"
+ );
+ var processedDirectory = Path.Combine(
+ Config.Get("processed-output-directory", Globals.DataFolder),
+ "alternative",
+ "quantconnect",
+ "ivrank"
+ );
+ return new OptionAdditionalFieldGenerator(processingDate, outputFolderRoot, addtionalFieldOutputDirectory, processedDirectory);
+ }
}
}
diff --git a/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestData/generated_samples.csv b/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestData/generated_samples.csv
new file mode 100644
index 0000000..9009f85
--- /dev/null
+++ b/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestData/generated_samples.csv
@@ -0,0 +1,21 @@
+,iv_30,iv_rank,iv_percentile,test_iv_rank,test_iv_percentile
+2019-01-05,0.4033288714285714,0.7602486820250108,0.9641434262948207,76.02486820250108,96.42857142857143
+2019-01-08,0.3413325714285714,0.5940866764233323,0.8764940239043824,59.408667642333235,87.6984126984127
+2019-01-09,0.4053031714285714,0.7655401856737672,0.9641434262948207,76.55401856737673,96.42857142857143
+2019-01-10,0.3356759,0.5789257092430145,0.8605577689243027,57.89257092430147,86.11111111111111
+2019-01-11,0.3325976714285714,0.5706754647424723,0.8565737051792828,57.06754647424723,85.71428571428571
+2019-01-12,0.3398087,0.590002407961335,0.8645418326693227,59.00024079613352,86.5079365079365
+2019-01-15,0.3296392428571428,0.562746307278916,0.848605577689243,56.2746307278916,84.92063492063492
+2019-01-16,0.3535012571428571,0.6267010941195853,0.8928571428571428,62.67010941195852,89.28571428571429
+2019-01-17,0.3306122,0.5653540195169305,0.8452380952380952,56.535401951693075,84.52380952380952
+2019-01-18,0.3346532142857142,0.5761847148709971,0.8492063492063492,57.61847148709971,84.92063492063492
+2019-01-19,0.2940956428571428,0.4674826244090702,0.7579365079365079,46.74826244090702,75.79365079365078
+2019-01-23,0.2845410428571428,0.4418744591036923,0.7091633466135459,44.18744591036924,71.03174603174604
+2019-01-24,0.3095117,0.5088006219883972,0.7928286852589641,50.880062198839724,79.36507936507937
+2019-01-25,0.3287171142857142,0.5602748253864291,0.8286852589641434,56.0274825386429,82.93650793650794
+2019-01-26,0.3125058285714285,0.5168254623168983,0.8047808764940239,51.68254623168983,80.55555555555556
+2019-01-29,0.2870498428571428,0.448598525534326,0.7131474103585657,44.859852553432596,71.42857142857143
+2019-01-30,0.3180328,0.5316388085565062,0.8127490039840637,53.16388085565064,81.34920634920636
+2019-01-31,0.3599641,0.6440227557515107,0.904382470119522,64.4022755751511,90.47619047619048
+2019-02-01,0.2263559428571428,0.2859272023781422,0.5059760956175298,28.592720237814223,50.39682539682539
+2019-02-02,0.2361797857142857,0.3122569903148188,0.545816733067729,31.225699031481884,54.36507936507936
diff --git a/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestData/test_comparison.py b/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestData/test_comparison.py
new file mode 100644
index 0000000..4f0b07a
--- /dev/null
+++ b/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestData/test_comparison.py
@@ -0,0 +1,48 @@
+from datetime import datetime, timedelta
+import pandas as pd
+from pathlib import Path
+
+OUTPUT_PATH = "/temp-output-directory/option/usa/universes/aapl"
+
+df = pd.DataFrame()
+for i, file in enumerate(Path.glob(Path.cwd(), "*.csv")):
+ if i == 0:
+ continue
+ try:
+ df_daily = pd.read_csv(file, usecols=[14, 15, 16])
+ series = df_daily.iloc[-1]
+ daily_entry = series.to_frame().T
+ daily_entry.index = [datetime.strptime(file.stem, "%Y%m%d") + timedelta(1)]
+ df = pd.concat([df, daily_entry])
+ except:
+ pass
+
+### Test comparison generation
+### Adapted from https://s3.amazonaws.com/tastytradepublicmedia/website/cms/SKINNY_ivr_and_ivp.txt/original/SKINNY_ivr_and_ivp.txt?_sp=50d826c5-0948-4c48-9851-02767cd310a9.1721559732707
+### Michael Rechenthin, Ph.D., tastytrade/dough Research Team
+# Extract IV column
+history = df[["iv_30"]]
+
+# calculate the IV rank
+# ---------------------------
+# calculate the IV rank
+low_over_timespan = history.rolling(252).min()
+high_over_timespan = history.rolling(252).max()
+iv_rank = (history - low_over_timespan) / (high_over_timespan - low_over_timespan) * 100
+
+# calculate the IV percentile
+# ---------------------------
+# how many times over the past year, has IV been below the current IV
+def below(df1):
+ count = 0
+ for i in range(df1.shape[0]):
+ if df1[-1] > df1[i]:
+ count += 1
+ return count
+count_below = history.rolling(252).apply(below)
+iv_percentile = count_below / 252 * 100
+
+df = pd.concat([df, iv_rank, iv_percentile], axis=1)
+df.columns = list(df.columns)[:3] + ["test_iv_rank", "test_iv_percentile"]
+
+df.dropna().to_csv("generated_samples.csv")
\ No newline at end of file