IV Rank

Lean.DataSource.DerivativeUniverseGenerator/AdditionalFieldGenerator.cs

@@ -0,0 +1,82 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2024 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Logging;
+using System;
+using System.IO;
+using System.Linq;
+
+namespace QuantConnect.DataSource.DerivativeUniverseGenerator
+{
+    /// <summary>
+    /// Generate additional fields that needed to calculate from the whoel derivative chain
+    /// </summary>
+    public class AdditionalFieldGenerator
+    {
+        protected readonly DateTime _processingDate;
+        protected readonly string _rootPath;
+
+        /// <summary>
+        /// Instantiate a new instance of <see cref="AdditionalFieldGenerator"/>
+        /// </summary>
+        /// <param name="processingDate"></param>
+        /// <param name="rootPath"></param>
+        public AdditionalFieldGenerator(DateTime processingDate, string rootPath)
+        {
+            _processingDate = processingDate;
+            _rootPath = rootPath;
+        }
+
+        /// <summary>
+        /// Run the additional fields generation
+        /// </summary>
+        /// <returns>If the generator run successfully</returns>
+        public virtual bool Run()
+        {
+            throw new NotImplementedException("AdditionalFieldGenerator.Run(): Run method must be implemented.");
+        }
+
+        /// <summary>
+        /// Write the additional fields to the Csv file being generated
+        /// </summary>
+        /// <param name="csvPath">Target csv file path</param>
+        /// <param name="additionalFields">The addtional field content</param>
+        protected virtual void WriteToCsv(string csvPath, IAdditionalFields additionalFields)
+        {
+            if (string.IsNullOrWhiteSpace(csvPath))
+            {
+                Log.Error("AdditionalFieldGenerator.WriteToCsv(): invalid file path provided");
+                return;
+            }
+
+            var csv = File.ReadAllLines(csvPath)
+                .Where(s => !string.IsNullOrWhiteSpace(s))
+                .ToList();
+            for (int i = 0; i < csv.Count; i++)
+            {
+                if (i == 0)
+                {
+                    csv[i] += $",{additionalFields.GetHeader()}";
+                }
+                else
+                {
+                    csv[i] += $",{additionalFields.ToCsv()}";
+                }
+            }
+
+            File.WriteAllLines(csvPath, csv);
+        }
+    }
+}

Lean.DataSource.DerivativeUniverseGenerator/IAdditionalFields.cs

@@ -0,0 +1,33 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2024 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+namespace QuantConnect.DataSource.DerivativeUniverseGenerator
+{
+    /// <summary>
+    /// Interface of additional fields which requires the whole derivative chain to calculate from
+    /// </summary>
+    public interface IAdditionalFields
+    {
+        /// <summary>
+        /// Convert the entry to a CSV string.
+        /// </summary>
+        string ToCsv();
+
+        /// <summary>
+        /// Gets the header of the CSV file
+        /// </summary>
+        string GetHeader();
+    }
+}

Lean.DataSource.DerivativeUniverseGenerator/Program.cs

@@ -13,20 +13,20 @@
  * limitations under the License.
 */
 
+using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
+using System.IO;
 using System.Linq;
-using System;
-
+using Newtonsoft.Json;
 using QuantConnect.Configuration;
 using QuantConnect.Logging;
 using QuantConnect.Util;
 using QuantConnect.Data;
 using QuantConnect.Interfaces;
 using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Lean.Engine.HistoricalData;
-using Newtonsoft.Json;
-using System.Collections.Generic;
 
 namespace QuantConnect.DataSource.DerivativeUniverseGenerator
 {
@@ -103,6 +103,23 @@ protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null)
                     Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating universe.");
                     Environment.Exit(1);
                 }
+
+                var universeOutputPath = Path.Combine(outputFolderRoot, securityType.SecurityTypeToLower(), market, "universes");
+                var optionsAdditionalFieldGenerator = GetAdditionalFieldGenerator(processingDate, universeOutputPath);
+
+                try
+                {
+                    if (!optionsAdditionalFieldGenerator.Run())
+                    {
+                        Log.Error($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Failed to generate additional fields.");
+                        Environment.Exit(1);
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Log.Error(ex, $"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): Error generating additional fields.");
+                    Environment.Exit(1);
+                }
             }
 
             Log.Trace($"QuantConnect.DataSource.DerivativeUniverseGenerator.Program.Main(): DONE in {timer.Elapsed:g}");
@@ -114,6 +131,8 @@ protected abstract DerivativeUniverseGenerator GetUniverseGenerator(SecurityType
             string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider,
             HistoryProviderManager historyProvider);
 
+        protected abstract AdditionalFieldGenerator GetAdditionalFieldGenerator(DateTime processingDate, string outputFolderRoot);
+
         /// <summary>
         /// Validate and extract command line args and configuration options.
         /// </summary>

Lean.DataSource.FuturesUniverseGenerator/Program.cs

@@ -43,5 +43,10 @@ protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUn
             return new FuturesUniverseGenerator(processingDate, market, dataFolderRoot, outputFolderRoot, dataProvider,
                 dataCacheProvider, historyProvider);
         }
+
+        protected override DerivativeUniverseGenerator.AdditionalFieldGenerator GetAdditionalFieldGenerator(DateTime dateTime, string _)
+        {
+            return null;
+        }
     }
 }

Lean.DataSource.OptionsUniverseGenerator/OptionAdditionalFieldGenerator.cs

@@ -0,0 +1,187 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2024 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Interfaces;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Logging;
+
+namespace QuantConnect.DataSource.OptionsUniverseGenerator
+{
+    public class OptionAdditionalFieldGenerator : DerivativeUniverseGenerator.AdditionalFieldGenerator
+    {
+        private const string _impliedVolHeader = "implied_volatility";
+        private const string _deltaHeader = "delta";
+        private const string _sidHeader = "#symbol_id";
+        private const string _tickerHeader = "symbol_value";
+        private readonly string _outputPath;
+        private readonly string _addtionalFieldDataPath;
+        private Dictionary<string, Dictionary<DateTime, decimal>> _iv30s = new();
+        private static readonly IMapFileProvider _mapFileProvider = new LocalZipMapFileProvider();
+
+        public OptionAdditionalFieldGenerator(DateTime processingDate, string rootPath, string newOutputPath, string addtionalFieldDataPath)
+            : base(processingDate, rootPath)
+        {
+            _outputPath = newOutputPath;
+            _addtionalFieldDataPath = addtionalFieldDataPath;
+            _mapFileProvider.Initialize(new DefaultDataProvider());
+        }
+
+        public override bool Run()
+        {
+            Log.Trace($"OptionAdditionalFieldGenerator.Run(): Processing additional fields for date {_processingDate:yyyy-MM-dd}");
+            var dataByTicker = new List<string>();
+
+            // per symbol
+            foreach (var subFolder in Directory.GetDirectories(_rootPath))
+            {
+                try
+                {
+                    _iv30s[subFolder] = new();
+                    var dateFile = Path.Combine(subFolder, $"{_processingDate:yyyyMMdd}.csv");
+                    var symbol = subFolder.Split(Path.DirectorySeparatorChar)[^1].ToUpper();
+                    if (!File.Exists(dateFile))
+                    {
+                        Log.Error($"OptionAdditionalFieldGenerator.Run(): no universe file found for {symbol} in {_processingDate:yyyy-MM-dd}");
+                        return false;
+                    }
+
+                    var ivs = GetIvs(_processingDate, subFolder);
+                    var additionalFields = new OptionAdditionalFields();
+                    additionalFields.Update(ivs);
+
+                    var sid = SecurityIdentifier.GenerateEquity(symbol, Market.USA, true, _mapFileProvider, _processingDate);
+                    dataByTicker.Add($"{sid},{symbol},{additionalFields.ToCsv()}");
+
+                    WriteToCsv(dateFile, additionalFields);
+
+                    SaveContentToFile(_outputPath, _addtionalFieldDataPath, $"{_processingDate:yyyyMMdd}", dataByTicker);
+                }
+                catch (Exception ex)
+                {
+                    Log.Error(ex,
+                        $"OptionAdditionalFieldGenerator.Run(): Error processing addtional fields for date {_processingDate:yyyy-MM-dd}");
+                }
+            }
+            return true;
+        }
+
+        private List<decimal> GetIvs(DateTime currentDateTime, string path)
+        {
+            // get i-year ATM IVs to calculate IV rank and percentile
+            var lastYearFiles = Directory.EnumerateFiles(path, "*.csv")
+                .AsParallel()
+                .Where(file => DateTime.TryParseExact(Path.GetFileNameWithoutExtension(file), "yyyyMMdd",
+                        CultureInfo.InvariantCulture, DateTimeStyles.None, out var fileDate)
+                    && fileDate > currentDateTime.AddYears(-1)
+                    && fileDate <= currentDateTime
+                    && !_iv30s[path].ContainsKey(fileDate))
+                .ToDictionary(
+                    file => DateTime.ParseExact(Path.GetFileNameWithoutExtension(file), "yyyyMMdd",
+                    CultureInfo.InvariantCulture, DateTimeStyles.None),
+                    file => GetAtmIv(file)
+                );
+            _iv30s[path] = _iv30s[path].Concat(lastYearFiles)
+                .ToDictionary(x => x.Key, x => x.Value);
+
+            return _iv30s[path].Where(x => x.Key > currentDateTime.AddYears(-1) && x.Key <= currentDateTime)
+                .OrderBy(x => x.Key)
+                .Select(x => x.Value)
+                .ToList();
+        }
+
+        private decimal GetAtmIv(string csvPath)
+        {
+            var lines = File.ReadAllLines(csvPath)
+                .Where(s => !string.IsNullOrWhiteSpace(s))
+                .ToList();
+            var headers = lines[0].Split(',');
+            var deltaIndex = Array.IndexOf(headers, _deltaHeader);
+            var ivIndex = Array.IndexOf(headers, _impliedVolHeader);
+            var sidIndex = Array.IndexOf(headers, _sidHeader);
+            var tickerIndex = Array.IndexOf(headers, _tickerHeader);
+
+            if (deltaIndex == -1 || ivIndex == -1 || sidIndex == -1 || tickerIndex == -1)
+            {
+                return -1m;
+            }
+
+            // Skip underlying row
+            var filtered = lines.Skip(2)
+                .Select(line =>
+                {
+                    var values = line.Split(',');
+                    var symbol = new Symbol(SecurityIdentifier.Parse(values[sidIndex]), values[tickerIndex]);
+                    var delta = decimal.Parse(values[deltaIndex]);
+                    var iv = decimal.Parse(values[ivIndex]);
+                    return (Expiry: symbol.ID.Date, Delta: delta, ImpliedVolatility: iv);
+                })
+                .Where(x => x.ImpliedVolatility != 0m)
+                .ToList();
+            if (filtered.Count == 0)
+            {
+                return -1m;
+            }
+
+            var expiries = filtered.Select(x => x.Expiry).ToList().Distinct();
+            var currentDateTime = DateTime.ParseExact(Path.GetFileNameWithoutExtension(csvPath), "yyyyMMdd", 
+                CultureInfo.InvariantCulture, DateTimeStyles.None);
+            var day30 = currentDateTime.AddDays(30);
+            var nearExpiry = expiries.Where(x => x <= day30).Max();
+            var farExpiry = expiries.Where(x => x >= day30).Min();
+
+            var nearIv = filtered.Where(x => x.Expiry == nearExpiry)
+                .OrderBy(x => Math.Abs(x.Delta - 0.5m))
+                .First()
+                .ImpliedVolatility;
+            if (nearExpiry == farExpiry)
+            {
+                return nearIv;
+            }
+            var farIv = filtered.Where(x => x.Expiry == farExpiry)
+                .OrderBy(x => Math.Abs(x.Delta - 0.5m))
+                .First()
+                .ImpliedVolatility;
+            // Linear interpolation
+            return (nearIv * Convert.ToDecimal((farExpiry - day30).TotalDays) + farIv * Convert.ToDecimal((day30 - nearExpiry).TotalDays))
+                / Convert.ToDecimal((farExpiry - nearExpiry).TotalDays);
+        }
+
+        private static void SaveContentToFile(string destinationFolder, string processedFolder, string name, IEnumerable<string> contents)
+        {
+            Directory.CreateDirectory(destinationFolder);
+            name = $"{name.ToLowerInvariant()}.csv";
+            var filePath = Path.Combine(processedFolder, name);
+
+            var lines = new HashSet<string>(contents);
+            if (File.Exists(filePath))
+            {
+                foreach (var line in File.ReadAllLines(filePath))
+                {
+                    lines.Add(line);
+                }
+            }
+
+            var finalLines = lines.OrderBy(x => x.Split(',').First()).ToList();
+            var finalPath = Path.Combine(destinationFolder, name);
+            File.WriteAllLines(finalPath, finalLines);
+        }
+    }
+}