v2.13 - Backtesting Added along with Date picker UI

- Fixed Inside bar detection and enhanced debug for dev release
- yfinance bumped to 0.2.32 from 0.1.87
- Data sources used for Nifty predictions are displayed

Author: pranjal-joshi

requirements.txt

@@ -22,7 +22,8 @@ scipy==1.11.2
 # ta-lib
 TA-Lib-Precompiled
 tabulate
-yfinance==0.1.87
+# yfinance==0.1.87
+yfinance==0.2.32
 alive-progress==1.6.2
 Pillow
 scikit-learn==1.3.2

src/classes/Changelog.py

@@ -7,7 +7,7 @@
 
 from classes.ColorText import colorText
 
-VERSION = "2.12"
+VERSION = "2.13"
 
 changelog = colorText.BOLD + '[ChangeLog]\n' + colorText.END + colorText.BLUE + '''
 [1.00 - Beta]
@@ -260,5 +260,9 @@
 1. Cosmetic Updates for Position Size Calculator
 2. Python base bumped to 3.11.6-slim-bookworm
 
+[2.13]
+1. Date based Backtesting Added for Screening
+2. Inside bar detection broken - bug fixed
+3. Auto enhanced debug on console in dev release
 
 ''' + colorText.END

src/classes/ConfigManager.py

@@ -8,6 +8,7 @@
 import sys
 import os
 import glob
+import re
 import configparser
 from datetime import date
 from classes.ColorText import colorText
@@ -195,3 +196,11 @@ def checkConfigFile(self):
             return True
         except FileNotFoundError:
             return False
+        
+    # Get period as a numeric value
+    def getPeriodNumeric(self):
+        import re
+        pattern = re.compile(r'\d+')
+        result = [int(match.group()) for match in pattern.finditer(self.period)][0]
+        return result
+

src/classes/Fetcher.py

@@ -11,6 +11,7 @@
 import requests
 import random
 import os
+import datetime
 import yfinance as yf
 import pandas as pd
 from nsetools import Nse
@@ -35,6 +36,22 @@ def __init__(self, configManager):
         self.configManager = configManager
         pass
 
+    def _getBacktestDate(self, backtest):
+        try:
+            end = backtest + datetime.timedelta(days=1)
+            if "d" in self.configManager.period:
+                delta = datetime.timedelta(days = self.configManager.getPeriodNumeric())
+            elif "wk" in self.configManager.period:
+                delta = datetime.timedelta(days = self.configManager.getPeriodNumeric() * 7)
+            elif "m" in self.configManager.period:
+                delta = datetime.timedelta(minutes = self.configManager.getPeriodNumeric())
+            elif "h" in self.configManager.period:
+                delta = datetime.timedelta(hours = self.configManager.getPeriodNumeric())
+            start = end - delta
+            return [start, end]
+        except:
+            return [None, None]
+
     def fetchCodes(self, tickerOption,proxyServer=None):
         listStockCodes = []
         if tickerOption == 12:
@@ -120,7 +137,7 @@ def fetchStockCodes(self, tickerOption, proxyServer=None):
         return listStockCodes
 
     # Fetch stock price data from Yahoo finance
-    def fetchStockData(self, stockCode, period, duration, proxyServer, screenResultsCounter, screenCounter, totalSymbols, printCounter=False, tickerOption=None):
+    def fetchStockData(self, stockCode, period, duration, proxyServer, screenResultsCounter, screenCounter, totalSymbols, backtestDate=None, printCounter=False, tickerOption=None):
         with SuppressOutput(suppress_stdout=True, suppress_stderr=True):
             append_exchange = ".NS"
             if tickerOption == 15:
@@ -131,7 +148,9 @@ def fetchStockData(self, stockCode, period, duration, proxyServer, screenResults
                 interval=duration,
                 proxy=proxyServer,
                 progress=False,
-                timeout=10
+                timeout=10,
+                start=self._getBacktestDate(backtest=backtestDate)[0],
+                end=self._getBacktestDate(backtest=backtestDate)[1]
             )
         if printCounter:
             sys.stdout.write("\r\033[K")
@@ -163,13 +182,15 @@ def fetchLatestNiftyDaily(self, proxyServer=None):
                 tickers="GC=F",
                 period='5d',
                 interval='1d',
+                proxy=proxyServer,
                 progress=False,
                 timeout=10
             ).add_prefix(prefix='gold_')
         crude = yf.download(
                     tickers="CL=F",
                     period='5d',
                     interval='1d',
+                    proxy=proxyServer,
                     progress=False,
                     timeout=10
                 ).add_prefix(prefix='crude_')

src/classes/ParallelProcessing.py

@@ -12,6 +12,7 @@
 import sys
 import os
 import pytz
+import traceback
 from queue import Empty
 from datetime import datetime
 import classes.Fetcher as Fetcher
@@ -59,7 +60,7 @@ def run(self):
             sys.exit(0)
 
     def screenStocks(self, tickerOption, executeOption, reversalOption, maLength, daysForLowestVolume, minRSI, maxRSI, respChartPattern, insideBarToLookback, totalSymbols,
-                     configManager, fetcher, screener, candlePatterns, stock, newlyListedOnly, downloadOnly, vectorSearch, printCounter=False):
+                     configManager, fetcher, screener, candlePatterns, stock, newlyListedOnly, downloadOnly, vectorSearch, isDevVersion, backtestDate, printCounter=False):
         screenResults = pd.DataFrame(columns=[
             'Stock', 'Consolidating', 'Breaking-Out', 'MA-Signal', 'Volume', 'LTP', 'RSI', 'Trend', 'Pattern'])
         screeningDictionary = {'Stock': "", 'Consolidating': "",  'Breaking-Out': "",
@@ -78,14 +79,18 @@ def screenStocks(self, tickerOption, executeOption, reversalOption, maLength, da
                     period = configManager.period
 
             if (self.stockDict.get(stock) is None) or (configManager.cacheEnabled is False) or self.isTradingTime or downloadOnly:
-                data = fetcher.fetchStockData(stock,
-                                              period,
-                                              configManager.duration,
-                                              self.proxyServer,
-                                              self.screenResultsCounter,
-                                              self.screenCounter,
-                                              totalSymbols,
-                                              tickerOption=tickerOption)
+                try:
+                    data = fetcher.fetchStockData(stock,
+                                                period,
+                                                configManager.duration,
+                                                self.proxyServer,
+                                                self.screenResultsCounter,
+                                                self.screenCounter,
+                                                totalSymbols,
+                                                backtestDate=backtestDate,
+                                                tickerOption=tickerOption)
+                except Exception as e:
+                    return screeningDictionary, saveDictionary
                 if configManager.cacheEnabled is True and not self.isTradingTime and (self.stockDict.get(stock) is None) or downloadOnly:
                     self.stockDict[stock] = data.to_dict('split')
                     if downloadOnly:
@@ -165,7 +170,7 @@ def screenStocks(self, tickerOption, executeOption, reversalOption, maLength, da
                     isConfluence = screener.validateConfluence(stock, processedData, screeningDictionary, saveDictionary, percentage=insideBarToLookback)
                 else:
                     isInsideBar = screener.validateInsideBar(processedData, screeningDictionary, saveDictionary, chartPattern=respChartPattern, daysToLookback=insideBarToLookback)
-                
+
                 with SuppressOutput(suppress_stderr=True, suppress_stdout=True):
                     if maLength is not None and executeOption == 6 and reversalOption == 6:
                         isNR = screener.validateNarrowRange(processedData, screeningDictionary, saveDictionary, nr=maLength)
@@ -261,6 +266,9 @@ def screenStocks(self, tickerOption, executeOption, reversalOption, maLength, da
         except KeyError:
             pass
         except Exception as e:
+            if isDevVersion:
+                print("[!] Dev Traceback:")
+                traceback.print_exc()
             if printCounter:
                 print(colorText.FAIL +
                       ("\n[+] Exception Occured while Screening %s! Skipping this stock.." % stock) + colorText.END)

src/classes/Screener.py

@@ -13,6 +13,7 @@
 import keras
 import time
 import classes.Utility as Utility
+from copy import copy
 from advanced_ta import LorentzianClassification
 from classes.Utility import isGui
 from sklearn.preprocessing import StandardScaler
@@ -253,6 +254,7 @@ def findBreakout(self, data, screenDict, saveDict, daysToLookback):
     # Validate 'Inside Bar' structure for recent days
     def validateInsideBar(self, data, screenDict, saveDict, chartPattern=1, daysToLookback=5):
         orgData = data
+        daysToLookback = int(daysToLookback)
         for i in range(daysToLookback, round(daysToLookback*0.5)-1, -1):
             if i == 2:
                 return 0        # Exit if only last 2 candles are left
@@ -610,6 +612,7 @@ def getNiftyPrediction(self, data, proxyServer):
           pass
         #
         model, pkl = Utility.tools.getNiftyModel(proxyServer=proxyServer)
+        datacopy = copy(data[pkl['columns']])
         with SuppressOutput(suppress_stderr=True, suppress_stdout=True):
             data = data[pkl['columns']]
             ### v2 Preprocessing
@@ -630,7 +633,7 @@ def getNiftyPrediction(self, data, proxyServer):
         print(colorText.BOLD + colorText.BLUE + "\n" + "[+] Nifty AI Prediction -> " + colorText.END + colorText.BOLD + "Market may Open {} next day! {}".format(out, sug) + colorText.END)
         print(colorText.BOLD + colorText.BLUE + "\n" + "[+] Nifty AI Prediction -> " + colorText.END + "Probability/Strength of Prediction = {}%".format(Utility.tools.getSigmoidConfidence(pred[0])))
         if isGui():
-            return pred, 'BULLISH' if pred <= 0.5 else 'BEARISH', Utility.tools.getSigmoidConfidence(pred[0])
+            return pred, 'BULLISH' if pred <= 0.5 else 'BEARISH', Utility.tools.getSigmoidConfidence(pred[0]), pd.DataFrame(datacopy.iloc[-1]).T
         return pred
 
     def monitorFiveEma(self, proxyServer, fetcher, result_df, last_signal, risk_reward = 3):

src/classes/Utility.py

@@ -377,6 +377,14 @@ def alertSound(beeps=3, delay=0.2):
             print('\a')
             sleep(delay)
 
+    def isBacktesting(backtestDate):
+        try:
+            if datetime.date.today() != backtestDate:
+                return True
+            return False
+        except:
+            return False
+
 def isDocker():
     if 'SCREENIPY_DOCKER' in os.environ:
         return True

src/release.md

@@ -7,12 +7,12 @@ Screeni-py is now on **YouTube** for additional help! - Thank You for your suppo
 
 ⚠️ **Executable files (.exe, .bin and .run) are now DEPRECATED! Please Switch to Docker**
 
-1. **Position Size Calculator** tab added for Better and Quick Risk Management!
-2. **Lorentzian Classification** (by @jdehorty) added for enhanced accuracy for your trades - - Try `Option > 6 > 7` 🤯
-3. **Artificial Intelligence v3 for Nifty 50 Prediction** - Predict Next day Gap-up/down using Nifty, Gold and Crude prices! - Try `Select Index for Screening > N`
-4. **US S&P 500** Index added for scanning US markets.
-5. **Search Similar Stocks** Added using Vector Similarity search - Try `Search Similar Stocks`.
-6. New Index - **F&O Stocks Only** Added for F&O traders with modified screening criterias.
+1. **Backtesting** Added for Screening Patterns to Develope and Test Strategies!
+2. **Position Size Calculator** tab added for Better and Quick Risk Management!
+3. **Lorentzian Classification** (by @jdehorty) added for enhanced accuracy for your trades - - Try `Option > 6 > 7` 🤯
+4. **Artificial Intelligence v3 for Nifty 50 Prediction** - Predict Next day Gap-up/down using Nifty, Gold and Crude prices! - Try `Select Index for Screening > N`
+5. **US S&P 500** Index added for scanning US markets.
+6. **Search Similar Stocks** Added using Vector Similarity search - Try `Search Similar Stocks`.
 7. New Screener **Buy at Trendline** added for Swing/Mid/Long term traders - Try `Option > 7 > 5`.
 
 ## Installation Guide
@@ -38,6 +38,10 @@ Screeni-py is now on **YouTube** for additional help! - Thank You for your suppo
 | ![Docker](https://img.shields.io/badge/docker-%230db7ed.svg?style=for-the-badge&logo=docker&logoColor=white) | `dev` | `docker pull joshipranjal/screeni-py:dev` | Command Line | `docker run -it --entrypoint /bin/bash joshipranjal/screeni-py:dev -c "run_screenipy.sh --cli"` |
 | ![Docker](https://img.shields.io/badge/docker-%230db7ed.svg?style=for-the-badge&logo=docker&logoColor=white) | `dev` | `docker pull joshipranjal/screeni-py:dev` | GUI WebApp | `docker run -p 8501:8501 joshipranjal/screeni-py:dev` |
 
+### Docker Issues? Troubleshooting Guide:
+
+Read this [troubleshooting guide](https://github.com/pranjal-joshi/Screeni-py/discussions/217) for Windows to fix most common Docker issues easily!
+
 **Why we shifted to Docker from the Good old EXEs?**
 
 | Executable/Binary File | Docker |

src/screenipy.py

@@ -23,7 +23,7 @@
 import urllib
 import numpy as np
 import pandas as pd
-from datetime import datetime
+from datetime import datetime, date
 from time import sleep
 from tabulate import tabulate
 import multiprocessing
@@ -170,13 +170,13 @@ def initExecution():
     return tickerOption, executeOption
 
 # Main function
-def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list = []):
+def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list = [], isDevVersion=None, backtestDate=date.today()):
     global screenCounter, screenResultsCounter, stockDict, loadedStockData, keyboardInterruptEvent, loadCount, maLength, newlyListedOnly, vectorSearch
     screenCounter = multiprocessing.Value('i', 1)
     screenResultsCounter = multiprocessing.Value('i', 0)
     keyboardInterruptEvent = multiprocessing.Manager().Event()
 
-    if stockDict is None:
+    if stockDict is None or Utility.tools.isBacktesting(backtestDate=backtestDate):
         stockDict = multiprocessing.Manager().dict()
         loadCount = 0
 
@@ -371,7 +371,7 @@ def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list
                 input('')
             sys.exit(0)
 
-        if not Utility.tools.isTradingTime() and configManager.cacheEnabled and not loadedStockData and not testing:
+        if not Utility.tools.isTradingTime() and configManager.cacheEnabled and not loadedStockData and not testing and not Utility.tools.isBacktesting(backtestDate=backtestDate):
             Utility.tools.loadStockData(stockDict, configManager, proxyServer)
             loadedStockData = True
         loadCount = len(stockDict)
@@ -380,7 +380,7 @@ def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list
               "[+] Starting Stock Screening.. Press Ctrl+C to stop!\n")
 
         items = [(tickerOption, executeOption, reversalOption, maLength, daysForLowestVolume, minRSI, maxRSI, respChartPattern, insideBarToLookback, len(listStockCodes),
-                  configManager, fetcher, screener, candlePatterns, stock, newlyListedOnly, downloadOnly, vectorSearch)
+                  configManager, fetcher, screener, candlePatterns, stock, newlyListedOnly, downloadOnly, vectorSearch, isDevVersion, backtestDate)
                  for stock in listStockCodes]
 
         tasks_queue = multiprocessing.JoinableQueue()
@@ -498,7 +498,7 @@ def main(testing=False, testBuild=False, downloadOnly=False, execute_inputs:list
 
         print(colorText.BOLD + colorText.GREEN +
                   f"[+] Found {len(screenResults)} Stocks." + colorText.END)
-        if configManager.cacheEnabled and not Utility.tools.isTradingTime() and not testing:
+        if configManager.cacheEnabled and not Utility.tools.isTradingTime() and not testing and not Utility.tools.isBacktesting(backtestDate=backtestDate):
             print(colorText.BOLD + colorText.GREEN +
                   "[+] Caching Stock Data for future use, Please Wait... " + colorText.END, end='')
             Utility.tools.saveStockData(