Benchmark of Python backends (#84)

* Python benchmark option
* Benchmark LTO & Ffloating point
* Series type fix 
* Cuml draft
* Benchmark - uncovered the faulty test

Author: mj-xmr

.github/workflows/build.yml

@@ -110,7 +110,11 @@ jobs:
     - name: build
       run: ./ci-default --compiler ${{ matrix.compiler }} --lto -j 1 ${{ matrix.options }} --native --unity --build-r
     - name: run demo
-      run: ./ci-default --compiler ${{ matrix.compiler }} --lto -j 1 ${{ matrix.options }} --native --unity --build-r --run-demo
+      run: ./ci-default --compiler ${{ matrix.compiler }} --lto ${{ matrix.options }} --native --unity --build-r --run-demo
+    - name: benchmark across frameworks
+      run: |
+        pip3 install darts
+        ./ci-default --compiler ${{ matrix.compiler }} --lto ${{ matrix.options }} --native --unity --build-r --benchmark
       
   build-macos:
     runs-on: ${{ matrix.os }}

README.md

@@ -152,6 +152,20 @@ export LD_LIBRARY_PATH=$R_HOME/lib
 ./tsqsim
 ```
 
+## Python backends
+`tsqsim` is able to wrap the Python 3rd party TSA frameworks. You may either write your own wrapper or use the already available ones.
+
+### Available python backends
+The following Python backends are currently available:
+
+| Name  | installation | script |
+| ----- | ------------ | ------ |
+| [statsmodels](https://www.statsmodels.org/) | pip install darts | scripts/py_darts.py |
+| [darts](https://github.com/unit8co/darts) |  pip install statsmodels  | scripts/py_statsmodels.py |
+
+### Extending python backends
+TODO! Wrap predict and convert from dataframe and return a timeseries.
+
 # Development
 For the development use case, it's recommended to turn on certain optimizations, that reduce the recompilation and linking time while changing the source code often. The optimizations are: dynamic linking (*shared*), precompiled headers (*pch*) and (optionally) a networked parallel compiler's wrapper, [icecc](https://github.com/icecc/icecream).
 

src/lib-base/src/CLI.cpp

@@ -27,6 +27,8 @@ EnjoLib::Result<CLIResult> CLI::GetConfigs(int argc, char ** argv) const
     const char * OPT_OUT_DIR   = "out";
     const char * OPT_LAGS  = "lags";
     const char * OPT_PER_SEASONAL  = "per-seasonal";
+    const char * OPT_BENCHMARK  = "benchmark";
+    const char * OPT_PLOT  = "plot";
 
 
     EnjoLib::ProgramOptionsState popState;
@@ -43,6 +45,9 @@ EnjoLib::Result<CLIResult> CLI::GetConfigs(int argc, char ** argv) const
 
     popState.AddInt(OPT_LAGS,           ConfigTS::DESCR_PLOT_LAGS_NUM);
     popState.AddInt(OPT_PER_SEASONAL,   ConfigTS::DESCR_PLOT_PERIOD_NUM);
+    
+    popState.AddBool(OPT_BENCHMARK,     ConfigTS::DESCR_BENCHMARK);
+    popState.AddBool(OPT_PLOT,          ConfigTS::DESCR_PLOT_PYTHON);
 
 
     popState.ReadArgs(argc, argv);
@@ -91,6 +96,8 @@ EnjoLib::Result<CLIResult> CLI::GetConfigs(int argc, char ** argv) const
     confTS.PLOT_LAGS_NUM        = pops.GetIntFromMap(OPT_LAGS);
     confTS.PLOT_PERIOD_NUM      = pops.GetIntFromMap(OPT_PER_SEASONAL);
     confTS.m_outDir             = pops.GetStrFromMap(OPT_OUT_DIR);
+    confTS.BENCHMARK            = pops.GetBoolFromMap(OPT_BENCHMARK);
+    confTS.PLOT_PYTHON          = pops.GetBoolFromMap(OPT_PLOT);
 
     //auto pluginName = pops.GetStrFromMap (OPT_PLUGIN);
 

src/lib-base/src/ConfigTS.cpp

@@ -2,6 +2,7 @@
 
 #include "ConfigDirs.h"
 #include "Consts.h"
+#include "PriceType.h"
 
 //#include <Util/Except.hpp>
 #include <Ios/Osstream.hpp>
@@ -11,14 +12,20 @@
 using namespace std;
 using namespace EnjoLib;
 
-const char * ConfigTS::DEFAULT_SCRIPT_FILE_NAME   = "ts-xform-script.txt";
-const char * ConfigTS::DEFAULT_SCRIPT_FILE_NAME_R = "r-custom.R";
+const char * ConfigTS::DEFAULT_SCRIPT_FILE_NAME     = "ts-xform-script.txt";
+const char * ConfigTS::DEFAULT_SCRIPT_FILE_NAME_R   = "r-custom.R";
+const char * ConfigTS::DEFAULT_SCRIPT_FILE_NAME_PY  = "py_statsmodels.py";
 const char * ConfigTS::DEFAULT_SCRIPT_FILE_NAME_GEN = "data-generation.txt";
 
 
 ConfigTS::~ConfigTS(){}
 ConfigTS::ConfigTS()
 {
+    const ConfigDirs dirs;
+    m_scriptPathTxt =      dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME;
+    m_scriptPathTxtR =     dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME_R;
+    m_scriptPathTxtGen =   dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME_GEN;
+    m_scriptPathTxtPy =    dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME_PY;
 }
 
 EnjoLib::Str ConfigTS::GetFileNameProt() const
@@ -35,17 +42,19 @@ void ConfigTS::RegisterAndReadBools(EnjoLib::Istream & f)
     RegisterAndReadBool(f, crashOnRecoverableErrors, 0, "Crash on err",     "Crash on recoverable errors");
     RegisterAndReadBool(f, PLOT_SERIES,              1, "Plot series",      "Plot output series after evaluation");
     RegisterAndReadBool(f, PLOT_BASELINE,            1, "Plot baseline",    "Plot baseline prediction in QT app");
-    RegisterAndReadBool(f, PLOT_PYTHON,              0, "Plot with Python", "Plot the series in Python backend");
+    RegisterAndReadBool(f, PLOT_PYTHON,              0, "Plot with Python", DESCR_PLOT_PYTHON);
     RegisterAndReadBool(f, PLOT_PYTHON_ACF,          0, "Plot ACF with Python", "Plot the Auto Correlation Function in Python backend");
     RegisterAndReadBool(f, MT_XFORM,                 1, "MT xform",         "Perform the transformations in a multithreaded way (still unstable)");
     RegisterAndReadBool(f, MT_REPORT,                0, "MT report",        "Generate report in multithreaded way (still unstable)");
     RegisterAndReadBool(f, USE_VECTOR_PRED,          1, "Opti vec pred",    "Use optimized vectored prediction");
+    RegisterAndReadBool(f, BENCHMARK,                0, "Benchmark",    DESCR_BENCHMARK);
+    
 }
 
 void ConfigTS::RegisterAndReadInts(EnjoLib::Istream & f)
 {
     RegisterAndReadInt (f, PRED_TYPE, 0);
-    RegisterAndReadInt (f, PRICE_TYPE, 0);
+    RegisterAndReadInt (f, PRICE_TYPE, static_cast<long int>(PriceType::HIGH));
     RegisterAndReadInt (f, PLOT_LAGS_NUM, 30);
     RegisterAndReadInt (f, PLOT_PERIOD_NUM, 30);
 }
@@ -54,13 +63,19 @@ void ConfigTS::RegisterAndReadFloats(EnjoLib::Istream & f)
 }
 void ConfigTS::RegisterAndReadStrs(EnjoLib::Istream & f)
 {
-    const ConfigDirs dirs;
-    RegisterAndReadStr(f, m_scriptPathTxt,      dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME);
-    RegisterAndReadStr(f, m_scriptPathTxtR,     dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME_R);
-    RegisterAndReadStr(f, m_scriptPathTxtGen,   dirs.DIR_SCRIPTS2 + DEFAULT_SCRIPT_FILE_NAME_GEN);
+    RegisterAndReadStr(f, m_scriptPathTxt,      m_scriptPathTxt);
+    RegisterAndReadStr(f, m_scriptPathTxtR,     m_scriptPathTxtR);
+    RegisterAndReadStr(f, m_scriptPathTxtGen,   m_scriptPathTxtGen);
+    RegisterAndReadStr(f, m_scriptPathTxtPy,    m_scriptPathTxtPy);
     //RegisterAndReadStr(f, m_outDir,   "");
 }
 
+void ConfigTS::SetScriptNamePy(const EnjoLib::Str & name)
+{
+    const ConfigDirs dirs;
+    m_scriptPathTxtPy = dirs.DIR_SCRIPTS2 + name;
+}
+
 PredictorType ConfigTS::GetPredType() const
 {
     return static_cast<PredictorType>(PRED_TYPE);
@@ -92,4 +107,16 @@ void ConfigTS::UpdateFromOther(const ConfigTS & cfgTSCmdLine)
     {
         PLOT_PERIOD_NUM = cfgTSCmdLine.PLOT_PERIOD_NUM;
     }
+    if (cfgTSCmdLine.PRED_TYPE >= 0)
+    {
+        PRED_TYPE = cfgTSCmdLine.PRED_TYPE;
+    }
+    if (cfgTSCmdLine.BENCHMARK)
+    {
+        BENCHMARK = cfgTSCmdLine.BENCHMARK;
+    }
+    if (cfgTSCmdLine.PLOT_PYTHON)
+    {
+        PLOT_PYTHON = cfgTSCmdLine.PLOT_PYTHON;
+    }
 }

src/lib-base/src/ConfigTS.h

@@ -25,27 +25,32 @@ class ConfigTS : public ConfigBase
 
         PriceType GetPriceType() const;
         void SetPriceType(const PriceType & type);
+        
+        void SetScriptNamePy(const EnjoLib::Str & name);
 
         void UpdateFromOther(const ConfigTS & cfgTSCmdLine);
 
 
         static const char * DEFAULT_SCRIPT_FILE_NAME;
         static const char * DEFAULT_SCRIPT_FILE_NAME_R;
+        static const char * DEFAULT_SCRIPT_FILE_NAME_PY;
         static const char * DEFAULT_SCRIPT_FILE_NAME_GEN;
 
         EnjoLib::Str m_scriptPathTxt;
         EnjoLib::Str m_scriptPathTxtR;
+        EnjoLib::Str m_scriptPathTxtPy;
         EnjoLib::Str m_scriptPathTxtGen;
         EnjoLib::Str m_outDir;
 
         bool crashOnRecoverableErrors = false;
         bool PLOT_SERIES = true;
         bool PLOT_BASELINE = true;
-        bool PLOT_PYTHON = false;
+        bool PLOT_PYTHON = false; constexpr static const char * DESCR_PLOT_PYTHON = "Plot the series in Python backend";
         bool PLOT_PYTHON_ACF = false;
         bool MT_REPORT = false;
         bool MT_XFORM = false;
         bool USE_VECTOR_PRED = false;
+        bool BENCHMARK = false; constexpr static const char * DESCR_BENCHMARK = "Benchmark the simulator against other frameworks";
 
         long int PRED_TYPE = 0;
         long int PRICE_TYPE = 0;

src/lib-base/src/IPredictor.cpp

@@ -78,3 +78,8 @@ void IPredictor::AssertNoLookaheadBias(const EnjoLib::VecD & data, const EnjoLib
     /// TODO: eps must be a percentage!
     Assertions::IsTrue (Logic::DoublesEqual(predLast,  predVec.Last(), 0.01),       "DoublesEqual(predLast,  predVec)");
 }
+
+bool IPredictor::IsCustomScript() const
+{
+    return false;
+}

src/lib-base/src/IPredictor.h

@@ -46,6 +46,8 @@ class IPredictor : public IOptimizable, public IHasLen
 
         virtual EnjoLib::VecD AssertNoLookaheadBiasGetVec(const EnjoLib::VecD & data) const;
         virtual void AssertNoLookaheadBias(const EnjoLib::VecD & data, const EnjoLib::VecD & predVec) const;
+        
+        virtual bool IsCustomScript() const;
 
         /// The error returned when there's not enough data for the prediction yet.
         static const double ERROR;

src/lib-base/src/PredictorAR.cpp

@@ -2,7 +2,7 @@
 #include "PredictorUtil.h"
 #include "ITSFun.h"
 
-//#include <Util/CoutBuf.hpp>
+#include <Util/CoutBuf.hpp>
 
 using namespace EnjoLib;
 
@@ -12,21 +12,38 @@ PredictorAR::PredictorAR(const IDataProvider & dat, const EnjoLib::Str & name)
 
 PredictorAR::~PredictorAR(){}
 
+/*
 EnjoLib::VecD PredictorAR::PredictVec(const EnjoLib::VecD & data) const
 {
     const int numSamples = GetLags();
     const EnjoLib::VecD & predAR = PredictorUtil().Regression(numSamples, data, false);
     return predAR;
 }
-
+*/
 double PredictorAR::PredictNext(const BufferDouble & datExpanding) const
 {
-    const VecD & vec = PredictVec(datExpanding.GetData());
-    return vec.Last();
+    const int lags = GetLags();
+    if (datExpanding.Len() < lags + 1)
+    {
+        return IPredictor::ERROR;
+    }
+    VecD ret;
+    for (int lag = 0; lag < lags; ++lag)
+    {
+        //const EnjoLib::VecD & predVec = PredictorUtil().RegressionProt(lag, datExpanding.GetData(), datExpanding.Len(), false);
+        //const double pred = predVec.Last();
+        const double pred = PredictorUtil().RegressionProt(lag, datExpanding.GetData(), datExpanding.Len(), false);
+        ret.Add(pred);
+        //LOGL << "Pred lag " << lag << " = " << pred << ", ret = " << ret.Mean() << Nl;
+    }
+    return ret.Mean();
+    
+    //const VecD & vec = PredictVec(datExpanding.GetData());
+    //return vec.Last();
 }
 
 unsigned PredictorAR::GetLags() const
 {
-    //return 11;
+    return 2;
     return GetLag1().GetVal();
 }

src/lib-base/src/PredictorAR.h

@@ -10,7 +10,7 @@ class PredictorAR : public PredictorBase
         PredictorAR(const IDataProvider & dat, const EnjoLib::Str & name = "AR");
         virtual ~PredictorAR();
 
-        EnjoLib::VecD PredictVec(const EnjoLib::VecD & data) const override;
+        //EnjoLib::VecD PredictVec(const EnjoLib::VecD & data) const override;
         double PredictNext(const BufferDouble & datExpanding) const override;
         unsigned GetLags() const override;
 

src/lib-base/src/PredictorARMA.cpp

@@ -16,6 +16,7 @@ PredictorARMA::PredictorARMA(const IDataProvider & dat)
 }
 PredictorARMA::~PredictorARMA(){}
 
+/*
 EnjoLib::VecD PredictorARMA::PredictVec(const EnjoLib::VecD & data) const
 {
     const int numSamplesMA  = GetLag1().GetVal();  // ParQ
@@ -27,11 +28,33 @@ EnjoLib::VecD PredictorARMA::PredictVec(const EnjoLib::VecD & data) const
 
     return predARMA;
 }
-
+*/
 double PredictorARMA::PredictNext(const BufferDouble & datExpanding) const
 {
-    const VecD & vec = PredictVec(datExpanding.GetData());
-    return vec.Last();
+    const int lags = GetLags();
+    if (datExpanding.Len() < lags + 1)
+    {
+        return IPredictor::ERROR;
+    }
+    const PredictorUtil util;
+    VecD ret;
+    for (int lag = 0; lag < lags; ++lag)
+    {
+        const EnjoLib::VecD & predAR = m_predAR.Predict(datExpanding.GetData());
+        const EnjoLib::VecD & errors = util.GetErrorsCorrected(predAR, datExpanding.GetData());
+        //const EnjoLib::VecD & predVec = PredictorUtil().RegressionProt(lag, datExpanding.GetData(), datExpanding.Len(), false);
+        //const double pred = predVec.Last();
+        const double pred = PredictorUtil().RegressionProt(lag, datExpanding.GetData(), datExpanding.Len(), false);
+        ret.Add(pred);
+        //LOGL << "Pred lag " << lag << " = " << pred << ", ret = " << ret.Mean() << Nl;
+    }
+    return ret.Mean();
+    
+    //const VecD & vec = PredictVec(datExpanding.GetData());
+    //return vec.Last();
+    
+    //const VecD & vec = PredictVec(datExpanding.GetData());
+    //return vec.Last();
 }
 
 unsigned PredictorARMA::GetLags() const