benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "45847435",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T02:57:01.430742Z",
     "start_time": "2022-04-03T02:57:00.295839Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from sktime.benchmarking.data import UEADataset, make_datasets\n",
    "from sktime.benchmarking.evaluation import Evaluator\n",
    "from sktime.benchmarking.metrics import PairwiseMetric\n",
    "from sktime.benchmarking.orchestration import Orchestrator\n",
    "from sktime.benchmarking.results import HDDResults\n",
    "from sktime.benchmarking.strategies import TSCStrategy\n",
    "from sktime.benchmarking.tasks import TSCTask\n",
    "from sktime.classification.interval_based import (\n",
    "    RandomIntervalSpectralEnsemble,\n",
    "    TimeSeriesForestClassifier\n",
    ")\n",
    "from sktime.series_as_features.model_selection import PresplitFilesCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ad64b3a9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:04:52.820479Z",
     "start_time": "2022-04-03T03:04:52.801173Z"
    }
   },
   "outputs": [],
   "source": [
    "import sktime\n",
    "\n",
    "DATA_PATH = os.path.join(os.path.dirname(sktime.__file__), \"datasets/data\")\n",
    "RESULT_PATH = \"results\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "035d6b3d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:04:53.280627Z",
     "start_time": "2022-04-03T03:04:53.269344Z"
    }
   },
   "outputs": [],
   "source": [
    "datasets = [\n",
    "    UEADataset(path=DATA_PATH, name=\"ArrowHead\"),\n",
    "    UEADataset(path=DATA_PATH, name=\"ItalyPowerDemand\")\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a924fc8d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:04:55.451028Z",
     "start_time": "2022-04-03T03:04:55.430802Z"
    }
   },
   "outputs": [],
   "source": [
    "datasets = make_datasets(\n",
    "    path=DATA_PATH, dataset_cls=UEADataset, names=[\"ArrowHead\", \"ItalyPowerDemand\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6652fc2d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:04:55.860585Z",
     "start_time": "2022-04-03T03:04:55.832338Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[TSCTask(features=None, metadata=None, target='target'),\n",
       " TSCTask(features=None, metadata=None, target='target')]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tasks = [TSCTask(target=\"target\") for _ in range(len(datasets))]\n",
    "tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ed1730a7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:04:56.560605Z",
     "start_time": "2022-04-03T03:04:56.542097Z"
    }
   },
   "outputs": [],
   "source": [
    "strategies = [\n",
    "    TSCStrategy(TimeSeriesForestClassifier(n_estimators=10), name=\"tsf\"),\n",
    "    TSCStrategy(RandomIntervalSpectralEnsemble(n_estimators=10), name=\"rise\")\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fc13044f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:04:57.210600Z",
     "start_time": "2022-04-03T03:04:57.200968Z"
    }
   },
   "outputs": [],
   "source": [
    "results = HDDResults(path=RESULT_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ff34ff40",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:06:38.400422Z",
     "start_time": "2022-04-03T03:06:32.271189Z"
    }
   },
   "outputs": [],
   "source": [
    "orchestrator = Orchestrator(\n",
    "    datasets=datasets,\n",
    "    tasks=tasks,\n",
    "    strategies=strategies,\n",
    "    cv=PresplitFilesCV(),\n",
    "    results=results\n",
    ")\n",
    "\n",
    "orchestrator.fit_predict(save_fitted_strategies=False, overwrite_predictions=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "58614895",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:08:18.140619Z",
     "start_time": "2022-04-03T03:08:17.852816Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>strategy</th>\n",
       "      <th>accuracy_mean</th>\n",
       "      <th>accuracy_stderr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rise</td>\n",
       "      <td>0.836560</td>\n",
       "      <td>0.020454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tsf</td>\n",
       "      <td>0.827677</td>\n",
       "      <td>0.020538</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  strategy  accuracy_mean  accuracy_stderr\n",
       "0     rise       0.836560         0.020454\n",
       "1      tsf       0.827677         0.020538"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator = Evaluator(results)\n",
    "metric = PairwiseMetric(func=accuracy_score, name=\"accuracy\")\n",
    "metrics_by_strategy = evaluator.evaluate(metric=metric)\n",
    "metrics_by_strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "11018aa0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:08:40.430649Z",
     "start_time": "2022-04-03T03:08:40.382414Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>strategy</th>\n",
       "      <th>accuracy_mean_rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rise</td>\n",
       "      <td>1.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tsf</td>\n",
       "      <td>1.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  strategy  accuracy_mean_rank\n",
       "0     rise                 1.5\n",
       "1      tsf                 1.5"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.rank()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "52f40e34",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:08:57.890447Z",
     "start_time": "2022-04-03T03:08:57.632433Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<Figure size 432x288 with 1 Axes>,\n",
       " <AxesSubplot:title={'center':'accuracy by strategy'}, xlabel='strategies', ylabel='accuracy'>)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEOCAYAAADc94MzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAgLElEQVR4nO3deXRU9R338c8sCRES1kagpoYQNiUiRlpEiAIRseGRJixOEmQTOGIbPaxCRCFCDAHhKBUPULD4nLBkAGWtCw1C0YhWwYBBliKbKQKBYDALDMPc5w8fpqYEHChDbsj7dU7Ombv87u97hzCf/O7c+Y3FMAxDAACYjLWqCwAAoDIEFADAlAgoAIApEVAAAFMioAAApkRAAQBMiYACTKh79+76+uuvb/hxV65cqaVLl960dsD/goACapDt27fr3LlzN60d8L+wV3UBwI3g8XiUkZGhnTt3qrS0VIZhKD09Xffff79KS0uVnp6uHTt2yGaz6ZFHHtHo0aNVVlZW6frU1FS1bNlSw4YNkyRNnDjRu9y9e3e1a9dO+/bt05gxY2S327VgwQK5XC4VFRUpPj5eo0aNkiStWrVKixcvltVqVYMGDTRjxgy9+eabatSokUaPHi1JWrt2rTZu3Kg333zzsnNatmyZ9u7dK5fLpaFDh6pfv3568cUXfWq/bNkyZWdnKyAgQLVq1dLUqVN16NAhffTRR8rNzVVQUJCKioqUl5enkydPqnXr1po4caImT56s06dPq7CwUHfccYdef/117dixo0K7AQMGaN68edq4caM8Ho/uuOMOTZkyRY0bN9aRI0f0wgsvqLi4WKGhoTIMQ71799aJEyd04MABzZ49W5L05ZdfKj09XWvWrPHTbwRuCQZwC9ixY4fx7LPPGhcvXjQMwzAWLFhgPP3004ZhGEZGRoYxevRow+12G+fPnzcGDBhgfPbZZ1dcP2HCBGPRokXeY/98uVu3bsbcuXMNwzAMj8djPPnkk8ahQ4cMwzCM48ePG3fddZdx+vRpY8+ePUbHjh2NY8eOGYZhGIsXLzZeeukl45tvvjE6d+5sXLhwwTAMw0hOTja2bt162fl069bNmDJlive4nTp1Mvbv3+9Te7fbbbRt29Y4ceKEYRiGsXr1aiM7O/uyc/nzn/9s9OzZ03ust99+21iwYIH33IYPH2689dZbl7VbvXq1MWrUKG+77OxsY/jw4YZhGMYTTzxhLF261DAMwzhw4IBx7733Gu+8845x6tQpIzo62jhz5oxhGIYxfvx4Y/ny5b/8D4sajREUbgn33Xef6tWrp+zsbH333Xf6/PPPVadOHUnSp59+qtTUVNlsNtlsNi1ZskSSlJ6eXun61atXX7WvDh06SJIsFovmz5+vLVu2aMOGDfr2229lGIbKy8u1bds2denSRU2bNpUkDRkyxNs+LCxMW7ZsUUREhE6ePKkuXbpU2k9iYqIkqXHjxurcubO2bdumQYMG/WJ7m82mxx57TImJieratau6dOmihx9+uNI+2rdvL7v9p5eBwYMH68svv9TixYt1+PBh/etf/9K99957WZvNmzfr66+/Vt++fSX9NHotLy9XcXGxdu3a5X0eIyMj9cADD0iSGjVqpK5du2rt2rWKj4/XJ598oilTplz1eQYIKNwStmzZoldeeUVDhw5VbGysmjdvrnXr1kmS7Ha7LBaLd9/vv/9eQUFBV1xvsVhk/GyKygsXLlToq3bt2pKksrIyJSQk6JFHHlGHDh3Ut29f5eTkyDAM2Wy2Csc+d+6c/v3vfysyMlIDBgzQO++8o2bNmumJJ56osN/PWa3/eYvY4/F4g8SX9rNmzdL+/fv16aef6i9/+YvWrl2rOXPmXLbfpXORpFdffVW7du1S37591bFjR7nd7grPw89rGT58uJKTkyVJLpdLxcXFstlsklShzaV1l+pOS0uT3W7Xo48+6v0DArgSbpLALSE3N1fdunVTcnKyoqKilJOTo4sXL0qSOnXqpNWrV8vj8cjlcum5557TF198ccX1DRo0UH5+viTpxIkT+uc//1lpn0eOHFFJSYlGjRql7t276/PPP5fL5ZLH41HHjh21bds2nTx5UpKUnZ2tV199VZLUs2dP7dmzRx9++KF3FFKZSyO5Y8eOadu2berUqZNP7YuKivTwww+rfv36GjJkiEaNGuW9I9Bms8ntdlfa3yeffKLBgwcrPj5ejRo10qeffup9Dn/erkuXLlq1apVKSkokSXPmzNHzzz+v4OBgRUdH691335Ukfffdd9q2bZs3QKOjo2W1WvXWW295R4fA1TCCwi0hMTFRY8eO1eOPPy63263OnTt738RPSUnRK6+8oj/84Q+6ePGi4uLi9Oijj6pLly6Vrr/nnns0btw49ezZU2FhYd7LVP+tdevW6tq1q37/+98rMDBQrVq1UosWLXTkyBHFxMRo/PjxGj58uCQpNDRUGRkZkqTAwED17NlTp06dUsOGDa94TufPn1dCQoIuXLigF198URERET61b9iwoZ555hkNGTJEQUFBstlsSk9PlyQ99NBDyszMrLS/P/3pT5o5c6bmzJmjgIAARUdH6+jRo5e1GzFihE6cOOEdvTVt2tS7bcaMGZo0aZKWLVumxo0bKywsTEFBQd4++vTpo/fee09t2rS58j8m8P9ZjMrG8AD8pqysTE8++aQmT56s9u3b3/T2/jRv3jw9+uijioyM1I8//qjevXtr4cKFatGihdxut1JSUtS7d2/FxcVVdamoBrjEB9xEH3/8sbp27aqYmJjrCpf/tb2/NWvWTKNHj1Z8fLySk5M1YsQItWjRQgcOHFCnTp3UoEEDPfbYY1VdJqoJRlAAAFNiBAUAMCUCCgBgStX+Lr5z584pPz9foaGhFT5zAQAwv4sXL6qwsFBRUVEV7viUboGAys/P14ABA6q6DADA/2Dp0qXeWVouqfYBFRoaKumnk2vSpEkVVwMAuBbHjx/XgAEDvK/lP1ftA+rSZb0mTZooLCysiqsBAFyPyt6i4SYJAIApEVAAAFMioAAApkRAAQBMiYACAJgSAQUAMCUCCgBgSgQUgBopKipKFovlmn6ioqKquuwahYACUCPl5+fLMIxKf8InbKh0fX5+flWXXaP4ZSYJj8ejtLQ07du3T4GBgUpPT1d4eLh3+5o1a/TWW28pJCRECQkJ6t+/vyQpPj5eISEhkqSwsDBNnz7dH+UBAKoBvwRUTk6OXC6XnE6n8vLylJmZqXnz5kmSioqKNGfOHK1evVp169bVkCFD1KlTJ+88TFlZWf4oCQBQzfjlEt/27dsVExMjSWrfvn2FYXFBQYHatGmj+vXry2q16p577tHOnTu1d+9elZeX66mnntKgQYOUl5fnj9IAANWEX0ZQJSUlCg4O9i7bbDa53W7Z7XaFh4frwIEDOnXqlOrUqaNt27apWbNmCgoK0rBhw9S/f38dPnxYI0aM0AcffCC7vdrPZwsAuA5+efUPDg5WaWmpd9nj8XiDpl69ekpNTdWzzz6rJk2aqG3btmrQoIEiIiIUHh4ui8WiiIgI1a9fX4WFhWratKn3OE6nU06ns0JfLpfLH6cAAKhifgmo6Ohobd68WXFxccrLy1OrVq2829xut3bu3KmlS5fK7XZr6NChGj16tFatWqX9+/crLS1NJ06cUElJyWXfD+JwOORwOCqsKygoUGxsrD9OAwBQhfwSUD169FBubq4SExNlGIYyMjK0fv16lZWVyeFwKCAgQH369FGtWrU0dOhQNWzYUP369VNqaqqSkpJksViUkZHB5T0AqMH8kgBWq1VTp06tsC4yMtL7OCUlRSkpKRW2BwYGavbs2f4oBwBQDfFBXQCAKRFQAABTIqBQqeuZp4y5ygDcSAQUKnU985QxVxmAG4mAAgCYEgEFADAlAgoAYEoEFADAlAgoAIApEVAAAFMioAAApkRAAQBMiYACAJgSAQUAMCUCCgBgSgQUAMCUCCgAgCkRUAAAUyKgAACmREABAEyJgAIAmBIBBQAwJQIKAGBKBBQAwJQIKACAKRFQAABTIqAAAKZEQAEATImAAgCYkl8CyuPxaPLkyXI4HBo4cKCOHDlSYfuaNWv0+OOPKzk5WStXrvSpDQCgZvFLQOXk5MjlcsnpdGrs2LHKzMz0bisqKtKcOXOUlZWlJUuWaP369SooKLhqGwBAzWP3x0G3b9+umJgYSVL79u2Vn5/v3VZQUKA2bdqofv36kqR77rlHO3fu1K5du67YBgBQ8/gloEpKShQcHOxdttlscrvdstvtCg8P14EDB3Tq1CnVqVNH27ZtU7Nmza7a5hKn0ymn01mhL5fL5Y9TAABUMb8EVHBwsEpLS73LHo/HGzT16tVTamqqnn32WTVp0kRt27ZVgwYNrtrmEofDIYfDUWFdQUGBYmNj/XEaAIAq5Jf3oKKjo7V161ZJUl5enlq1auXd5na7tXPnTi1dulQzZszQwYMHFR0dfdU2AICaxy8jqB49eig3N1eJiYkyDEMZGRlav369ysrK5HA4FBAQoD59+qhWrVoaOnSoGjZsWGkbAEDN5ZeAslqtmjp1aoV1kZGR3scpKSlKSUn5xTYAgJqLD+oCAEzJLyMoADCDe1/eqOLyC9fVttnEv13T/vVuC9DOKY9eV1+oHAEF4JZVXH5BhzN73ZS+rjXQ8MsIqBruev/CvJ7/jPyFCeBaEFA1HH9hAjArbpIAAJgSAQUAMCUCCgBgSgQUAMCUCCgAgCkRUAAAUyKgAACmREABAEyJgAIAmBIBBQAwJQIKAGBKBBQAwJQIKACAKRFQAABTIqAAAKZEQAEATImAAgCYEgEFADAlAgoAYEoEFADAlAgoAIApEVAAAFMioAAApmT3x0E9Ho/S0tK0b98+BQYGKj09XeHh4d7t69at0+LFi2W1WtW3b18lJydLkuLj4xUSEiJJCgsL0/Tp0/1RHgCgGvApoC5cuKCAgACfD5qTkyOXyyWn06m8vDxlZmZq3rx53u0zZ87Uhg0bVLt2bfXq1Uu9evVSUFCQJCkrK+saTwEAcCvy6RJfnz599Morr2j//v0+HXT79u2KiYmRJLVv3175+fkVtrdu3Vo//vijXC6XDMOQxWLR3r17VV5erqeeekqDBg1SXl7etZ0JAOCW4tMIau3atfr44481d+5cnTlzRr1791ZcXJzq1KlT6f4lJSUKDg72LttsNrndbtntP3XXsmVL9e3bV7fddpt69OihunXrKigoSMOGDVP//v11+PBhjRgxQh988IG3DQCgZvHp1d9qteqhhx6SJK1atUpZWVl65513lJCQIIfDcdn+wcHBKi0t9S57PB5v0Ozdu1dbtmzRpk2bVLt2bY0fP17vv/++YmNjFR4eLovFooiICNWvX1+FhYVq2rSp9zhOp1NOp7NCXy6X69rPGgBgej4F1MyZM7Vp0yb97ne/04gRI9SuXTt5PB716dOn0oCKjo7W5s2bFRcXp7y8PLVq1cq7LSQkREFBQapVq5ZsNpsaNmyos2fPatWqVdq/f7/S0tJ04sQJlZSUKDQ0tMJxHQ7HZf0VFBQoNjb2es4dAGBiPgVUs2bNtHr1atWuXVsXLlyQ9NOoau7cuZXu36NHD+Xm5ioxMVGGYSgjI0Pr169XWVmZN2SSk5MVEBCgO++8UwkJCZKk1NRUJSUlyWKxKCMjg8t7AFCD+ZQAhmHo9ddf1wsvvKCnn35avXv3Vnx8vMLCwird32q1aurUqRXWRUZGeh8nJSUpKSnpsnazZ8++ltoBALcwn+7iy87O1tixYyVJCxYs0PLly/1aFAAAPgWU1WpVrVq1JEkBAQGyWCx+LQoAAJ8u8cXGxio5OVnt2rXT7t271b17d3/XBQCo4XwKqD/+8Y/q1q2bDh06pPj4eLVp08bfdQEAajifLvEdOXJEW7du1cGDB5WTk6PJkyf7uy4AQA3nU0BNmDBBkrRjxw4VFBTohx9+8GdNAAD4dokvKChITz/9tA4fPqzp06d7Zx9H9Vc74jXd838n3qS+GkvqdVP6AlD9+fw5qMLCQpWVlamsrEzFxcX+rgs3Sdmh0TqceXNCo9nEv92UfgDcGny6xJeSkqKcnBz17t1bsbGx3nn5AADwF59GULt27dKwYcMkiXnvAAA3hU8jqH/84x+6ePGiv2sBAMDLpxHUmTNnFBMTo7CwMFksFlksFmVnZ/u7NgBADeZTQM2fP9/fdQDADcddqtWbTwG1evXqy9alpKTc8GIA4EbiLtXqzaeA+tWvfiXpp9vNv/nmG3k8Hr8WBQCATwGVmJhYYXn48OF+KQYAgEt8CqhDhw55HxcWFur777/3W0EAAEg+BtTkyZNlsVhkGIaCgoL0/PPP+7suAEAN51NALVq0SN9++63uvvtu5eTk6MEHH/R3XQCAGs6nD+qOHz9eO3fulPTT5b6JE2/ObZsAgJrLp4A6ceKEkpKSJEkjRozQyZMn/VoUAAA+BZT0nxsljh49ym3mAAC/8+k9qBdeeEGjRo3S6dOndfvtt+vll1/2d10AgBrOp4C66667NH36dO9NEm3atPF3XQCAGs6nS3zjxo3jJgkAwE3FTRIAAFO65pskjhw5wk0SAAC/8+k9qEmTJmn06NE6deqUbr/9dqWlpfm5LABATefTCGr37t0qLy9XYGCgfvjhB40bN87fdQEAajifAmrlypXKysrSww8/rOnTp6tFixZX3d/j8Wjy5MlyOBwaOHCgjhw5UmH7unXrlJCQoL59+2rZsmU+tQEA1Cw+BVSDBg10++23q7S0VB07dlRxcfFV98/JyZHL5ZLT6dTYsWOVmZlZYfvMmTO1ePFiLV++XIsXL1ZxcfEvtgEA1Cw+vQcVEhKinJwcWSwWZWdnq6io6Kr7b9++XTExMZKk9u3bKz8/v8L21q1b68cff5TdbpdhGLJYLL/YBgBQs/g0gkpPT9evf/1rjR07VocPH/7FmyRKSkoUHBzsXbbZbHK73d7lli1bqm/fvurVq5e6du2qunXr/mIbAEDN4tMIKjg4WHfffbck+fQh3eDgYJWWlnqXPR6P7Pafutq7d6+2bNmiTZs2qXbt2ho/frzef//9q7a5xOl0yul0Vljncrl8OQUAQDXjU0Bdq+joaG3evFlxcXHKy8tTq1atvNtCQkIUFBSkWrVqyWazqWHDhjp79uxV21zicDjkcDgqrCsoKFBsbKw/TgMAUIX8ElA9evRQbm6uEhMTZRiGMjIytH79epWVlXlDJjk5WQEBAbrzzjuVkJAgu91+WRsAQM3ll4CyWq2aOnVqhXWRkZHex0lJSd6pk37uv9sAAGoun6c6AgDgZiKgAACmREABAEyJgAIAmBIBBQAwJQIKAGBKBBQAwJT88jkoVC/NJv7tpvRT77aAm9IPgFsDAVXDHc7sdc1tmk3823W1A4BrwSU+AIApEVAAAFMioAAApkRAAQBMiYACAJgSAQUAMCUCCgBgSgQUAMCUCCgAgCkRUAAAUyKgAACmREABAEyJgAIAmBIBBQAwJb5uA8Atje87q74IKAC3rOv93jK+88wcuMQHADAlAgoAYEoEFADAlAgoAIAp+eUmCY/Ho7S0NO3bt0+BgYFKT09XeHi4JKmwsFBjxozx7rtnzx6NHTtWSUlJio+PV0hIiCQpLCxM06dP90d5AIBqwC8BlZOTI5fLJafTqby8PGVmZmrevHmSpNDQUGVlZUmSvvrqK7322mt64okndP78eUnybgMA1Gx+ucS3fft2xcTESJLat2+v/Pz8y/YxDEPTpk1TWlqabDab9u7dq/Lycj311FMaNGiQ8vLy/FEaAKCa8MsIqqSkRMHBwd5lm80mt9stu/0/3X300Udq2bKlmjdvLkkKCgrSsGHD1L9/fx0+fFgjRozQBx98UKGN0+mU0+ms0JfL5fLHKQAAqphfAio4OFilpaXeZY/HUyFoJGndunUaNGiQdzkiIkLh4eGyWCyKiIhQ/fr1VVhYqKZNm3r3cTgccjgcFY5TUFCg2NhYf5wGAKAK+eUSX3R0tLZu3SpJysvLU6tWrS7bZ/fu3YqOjvYur1q1SpmZmZKkEydOqKSkRKGhof4oDwBQDfhlBNWjRw/l5uYqMTFRhmEoIyND69evV1lZmRwOh4qKilSnTh1ZLBZvm379+ik1NVVJSUmyWCzKyMi4bNQFAKg5/JIAVqtVU6dOrbAuMjLS+7hhw4Zau3Zthe2BgYGaPXu2P8oBAFRDfFAXAGBKBBQAwJQIKACAKRFQAABTIqAAAKZEQAEATImAAgCYEgEFADAlAgoAYEoEFCoVFRUli8VS6c+RGf/nituioqKqunQAtwgmu0OlKvsOLwC4mRhBAQBMiYACAJgSAQUAMCUCCgBgSgQUAMCUCCgAgCkRUAAAUyKgAACmREABAEyJgAIAmBIBBQAwJQIKAGBKBBQAwJQIKACAKRFQAABTIqAAAKZEQAEATMkv36jr8XiUlpamffv2KTAwUOnp6QoPD5ckFRYWasyYMd599+zZo7Fjx8rhcFyxDQCg5vFLQOXk5MjlcsnpdCovL0+ZmZmaN2+eJCk0NFRZWVmSpK+++kqvvfaannjiiau2AQDUPH4JqO3btysmJkaS1L59e+Xn51+2j2EYmjZtmmbNmiWbzeZTGwBAzeGX96BKSkoUHBzsXbbZbHK73RX2+eijj9SyZUs1b97c5zYAgJrDLyOo4OBglZaWepc9Ho/s9opdrVu3ToMGDbqmNk6nU06ns8I6l8t1I0sHAJiEXwIqOjpamzdvVlxcnPLy8tSqVavL9tm9e7eio6OvqY3D4ZDD4aiwrqCgQLGxsTf+JAAAVcovAdWjRw/l5uYqMTFRhmEoIyND69evV1lZmRwOh4qKilSnTh1ZLJartgEAf4mKitLu3buvuN0y4/J1bdu25f3xm8hiGIZR1UX8Ly6NoDZt2qSwsLCqLgcAcA2u9hrOB3UBAKZEQAEATImAAgCYEgEFADAlAgoAYEoEFADAlPzyOaib6eLFi5Kk48ePV3ElAIBrdem1+9Jr+c9V+4AqLCyUJA0YMKCKKwEAXK/CwsLLvmKp2n9Q99y5c8rPz1doaKhsNltVl1MjjBw5UvPnz6/qMgC/4Xf85rl48aIKCwsVFRWloKCgCtuq/QgqKChIHTp0qOoyapTAwEBm7cAtjd/xm+tKX07LTRIAAFMioAAApkRAAQBMiYDCNfvv7+QCbjX8jptDtb+LDwBwa2IEBQAwJQIKAGBKBBR+UUpKSlWXAPjF+fPntXLlykq3Xbx4UcOGDVNSUpKKi4tvcmWQCCj4YO7cuVVdAuAXhYWFVwyowsJCnTlzRsuXL1e9evVucmWQboGZJOAf7777rt555x15PB4dOnRIn332mZYuXao1a9bIarUqOjpaEyZM0Pfff6+XXnpJ58+fV61atTRt2jQ1bdq0qssHfDJ//nwdOHBAc+fO1datW2W321W3bl3NmjVLL730kg4fPqzJkydr6tSpVV1qjcQICldUt25dLV++3DvH4bvvvqtJkybJ6XTqN7/5jdxut2bMmKGBAwcqKytLw4YN06xZs6q4asB3I0eOVIsWLVRaWqoePXpoyZIl6tevn86ePaspU6aoRYsWhFMVIqBwRRERERWWp0+fruzsbD355JM6duyYDMPQ/v37tWDBAg0cOFBvvvmmioqKqqha4PqNHDlSRUVFGjx4sD744APZ7VxcMgP+FXBFVmvFv19WrFihl19+WbVq1dKwYcP01VdfqXnz5nrqqacUHR2tb7/9Vl988UUVVQtcO6vVKo/Ho/Xr1yshIUETJkzQggULtGLFCsXHx1d1eTUeAQWftW7dWv369VODBg3UuHFj3XvvvZowYYLS0tJ0/vx5nTt3TpMmTarqMgGfNWrUSBcuXNCHH36od999V7Vr11ZAQACX9UyCmSQAAKbEe1AAAFMioAAApkRAAQBMiYACAJgSAQUAMCUCCvCzJUuW+Lzv1SYvvZqtW7fK6XReczvAzLjNHPCzzp07Kzc316d9CwoKNGbMGK1YscLPVQHmxwd1gRvo0KFDSk1Nld1ul81m0wMPPKDi4mKlpaWpXbt23gl4n3vuOX377bfauHGj3G63QkJC9MYbb1SYvHTw4MGaNGmSzpw5I0l68cUX1bp1a61cuVJLly5VvXr1FBAQoLi4OEnSwYMHNW7cOGVlZWnDhg2yWCyKi4vToEGDtHHjRi1cuFB2u1133HGHZs6cedlMIYDZ8BsK3ECffvqp2rZtq8WLF2vkyJGKjY1VvXr1lJaWJuk/E/B27NhRP/zwg95++20tW7ZMbrdbX3/9tXfy0pSUFM2fP18PPPCAsrKyNG3aNKWlpamoqEiLFi3S8uXL9de//lXl5eUV+j9w4IDee+89LVu2TMuWLVNOTo4OHjyoDRs2aMiQIVq+fLm6dOmikpKSKnh2gGvDCAq4gfr166eFCxdq+PDhCgkJ0ejRoytsvzQBr9VqVUBAgMaMGaPatWvr+PHjcrvdFfbdv3+/PvvsM73//vuSpLNnz+ro0aOKjIzUbbfdJkm67777Lmtz7NgxDRkyRJJUXFyso0ePKjU1VQsWLNDy5cvVvHlzPfLII/44feCGIqCAG2jTpk26//77lZKSog0bNmjRokX6+du8ly6r7d27Vzk5OVq5cqXKy8vVp08fGYbhnbxUkpo3b67evXvr8ccf1+nTp7Vy5UrdeeedOnjwoM6dO6fAwEDt2rVLzZs39x6/efPmatGihRYtWiSLxaK3335brVq1ktPp1LPPPqtGjRpp8uTJ+vvf/66EhISb++QA14iAAm6gqKgojR8/Xm+88YasVqtSU1NVUFCgcePG6cEHH/TuFx4erttuu019+vRRYGCgQkNDdfLkSd133326cOGCXn31VY0cOVKTJk3SihUrVFJSopSUFDVs2FAjRoxQcnKy6tevr/Pnz8tut3tHX23atFGnTp2UlJQkl8uldu3aqXHjxmrXrp2GDh2q+vXrq06dOuratWsVPUOA77iLD6hG3G63Fi5cqGeeeUaSNGDAAI0aNUq//e1vq7gy4MZjBAVUI3a7XeXl5UpISFBAQIDatWunDh06VHVZgF8wggIAmBK3mQMATImAAgCYEgEFADAlAgoAYEoEFADAlAgoAIAp/T/d+8ZHN8TGIQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluator.plot_boxplots()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3b6b0854",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:09:36.760135Z",
     "start_time": "2022-04-03T03:09:36.005000Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:207: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  t_df = t_df.append(t_test, ignore_index=True)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:207: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  t_df = t_df.append(t_test, ignore_index=True)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:207: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  t_df = t_df.append(t_test, ignore_index=True)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:207: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  t_df = t_df.append(t_test, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(  estimator_1 estimator_2    t_stat     p_val\n",
       " 0        rise        rise  0.000000  1.000000\n",
       " 1        rise         tsf  0.052996  0.962553\n",
       " 2         tsf        rise -0.052996  0.962553\n",
       " 3         tsf         tsf  0.000000  1.000000,\n",
       "           rise                 tsf          \n",
       "         t_stat     p_val    t_stat     p_val\n",
       " rise  0.000000  1.000000  0.052996  0.962553\n",
       " tsf  -0.052996  0.962553  0.000000  1.000000)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.t_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "1339d0e2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:10:09.740459Z",
     "start_time": "2022-04-03T03:10:09.653053Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:248: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  sign_df = sign_df.append(sign_test, ignore_index=True)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:248: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  sign_df = sign_df.append(sign_test, ignore_index=True)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:248: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  sign_df = sign_df.append(sign_test, ignore_index=True)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:248: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  sign_df = sign_df.append(sign_test, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(  estimator_1 estimator_2  p_val\n",
       " 0        rise        rise    0.5\n",
       " 1        rise         tsf    1.0\n",
       " 2         tsf        rise    1.0\n",
       " 3         tsf         tsf    0.5,\n",
       " estimator_2  rise  tsf\n",
       " estimator_1           \n",
       " rise          0.5  1.0\n",
       " tsf           1.0  0.5)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.sign_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b1c7dfaf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:10:29.680162Z",
     "start_time": "2022-04-03T03:10:29.588642Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:281: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  ranksum_df = ranksum_df.append(ranksum, ignore_index=True)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:281: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  ranksum_df = ranksum_df.append(ranksum, ignore_index=True)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:281: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  ranksum_df = ranksum_df.append(ranksum, ignore_index=True)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:281: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  ranksum_df = ranksum_df.append(ranksum, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(  estimator_1 estimator_2  t_stat  p_val\n",
       " 0        rise        rise     0.0    1.0\n",
       " 1        rise         tsf     0.0    1.0\n",
       " 2         tsf        rise     0.0    1.0\n",
       " 3         tsf         tsf     0.0    1.0,\n",
       "        rise          tsf      \n",
       "      t_stat p_val t_stat p_val\n",
       " rise    0.0   1.0    0.0   1.0\n",
       " tsf     0.0   1.0    0.0   1.0)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.ranksum_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "c2a97aa7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:10:48.060316Z",
     "start_time": "2022-04-03T03:10:47.974860Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:207: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  t_df = t_df.append(t_test, ignore_index=True)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:207: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  t_df = t_df.append(t_test, ignore_index=True)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:207: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  t_df = t_df.append(t_test, ignore_index=True)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:207: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  t_df = t_df.append(t_test, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rise</th>\n",
       "      <th>tsf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rise</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tsf</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       rise    tsf\n",
       "rise  False  False\n",
       "tsf   False  False"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.t_test_with_bonferroni_correction()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e674b4f7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:10:54.830389Z",
     "start_time": "2022-04-03T03:10:54.771053Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\evaluation.py:353: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  wilcoxon_df = wilcoxon_df.append(w_test, ignore_index=True)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>estimator_1</th>\n",
       "      <th>estimator_2</th>\n",
       "      <th>statistic</th>\n",
       "      <th>p_val</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rise</td>\n",
       "      <td>tsf</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  estimator_1 estimator_2  statistic  p_val\n",
       "0        rise         tsf        1.0    1.0"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.wilcoxon_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "55c99c05",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:11:20.900336Z",
     "start_time": "2022-04-03T03:11:20.872514Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rise</th>\n",
       "      <th>tsf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rise</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tsf</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      rise  tsf\n",
       "rise   1.0  1.0\n",
       "tsf    1.0  1.0"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.nemenyi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "9cc574c7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:17:28.050342Z",
     "start_time": "2022-04-03T03:17:28.011001Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>y_true</th>\n",
       "      <th>y_pred</th>\n",
       "      <th>fit_estimator_start_time</th>\n",
       "      <th>fit_estimator_end_time</th>\n",
       "      <th>predict_estimator_start_time</th>\n",
       "      <th>predict_estimator_end_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2022-04-03 11:06:32.340581</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.731220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2022-04-03 11:06:32.340581</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.731220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>38</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2022-04-03 11:06:32.340581</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.731220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2022-04-03 11:06:32.340581</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.731220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2022-04-03 11:06:32.340581</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.731220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170</th>\n",
       "      <td>206</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2022-04-03 11:06:32.340581</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.731220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>207</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2022-04-03 11:06:32.340581</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.731220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>208</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2022-04-03 11:06:32.340581</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.731220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>209</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2022-04-03 11:06:32.340581</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.731220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>210</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2022-04-03 11:06:32.340581</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.442372</td>\n",
       "      <td>2022-04-03 11:06:32.731220</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>175 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index  y_true  y_pred    fit_estimator_start_time      fit_estimator_end_time predict_estimator_start_time  predict_estimator_end_time\n",
       "0       36       0       0  2022-04-03 11:06:32.340581  2022-04-03 11:06:32.442372   2022-04-03 11:06:32.442372  2022-04-03 11:06:32.731220\n",
       "1       37       0       2  2022-04-03 11:06:32.340581  2022-04-03 11:06:32.442372   2022-04-03 11:06:32.442372  2022-04-03 11:06:32.731220\n",
       "2       38       0       0  2022-04-03 11:06:32.340581  2022-04-03 11:06:32.442372   2022-04-03 11:06:32.442372  2022-04-03 11:06:32.731220\n",
       "3       39       0       0  2022-04-03 11:06:32.340581  2022-04-03 11:06:32.442372   2022-04-03 11:06:32.442372  2022-04-03 11:06:32.731220\n",
       "4       40       0       0  2022-04-03 11:06:32.340581  2022-04-03 11:06:32.442372   2022-04-03 11:06:32.442372  2022-04-03 11:06:32.731220\n",
       "..     ...     ...     ...                         ...                         ...                          ...                         ...\n",
       "170    206       2       2  2022-04-03 11:06:32.340581  2022-04-03 11:06:32.442372   2022-04-03 11:06:32.442372  2022-04-03 11:06:32.731220\n",
       "171    207       2       2  2022-04-03 11:06:32.340581  2022-04-03 11:06:32.442372   2022-04-03 11:06:32.442372  2022-04-03 11:06:32.731220\n",
       "172    208       2       1  2022-04-03 11:06:32.340581  2022-04-03 11:06:32.442372   2022-04-03 11:06:32.442372  2022-04-03 11:06:32.731220\n",
       "173    209       2       2  2022-04-03 11:06:32.340581  2022-04-03 11:06:32.442372   2022-04-03 11:06:32.442372  2022-04-03 11:06:32.731220\n",
       "174    210       2       2  2022-04-03 11:06:32.340581  2022-04-03 11:06:32.442372   2022-04-03 11:06:32.442372  2022-04-03 11:06:32.731220\n",
       "\n",
       "[175 rows x 7 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"results/tsf/ArrowHead/tsf_test_0.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "8a7ef8ce",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-03T03:18:29.820474Z",
     "start_time": "2022-04-03T03:18:29.740939Z"
    }
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "No dataset found at path: .\\uea\\uea_TRAIN.ts",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Input \u001b[1;32mIn [38]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mUEADataset\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m.\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43muea\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mC:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\data.py:39\u001b[0m, in \u001b[0;36mUEADataset.__init__\u001b[1;34m(self, path, name, suffix_train, suffix_test, fmt, target_name)\u001b[0m\n\u001b[0;32m     37\u001b[0m filename \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpath, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mname, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mname)\n\u001b[0;32m     38\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_train_path \u001b[38;5;241m=\u001b[39m filename \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_suffix_train \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fmt\n\u001b[1;32m---> 39\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_path\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_train_path\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     41\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_test_path \u001b[38;5;241m=\u001b[39m filename \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_suffix_test \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fmt\n\u001b[0;32m     42\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_validate_path(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_test_path)\n",
      "File \u001b[1;32mC:\\ProgramData\\Miniconda3\\envs\\time_series\\lib\\site-packages\\sktime\\benchmarking\\base.py:45\u001b[0m, in \u001b[0;36mHDDBaseDataset._validate_path\u001b[1;34m(path)\u001b[0m\n\u001b[0;32m     43\u001b[0m \u001b[38;5;66;03m# check if path already exists\u001b[39;00m\n\u001b[0;32m     44\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(path):\n\u001b[1;32m---> 45\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNo dataset found at path: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[1;31mValueError\u001b[0m: No dataset found at path: .\\uea\\uea_TRAIN.ts"
     ]
    }
   ],
   "source": [
    "UEADataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b4ac5ac",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:time_series] *",
   "language": "python",
   "name": "conda-env-time_series-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.11"
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}