runtime.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Runtime measurements\n",
    "\n",
    "## Preparations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import joblib\n",
    "import random\n",
    "import glob\n",
    "import yaml\n",
    "import timeit\n",
    "import math\n",
    "\n",
    "from sklearn.base import BaseEstimator\n",
    "from sklearn.model_selection import KFold, train_test_split, cross_val_score, GridSearchCV\n",
    "from sklearn.linear_model import LinearRegression, Ridge\n",
    "from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.preprocessing import LabelEncoder, StandardScaler, PolynomialFeatures\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "\n",
    "sns.set(font_scale=1.1, style='white')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FixedModel(BaseEstimator):\n",
    "    \"\"\"\n",
    "    sklearn model of the fixed model, \n",
    "    always allocating/predicting a fixed, pre-defined amount of resources.\n",
    "    For use in sklearn functions like k-fold CV\n",
    "    \"\"\"\n",
    "    def __init__(self, fixed_value):\n",
    "        self.fixed_value = fixed_value\n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        \"\"\"There's nothing to fit here\"\"\"\n",
    "        return self\n",
    "    \n",
    "    def predict(self, X):\n",
    "        \"\"\"Always 'predict' the specified fixed value\"\"\"\n",
    "        n_samples = X.shape[0]\n",
    "        return [self.fixed_value for _ in range(n_samples)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function for processing and simplifying the dataset\n",
    "def select_and_rename(df, mapping):\n",
    "    \"\"\"\n",
    "    Helper: Selects columns of df using the keys of the mapping dict.\n",
    "    It renames the columns to the values of the mappings dict.\n",
    "    \"\"\"\n",
    "    # select subset of columns\n",
    "    dff = df[list(mapping.keys())]\n",
    "    # rename \n",
    "    for k, v in mapping.items():\n",
    "        dff.rename(columns={k: v}, inplace=True)\n",
    "    return dff\n",
    "\n",
    "def replaceSize(df):\n",
    "    df[\"size\"] = df[\"size\"].str.replace(\"ab -c 1 -t 60 -n 99999999 -e /tngbench_share/ab_dist.csv -s 60 -k -i http://20.0.0.254:8888/\", \"small\")\n",
    "    df[\"size\"] = df[\"size\"].str.replace(\"ab -c 1 -t 60 -n 99999999 -e /tngbench_share/ab_dist.csv -s 60 -k http://20.0.0.254:8888/bunny.mp4\", \"big\")\n",
    "    df[\"size\"] = df[\"size\"].str.replace(\"ab -c 1 -t 60 -n 99999999 -e /tngbench_share/ab_dist.csv -s 60 -k -i -X 20.0.0.254:3128 http://40.0.0.254:80/\", \"small\")\n",
    "    df[\"size\"] = df[\"size\"].str.replace(\"ab -c 1 -t 60 -n 99999999 -e /tngbench_share/ab_dist.csv -s 60 -k -X 20.0.0.254:3128 http://40.0.0.254:80/bunny.mp4\", \"big\")\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\stefan\\git-repos\\work\\ai\\venv\\lib\\site-packages\\pandas\\core\\frame.py:4133: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    }
   ],
   "source": [
    "# Load data from path\n",
    "web1 = pd.read_csv(\"vnf_data/csv_experiments_WEB1.csv\")\n",
    "web3 = pd.read_csv(\"vnf_data/csv_experiments_WEB3.csv\")\n",
    "\n",
    "# do processing, renaming and selection\n",
    "mapping = {\n",
    "    \"param__func__mp.input__cmd_start\": \"size\",\n",
    "    \"metric__mp.input.vdu01.0__ab_transfer_rate_kbyte_per_second\": \"Max. throughput [kB/s]\",\n",
    "}\n",
    "\n",
    "mapping01 = mapping.copy()\n",
    "mapping01[\"param__func__de.upb.lb-nginx.0.1__cpu_bw\"] = \"CPU\"\n",
    "mapping01[\"param__func__de.upb.lb-nginx.0.1__mem_max\"] = \"Memory\"\n",
    "\n",
    "mapping03 = mapping.copy()\n",
    "mapping03[\"param__func__de.upb.px-squid.0.1__cpu_bw\"] = \"CPU\"\n",
    "mapping03[\"param__func__de.upb.px-squid.0.1__mem_max\"] = \"Memory\"\n",
    "\n",
    "web1 = select_and_rename(web1, mapping01)\n",
    "web3 = select_and_rename(web3, mapping03)\n",
    "\n",
    "web1 = replaceSize(web1)\n",
    "web3 = replaceSize(web3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select sub-datasets with small and large flows\n",
    "# and with specific memory\n",
    "mem = 128\n",
    "\n",
    "web1_small = web1.loc[(web1[\"size\"] == \"small\") & (web1[\"Memory\"] == mem)]\n",
    "web1_small = web1_small[[\"Max. throughput [kB/s]\", \"CPU\"]]\n",
    "web3_small = web3.loc[(web3[\"size\"] == \"small\")  & (web3[\"Memory\"] == mem)]\n",
    "web3_small = web3_small[[\"Max. throughput [kB/s]\", \"CPU\"]]\n",
    "\n",
    "# add 20 \"measurements\" at 0 CPU and throuhgput\n",
    "num_measures = 20\n",
    "measures = [0 for _ in range(num_measures)]\n",
    "web1_small = web1_small.append(pd.DataFrame({'Max. throughput [kB/s]': measures, 'CPU': measures}), ignore_index=True)\n",
    "web3_small = web3_small.append(pd.DataFrame({'Max. throughput [kB/s]': measures, 'CPU': measures}), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_data(data):\n",
    "    \"\"\"Split return data\"\"\"\n",
    "    X = data[['Max. throughput [kB/s]']]\n",
    "    y = data['CPU']\n",
    "    X = X.fillna(X.median())\n",
    "\n",
    "    scaler = MinMaxScaler()\n",
    "    scaler.fit_transform(X)\n",
    "    \n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "def barplot(times, labels, ylabel='Time [s]', filename=None):\n",
    "    assert len(times) == len(labels)\n",
    "    \n",
    "    times_mean = [np.array(t).mean() for t in times]\n",
    "    print(times_mean)\n",
    "    times_std = [np.array(t).std() for t in times]\n",
    "    x = np.arange(len(labels))\n",
    "    \n",
    "    sns.set(font_scale=1.1, style='white')\n",
    "    fig, ax = plt.subplots(figsize=(8, 4))\n",
    "    plt.bar(x, times_mean, yerr=times_std, capsize=5, color='gray')\n",
    "    \n",
    "    ax.set_xticks(x)\n",
    "    ax.set_xticklabels(labels)\n",
    "    ax.set_ylabel(ylabel)\n",
    "    \n",
    "    if filename is not None:\n",
    "        fig.savefig(f'plots/{filename}.pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare data\n",
    "X_nginx, y_nginx = prepare_data(web1_small)\n",
    "X_squid, y_squid = prepare_data(web3_small)\n",
    "\n",
    "X = X_nginx\n",
    "y = y_nginx\n",
    "vnf_name = 'nginx'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate synthetic data\n",
    "def gen_benchmark(cpu, coeff1=1, coeff2=1):\n",
    "    \"\"\"Generate max throughput for given CPU time according to log function\"\"\"\n",
    "    return coeff1 * math.log2(1 + coeff2 * cpu)\n",
    "\n",
    "def synthetic_benchmark(n):\n",
    "    \"\"\"Generate n synthetic benchmark results for CPU time 0-100%\"\"\"\n",
    "    cpu_list = [random.random() for _ in range(n)]\n",
    "    thr_list = [gen_benchmark(cpu, coeff2=100) for cpu in cpu_list]\n",
    "    X = pd.DataFrame(data={'Throughput': thr_list})\n",
    "    y = np.array(cpu_list)\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_times(models, X, y, scaler):\n",
    "    # measure training times\n",
    "    times = []\n",
    "    for model in models:\n",
    "        print(type(model).__name__)\n",
    "        X_scaled = scaler.transform(X)\n",
    "        t = %timeit -o model.fit(X, y)\n",
    "        # save the best time, which is the most comparable and reproducible\n",
    "        times.append(t)\n",
    "    return times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = ['Linear', 'Ridge', 'SVR', 'Forest', 'Boosting', 'MLP', 'Fixed']\n",
    "models = [LinearRegression(), Ridge(), SVR(verbose=True), RandomForestRegressor(n_estimators=10),\n",
    "          GradientBoostingRegressor(), MLPRegressor(max_iter=1500), \n",
    "          FixedModel(fixed_value=0.8)]\n",
    "model_names = [type(model).__name__ for model in models]\n",
    "models = [joblib.load(f'ml_models/{vnf_name}/{name}.joblib') for name in model_names]\n",
    "scaler = joblib.load(f'ml_models/{vnf_name}/scaler.joblib')\n",
    "\n",
    "# X_rand = pd.DataFrame(data={'Rand max. throughput': [random.randrange(0, 3000) for _ in range(100000)]})\n",
    "# y_rand = np.array([random.random() for _ in range(100000)])\n",
    "\n",
    "# X_synth, y_synth = synthetic_benchmark(200000)\n",
    "\n",
    "# times = training_times(models, X_synth, y_synth, scaler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # plot\n",
    "# # timeit runs 7 runs with variable number of loops; calc time per loop for each run of each model\n",
    "# all_times = [[i / t.loops for i in t.all_runs] for t in times]\n",
    "# barplot(all_times, labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load trained models and scaler\n",
    "# model_names = [type(model).__name__ for model in models]\n",
    "models = [joblib.load(f'ml_models/{vnf_name}/{name}.joblib') for name in model_names]\n",
    "scaler = joblib.load(f'ml_models/{vnf_name}/scaler.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearRegression\n",
      "The slowest run took 4.49 times longer than the fastest. This could mean that an intermediate result is being cached.\n",
      "83.2 µs ± 63.5 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n",
      "Ridge\n",
      "64.5 µs ± 17.2 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n",
      "SVR\n",
      "86.2 µs ± 19 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n",
      "RandomForestRegressor\n",
      "7.29 ms ± 170 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n",
      "GradientBoostingRegressor\n",
      "123 µs ± 1.98 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n",
      "MLPRegressor\n",
      "115 µs ± 5.04 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n",
      "FixedModel\n",
      "1.13 µs ± 135 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)\n"
     ]
    }
   ],
   "source": [
    "# time single prediction for rand input\n",
    "X_rand = pd.DataFrame(data={'Rand max. throughput': [random.randrange(0, 3000) for _ in range(1)]})\n",
    "times = []\n",
    "for model in models:\n",
    "    print(type(model).__name__)\n",
    "    X_scaled = scaler.transform(X_rand)\n",
    "    t = %timeit -o model.predict(X_scaled)\n",
    "    # save the best time, which is the most comparable and reproducible\n",
    "    times.append(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.041868121428571416, 0.04201067571428731, 0.059522719999998634, 0.8912386999999821, 0.10723779999999741, 0.06644656000000201, 0.0009462411285714519]\n"
     ]
    },
    {
     "data": {
      "text/plain": "<Figure size 576x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAEDCAYAAAA/VN3hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XtYVHX+B/D3wDDOKMhFEBVCyTbcFAUEgRElzQvCz0uSrGiirYo3vC15C4kkTTPXMIPU7OJts9wsJF3RSK3HREUxfrZiLq7zAxGDQUMQGC7z+8PHszuheMg5XDrv1/PwPJzvfM+cz1fG857vOWfOKIxGoxFERET0u2fR0gUQERFR82DoExERyQRDn4iISCYY+kRERDLB0CciIpIJhj4REZFMMPSJiIhkgqFPREQkEwx9IiIimWDoExERyYSypQswh6qqKly8eBFOTk6wtLRs6XKIiIgkVVdXh+LiYvTp0wdqtVr0er+L0L948SImT57c0mUQERE1qz179sDX11d0/99F6Ds5OQG4N/guXbq0cDVERETSKioqwuTJk4X8E+t3Efr3D+l36dIFrq6uLVwNERFR82jqKW1eyEdERCQTDH0iIiKZYOgTERHJBEOfiIhIJhj6REREMsHQJyIikgmGPhERkUww9ImIiGTid3FzHiJ6uNraWiiV5vmvHh4ejszMzEf2CwgIwOeff26WbZqzfiK54/8kot85pVKJVatWmeW5+vbti759+4rqa65tJiQkmOV5iIiH94mIiGSDoU9ERCQTDH0iIiKZYOgTERHJBEOfiIhIJhj6REREMsHQJyIikgmGPhERkUww9ImIiGSCoU9ERCQTDH0iIiKZYOgTERHJBEOfiIhIJhj6REREMsHQJyIikolmCf2srCyEhYXBy8sLc+fORXl5eYM+ubm5CA8Ph4+PD8aPH4+cnJzmKI2IiEg2JA/9yspKLFiwAEuXLkVmZibUajWSk5Mb9Fu6dCmmTJmCc+fOISIiAkuWLJG6NCIiIlmRPPQzMzPh4uKC4OBgqNVqxMTEIDU1tUG//Px8GI1GGI1GWFhYoF27dlKXRkREJCtKqTeg0+nQo0cPYdnNzQ16vR63b9+GnZ2d0P7SSy9hxYoViIuLQ7t27bBjxw6pSyMiIpIVyWf6FRUV0Gg0wrJSqYSVlRWqqqpM+imVSmzevBkXLlzAyy+/jEWLFqG6ulrq8oiIiGRD8tDXaDQm4V1bW4uamhqTNwI5OTn4+uuvMXz4cKhUKkyePBkqlQpnz56VujwiIiLZkDz03d3dodPphGWdTgd7e3vY2toKbTdu3EBNTY3JekqlEkql5GcfiIiIZEPy0A8ICIBOp0NGRgaqqqqQkpKCUaNGmfTx8vJCQUEB9u/fj/r6enz55ZcoKytD3759pS6PiIhINprl8H5ycjKSkpKg1WpRXV2N2NhYFBYWwtvbG4WFhXB2dkZycjJ27NgBPz8/fPLJJ9i6dSvat28vdXlERESy0SzHz318fJCWlmbSZm1tjezsbGFZq9U+8KN8REREZB68DS8REZFMMPSJiIhkgqFPREQkEwx9IiIimWDoExERyQRDn4iISCYY+kRERDLB0CciIpIJhj4REZFMMPSJiIhkgqFPREQkEwx9IiIimWDoExERyQRDn4iISCYY+kRERDKhbOzBnTt3NunJoqKiHqsYIiIikk6job9u3Tr0799f1BOdP3+eoU9ERNSKNRr67dq1w65du0Q9kbe3t1kKIiIiImk0ek7/wIEDD2yvq6sT3ZeIiIhah0ZD/4knngAA5OfnIz4+HgBw7Ngx+Pj4ICgoCBcuXGjQl4iIiFonUVfvv/baa6irq4PRaMT69esxb948xMTEYPXq1VLXR0RERGbS6Dn9+3Jzc7Ft2zYUFBTg//7v/zBp0iR06NAB69evl7o+IiIiMhPRn9OvrKzE8ePH4enpCWtraxQVFaF9+/ZS1kZERERmJGqmHxYWhgkTJqC4uBjx8fG4cuUKYmJiMG7cOKnrIyIiIjMRFforVqyAr68vbGxsEBgYiIKCAsycORPh4eFS10dERERmIurwvkKhQGBgIDp16oTc3FyUl5ejT58+uHz5stT1ERERkZmImul//PHHeOuttxp8Pl+hUODSpUuSFEZERETmJSr0t2zZgs2bN2PIkCFQKBRS10REREQSEHV4X6VSYdCgQQx8IiKiNkzUTH/27NlISEjA1KlTYWNjY/JYt27dJCmMiIiIzEtU6NfX1yM1NRX79+8HcO9cvtFo5Dl9IiKiNkRU6G/evBkbNmxAYGAgLC0tpa6JiIiIJCAq9K2srDB8+HAolaK6ExERUSsk6kK+2bNnIz4+Hvn5+bhz5w7Ky8uFHyIiImobRE3dN23ahDt37uDLL78U2nhOn4iIqG0RFfr/HfZERETUNjV6eP/+vfVdXFwe+vPrvkRERNQ6NTrT/+mnn7Br1y4YjcZHPtGVK1ce+lhWVhYSEhJw/fp1aLVarF+/HtbW1iZ9SktLkZCQgDNnzsDOzg4rV67EoEGDRA6DiIiIHqXR0Pfy8sKRI0dEPVG/fv0e2F5ZWYkFCxZg7dq18Pf3xyuvvILk5GQsW7bMpN/ixYvRq1cvnDx5EqdPn8bChQtx8uRJtGvXTuRQiIiIqDGNhv6uXbseewOZmZlwcXFBcHAwACAmJgYvvviiSegXFhYiNzcX27dvh1KpxMCBA7F7927e9peIiMiMRH1k73HodDr06NFDWHZzc4Ner8ft27eFttzcXLi7u2Pjxo3QarUYN24cKioqoFKppC6PiIhINiQP/YqKCmg0GmFZqVTCysoKVVVVQtudO3eQk5MDe3t7HD9+HNHR0YiJicGdO3ekLo+IiEg2JA99jUaD6upqYbm2thY1NTUmbwSsrKzQvn17zJw5EyqVCqGhoXB0dEROTo7U5REREclGk0K/pqYGOp0ORqNR1BX9AODu7g6dTics63Q62Nvbw9bWVmjr0aMHqqqqUFNTI7TV19eL3gYRERE9mqjQr6qqQlxcHPr164exY8fi6tWrCAkJQX5+/iPXDQgIgE6nQ0ZGBqqqqpCSkoJRo0aZ9PnjH/8INzc3vP3226irq8PBgwdRWloKX1/f3zYqIiIiakBU6K9duxZlZWX46quvYGVlBTc3NwQFBWHVqlWPXFej0SA5ORlJSUnQarWorq5GbGwsCgsL4e3tjcLCQigUCrz//vv46aef4O/vj/feew/vvvsu1Gr1Yw+QiIiI7hF1G96MjAwcPnwY1tbWUCgUsLKywrJlyxAUFCRqIz4+PkhLSzNps7a2RnZ2trDs4uKCDz74oAmlExERUVOImukrlUoYDAYAEM6z37171+RiPCIiImrdRIX+yJEjsXDhQuFq+mvXriEuLg7Dhw+XtDgiIiIyH1GhHxsbCw8PD0RFReHOnTsYPXo07OzssHjxYqnrIyIiIjMRdU5fpVJh5cqVWLlyJUpLS2FnZwcLC8k/4k9ERERmJCr07969iy+++AIFBQWor683eWzFihWSFEZERETmJSr0Fy1aBJ1OBy8vL87wiYiI2ihRoZ+VlYXjx4+jY8eOUtdDREREEhE1be/ZsyfKysqkroWIiIgkJGqmHx8fj6lTp2LYsGGwsbExeSwmJkaSwoiIiMi8RIV+SkoKamtrodPpYGlpKbQrFArJCiMiIiLzEhX6p0+fxokTJ3hOn4iIqA0TdU6/R48eKC8vl7oWIiIikpComf7IkSMRGRmJ8ePHw87OzuSwflRUlGTFERERkfmICv2TJ0/Czc0NWVlZJu0KhYKhT0RE1EaICv1du3ZJXQcRERFJrNHQ/+STTxAZGYmdO3c+tA9n+kRERG1Do6H/zTffIDIyEkePHn3g4zy8T0RE1HY0Gvrvv/8+AGDjxo1wcnJq8PiVK1ekqYqIiIjMTtRH9kaOHNmgzWAwICIiwuwFERERkTQeOtMvKChAREQEamtrcffuXQwYMMDkcYPBgF69ekleIBEREZnHQ0Pf1dUVH374IX755RdER0fj3XffNXlcpVLBw8ND8gKJiIjIPBo9p39/Jn/s2DE4ODg0S0FEREQkDVHn9Bn4REREbZ+o0CciIqK2j6FPREQkE6Juw3vf3bt3UV9fb9JmbW1t1oKIiIhIGqJC/+uvv8aqVatQUlIitBmNRigUCly6dEmy4oiIiMh8RIV+UlISIiIiMGbMGCiVTTo4QERERK2EqAS/fv065s6dC0tLS6nrISIiIomIupAvMDAQJ0+elLoWIiIikpComb5Go8HcuXPRp08fODo6mjz26zv1ERERUeskKvTd3d0xe/ZsqWshIiIiCYkK/ZiYGOF3vV6Pjh07wsrKSrKiiIiIyPxEndOvrq5GQkIC+vXrh6CgIPj4+GDJkiWorKyUuj4iIiIyE1Ghv2HDBuTl5WHnzp347rvvsGPHDty4cQNvvfWW1PURERGRmYgK/SNHjuCdd95Bv3794OjoCB8fHyQlJeHQoUNS10dERERmIir0DQYD1Gq1SZtarYaFhbhb92dlZSEsLAxeXl6YO3cuysvLH9r36tWr8PT0RHFxsajnJiIiInFEpXZQUBBWrlyJ0tJSAPcu5ktISIBWq33kupWVlViwYAGWLl2KzMxMqNVqJCcnP7BvfX09Vq5cCYPB0IQhEBERkRiiQn/58uUoLCyEVqsVLua7desWVqxY8ch1MzMz4eLiguDgYKjVasTExCA1NfWBfXfu3IlnnnmmaSMgIiIiUUR9ZK9Tp07Yu3cv8vPzodfr0bVrVzg7O4vagE6nQ48ePYRlNzc36PV63L59G3Z2dkJ7fn4+9u3bh88++wy7du1q2iiIiIjokRoN/U8++QSRkZHYuXOnSXtOTo7we1RUVKMbqKiogEaj+c8GlUpYWVmhqqpKaDMajVi5ciWWLVuGDh06NGkAREREJE6jof/NN98gMjISR48efeDjCoXikaGv0WhQXV0tLNfW1qKmpsbkjcCnn36Kzp07Y/DgwU2pnYiIiJqg0dB///33AQAbN26Ek5NTg8evXLnyyA24u7vjyJEjwrJOp4O9vT1sbW2FtqNHjyI7Oxu+vr5CW0hICLZu3WrSRkRERL+dqAv5Ro4c2aDNYDAgIiLikesGBARAp9MhIyMDVVVVSElJwahRo0z6fPDBBzh//jyysrKQlZUFADh8+DADn4iIyIweOtMvKChAREQEamtrcffuXQwYMMDkcYPBgF69ej1yAxqNBsnJyUhISMCSJUug1WoRGxuLwsJChIWF4eDBg+jWrdvjj4SIiIga9dDQd3V1xYcffohffvkF0dHRDb5CV6VSwcPDQ9RGfHx8kJaWZtJmbW2N7OzsB/a/fPmyqOclIiIi8Ro9p39/Jn/s2DEYDAbY2tpCo9EgNzcXNjY2JhfjERERUesm6pz+uXPnEBISgn//+98A7t1wZ+zYsThx4oSkxREREZH5iLo5z8aNG/Hee+8Jd8ubNm0aPDw8sGbNGgQHB0taIBEREZmHqJl+UVER/P39Tdr8/f1RVFQkSVFERERkfqJC38PDA5999plJ2xdffIGnnnpKkqKIiIjI/EQd3l+2bBmio6OxZ88edO3aFUVFRbh586Zw8x4iIiJq/USFvre3N9LT03H8+HGUlJTA2dkZwcHBJl+YQ0RERK1bo6FfXl4Oa2trlJeXQ6VSYcSIEQ98nIiIiFq/RkN/8ODBOH/+PHx9faFQKEweMxqNUCgUuHTpkqQFEhERkXk0GvoHDx4EAGRkZDRLMURERCSdRkPfxsYG5eXlJt+IR0RERG1To6H/oMP6v8bD+0RERG1Do6F//7D+/v378eOPP2LRokVwdXVFUVERkpKS8Mc//rFZiiQiIqLH12jou7i4AAA+/fRTpKeno0OHDgCAp556Cm+++SaGDRuGefPmSV8lERERPTZRd+Srra3FrVu3TNqKiopgYSFqdSIiImoFRN2cJyIiAtOmTcOLL74IZ2dnFBYWYvfu3Zg6darU9REREZGZiAr9xYsXw9nZGf/4xz9QXFyMzp07Y+HChRg3bpzU9REREZGZiAp9hUKByZMnY/LkyTAYDFCpVFLXRURERGYm6qR8fX093nvvPQwePBj+/v7Iz89HVFQUSktLpa6PiIiIzERU6G/atAnffvst1qxZA6VSCQcHB9ja2iIxMVHq+oiIiMhMRB3eP3DgAPbt2wdHR0coFAp06NABa9euxXPPPSd1fURERGQmomb6NTU1aN++PYB7X7QDABYWFlAqRb1nICIiolZAVOgPGjQIcXFxKC0thUKhgMFgwPr16xEYGCh1fURERGQmokJ/xYoVqKyshFarRVlZGby9vaHT6fDKK69IXR8RERGZiajj80VFRdiyZQv0ej2uX78OZ2dnODs7S10bERERmZGomX5UVBQqKirQqVMn9O3bl4FPRETUBokKfXd3d1y4cEHqWoiIiEhCoi+/nzFjBuzt7dG5c2coFAqh/YsvvpCkMCIiIjIv0V+4ExERIXUtREREJKFHhr7RaMSQIUNgZ2fXHPUQERGRRBo9p3/lyhUMGTIEgYGBGDNmDHQ6XXPVRURERGbWaOi/+eabCAsLQ1paGry8vPDmm282V11ERERkZo0e3r9w4QK2bt0KS0tLxMbGIiwsrLnqIiIiIjNrdKZvNBphaWkJALC1tYXBYGiWooiIiMj8Hhn6RERE9PvQ6OF9o9GIy5cvC+FfV1dnsgwAvXr1krZCIiIiMotGQ7+yshLjxo0zCfmxY8cKvysUCly6dEm66oiIiMhsGg393Nxcs2wkKysLCQkJuH79OrRaLdavXw9ra2uTPufPn8fq1atx7do1dO/eHfHx8fDx8THL9omIiEjkvfcfR2VlJRYsWIClS5ciMzMTarUaycnJJn2qqqoQExODWbNmISsrCzNnzsS8efNQVVUldXlERESyIXnoZ2ZmwsXFBcHBwVCr1YiJiUFqaqpJn59//hlBQUEYOXIkLCwsEBoaivr6euTn50tdHhERkWxIHvo6nQ49evQQlt3c3KDX63H79m2TtvXr1wvLOTk5qK6uxhNPPCF1eURERLIheehXVFRAo9EIy0qlElZWVg89dH/jxg0sXLgQixYtglqtlro8IiIi2ZA89DUaDaqrq4Xl2tpa1NTUmLwRuC8vLw+RkZEYM2YMpk2bJnVpREREsiJ56Lu7u5t8UY9Op4O9vT1sbW1N+v3444+YPHkypk6disWLF0tdFhERkexIHvoBAQHQ6XTIyMhAVVUVUlJSMGrUKJM+ZWVlmDVrFubPn4+XXnpJ6pKIiIhkqVkO7ycnJyMpKQlarRbV1dWIjY1FYWEhvL29UVhYiPT0dBQXF2PDhg3w9vYWfnJycqQuj4iISDYavTmPufj4+CAtLc2kzdraGtnZ2QCACRMmYMKECc1RChERkWxJPtMnIiKi1oGhT0REJBMMfSIiIplg6BMREckEQ5+IiEgmGPpEREQywdAnIiKSCYY+ERGRTDD0iYiIZIKhT0REJBMMfSIiIplg6BMREckEQ5+IiEgmGPpEREQywdAnIiKSCWVLF0BEZE61tbVQKs2zawsPD0dmZuYj+wUEBODzzz83yzbNWT/Rr/GVRUS/K0qlEqtWrTLLc/Xt2xd9+/YV1ddc20xISDDL8xA9CA/vExERyQRDn4iISCYY+kRERDLB0CciIpIJhj4REZFMMPSJiIhkgqFPREQkE/ycPhFRG9aWb0bEGxE1P/5rExG1YW35ZkS8EVHzY+iTLJlrhsHbtBJRW8I9B8mSuWZHvE0rEbUlvJCPiIhIJhj6REREMsHD+w/Rlq+IBZpev9zGS0QkR9xLPkRbviIWaPp5X7mNl4hIjnh4n4iISCYY+kRERDLB0CciIpIJhj4REZFMNEvoZ2VlISwsDF5eXpg7dy7Ky8sb9PnXv/6FF154AV5eXnjxxRdRVFTUHKURERHJhuShX1lZiQULFmDp0qXIzMyEWq1GcnKySR+j0YhFixbhhRdewJkzZ+Dp6Yk33nhD6tKIiIhkRfLQz8zMhIuLC4KDg6FWqxETE4PU1FSTPleuXMGtW7cwceJEqFQqzJ8/H8ePH3/gEQEiIiL6bST/nL5Op0OPHj2EZTc3N+j1ety+fRt2dnZCn+7duwt92rdvD3t7e+h0OvTu3fuR26irqwMAs58SaMtvOgoKCpq8DsfbNshprADHK0ZbHe9vGSvdcz/v7uefWJKHfkVFBTQazX82qFTCysoKVVVVD+0DAGq1GpWVlaK2UVxcDACYPHmyGSr+fThw4EBLl9Cs5DReOY0V4Hh/z+Q0VqkUFxebTJofRfLQ12g0qK6uFpZra2tRU1NjEvK/7gMAVVVVaN++vaht9OnTB3v27IGTkxMsLS3NUzgREVErVVdXh+LiYvTp06dJ60ke+u7u7jhy5IiwrNPpYG9vD1tbW5M+Op1OWL579y70ej3c3NxEbUOtVsPX19d8RRMREbVyTZnh3yf5hXwBAQHQ6XTIyMhAVVUVUlJSMGrUKJM+Tz/9NGxsbLBnzx4YDAa8++670Gq1sLa2lro8IiIi2ZA89DUaDZKTk5GUlAStVovq6mrExsaisLAQ3t7eKCwsBABs3rwZBw4cgL+/Py5evIjXX39d6tKIiIhkRWE0Go0tXQQRERFJj7fhJSIikgmGPhERkUww9ImIiGSCoU9ELeb+hbxE1DwY+r9RQUEBPD09G7S/+uqr+Oijj1qgIul5eHjAy8sL3t7ewk9UVJSw496yZQvWrFnzwHWHDh2KCxcuNGe5ZpeXl4cZM2bAx8cH/fv3x8yZM5GXl4fY2FgkJCQ06H/q1CkMHjwY9fX1mDJlCjw9PYV/Ny8vL4SFheHUqVMtMBJTD/u7Sq2kpATPP/+85NtpDm3pzUtBQQE8PDwwZcqUBo+9/vrr8PDweOj+Dbj3SavevXubvF4mTpyIixcvSl26aI29psPCwpCTk2OW7WzevBmvvvqqWZ6ruUh+cx65SUxMbOkSJHX06FE4OTkBuHe/77i4OKxbtw7vvPMOZs+e3cLVSaeurg4zZ87ErFmzsGXLFtTX1+Ojjz7CzJkz8frrryM2Nhbx8fFQKv/zXyotLQ1jx46FhcW999br1q1DWFgYgHt3ptyxYwdefvllfPvtty1+J8n//rs2l6qqKty9e1eS5y4oKMBzzz1nclfP3r17Y/Xq1SbfBWIOu3fvxk8//YTExEQcOHAA6enpDb5JtLVRKpXIzc2FXq9Hp06dANz7ttMTJ06Iei2Gh4cL+zqj0Yi9e/dizpw5OHbsmMn/gZb0sNf0wYMHW6Ca1oMzfTNbvnw5tm3bBgB45pln8OGHHyIwMBDBwcH44osvhH7Z2dkYP348/Pz8EB0djZs3bwK4F6RLlizBkCFD0LdvX0RFRaGkpAQAMGXKFCxbtgz+/v7YsmVL8w/uV6ytrTF69Gjk5eUBMH3Xm5eXhxdeeAHe3t6Ij483+VKIb775BsOGDcPAgQOxdetWPPPMM8Jj+/btw7Bhw6DVarF69WoYDIbmHdRD3Lp1C4WFhQgNDYVSqYRKpUJ0dDSCgoLg4eGBdu3a4fvvvxf6GwwGHD169KEzWaVSiYiICJSUlOCXX35prmE02fXr1zFz5kz4+voiNDQUGRkZAO6F6pAhQxAdHQ1/f3/odDpcvXoVU6ZMgZ+fHyZNmiS8LsrLyzFnzhz4+flh+PDh+PDDDwEAU6dOhcFggLe3tyR/Z5VKhezsbGRnZ+Ps2bPw9PREfHy82bdz69Yt4fcxY8a0+sAHAAsLCwwaNEj4ewLAuXPn4OHh0eQ3oAqFAuPGjcPPP/+MsrIyc5dqdvePOu7evRsjRoyAwWBAfX09/vSnPwn77oftn2tqapCQkID+/ftj9OjRuHbtWguO5Ldh6Euorq4OV69exYkTJ7B8+XIkJiaiuroaZWVlmDNnDmJiYvD999/Dy8sLy5YtAwC8//77MBgMSE9Px6lTp6BQKLB3717hOYuKinD8+HG8+OKLLTUswa1bt/DJJ5888BbIixYtwogRI3DmzBn06NFD+Eao4uJivPzyy1i1ahUyMjJw6dIl4Q3B2bNn8e6772Lbtm04cuQICgoKsH379mYd08M4OjrC09NT2DHk5OSgvr4eiYmJcHR0xNixY3Ho0CGh/zfffIOePXviySeffODzGQwGbN++HT179oSDg0NzDaPJlixZAg8PD3z//fdITEzEsmXLhDAvLCzEc889h4yMDLi4uGDOnDkIDQ3FqVOnMGHCBMybNw91dXXYu3cv2rdvj8zMTGzfvh1bt27FzZs3sWPHDiGYVSqVpONQKpUYM2YMcnNzAQDp6ekICQmBn58fZs+eLezUjUYjNm3ahKCgIAQFBZm88Tx9+jTCwsLg5+eHiRMn4scff8T333+PrVu34vPPP8fy5cuxf/9+TJ8+HcC9N+mbN2/G8OHDMWDAAKxevRr3b4uSnZ2N0aNHw9/fH6tXr8bQoUOb/RvnRo4cifT0dGH58OHDCAkJafLzVFVVYefOnXB3d2/Vr+VfmzRpEuzs7LBjxw7s3r0btbW1mD59eqP75927d+Of//wnMjIysHHjRpw8ebKFR9F0DH2JRUVFQaVSYdSoUTAYDNDr9Th+/DieeeYZDB06FFZWVpg1axZ++OEH/Pzzz5g2bZpwN8IbN27A1tZW+BZB4N67VI1G02K3KA4JCYGPjw/69OmD//mf/8HTTz+N5cuXm/TR6XQoKCjAn//8Z1hZWeGll14SdgYnTpyAt7c3Bg4cCLVajcWLFwvrpaamIjIyEk8++SSsra0xe/ZspKamNuv4GvPRRx9h1KhRSEtLw4QJExAUFIS//e1vAIDnn38eX3/9tRAQaWlpGD9+vMn6cXFx8PX1haenJ/z8/JCfny/MLFpaSEgIfH19hZ9r167h5s2b+Oc//4kFCxZApVLB19cXI0aMwOHDh4X1QkNDYW1tjR9++AEAEBkZCaVSKRzhyMnJQceOHXHx4kWkp6ejU6dOyMzMhLOzc7OOr6amBl9++SUGDx6MvLw8vPrqq1izZg1OnjwJd3d3/OUvfwEA/P3vf8fXX3+Nv//97/jqq69w+fJlpKSkALh36m7p0qXr+upwAAAI50lEQVQ4e/Yshg4dik2bNkGr1WLWrFkIDw/HunXrGmz32LFj+PTTT/HZZ58hNTUV586dQ3V1NebPn49p06bhu+++g1KpxPXr15v13wMAgoODkZOTg7KyMuHQ/pAhQ0Stu3//fvj6+sLPzw/BwcHIzMzEpk2bJK64aR70mv5vFhYWSExMxPbt25GcnIy1a9fC0tKy0f3zkSNHMG3aNNjZ2eEPf/gDxo0b1zKDewyt4+TL75i9vb3wu4WFBerr61FUVIQzZ86YzJDr6upQVFQEGxsbrFy5Enl5efDw8EB1dTXs7OyEfvfPv7WUw4cPw8nJCZmZmfjLX/6CgICABl+LrNfr4eDgIJzbs7CwEHbyer0enTt3Fvp27dpV+L2oqAhfffWVMLs3Go1QKBRSD0k0a2trzJ8/H/Pnz0dpaSnS09Oxbt069OzZE/7+/ujZsye+/fZbDBgwAGfOnMGbb75psv6aNWsQFhaGgoICzJkzB+7u7nB1dW2h0Zi6/3f9bz/88AMcHBxMZuFdu3YVZsVWVlawsbEBANy8eRMFBQUmr+mamhoUFRVhwoQJuHXrFjZt2oSlS5dizJgxD7zw0dwMBoNQT2VlJRQKBbZt24ajR49i2LBh6N+/PwBg8eLF8PPzw40bN/CPf/wDM2bMQJcuXQAACxcuxPLly7Fo0SLY2Njgq6++gq2tLaZPn47o6OhH1jB+/Hg4ODjAwcEBvXv3RkFBAWpqatChQweEh4cDuHdUbNeuXRL9KzycWq1GYGAgjh07BldXVzz99NOiJxPjx49v9dcvPeg1/Wu9evWCq6sr6uvr8fTTTwNAo/vn0tLSBvuv+0e+2gqGfgtwcnLCkCFDsHnzZqEtLy8P3bt3x4wZMzB48GDs3r0bCoWiwXnt1hKCAQEBWLJkCV5++WUcOHDAJLydnJyg1+thMBiEwLh/XYKzszPOnDkj9L0fIPfXi42NFa4qrqysFNZraQcPHsTHH3+Mffv2AQAcHBwQGRmJY8eO4aeffoK/vz+ef/55HDp0CLdu3UJwcPBDd6Curq5ISkrC+PHj0bt3bwwdOrQ5hyKak5MTSktLTf6OhYWFwk7vv1+Ljo6O8PDwwP79+4U2nU6HLl26IC8vD2PGjMGsWbNw9epVLFiwAKmpqdBqtZLWr1KpkJWVBQCor6/H8ePHMW/ePPTv3x9eXl4m/Tp16oSioiLcuHHD5LXcpUsX4dTUxo0b8de//hUzZsyAWq3GsmXLMHr06EZr+O83/ZaWlqivr4derzc50qFWq036NaeQkBAcOnQI3bp1a/BFaHJw6NAhVFRUoF27dvj8888RHh7e6P7Z0dERN27cENpby/6pKXh4/zEVFRWZ/NTU1DxyneDgYGRlZeHUqVMwGo348ssvERkZiZqaGpSXl6Ndu3ZQKBQ4e/Ys9u/fj9ra2mYYSdM9//zz6NOnD9544w2T9ieeeAJPPfUUUlJSUFNTgz179ginKJ599lnk5OQgMzNT+EbF+0JDQ7F7927k5+fDYDBg7dq1eO2115pzSA8VEBCAa9euYdu2bdDr9aitrUVWVhb+93//Vwiv0NBQnD59GgcPHmxwaP/XevbsiVmzZiExMRGVlZXNMYQm69atG3r16oV33nkHBoMB586dw+HDhzFixIgGfb28vHD79m2kpaWhvr4ep06dwpgxY6DX65GWlobXX38dlZWVcHR0hEKhQMeOHaFSqVBXV4fq6mrJx2JhYYGhQ4fCzc0NXl5eJjtug8GAkpISODg4wMnJyeSxwsJCODg4oK6uDjqdDn/961+RmZmJV155BStWrPhNfztnZ2eTN7sGgwG3b99+vAH+Rs8++yzOnj370EP7v96/tZYLa82hrKwMb7zxBuLj47Fy5UqsX78epaWlje6fQ0ND8cEHH6CkpATXrl0zuTi7rWDoPwaDwYDg4GCTH71e/8j1HBwc8M4772D9+vXo378/Pv74Y6SkpECj0WDJkiX429/+Bh8fH6xduxYTJ07Ev//972YYzW8THx+PY8eO4bvvvjNpf/vtt3HmzBn4+fnh9OnTwqEzOzs7rF27FsuXL8ezzz4LBwcHWFlZAQAGDRqEqKgoTJs2DYGBgSgsLGzwhqKldOrUCbt27cLZs2cxcuRI+Pr6Yt26dXjrrbfQs2dPAEDHjh0xYMAAXLt2DQEBAY98zhkzZqBdu3bCOePWaOPGjbh8+TK0Wi1WrFiB1atXo3fv3g36qVQqvPfee9i7dy/8/Pzw2muvYcOGDejWrRuio6NhZWWFwYMHY/jw4Rg4cCBGjhwJJycn+Pv7IyAgAD///LPkYzl79izy8/MxfPhwpKen4/z58zAYDHj77bfx5JNPonv37ggLC8P27dtRVFSE27dvY/PmzQgJCYGFhQWWL1+OtLQ0KJVK2NvbQ6PRCJ/kqKioEF2Hl5cXDAYDUlNTUVNTg+TkZFGTBSm0b98efn5++MMf/oAOHTqYPPag/Zu5Pt/eGmzYsAE+Pj4YOHAg/Pz8MGjQIKxdu7bR/XNkZCS0Wi1CQkIwa9YsDB48uKWH0WT8lj1qVnq9HiUlJfDw8AAAXLt2DZMmTTL5uBvR43rQ5/S7du2KmJgYhIaG4vDhw0hKSsLNmzfh4+ODVatWwdXVFXV1dUhOTsa+fftQXV2NUaNGYcWKFVCr1cjKykJiYiLy8/Ph7OyMuLg4DBo0CJcuXcKf//xn9O/fH0OHDsXBgwfxwQcfYMqUKZg4caJwb4bp06cjLCwM48ePx7lz5xAXF4fS0lKEh4dj165dyMjIaPYLHEl+GPrUrPLz8/GnP/0J+/btQ+fOnbFmzRqUlZVh48aNLV0aUbOoqKjAv/71L/Tr1w/AvRm1l5cXzp071+CiWCJz4+F9alZPPPEEYmJiMGnSJAQEBODmzZuIi4tr6bKImo2lpSWmT5+OS5cuCXd29PT0ZOBTs+BMn4iomR05cgRvvfUWSkpK0KdPH6xevRrdu3dv6bJIBhj6REREMsHD+0RERDLB0CciIpIJhj4REZFMMPSJiIhkgqFPREQkE/8PA+KlQIRtNJYAAAAASUVORK5CYII=\n"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "all_times_ms = [[i * 1000.0 / t.loops for i in t.all_runs] for t in times]\n",
    "\n",
    "# there were some changes in RandomForest in sklearn that affect prediction times\n",
    "# these are the times from the original paper:\n",
    "# see https://github.com/CN-UPB/ml-for-resource-allocation/blob/54e9bc744bbe830b74a668fcf6dbc2eaa5ce5d69/runtime.ipynb\n",
    "all_times_ms = [0.041868121428571416, 0.04201067571428731, 0.059522719999998634, 0.8912386999999821, 0.10723779999999741,\n",
    "                0.06644656000000201, 0.0009462411285714519]\n",
    "labels = ['Linear', 'Ridge', 'SVR', 'Forest', 'Boosting', 'MLP', 'Fixed']\n",
    "\n",
    "barplot(all_times_ms, labels, ylabel=\"Prediction time [ms]\", filename='prediction_times')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)\n",
      "Ridge(alpha=1, copy_X=True, fit_intercept=True, max_iter=None, normalize=False,\n",
      "      random_state=None, solver='auto', tol=0.001)\n",
      "SVR(C=10, cache_size=200, coef0=0.0, degree=3, epsilon=0.01,\n",
      "    gamma='auto_deprecated', kernel='rbf', max_iter=-1, shrinking=True,\n",
      "    tol=0.001, verbose=False)\n",
      "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
      "                      max_features='auto', max_leaf_nodes=None,\n",
      "                      min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                      min_samples_leaf=1, min_samples_split=2,\n",
      "                      min_weight_fraction_leaf=0.0, n_estimators=100,\n",
      "                      n_jobs=None, oob_score=False, random_state=None,\n",
      "                      verbose=0, warm_start=False)\n",
      "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
      "                          learning_rate=0.3, loss='ls', max_depth=3,\n",
      "                          max_features=None, max_leaf_nodes=None,\n",
      "                          min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                          min_samples_leaf=1, min_samples_split=2,\n",
      "                          min_weight_fraction_leaf=0.0, n_estimators=10,\n",
      "                          n_iter_no_change=None, presort='auto',\n",
      "                          random_state=None, subsample=1.0, tol=0.0001,\n",
      "                          validation_fraction=0.1, verbose=0, warm_start=False)\n",
      "MLPRegressor(activation='relu', alpha=0.001, batch_size='auto', beta_1=0.9,\n",
      "             beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
      "             hidden_layer_sizes=256, learning_rate='constant',\n",
      "             learning_rate_init=0.01, max_iter=1500, momentum=0.9,\n",
      "             n_iter_no_change=10, nesterovs_momentum=True, power_t=0.5,\n",
      "             random_state=None, shuffle=True, solver='adam', tol=0.0001,\n",
      "             validation_fraction=0.1, verbose=False, warm_start=False)\n",
      "FixedModel(fixed_value=0.8)\n"
     ]
    }
   ],
   "source": [
    "for model in models:\n",
    "    print(model)\n",
    "    \n",
    "# reasons for relatively high prediction time of random forrest - in contrast to gradient boosting \n",
    "# both tree-based ensemble methods; both n_estimators=10 --> same number of decision trees\n",
    "# unlimited max_depth of forest; max_depth 3 for gradient boosting --> deeper decision trees\n",
    "\n",
    "# mlp is fast because it's pretty shallow (128,)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Placement times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sum_cpu(node_res):\n",
    "    \"\"\"Return sum of CPU resources allocated to all nodes\"\"\"\n",
    "    cpu = sum([v['cpu'] for v in node_res])\n",
    "    return cpu\n",
    "\n",
    "def read_placement(placement, df_data, flow_dr=250):\n",
    "    \"\"\"Read placement dict and write it to df_data. Then return.\"\"\"\n",
    "    df_data['num_flows'].append(placement['input']['num_flows'])\n",
    "    df_data['num_sources'].append(placement['input']['num_sources'])\n",
    "    df_data['source_dr'].append(placement['input']['num_flows'] * flow_dr)\n",
    "    df_data['num_instances'].append(placement['metrics']['num_instances'])\n",
    "    df_data['max_e2e_delay'].append(placement['metrics']['max_endToEnd_delay'])\n",
    "    df_data['total_delay'].append(placement['metrics']['total_delay'])\n",
    "    df_data['runtime'].append(placement['metrics']['runtime'])\n",
    "    df_data['total_cpu'].append(sum_cpu(placement['placement']['alloc_node_res']))\n",
    "    return df_data\n",
    "\n",
    "def read_results(results):\n",
    "    \"\"\"Read result files matching the pattern and return df containing their metrics\"\"\"\n",
    "    data = {'num_sources' : [], 'num_flows': [], 'source_dr': [], 'num_instances': [], \n",
    "            'max_e2e_delay': [], 'total_delay': [], 'runtime': [], 'total_cpu': []}\n",
    "\n",
    "    # iterate through result files\n",
    "    for res in glob.glob(results):\n",
    "        # open and save metrics of interest\n",
    "        with open(res, 'r') as f:\n",
    "            placement = yaml.load(f, Loader=yaml.SafeLoader)\n",
    "            data = read_placement(placement, data)\n",
    "\n",
    "    return pd.DataFrame(data).sort_values(by=['num_flows'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read results\n",
    "dataset = 'web_data'\n",
    "sources = 'runtime'\n",
    "results = f'placement_data/{dataset}/{sources}/'\n",
    "\n",
    "df_results = []\n",
    "df_results.append(read_results(results + 'linear/*.yaml'))\n",
    "df_results.append(read_results(results + 'ridge/*.yaml'))\n",
    "df_results.append(read_results(results + 'svr/*.yaml'))\n",
    "df_results.append(read_results(results + 'forest/*.yaml'))\n",
    "df_results.append(read_results(results + 'boosting/*.yaml'))\n",
    "df_results.append(read_results(results + 'mlp/*.yaml'))\n",
    "df_results.append(read_results(results + 'fixed/*.yaml'))\n",
    "# df_results.append(read_results(results + 'ml/*.yaml'))\n",
    "\n",
    "labels = ['Linear', 'Ridge', 'SVR', 'Forest', 'Boosting', 'MLP', 'Fixed']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.0285263538360596, 0.9558404763539632, 1.021316393216451, 107.67214407920838, 1.906393082936605, 1.0488584121068318, 1.2573257525761923]\n"
     ]
    },
    {
     "data": {
      "text/plain": "<Figure size 576x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAEDCAYAAABDBLzJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3X1czff/P/BH151hs9JclC5ky4cQdVSuIherTCPbODONyVVzHZOv5SIsGxk1F2uMKc1mazW5rjHbFMLExnw+UQoVYaQ6J6fz+8PN+7ezJHTOeaf34367ndvN+/W+er47b+/347wvjTQajQZEREQkKcZiF0BERESGxwBAREQkQQwAREREEsQAQEREJEEMAERERBLEAEBERCRBDABEREQSxABAREQkQQwAREREEsQAQEREJEGmYhegaxUVFThz5gxsbGxgYmIidjlERER6pVarce3aNbi6usLS0vKxx2twAeDMmTMYOXKk2GUQEREZ1NatW+Hh4fHYwze4AGBjYwPg/h+iRYsWIldDRESkX4WFhRg5cqSw/3tcDS4APDjs36JFC9jZ2YlcDRERkWE86WlvXgRIREQkQQwAREREEsQAQEREJEEMAERERBLEAEBERCRBDABEREQSxABAREQkQQYNABs3bsT8+fOrtSclJcHPz0+rbfXq1fD09IS3tze2bNliqBKJiIgkwSABoLKyEjExMVixYkW1fteuXcPHH3+s1bZ7926kpaVh165dSEhIQFxcHM6fP2+IUonoMUVHR8PW1rbWT3R0tNilEtFDGCQAREVF4cyZMxg+fHi1fgsXLsTgwYO12nbv3g2FQgFra2s4Oztj6NChSElJMUSpRPSYwsLCcPnyZeHj7e0Nb29vrbbLly8jLCxM7FKJ6CEMEgAmTZqEuLg4WFtba7Xv3LkTJiYmGDBggFZ7bm4unJychG57e3vk5OQYolQiIiJJMEgAeNgLCm7cuIGYmBhERERU61dWVqb1SkNLS0tUVFTotUYiIiIpEe0ugCVLliAkJOSh4UAmk0GpVArdFRUVkMlkhiyPiIioQRPtbYBpaWk4dOgQPv74Y6jValRUVMDDwwNZWVlwcnLCpUuX4OXlBQDIy8uDo6OjWKUSERE1OKIdAcjOzkZWVhaysrKwfv16ODg4ICsrCwDg5+eH+Ph4FBcX48KFC0hOToa/v79YpRIRETU4oh0BeBR/f3/k5OQgKCgIGo0GoaGh6NSpk9hlERERNRgGDQBTpkx5aLunpyf27NkjdBsZGWHKlCk1Dk9ERER1w0cBExERSRADABERkQQxABAREUkQAwAREZEEMQAQERFJEAMAERGRBDEAEBERSRADABERkQQxABAREUkQAwAREZEEMQAQERFJEAMAERGRBDEAEBERSRADABERkQQxABAREUkQAwAREZEEMQAQERFJEAMAERGRBDEAEBERSZBBA8DGjRsxf/58oXv79u3o168fPDw8MGHCBBQVFQn9Vq9eDU9PT3h7e2PLli2GLJOIiKjBM0gAqKysRExMDFasWCG0nT59GqtWrUJcXBwyMjLQunVrIRzs3r0baWlp2LVrFxISEhAXF4fz588bolQiIiJJMEgAiIqKwpkzZzB8+HChraioCGPGjIGzszPMzMzw1ltv4dSpUwDuBwCFQgFra2s4Oztj6NChSElJMUSpREREkmCQADBp0iTExcXB2tpaaOvfvz9CQkKE7kOHDuHll18GAOTm5sLJyUnoZ29vj5ycHEOUSkREJAmmhpiJjY3NI/sfPnwYa9euxaZNmwAAZWVlsLS0FPpbWlqioqJCrzUSERFJieh3Aezbtw9Tp07FihUr0LlzZwCATCaDUqkUhqmoqIBMJhOrRCIiogZH1ADw3XffISIiAmvWrIGvr6/Q7uTkhEuXLgndeXl5cHR0FKFCIiKihkm0AHDy5EksXboUGzZsgKenp1Y/Pz8/xMfHo7i4GBcuXEBycjL8/f1FqpSIiKjhMcg1AA+zdetWVFRUIDg4WGgzNzfHkSNH4O/vj5ycHAQFBUGj0SA0NBSdOnUSq1QiIqIGx0ij0WjELkKXCgoK0K9fP6Snp8POzk7scogk44033gBw/9QeERnO0+73RL8IkIiIiAyPAYCIiEiCGACIiIgkiAGAiIhIghgAiIiIJIgBgIiISIIYAIiIiCSIAYCIiEiCGACIiIgkiAGAiIhIghgAiIiIJIgBgIiISIIYAIgk5N69e2KXUCfPev1E9YlorwMmIsMzNTXFokWL9DLt3NxcANDb9AFgwYIFeps2kdTwCAAREZEEMQAQERFJEAMAERGRBDEAEBERSRADABERkQQZNABs3LgR8+fPF7r37dsHX19fdOnSBXPnzkVlZaXQb/Xq1fD09IS3tze2bNliyDKJiIgaPIMEgMrKSsTExGDFihVCW2FhIT788EPExMTg0KFDKCgowLfffgsA2L17N9LS0rBr1y4kJCQgLi4O58+fN0SpREREkmCQABAVFYUzZ85g+PDhQltaWhq6d+8OV1dXNGnSBOPHj0dycjKA+wFAoVDA2toazs7OGDp0KFJSUgxRKhERkSQYJABMmjQJcXFxsLa2Ftpyc3Ph6OgodDs4OCAnJ0fo5+TkJPSzt7cX+hEREVHdGSQA2NjYVGsrKyuDTCYTui0sLFBRUSH0s7S0FPpZWloK/YiIiKjuRLsLQCaTQalUCt1KpVIIBP/uV1FRoRUWiIiIqG5ECwBOTk7Iy8sTuv95SsDJyQmXLl0S+uXl5WmdLiAiIqK6ES0A9OvXD7/++it+//133LlzB1988QX8/f0BAH5+foiPj0dxcTEuXLiA5ORkoR8RERHVnWhvA2zZsiWWLl2K2bNn4+bNm/D398fo0aMBAP7+/sjJyUFQUBA0Gg1CQ0PRqVMnsUolIiJqcAwaAKZMmaLV3b9/f/Tv37/acEZGRpgyZUq14YmIiEg3+ChgIiIiCWIAICIikiAGACIiIgliACAiIpIgBgAiIiIJYgAgIiKSIAYAIiIiCWIAICIikiAGACIiIgmq8UmApaWlTzShxo0b17kYIiIiMowaA4BcLn+sCWg0GhgbG+PPP//UWVFERESkXzUGAEtLS6SmptY6AY1Gg8GDB+u0KCIiItKvGgPAuHHjYGtr+1gTGTdunM4KIiIiIv2r8SLA0NBQ4d9ZWVkAgGvXriEiIgIrVqxAWVnZQ4clIiKi+q/WuwA++eQTzJo1CwAwf/58XLx4EX/88QciIyP1XhwRERHpR42nAB5IS0vDN998g7t37+KXX35BamoqbGxs4Ovra4j6iIiISA9qPQJw69YtNG/eHJmZmWjevDkcHR1hbm6OqqoqQ9RHREREelDrEQAXFxd8+umnOHLkCPr164fS0lKsWrUKHTt2NER9REREpAe1HgFYunQp/vrrL7Ru3RrTpk3DuXPncPbsWSxatMgQ9REREZEe1BgANm/eDACwt7fH+vXrsXz5cjRq1AgeHh7YunUrWrduXW3Yp5GRkYGAgAC4u7tj1KhRyM3NBQAkJiaiZ8+ekMvliI6OfurpExERUXU1BoCYmBiUlpbW+rlz5w5iYmKeauZqtRrTp0/HokWLcOzYMXTp0gWRkZHIzs7GunXrkJiYiNTUVKSnp+PgwYNPu4xERET0LzVeA1BWVvZYjwPWaDQwMjJ6qpn//fffuHXrFqqqqoRHCltYWGD37t0IDAyEvb09ACA4OBjJycno06fPU82HiIiItNUYANLT0/U+cysrKwQFBSE4OBgmJiZo2rQpvvnmG3z00Udatxk6ODhg69ateq+HiIhIKmoMAI/7GOC6UKvVaNq0KeLj49G5c2fExsZi1qxZsLS0hEwmE4azsLBARUWF3ushIiKSilrvAtCnvXv3oqCgAN26dYOFhQVmzJiB8+fPw8zMDEqlUhhOqVRqBQIiIiKqG1EDQGFhISorK4VuY2NjGBkZQSaT4dKlS0J7bm4uHB0dRaiQiIioYRI1AHh5eSEjIwO//vor7t27h/Xr18PBwQEhISFISkrCxYsXUVxcjPj4ePj7+4tZKhERUYNS65MAAeD48eP4+uuvUVxcjJUrV2Lbtm14//33n/rq/wfat2+PpUuXYvHixSgpKUGnTp0QGxsLOzs7hIaGYsyYMSgvL4dCoWAAICIi0qFaA0BqaiqWLVuGoKAgHDhwAFVVVdixYwdUKhVmzpxZ5wJee+01vPbaa9XaFQoFFApFnadPRERE1dV6CmDt2rVYv349Zs6cCWNjY7z00kvYsGEDfvjhB0PUR0RERHpQawC4fv062rdvDwDCIf9WrVppXaVPREREz5ZaA0CHDh2wZcsWrbYff/wRLi4ueiuKiIiI9KvWawDmzZuH9957D9u3b0dZWRlGjBiB/Px8bNiwwRD1EVE9lZWVhRMnTlRrj4uL0+ru2rUrPDw8DFUWET2mWgNA27ZtsWfPHhw8eBBXrlzBSy+9BB8fH7zwwguGqI+I6ikPDw/u2ImeYY91G6BarUbbtm3Rpk0bAMDVq1dx9epVtGvXTq/FERERkX7UGgA2b96M5cuXQ61Wa7UbGRnh7NmzeiuMiIiI9KfWALB+/XrExsaib9++dX7wDxEREdUPtd4FYG5ujl69enHnT0RE1IDUegRg4sSJWLBgAd599100adJEq1+rVq30VhgRERHpT60BoKqqCikpKUhKSgJw/9y/RqPhNQBERETPsFoDQGxsLFasWAFvb2+YmJgYoiYiIiLSs1oDgJmZGQYMGABT08e6Y5CIiIieAbVeBDhx4kREREQgPz8fd+7cQWlpqfAhIiKiZ1OtP+tXr16NO3fuIDk5WWjjNQBERETPtloDwD93/ERERNQw1BgASktL0bhxYz7zn4iIqAGqMQD07t0bJ06cgIeHR7WHAPEUABER0bOtxgCwc+dOAEB6errBiiEiIiLDqPEugAULFgAAbG1ta/zoQn5+PsaMGQO5XI4hQ4bgzJkzAIDExET07NkTcrkc0dHROpkXERER3VdjAMjKytL7zNVqNUJCQuDj44MjR45g5MiRCAsLQ3Z2NtatW4fExESkpqYiPT0dBw8e1Hs9REREUiHq031OnDgBc3NzjB49GgAwbNgw/Oc//0FqaioCAwNhb28PAAgODkZycjL69OkjXrFEREQNSI0BQKVSISoq6pEjz507t04zP3fuHBwdHTF37lz89NNPcHZ2xuLFi5GXlwdfX19hOAcHB2zdurVO8yIiIqL/75FPArx9+/YjP3V1584dpKenw9vbG7/88gt8fX0xefJklJaWQiaTCcNZWFigoqKizvMjIiKi+2o8AmBubl7rEYC6MjMzQ9u2bREYGAgAGDt2LNatWwdbW1solUphOKVSqRUIiIiIqG5qPAKg0Wj0PnNHR0etdwpoNBpUVVXB0tISly5dEtpzc3Ph6Oio93qIiIikosYA4OHhofeZd+/eHWVlZYiPj4darcbGjRthZ2eHcePGISkpCRcvXkRxcTHi4+Ph7++v93qIiIikosYA8MUXX+h95o0aNcLmzZuxc+dOyOVy7N+/H6tWrULnzp0RGhqKMWPGYPDgwRg4cCADABERkQ6JehsgALRr1w7btm2r1q5QKKBQKESoiIiIqOF75F0ARERE1DAxABAREUkQAwAREZEEMQAQERFJEAMAERGRBDEAEBERSRADABERkQQxABAREUkQAwAREZEEMQAQERFJEAMAERGRBDEAEBERSRADABERkQQxABAREUkQAwAREZEEMQAQERFJEAMAERGRBDEAEBERSRADABERkQQxABAREUlQvQgAmZmZ6Ny5s9CdmJiInj17Qi6XIzo6WsTKiIiIGibRA0B5eTkWLFiAqqoqAEB2djbWrVuHxMREpKamIj09HQcPHhS3SCIiogZG9ACwcuVK9OrVS+jevXs3AgMDYW9vj+bNmyM4OBjJyckiVkhERNTwiBoATp48iezsbLz77rtCW25uLhwdHYVuBwcH5OTkiFAdERFRw2Uq1oxVKhUWLFiA5cuXw8jISGgvKyuDTCYTui0sLFBRUSFGiURERA2WaEcAYmNj4evrCxcXF612mUwGpVIpdCuVSq1AQERERHUn2hGA/fv349q1a0hISEBVVRVUKhU8PDwwYMAAXLp0SRju36cEiIiIqO5ECwB79uwR/l1QUAB/f39kZWXh1KlTmDx5MoYMGYJGjRohPj4eU6ZMEatMIiKiBkm0AFCTzp07IzQ0FGPGjEF5eTkUCgX8/f3FLouIiKhBqRcBwM7ODqdPnxa6FQoFFAqFiBURERE1bKI/B4CIiIgMjwGAiIhIghgAiIiIJIgBgIiISIIYAIiIiCSIAYCIiEiCGACIiIgkiAGAiIhIghgAiIiIJIgBgIiISIIYAIiIiCSIAYCIiEiCGACIiIgkiAGAiIhIghgAiIiIJIgBgIiISIIYAIiIiCSIAYCIiEiCGACIiIgkSNQA8NNPPyEgIADu7u4YOXIkcnJyAACJiYno2bMn5HI5oqOjxSyRiIioQRItABQWFiI8PByLFy/G0aNH0bdvX0ybNg3Z2dlYt24dEhMTkZqaivT0dBw8eFCsMomIiBok0QLA1atXMWzYMLi7u8PExAQKhQL//e9/kZqaisDAQNjb26N58+YIDg5GcnKyWGUSERE1SKIFgC5dumDOnDlC96FDh2Bra4v8/Hw4OjoK7Q4ODsKpASIiItKNenER4Llz57Bw4UKEh4ejrKwMMplM6GdhYYGKigoRqyMiImp4RA8AWVlZePfddzFjxgwMHDgQMpkMSqVS6K9UKrUCAREREdWdqZgzP3ToEGbOnInIyEgEBAQAAJycnHDp0iVhmNzcXK1TAkRERFR3oh0ByM/Px/Tp0/HJJ58IO38A8PPzQ1JSEi5evIji4mLEx8fD399frDKJiIgaJNGOAHz33XcoKytDWFiYVntaWhpCQ0MxZswYlJeXQ6FQMAAQERHpmGgBYMaMGZgxY8ZD+ykUCigUCgNXREREJB2iXwRIREREhscAQEREJEEMAERERBLEAEBERCRBDABEREQSxABAREQkQQwAREREEsQAQEREJEEMAERERBLEAEBERCRBDABEREQSxABAREQkQQwAREREEsQAQEREJEEMAERERBLEAEBERCRBDABEREQSxABAREQkQQwAREREElRvA0BWVhYGDRoENzc3hIaGorS0VOySiEjCoqOjYWtrW+snOjpa7FKJHku9DADl5eWYOnUqPvjgA2RmZsLS0hJr1qwRuywiesbcu3dPZ9MKCwvD5cuXhY+3tze8vb212i5fvoywsDCdzVOX9VPNpBruTMUu4GEyMzNha2sLHx8fAMDkyZPxzjvvYM6cOaLUc+/ePZia6uZPFR0djZUrV9Y63MyZM3W2IdFl/U9KjOV9Us/y9yvmd/ssMDU1xaJFi3QyraysLJw4caJau62trVZ3165d4eHhoZN5Lliw4ImGf5bXZeDJ6tflsoaFhWktwxtvvAEA+O6773Qy/YepD/936+WWIy8vD46OjkK3vb09SkpKcOvWLTRt2vSR46rVagBAYWGhTmtavXq1TqZz+vRpGBkZ1Trc/v37dbYM06ZNe6Lh1Wo1TExMdDLvO3fuPNby3rlzBwUFBTqZ59PU/6x+v0/63QJ4pk+nPc06oqvlbdeuHdq1a2fQeT7N8j6r6zLw5OuzLpf1zJkz1drt7Oy0ul1dXdGxY0edzPNp/u/W5MHf/8H+73EZaTQajc6q0JE1a9agqKgIkZGRQpurqyvS0tLQokWLR46blZWFkSNH6rtEIiKiemXr1q1PdPSpXh4BkMlkUCqVQve9e/dQWVkJmUxW67iurq7YunUrbGxsdPYrloiIqL5Sq9W4du0aXF1dn2i8ehkAnJycsG/fPqE7Ly8PL774Il544YVax7W0tNTZ+TciIqJngYODwxOPUy/vAvDy8kJeXh7S09NRUVGBtWvXwt/fX+yyiIiIGox6eQ0AAJw4cQILFizA5cuX0b17dyxbtgyNGzcWuywiIqIGod4GACIiItKfenkKgIiIiPSLAYCIiEiCGACIiIgkiAGAiOqFK1euiF0CkaQwAOhAQUHBQx8POX/+fGzatEmEivTPxcUFbm5u6NKli/AJDg4WNuLr16/H0qVLHzqur68vfv/9d0OWq3M5OTkICQlB165d4e7ujnHjxiEnJwdhYWEPfX57RkYGevfujaqqKowaNQodO3YU/m5ubm4YNGgQMjIyRFgSbTV9r/p2/fp1DB06VO/zMYRnKcgUFBTAxcUFo0aNqtZv8eLFcHFxqXH7BgCxsbHo0KGD1voyYsSIhz5WV0yPWq8HDRqE7OxsncwnNjYW8+fP18m0DKFePgioofjno4wbov3798PGxgbA/Wefz5s3D8uWLUNMTAwmTpwocnX6o1arMW7cOEyYMAHr169HVVUVNm3ahHHjxmHx4sUICwtDRESE1os+duzYgddffx3Gxvcz97JlyzBo0CAA9590+dVXX2HWrFk4dOiQ6E+w/Of3aigVFRUoKyvTy7QLCgrQr18/PPfcc0Jbhw4dsGTJEq13juhCQkICzp8/j8jISPz444/Yu3dvvX+TqampKc6dO4eSkhJYW1sDADQaDX7++efHWheHDRsmbOs0Gg22bduGSZMm4cCBA6K/7Oafalqvd+7cKUI19QOPAOhReHg44uLiAADt27fHl19+CW9vb/j4+OCHH34Qhjt58iSCgoIgl8sxfvx4FBUVAbi/U509ezb69u2LTp06ITg4GNevXwcAjBo1CnPmzIGnpyfWr19v+IX7l8aNG2Pw4MHIyckBoJ2Ec3Jy8MYbb6BLly6IiIjQemHFTz/9hP79+6NHjx74/PPP0b59e6Hf9u3b0b9/f3Tv3h1LliyBSqUy7ELV4ObNm7hy5QoCAgJgamoKc3NzjB8/Hj179oSLiwssLCxw+PBhYXiVSoX9+/fX+AvX1NQUb731Fq5fv46///7bUIvxxC5fvoxx48bBw8MDAQEBSE9PB3B/B9u3b1+MHz8enp6eyMvLw4ULFzBq1CjI5XK8/fbbwnpRWlqKSZMmQS6XY8CAAfjyyy8BAO+++y5UKhW6dOmil+/Z3NwcJ0+exMmTJ3Hs2DF07NgREREROp/PzZs3hX8HBgbW+50/ABgbG6NXr17C9wkAx48fh4uLyxOHUSMjIwwZMgTFxcW4ffu2rkvViwdHJBMSEjBw4ECoVCpUVVVh+PDhwva7pm10ZWUlFixYAHd3dwwePBi5ubkiLsmTYwAwELVajQsXLuDnn39GeHg4IiMjoVQqcfv2bUyaNAmTJ0/G4cOH4ebmJrz2+IsvvoBKpcLevXuRkZEBIyMjbNu2TZhmYWEhDh48iHfeeUesxRLcvHkTX3/99UMfwzx9+nQMHDgQR48ehaOjo/DmqmvXrmHWrFlYtGgR0tPTcfbsWSEcHDt2DJ999hni4uKwb98+FBQUYMOGDQZdppo0a9YMHTt2FDYQ2dnZqKqqQmRkJJo1a4bXX38du3btEob/6aef4OzsjDZt2jx0eiqVChs2bICzszOsrKwMtRhPbPbs2XBxccHhw4cRGRmJOXPmCDv2K1euoF+/fkhPT4etrS0mTZqEgIAAZGRk4M0338T7778PtVqNbdu24bnnnkNmZiY2bNiAzz//HEVFRfjqq6+EnbS5ublel8PU1BSBgYE4d+4cAGDv3r3w8/ODXC7HxIkThY27RqPB6tWr0bNnT/Ts2VMrhB45cgSDBg2CXC7HiBEj8Mcff+Dw4cP4/PPP8f333yM8PBxJSUkYO3YsgPuBPTY2FgMGDEC3bt2wZMkSPHgEy8mTJzF48GB4enpiyZIl8PX11dmbMR/Xq6++ir179wrde/bsgZ+f3xNPp6KiAlu2bIGTk1O9Xpcf5u2330bTpk3x1VdfISEhAffu3cPYsWMfuY1OSEjAn3/+ifT0dKxcuRK//fabyEvxZBgADCg4OBjm5ubw9/eHSqVCSUkJDh48iPbt28PX1xdmZmaYMGECTp06heLiYowePRqLFy8GAFy9ehUvvPACrl27JkzP19cXMplMtCck+vn5oWvXrnB1dcVrr72GV155BeHh4VrD5OXloaCgAO+99x7MzMwwZswYYcPw888/o0uXLujRowcsLS0xY8YMYbyUlBQoFAq0adMGjRs3xsSJE5GSkmLQ5XuUTZs2wd/fHzt27MCbb76Jnj17IjExEQAwdOhQpKWlCTuLHTt2ICgoSGv8efPmwcPDAx07doRcLkd+fr7wa0Nsfn5+8PDwED65ubkoKirCn3/+ialTp8Lc3BweHh4YOHAg9uzZI4wXEBCAxo0b49SpUwAAhUIBU1NT4chHdnY2nn/+eZw5cwZ79+6FtbU1MjMz0bx5c4MuX2VlJZKTk9G7d2/k5ORg/vz5WLp0KX777Tc4OTlh5syZAO6/Cz4tLQ3fffcdUlNT8ddff2Ht2rUA7p/e++CDD3Ds2DH4+vpi9erV6N69OyZMmIBhw4Zh2bJl1eZ74MABfPPNN/j222+RkpKC48ePQ6lUYsqUKRg9ejR++eUXmJqa4vLlywb9ewCAj48PsrOzcfv2beHwf9++fR9r3KSkJHh4eEAul8PHxweZmZk6e02vLj1svf4nY2NjREZGYsOGDVizZg2ioqJgYmLyyG30vn37MHr0aDRt2hQvv/wyhgwZIs7CPaX6c4JGAl588UXh38bGxqiqqkJhYSGOHj2q9ctZrVajsLAQTZo0wYcffoicnBy4uLhAqVSiadOmwnAPzteJZc+ePbCxsUFmZiZmzpwJLy+vam9sLCkpgZWVlXAu0NjYWNjgl5SU4KWXXhKGbdmypfDvwsJCpKamCr/6NRrNY72b3FAaN26MKVOmYMqUKbhx4wb27t2LZcuWwdnZGZ6ennB2dsahQ4fQrVs3HD16FB9//LHW+EuXLsWgQYNQUFCASZMmwcnJqdq7x8Xy4Hv9p1OnTsHKykrr13nLli2FX8tmZmZo0qQJAKCoqAgFBQVa63RlZSUKCwvx5ptv4ubNm1i9ejU++OADBAYGPvSiSV1TqVRCPeXl5TAyMkJcXBz279+P/v37w93dHQAhIzbFAAAHDElEQVQwY8YMyOVyXL16Fbt370ZISIjwCvJp06YhPDwc06dPR5MmTZCamooXXngBY8eOxfjx42utISgoCFZWVrCyskKHDh1QUFCAyspKNGrUCMOGDQNw/2hZfHy8nv4KNbO0tIS3tzcOHDgAOzs7vPLKK4/9wyIoKOiZuN7pYev1v7Vr1w52dnaoqqrCK6+8AgCP3EbfuHGj2jbswVGxZwEDgMhsbGzQt29fxMbGCm05OTlwcHBASEgIevfujYSEBBgZGVU7D15fdoheXl6YPXs2Zs2ahR9//FFrR25jY4OSkhKoVCph5/HgOobmzZvj6NGjwrAPdiYPxgsLCxOuTi4vLxfGE9vOnTuxefNmbN++HQBgZWUFhUKBAwcO4Pz58/D09MTQoUOxa9cu3Lx5Ez4+PjVuTO3s7LBq1SoEBQWhQ4cO8PX1NeSiPDYbGxvcuHFD63u8cuWKsPH757rYrFkzuLi4ICkpSWjLy8tDixYtkJOTg8DAQEyYMAEXLlzA1KlTkZKSgu7du+u1fnNzc2RlZQEAqqqqcPDgQbz//vtwd3eHm5ub1nDW1tYoLCzE1atXtdblFi1aCKevVq5ciejoaISEhMDS0hJz5szB4MGDH1nDP38AmJiYoKqqCiUlJVpHQCwtLbWGMyQ/Pz/s2rULrVq1kuzL13bt2oW7d+/CwsIC33//PYYNG/bIbXSzZs1w9epVob2+bKMeF08B6FBhYaHWp7KystZxfHx8kJWVhYyMDGg0GiQnJ0OhUKCyshKlpaWwsLCAkZERjh07hqSkJNy7d88AS/Lkhg4dCldXV3z00Uda7a1bt0bbtm2xdu1aVFZWYuvWrcJpjD59+iA7OxuZmZlQqVT47LPPhPECAgKQkJCA/Px8qFQqREVFYeHChYZcpBp5eXkhNzcXcXFxKCkpwb1795CVlYXTp08LO7KAgAAcOXIEO3furHb4/9+cnZ0xYcIEREZGory83BCL8MRatWqFdu3aISYmBiqVCsePH8eePXswcODAasO6ubnh1q1b2LFjB6qqqpCRkYHAwECUlJRgx44dWLx4McrLy9GsWTMYGRnh+eefh7m5OdRqNZRKpd6XxdjYGL6+vrC3t4ebm5vWBlylUuH69euwsrKCjY2NVr8rV67AysoKarUaeXl5iI6ORmZmJv7v//4Pc+fOfarvrnnz5lrBV6VS4datW3VbwKfUp08fHDt2rMbD///evtWXi3J15fbt2/joo48QERGBDz/8EJ988glu3LjxyG10QEAANm7ciOvXryM3N1fr4u5nAQOAjqhUKvj4+Gh9SkpKah3PysoKMTEx+OSTT+Du7o7Nmzdj7dq1kMlkmD17NhITE9G1a1dERUVhxIgRuHjxogGW5ulERETgwIED+OWXX7TaP/30Uxw9ehRyuRxHjhwRDq01bdoUUVFRCA8PR58+fWBlZQUzMzMAQK9evRAcHIzRo0fD29sbV65cqRYuxGJtbY34+HgcO3YMr776Kjw8PLBs2TIsX74czs7OAIDnn38e3bp1Q25uLry8vGqdZkhICCwsLIRzzPXRypUr8ddff6F79+6YO3culixZgg4dOlQbztzcHOvWrcO2bdsgl8uxcOFCrFixAq1atcL48eNhZmaG3r17Y8CAAejRowdeffVV2NjYwNPTE15eXiguLtb7shw7dgz5+fkYMGAA9u7dixMnTkClUuHTTz9FmzZt4ODggEGDBmHDhg0oLCzErVu3EBsbCz8/PxgbGyM8PBw7duyAqakpXnzxRchkMuGOkLt37z52HW5ublCpVEhJSUFlZSXWrFnzWD8c9OG5556DXC7Hyy+/jEaNGmn1e9j2TVf3ztcXK1asQNeuXdGjRw/I5XL06tULUVFRj9xGKxQKdO/eHX5+fpgwYQJ69+4t9mI8Eb4NkERTUlKC69evw8XFBQCQm5uLt99+W+sWOqK6ethzAFq2bInJkycjICAAe/bswapVq1BUVISuXbti0aJFsLOzg1qtxpo1a7B9+3YolUr4+/tj7ty5sLS0RFZWFiIjI5Gfn4/mzZtj3rx56NWrF86ePYv33nsP7u7u8PX1xc6dO7Fx40aMGjUKI0aMEJ79MHbsWAwaNAhBQUE4fvw45s2bhxs3bmDYsGGIj49Henq6wS+OJOlhACDR5OfnY/jw4di+fTteeuklLF26FLdv38bKlSvFLo3IIO7evYv//e9/6Ny5M4D7v7Td3Nxw/PjxahfUEukaTwGQaFq3bo3Jkyfj7bffhpeXF4qKijBv3jyxyyIyGBMTE4wdOxZnz54VnijZsWNH7vzJIHgEgIhIRPv27cPy5ctx/fp1uLq6YsmSJXBwcBC7LJIABgAiIiIJ4ikAIiIiCWIAICIikiAGACIiIgliACAiIpIgBgAiIiIJ+n9N2FZDY0OnpwAAAABJRU5ErkJggg==\n"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "placement_times = [df['runtime'] for df in df_results]\n",
    "\n",
    "barplot(placement_times, labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combined Plot: Prediction and Placement times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "def comparison_barplot_one(pred_times, place_times, labels, filename=None):\n",
    "    \"\"\"Plot both in one bar plot\"\"\"\n",
    "    sns.set(font_scale=1.1, style='white')\n",
    "    assert len(pred_times) == len(place_times) == len(labels)\n",
    "    \n",
    "    # prepare data\n",
    "    pred_times_mean = [np.array(t).mean() for t in pred_times]\n",
    "    pred_times_std = [np.array(t).std() for t in pred_times]\n",
    "    place_times_mean = [np.array(t).mean() for t in place_times]\n",
    "    place_times_std = [np.array(t).std() for t in place_times]\n",
    "    x = np.arange(len(labels))\n",
    "    width = 0.35\n",
    "\n",
    "    # plot: separate axes\n",
    "    fig, ax1 = plt.subplots(figsize = (8, 5))\n",
    "    ax1.bar(x - width/2, pred_times_mean, width, yerr=pred_times_std, capsize=5, color='gray', label='Prediction')\n",
    "    ax1.set_ylabel(\"Prediction time [ms]\")\n",
    "    ax1.set_xticks(x)\n",
    "    ax1.set_xticklabels(labels)\n",
    "    \n",
    "    ax2 = ax1.twinx()\n",
    "    ax2.bar(x + width/2, place_times_mean, width, yerr=place_times_std, capsize=5, color='lightgray', label='Placement')\n",
    "    ax2.set_ylabel(\"Placement time [s]\")\n",
    "        \n",
    "    # labels\n",
    "    ax1.legend()\n",
    "    \n",
    "    if filename is not None:\n",
    "        fig.savefig(f'plots/{filename}.pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 576x360 with 2 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiUAAAE6CAYAAADJBSI4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3XlUVPX/P/DnIGsi4oJkEEKWmLmAQiyRqKWiJgqmOaWo5QaCYuSSGwnuuaAIuFYKqKWZC+aK2SoqSqEV5pdydEQUQUJkGWDm94c/76crixdkYITn45w5h/u+977v6+I48+Qu7yvTaDQaEBEREdUzvfougIiIiAhgKCEiIiIdwVBCREREOoGhhIiIiHQCQwkRERHpBIYSIiIi0gkMJURERKQTGEqIiIhIJzCUEBERkU5gKCEiIiKdwFBCREREOkG/vguoDbm5uUhISED79u1hYGBQ3+UQERFpVUlJCdLT0/HWW2/B3Ny8vsupNQ0ilCQkJCA8PLy+yyAiIqpzo0aNqu8Sak2DCCUvvPACAGD+/Pno2LFjPVdDRESkXWlpaQgPDxe+/xqKBhFKDA0NAQAdO3aEk5NTPVdDRERUNx5+/zUUvNCViIiIdAJDCREREekEhhIiIiLSCQ3imhIiItIOtVoNpVKJ+/fv13cpjVLTpk1hbW0NPb3GcQyBoYSIiCp1584dyGQy2NvbN5ovRl2hVqtx48YN3LlzB23atKnvcuoE32FERFSp3NxcWFpaMpDUAz09PVhaWuLff/+t71LqDN9lRERUqbKyMo6UXY8MDAxQWlpa32XUGYYSIiKqkkwmq+8SdF5JSQmysrJqvd/G9rtnKCEioqeWUqmEvb09HB0dhZenpyc+//zzJ+o3MjISCxYsAACMHz8eR48erXL5kJAQnD17FgCwYMGCJ95+Y8ULXYmIqFpKS0uhr6+9r4/q9m9oaIiUlBRh+tq1a3j33XfxwgsvwNPT84nr2bJly2OXuXv3rvBzWFjYE29TW7Zu3QqFQlGuxr1792LTpk04cuSI0LZ27Vrs2LEDenp68Pf3h5+fn9brYyghakQq+7AfNmwYkpKSHru+q6srvv76a0l9rlq1CqtXr35snx9++CFCQkIeuxzpDn19fSxcuFBr/YeGhj7R+jY2NujRoweuXLmCdevWQU9PDzdv3sSRI0dw5coVhIeH4/r163B0dER4eDgsLS1RUlKCRYsWISEhAc899xw6dOiApk2bAgBGjx6NkSNHYtCgQfjtt9+wcOFC/PPPP+jYsSOWLVuG/fv3Izk5Gb/++isKCgpw/vx5vPDCC5g4cSLS0tIQFhaGy5cvo127dpg3bx66d++OM2fOYN26dbCyssLx48fx/PPPY8mSJejcuXNt/ArLKSkpQUxMDGJiYjB8+HDRvKysLCxfvhwtWrQQ2g4fPowTJ07g22+/RW5uLsaMGQNXV1d06NBBK/U9xFBC1IhU9mXStWtXdO3aVVIfj65f2RdISEiIKGy8/fbbAIA9e/ZILZeo2tRqNX7//XecPXsWnTp1wqVLl7B161Y4ODhArVbD398fS5Ysweuvv47Nmzdj1qxZ+OKLLxAXF4c//vgDiYmJyMrKwujRo9GvXz9R34WFhQgICMD06dPh4+ODzz77DHPmzEF8fDzOnTsnBJfz588DAFQqFaZOnYoxY8Zg+/btOHnyJPz9/XH48GEAQHJyMoYNG4ZFixbh008/xcqVK/HFF19o5feydOlSKJVKvPPOO1Cr1aJ5n3zyCQYPHoyffvpJaDt8+DDkcjlatWqFVq1awcfHB/v378eMGTO0Ut9DDCVERPRUU6lUooextmnTBkFBQWjfvj3MzMzg4eEBADhw4AA6deqEPn36AAAmTZqEzZs34/bt2zh27BjGjh0Lc3NzmJubY+jQoSgoKBBtJyUlBaampkLAHjNmDNzc3Cqt69KlS9BoNHjvvfcAAP369UN8fDx++OEHtG3bFubm5vD19QUA9O/fHydPnqz2vmdlZUGpVIrazMzMYGZmJmrz9/eHhYUFIiMjRRfkHjp0CE2aNEHfvn1FoeTq1auQy+XCtI2NDRITE6tdX3UxlBAR0VPN0NAQycnJ5drPnDmDli1bCtOZmZk4e/asKMCUlZUhMzMTOTk5ogHK2rZti/T0dFF/OTk5sLS0FG23qtMtmZmZePbZZ0Vtbdu2xa1bt9C2bVvR6RJ9fX1oNBoJeysWHBxcri0wMBBBQUGiNgsLi3LL5eTkYN26dYiLi8Pff/8tmldQUABjY2Nh2tjYGEVFRdWur7oYSoiIqMH67y21FhYW6N27NyIjI4W29PR0tGvXDq1bt8bNmzeF9jt37pTrq02bNrh9+7YwrVKpsGbNGnz00UcVbtvCwgKZmZmiths3bqBHjx413p9HRUREoEuXLqK2R4+SVGbRokUYP348LCwsyoUSExMTFBcXC9NFRUUwMTF58oIfg7cEExFRo+Dp6Ynk5GScPn0aGo0G+/btg1wuR0lJCQYOHIitW7fizp07uHr1Kr755pty63fr1g1FRUXYt28fysrKEBcXh9TUVDRp0gSGhoblng/UrVs3lJWVIT4+HqWlpTh+/DguXbqEnj171to+WVhYwNraWvSSGkpOnDiB5cuXw8nJCZMnT4ZCoRCOItnZ2eHatWvCsgqFAra2trVWd2V4pISIiBqFli1bYt26dViyZAkUCgVsbGwQHR0NExMTyOVyKJVKeHl5oVWrVhUGByMjI0RFReGTTz5BeHg4OnXqhOXLlwMAvLy8sGTJEtF1KIaGhoiJicHChQuxcuVKPP/884iOjoalpSWuXr1aV7tdqdTUVOHnM2fOIDQ0VLgl2MvLCzExMejVqxfy8/Oxb98+REdHa70mhhIiIqqW0tLSJ75t93H9Sx2nxNraGhcvXqxwnouLi2jcDQBwdnau8CiInp4eZs2ahVmzZpWbFxsbK/z8yiuvYPfu3eWWGT58eLlbbQHA3t4eO3bseGxtDg4ONbrQVVsGDBiA9PR0+Pr6QqPRICAgQPIdek+CoYSIiKpFmwOn1UX/jdmjF8A+9GhIkslkCAoKqnR5beE1JURERKQTGEqIiIhIJzCUEBERkU5gKCEioirVZFAvqh2N7XfPUEJERJVq0qQJSkpK6ruMRqukpKRRXfjLUEJERJUyNzfHrVu3yj3EjbRPrVbj1q1baN68eX2XUmcaT/wiIqJqa926NZRKJS5fvlzfpTRKTZs2RevWreu7jDrDUEJERJXS09ODjY1NfZdBjQRP3xAREZFOYCghIiIincBQQkRERDqBoYSIiIh0AkMJERER6YQ6CSXJyckYNGgQHBwcEBAQgPz8/HLLpKWlYdiwYejevTt8fX2RmppaF6URERGRjtB6KCksLMTUqVMxc+ZMJCUlwdjYGFFRUeWWmzlzJkaPHo3z589jxIgRmDFjhrZLIyIiIh2i9VCSlJQEKysreHp6wtjYGIGBgdi/f3+55a5fvw6NRgONRgM9PT0YGRlpuzQiIiLSIVofPE2hUMDW1laYtrGxQXZ2NnJzc2Fubi60jxs3Dh9//DHmzp0LIyMjbNu2TdulERERkQ7Reii5f/8+TExM/rdBfX0YGBigqKhIXIi+PiIjI+Hp6Yndu3cjODgYhw8fLnfEJC8vD3l5eaK2rKws7e0AERER1QmthxITExMUFxcL06WlpSgpKREFldTUVJw4cQIBAQEAgPfeew+xsbE4d+4cPDw8RP1t27YN69ev13bZREREVMe0Hkrs7Oxw7NgxYVqhUKBFixaipx7evHmz3KOx9fX1K3xc85gxY+Dj4yNqu3jxIoKDg2u5ciIiIqpLWr/Q1dXVFQqFAomJiSgqKkJ0dDQGDBggWsbBwQFKpRJ79+6FWq3Gvn37kJeXh65du5brz8zMDNbW1qKXhYWFtneDiIiItEzrocTExARRUVGIiIiAu7s7iouLERISgoyMDDg6OiIjIwOWlpaIiorCtm3b4OzsjJ07d2Ljxo145plntF0eERER6Qitn74BgO7du+PgwYOiNlNTU6SkpAjT7u7uFd4qTERERI0Dh5knIiIincBQQkRERDqBoYSIiIh0AkMJERFRI7F161YsWLBAmN69ezfeeOMNODk5YdKkSbh165Ywb+3atXBxcYGbmxu2b99eJ/UxlBARETVwJSUlWLduHVauXCm0Xbx4EREREdi0aRNOnz6N559/Xggshw8fxokTJ/Dtt98iLi4OmzZtwl9//aX1OhlKiIiIGrilS5fi0qVLeOedd4S2W7duYdy4cWjfvj0MDAwwYsQI/PbbbwAehBK5XI5WrVqhffv28PHxqZM7ZBlKiIiIGjh/f39s2rQJrVq1EtrefPNNjB8/Xpj+4Ycf8NJLLwEArl69Cjs7O2GejY0N0tPTtV5nnYxTQkRERLUvKysLSqVS1GZmZgYzMzNR2+NGPv/ll18QHR2Nzz//HABQUFAAY2NjYb6xsXG5B+lqA0MJERHRU6qi574FBgYiKChIch/Hjh3DnDlzsHLlSnTr1g1A+YfpFhUViR6kqy0MJURERE+piIgIdOnSRdT26FGSquzZsweffvopoqKi4OLiIrTb2dnh2rVrcHV1BfDgYbq2tra1UnNVGEqIiIieUhYWFrC2tq7RuikpKVi8eDG2b99eLth4eXkhJiYGvXr1Qn5+Pvbt24fo6OjaKLlKDCVERESNUHx8PIqKiuDn5ye0GRoa4syZMxgwYADS09Ph6+sLjUaDgIAAdO3aVes1MZQQERE1Ev+91mTlypWicUv+SyaTISgoqFrXptQG3hJMREREOoGhhIiIiHQCQwkRERHpBIYSIiIi0gkMJURERKQTGEqIiIhIJzCUEBERkU5gKCEiIiKdwFBCREREOoGhhIiIiHQCQwkRERHpBIYSIiIi0gkMJURERKQTGEqIiIhIJzCUEBERkU5gKCEiIiKdwFBCREREOoGhhIieiFqtfir6JCLdp1/fBRDR001PTw+XLl167HL3798HAEnLdu7c+YnrIqKnD4+UEBERkU5gKCEiIiKdUOXpm+3bt1erMz8/vycqhoiIiBqvKkPJsmXL0KNHD0kdXbhwgaGEiIiIaqzKUGJkZITY2FhJHTk6OtZKQURERNQ4VXlNyYEDBypsLysrk7wsERERkRRVhpLnn38eAHD9+nXMnz8fAPDdd9+he/fu8PDwwK+//lpuWSIiImo88vPzq/WqiqRxSj755BNYWlpCo9FgxYoVmDJlCszMzLBo0SLs2bOnVnaKiIiInj7Ozs6SltNoNNDT08Mff/xR6TKSQklaWho2bdoEpVKJa9eu4d1330XTpk2xYsUKaRUTERFRvdu6dSsUCgXCwsIAAMeOHcOyZctw9+5deHl5ISwsDAYGBgCAtWvXYseOHdDT04O/v3+lN7MYGxsjISHhsdvWaDQYPHhwlctIHtG1sLAQp06dQpcuXWBqaoqbN2/imWeekbo6ERER1ZOSkhLExMQgJiYGw4cPBwBkZmZi3rx5+Oyzz9CuXTsEBATgq6++wnvvvYfDhw/jxIkT+Pbbb5Gbm4sxY8bA1dUVHTp0KNf3hAkTYGVlJamOCRMmVDlf0uBpgwYNwvDhw7F27VrI5XJcuXIFY8eOxdChQyUVQURERPVn6dKluHTpEt555x2h7cSJE3B3d0fnzp3RrFkzTJw4Efv27QMAHD58GHK5HK1atUL79u3h4+OD/fv3V9h3QECA8HNycjIAICsrC/Pnz8fKlStRUFBQ4bIVkXSk5OOPP4aTkxOaNWsGNzc3KJVKTJgwAcOGDZOyOhEREWlBVlYWlEqlqM3MzAxmZmaiNn9/f1hYWCAyMhJZWVkAgKtXr8LW1lZYpl27dkhPTxfmyeVyYZ6NjQ0SExOrrGXFihX49ttvcerUKSxYsAD37t2DgYEBwsLCsGzZMkn7IymUyGQyuLm54ebNm0hLSwPw4IFZly9fRseOHSVtiIiIiGpXcHBwubbAwEAEBQWJ2iwsLMotV1BQIGo3MjJCUVGRMM/Y2FiYZ2xsLMyrzIkTJ/Dll1/i/v37+PHHH5GQkAALCwv06dNH8v5ICiVffPEFPv3003Ljk8hkMvz555+SN0ZERES1JyIiAl26dBG1PXqUpDImJiYoLi4WpouLi2FiYlLhvKKiImFeZXJzc2FpaYnExERYWlrC1tYWJSUlUKvVUndHWijZsGEDIiMj0bt3b8hkMsmdExERkfZYWFjA2tq6Ruva2dkhJSVFmP7v6Rw7Oztcu3YNrq6uAACFQiE61VMRe3t7rFmzBmfOnMEbb7yB/Pz8CkNTVSRd6GpoaIjXX3+dgYSIiKiBeOONN/DTTz/h119/xb1797B582YMGDAAAODl5YXY2Fjcvn0bf//9N/bt2yfMq8zixYtx+fJlPP/885g2bRrS0tLw559/YuHChZJrkhRKJk+ejNDQUFy+fBkZGRmilxTJyckYNGgQHBwcEBAQUOGIbjk5OQgKCoKLiwv69++PH3/8UfJOEBERUfW0bdsWixcvxowZM9C7d2/Y2tpi7NixAIABAwagX79+8PX1xejRo+Hv74+uXbtW2M8XX3wB4MHFsBs2bMCnn36Kpk2bwsnJCfHx8aIR3x8uWxlJp2/UajX279+PvXv3AnhwLYlGo5F0TUlhYSGmTp2KpUuXwsXFBXPmzEFUVBRmzZolWm769Ono2LEjfv75Z5w5cwbTpk3Dzz//DCMjIyklEhER0WM8egHsm2++iTfffLPccjKZDEFBQeWWr8i6devw9ttvP3Y5jUaDdevWCcGnIpJCSWRkJFauXAk3Nzc0adJEyiqCpKQkWFlZwdPTE8CDq4JHjRolCiUZGRlIS0vDli1boK+vj9deew1xcXE8XURERKTjCgoKJA01//BgRlUkhRIDAwP07dsX+vqSB4AVPHpxjI2NDbKzs5Gbmwtzc3MAD4axt7Ozw+rVq7F//360adMG8+fPh6GhYbW3R0RERHXnceOXVIeklDF58mTMnz8fAQEBMDc3FyUdU1PTKte9f/++6DYifX19GBgYiO53vnfvHlJTU9GnTx+cOnUKJ06cQGBgII4dO4ZmzZqJ+svLy0NeXp6o7eFAMERERFS3pA4xL4WkULJ27Vrcu3dPGH4WgORrSh6917m0tBQlJSWioGJgYIBnnnkGEyZMgEwmw8CBAxETE4PU1FS89tprov62bduG9evXS9o5IiIienpICiX/DSPVZWdnh2PHjgnTCoUCLVq0QPPmzYU2W1tbFBUVoaSkRDhlo1arodFoyvU3ZswY+Pj4iNouXrxY4ah2RERE9PSo8pbgh8+2sbKyqvT16LKPcnV1hUKhQGJiIoqKihAdHV3uXueXX34ZNjY2WLNmDcrKynDo0CHk5OTAycmpXH9mZmawtrYWvSoaPpeIiIieLlUeKfnrr78QGxtb4RGLR125cqXCdhMTE0RFRSE0NBQzZsyAu7s7QkJCkJGRgUGDBuHQoUN47rnnsHnzZixYsAAuLi549tlnsX79etG4+0RERKTbzp8/j507d+L27dtYvXo1du3ahSlTpki+m7bKUOLg4CA69VKVbt26VTqve/fuOHjwoKjN1NRUNLytlZUVtm7dKmlbREREpFsSEhKwbNky+Pr64rvvvoNarcbBgwehUqnw4YcfSuqjylASGxtbK4USERFRwxYdHY0NGzagc+fO2LlzJ9q0aYMtW7bg3XfflRxKJA0zT0RERFSVO3fuoFOnTgAgnK557rnnRHfgPg5DCRERET2xV155Bdu3bxe1HThwAPb29pL7qP4QrURERESPmDt3Lt5//33s3r0bBQUFGDlyJK5fv44tW7ZI7qNaoaSkpAQZGRmwsbEBAD6bhoiIiAAAL774Io4cOYJTp04hIyMDbdq0gaenp2hcsseRdPqmqKgIc+fORbdu3TBkyBD8/fff8PLywvXr12tcPBERETUsZWVlePHFF+Hh4YEOHTrg5s2bSEtLk7y+pCMlS5cuRV5eHhISEvDOO+/AxsYGHh4eWLhwYbUOyxAREVHD9MUXX+DTTz9FWVmZqF3KI2kekhRKEhMTceTIEZiamkImk8HAwACzZs2Ch4dH9asmIiKiBmfDhg2IjIxE7969a3x5h6TTN/r6+lCpVAAgjO5aUFAgeqgeERERNV6GhoZ4/fXXn+h6U0mhpH///pg2bRpSU1MBAFevXsXcuXPRt2/fGm+YiIiIGo7JkycjNDQUly9fRkZGhugllaTTNyEhIVixYgX8/PxQVFSEwYMHw9vbG9OnT69x8URERNRwqNVq7N+/H3v37gXw4FoSjUZT+9eUGBoaYt68eZg3bx5ycnJgbm4OPT2Ou0ZEREQPREZGYuXKlXBzc0OTJk1q1IekUFJQUIBvvvkGSqUSarVaNO/jjz+u0YaJiIio4TAwMEDfvn2hr1/zcVklrRkcHAyFQgEHBwceISEiIqJyJk+ejPnz5yMgIADm5uaiC15NTU0l9SEplCQnJ+PUqVMwMzOrWaVERETUoK1duxb37t3Dvn37hDatXFPSvn175OXlMZQQERFRhf4bRmpKUiiZP38+xowZgzfffBPNmjUTzQsMDHziIoiIiOjplJ+fD1NT02o946YykkJJdHQ0SktLoVAoRFfU8oF8REREjVvPnj1x4cIFODk5lcsFWjl9c+bMGXz//fc8fUNERPQUOn36NMLDw3Hr1i106tQJ4eHhsLW1xY4dOxAdHY3i4mKMHDkSISEh1e770KFDAB48kuZJSbqVxtbWFvn5+U+8MSIiIqpbZWVlCA4OxsKFC3Hu3Dk4OjoiLCwMqampiImJwY4dO5CQkIDExEScOnWq2v23bdsWALBo0SJYWVmVe82cOVNyX5KOlPTv3x9yuRy+vr7lbvPx8/OrZvlERERUV/7991/k5uZCrVZDo9FAT08PRkZGOHz4MLy9vWFjYwPgwff5vn370KtXL8l937hxA9u3bwcA/PTTT1i6dKlo/r1795Ceni65P0mh5Oeff4aNjQ2Sk5NF7TKZjKGEiIhIh7Vs2RK+vr7w8/NDkyZNYG5uji+//BJLlixBnz59hOXatWuH+Pj4avVtZWUFfX195OTkQKPRIC8vTzTfwMAAa9askdyfpFASGxtbrSKJiIhI+7KysqBUKkVtZmZmomtAy8rKYG5ujtjYWHTr1g2RkZH46KOPYGxsDBMTE2E5IyMjFBUVVbuGGTNmAADs7e0xduzYmu3I/1dlKNm5cyfkcrlwaKYiPFJCRERUP4KDg8u1BQYGIigoSJg+evQolEolXn31VQDA9OnT4eTkhB49eqC4uFhYrri4WBRSqutJAwnwmFBy8uRJyOVyHD9+vML5PH1DRERUfyIiItClSxdR26N3ymZmZqKkpESY1tPTg0wmg4mJCa5duya0X716Fba2tlqt93GqDCWbN28GAKxevRoWFhbl5l+5ckU7VREREdFjWVhYwNrausplXF1dsXbtWvz0009wdXXF5s2b0a5dO4wfPx6BgYEYOnQomjZtitjYWNERlvog6Zbg/v37l2tTqVQYMWJErRdEREREtadTp05YvHgxwsPD4erqinPnziEyMhLdunVDQEAAxo0bh8GDB6Nfv34YMGBAjbdz+PDhCtt37doluY9Kj5QolUqMGDECpaWlKCgoEM5FPaRSqdCxY0fJGyIiIqL68dZbb+Gtt94q1y6XyyGXy2vc771793Djxg0AwJw5c/DCCy9Ao9EI8/Pz87F8+XKMHDlSUn+VhhJra2t89tln+PfffzFx4kSsX79eNN/Q0BD29vY12QciIiJqIN5//33k5OQAAIYMGSKaZ2BggLfffltyX1VeU/LwSMh3332Hli1bVrdOIiIiasCaNWuGX375BQDg6+uLvXv3PlF/kq4pYSAhIiKiqjxpIAEkDp5GREREVJWUlBQsW7YMCoUCarVaNO/s2bOS+mAoISIioicWHh6Ol19+GdOmTYO+fs3iRbXWKigoKJd+TE1Na7RhIiIiajj++ecf7Nq1C4aGhjXuQ1IoOXHiBBYuXIg7d+4IbRqNBjKZDH/++WeNN05EDVdsbCzi4uLKtT867tGoUaMwevTouiqLiLSkc+fO+Ouvv9C5c+ca9yEplERERGDEiBHw9vau8SEZImpcRo8ezbBB1Ih06NABY8aMQa9evdC6dWvRvI8//lhSH5ISxo0bNxAQEIAmTZpUv0oiIiJq8AoKCtCvXz8AQF5eXo36kBRK3Nzc8PPPP6Nnz5412ggRERE1bEuXLn3iPiSFEhMTEwQEBKBz587lDsk8OtIrERERNU4HDhzArl27cPv2bcTFxWHt2rUIDQ2FsbGxpPUlDZ5mZ2eHyZMnw8PDAx07dhS9iIiIiL744gvExMRgyJAhyM3NhbGxMa5evYolS5ZI7kPSkZLAwEDh5+zsbJiZmcHAwKD6FRMREVGDtGPHDmzZsgU2NjZYtWoVzM3NERUVhbfeegthYWGS+pB0pKS4uBihoaHo1q0bPDw80L17d8yYMQOFhYVPtANERETUMOTl5aFt27YAIDwp2NTUtNz4ZlWRFEpWrlyJ9PR0bN++HT/++CO2bduGmzdv4tNPP61B2URERNTQODk5YeXKlVCr1ZDJZACArVu3wsHBQXIfkk7fHDt2DN98843wYL7WrVsjIiICb731FhYsWFCD0omIiKghmT9/PiZPnoxXX30VBQUF6NmzJ0xNTbFx40bJfUgKJSqVqtyVs8bGxtDTk3SghYiIiBo4S0tLfP3117h06RJu3LiBNm3aoGvXrtW6BlVSqvDw8MC8efOQk5MD4MHFrqGhoXB3d69Z5URERNTgpKenIzs7G4aGhsjNzcUPP/yAxMREyetLOlIye/ZsTJkyBe7u7jAyMoJKpYKbmxuvKSEiIiIAwPLly7F9+3a0adNGuKYEAGQyGd544w1JfUgKJa1atcKuXbtw/fp1ZGdno23btrC0tKxZ1URERNTg7NmzB7t370anTp1q3EeVoWTnzp2Qy+XYvn27qD01NVX42c/P77EbSU5ORmhoKG7cuAF3d3esWLECpqamFS77999/Y8iQITh58iQsLCyk7AMRERHVs+bNm8PW1vaJ+qi6cEVOAAAgAElEQVQylJw8eRJyuRzHjx+vcL5MJntsKCksLMTUqVOxdOlSuLi4YM6cOYiKisKsWbPKLatWqzFv3jyoVKpq7AIRERHVt1mzZmH69Ol455130KxZM9E8Z2dnSX1UGUo2b94MAFi9enWFRy2uXLny2A0kJSXBysoKnp6eAB6MDjtq1KgKQ8n27dvRqVMnnD9/XlLxREREpBsuXryI77//HufOnUOTJk2EdplMhrNnz0rqQ9I1Jf3798eFCxdEbSqVCiNGjEBKSkqV6yoUCtHhHBsbG2RnZyM3Nxfm5uZC+/Xr17F792589dVXiI2NlVQ8ERER6Yb4+Hh89dVX6Nq1a437qDSUKJVKjBgxAqWlpSgoKMCrr74qmq9SqSQ9kO/+/fswMTH53wb19WFgYICioiKhTaPRYN68eZg1axaaNm1aZX95eXnIy8sTtWVlZT22DiIiItKe5s2bw97e/on6qDSUWFtb47PPPsO///6LiRMnYv369aL5hoaGkjZuYmKC4uJiYbq0tBQlJSWioPLll1+iTZs26Nmz52P727ZtW7laiIiIqH75+/sjICAAEyZMEJ0JASDpIAbwmNM3Dzv57rvvoFKp0Lx5c5iYmCAtLQ3NmjUTBYvK2NnZ4dixY8K0QqFAixYt0Lx5c6Ht+PHjSElJgZOTk9Dm5eWFjRs3itoAYMyYMfDx8RG1Xbx4EcHBwY+thYiIiLRj/vz5AICff/5Z1C6TyfDnn39K6kPSiK7nz5+Hl5cX/vnnHwAPLl4dMmQIvv/++8eu6+rqCoVCgcTERBQVFSE6OhoDBgwQLbN161ZcuHABycnJSE5OBgAcOXKkXCABADMzM1hbW4tevHWYiIioctevX8e4cePg7OyMoUOH4tKlSwCAHTt2wMPDA87Ozli1atUTbSMtLa3Cl9RAAkgMJatXr0ZMTIwwIMrYsWMRGRkpaURXExMTREVFISIiAu7u7iguLkZISAgyMjLg6OiIjIwMycUSERFR9ZSVlWH8+PHw9PTEmTNn8N577yEkJASpqamIiYnBjh07kJCQgMTERJw6deqJtpWfn4/du3cjMjIS9+/fl3zXzUOS7r7JzMyEi4uLqM3FxQWZmZmSNtK9e3ccPHhQ1GZqalrpnTuXL1+W1C8RERFV7cKFCzA0NMTYsWMBAMOGDcPLL7+MhIQEeHt7w8bGBsCDwVD37duHXr161Wg7f/75J95//320a9cOly9fhre3NyZMmIBFixZh8ODBkvqQdKTE3t4eX331lajtm2++wYsvvlj9qomIiKhWZGVlQalUil6P3qGalpYGW1tbfPzxx3BxccGoUaNgYmJSbsiOdu3aIT09vca1hIeHY86cOdi1axf09fXRrl07bNiwAdHR0ZL7kHSkZNasWZg4cSLi4+PRtm1bZGZm4tatW8LgakRERFT3KrrJIzAwEEFBQcL0vXv3kJiYiGXLlmHhwoXYvn07AgMD0bp1a9ENK0ZGRqLhOqrrypUrGDRoEAAID+Rzc3PD7du3JfchKZQ4Ojri6NGjOHXqFO7cuQNLS0t4enqWu+WHiIiI6k5ERAS6dOkiajMzMxNNGxgY4MUXX4S3tzcA4IMPPkBMTAysrKxEQ3YUFxdLuqu2Ms899xzOnTsnutwjNTUVzz33nOQ+qgwl+fn5MDU1RX5+PgwNDdGvX78K5xMREVHds7CwgLW1dZXL2NraIj8/X5jWaDRQq9UwNjbGtWvXhParV68+0QP1pk6dCn9/fwwZMgQqlQpr1qzBnj17MHfuXMl9VHlNycPBzJycnODs7Cx6PWwjIiIi3eXu7o6CggLExsairKwMW7duhbW1NSZMmIC9e/fin3/+we3btxEbG1tuyI7qeOONN7Bt2zZoNBq8+uqryMnJwdq1azFw4EDJfVR5pOTQoUMAgMTExBoXSURERPWnadOm+OKLL/DJJ59gzZo1ePHFFxEREYH27dsjICAA48aNQ2FhIeRy+ROFEuDB7cfTp09H8+bNcebMGdGD+aSoMpQ0a9YM+fn5otFXiYiI6OnSsWNH7Nq1q1y7XC6HXC6vlW3s3LkTa9asQXx8PJo3b46MjAwsW7YMs2fPLjcSe2WqDCVOTk7CFbSVqc5IbURERNQwbd68GbGxsXjppZcAAD4+PujUqROmTJlSO6Hk4WmbvXv34vfff0dwcDCsra2RmZmJiIgIvPzyy0+4C0RERNQQ/Pvvv2jfvr2orX379uXGTalKlRe6WllZwcrKCl9++SVWrVqFjh07wtTUFC+++CKWL1+OuLi4mlVOREREDUq3bt0QGRkJtVoN4MFdPhs2bEDnzp0l9yFpnJLS0lLcvXsXTZs2FdoyMzOhpydpQFgiIiJq4ObPn4/x48dj586dsLCwwJ07d2Bubo4NGzZI7kNSKBkxYgTGjh2LUaNGwdLSEhkZGYiLi8OYMWNqXDwRERE1HHZ2djhy5AguXLiA7OxstGnTBt26dYOBgYHkPiSFkunTp8PS0hKHDx9GVlYW2rRpg2nTpmHo0KE1Lp6IiIgaln///Rfm5ubCXbtpaWm4cuUKfH19Ja0vKZTIZDK89957eO+996BSqWBoaFjziomIiKjB2bZtG1asWCG6pkQmk8He3l5yKJF0UYharUZMTAx69uwJFxcXXL9+HX5+fsjJyal59URERNRgfP7554iMjMTGjRsxbNgwnD17Ft7e3nBzc5Pch6RQsnbtWvzwww9YvHgx9PX10bJlSzRv3hxhYWE1Lp6IiIgajry8PPTp0wedOnXCxYsXYWZmhjlz5uDw4cOS+5B0+ubAgQPYvXs3WrduDZlMhqZNm2Lp0qV44403alw8ERERNRzPPvsssrKyYGFhgczMTKhUKjRt2rRa45RICiUlJSV45plnADw4RwQAenp60NeXtDoRERE1cAMHDsSoUaMQHx+P1157DR9++CGMjIzQoUMHyX1IOn3z+uuvY+7cucjJyYFMJoNKpcKKFSuqdZ6IiIiIGq7AwEAEBQXBxMQEoaGhaNmyJTQaDZYvXy65D0mHOj7++GPMnDkT7u7uAABHR0e8+uqrWLVqVc0qJyIiogbnrbfeEn6uyXWnkkJJZmYmNmzYgOzsbNy4cQOWlpawtLSs9saIiIioYRk6dOhjH977zTffSOpLUijx8/NDYmIiWrVqhVatWknqmIiIiBq+2hzdXVIosbOzw6+//orXXnut1jZMRERETz8fHx9oNBphNNcnIfn2mfHjx6NFixZo06aN6DCN1EMyRERE1PBcuXIFEyZMwK1bt/DSSy8hMjIS7dq1q1Ffkh/IN2LEiBptgIiIiBqu5cuXY9CgQfDx8cH27duxfPlyREdH16ivx4YSjUaD3r17P/EhGSIiImp4fv31V2zcuBFNmjRBSEgIBg0aVOO+qhyn5MqVK+jduzfc3Nzg7e0NhUJR4w0RERFRw6PRaNCkSRMAQPPmzaFSqWrcV5Wh5OEhmYMHD8LBwaFaA6AQERFRw/dwpPfaUOXpm9o8JENEREQNj0ajweXLl4VwUlZWJpoGgI4dO0rqq8pQUpuHZIiIiKjhKSwsxNChQ0UhZMiQIcLPMpkMf/75p6S+HhtKiIiIiCqTlpZWa309NpTU1iEZIiIioqpUGUpq85AMERERUVWqDCW1eUiGiIiI6ldSUhImTZqE3377DQCwY8cOREdHo7i4GCNHjkRISEi91lflLcFERETUMBQWFiI0NBRqtRoAkJqaipiYGOzYsQMJCQlITEzEqVOn6rVGhhIiIqJGYPXq1Xj99deF6cOHD8Pb2xs2NjawtLSEn58f9u3bV48VMpQQERE1eCkpKUhNTcWYMWOEtqtXr8LW1laYbteuHdLT0+uhuv+R/JRgIiIi0i1ZWVlQKpWiNjMzM5iZmQnTKpUKoaGh+PTTTyGTyYT2goICmJiYCNNGRkYoKirSftFVYCghIiJ6SgUHB5drCwwMRFBQkDAdGRmJPn36wN7eXhRgTExMUFxcLEwXFxeLQkp9YCghIiJ6SkVERKBLly6itv8eJQGA48ePIysrC3FxcVCr1VCpVHByckLfvn1x7do1YblHT+fUB4YSIiKip5SFhQWsra2rXObIkSPCz0qlEgMGDEBycjJ+++03BAYGYujQoWjatCliY2NFR1jqA0MJERFRI9StWzcEBARg3LhxKCwshFwux4ABA+q1JoYSIiKiRsLa2hoXL14UpuVyOeRyeT1WJMZbgomIiEgnMJQQERGRTmAoISIiIp3AUEJEREQ6gaGEiIiIdEKdhJLk5GQMGjQIDg4OCAgIQH5+frllLly4AF9fX3Tv3h0+Pj64cOFCXZRGREREOkLroaSwsBBTp07FzJkzkZSUBGNjY0RFRYmWKSoqQmBgICZNmoTk5GRMmDABU6ZMqfcx+ImIiKjuaD2UJCUlwcrKCp6enjA2NkZgYCD2798vWub27dvw8PBA//79oaenh4EDB0KtVuP69evaLo+IiIh0hNZDiUKhEI2lb2Njg+zsbOTm5oraVqxYIUynpqaiuLgYzz//vLbLIyIiIh2h9RFd79+/L3rqoL6+PgwMDCo9NXPz5k1MmzYNwcHBMDY2Ljc/Ly8PeXl5orasrKzaLZqIiIjqnNZDyaOPRi4tLUVJSUmFj0dOT0/HBx98gCFDhmDs2LEV9rdt2zasX79eW+USERFRPdF6KLGzs8OxY8eEaYVCgRYtWqB58+ai5X7//Xd88MEHmDRpEsaNG1dpf2PGjIGPj4+o7eLFiwgODq7dwomIiKhOaf2aEldXVygUCiQmJqKoqAjR0dHlnkKYl5eHSZMmISgoqMpAAgBmZmawtrYWvSwsLLS5C0RERFQHtB5KTExMEBUVhYiICLi7u6O4uBghISHIyMiAo6MjMjIycPToUWRlZWHlypVwdHQUXqmpqdouj4iIiHSE1k/fAED37t1x8OBBUZupqSlSUlIAAMOHD8fw4cProhQiIiLSURxmnoiIiHQCQwkRERHpBIYSIiIi0gkMJURERKQTGEqIiIhIJzCUEBERkU5gKCEiIiKdwFBCREREOoGhhIiIiHQCQwkRERHpBIYSIiIi0gkMJURERKQTGEqIiIgauJMnT2LgwIHo0aMH3nvvPaSnpwMAduzYAQ8PDzg7O2PVqlX1XCVDCRERUYOWmZmJ2bNnIzw8HGfPnkXv3r0xbdo0pKamIiYmBjt27EBCQgISExNx6tSpeq2VoYSIiKgBu3nzJoYNG4YePXqgSZMmkMvluHLlChISEuDt7Q0bGxtYWlrCz88P+/btq9da9et160RERFRjWVlZUCqVojYzMzOYmZkJ046OjnB0dBSmf/jhB1hZWeH69evo06eP0N6uXTvEx8drv+gqMJQQERE9pYKDg8u1BQYGIigoqMLl09LS8MknnyA8PBzx8fEwMTER5hkZGaGoqEhrtUrBUEJERPSUioiIQJcuXURt/z1K8l/JycmYMmUKpk+fjn79+mHv3r0oLi4W5hcXF4tCSn1gKCEiInpKWVhYwNra+rHL/fDDD/jwww8RFhaGgQMHAgDs7Oxw7do1YZmrV6/C1tZWW6VKwgtdiYiIGrDr168jODgYK1asEAIJAHh5eWHv3r34559/cPv2bcTGxmLAgAH1WCmPlBARETVoe/bsQUFBAUJCQkTtJ06cQEBAAMaNG4fCwkLI5XKGEiIiItKe6dOnY/r06RXOk8vlkMvldVxR5RhKiKhBKy0thb6++KNu2LBhSEpKeuy6rq6u+PrrryX1SURPjv+riKhB09fXx8KFC0VtXbt2RdeuXSWt/+i6ABAaGlortRGRGC90JSIiIp3AUEJEREQ6gaGEiIiIdAJDCREREekEhhIiIiLSCQwlREREpBMYSoiIiEgnMJQQERGRTmAoISIiIp3AUEJEREQ6gcPMExE1IJU9l+dJnvfDZ/1QXeG7jIioAanoWT/Akz3vh8/6obrCUEKNljb+oqyqXyIiqho/OanR0sZflAD/qiQiqile6EpEREQ6gaGEiIiIdAJDCREREekEXlNSicZ4W11F9fGiTyIiqiv8tqhEY7ytrqJ95kWfRERUV3j6hoiIiHQCQwkRERHpBIYSIiIi0gkMJURERKQTGEqIiIhIJzCUEBERkU6ok1CSnJyMQYMGwcHBAQEBAcjPzy+3zP/93//h7bffhoODA0aNGoXMzMy6KI2IqNrUanW5tlWrVsHKyuqxr1WrVknuk6i2SPke1gVaDyWFhYWYOnUqZs6ciaSkJBgbGyMqKkq0jEajQXBwMN5++22cPXsWXbp0wZIlS7RdGhFRjejp6eHSpUui1+3btyWte/v27XLrXrp0CXp6PHCt6540eNYXKd/DukLrg6clJSXBysoKnp6eAIDAwECMGjUKs2bNEpa5cuUK7t69i5EjRwIAgoKC4Orqivz8fJiammq7xDqjVqsr/OBZtWoVVq9e/dj1P/zwQ4SEhEjuVxdoY591eX+Biut70pFxdX2fCRg9ejRGjx5d32VoRWXvv4b8vtZGbfW1v1K+h3WF1kOJQqGAra2tMG1jY4Ps7Gzk5ubC3NxcWKZdu3bCMs888wxatGgBhUKBV155RdRfXl4e8vLyRG03btwAAKSlpdVq7VL/8pEqIyMDf/31V7n2f/75BzKZ7LHr//PPPzh58mS59g4dOtTa6a6nYZ91eX+Bivd5ypQpmDJliqT1n7Z/Y6VSqZU+a/MUbmPb57r6f6xL7+va9uyzz5bbZ0dHR8TGxkpaX9v7+/D77saNG3j22WdF88zMzGBmZiZMS/ke1hVaDyX379+HiYnJ/zaorw8DAwMUFRVVugwAGBsbo7CwsFx/27Ztw/r16yvcVnh4eC1VrR0nTpyodF7Tpk0fu/7x48dx/Pjx2ixJ67jPDV9j21+g8e1zY9vfp8nMmTPLtQUGBiIoKEiYlvI9rCu0HkpMTExQXFwsTJeWlqKkpET0C3p0GQAoKirCM888U66/MWPGwMfHR9SWn5+Pn376Ca+88goMDAxqeQ+IiIh0S0lJCX7//Xe89tpraNasmWjef4+SANK+h3WF1kOJnZ0djh07JkwrFAq0aNECzZs3Fy2jUCiE6YKCAmRnZ8PGxqZcf48elnqoY8eOtVw5ERGR7nJzc5O0nJTvYV2h9StuXF1doVAokJiYiKKiIkRHR2PAgAGiZTp06IBmzZohPj4eKpUK69evh7u7e4O6yJWIiKg+SPke1hVaDyUmJiaIiopCREQE3N3dUVxcjJCQEGRkZMDR0REZGRkAgMjISBw4cAAuLi64dOmSzl8fQkRE9DSo7HtYF8k0Go2mvosgIiIi0s0bxImIiKjRYSghIiIincBQQkRERDqBoYQey97eHg4ODnB0dBRefn5+wkXKGzZswOLFiytct0+fPvj111/rslx6ijx8DzUEDWlfiOoLQ0k1KZVKdOnSpVz7ggUL8Pnnn9dDRXXj+PHjSElJQUpKCn788Ue0aNECy5YtAwBMnjwZc+fOrecKtSs9PR3jx49H9+7d0aNHD0yYMAHp6ekICQlBaGhoueVPnz6Nnj17Qq1WY/To0ejSpYsQ6BwcHDBo0CCcPn26HvakvMpCp7bduXOn3ECItUmpVMLe3l60X6NGjcLVq1drfVtxcXHYsGEDAODAgQOSh16vTw9/PxU9ryc8PBz29vaVft4BD+6YfOWVV0S/35EjR+LSpUvaLr1aqnp/Dxo0CKmpqbWyncjISCxYsKBW+mrMGEpqSVhYGMaNG1ffZdQJU1NTDB48GOnp6QDE/xnT09Px9ttvw9HREfPnz0dZWZmw3smTJ/Hmm2/itddew8aNG9GpUydh3u7du/Hmm2/C3d0dixYtgkqlqtudqkJZWRkmTJiAvn374uzZszh9+jScnJwwYcIE+Pr64ujRoygtLRWtc/DgQQwZMkR4+NayZcuEUJecnAxfX1989NFHot9Pffpv6ExJScH27du1vs2ioiIUFBRodRuGhobCPp07dw5dunTB/Pnza307d+/eFX729vbW2SewPkpfXx9paWnIzs4W2jQaDb7//ns0adLksesPGzZM+P1euHABQ4YMgb+/f7n/D/Wtsvf3oUOH0LVr13qujv6LoaSWzJ49G5s2bQIAdOrUCZ999hnc3Nzg6emJb775RlguJSUFvr6+cHZ2xsSJE3Hr1i0AD4bKnzFjBnr37o2uXbvCz88Pd+7cAfDg6aOzZs2Ci4uL8NdYfbp79y527twJJyencvOCg4PRr18/nD17Fra2tsLDp7KysvDRRx9h4cKFSExMxJ9//il8IZ87dw7r16/Hpk2bcOzYMSiVSmzZsqVO96kqd+/eRUZGBgYOHAh9fX0YGhpi4sSJ8PDwgL29PYyMjPDLL78Iy6tUKhw/frzSowD6+voYMWIE7ty5g3///beudqNGbty4gQkTJsDJyQkDBw5EYmIigAd/Zffu3RsTJ06Ei4sLFAoF/v77b4wePRrOzs549913hdCan58Pf39/ODs7o2/fvvjss88APHhkhEqlgqOjY52EUH19fXh7ewsPMjt69Ci8vLzg7OyMyZMnC/8XNRoN1q5dCw8PD3h4eIhC8pkzZzBo0CA4Oztj5MiR+P333/HLL79g48aN+PrrrzF79mzs3bsXH3zwAYAH/3cjIyPRt29fvPrqq1i0aBEejsKQkpKCwYMHw8XFBYsWLUKfPn2gVCq1/nv4Lz09Pbz++uvCvysAnD9/Hvb29pJCyX/JZDIMHToUt2/fLvfQVF318PRyXFwc+vXrB5VKBbVajXfeeUf4PK/sM7ukpAShoaHo0aMHBg8erJUjcI0RQ4kWlJWV4e+//8b333+P2bNnIywsDMXFxcjLy4O/vz8CAwPxyy+/wMHBQXh09ObNm6FSqXD06FGcPn0aMpkMu3btEvrMzMzEqVOnMGrUqHrZJy8vL3Tv3h2dO3fGW2+9hQ4dOmD27NmiZRQKBZRKJd5//30YGBhg3LhxaNmyJQDg+++/h6OjI1577TUYGxtj+vTpwnr79++HXC7HCy+8AFNTU0yePBn79++v0/2rSuvWrdGlSxfhgyo1NRVqtRphYWFo3bo1hgwZgm+//VZY/uTJk2jfvj1eeOGFCvtTqVTYsmUL2rdvL/x+dNWMGTNgb2+PX375BWFhYZg1a5YQNjIyMvDGG28gMTERVlZW8Pf3x8CBA3H69GkMHz4cU6ZMQVlZGXbt2oVnnnkGSUlJ2LJlCzZu3Ihbt25h27ZtwpEMQ0NDre9LSUkJ9u3bh549eyI9PR0LFizA4sWL8fPPP8POzg4ffvghAGDPnj04ceIE9uzZg4SEBFy+fBnR0dEAHhwRnTlzJs6dO4c+ffpg7dq1cHd3x6RJkzBs2DDhlOZ/fffdd/jyyy/x1VdfYf/+/Th//jyKi4sRFBSEsWPH4scff4S+vr7wtPO61r9/fxw9elSYPnLkCLy8vKrdT1FREbZv3w47Ozudf18/6t1334W5uTm2bduGuLg4lJaW4oMPPqjyMzsuLg5//PEHEhMTsXr1avz888/1vBcNA0OJlvj5+cHQ0BADBgyASqVCdnY2Tp06hU6dOqFPnz4wMDDApEmT8Ntvv+H27dsYO3asMIrtzZs30bx5c2RlZQn99enTByYmJvU29P6RI0dw4cIFbNmyBRqNBq6uruUe5pSdnY2WLVtCX//BI5X09PRgaWkpzGvTpo2wbNu2bYWfMzMzsWHDBjg5OcHJyQkffPCB6HCyLvj8888xYMAAHDx4EMOHD4eHhwd27NgBAPDx8cGJEyeEv6YPHjwIX19f0fpz586Fk5MTunTpAmdnZ1y/fl34S0wXeHl5Cb9/JycnXL16Fbdu3cIff/yBqVOnwtDQEE5OTujXrx+OHDkirDdw4ECYmprit99+AwDI5XLo6+sLR4lSU1NhZmaGS5cu4ejRo2jVqhWSkpKE94W2qVQqYZ8cHBwQHx+PYcOG4fjx43jzzTfRo0cPGBoaYvr06bh06RJu3ryJw4cPY/z48Xj22Wdhbm6OadOmISEhAQDQrFkzJCQk4Ndff8UHH3wg6d/Q19cXLVu2hK2tLV555RUolUpcuHABTZs2xbBhw2BoaIjg4GDh/01d8/T0RGpqKvLy8oRTN71795a07t69e+Hk5ARnZ2d4enoiKSkJa9eu1XLF1VfR+/u/9PT0EBYWhi1btiAqKgpLly5FkyZNqvzMPnbsGMaOHQtzc3O89NJLGDp0aP3sXANTP/8LGoEWLVoIP+vp6UGtViMzMxNnz54VnfYoKytDZmYmmjVrhnnz5iE9PR329vYoLi6Gubm5sFyrVq3qtP7KuLq6YsaMGfjoo49w4MABUbiwsLBAdnY2VCqV8Jfvw1NQlpaWOHv2rLDsw0OgD9cLCQkRLrgrLCwU1tMVpqamCAoKQlBQEHJycnD06FEsW7YM7du3h4uLC9q3b48ffvgBr776Ks6ePYvly5eL1l+8eDEGDRoEpVIJf39/2NnZwdraup72prwjR47AwsJC1Pbbb7+hZcuWoqMYbdu2Ff7tDAwMhKeT3rp1C0qlUvTeLikpQWZmJoYPH467d+9i7dq1mDlzJry9vSu8OFgbDA0NkZycDABQq9U4deoUpkyZgh49esDBwUG0XKtWrZCZmYmbN2+K3tfPPvuscBpy9erVWLVqFcaPHw9jY2PMmjULgwcPrrKG/34WNGnSBGq1GtnZ2aJgZmxsLFquLhkbG8PNzQ3fffcdrK2t0aFDB8l//Pj6+iIsLEzLFT65it7fj+rYsSOsra2hVqvRoUMHAKjyMzsnJ6fcH1oPjyJSzfFISR2ysLBA7969kZycLLy++eYbdOrUCQsXLkTPnj1x+vRpbNu2DZ07dxatK5PJ6qnq8nx8fNC5c2csWbJE1P7888/jxRdfRHR0NEpKShAfHy8c7enVqxdSU1ORlJQkPHTxoYEDByIuLg7Xr1+HSnK0VMMAAAVySURBVKXC0qVL8cknn9TlLlXp0KFDGD58uDDdsmVLyOVyuLi44K+//gLw4Hfy7bff4ujRo/D09Kz0Q93a2hoRERHYsmULTp48WSf115SFhQVycnJE13tkZGQIX57/fU+2bt0a9vb2ovf2gQMH0KdPH6Snp8Pb2xtHjx7FgQMHkJqaWi+n5/T09NCnTx/Y2NjAwcEBN2/eFOapVCrcuXMHLVu2hIWFhWheRkYGWrZsibKyMigUCqxatQpJSUmYM2cOPv74YxQWFla7FktLS1EwV6lUyM3NfbIdfAJeXl44fvw4jh49qrMPatO2b7/9Fvfv34darcbXX38NoOrP7NatW4veJ7r2h9TTiqGkhjIzM0WvkpKSx67j6emJ5ORknD59GhqNBvv27YNcLkdJSQny8/NhZGQEmUyGc+fOYe/evTp3Bft/zZ8/H9999x1+/PFHUfuaNWtw9uxZODs748yZM8JfHObm5li6dClmz56NXr16oWXLljAwMAAAvP766/Dz88PYsWPh5uaGjIyMcoGnPrm6uuLq1avYtGkTsrOzUVpaiuTkZFy8eBHu7u4AHgSrM2fO4NChQ+VO3Tyqffv2mDRpEsLCwmr0hVZXnnvuOXTs2BHr1q2DSqXC+fPnceTIEfTr16/csg4ODsjNzcXBgwehVqtx+vRpeHt7Izs7GwcPHkR4eDgKCwvRunVryGQymP2/9u4vlPU/DOD425hCKduJqJlGuSFaYchIyXAl5N8ludqVUsLQ2qbsj5pMiSi5crVczZV7sUspFxMXViOtdrH507nQWcfv/Pyc38mZdc7zuvys73o+N8/3+X6ez+f7zc8nOzubp6cn4vF4yuZ0fHzM1dUVHR0dBAIBTk9PSSQSrKysoNPp0Gq19PT0sLm5yc3NDff396yurmIymVAoFExPT3NwcEBWVhYFBQXk5OQkNz/HYrGfjqO2tpZEIoHf7+fh4YG1tbWfyiG/S1tbG8fHx2+2bv6Z79LpdNxHiEajOBwOLBYLc3NzLC8vc3d39585u7u7m62tLSKRCKFQ6NWBBvHrpH3zCxKJBK2tra/GGhsb371OpVLh9XpxOBxcXl5SWlqKz+cjJyeHqakpLBYLXq+XsrIyhoaGCAaDv2sK/8v5+fkPYzqdLvk+gpaWluS4RqNJ7rX43u3tLRqNhqOjIwBCodCrp+XR0VFGR0c/OPKPoVar2d3dxe12s7GxwePjIxUVFTidTsrLywHIz8+nvr6eYDCIwWB49z/Hx8fx+/34fL60/VonvLQrFhYWaGpqQqVSYbPZkvsivpednc36+jpWq5XFxUW+fPmCy+WipKSEiYkJZmZmMBqNKBQKent76ezsBKChoQGDwUAgEHi1FP5Rvp3u+aa4uBibzUZlZSU2m42ZmRnC4TB6vT65etff3084HGZgYIB4PE5XVxeTk5NkZGTgdruxWq3Mz89TVFSEx+NBqVRiNBrZ2dnBbDbT3t7+blxKpRKXy8Xs7Cx2u52+vj6USmWyUE+13Nxc6urqAMjLy3v127/lu729vZTFlgoulwu9Xk9zczPwktOWlpZwOp1v5uzh4WGur68xmUyo1WqMRuMnz+LPIF8JFilxdXXF4OAg+/v7FBYWYrfbiUajeDyezw5NiJSLxWJcXFxQU1MDvNz4a2trOTk5+WEDuRB/E2nfiJTQaDSYzWZGRkYwGAyEw+E//i2wQrwlMzOTsbExzs7OeH5+Znt7m+rqailIxF9PVkqEEOITHB4e4nQ6iUQiVFVVYbPZ0Gq1nx2WEJ9KihIhhBBCpAVp3wghhBAiLUhRIoQQQoi0IEWJEEIIIdKCFCVCCCGESAtSlAghhBAiLUhRIoQQQoi08BU7s75qMytkHwAAAABJRU5ErkJggg==\n"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "comparison_barplot_one(all_times_ms, placement_times, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib.ticker import ScalarFormatter\n",
    "\n",
    "\n",
    "def comparison_barplot_two(pred_times, place_times, labels):\n",
    "    \"\"\"Plot both in two bar plots\"\"\"\n",
    "    sns.set(font_scale=1.1, style='white')\n",
    "    assert len(pred_times) == len(place_times) == len(labels)\n",
    "    \n",
    "    fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, sharey='all', figsize=(8, 4))\n",
    "    \n",
    "    # prepare data\n",
    "    pred_times_mean = [np.array(t).mean() for t in pred_times]\n",
    "    pred_times_std = [np.array(t).std() for t in pred_times]\n",
    "    # 95% CI (not bootstrapped like seaborn does): https://en.wikipedia.org/wiki/Confidence_interval#Basic_steps\n",
    "    pred_times_ci95 = [1.96 * std / np.sqrt(len(pred_times_std)) for std in pred_times_std]\n",
    "\n",
    "    place_times_mean = [np.array(t).mean() for t in place_times]\n",
    "    place_times_std = [np.array(t).std() for t in place_times]\n",
    "    place_times_ci95 = [1.96 * std / np.sqrt(len(place_times_std)) for std in place_times_std]\n",
    "    x = np.arange(len(labels))\n",
    "\n",
    "    # plot: separate axes\n",
    "    ax1.barh(x, pred_times_mean, xerr=pred_times_ci95, capsize=5, color='gray', label='Prediction')\n",
    "    ax1.set_xlabel(\"Prediction Time [ms]\")\n",
    "#     ax1.set_xscale('log')\n",
    "    ax1.set_yticks(x)\n",
    "    ax1.set_yticklabels(labels)\n",
    "    ax1.tick_params(axis='both', direction='inout', length=5, bottom=True, top=True)\n",
    "\n",
    "    \n",
    "    ax2.barh(x, place_times_mean, xerr=place_times_ci95, capsize=5, color='lightgray', label='Placement')\n",
    "    ax2.set_xlabel(\"Total Time [s]\")\n",
    "    # log scale; avoid \"power 10\" labels\n",
    "    ax2.set_xscale('log')\n",
    "    ax2.tick_params(axis='both', direction='inout', length=5, bottom=True, top=True)\n",
    "    ax2.xaxis.set_major_formatter(ScalarFormatter())\n",
    "    \n",
    "    plt.tight_layout()\n",
    "    fig.savefig(f'plots/runtimes.pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 576x288 with 2 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjEAAAERCAYAAACKK/ZfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XlcVPXeB/APOAwQaIoCKkiaT1CGN0ZBtpuoqSBcMaHFBVxy300g8VG0EJVSMECtxK4LkJm75oJ7WYlAYuq9KU/4yJWQ1ZBwYWBmnj98nFco4qAzc+Ywn/frxUvmLL/z/f1gjh/OOXOOiUqlUoGIiIhIZEyFLoCIiIjoaTDEEBERkSgxxBAREZEoMcQQERGRKDHEEBERkSgxxBAREZEoMcQQERGRKBlViKmurkZKSgqqq6uFLuWZsB+Ghf0wThwvYXH8hWUo4y8RdOt6VlFRgTVr1qBPnz5wcHAQupynVlJSgjVr1sDb2xsdO3YUupynxn4YFkPph0KhQHl5OVxdXWFhYaHVtu/du4dLly7B1tYWrVq1eqa2DGW8jBXHX1i6HP/m7ANMjOmOvYcOHcLcuXOFLoOINJCRkQF3d3ettpmbm4vRo0drtU0i0g1N9gFGdSTGxsYGAPDpp5+iZ8+eAlfz9EpKSjB69GhkZGSI+i8Q9sOwGEo/HtRha2ur9bYftKmNPhrKeBkrjr+wdDn+zdkHGFWIeXD42NbWFo6OjgJX8+w6duzIfhgQ9kO7nvV0T1NtarOPhjJexorjLyxdjr8m+wCjurDX2tq6wb9i1aZNG8ycORNt2rQRupRnwn4YlpbSD33heAmL4y8sQxl/o7ompqioCG+88QaOHz/O5E5koHT5PuU+gMjwNed9alRHYoiIiKjlYIghIiIiUWKIISIiIlFiiCEiIiJRYoghIiIiUWKIaUJ9fb3QJRCRCCiVSqFLIDJKRnWzuweSkpI0ulfMkiVL9FANEelbfn4+qqqqtNaeq6ur1toiIs3xSAwRERGJEkMMERERiRJDDBEREYkSQwwRERGJEkMMERERiRJDTBNCQ0OFLoGIBJaWlvZM6yckJGipEiJ6GENME7KysoQugYgElp6e/kzrJyYmaqkSInqYTkJMUVERevbs+cj0xYsXY+PGjbrYJBERERkZvd7sLjY2Vp+bIyIiohZMryEmOjoaL774IiZPnowePXogMjISqampkEqlmDt3LoYPHw4AyMvLw9KlS3H9+nXIZDIsXboU9vb2qKmpwUcffYTc3FxUVlbCzc0NiYmJ6NChA8LDw9G5c2ecOnUK48ePx9SpU/XZNSJqwaKiopqcb2VlpadKiOivBLsmRqFQ4OrVq/juu+8QHR2N2NhY1NbWorq6GtOmTcPMmTPx008/wc3NDfPnzwcApKamQi6XIzMzE2fOnIGJiQm+/vprdZslJSU4deoUwsLChOoWERER6Ymgz04aM2YMpFIphgwZgsjISFRWViI3Nxc9evTAgAEDAABTpkxBamoqysrKMG7cOLRq1QoAcOPGDTz//PMoLy9XtzdgwABYWloK0hciarlWrlzZ5Pymnp3k4OCg7XKI6P8JGmLatWun/t7U1BRKpRIlJSXIzs6Gu7u7ep5CoUBJSQlat26NRYsWoaCgAC4uLqitrUXbtm3Vy7Vv316v9RMREZFwDO4p1ra2tujfvz9SUlLU0woKCvDCCy9g4sSJ6Nu3L9LT02FiYoK4uDjI5XL1ciYmJkKUTERERALQ6TUxJSUlDb7q6uqeuI6fnx9yc3Nx5swZqFQq7NmzByNHjkRdXR1qampgbm4OExMT5OTkYNeuXaivr9dlF4iIiMhA6exIjFwuh5+fX4Np3t7eT1zPxsYGycnJWL58OQoLC+Hk5IR169bB0tISUVFRiImJQXJyMrp27YoRI0YgLy9PV12Al5eXztomInF41g8KzJs3T0uVENHDdBJiHB0dceXKlSaXeXj+xYsX1d97eHhg9+7dj6zj6emJI0eONNres94avDE7d+7UeptEJC7h4eHPtH5ERISWKiGih/GxA0RERCRKDDFEREQkSgwxREREJEoMMURERCRKDDFEREQkSgZ3szt9mDNnDhwdHZ+4XH19PSQSoxwiohbN2dlZo32AppRKJUxN+Tchkb7xXdcEBhgi0gQDDJEw+M4jIiIiUWKIISIiIlFiiCEiIiJRYoghIiIiUWKIaQKfkE1Ej6NUKoUugcjoGeXHb5KSkmBtbf3E5ZYsWaKHaohI3/Lz81FVVfVMbbi6umqpGiJ6WjwSQ0RERKLEEENERESixBBDREREosQQQ0RERKLEEENERESixBDThNDQUKFLICKBpKWlPdV6CQkJWq6EiB5H5yHGxcUFbm5ukMlk6q8xY8aguLgYAPD5559j2bJlja47YMAAnD9/XtclPlZWVpZg2yYiYaWnpz/VeomJiVquhIgeRy/3iTl69ChsbW0BADU1NVi4cCHi4+ORnJyMqVOn6qMEIiIiamH0fjrJ2toaQ4cORUFBAQAgJSUFixcvBgAUFBTgrbfegkwmQ0xMDBQKhXq9EydOYODAgfD19cUXX3yBHj16qOdt374dAwcOhI+PD+Li4iCXy/XbKSIiItI7vd+x948//sDWrVvh7u7+yLy5c+di6NChGD9+PLZs2YJvvvkGAFBeXo7IyEikpKSgd+/eiI6OVgecnJwcrFmzBhs3boSdnR0iIyOxYcMGTJ8+Xa/9IqKWJyoq6rHzrKys9FgJETVGL0diAgIC0KtXL7i6uuIf//gHnJ2dER0d3WCZwsJCFBUV4b333oOZmRnGjx8PGxsbAMB3330HmUwGX19fWFhY4P3331evt3fvXowcORIvvvgirK2tMXXqVOzdu1cf3SIiIiIB6eVIzOHDh2Fra4usrCzMmzcPXl5esLS0bLBMZWUlbGxsIJHcL8nU1BT29vbqeXZ2duplO3XqpP6+pKQE3377LTZs2AAAUKlUMDEx0XWXiMgIrFy58rHzHvfsJAcHB12VQ0QP0evpJC8vL0RFRSEyMhL79u1rEEZsbW1RWVkJuVwOqVQKAKioqAAA2NvbIzs7W71saWlpg/UiIiIQHh4OALh79656PSIiImq59H5h7/Dhw+Hq6orly5c3mN6lSxf813/9F9atW4e6ujpkZGSgvLwcANCvXz9cuHABWVlZkMvlWLNmjXq9wMBApKen4/r165DL5VixYgU+/PBDfXaJiIiIBCDIze5iYmJw8uRJnD59usH01atXIzs7Gx4eHjh79iycnZ0BAG3btsWKFSsQHR2Nfv36wcbGBmZmZgCA119/HWPGjMG4cePg7e2N4uLiRwISERERtTw6P5105cqVR6a9+OKLuHTpEoD7IeSBLl264Kuvvnpk+crKSnTp0gWnTp0CAFy7dq3BxbujR4/G6NGjtVz5/dNfRGScwsLCnmq9efPmabkSInocUTx24M6dOxg/fjx+//131NXVYdOmTXoJGDt37tT5NojIMD24zq65IiIitFwJET2O3u8T8zS6dOmCmTNnYtSoUaipqUGfPn0QFxcndFlEREQkIFGEGAAYNWoURo0aJXQZREREZCBEcTqJiIiI6GEMMURERCRKDDFEREQkSqK5Jkab5syZA0dHxycuV19fr34MAhG1HM7OzhrtA5qiVCphasq/A4mExHdgExhgiOhxGGCIhMd3IREREYkSQwwRERGJEkMMERERiRJDTDPU19cLXQIRGRilUil0CURGyyivXE1KSoK1tXWz11uyZIkOqiEifcvPz0dVVZVW2nJ1ddVKO0TUfDwSQ0RERKLEEENERESixBBDREREosQQQ0RERKLEEENERESixBDTDKGhoUKXQEQCSEtLa/Y6CQkJOqiEiP6KIaYZsrKyhC6BiASQnp7e7HUSExN1UAkR/ZWgIaagoAATJ05Er1690Lt3b0yaNAkFBQWIiIho9J4sZ86cQd++faFUKhEeHo6ePXtCJpNBJpPBzc0NQUFBOHPmjAA9ISIiIn0TLMQoFApMmjQJgwYNQnZ2Ns6cOQN3d3dMmjQJISEhyMzMfOQOufv378ewYcPUT4+Nj49HXl4e8vLykJubi5CQEERGRkKhUAjRJSIiItIjwe7Y+8cff6C4uBiBgYGQSO6XMXnyZPz+++9wcXGBubk5fvrpJ/Tt2xcAIJfLcfToUWzbtq3R9iQSCd555x188sknuHXrFmxsbPTWFyJq+aKiohqdbmVlpedKiOgBwY7EdOjQAT179sS7776L9evX48KFC1AqlYiNjUWHDh0wbNgwHDx4UL38iRMn0L17d7z44ouNtieXy7FhwwZ0796dAYaIiMgICPrspI0bN2Ljxo3Yv38/EhISYGNjg1mzZmHUqFEYPnw43n77bcjlckilUuzfvx8hISEN1l+4cCGWLFmC2tpamJqa4o033sD69esF6g0RtWQrV65sdPrjnp3k4OCgy3KICAJf2GttbY1Zs2Zh//79OHPmDGbPno2PP/4YZ8+eRbdu3dC9e3d8//33qK6uRnZ2NgIDAxusv2zZMuTm5uLQoUNwcnJCt27d4OjoKFBviIiISJ8ECzEHDhzA22+/rX5tY2ODkSNHwtPTE/n5+QCA4cOH4+DBg8jMzISfn99jnzzt6OiITz/9FBs2bMCJEyf0Uj8REREJS7AQ4+XlhWvXrmH9+vWorKxEfX09cnNzcfHiRfj4+AAAAgMDcfbsWRw4cOCRU0kP6969O6ZMmYLY2FjcvXtXH10gIiIiAQkWYtq3b4+0tDTk5OTA398f7u7uiI+Px8qVK9G9e3cAQJs2bdCnTx9cu3YNXl5eT2xz4sSJMDc3x7p163RSsyY1EFHLExYW1ux15s2bp4NKiOivBL2w9+WXX0ZqamqTy6xevbrR6Y3dBlwqlSIzM1MrtTVm586dOmubiAxXeHh4s9eJiIjQQSVE9Fd87AARERGJEkMMERERiRJDDBEREYkSQwwRERGJEkMMERERiZKgn04Sypw5c57qzr719fXqh1USkXg5Oztr7e7eSqUSpqb8e5BICHznNQMDDBE9jAGGSDh89xEREZEoMcQQERGRKDHEEBERkSgxxBAREZEoMcQQkV7U19cLXQIR6YhSqRRku0b5cZukpCRYW1sLXQaRUVmyZInQJajl5+ejqqpK6DKIWgxXV1dBtssjMURERCRKDDFEREQkSgwxREREJEoMMURERCRKDDFEREQkSi0uxBQXFwtdAhE1IjQ0VOgSiKgZ0tLStNpeQkKCVtsDdBRiXFxc4ObmBplMpv4aM2aMLjbVQEVFBYYPH67z7RBR82VlZQldAhE1Q3p6ulbbS0xM1Gp7gA7vE3P06FHY2trqqvlG3bt3D3fu3NHrNomIiEgYej+d9Pvvv2PSpElwd3dHYGAgjh8/DgAoKipC//79MXnyZHh6eqKwsBBXr15FeHg4PDw8MGrUKBQUFAAAampqMG3aNHh4eGDQoEH45z//CQAYO3Ys5HI5ZDIZ5HK5vrtGREREeqT3O/ZGRUWhV69eWLt2LS5cuICpU6di27ZtMDc3R3FxMaZOnYrExERYWFggKCgI48aNw8aNG7F//37MmDEDBw4cwNdff43nnnsOWVlZKCoqwjvvvIOgoCBs3rwZQ4YMQV5enr67RURE1OJERUVptJyVlZWOK2mczo7EBAQEwN3dXf117do1lJaW4t///jdmz54NqVQKd3d3DB48GIcPH1avFxgYCGtra/zyyy8AgJEjR0Iikaivdblw4QLatGmDS5cuITMzE+3bt0dWVhbs7e111RUiIiIyQDo7EnP48OFHron55ZdfYGNjA6lUqp7WqVMnlJaWAgDMzMzQunVrAEBpaSmKiorg7u6uXraurg4lJSV4++238ccffyApKQkffPABgoODDeq5LERERC3BypUrNVpOk2cnOTg4PGs5j9Dr6SRbW1vcvHkTcrlcHWSKi4thZ2cHADAxMVEv26FDB7i4uGDXrl3qaYWFhejYsSMKCgoQHByMKVOm4OrVq5g9ezb27t0LHx8ffXaHiIiIBKTXC3s7d+6Ml19+GcnJyZDL5fj5559x+PBhDB48+JFl3dzcUFVVhf3790OpVOLMmTMIDg5GZWUl9u/fj6VLl+Lu3bvo0KEDTExM0KZNG0ilUigUCtTW1uqzW0RERCQAvX86KTExEVeuXIGPjw8WLFiAuLg4vPrqq48sJ5VK8dlnn+Hrr7+Gh4cHPvzwQ6xatQqdO3fG5MmTYWZmhr59+2LQoEHw9fWFv78/bG1t4enpCS8vL5SVlem7a0RERKRHOjmddOXKlcfO69y5M1JTUx+Z7ujoiIsXLzaY5uLigoyMjEeWtbKyQlJSUqPtb9y4sZnVEpE+eHl5CV0CETVDWFiYVtubN2+eVtsDWuBjB4jIMO3cuVPoEoioGcLDw7XaXkREhFbbAxhiiIiISKQYYoiIiEiUGGKIiIhIlBhiiIiISJQYYoiIiEiU9P4ASEMwZ84cODo6Cl0GkVGpr6+HRGIYuxxnZ2fuA4i0SKlUwtRU/8dFeCSGiPTCUAIMEWmfEAEGYIghIiIikWKIISIiIlFiiCEiIiJRYoghIiIiUWKI0aH6+nqhSyCiFkKpVApdApHBMcqPCyQlJcHa2lrn21myZInOt0FEzZefn4+qqiqhy2gWV1dXoUsgMjg8EkNERESixBBDREREosQQQ0RERKLEEENERESixBCjQ6GhoUKXQEQikpaW9lTrJSQkaLkSInEwuBBTXFwsdAlak5WVJXQJRCQi6enpT7VeYmKilishEocnhpiioiK4uLhAJpOpv8LCwnDt2jWtF5Oeno7PP/8cALBv3z7MmDFD69sgIiKilkGjIzFSqRR5eXnIy8tDTk4OevbsiZiYGK0X88cff6i/Dw4Oxtq1a7W+DSIiImoZmn2zO4lEguDgYOzYsQMAkJmZidWrV6OyshK9e/fGRx99BHt7e6hUKiQnJ2P79u0AgICAAHzwwQeQSqU4e/YsYmNjUVZWhu7duyMmJga3bt3CF198AZVKBblcjj59+uDAgQP48ssvER4ejj59+mDfvn24desWgoODsXDhQpiYmCAvLw+LFy9GWVkZhg4dihMnTmDLli1wdHTU7kgREelBVFRUo9OtrKz0XAmR4Wv2NTF1dXXYs2cP+vbti4KCAixevBjLli3Djz/+iG7dumHevHkAgB07duDYsWPYsWMHvv32W1y5cgXr1q0DAMTGxuKDDz5ATk4OBgwYgKSkJPj4+GDKlCkIDQ1FfHz8I9s9efIktm3bhm+++QZ79+7Fzz//jNraWsyaNQvjxo3D6dOnIZFI8Pvvvz/jkBAREZEYaHQkRi6Xw93dHQBw9+5dmJiYYP369Th69CgGDhyI3r17AwDef/99eHh44MaNGzh06BAmTpyIjh07AgDmzJmD6OhozJ07F61bt8a3336L559/HhMmTMDkyZOfWENISAhsbGxgY2ODV199FUVFRairq4OVlZX6U0Bz58596qv7iYgMwcqVKxud3tRjBxwcHHRVDpFB0/iamNzcXOTm5uLixYtITk7GjBkzkJubi06dOjVYrn379igpKcGNGzcazOvYsSNKSkoA3L+SXqlUYuLEifDz88P+/fufWEO7du3U37dq1QpKpRKVlZWwt7dXT7ewsGiwHBEREbVczb4mxtTUFAMGDICTkxPc3Nxw48YN9Ty5XI6KigrY2NjA1ta2wbzi4mLY2NhAoVCgsLAQCQkJqK+vx5EjR/DBBx9g4MCBzS7e3t4epaWlDbYvtoe6ERER0dN5qvvE5OTk4Pr16xg0aBAyMzNx7tw5yOVyrF69Gi+++CJeeOEFBAUFYcOGDSgpKUFVVRVSUlIQEBAAU1NTREdHY//+/ZBIJGjXrh0sLS0hkUgglUpx+/Ztjetwc3ODXC7H3r17UVdXh7Vr16Kuru5pukREREQio/E1MTKZTP26U6dOiIuLg4uLC+Li4vDf//3fKC0tRa9evbBmzRoAwFtvvYXS0lK8/fbbqK2txZAhQzBv3jyYmJggISEBsbGxWLx4Mezt7ZGYmAgzMzP07dsXmzZtwsyZMzFgwIAn1mVmZoZVq1Zh4cKFWLZsGUJDQ2FmZgYzM7OnHA7t8vLyEroEIhKRsLCwp1rvwQcqiIyNiUqlUgldxNO6ffs2fvvtN7z22msA7octNzc3/Pzzz7C0tHxk+aKiIrzxxhsIDg6GtbW1zutbsmSJzrdB1NI8eJ8eP35c67dKeND2Z599Bjs7O622rWtNXdhL1JI0Zx9gcI8daI5WrVphwoQJ+PXXX6FUKrFx40b07Nmz0QBDRERELUuzL+w1JBYWFli+fDlmz56NiooKuLq64pNPPhG6LCIiItIDUYcYABg8eDAGDx4sdBlERESkZ6I+nURERETGiyGGiIiIREn0p5Oexpw5c/TygMj6+npIJEY5xEQGzdnZWXQPiVUqlTA15d+dRH/Fd4QOMcAQkbYwwBA9iu8KIiIiEiWGGCIiIhIlhhgiIiISJYYYIiIiEiWGmKdQX18vdAlEZECUSqXQJRAZJaP8+ExSUtIzPQCSD3YkErf8/HxUVVVprT0+nJFIGDwSQ0RERKLEEENERESixBBDREREosQQQ0RERKLEEENERESixBDzFEJDQ4UugYgElpaW1uj0hIQEPVdCZLwECzFFRUVwcXFBeHj4I/OWLl0KFxcXFBUVoWfPno2un5KSgldffRUymUz9NWLECFy6dEnXpSMrK0vn2yAiw5aent7o9MTERD1XQmS8BD0SI5FIcPnyZVRWVqqnqVQqfPfdd2jVqtUT1w8NDUVeXh7y8vJw7tw5DBs2DNOmTePN6IiIiIyAoCHG1NQUr7/+Oo4fP66e9vPPP8PFxUWjEPNXJiYmePPNN1FWVobq6mptl0pEREQGRvA79vr7++Obb77BO++8AwA4fPgwAgIC8P333zernXv37mHLli3o1q0bbGxsdFEqEVEDUVFRAAArKyuBKyEyToJf2Ovn54cLFy6gurpafSqpf//+Gq27a9cuuLu7w8PDA35+fsjKykJSUpKOKyYiIiJDIPiRGAsLC3h7e+PkyZNwdHSEs7Ozxs81CgkJQWxsrI4rJCJq3MqVKwE0fHaSg4ODUOUQGR3BQwwABAQE4ODBg+jcuTOGDBkidDlEREQkAoKfTgKAfv36IScn57GnkkpKShp8yeVyAaokIiIiQ2IQR2Kee+45eHh4AHj0Ajm5XA4/P78G0zIyMvRWGxERERkmwUKMo6MjLl68qH69Zs2aBvMfzLty5Uqj67u7u+uuuCfw8vISbNtEZBjCwsIanT5v3jw9V0JkvAzidJLY7Ny5U+gSiEhgjd1tHAAiIiL0XAmR8WKIISIiIlFiiCEiIiJRYoghIiIiUWKIISIiIlFiiCEiIiJRMoj7xOjbnDlz4Ojo+NTr19fXQyIxyqEjahGcnZ2faR/wMKVSCVNT/k1IpG981z0FBhgi+isGGCJh8J1HREREosQQQ0RERKLEEENERESixBBDREREosQQQ0SkY0qlUugSiFokfsyGiIxOfn4+qqqq9LY9V1dXvW2LyJjwSAwRERGJEkMMERERiRJDDBEREYkSQwwRERGJEkMMERERiRJDDBFRM6WlpWm9zYSEBK23SdTS6S3EuLi4wM3NDTKZTP01ZswYAEBQUBAuXLigle2kpKRg8eLFWmmLiKgx6enpWm8zMTFR620StXR6vU/M0aNHYWtr+8j0AwcO6LMMIiIiagEM4nTSgAEDcP78eaSnp2Pw4MGQy+VQKpV49913sX79egBAXl4eQkJC4OHhgcmTJ6O0tBQAUFdXhyVLlqB3794YOnQorl27JmBPiIiISF8M6o69o0aNwr59+7B582aYm5ujvr4eEyZMQHV1NaZNm4bly5fj9ddfR2pqKubPn49NmzYhPT0d//73v3H8+HGUl5cjPDwcgwcPFrorRNTCRUVFabyslZWVDishMl56DTEBAQEwMTFRv96xYwe6du2qfm1qaorY2FiMHTsWwP2L51q1aoVTp06hR48eGDBgAABgypQpSE1NRVlZGY4cOYJx48ahbdu2aNu2Ld58803cuXNHn90iIiIiAeg1xBw+fLjRa2L+6uWXX4ajoyOUSiWcnZ0BACUlJcjOzoa7u7t6OYVCgZKSEty8eRN2dnbq6Z06dUJBQYFuOkBE9P9Wrlyp8bKaPDvJwcHhWcohMkoGdToJAA4ePIjbt2/D3NwcO3fuRGhoKGxtbdG/f3+kpKSolysoKMALL7yADh064MaNG+rpFRUVQpRNREREemYQF/Y+UF1djeXLlyMmJgaLFi3CJ598gps3b8LPzw+5ubk4c+YMVCoV9uzZg5EjR6Kurg6BgYH48ssvUVFRgWvXrmH37t1Cd4OIiIj0wKCOxKxatQq9evWCr68vAOD111/HihUrsHLlSiQnJ2P58uUoLCyEk5MT1q1bB0tLS4wcORJFRUUICAhA+/bt0bdvX4F7QURERPqgtxBz5cqVx847ceIEAMDNza3B9FWrVqm/9/DwaPQoi6mpKebPn4/58+drqVIioqaFhYVpvc158+ZpvU2ils6gTicREYlBeHi41tuMiIjQeptELR1DDBEREYkSQwwRERGJEkMMERERiRJDDBEREYmSQX3EmohIH5ydneHo6Ki37SmVSpia8m9GIm3ju4qISMcYYIh0g+8sIiIiEiWGGCIiIhIlhhgiIiISJYYYIiIiEiWGGCIiIhIlhhgiIiISJaMKMTU1NQ3+Favq6mqkpKSgurpa6FKeCfthWFpKP/SF4yUsjr+wDGX8jepmd7du3QIA/O///i+sra0FrubplZSUYM2aNfD29kbHjh2FLuepsR+GxVD6UVJSAgBQKBRab/tBmw+28SwMZbyMFcdfWLoc/+bsA0xUKpVKq1s3YIcOHcLcuXOFLoOINJCRkQF3d3ettpmbm4vRo0drtU0i0g1N9gFGdSTGxcUFALBlyxY4ODgIXM3TKykpwejRo5GRkSHqv0DYD8NiKP1QKBQoLy+Hq6ur1tt2dXVFRkYGbG1t0apVq2dqy1DGy1hx/IWly/Fvzj7AqEKMVCr4m7XDAAASlElEQVQFADg4OOj1uSm60rFjR/bDgLAf2vPCCy/opF0LCwutH90xhPEyZhx/Yelq/DXdB7T68MMPP9T61g2Yubk5PD09YW5uLnQpz4T9MCzsh3HieAmL4y8sQxh/o7omhoiIiFoOo/qINREREbUcDDFEREQkSi0yxOTm5iIoKAhubm6YPn16oze3++233/DWW2/Bzc0NYWFhWrlvhLZp0o9z584hJCQEvXr1wvDhw3Hu3DkBKm2aJv144OrVq+jZsyfKy8v1WKFmNOnHzZs3MWvWLHh6esLf3x+nT58WoNKmadKPy5cvIzQ0FL169UJISAguXLggQKWG78svv8TixYuFLsOoPDzmR44cwYABAyCTybBgwQLU1dUJWF3L05zxTkpKgqenJ7y9vbFlyxb9FKhqYe7cuaPy9vZWnTp1SnX37l3V+++/r4qPj2+wjFKpVAUFBam2bt2qqq2tVcXHx6tmzZolUMWN06Qfd+/eVXl7e6sOHz6sUigUqgMHDqi8vLxUd+/eFajqR2nSjwcUCoVq5MiRKmdnZ1VZWZmeK22apv0YM2aMavny5aq6ujrVDz/8oOrdu7fq3r17AlTcOE37MXToUNXu3btVSqVStXXrVtXgwYMFqNZwyeVyVVJSkurll19WxcTECF2OUWhszG/cuKHy8PBQXbx4UVVdXa0KCwtTpaenC1xpy9Dc8T548KDqH//4h6qiokL122+/qXx9fVVXrlzReZ0t7khMVlYWHBwc4OfnBwsLC8ycORN79+5tsMz//M//4I8//sCIESMglUoxa9YsnDp1yqAeR6BJP8rKyvD3v/8d/v7+MDU1RWBgIJRKJa5fvy5Q1Y/SpB8PbNmyBT169NBzhZrRpB/FxcW4fPkyIiMjIZFI4Ovri/T0dJiYmAhU9aM0/Xlcv34dKpUKKpUKpqam/PTHQ1asWIFLly7h3XffFboUo9HYmB87dgw+Pj5wdXVF69atMXnyZOzZs0fAKluO5o73oUOHMHLkSLRv3x7du3fH8OHDH7uv16YWF2IKCwvRtWtX9WsnJydUVlaiqqqqwTJ//Qz6c889h3bt2qGwsFCfpTZJk344OTnhk08+Ub++cOECamtr0aVLF32W2iRN+gHc/09z+/bteP/99/VcoWY06cfly5fRrVs3JCYmwsfHB2+++SZu376tvj+RIdD05zF+/HgsWLAArq6uWLFiBeLi4vRcqWGbNm0a1q9fj/bt2wtditFobMyvXbvW4Pf5hRdeQEFBgQDVtTzNHe9r166hW7du6nlOTk56+Vm0uBBz+/ZtWFpaql9LJBKYmZnh3r17j10GuH8TrLt37+qtzifRpB9/dePGDcyZMwdz586FhYWFvsp8Ik36oVKpsGjRIsyfPx9WVlZClPlEmvTjzz//xIULF9CuXTucOnUKkydPxsyZM/Hnn38KUXKjNP29kkgkSElJwfnz5xEZGYm5c+eitrZW3+UaLFtbW6FLMDqNjfmdO3ca/D6bm5s/dh9JzdPc8b5z506D/3ssLCz08rNocSHG0tKywc62vr4edXV1DQb+4WUA4N69e3juuef0VueTaNKPBwoKCjBy5EgEBwdj3LhxeqzyyTTpx7Zt22BnZ4e+ffsKUaJGNOmHmZkZnnvuOUyaNAlSqRSBgYHo0KGDQV0Uq0k/Lly4gGPHjmHQoEGQSqUYPXo0pFIpcnJyhCiZ6LEe/n2ura1tdB9J2tHUeD887969e3r5WbS4ENOtW7cGp4UKCwvRrl07PP/8849d5s6dO6isrISTk5Nea22KJv0AgH/9618YPXo0xo4da5CnYjTpx9GjR3H8+HG4u7urbwkfEBCA3Nxcvdf7OJr0o2vXrrh3716Dq/WVSiVUBnQ/SU36cePGjUc+4SGRSCCRGNVTSkgEHv59fvh0B2lXU+PdrVs3/Oc//1HPe/jUta60uBDj5eWFwsJCHD9+HPfu3cO6deswZMiQBss4OzujdevWyMjIgFwux5o1a+Dj4wNra2uBqn6UJv2orq7GlClTMGvWLIwfP16gSpumST++/PJLnDt3Drm5uergcvjwYa0/4+ZZaNKPV155BU5OTli9ejUUCgUOHDiAmzdviq4fbm5uKCoqwq5du6BUKrFnzx5UV1fjb3/7m0BVEzXujTfewA8//IDz58/jzz//RGpq6iO/z6Q9TY13QEAA0tLSUFZWhqtXr2LPnj16+Vm0uBBjaWmJtWvX4tNPP4WPjw9qa2sRERGB4uJiyGQyFBcXAwBSUlKwb98+eHp64tKlS1i6dKnAlTekST8yMzNRXl6OVatWQSaTqb8M7fSFJj8PQ6dJP0xMTJCamor8/Hx4enris88+w5o1awzqGiVN+mFvb4+1a9di8+bN8PDwwNatW/HFF18Y1OlWIgDo1KkTli1bhqioKPTv3x9du3Y1uFPqLUlT4z1kyBAMHjwYISEhCA8Px7Rp0/Tyhw+fnURERESi1OKOxBAREZFxYIghIiIiUWKIISIiIlFiiCEiIiJRYoghIiIiUWKIISIiIlFiiCG9MqQnbBOR/vz+++9Cl0AtEENMC+Xi4oLXXntNfQO8Xr16YcKECcjPz9dK+0VFRXBxcUF1dbX6RmlPetDhr7/+infeeUf9euLEicjIyNBKPQDUdTz4engM9u3bh88//xzz5s3T2jYbk5KSgh49ekAmk6GyslIrbfr7+8PV1RXTp0/XSntEzTFx4kT1++jVV1+Fq6ur+vXEiROfuP6KFSuQnp6u0bZGjhzZ6LLvvffeY2uYOnUqrl+/DplMhjt37jS7f5oqLCyEi4sLZDIZtmzZ8tjlkpOT4ebmBhcXF9y+fVtn9RDAh6G0YF9//TVeeeUVAEBdXR1Wr16NSZMm4cSJE2jVqpXWttO5c2fk5eU9cbnq6uoGz+TZsGGD1mporA4XF5cGY6BP/fr1w7p167TWXmZmJlJSUvDrr79qrU0iTf31vRodHY3WrVtj4cKFGq9fWVn5zE/+/uc//6n+PjIyEra2tpg/f36DZTTZD2nDDz/8ACsrq8fOnz17NoYNG4bBgwfrpR5jxiMxRsLMzAzDhw9HSUkJbt26hV27dmHEiBEYMWIEPD09cfnyZdy6dQsLFiyAr68v/Pz8kJiYiPr6egD3H2SYmJgIT09P+Pr6YteuXeq2/3pUBri/I3n33Xchk8ng7++PI0eOoLKyEpMmTcKff/4JmUyG0tJShIeHY9OmTQCAiooKREZGwsvLC6+//jri4uJw9+5dAPePbERERGDatGmQyWQIDAzEsWPHnmocUlJS1EczUlJSsGDBAkyfPh0ymQxDhw7F+fPnMXv2bPXrK1euqNf95ptv4O/vDw8PD0yYMEHjU2MDBgzA5s2b4e/vDzc3N8yfPx/Z2dkICgqCTCZDREQEFAoFAGDnzp0YOHAgPDw8EBoaiu+///6p+kmkb6dPn0ZISAh69eqFoUOH4ujRowCA1NRUHDp0CGlpaZgxYwYA4NChQwgJCUGfPn3Qp08fLFy4UL2veVoPjpLcvn0bhYWF8PT0xKZNm+Dr6wtPT09kZGQgLS0Nf//73+Hp6Ym0tDT1uvn5+Rg7diw8PDwwZMgQHDx4UKNtXr16FaNHj4a7uzsGDhyIlStXGtQDX40BQ4yRuHXrFtLS0vDSSy/BxsYGwP2wMX36dBw/fhwuLi6YP38+bt++jczMTGzfvh3Z2dn44osvAABbt27Ft99+i+3btyMzMxP/+te/Gt3OzZs3MWnSJAQHByMnJwcffvghIiMjUVtbi9TUVLRu3Rp5eXmwt7dvsN7MmTNRX1+PY8eOYffu3fj111+xYsUK9fxDhw7h3XffRXZ2NgICAvDhhx9qZWexb98+vPPOO8jNzYWDgwPCwsIQGhqKs2fP4qWXXsKaNWsAAEeOHEFycjISExPx448/ok+fPpg0aZLGO94dO3Zg69atOHDgAA4fPoz4+Hhs3rwZ3377LU6fPo3vvvsON2/eRExMDD777DPk5ORgxIgRiIuL406RDN7ly5cxffp0TJkyBdnZ2YiOjkZUVBTOnz+PSZMmYciQIQgPD8fatWtx/fp1REdHIyYmBtnZ2di2bRuOHTuGkydParWmqqoqXLlyBadOncKiRYuwbNkyXL58GSdOnMDSpUsRHx+Pmpoa1NTU4L333kP//v3x008/IT4+HrGxsRod1Vm+fDnc3d2Rk5ODzZs3Y+/evTh37pxW+0FNY4hpwUaNGgV3d3e4u7tjyJAhKCsrQ0pKinp+u3bt0LdvX1hbW6OyshInT57E4sWLYW1tDTs7O8yYMQNff/01AODgwYMICwuDk5MTrK2tMXfu3Ea3efLkSdjb22P06NGQSCTw9vbGV199heeff/6xdf7nP/9BXl4eFi1aBGtra3To0AFRUVHYvXs3lEolAKBnz57o168fzMzMEBwcjPLycq2ca/7b3/6Gfv36oVWrVujTpw+6d+8OPz8/SKVS+Pj4oKioCMD9ozBjxozBq6++CqlUismTJ6OmpgZnz57VaDtjxoyBjY0NHBwc4OTkhODgYHTo0AEODg546aWXUFRUBKlUColEgh07duDixYsICQlBZmYmTExMnrmfRLp04MAB+Pr6wt/fHxKJBL6+vggKCsLu3bsfWdbe3h4HDhxQX0dXVVWFtm3boqysTOt1zZgxA2ZmZvD29oZCocDYsWMhlUrRr18/1NfXo6SkBCdOnEC7du0wbtw4mJmZ4bXXXsPw4cPV+76mtG7dGmfPnsWJEyfQtm1bfP/99+jdu7fW+0GPx2tiWrCvvvqqyetB7Ozs1N8/eJp0QECAeppKpUJdXR1qa2tRUVGBjh07quc5Ojo22mZlZSU6derUYJqrq2uTdVZWVkIqlaJDhw7qaQ4ODpDL5eoLY9u3b6+eJ5Hc/7V9EHCeRdu2bdXfm5qaok2bNg1eP9hGcXEx1q1bh/Xr16vn19XVafwUbk22Y21tjc2bN+OLL77AmDFjYGFhgbFjx2LKlCkMMmTQKisr0blz5wbTHBwc8MsvvzyyrEQiwfbt27Fz505YWlrilVdeQW1trU6OOD5435ma3v97/cH77sFrpVKJ4uJiXL16Fe7u7ur1FAqFRk9gjo2Nxaeffoq4uDiUl5ejb9++iI2NbbAvI91iiDFif/2P0c7ODqampjh9+jQsLS0BADU1NaisrIS5uTns7OwafESytLS00Tbt7Owembdp0yZ4eno+to7OnTtDLpejvLxcffHf9evXYWZm1uQRHG3QNBzY2dlhzJgxGDFihHpaQUHBIzvuZ9nOrVu3oFAo8Pnnn6Ourg4//vgjZs2apT6aRmSoOnfujIsXLzaYdv369QZ/fDywb98+HDx4ELt371a/34OCgnRSlybvOzs7O7z22mv46quv1NNKS0vVQacply9fxty5cxETE4OCggIsWrQIycnJiI2Nfaa6SXM8nUQAgI4dO6JPnz6Ij4/H7du3UVNTgwULFmDRokUAgJCQEKSnp6OgoAC3b9/Gp59+2mg7fn5+KC0txfbt26FQKHDmzBkkJyfD2toaUqkUcrkctbW1Ddaxt7eHt7c3li9fjpqaGlRUVCAhIQH+/v6QSqU677sm3nzzTWzcuBEFBQVQqVTYv38/hg0b9tgw9zRu3ryJCRMmIDs7G2ZmZrC3t4eJiYnOgxzRswoKCsKZM2eQmZkJhUKBH3/8EQcPHsTQoUMBAFKpVH0LhpqaGrRq1QpmZmaQy+XYtGkTfvvttwafXNSnfv36oaCgAHv27IFCocD169cRFhaG7du3P3HdVatWISUlBXK5HPb29pBIJA2OupLu8UgMqSUkJGDFihUYNGgQ6uvr4eXlpQ4rw4cPR1lZGcLDw6FQKBAeHq7+9MFftWvXDuvXr8eKFSsQHx8Pe3t7JCQkoEuXLmjfvj1eeeUVeHp6Ytu2bQ3WW7VqlXrbSqUSAQEB+OCDD/TSb028+eabqK6uxvTp01FWVgYnJyesXbsWXbt21do2unXrho8++ggxMTEoKytDu3btsGjRIrz00kta2waRLnTr1g1r167F6tWrER0djU6dOiEuLg7e3t4AgCFDhiAiIgKFhYVYv349srOz8cYbb0AqlcLd3R2BgYFau4dVc9nY2GDDhg34+OOPsWzZMpibmyMkJARTpkx54roff/wxlixZAh8fH5iamqJ///4arUfaY6LiRx+ItOrB/Vy0eZ8YXbZLRJopLCzE4MGDce7cuSbvE9PcZenp8XQSERERiRJDDJEOnDp1SuuPHXhwzx4iEtbf//73Jz52YNiwYXqsyHjxdBIRERGJEo/EEBERkSgxxBAREZEoMcQQERGRKDHEEBERkSgxxBAREZEo/R/JodaJEQIa/QAAAABJRU5ErkJggg==\n"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "comparison_barplot_two(list(reversed(all_times_ms)), list(reversed(placement_times)), list(reversed(labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}