ScaledHybridSVD_YaMus.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "%env MKL_NUM_THREADS=16\n",
    "%env OMP_NUM_THREADS=16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from ipypb import track\n",
    "\n",
    "from polara.evaluation import evaluation_engine as ee\n",
    "from polara.evaluation.pipelines import (find_optimal_config,\n",
    "                                         set_config)\n",
    "\n",
    "from hybrids import ScaledHybridSVD, ScaledHybridSVDColdStart\n",
    "from data_preprocessing import (get_yahoo_music_data,\n",
    "                                get_similarity_data,\n",
    "                                prepare_data_model,\n",
    "                                prepare_cold_start_data_model)\n",
    "from utils import (report_results, save_results,\n",
    "                   apply_config, read_config, print_data_stats,\n",
    "                   save_training_time, save_cv_training_time)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from polara.recommender import defaults\n",
    "defaults.memory_hard_limit = 15 # allowed memory usage during recommendations generation\n",
    "max_test_workers = 6 # use this manyparallel thread for evaluation each using up to {memory_hard_limit} Gb of RAM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_name = 'scaledhybridsvd'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_labels = ['YaMus']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranks_grid = [1, 15, 30, 50, 75, 100, 150, 200, 250, 300, 400,\n",
    "              500, 750, 1000, 1250, 1500, 2000, 2500, 3000]\n",
    "svd_ranks = {'YaMus': ranks_grid}\n",
    "\n",
    "hybridsvd_weights = [0.1, 0.3, 0.5, 0.7, 0.9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "topk_values = [1, 3, 10, 20, 30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_metric = 'mrr'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dict = dict.fromkeys(data_labels)\n",
    "meta_dict = dict.fromkeys(data_labels)\n",
    "similarities = dict.fromkeys(data_labels)\n",
    "sim_indices = dict.fromkeys(data_labels)\n",
    "feature_idx = dict.fromkeys(data_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = [data_dict, similarities, sim_indices, meta_dict]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Yahoo Music"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "lbl = 'YaMus'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dict[lbl], meta_dict[lbl] = get_yahoo_music_data('/data/recsys/yahoo_music/yamus_train0_rating5.gz',\n",
    "                                                      meta_path='/data/recsys/yahoo_music/yamus_attrs.gz',\n",
    "                                                      implicit=True,\n",
    "                                                      pcore=5,\n",
    "                                                      filter_data={'genreid': [0]}, # filter unknown genre\n",
    "                                                      filter_no_meta=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "similarities[lbl], sim_indices[lbl], feature_idx[lbl] = get_similarity_data(meta_dict[lbl])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(meta_dict[lbl].applymap(len).sum(axis=1)==0).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "YaMus\n",
      "{'userid': 183003, 'songid': 134059}\n",
      "density 0.09740952587383789\n",
      "similarity matrix density 0.4576464914574314\n"
     ]
    }
   ],
   "source": [
    "print_data_stats(data_labels, all_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Standard experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaling_params = {lbl: read_config('scaledsvd', lbl)['PureSVDs']['col_scaling'] for lbl in data_labels}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_recommender_models(data_label, data_models, config):\n",
    "    data_model = data_models[data_label]\n",
    "    svd = ScaledHybridSVD(data_model)\n",
    "    svd.col_scaling = scaling_params[data_label]\n",
    "    svd.max_test_workers = max_test_workers\n",
    "    models = [svd]\n",
    "    apply_config(models, config, data_label)\n",
    "    return models\n",
    "\n",
    "\n",
    "def fine_tune_hybridsvd(model, ranks, weights, label, record_time=False):\n",
    "    model.col_scaling = scaling_params[label]\n",
    "    model.max_test_workers = max_test_workers\n",
    "    param_grid = [(w, r) for w in weights for r in reversed(list(sorted(ranks)))]\n",
    "    param_names = ('features_weight', 'rank')\n",
    "    best_svd_config, svd_scores = find_optimal_config(model, param_grid, param_names,\n",
    "                                                      target_metric,\n",
    "                                                      return_scores=True,\n",
    "                                                      force_build=False,\n",
    "                                                      iterator=lambda x: track(x, label=label))\n",
    "    model_config = {model.method: dict(zip(param_names, best_svd_config))}\n",
    "    model_scores = {model.method: svd_scores}\n",
    "    try:\n",
    "        if record_time:\n",
    "            max_rank = max(ranks)\n",
    "            save_training_time(f'{experiment_name}_rank_{max_rank}', model, svd_scores.xs(max_rank, level='rank').index, label)\n",
    "    finally:\n",
    "        return model_config, model_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {}\n",
    "scores = {}\n",
    "data_models = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
       "<progress style=\"width:60ex\" max=\"1\" value=\"1\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">1/1</span>\n",
       "<span class=\"Time-label\">[15:00:45<15:00:45, 54045.32s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[A\u001b[2K\r",
       " [████████████████████████████████████████████████████████████] 1/1 [15:00:45<15:00:45, 54045.32s/it]\u001b[B"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:15ex; max-width:15ex; vertical-align:middle; text-align:right\">YaMus</span>\n",
       "<progress style=\"width:45ex\" max=\"95\" value=\"95\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">95/95</span>\n",
       "<span class=\"Time-label\">[15:00:18<00:54, 568.61s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[2K\r",
       "          YaMus [█████████████████████████████████████████████] 95/95 [15:00:18<00:54, 568.61s/it]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for label in track(data_labels):\n",
    "    data_models[label] = prepare_data_model(label, *all_data, seed)\n",
    "    config[label], scores[label] = fine_tune_hybridsvd(ScaledHybridSVD(data_models[label]),\n",
    "                                                       svd_ranks[label],\n",
    "                                                       hybridsvd_weights,\n",
    "                                                       label, record_time=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/py36/lib/python3.6/site-packages/pandas/plotting/_core.py:1001: UserWarning: Attempted to set non-positive left xlim on a log-scaled axis.\n",
      "Invalid limit will be ignored.\n",
      "  ax.set_xlim(left, right)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEXCAYAAABBFpRtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnXd8XOWV979nmuqoS5Zl2ZbcKxgwdjDFNiT0pSQhYAiQBAIpkGwK75tsEpbwJrubTTbsBtgFJyTUrCEkSwglhCwuQGyDDaa427KNe5NtyUWWZuZ5/7hzpbE8kqbcadL5fj76eObOvc99pvjcc3+nPGKMQVEURRkYuDI9AUVRFCV9qNFXFEUZQKjRVxRFGUCo0VcURRlAqNFXFEUZQKjRVxRFGUCo0VcURRlAqNFX+jUi8pSI/Lrbtpkisl9EBvdynEdEjIjsEBF3xHZf+NhAKuetKKlCjb7S3/kacKmIfAJARPKBXwLfMsbsjOH4VuDCiOeXA/scn6WipAk1+kq/xhizH7gTmCsiRcA/AhuNMY+KyFkiskREDorIThH5hYh4uw3xBHBTxPObgMcjdxCRbSIyK+L5j0Tk0fDjQhH5bfju4KCIvCUiVc6/U0WJDTX6Sr/HGPM7YDnw38BtwO3hlwLA14Eq4Gzg4ojXbP4AnC8iJSJSCXwMeCGO038eKATqgUrgK0BbYu9EUZLHk+kJKEqa+CqwEfieMeYjAGPM2xGvN4nIXGAm8EDE9qPAy8A1QAHwP8DxOM7bgXVRGWWM+QBYlvA7UBQHUKOvDAiMMbtFZB+w0t4mIuOAfwPOwPLGPcDSKIc/jiULFWDdGcTDo0Ad8IyIlGDJRd83xmggWMkIKu8oA5mHgQ+xvPAS4G5Aouw3HxgOlBljFkd5/QjWRcOm1n5gjGk3xtxjjBkPnANcDdzg0PwVJW7U6CsDGT9wCDgiIuM5Wc8HwFj9xy8HruphnBXAdeE0z2nAJ+0XROR8EZkkIi6gBUvuCTr4HhQlLtToKwOZbwE3Y6VlPgw83dOOxpgPjTGrenj5e8A44CDwA+C3Ea/VYQWDW7Ckpb9iBZQVJSOILqKiKIoycFBPX1EUZQChRl9RFGUAoUZfURRlABGT0ReRi0VkrYhsEJHvRHk9T0SeDr++VEQawtsbROSYiKwI/z3k7PQVRVGUeOizOCvcYfBB4BPANuBtEXm+WybDLcABY8woEbkO+Alwbfi1jcaYKbFOqKqqyjQ0NMS6u6IoigIsX758nzGmuq/9YqnInQZsMMY0AYjIPOBKINLoXwncE378LPCAiEQrcumThoYGli3TSnVFUZR4EJEtsewXi7wzBNga8XxbeFvUfcLl5YewmksBNIrIuyKyUETO7WGyt4nIMhFZtnfv3ljmrSiKoiRALEY/msfePbm/p312AsOMMacB3wR+G+4/cuKOxsw1xkw1xkytru7z7kRRFEVJkFiM/jZgaMTzemBHT/uIiAcoBZqNMcfD/cwxxizH6nI4JtlJK4qiKIkRi6b/NjBaRBqB7cB1wPXd9nkeq5x9MfBp4DVjjBGRaizjHxSREcBooMmx2SuKkjE6OjrYtm0bbW26PEA6yc/Pp76+Hq+3+3o/sdGn0TfGBETkDuAVwA382hizUkTuBZYZY54HHgGeEJENQDPWhQHgPODe8HqiQeBLxpjmhGaqKEpWsW3bNvx+Pw0NDSSYt6HEiTGG/fv3s23bNhobGxMaI6Z++saYl4CXum27O+JxG9YiE92P+z3w+4RmpihKVtPW1qYGP82ICJWVlSST8KIVuYqiJIwa/PST7GeuRl9RFGUAoUZfUZScpbi4+ITnjz76KHfccUevx8yaNSumAtBly5bxta99LeprDQ0N7Nu3D4Af//jHTJw4kVNOOYUpU6awdOlS7rnnHr773e+ecMyKFSsYP3585/GTJ09m8uTJTJgwge9///scPx7P0suJo0ZfURSlG4FAgKlTp/KLX/yi1/0WL17MCy+8wDvvvMP777/PX//6V4YOHcqcOXN4+ukT1+SZN28e11/flfg4f/58PvjgA9566y2ampq47bbbUvJeuqNGX1GUfkdrayuNjY10dHQA0NLSQkNDQ+fzJ598khkzZjBp0iTeeustAO655x5uu+02LrzwQm666SYWLFjA5ZdfDsD+/fu58MILOe2007j99tuxF5/auXMnVVVV5OXlAVBVVUVdXR1jx46lrKyMpUuXds7pmWee4brrrqM7xcXFPPTQQzz33HM0Nzezc+dOzjvvPKZMmcKkSZN4/fXXHf1sYsreURRF6Y0f/mklq3a0ODrmhLoS/vHvJva6z7Fjx5gypaufY3NzM1dccQV+v59Zs2bx4osvctVVVzFv3jw+9alPdea2HzlyhL/97W8sWrSIL3zhC3z44YcALF++nDfeeIOCggIWLFjQ9f5++EPOOecc7r77bl588UXmzp0LwIUXXsi9997LmDFj+PjHP861117LzJkzAZgzZw7z5s1j+vTpLFmyhMrKSkaPHh31fZSUlNDY2Mj69et54403uOiii/je975HMBjk6NGjCX+G0VBPX1GUnKWgoIAVK1Z0/t17772dr91666385je/AeA3v/kNn//85ztfmzNnDgDnnXceLS0tHDx4EIArrriCgoKCk86zaNEiPvvZzwJw2WWXUV5eDlhe+vLly5k7dy7V1dVce+21PProowBcd911PPvss4RCIebNm9d5zp6w7x7OPPNMfvOb33DPPffwwQcf4Pf7E/loekQ9fUVRkqYvjzwTnH322WzevJmFCxcSDAaZNGlS52vd0x7t50VFRT2O11OqpNvtZtasWcyaNYvJkyfz2GOP8bnPfY6hQ4fS0NDAwoUL+f3vf8/ixYt7HLu1tZXNmzczZswYysvLWbRoES+++CI33ngjd911FzfddFM8b71X1NNXFKXfctNNNzFnzpwTvHygM8j6xhtvUFpaSmlpaa/jnHfeeTz11FMAvPzyyxw4cACAtWvXsn79+s79VqxYwfDhwzufz5kzh2984xuMHDmS+vr6qGMfPnyYr3zlK1x11VWUl5ezZcsWampq+OIXv8gtt9zCO++8E/8b7wX19BVF6bfccMMNfP/73z9JWikvL2fGjBm0tLTw61//us9x/vEf/5E5c+Zw+umnM3PmTIYNGwZYBvvOO+/k4MGDeDweRo0a1an3A1xzzTV8/etf5/777z9pzNmzZ2OMIRQKcfXVV/ODH/wAgAULFvDTn/4Ur9dLcXExjz/+eDIfwUmIrSNlC1OnTjW6iIqiZD+rV6/uzDvPVp599ln++Mc/8sQTT2R6Ko4S7bMXkeXGmKl9HZt1nv6e1uM8OH9DwscX53k4Z3QVI6qKtERcUQYwd955Jy+//DIvvfRS3zsPILLO6O9uaeOnr6xNepyhFQXMHlvDrLHVnDWiigKf24HZKYqSK0STVJQsNPqT6kpZ/KNLEj5+d0sbC9ftZcHaPfxu2TYeX7wFn8fFWSMqmTW2mtlja2io6jlCryhK7Bhj9I46zSQryfdrTb+tI8jbm5uZv2YvC9btoWnvEQAaKguZFb4L+NiISvK9ehegKPGyadMm/H4/lZWVavjThN1P3644jiRWTb9fG/3ufLT/KAvW7WH+mj0sbtpPW0eIfK91FzB7XA2zxtQwrLIwJedWlP6GrpyVGXpaOUuNfh+0dQRZ0rSfBWstKWjzfqvUeUR1UWcsYFpjBXkevQtQ+j8PL9zIoJJ8rjptSKanoiRIzmbvpIt8rzss8dQAE9m07wgL1u5h/tq9PLFkC4+8sYlCn5sZIys7paD6cr0LUPon897eSsgYrpxSp1JNP2fAGv3uNFYV0VjVyOfPbuRYe5DFTftYsHYv89fu4a+r9wAwuqY4LANVM7WhAp9HC5qV/kFHMMS2A8dYt/swY2ud7fWiZBdq9KNQ4HNz/rhBnD9uEMYYmvYdYf6aPSxct5dH39zM3EVNFPncnD2qyroIjK1mcOnJTZoUJVcIBC2Z9y8rd6nR7+eo0e8DEWFkdTEjq4u59dwRHDke4G8b97Ng7R4WrN3LX1btBmBcrZ+Z4ZTQM4aX43XrXYCSOwRCIQD+smo3d14Qvf2v0j9Qox8nRXkePjFhEJ+YYN0FbNhzmPlr9zB/zV4eeX0TDy9swh+uCp49toaZY6sZVJKf6WkrSq90BA1et/DB9kPsOHiMujK9c+2vqNFPAhFh9CA/owf5ue28kbS2dfDmhv0sXGddBF7+cBcAEwaXWIVh42o4bWgZHr0LULKMjmCIc0dX89qaPby6ajc3z2jI9JSUFKFG30H8+V4unlTLxZNqMcawdncr89dYweCHFzXxnws2UpLv4dwx1cwaU83MsdXU+PUuQMk8gaBhzCA/W/Yf4S+rdqnR78eo0U8RIsK42hLG1Zbw5VkjOXSsgzc37OuMBbz4/k4AJg8pZdbYamaNrWHK0DLcLk2XU9JPRyiE1y1cOLGWuYuaOHS0g9JCb98HKjmHGv00UVrg5dLJg7l08mCMMaza2dJZGPbg/A3c/9oGygq9nDe6mk+ePoSZY6o1X1pJC8GQwRjwuFycP66K/1qwkdfW7ubq06Iv+pGNtLR1cPvjy/nJp07Rqvo+UKOfAUSEiXWlTKwr5auzR3HoaAevb9jL/DV7WbhuD8+/t4MZIyv5h0vHM2lI7yv6KEqydAStzB2PWzi1vowafx5/WZlbRn/LvqMsbtrPwvV7ubFyeN8HDGDU6GcBpYVeLj+ljstPqaM9EOLJJVv4xWvrufz+N7j6tCF8+6KxDNFsCiVFBEJWjr7P7cLlEi6cOIg/vLOdto5gzjQjDIXbyWwKN1VUekbTSLIMn8fFF85pZOFds/nSzJG8+MFOZv9sAf/88moOHevI9PSUfkhHoMvTB/j4+EEcbQ/y1qbmTE4rLoK20d93OMMzyX7U6GcppQVevnPJOOZ/exaXTx7M3EVNzPrpfH79xibaw/9JFcUJOkK20bfMwRnDyxGBFVsPZnJacREK36007VNPvy/U6Gc5Q8oK+Pm1U/jTHecwoa6Ee19Yxcd/vpAX3t+R9GIKigJdLRi84cwxf76XEVVFvL8th4x++L/C1uaj6hT1gRr9HGHSkFKevGU6j37+TAq8bu747btc/Z9/4+3NuXMLrmQnttGPLBo8tb6M97YdyhnHIhi2+iEDHzWnx9s3xnSeN5dQo59DiAizxtbw0tfP5V8/dQo7Dx3jmocWc9vjy9i4V7VMJTFsecfr7koRPnVoGXtbj7OrJTcWSAlFXJya0hTM/cR9i5j6o1fTci4nUaOfg7hdwmfOHMr8b8/iW58Yw5sb9nHhfYv4wXMfsu/w8UxPT8kxOuWdCE//lHorVfi9HNH1TzD6adL1N+w5zIGjuZdcEZPRF5GLRWStiGwQke9EeT1PRJ4Ov75URBq6vT5MRA6LyLedmbYCUOjzcOcFo1lw12yunzaM3771ETP/dT73/+96jrUHMz09JUfozNOPqAYfP7gEj0t4b9uhTE0rLiJllnSnbYZyTOLp0+iLiBt4ELgEmADMEZEJ3Xa7BThgjBkF3Af8pNvr9wEvJz9dJRrV/jz+31WT+Ms3zuPsUVX826vrmPWz+Tzz9tac1ByV9GIb/UhPP9/rZvzgkpwJ5tqefr7XxaY0Z/DsP9Ke1vMlSyye/jRggzGmyRjTDswDruy2z5XAY+HHzwIXSLiHgIhcBTQBK52ZstITI6uLmXvTVJ65/SwGlxbwf37/Ppf94nUWrN2TMwE5Jf3YxVke94ltP06pL+X9bYdywpMNhyUYUVVMU5pz9XceOubIOG+s38f3n/vAkbF6IxajPwTYGvF8W3hb1H2MMQHgEFApIkXA/wV+2NsJROQ2EVkmIsv27t0b69yVHpjWWMH/fGUGD1x/Gkfbg3zuN29z4yNvsXJHbtyqK+mlS9450RycWl9Ga1uATfuzP/fdLs4aPaiYfYfb01LIaC+XuvOQM8Huzz6ylCeXfJTyu/NYjH60rl/dZ9XTPj8E7jPG9HrpNcbMNcZMNcZMra6ujmFKSl+ICJefUsdfvzmTuy+fwIc7DnH5/W/wzadXsP2gM56J0j/oCuR28/SHWsHcXJB47LuRxqoiwMrXTzW14cWRdjr0/8kXltcOHE2tXBSL0d8GDI14Xg/s6GkfEfEApUAzMB34VxHZDPw98A8ickeSc1biILKtw23njeCFcFuHf3l5DS1tuZd5oDhPIHSypg8wusZPoc/Ne1uz/w7Rdo7ry60OmzvS4NiUh1tPO+Xpl4XHa05xjCAWo/82MFpEGkXEB1wHPN9tn+eBm8OPPw28ZizONcY0GGMagH8H/skY84BDc1fioLTAy3cvGc9r35rJZZMH89DCjcz81/n85k1t6zDQaQ9E1/TdLmFSXSnv5YCnb8s79eVWY0KnDHFv2HKHU+eqKPIBsP9who1+WKO/A3gFWA08Y4xZKSL3isgV4d0ewdLwNwDfBE5K61Syg/ryQu67dgov3HkO4weX8MM/reLC+xbyyspdGuwdoPTk6YMVzF21o6VT989WbHmn2p+Hz+NKi6dvZww5FcjNJk8fY8xLxpgxxpiRxpgfh7fdbYx5Pvy4zRhzjTFmlDFmmjGmKcoY9xhjfubs9JVEmTSklKdunc5vPn8mXreL259YzvW/XMqqHS2ZnpqSZjrbMERZte3UoWUcD4RYu6s13dOKC9sAe1xCXWl+WuJWdsaQ055+85HUFlhqRe4ARkSYPbaGl79+Lv/vyoms2dXCZfe/znd+/z57W7Wyd6AQLU/f5tT6MgDez/IiLTvjxSXC4NKCtMg79oVmd0ubI3fJpQWWp5/qvH81+goet4sbz2pgwbdnc8vZjTy7fBuzf7aA/1qwkbYOrezt7/SUpw8wtKKA8kJv1rdjsA2w2yXUlRWkRd6x6QgaWo8Hkh7HFV4eNSvkHWVgUFro5fuXT+Av3ziPj42o5Cd/XsMn7lvIyx/sVL2/HxPoxdMXESYNKeWD7dnt6dvZOy4RhpTls7ulLeVxiMh+PwccMNT2aOrpK2lnRHUxv7p5Kk/eMp1Cr4cvP/UO185dwodZ/h9fSYz2zn760c3B5CGlrNvdyvFA9t71dco7LqgrKyBkLNkllYQMFISXk3TCO7evIc2Zzt5RBi7njK7ixa+dw4+vnsSGPYf5uwfe4K7fvceeHGm3q8RGIHjicondmTSklEDIZHUwt1PeEWFwWXrSNkPGUFlsBV8POtBt076bVnlHySget4sbpg9nwV2z+OK5I3huxXZm/WwBD87foHp/P6E3TR8sTx/gw+3Zm9llp2y6XZa8A6kv0DIGKjszbpzz9A8eU6OvZAEl+V7+4dLxvPqNmZw7uoqfvrKWC/5tIX96T5dtzHU6s3d6kHfqywsoyfdkta4fVqiQcPYOkPK0TcvTzwOcaZ1g3620tiUfFO4NNfpKXDRUFfHwjVP57RenU1Lg5c7/fpdrHlqc9dkdSs8EggaXgCtKnj50BXOzuWFfpKdflOehrNCbck8/ZAwl+R7cLnHE6Nuu09H2YKfklgrU6CsJMWNkFS/ceQ7/8snJbN5/hCsffJNvPrOCXWnIj1acpSMUipq5E8nkIaWs2dmatZW5wQhNH7By9Q+m9rdojHWhLC/00nzECU2/6/FhB1JAe0KNvpIwbpdw3bRhzP/2LL48ayQvvGc1c/uPv67P6kwP5UQ6AqZPoz9xSCntwRDrdmdnMNeWRsI2nyFlqa/KNcZKES0r9HHQCU8/wuqnUuJRo68kjT/fy/+9eBx//eZMZo+r5r6/ruP6Xy7Vqt4cIRAK9RjEtZlUVwLAyiwN5kbKO0BaqnJDxpLFKgp9zgRyIx6r0VdygmGVhfznDWfw4PWns2pHC1c88AYfZHn5vmJVlHZfQKU7DZVFFOdlbzDXVp1seaeqOI9DxzpS2kHWMvpCWaHXkZTN0AmefuranqvRVxznslMG8+yXz8Ilwqcf+ht/XLE901NSeiEQDJ20gEp3XC5hQl0JH2ZpMLe7vFPtt7Jq9qeweVnIWEHuiiIfzY7IO12P1dNXco6JdaX88Y6zObW+jK/PW8G/vLxGF2nPUgIh06e8A1Ywd/XOlpRmliSKLbVIp6dv5c/va01dzrsxBhEoK/Rx4Eh70qnLofB4AK3H1dNXcpCq4jyevHU6108fxkMLN/LFx5fpal1ZSEew7+wdgElDSmjrCLFxb/atmRsMmU49H6Aq7OnvPZw6Xd8K5EJFkZdAyCSdcWOw6mFAPX0lh/F5XPzT1ZP50VWTWLRuL1c/+CZNe3tdMllJMx3BUI+FWZFMqrMrc7NP4rGlFpvqcNFUKj39Lk3fuqs4kGzapoGSAg+gRl/pB3z2Y8N58tbpHDjawZUPvsmCtXsyPSUlTCAYm7wzorqYAq87K4O5IWM6g7jQpenvPZxaTd8lQnnY6CfbPiFkDPkeNz63S42+0j/42IhK/vjVs6kvL+QLj77NLxc1aQuHLKAjZPDEIO+4w8HcbKzM7S7v5HvdFOd52JdSo29p8P58Z7xzO+/fn+/R7B2l/zC0opDff/ksLpk0mB+/tJpvPfOeNm7LMIFgCG8PLRi6M3lIKR9ub8m64rvIIKhNVbEvpbUikUYakk+zjLyIqKev9CsKfR4euP40vvWJMfzh3e1c+/Bibd+QQWKVdwDOGVXFsY4gyzYfSPGs4iPUzdMHS+JJtafvkq7ga0uynn74X3++Vz19pf8hItx5wWjm3ngGG/Yc5ooH3uCdj7LLkAwUYum9YzNjVCU+t4v5a7IrJhPspumDlT22L4ULklieuaRI3lFPX+mnXDixlj985WzyvW6ue3gJzy7flukpDThiTdkE6y5t+ogK5mdZIL579g5YRj/V8o4IFOVZRv9w0kbfkncKfR6OtKdOPlOjr2ScsbV+/vjVszmzsZxv/+49fvziKg3wppFA0OCJUdMHmD22ho17j/DR/qMpnFV8WPLOiduq/altxWB75l63iwKvO2lJxmCNV5Tn5li7evpKP6e8yMdjn5/GzWcN55evb+KfXlqthj9NxOPpA8weVwPAgnXZ4+0HQ9HlHUhdKwZb0wdngq8h9fSVgYbH7eKeKybyuRkN/PL1Tdz/2oZMT2lAEGsbBpvGqiIaKguzStePLu+kthWDXZwFYaOfZOsEE34PhT43x1Jo9D0pG1lREkBEuPvyCRw+HuDnr66jOM/DF85pzPS0+jWBGLpsdmfW2Br++62PaOsIku91p2hmsRMy0bN3wG7FUJqCc3ZdaKyMGwc8faDI5+ZIeyCs8cd+MY4V9fSVrMPlEv7lk5O5ZFIt976wimfe3prpKfVrOoIhfJ74jMvscTUcD4RY3LQ/RbOKj+7FWdAl76TC07elR/uM/nxP0imbYAWGC3wejIG2jtTEItToK1mJx+3i36+bwnljqvnOH97nxfd3ZnpK/ZaOYChuT396YwX5XhcLskTiiVaclcpWDHbDWFveKcn3cjjZQK7pCuQCHE1RMFeNvpK15HncPPzZMzhjeDl///S7WZcm2F+IpzjLJt/r5uyRVcxfuzcrAu7de+9AVyuGVKRt2u/Z8UAuUOC1jX5qdH01+kpWU+Bz88jnzmRsrZ8vPbGcJVkiJ/Qn4inOimTm2Go+aj7K1ubUrkUbC9HkHYCakryUVHt3evquiECuQ8VZdt6/Gn1lwFKS7+Wxz09jaEUhtz62jPe3Hcz0lPoV8ebp25zZUAHAsi3NTk8pboKhk7N3AEZWF7MxBa28u6/U5c/3cqwjSEcSC8yEjAGBQp/l6R9ReUcZyFQW5/HkLdMpL/Jy06/fYt3u1kxPqV9gjAmnbMZvCsYM8uPP87BsS+bbZxhzcnEWwOiaYjbtO5KUMY5+PuvfyJRNSK4q12AFhgt9YU//uHr6ygCntjSfp275GD63i8/+ailb9mffCk65RiCsU/ji1PTBarV82vBylmdB87VovXcARg8qJhAyjv9WQidp+g6seBWWd2xPXwO5igIMqyzkyVun0xEMccOvlrLzUOb15FzG9oAT8fQBpg4vZ92eVg4dy+wymMFQ9Jz20TV+ANbvdlbi6ZR3wkmbxWEdPpkCra6K3CwI5IrIxSKyVkQ2iMh3oryeJyJPh19fKiIN4e3TRGRF+O89Ebna2ekrA5Exg/w8/oXpHDzawWd/tZT9KWyf29/pCFrGKxFNHyyjbwwZ75BqDFEDuSOrixGB9XucNfp2vpJ9nSlxoNNmV++dDAdyRcQNPAhcAkwA5ojIhG673QIcMMaMAu4DfhLe/iEw1RgzBbgYeFhEtApYSZrJ9aU8cvNUth04xk2/fivjnmauEgh7+olk7wBMGVaG2yUZl3ii9d4BK/trSFmB80Y/HCJwRVTkQnJG3/b0C7JA3pkGbDDGNBlj2oF5wJXd9rkSeCz8+FngAhERY8xRY4w983y6LpCKkjTTR1Ty0I1nsG53K1/773ezIl8817A1/Xjz9G0KfR4mDC7JeAZP0Bh6qi8bXVPMBoeNfndN317QPBnno7P3TjhP/0gGA7lDgMg6+G3hbVH3CRv5Q0AlgIhMF5GVwAfAlyIuAp2IyG0iskxElu3duzf+d6EMWGaPreH/XjyOhev2smCt/nbixdb0vXFW5EZyxvByVmw96HiGTDyYiOZn3Rk9yM/GvYcJhpxzCjqNftjq1/jzAdjdknhNgAkXZ3ncLvI8Lo52ZM7Tj/ZJdv/0etzHGLPUGDMROBP4rojkn7SjMXONMVONMVOrq6tjmJKidHHTWQ00VBby45dWd8oVSmwEwpq+N87eO5FMbSinrSPEqh0tTk0rbnoqzgIYVVNMeyDE1mbn+v/b1w87eFzgc1Ne6E0qscDQFSMo9LkzmrK5DRga8bwe2NHTPmHNvhQ44X7PGLMaOAJMSnSyihINn8fFdy4Zz4Y9h5mnzdniojN7J0lPH8hovn7Q0KOnP6qmGHA2mNu94RrA4NICdh5MxtPveg+FPk9Gs3feBkaLSKOI+IDrgOe77fM8cHP48aeB14wxJnyMB0BEhgNjgc2OzFxRIrho4iCmNVRw36vrUrqodH/Dzt7xJqjpg2XshpQVsDyDur6JWNCkOw2VRQB8lAIwxP2rAAAgAElEQVRPP/JCM7g0nx1JtHywe+8AFOW5OZxkf/6e6NPohzX4O4BXgNXAM8aYlSJyr4hcEd7tEaBSRDYA3wTstM5zgPdEZAXwP8BXjDH7nH4TiiIifO+y8ew/0s5DCzdmejo5QyCUvKcPlre/fMuBjAXTe5N3ygu9FPncbDvgnNE3nBjIBRhclp+cvBPRn7+s0MeBo6kx+jGlTxpjXgJe6rbt7ojHbcA1UY57AngiyTkqSkycOrSMK6fU8avXN3HD9OHUlRVkekpZT2eefhKePlhG//n3drDzUFtGPvdgqOdArohQX17ItgPOFfJF9/QLOHi0g2Ptwc60y/jG7GoPXVnkczzN1EYrcpV+xV0XjcUAP31lbaankhMkm6dvM6GuBIDVOzMTzDW9aPoA9eUFzhr90IkN18CSd4CkvH17uIoiX8qKDtXoK/2K+vJCbjmnkf95d7t244wBO08/WaM/rtZqd5Apox+MslxiJPXlBWxrPuqY/NS94RpYnj7AzgR1/cgLV2VxHgePdTiaZmqjRl/pd3xl1kgqi3z86MXVWrDVB+2dvXeSk3f8+V6GVRSyKkNGPxQynTnz0RhaUUjr8QAtx5zJfe/K0+/aVldmefo7Dibm6XeXd4yBA0edX+pRjb7S7/Dne/n7j4/mrU3NvLpqd6ank9V05uknGcgFmDC4hNU7M9PyOtRL9g5Ynj7AVoeCud0brgEMKrHlnQQ9fbo8/YoiHwDNR9ToK0pMzJk2jJHVRfzLy2syWima7QQc8vTB0vU37z/CkeOpqSTtjZ5aK9vUlxcCOJbB073hGljLM5YXehOuyrUXUQHL0wfYf1iNvqLEhMft4h8uHU/TviM8tWRLpqeTtXSEks/Ttxk/uARjYM2u9Hv7oRC9yzudRt+ZYG7XGrknnrPGn8+eRNfkjdD0K4rV01eUuDl/XA0zRlbyH/+7Xrtw9kDAgYpcGzuDJxO6fl/yTkmBB3+exzGjHy1lE6w1eRM1+pHFWV3yjvMZPGr0lX6LiPAPl47n4LEOHpy/IdPTyUq6eu8kbwrqSvMpyfdkJIOnt+IssH4LQ8oLHJN3unfZtKnx57MnQXknsvdOeWFY3lFPX1HiY9KQUj55Wj2PvrnZ0YZb/YX2zi6bycs7IsKEupKMNF4L9dJl02ZoRSFbmx3y9MNhou6rddWU5LG39XhnHn88RKZset0uSgu8Ku8oSiLcddFYXC74yZ/XZHoqWUcgyeUSuzN+cAlrd7WmJL+8N0J9FGeBXaDlTK5+Z/bOSZ5+HoGQSSjVMlLeASuYq56+oiRAbWk+t507ghfe35nxZf2yjWQXUenOhMElHOsIsjnNi9b3Je8ADCkr4Eh70JH4TrTiLOhK20xE14/svQNQXuSjWbN3FCUxbps5kqriPH70wiot2Iqgw8E8fbA8fUh/ZW6ol947NnauvhPB3GgN18Dy9CFRo29OuHOoKPJpcZaiJEpxnodvXTiGdz46yPPvdV8OYuDiZJ4+wOhBxXhcknZdP9hH9g7Q2Qgu0YrZSHrM3gmvoJVIMNeAyjuK4iSfmTqUU+tLuevZ91m4TpdWhK48fY8DgVyAPI+bUTXF6ff0++i9A5a8A7DdEaPfg6Zfkoynf+JFpKLIx4Ej7Y7fmarRVwYMbpfw6OenMbK6mC8+vkwNP9bKWV63nJSFkgwTBpekPVe/r+IssIxovtfliKffU3FWvteNP9+TkKcfiiLvBELGsX5BNmr0lQFFeZGP3946XQ1/mEAw5EhhViRjav3sbjme1oK4WOQdEaGurMAhT9/6N1ococafWIGWlad/oqcP0Oywrq9GXxlwqOHvoiNoHNPzbcYMCq9Juzt97RhCffTesRlSVsD2JNax7TxflH76NlXFeQn1zIkWyAXnq3LV6CsDEjX8FoFQKOle+t0ZXWP11l+3OzUrP3XHGGPp4THEJYaUFbDdkewdi56M/r4EDLWl6Xc9ryyy4gNON11To68MWGzDP2oAG/5A0DgWxLUZUlZAoc/NujR5+nYhWF8pm2Bl8Ow7fJy2jmBS5wz1oOkDVBX7EjLUVnFWhLyToqZravSVAU15kY+nBrDh7wgaxz19l0sYXVPM+j3pMfq2vt5X9g50ZfAk2vPepqfiLLBWvTp0rIP2QHwtvSN774C1oDvAQYdjI2r0lQHPQDb8dvaO04we5E+bvNOb192dIeXO5Or31HANoDJBD717ymaB143HJbSo0VcU5xmohj8QCjnWdyeSMYOK2dt6nIMpqCjtTpe80/e+nbn6Ser69t1FtFRXW4vfF+fC5qFu+fgiQmmB1/EsKDX6ihIm0vDf/sQy1uzKzHqv6aQjBZo+WJ4+pCeYaxvLWOSd2tJ8RJIv0OqpOAug2p9gW+QoTeNKCry0tGmevqKkjPIiH499YRol+V6+/OQ7tLT178VXAkHns3cAxnQa/dTr+nab41jkHa/bRY0/L2l5p6fiLIjw9OPM1e9enAWW0VdPX1FSTLU/jwdvOJ2Pmo/yf373fr9u0BYIOZ+nD9aCKsV5nrTk6gfj8PQBakuSWNIwTFcg9+TXbE1/f5xpm9177wCU5HvU6CtKOjizoYLvXjKOP6/cxa9e35Tp6aSMjhR5+iLCqJritMo7sapU1f78hBcv7zon4XOefNLiPA8+jyvutM1otQalBV5a1egrSnq45ZxGLplUy7/8eQ1Lm/ZnejopwUrZdN7TByuYm460Tbs6NpbiLIBBSaxj23nOXjR9EaG6OI+9CQRyT/L0Vd5RlPQhIvzrp09hWEUhd/z3u+xpTb58P9tIRe8dmzGD/Ow73J6SJf8i6ZR3YmwaV+PPp/lIe9x59JH0pumDJfHE7elzcjZQaYGXlrYORyVGNfqK0gv+fC//9dnTaW3r4I7fvtvZf76/kEpPf3Sagrm9SS3RGBRufxyvJx7POSuLfHGnbHbvvQOW0e8IGo4lWUEciRp9RemDcbUl/PMnJ/PWpmZ++pe1mZ6OowRCqfT009N4LX55x1roJBldvzd5B6CiKC/B4qwTt5XkW1W5TrZXVqOvKDFw9Wn13DB9GA8vbOLPH+7K9HQcI5CCLps2tSX5+PM8KQ/m2sVZscajq+0lDVsS9/R7y96BcP+dOBdA6d57ByxPH3BU11ejrygxcvffTeCU+lLu+t17bNqX3oW/U0VHKIQvBdk7YOnTE+pKeHdrahejj6cNA0QuXu6Epx/9nBVFPtoDIY60xy7LdO+9A1BS4AFwtF5Ejb6ixEiex81/3nA6brfw5SeXc+ho7hdudQRS5+kDzBhZxcodLSltxxCv0a8s8uF2SVLyTm8N16CrF/7+OHR9Y6IHcgEOOvhbU6OvKHFQX17Iv187hQ17DnPZ/a/z7kep9WJTTap679icPaoSY2DxxtSlvNqx9ViLs1wuK6UyGXmnr9qAquJwL/wYdX1bBuo+XFmBdfFw8qIZ07ctIheLyFoR2SAi34nyep6IPB1+famINIS3f0JElovIB+F/z3ds5oqSIWaNreF3XzoLY+CahxYzd9HGzmBirtERNHhT0HvH5tShZRT53Ly5cV/KzhFvcRZYGTy7k8jV7yt7p3PVqxjTNnu6cygryoCmLyJu4EHgEmACMEdEJnTb7RbggDFmFHAf8JPw9n3A3xljJgM3A084NXFFySSnDSvnpa+dywXja/inl9Zw6+PLUp6PngoCwdR6+l63i+kjKvnbhlR6+vHJO2BV5SayeLlN39k78bVi6Gk8f54Ht0s4kGZPfxqwwRjTZIxpB+YBV3bb50rgsfDjZ4ELRESMMe8aY3aEt68E8kUkz4mJK0qmKS308tBnz+CHV0zkjfX7uPQ/XuetTc2ZnlZcdKSo904kM0ZW0rTvCDsPJb9MYTTi6bJpk2xVrukjkNvVfydGTz/8b/fRRISyAm/aNf0hwNaI59vC26LuY4wJAIeAym77fAp41xjj7Cq/ipJBRISbZzTwh6/MIM/r4rq5i3ngtfU5I/cEgqnL3rE5e1QVAG+myNtPxNO3q3KPBxIreuorZbPQ56HA645f3okyYGlh+o1+tLfV/Rfd6z4iMhFL8rk96glEbhORZSKybO/egbF4hdK/mDSklBfuPIfLTqnjZ39ZxzefWZH13TmDIUPIkLLiLJuxg/xUFvl4c0NqdP1QLwazJzqrchP09mOpAq4M5+rHNl7Pv5XyQh8Hj6VX3tkGDI14Xg/s6GkfEfEApUBz+Hk98D/ATcaYjdFOYIyZa4yZaoyZWl1dHd87UJQswZ/v5RfXTeEbHx/Dcyt2cP9rGzI9pV7pCKe9pFrecbmEs0ZW8uaGfSm5EIbi7L0DVhYWwEf7jyZ1zt5OWVkUu9G3iXYRyYS88zYwWkQaRcQHXAc8322f57ECtQCfBl4zxhgRKQNeBL5rjHnTqUkrSrYiInztglF88rQh/PzVdbz4/s5MT6lHAmF3NVW9dyI5e1QVe1qPs3Gv89W58SyXaDOm1moRsWZXYi0i+mq4BlYwtznJQC5AWaEvvUY/rNHfAbwCrAaeMcasFJF7ReSK8G6PAJUisgH4JmCndd4BjAJ+ICIrwn81js1eUbIQEeGfPzWZM4aX863freD9bQczPaWo2M3jUi3vAJwT1vVfW7PH8bE7UzbjsPrVxXlUFPlYm6DRj03eyWNfa7wpmye/VlboTX+evjHmJWPMGGPMSGPMj8Pb7jbGPB9+3GaMucYYM8oYM80Y0xTe/iNjTJExZkrEn/PfuqJkGXkeNw/feAaVRXnc+tgydh3KvrbMHcH0efpDKwqZMLiEV1budnzsUJzFWWBdmMcO8ie8DnKoh2KqSOrKCtjd2hZTC+eu8U4esbzQy5H2YFKtoCPRilxFSRFVxXk88rmpHDke4NbH3+Zou7MLXCdLIGwtU7FyVjQumVTL8i0HksqPj0YwgeIsgHGD/azbfTihTKu+2jAADC0vwBhiWo+3M2UzynClheGqXIeCuWr0FSWFjKst4RdzTmPljha+9cx7WZXK2RGw5pLK4qxILp5UC8ArK53tUhpv7x2bcbV+jnUE+ag5/mBup2fey0dnB4u3HYjB6Id/FtHy/qvChV6JZhp1R42+oqSYC8YP4nuXjuflD3dx31/XZXo6nXR0evqpl3cARtUUM6K6iD87bfQ7WyvH9z7G1pYAiQVzY/L0KwoA2Hqg74tKT713AGpLk+//H4kafUVJA7ec08i1U4dy/2sbeGN96vrQxEMgrOmnI5ALlhd7yaRaljQ1c8DBlhWJFGeBtciLCAkFc2Pp91Nbko/HJWyN4U6it0Du4FLr4rHTobiQGn1FSQMiwr1XTaS2JJ8H52dH/n668vQjuXjiYIIhw6urnQvoxrtcok2hz8OwikJW74w/mBvLOT1uF4PL8tkag7zTW3/+an8ebpc4lgygRl9R0kSex80t5zSyuGk/723NfBpnOvP0bSYNKWFIWQEvf+Bc/UIivXdsTh9Wztubm+OOtcRSnAUwtLyQbbHIO+F/o43ndgk1/jz19BUlF5kzfRj+fA8PLYxanJ5W7Dz9dGXvgOXJfvL0Icxfu5dF65xpuZJIcZbNjJGV7D/Szto41/E1vaRYRjK0vJCtzckFcsHS9dXTV5QcpDjPw40fG86fV+7K+JKL7Wkszorkq7NHMaqmmLuefc+RoqNEirNsuprBxRdn6avhms3wqkL2HT7eZz/83gK5AINL8x3rUqpGX1HSzOfPbsTrdjF3UVNG5xFIY3FWJPleN/9+7RT2H27nB39cmfR4ifTesakrK2BEVRF/i3Nlr1jjCOPDGUJ9BYtteaen8WpLCth5qM2R3kVq9BUlzVT78/j0GfX8/p1tSS3OnSx2cVa68vQjmTSklL//+Gj+9N4OFqxNrkjfXi4x3kCuzYxRlSxt2t8Z2I6FWDX98YMto99XsLiv8YZVFHC0PehIrr4afUXJAF88dwQdwRCPvrk5Y3Po6EzZTK+nb3P7zJGUFXp57t3tSY3TJe8kdvw5o6o40h6MK7hujEGkZw3eZlBJHmWF3j7bPfQlF42othrENTkgCarRV5QM0FhVxCWTanliyRZa25zroBgPXfJOZsyA1+3iwgmD+OvqPbR1JLaYCSRenGUzvbESkfgWbw+Z2O4sRIRxtX5W7+xd3umt9w7AiOoiAJr2qtFXlJzl9vNG0toWYN5bW/veOQUE0lyRG41LJw/m8PFAUgVrwSQ0fYDyIh/ja0vi0vVDxvSRt9PF+MElrN3V2pllFI1Oqb6HQetKC8j3umhyoDW1Gn1FyRCnDi3jrBGVPPLGJsc6KMaDfc5MefpgZc+UFnh5KYm8/VAf6Y6xMGNkJcs/OhDzHYch9hjC+NqSmHv89DSmyyU0VBapvKMouc6XZo1kV0sbf1yRnK6dCHZxVjorcrtjSzyvrtqd8Hq1yco7YAVz2wMh3vnoQGznDGv6sRBLMDeWVs0jqoscSfNVo68oGeS80VWMH1zCw4ua0t6BM52LqPTGpacMpjUJiceWTRKVdwDObKjA7RKWxCjxmBg1fYDRg4pxSe9Gv2th9J7HaagsYmvz0c7vLVHU6CtKBhERvjRzBBv2HOalD9O7tGI6F1HpjbNHVlGS7+GlDxLrvhlLm+O+8Od7OaW+lEUxXnhCIRNzBXC+182I6uIYPf2eB22oKiIQMmyPoT9/b6jRV5QMc9nkwUysK+H7z33oWNVlLGQyTz8Sn8fFeWOqE144PZnirEhmjanhvW0H2X+471z4WLN3bMYPLuk1g6e33js2DZVWBk+yEo8afUXJMB63i/vnnEZ7IMTX/3tF0rfvsZItnj7Ax0ZUsquljS3741/QJJjAconRmD2uGmNg0fq+ewLFo+kDjB/sZ/vBYxzqYYFz00uXTZuGKmtRlkQ+o0jU6CtKFjCiupgfXz2JtzY384vX0tN62a5A9WZY0wfL6AMs3RRfOwSIvTq2LybVlVJVnMdra/o2+lZxVuwnPH1YOQBLenh/nQ3XehmjujiPIp9bPX1F6S9cfVo9nzq9nvtfW8/fNqZ+oZVA0NKlE2lU5jQjq4uoKs5jSVNz3MeGHAjkgvU5zBpbzcK1e/q827JSNmMf+/Rh5RT63D0Gq/vqvQPWXcCoJBZzt1GjryhZxL1XTqSxsohvPL0iJm05GTpCoYzr+TYiwvQRFSxp2h+3rt9ZnOXAxeu8MdW0tAX6XEIxZExcmr7P42J6YwVv9NDNM9a7lclDSli5vSWpTK/s+MYVRQGgKM/D/defxoEjHXz7d6ldSD0QNHizwMu3+VhjBTsPtcXUfz4S+zNKpjjLZnytH4D1e/oy+vGf75zR1WzadyRqX/xYWzVPHlJK6/EAm/cnLvGo0VeULGNiXSnfu2w889fu5ddvbkrZeQLB7PH0oUvXX9IUf5tjJ7x8gOGVRXhcwvrdvbc7MCb2lE2bMYOspmlbohjsUF99GMJMGlIKwAfbD8V38giy5xtXFKWTm84azoUTBvGTP69h5Y7E/4P3RkfIZLQFQ3dG1RRTWeSL2+gHjUlaz7fxeVw0VhWxrg+jHwrF38p5cGk+ALtaevb0+xpyVI114di8L/EMnuz5xhVF6URE+MmnTqGs0Me3nnkv4RYFvdERCGVFuqZNorp+KBRf+mRfjB5UzIY+5Z34zzm4tACAHQd7XkOhrwtJnsdNeaE3qXUY1OgrSpZSXuTjJ5+azJpdrdz36nrHxw+ETEb77kTjrJFV7DjUFldjsZAxjsk7AKNr/GxpPtpr87V4Gq7ZFOV5KMn3RC3Ai6X3jk2NP589SSymokZfUbKY88cN4tqpQ5m7aCPLt8SfztgbHcFQVuToRzJ7bDUA89fEvppWMJR8umYkowcVYwxs7KWNcSKePljLM+7sLZAbw9dRU5KnRl9R+jPfv3w8dWUFfPOZ9zjaHnBs3EAw+zz9+vJCxgwq5rU4jH6iBrgnxgwKZ/D0ouvH03AtktoeFjiPpfeOTbU/jz1R4gKxokZfUbIcf76Xn11zKh81H+WfX1rj2LiBUCjjHTajMXtcDW9vbo55RTGn5Z2GcAbPut096/qhBLJ3wNL1d0bR9GPpvWNT489nb+vxhNN5s+8bVxTlJD42opIvnN3IE0u28HoMvWFioSNo8HqyzwTMHltDR9DwZg+FTN0Jhpw1+j6Pi1E1xazc0VtXzMQ8/brSfPYfaT8pXhBL7x2bQSV5BEKGA0fb4z4/qNFXlJzhrovGMqqmmLt+9z77HKjWtTT97JJ3AM4YXo4/3xOzxBOKsw9OLJxSX8oH2w/1mEWUqKRUa6dtdtP1Y+m9Y1Pjt8ZIVNdXo68oOUK+1819n5nCwWPtfObhxUm3Yc5GTR+s1bTOG1PN/LV7Y5IwQg4HcgEm15fRfKSdbQeif8bxNlyzaayK3h45lt47NrWleUD0fP9YUKOvKDnE5PpSHv/CdPa2HOfT/7U4anVnrHSEQllVnBXJ+WNr2Nt6nFdW9r2wStBhTR/glD4qX61Abvzjjq6xgsTd4wVdrST6HqOuzMr3397DBakvYvrGReRiEVkrIhtE5DtRXs8TkafDry8VkYbw9koRmS8ih0XkgYRmqCjKCUxrrOC3X/wYR9sDXPPQ4l4Djr0RCBo8WSjvAFwyuZZT6kv5+rwVLFzXewzD6eIsgHGD/Xjdwvvbohv9eBuu2ZQWeqnx551U8RtvINfjkh7vQvqiT6MvIm7gQeASYAIwR0QmdNvtFuCAMWYUcB/wk/D2NuAHwLcTmp2iKFGZXF/K07efBcBnHl7M+9sOxj1GR5b13omk0Ofh8S9MY1RNMbc9vowPejC+4Hz2DliVr+NqS/hge/TPNZGGazZjBvlPqviNJ2XT7RLqygoSXjYxlm98GrDBGNNkjGkH5gFXdtvnSuCx8ONngQtERIwxR4wxb2AZf0VRHGTMID+/+9JZFOd5uOFXS+NuxRwIGXxZavQBygp9PHnrdLxuF08t3dLjfkHjvKYPcOrQUt7edICfv7rupP76iTRcsxlVU8z6PYdPjFfE2HvHZkhZAdsPJNZ/J5ZvfAiwNeL5tvC2qPsYYwLAIaAyoRkpihIzwyuL+NXNU2ltCzDv7a19HxCB5elnp7xjU1Hk4/xxNfxl1W6CPQR1UyHvAHzt/NGcP66GX/zvel768MTYQqIpm2BdrI+2B/ntWx91bosnkAswpDy1nn60WXT/9GPZp+cTiNwmIstEZNnevc7kICvKQGFcbQkzRlby1JItca2va2n62evp21w8qZbmI+28vTl6G4pUyDsANSX5PHD9aeR5XLy/9USZJ5kq4Esm1XJqfSnff+5DVu9s6RwPYvf0h5YXsrvlOP/00uq4zx/LN74NGBrxvB7Y0dM+IuIBSoGYG4UYY+YaY6YaY6ZWV1fHepiiKGFuntHAjkNtvLpqd8zHdASzq8tmT8wcU43P4+LPH0bP5AmGEguqxoLH7WJcrf+kQi2ThKZfXuTjp9ecCnRl8cS6iIrNnGlDqSrO448rtsd9/liM/tvAaBFpFBEfcB3wfLd9ngduDj/+NPCaiXfNM0VREubj4wcxpKyAxxZvjvmYbOyyGY2iPA/nja7mLyt3RS2WSjSTJlYm1JWycseJhVqJtmGwGVZRiEhXvn6si6jY1JTkc8P0YexpPU57IPa7O4jB6Ic1+juAV4DVwDPGmJUicq+IXBHe7RGgUkQ2AN8EOtM6RWQz8HPgcyKyLUrmj6IoSeJ2CTeeNZwlTc0xL5zdEczO3jvRuHhSLTsOtUXNm3e6DUN3JtaV0NIWOEFDT7Thmk2+101daUGn0e/S9GMfY0h5AcYQd5FeTN+4MeYlY8wYY8xIY8yPw9vuNsY8H37cZoy5xhgzyhgzzRjTFHFsgzGmwhhTbIypN8asimuGiqLExLVTh5LncfHY33rOdIkkEDT4srD3TjQ+Pr4Gt0v4axT5KmTAlWKjD5wg8STr6YNVnbvZNvpx9N6xqU+wSCs3vnFFUfqkvMjHlVPqeO7d7Rw62neHSsvTz355B6z0zQmDS1i66eRQYcgYUqlSjastwSUnG/1k+/00VhXRtO8Ixpi4eu/YDCkPG/04s3jU6CtKP+LmGQ0c6wjyzLLe0zeNMWFNP3dMwJkNFazYevAkDTuVgVyAAp+bEdXFrDrB6CfWhiGShqoiWtsC7D/SHhHIjX1Qu3mbGn1FGcBMrCvlzIZyHl+yuce8drCCuEBWdtnsiWmN5RwPhE7S9UPGpFTeARhb62ft7i6jb4yJqXq2NxqrCgHYsv9I3CmbYFUN1/jz2KFGX1EGNjfPaGBr87FelxwMBC0jk0ue/hnDKwBOytdPRZfN7owb5Gdr8zGOHLdWLjMmtqUNe2NYhdVxc8v+o3H13omktjSfXS3xVWLnzjeuKEpMXDSxlrrSfP7p5dUcPh59ecWOkCWR5EKevk21P48RVUUs62b0g8YkbYD7Ykztid0xnUgTrS8vQAQ+aj7aFciN8+5hUEk+u6OsudsbavQVpZ/hdbv4t89MYfO+I3z3Dx9EzW23Pf1sba3cE1Mbynl784ET+takOk8fYNxJRj/x4iybfK+b2pJ8Ptp/tCuQG6+nX5Ifd1/93PrGFUWJibNGVvKtC8fyp/d28NTSj056vSPcriEXirMiObOhgkPHOli/p6s1cSjFefpgtT0o8LpZs8uuoE0+ZROsIq0tzUfj7r1jU1uaz6FjHSctv9gbavQVpZ/y5ZkjmTmmmnv/tIoPuwU/baPvzZHiLJtpjZau/9am/Z3bgmnw9F0uYcygYtbu6vL0nTjn8MpCtuw/mlAgFyx5B2B3HN5+bn3jiqLEjMsl3HftFCqLfXzlqXc4dKwrd78rkJtbnv6wikJGVBXx2OKu5nKhkDMGuC/G1vpZs6uVUMhYefoOjDm8soh9h49z9Ljlqcd79zCoJLx0Yhy6vhp9RenHVBT5eOD609hx8Bh3//HDzu2BkC3v5JYJEBH+z8Vj2X3NJ0sAAA5XSURBVLDnME+HaxGsLpupP/dZIys7u30m03AtklE1xQC837lYS3xjDk4gVz+3vnFFUeLmjOEVXHvmUF5dtbszANoRzL08fZuLJtYyraGC+15dR2tbR8qLsyLPW5zn4dnl2xxpwwDwsRGVuF3SuSRkvGM2VBZR5HPzzkcHYj5Gjb6iDAAmDynlaHuQreHVlnI1ewcsD/t7l41n3+F2nn57a1qKs8BawvGyyYN58YOdHD4ecORCU1rg5bShZWxttjz1eO8ePG4XZzZWsKQp5k72avQVZSAwNpxyaGeftOdo9o7NqUPLqCzysWnfEUIpWi4xGpedMpij7UG2HTjmWG3AuaO71hBJ5F1Mb6xkw57Dfe8YRo2+ogwAxgwK55mHjb4dBM1FT9+mqjiPva3Hw/JOes5pZ8uAM5o+wOCyrjETuXuoizg+FnL3G1cUJWaK8jwMqyhkTbi4yO69kytdNqNR5fex7/DxtMk7ACUFns7HTsURinxdYyYyZL7XHdf+avQVZYAwttbfmWfeVZyVuyagqjiPfYfbreKsNMk7JfnezsdOnbEwr8toq9FXFMUxxtX62bTvCMcDwYhAbu56+tW2vJOG4iybQp+7s/rXqZuLEz39+ActUKOvKEo0xgzyEwwZNuw53Jmnn9Oavj+PYx1BDrcF0ibviAj+fMtIO3WhKfRFePoJHK9GX1GUqNhNw9buaqW9H3j6VcVWNeqR9mBairNsbInHqUBuUV5ycYJ8b3xvXo2+ogwQGqqK8LldrN3V2pm9kysLo0ejqtjX+Thd8g50BXOdk3dU01cUJQV43S5G1hSzZldrzvbeiaTan9f5OK1GP+zpOybv5Gn2jqIoKWJcOIOnox9o+tXFXUY/1a2VI7E1faeuM5GafCJLMBb41OgritIDY2v97GppY//hdiC38/Qrinydhjedb8NpTT/ygpXI+8j3qKavKEoP2O0YVu6w+ut74zQY2YTH7aK80NL105W9A1BSYMs7zo+dyIXE43bFFZDP3W9cUZS4sTN4PtzeAuTeIirdsSWedBVnQZenHzp5FcqkSfRd5Htil3hy+xtXFCUuakvyKcn3dPZfz+VALlitGCAz2TtH26MvOp8Mib6P/Dh0fTX6ijKAEJFOiQdyW9OHrlz9dMo7/rCnf7jNeaOfqKsfT66+Gn1FGWDYRt/jEseCkZmiKiPyjuXptx5Phaef2HHxVOWq0VeUAcbY2hIg96UdiPD005m9U5A6Tz/Ri3A8ufpq9BVlgGEHc3M5R9/GLtBKa/aOLe+kwNNPOJCrRl9RlJ6wF1TpD0bfbsWQzuIsO5CbCqOfcCBXjb6iKD1RWuClrjQ/54O4kBlNP5WB3ETfRoEGchVF6Y2xtf5+4enXZEDe8Yd75UQufpIsdaXxLXnYnXgCuZ6+d1EUpb9xx/mjO3P1c5lqfx4/vGIiF02sTds5XS7hp58+hTOGlzs25tO3n8Wi9Xvjbp5mM2NUFf8R475iTArKypJg6tSpZtmyZZmehqIoSk4hIsuNMVP72i+m+zsRuVhE1orIBhH5TpTX80Tk6fDrS0WkIeK174a3rxWRi+J5E4qiKIqz9Gn0RcQNPAhcAkwA5ojIhG673QIcMMaMAu4DfhI+dgJwHTARuBj4z/B4iqIoSgaIxdOfBmwwxjQZY9qBecCV3fa5Engs/PhZ4AKxqgyuBOYZY44bYzYBG8LjKYqiKBkglkDuEGBrxPNtwPSe9jHGBETkEFAZ3r6k27FDup9ARG4Dbgs/PSwiu4BDvcyptJfXq4B9vRybC/T2/nLlfMmMmcix8RwTy7597dPX6/o7zPz5kh0vlb9Dp/aLfH14DOOBMabXP+Aa4FcRz28E7u+2z0qgPuL5Riyj/yDw2YjtjwCfiuGccxN9HVjW1/jZ/tfX+8+F8yUzZiLHxnNMLPsm8xsMv66/wwyfL9nxUvk7dGq/ROYYi7yzDRga8bwe2NHTPiLiwbr6NMd4bDT+lOTruU66318qzpfMmIkcG88xsew70H+DkPu/w2THS+Xv0Kn94p5jnymbYSO+DrgA2A68DVxvjFkZsc9XgcnGmC+JyHXAJ40xnxGRicBvsXT8OuB/gdHGmGC8E40VEVlmYkhbUpRUor9DJVvpU9M3lkZ/B/AK4AZ+bYxZKSL3Yt3CPo8l2zwhIhuwPPzrwseuFJFngFVAAPhqKg1+mLkpHl9RYkF/h0pWknXFWYqiKErqyP3mG4qiKErMqNFXFEUZQKTN6ItIgYgstCtyReRmEVkf/ru5h2OuEZGVIhISkT6DYiIyVETmi8jq8HFfj3jtHhHZLiIrwn+XRrwWtVVET+0nRGSeiIxO9LNQMk+u/R5FxCcii8KJFYqSOGnMwf0q8PXw4wqgKfxvefhxeZRjxgNjgQXA1BjOMRg4PfzYj5V1NCH8/B7g21GOmQC8B+QBjVg1Bu7w30ZgBOAL72OPNRP4Zbo+O/3T32P4tX8Ebsj0Z6d/uf2XTnnnBuCP4ccXAa8aY5qNMQeAV7F685yAMWa1MWZtrCcwxuw0xrwTftwKrCZKBXA3emoV0Vv7ideBj6vXldPk2u8R4LnwvBUlYdJi9EXEB4wwxmwOb4rW2qGv/wzxnrMBOA1YGrH5DhF5X0R+LSJ2M+ye5tLjHI0xIaz/jKc6OWclPeTo7xHgQ+BMJ+elDDzS5elXAQcjnkdb5sax3FERKQZ+D/y9MaYlvPm/gJHAFGAn8G99zKWvOe7BKjhTco9c/D1irBqXdhHxOzU3ZeCRLqN/DIhcDyzR9gx9IiJerP9gTxlj/mBvN8bsNsYEw176L+m6Ze5pLn3NMR/rfSm5Ry7+Hm3ygDYn5qYMTNJi9MM6qVtE7P9orwAXikh5+Lb2wvC2mBCRISLyv1G2C1Z18GpjzM+7vTY44unVWLfKAM8D14UXgmkERgNvYbWbGC0ijWE54LrwvjZjsBrNKTlGjv4eEZFKYK8xpiPWuSlKd9IZyP0LcA6AMaYZ+H9YhvVt4N7wNkTkV3Y6nIhcLSLbgLOAF0XE/o84GKutQ3fOxuoCen6UVLh/FZEPROR9YDbwjfBcVgJ2q4g/E24VYYwJAHb7idXAM+F9EZFBwDFjzE6nPhwl7eTU7zF8zGzgJcc+AWVAkrY2DCJyGvBNY8yNDox1B/CRsfr+pB0R+QbQYox5JBPnV5InF3+PIvIH4LvxZBApSnfSlnJojHk3XKjiNkk2XTPGPODUvBLkIPBEhuegJEGu/R7DEuNzavCVZNGGa4qiKAMI7b2jKIoygFCjryiKMoBQo68oijKAUKOvKIoygFCjr6QUEflauLXwU3Ee1yAi16dqXqlAROpE5NkY9jvcw/arRGSC8zPr/bzKwEKNvpJqvgJcaoyJtztkAxC30bf742cCY8wOY8ynkxjiKqzWyr2i3V2VZFCjr6QMEfn/7Z1taFZlGMd//9lQpjQ/JH4ITRBNcoVQBJOKBWofVotqWB96mX0wrCQEK8GCaPQ2owiGKUUsKuiFImIfGjFW7SVHOnNbQgVmREUfDKUFVht3H67rsLOn55nPPmyDPdcPDlznPufcL2fbdd/nOjv/6xCWj+ATSftdTfJrSccl3ernrJHUK2nIt81++fPA9f4V6x5JLZLac3V3Smpwe0zS05IGgXpJV8sSpByT1JVJHvhTx0lXtnx3mn6PSFou44yke738LUlbJC2SdMDHMizpgdxYRt2ukfS+H39P0qByiVckPSPphKQjklb6uJuAAz7mtQV96pD0kqQe4AVJ10oa8Hs5IOlyP69F0keSPpUlhGkrMr5LJH0lqXFGP9BgYTDfgv6xLewNOI2pWj4L3O1ly7GEIkuBGmCJl68DjrrdAHTm6mkB2nP7nUCD2wnY7nY1MACs8P07gTfc/hVYnPVhmj4fAhqBOkyW4TUv/wFYBuwEnvCyxcBRLOHJGmDUy/cCh92uw2Qarsn19xa323J1dQDNJfrU4WPOEqpcDFzk9hbgw9x9OgXUYqJyPwGr/NgYsBKTd946378bsc3PFo+JwVyxDWiStNf3lwCrMUfcLmkTMIEJ2c2UCUzJEiyzVR3wmemdsQiTLgYYBt6R9DGWkKQUvcANmMN8Fdgp6VLgj5TSmKRtwFWSslBOLTZhfZ+r4zrgFYCU0qhr7GT8gzlwgGPA1jLH+UGa/Hq4FnhTlrYzYZNdRndK6RyApJPAZZhGfzXQjen5fFFmm8ECI5x+MFcIuCMVyAhIegr4HUtIU0Vp2eBxpoYj89LI53POUMC3KaX6InU0Ys68CXhS0sZkwnqFfImlU1wN7MdUMJuxySBrY3dKaYoSpyxRCrlzSvFvSin7FH6C8v8O/8rZrUBPSuk2b/fz3LG/c3a+/nFskrkJCKdfoURMP5gruoDd8uW3C56BrVh/S6Yrfw+2Mgf4E8srm3Ea2CSpStIqJvXnC/kOWCGp3tuplrRRUhUW5ugBHsNCTMuKVZBS+hkLSa1LKZ0C+rBwTeb0u4BdMq18JK2XtLSgmj5gux+/Ariy5J2ZZMqYJT3sYm7FqAV+cbuljLrBngjuBzZI2lfmNcECI5x+MFe0YuGFYX/Z2erlB4H7JB3BQjvZanYYGPeXnXuAfuBHYAR4ERgq1kiyfMbN2MvOE8A3wGZsMnlb0ghwHHg5pXS2WB3OIJPhml4sZWGf77+OSR8P+VgO8//V+kFs8hkGHvfxnJumPbA8zI/6y9m1wAbgTIlz24DnJPUzOVFeEH8iugu4UdKD5V4XLBxCcC0IZgH/19HqlNJ5d+DdwHqflMqtoxO4fSbXBMGFiJh+EMwONUCPh4AE7Jqp804p3TwrPQsqmljpBxWLpB3AIwXF/Smlh+ajP0EwF4TTD4IgqCDiRW4QBEEFEU4/CIKggginHwRBUEGE0w+CIKgg/gNmKy157xizNgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "report_results('rank', scores);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'YaMus': {'HybridSVDs': {'features_weight': 0.1, 'rank': 3000}}}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### saving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(experiment_name, config=config, tuning=scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
       "<progress style=\"width:60ex\" max=\"1\" value=\"0\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>0%</strong></span>\n",
       "<span class=\"Iteration-label\">0/1</span>\n",
       "<span class=\"Time-label\">[0<0, 0.00s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[2K\r",
       " [############################################################] 0/1 [0<0, 0.00s/it]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:15ex; max-width:15ex; vertical-align:middle; text-align:right\">YaMus</span>\n",
       "<progress style=\"width:45ex\" max=\"5\" value=\"4\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>80%</strong></span>\n",
       "<span class=\"Iteration-label\">4/5</span>\n",
       "<span class=\"Time-label\">[11:06:17<02:49:04, 9994.16s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[2K\r",
       "          YaMus [████████████████████████████████████#########] 4/5 [11:06:17<02:49:04, 9994.16s/it]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "result = {}\n",
    "for label in track(data_labels):\n",
    "    models = prepare_recommender_models(label, data_models, config)\n",
    "    result[label] = ee.run_cv_experiment(models,\n",
    "                                         fold_experiment=ee.topk_test,\n",
    "                                         topk_list=topk_values,\n",
    "                                         ignore_feedback=True,\n",
    "                                         iterator=lambda x: track(x, label=label))\n",
    "    save_cv_training_time(experiment_name, models, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "report_results('topn', result, target_metric);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### saving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(experiment_name, cv=result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cold start"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaling_params = {lbl: read_config('scaledsvd_coldstart', lbl)['PureSVDs(cs)']['col_scaling'] for lbl in data_labels}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_cold_start_recommender_models(data_label, data_models, config):\n",
    "    data_model = data_models[data_label]\n",
    "    svd = ScaledHybridSVDColdStart(data_model, item_features=meta_dict[data_label])\n",
    "    svd.col_scaling = scaling_params[data_label]\n",
    "    svd.max_test_workers = max_test_workers\n",
    "    models = [svd]\n",
    "    apply_config(models, config, data_label)\n",
    "    return models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config_cold = {}\n",
    "scores_cold = {}\n",
    "data_models_cold = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for label in track(data_labels):\n",
    "    data_models_cold[label] = prepare_cold_start_data_model(label, *all_data, seed)\n",
    "    model = ScaledHybridSVDColdStart(data_models_cold[label],\n",
    "                                     item_features=meta_dict[label])\n",
    "    model.use_raw_features = True\n",
    "    config_cold[label], scores_cold[label] = fine_tune_hybridsvd(model,\n",
    "                                                                 svd_ranks[label],\n",
    "                                                                 hybridsvd_weights,\n",
    "                                                                 label)\n",
    "    del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/py36/lib/python3.6/site-packages/pandas/plotting/_core.py:1001: UserWarning: Attempted to set non-positive left xlim on a log-scaled axis.\n",
      "Invalid limit will be ignored.\n",
      "  ax.set_xlim(left, right)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEXCAYAAACtTzM+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnXmcHGWd8L+/vue+cyczIQlHQiBACPclLoRFIShXRNAVZRURV3e9VmGFV3fFi3dd8cBVkcuAqC8RgqjLzUIggXAECDnIMTknmSSTSTIzfTzvH1XV0zPpnq7uru6ZTn7fz2c+01311FNPT/XUr363GGNQFEVRFN9wL0BRFEUZGahAUBRFUQAVCIqiKIqNCgRFURQFUIGgKIqi2KhAUBRFUQAVCIqiKIqNCgTlkEVE7hORXw3adpaI7BCRsUMcFxARIyKbRMSfsj1kHxsr5roVpVioQFAOZW4E/l5E/g5ARCLAL4B/NsZsdnH8HuC8lPcfALZ7vkpFKREqEJRDFmPMDuBzwJ0iUgX8G7DaGHOXiJwiIi+KyC4R2SwiPxKR4KAp7gGuSXl/DXB36gARaReRs1Pef0tE7rJfV4rI/bZWsUtEXhKRZu8/qaK4QwWCckhjjPkdsBT4LXAd8I/2rhjweaAZOA2Ym7LP4Q/A+0SkVkSagJOBR3I4/T8AlcAEoAm4HujJ75MoSuEEhnsBijIC+CywGvi6MWY9gDHm5ZT9a0TkTuAs4Mcp2/cBjwGXARXAH4HeHM4bxRI4U40xbwBL8v4EiuIBKhCUQx5jzFYR2Q4sd7aJyJHAD4ATsJ7iA8DiNIffjWVqqsDSKHLhLmAc8KCI1GKZoL5hjFGntDIsqMlIUdLzc+BNrKf3WuBmQNKMexJoBeqNMS+k2b8XS6A4jHFeGGP6jDHfNMYcBZwOXAJc5dH6FSVnVCAoSnpqgN3AXhE5igP9BwAYq378B4B5GeZZBlxph6rOAT7k7BCR94nI0SLiA7qwTEhxDz+DouSECgRFSc8/Ax/DCi39OfBApoHGmDeNMW9l2P114EhgF3ATcH/KvnFYjukuLHPV37Cc24oyLIg2yFEURVFANQRFURTFRgWCoiiKAqhAUBRFUWxUICiKoihAmSWmNTc3m7a2tuFehqIoSlmxdOnS7caYlmzjykogtLW1sWSJZvcriqLkgoisczNOTUaKoigKoAJBURRFsVGBoCiKogAqEBRFURQbFQiKoigKoAJBURRFsVGBoCiKogAuBYKIzBWRFSKySkS+mmZ/WEQesPcvFpG2QfsniUi3iPyL2zkVRVGUflZs2cMH/utZPnvfK8QTxalSnVUgiIgfuAO4AJgOzBeR6YOGXQvsNMZMBW4Hbhu0/3as3rO5zKkoiuI5xhgWr9nBojc2F/1ct/35HU789t/42h/eKHiuJ1ds482NXTz6xmbe277Xg9UdiBsNYQ6wyhizxhjTBywALh405mLgN/brh4BzRUQARGQesIaUfrUu51QUpQwwxrB03U5++JcVbN69f7iXMyTbunp4/w+f5oo7X+T6+15h+abdRTvXhs59/PSp1XT3xHj09U0kCnyqX7FlT9rXXuJGIIwHNqS8b7e3pR1jNwjfDTSJSBXwFeCWPOYEQESuE5ElIrKko6PDxXIVRSkF+/vi/OSpVZz7w6f58E//lx89sYrLf/4CGzr3DffSMnLf4vWs2b6Xb19yNJUhP79+fm3RzvX48i0A/ONZh9HVE2PF1sJu4iu27OGkyY34fcI7W7q8WOIBuBEI6RqLDxZ1mcbcAtxujOnOY05rozF3GmNmG2Nmt7Rkrc2kKEqJuOnhN/nun1fQVBXiux8+ht9+6mR274tyxc9fYG2RTBqFEE8YfrdkA6dPbeaqk1r58PETWLhsE9u7e4tyvj+/uYWjxtby4eMnAPDy2s6854rFE6zq6OaYCXUc1lzF25uHT0NoByamvJ8AbMo0RkQCQB3QCZwEfFdE1gL/BPyriNzgck5FUUYoS9Z28tDSdj591hR+9+lTufzEiZwypYnfXncy+6NxLv/5C0Wzc+fLMys72LS7h/lzJgHw8dPa6IsnuO/F9Z6fq3NvH0vX7+T8GaOZ0FDBmNoIL72Xv0BY17mPvliCI8bUcuTYWt7ePHwawsvANBGZLCIh4Epg4aAxC7EakgNcCjxhLM4wxrQZY9qA/wv8uzHmxy7nVBRlBBKLJ/jG/3uTcXURbjx36oB9M8bVseC6U4jGE3zm3qX0ROPDtMoDWfDSepqqQrz/qNEATGmp5uwjWrh38Tr6YglPz/Xmxt0YAydNbkJEmDO5kedWbadzb19e863aZhlZpo6qZk5bAxt37efdAk1Q6cgqEGyfwA3A48DbwIPGmOUicquIXGQP+yWWz2AV8EVgyDDSTHPm/zEURSkV97y4jne27OHmD06nMnRgBf0jxtTwg8uP5Z0te/iPRW8PwwoPZPf+KP/z9jY+dPx4QoH+297HT22jY0+v5xFHjo3/yDE1AHzm7Cns643zr3lGGzkmuMnNVZx/9BhEKEqUlKt+CMaYRcCiQdtuTnndA1yWZY5vZptTKT3GGLp7Y+zaF2Xnvj527ouya18fO/f2sXt/jOpIgHF1EcbWVzCuLkJzdRifL50LSDkU2Lanhx/+5V3OPLyF82eMyTjufUeO5hOnTeZXz7/HaVObOW+IsaXg2ZUdxBLmgDWfOa2Fw1qq+PXz73HxrHHYwZEF886WPYyqCdNQFQLgqLG1fPqsw/jRE6vY2tXD6NpITvOt3bGXxqoQdRVBIMjs1gb+5+1t/NP7D/dkvQ5l1SBHGZpoPMEu54Zu3+AHvN7rbBt484/lEA4X9AujayOMq6tgbH2EsXUVjLN/j62LMK6+gobKoGf/WMrIIZEw3LLwLXpjCW65aEbWa/yVC47gpbU7+NJDr3P0+DrG1VeUaKUH8sQ726ivDHLcpIYB230+4eOntnHzw8tZtmHXAfvz5Z3NezhybO2AbadObeZHT6zirc1dOQuE97bvpa2pMvn+6PF1LHhpA4mE8fQBTQXCCCaeMOzo7mXT7h627N7P1q7etDd052a/pzeWca5QwEdDZZCGyhD1lUGmjqqmvjI0YFtDZYiGqqC9PURtJMCenhibdu9n864eNu/ez6bdPWzeZf1+Zf1OtuzeTDQ+UKBEgr6kgBggMOojNFWFqAoHqA4HqAoHqAz6VeMoA2LxBF/+/es8+sZmvjz3CCY3V2U9Jhzw81/zj+f8//sMP31qNf9n3tElWOmBJBKGp1d0cNbhLfjTfNfmHTeeby5czpMrOjwRCE5E0OnTmgdsP8oWEG9v7uKcI0blNOfa7fs4dWpT8v1hLdXsj8bZ0tXjqaBVgTBMxBOG7d29bLZvsJt397Clq4dNu/azZXcPm3f3sLWrJ+3Te20kQENViPrKEI1VIaa0VPff0Cv7b+j1lUEaqqxtFUF/Xk/tDVUhGqpCzBhXl3Z/ImHYvre3X2DYvzfbn+GF1dvZuqc3Y6q9CFQG/UkhUR0JUBUK2O/9A4RHXUWQ9x05iomNlWnnUopDXyzB5xe8ymNvbuGf/+5wPnPWFNfHTm6u4sKZY/njqxv56gVHUhUu/S3ntfZd7Njbx/uOTH8Tro0EmTGujpfe2+HJ+dbu2EtfLJH0HzjUVQQZX1+Rc8jo/j7rxj+5qV8IT2mxXq/p2KsCYaST7mbv3CSHutmHA77kU/VJkxsZWx9hTF0FY2sjjK2PMLo2Qn1FkIB/5NQk9PmEUTURRtVEOHZifdoxsXiCju5eNu3qYde+Prp7Y3T3xtjbG6O7N87e5Ov+7Rt37U9u39MbS0aBfPNPyzn78BauPqWVsw4flfaJT/GOnmicT9+7lKdWdHDTB6Zz7emTc57joydP4o+vbmTha5uSIZ+l5Ml3tuETOOvwzHlMJ7Y1ct/idfTG4oQD/oLO9+5WKyLo8NE1B+w7amwtb+WYHb1xl5X9nfogNKWlGoA12w/URApBBUKODHWzd274udzsx9VFGGNvP1ht7wG/Y0LK/0kmGk+wZXcPv1uygd++vIFP3LWE8fUVXHXyJC6fPZHm6rCHK1Ycfre0nadWdPDvl8zkIyfldzM/flIDR46p4d4X13HliRNL/h1/fvUOZk2sp74ylHHMnMmN/Or593hz425OaG0s6Hz7+6xQ29pI8IB908fW8MQ7W+mJxokE3QmeWMJ6GEqNjhpVE6Yq5Gf1tsE5v4WhAiEF52afarbRm/3IIOj3MbGxki+edwSfO3caf1m+lXteXMt3/7yC//vXlVwwcwxXn9zKCa0N+nf2kF123PzlsyfkPYeIcNVJk7jp4eW83r47oyZZLLp7Yll9Hie2Wb6Dxe91FiwQEsa6P6T7Gh45tpaEgZVbu5k5Ib0Z9oD57BSJVGVYRGhrrmK9x2VCDhmBMPhm7zhqC73Zj6uroF5v9iUl6Pdx4TFjufCYsazcuof7Fq/n90vbeXjZJo4cU8NHT25l3nHjqR4Ge/XBRjRu3Y0KNc3NO248//HYO9y3eF3JBULcmKzrb6oOM3VUNS+/1wlnF3Y+Wx6kFQiOqee9HXtdCwSTrOozcMIxtRE27e7Jd5lpKav/mDc27qbtq496Np/e7MufaaNr+OZFM/jy3CNYuGwT97y4jm/8vzf5zmPvcMlx47n6lNa0tlzFHdGEIeT3Ffw/UBMJcvGscfzx1Y18/cLpdjx9aXAbmjlnciMLl20q2I/g3MB9af5mrU2ViMB7He7LemQSMKPrIry6YVfe60xHWQmEUTVhPn/utLyOFbGeAvRmf3BSGQpw5ZxJXHHiRJZt2MU9L67jgSUbuOfFdXzouPF8ee6RjKnLLfZbgWgsQcDvzf/IR09u5bcvbeDHT6zk6xeWrv1JwhjcKDjnTR/N/YvX8+Q7Hcw9Ov9EOsfIkE4gRIJ+xtVV8N5297Z/k2G+MbUROvf25eSPyEZZCYTRtRG+8HfeZuYpBxciwnGTGjhuUgM3XTidO59dwy+ffY/H3tzCZ8+ZwifPOMyzf55DgVjCEPQoqm3GuDrmz5nEL597j4tnjefo8e5MJoUSNwa/iwe/06c201wd5g+vtBcoEBwNIf3+w1qqcir852gcg6dzHnC2dfUyqcmbUOyRE7+oKB7TUBXiK3OP5G9fPIuzDm/h+395l3N/8DSPvr4ZY4rTgvBgoy+eIOiRhgDw1QuOpKk6zFd+/zqxuLcF5TKRSODKZBTw+5g3axxPrtjGzjyL0EG/hpC2yD9Wbsaa7XtdfwczmYzG2NnOW7q88yOoQFAOeiY1VfKzq0/gt586mdqKIJ+9/xWu+PmLvLmxeN2yDhZi8YRnGgJYyVm3XDSD5Zu6itqcJhW3JiOAS44fTzRueOT1/KvxG5PZhwCWQNjTE2OHS6GTyDCfoyF42aVOBYJyyHDKlCYe+dzp/PslM1nd0c0Hf/wcX3nodTr2FKdBysFANG488yE4XHD0GN5/1Ch++Nd32ebh020m4onsUUYO08fWMrm5iidX5N+dMZPN38EJgV3j0rGc1CMGO5VtDWGragiKkh9+n/CRkybx5JfO5pOnT+YPr7Zzzvef4mdPr6Y3NnJq948Uoh5rCGD5eW76wHT64gl++vRqT+dOh6UhuBMIIsKpU5pYvGZHMuQ2n/NBZh+CE/W2wmUbzEwCpjZilXVp36kagqIURG0kyNcvnM5fvnAWJx/WyHcee4fzbn+GP7+5Rf0LKUTjCYI+728TrU1VfOi48dy3eD1bPI6lH0wuGgLAaVOb2dsX5/X2/EI6E0mbf/pzjq2LUFcR5C2XNY2c7+Pg2USE1qZK1u3wLjlNBYJySDO5uYr//tiJ3P2JOYQDPj5971Lm/0L9Cw6xuCEYKE5o9ufeN41EwvDTp1YVZX6HhMlsvknHKYc1IQLPr8qv2J0ZIlPZ2i4cOabGdRvMpI86zXxtTVWs2+Fdq1JXAkFE5orIChFZJSIHdEMTkbCIPGDvXywibfb2OSKyzP55TUQuSTlmrYi8Ye9b4tUHUpR8OPPwFhbdeAbfmnc07261/Atffug1tu0pvo17JNMXTxAogoYAlrP/0hMm8NuXNrBpl3dmj8EkEu5NRmBFp00fW8vzq7bnd74sTmWwityt2LInYxXgVIbySbQ2VdK+c79nEVtZr7SI+IE7gAuA6cB8ERmcVXItsNMYMxW4HbjN3v4mMNsYMwuYC/xcRFJzH84xxswyxswu8HMoSsEE/D4+enIrT/6L5V/446sbOed7T3HHk6tGVG/gUhKLW5nKxeKz50zFYPh5EX0JVumK3I45bWozr67flSxUlwv9N/DMY6aPrWV/NO6qFlEig8kIoK25iljCsGmXNw8ubv5Mc4BVxpg1xpg+YAFw8aAxFwO/sV8/BJwrImKM2Wf3TwaIkOIwV5SRSl2F5V/46xfO4rSpzXzv8RWc+4On+dNrmw45/0I07l2mcjomNlby/qNG8/jyrUX728Zz1BAAjptYT188kWxunwtDZSo7OM1y3nFhNjJD5DW02T0S1npkNnIjEMYDG1Let9vb0o6xBcBuoAlARE4SkeXAG8CnUwSEAf4iIktF5Lr8P4KiFIe25iruvGY293/qJGorgnzut69y6c9e4N2tuTU4KWeiHmYqZ+LMw1vY0tXD6g5vSzk7GOMuMS2VaaOtInQrt+V+rYeqdurQ2mxlFrvREIaqjTShwSopv2GnN45lN1c63ccaLMozjjHGLDbGzABOBL4mIk5BmdOMMcdjmaI+KyJnpj25yHUiskRElnR05B8brCj5cuqUZh753Onc9uGZrN2+lw/813P88rn3SOTQi7pcica8zVROx+lTrQYvz7ybn80+G25LV6TS2lRFwCeszEND6I8KynzO2kiQmkgg2fxm6Pmw5zuQlpowIrC1y5tcGjcCoR2YmPJ+AjA4jS85xvYR1AGdqQOMMW8De4Gj7feb7N/bgD9imaYOwBhzpzFmtjFmdktL5o5HilJM/D7hihMn8fgXzuTMac38n0fe4upfLfY0S3QkEkt4n4cwmImNlUxuruLZlcV54Ivn0Yg+6PcxubmKlVvzEQjW72ynHF9fwUYXOQRmiDDWoN9Hc3XYswQ/N1f6ZWCaiEwWkRBwJbBw0JiFwMfs15cCTxhjjH1MAEBEWoEjgLUiUiUiNfb2KuA8LAe0ooxomqvD/OKa2fzHh2by6vpdnH/7Mzy8bONwL6toWJnKxY9OP2NaMy+u6fQ8OdBkSRIbimmjq1mVl8kI+5xDn3RCQ4U7DYGhP8Po2rBn9YyyXmnb5n8D8DjwNvCgMWa5iNwqIhfZw34JNInIKuCLgBOaejrwmogsw9ICrjfGbAdGA8+JyGvAS8Cjxpg/e/KJFKXIiAjz50xi0Y1nMGVUNZ9fsIwbf/squ/dFh3tpnhP1uLhdJk6f2sz+aJxX1nlb398J68zVZAQwdVQN6zv35Rxh5saHAO41hP5Et/T7x9RGPEvuc1X+2hizCFg0aNvNKa97gMvSHHcPcE+a7WuAY3NdrKKMJNqaq/jdP57CT55azX/+z0peXtvJDy47llOnetf0fLgpVqbyYE6Z0oTfJzy3qoNTpjR5Nm/c0RDyUBGmjaomYeC97XuTUUFuMMYgkjlT2WF8QwV7emPs3h8dsmGQGSrMCKum0SvrvRGkmqmsKAUQ8Pu48dxp/OEzp1IR9POR/17MbX9+x1XCUTlQzEzlVGoiQY6fVM+zK711LPf3I85DICQjjXLzIyRMxsrXAxhfb0UaZUvKc75JmU1GVqMcL8xtKhAUxQOOnVjPozeewZUnTuSnT63mk795ma6e8jchFTNTeTCnT23hjY27C+pFMBjHfJOPG2RycxU+gZU5hhkb3OU9jLdDRrOZjfpLYaSf0+mLsM2DSCMVCIriERUhP9/58DF8a97RPLtyO/PueL5osfWlIhY3hAKluU2cc2QLxsCiNzd7NmfcRRmJTIQDfqa0VLN8k7uaQw5uaye5zSEYKuwUrN7K4E2jHBUIiuIxHz25lXs/eRK79kWZd8fzPLli23AvKW+i8QSBfEJ08mDm+DqOGlvLvS+u9yxr2ckVyaXaaSrHT2rglfU7c8o5Sdg+hGw0VYWoiQSyPjRk66+Q7JzmgWNZBYKiFIGTD2ti4Q2nMaGhkk/c9TI/e3p12ZW9MMZ42lM5GyLCR0+exNubuzxzkjq+nHw0BIATWhvYtS/Kmlx6ILvUEESEaaOqs5bHyBa15HRO86JRjgoERSkSExoq+f1nTuHvZ47lO4+9wz89sKysiuRF49aNqBRhpw7zZo2nOhzg3hfXeTJfMicgXw2htQGAV9btdH/OhDsNAWDqqGpWbRta2GR7jKiNBKgI+tmsGoKijGwqQwF+PP84vnT+ESx8bRNX3PliXhU0h4OYHaJTKg0BoCoc4EPHj+fR1zfT6YFzOelUzlNDOKy5ivrKIEtzEAgG9xrJtFE1bO/uZde+zJ81m8lIRBhTF1EfgqKUAyLCZ8+Zyk8+cjyvbdjFrY8sH+4luSIas+5EpchUTuWjJ7fSF0/w4JIN2Qdnod9klN/xPp9w/KQGlq7PQUNw6UMAS0MAhjQbZWu4A5YfYatqCIpSPlwwcyyfOXsKv31pQ1mUu4jaGkKohCYjsHoOHz+pnsfeKDzaKFFAYprDCa0NrNrWze797sKI3foQoF8gDJXrMFTHNIcxdRE1GSlKufHPf3c4s1sb+Nc/vMF7OTgqhwOnyXypNQSwKsy+uamLvb2x7IOHwElMy9dkBFbGMsBal9crYYxrjWR8fQUBnwxZBjubyQgsgbBtT0/BFXhVIChKCQn4ffxo/nEEAz4+e98rI9rJHEs6lUt/mzhxciPxhGHZhsKijeKmsLBTsEphA6xz0bsAHJORu/P5fEJLTZiOPZmTyobqmObQXB0mGjfs6SlMgKpAUJQSM66+gu9feixvbe7iPxa9PdzLyUhf3HEql9ZkBHD8pHp8Ai+915l98BA4PoQCFAQmNVolJta51BAsk5H7+UfVhNk2hEBwYzJqrLJqIe3YW1i2sgoERRkG3j99NNeePpnfvLDOE1t5MRhODaEmEuSosbW8vLYwgWA80BAqQn7G1EZYu8OthpC9sF0qLTVD9zPIVroCoKEyBMDOIaKV3KACQVGGia/MPZJjJ9Tx5d+/zgaX5ohSkvQhlChTeTAntjXy6vpdyXXkQ7zAsFOHSU2VrO90qyG49yEAtNREhjQZZStdAdBUFQZgR7cKBEUpS0IBHz/+yPEA3PDbV+mL5X/jKwbOjThYolpGgzmxrZH90XjOtYRSSYadFijU2poqc9AQ3BW3cxhVE6ZzX19Gwec0yBlSQ7BNRqohKEoZM7Gxku9++Bhe27CLH/713eFezgCSmcolqnY6mBMnW1nCLxfgR3AToeOG1qYqOvb0uop6Mi7LXzuMqg1jTOanezctOZMaQoHJfCoQFGWYuWDmWC47YQK/fG7NiApFjQ2jUxlgVE2EtqbKgvwIyY5pBd7pWpssx/JQ4aEOufoQRtXY5av3pPcjJDumDSFmKkJ+IkFfwaXDXf2ZRGSuiKwQkVUi8tU0+8Mi8oC9f7GItNnb54jIMvvnNRG5xO2cinIo8aW5RxD0+/jOYyMn6qhvGPMQHGa3NbJk3c68CwMWUv46lTYn9HRHdoFtjCEXpWpUjfV0n6mfgZtMZbC0hKJrCCLiB+4ALgCmA/NFZPqgYdcCO40xU4Hbgdvs7W8Cs40xs4C5wM9FJOByTkU5ZBhVE+H6s6fw+PKtvLB6x3AvB+iPMgoNo0CY09ZI596+vPtKJAqsduowydYQ3PgRcvYh1NoCIYNj2U3YKVh+hFJoCHOAVcaYNcaYPmABcPGgMRcDv7FfPwScKyJijNlnjHGMbhH6P5ubORXlkOKTZxzGuLoI33r0rRHRgrM/U3l4TEYAs9tsP8Ja97WEUnH+jIWEnQLURoI0V4dY40IwuW2h6dBc7QiE9CYjN2GnAI1V4YILAroRCOOB1CpT7fa2tGNsAbAbaAIQkZNEZDnwBvBpe7+bObGPv05ElojIko6ODhfLVZTyJBL085ULjmT5pi7+8Er7cC+HaGL48hAcJjdX0VwdytuPUGg/hFSmtGTvXQC5VTsF6+87qiacsbeym7BTgOaqENtLEHaabh2DH18yjjHGLDbGzABOBL4mIhGXc2Iff6cxZrYxZnZLS4uL5SpK+XLRseOYNbGe7z2+ouA6PoUSjQ2vUxmsp+LZrY15C4SEB4lpDtNGV7NyW3dWf0Yu1U4dJjRUsKEzg0Cwf2cTMi21VgmMQhoxuREI7cDElPcTgE2ZxohIAKgDBlxBY8zbwF7gaJdzKsohh4hw0weOYtueXn7+zJphXctw9ENIx4mTG9nQuT+vFpGFlr9OZWpLNXt6YkMmkYGTmJbbCSc2VtK+K71/IlvHNIdRNRH64gl27XNXlTUdbq70y8A0EZksIiHgSmDhoDELgY/Zry8FnjDGGPuYAICItAJHAGtdzqkohyQntDbygWPGcuczq9m8O/1TYynoizv9EIZPQwA4MelHyF1L8KL8tcO00TXA0L0LwKqwmrNAaKhk066eZKhvKm5NRk60Ukd3/vWMsgoE2+Z/A/A48DbwoDFmuYjcKiIX2cN+CTSJyCrgi4ATRno68JqILAP+CFxvjNmeac68P4WiHGR8Ze6RJAx8788rhm0Nzs1pOKOMAKaPraUy5GdJAQKh0NIV4K53gXPOfExG8YRJ29PArVM5W/iqGwJuBhljFgGLBm27OeV1D3BZmuPuAe5xO6eiKBYTGyu59vTJ/PSp1Xzs1DaOnVhf8jUMZz+EVAJ+H8dPauClPCKNnAduL5zKo2rC1IQDWTUEQ26JaWBdb4D2nfuTr1PnAxcmo9qhE9zcoJnKijJCuf7sKTRXh7jtz+8My/mTpSuG2WQEVvjpO1u66OrJzT7ebzIqfA0iwtTR1azctmfIcbkWtwPLZASwYeeBfoRcTUZDldLOhgoERRmh1ESCfPKMw/jf1TtYsWXom1AxSBa3G6ZaRqnMaWvEGHJqdg/9iWleRBmB5VhetW3obOVEDi00HcbWR/AJtKcpjZFwmW1dFQ5QFfIXZDIa/iutKEpGLp/7VGSoAAAgAElEQVQ9kVDAx70vriv5uaPxBH6feOKQLZRjJ9YjAm+0787pOK/KXztMbKxke3cvvbHMne5yaaHpEPT7aKwK05EmjyCpIbiYs6k6TGcBTXJUICjKCKaxKsQHjhnLH15pZ0+O5pJCicXNsPVCGExVOEBrYyVvb86tFHZ/xzRvPscYx04/xFO4sZwIOc/dmKH0RL8PIfuctRWBgtpoqkBQlBHONae0sbcvzv97dWNJz9sXTwx7hFEqR42tzVkgOE/XXpmMnLpDW4focJaPhgBW17PONP0M3Ba3A6gJB3P2s6Qycq62oihpOXZCHTPH13H3C+sKykLNlVjcDHsOQipHja1lXee+nDK4k+WvvdIQ6iwNYcuQLS/zi2pqqAyl1xBcOpVBNQRFOegREa4+pZWV27p5cU1hPYZzIRpPDHuWcipHja3FGHgnBwd7PIenazc4JqOhsqbz1hCqQmk7njkd09wImZpIkK79qiEoykHNRceOo64iWFLncjRuRphAsDKFczEbeR1lVFcRJBTwDRnaaSWm5elD2Bc9QAtM5OBUro0E6VINQVEObiJBP5fPnsDjy7cMab/2EktDGDkmo/H1FdRGArkJBI99CCLCmNrIkBpCri00HRoqQ8QT5oAber/JyI2GEKC7N5Z3+XQVCIpSJnz05FbixvDbl9aX5HyxRGLYs5RTERGOzNGx7FXHtFTG1EaK4kNorAoBHOBHcExGrpzKEav4RHeeWsLIudqKogxJa1MVZx3ewv2L1yeTxopJX2xkmYzAqmv0zpY9SVNQNhIeVjt1GF0XyR5llMefraHSEgiDI41yyUOorQgC5B1pNLKutqIoQ3L1ya1s29PLX9/aWvRzxRIjy2QElh9hX1/cVbN78LYfgsPomjBbu3oyRnzl2kLToSGThuA4xl2YjGptDUEFgqIcApx9xCgmNFRw9wtri36ukRZlBFakEbh3LCf7IXgoEMbUReiJJujan94skzD5JcI1OhrCAQLB+u3mI9RGLA0h39DTkXW1FUUZEr9PuOqkVl5c08m7W4tb3yg6gjKVHQ4fXYNP3AsEt3WAcmF07dC5CIY8ncpV1s18cIOb/igjd2GnQN6hpyoQFKXMuOLEiYQDPn5R5I5q0XiCUGBk3SIiQT+Tm6t422UuguNq8SoxDfoFQiY/Qj7VTgGqwwGCfmFHJqeyizlqKxyTkWoIinJI0FgVYv6cSfzx1Y1szNCY3QtGUi2jVI4cU+taO/Ky/LXDWDtbOVM3u3x9CCKSNls5J6eyagiKcuhx3ZmHIQJ3Pr26aOcYiT4EsMxG6zv3sa8v+1NwwuPSFWBpCCKwaVd6DSGRyL+YXmNV6EANwWXHNOiPMtpdTIEgInNFZIWIrBKRr6bZHxaRB+z9i0Wkzd7+dyKyVETesH+/L+WYp+w5l9k/o/L6BIpyCDKuvoIPHTeBBS9vyNr0PV9GqkA4YkwNxsDKrUN3LoPi5CGEAj5G1YTZlEE7y7d0BUBTdeiA8tW5FE/1+4SaSKB4AkFE/MAdwAXAdGC+iEwfNOxaYKcxZipwO3CbvX078EFjzEzgYxzYTvMqY8ws+2dbXp9AUQ5RPnP2FKLxBP/9XHF8CVbpipFnMjpijFXCYoULs5HjkPW6p8PYuoq0/Y8d8pU/jVXhtFFGuQi0uopgUTWEOcAqY8waY0wfsAC4eNCYi4Hf2K8fAs4VETHGvGqM2WRvXw5ERCSc10oVRRlAW3MVHzhmHPe+sI5daYqiFUosPrIylR0mNVYSCfpcdZFLJIynOQgO4+srsmgI+Z2zKY3JKGFMTlFL9ZXBvL8Pbq72eGBDyvt2e1vaMcaYGLAbaBo05sPAq8aYVH3o17a56CbJYCATketEZImILOno6HCxXEU5dLj+nCns7Ytz1/+u9XzuvhFW3M7B7xOmjapx5ViOG+Op/8BhXH2Ejbv2p01Oy6eFpkNjVYg9PTH6Yv2Z6Ln22ym2hpBuKYP/CkOOEZEZWGakf0zZf5VtSjrD/rk63cmNMXcaY2YbY2a3tLS4WK6iHDocOaaW9x81ml8/v5buHPoEuGEkZio7HDGmxrWGUAR5wLj6CnpjiQPMO+BUO81vXqeeUeq8JsdEt/qKELuKKBDagYkp7ycAmzKNEZEAUAd02u8nAH8ErjHGJEMijDEb7d97gPuxTFOKouTIDe+byu79Ue7zuDR2NDYyncoAR4yuYdue3rQNZVJJmOKYjMbWVQCk9yMUoCE02QJhR4pj2eRoMqqrzL8ngpur/TIwTUQmi0gIuBJYOGjMQiynMcClwBPGGCMi9cCjwNeMMc87g0UkICLN9usg8AHgzbw+gaIc4syaWM/pU5v5xbPv0RPN3Pw9V6KJkdUxLRW3juV4wtuQU4fx9ZZASJcH4rmGQO4mo11p+iq4IatAsH0CNwCPA28DDxpjlovIrSJykT3sl0CTiKwCvgg4oak3AFOBmwaFl4aBx0XkdWAZsBH4Rc6rVxQFgM+eM5Xt3b38bsmG7INdEh1hPZVTSQqELGajQm7OQzGu3kpOS+dYLsSH0FSdzmSUm5O6viJILGHY15f7w0HAzSBjzCJg0aBtN6e87gEuS3Pct4BvZZj2BPfLVBRlKE4+rJGZ4+t4YMkGrj6lreD54gmDMRDwMsXXQ0bVhKmvDLrQEIpjMmqsChEO+NKajArTEKwgzB3d/QIhkWPDnTo7OW3X/ihVYVe3+CQj82oripITIsLFs8bx5sYu1nRkT9jKhtNvIRgYmSYjEeHw0TVZi9wVy4cgIoyrr2DjzgM1hHwb5ID1dO+TAp3KlU6RvNxDT1UgKMpBwoXHjEUEHnl9c8FzJQXCCNUQwPKdLN/URW8ss2mkkJyAbExoqGDDzgP7MuRb3A6sBLrGqjDbu1OcyuSmcdRVWGanfEJPR+7VVhQlJ8bWVXBiWyMLX9uUl0MxlWjcOn6khp0CnNDaQF8swZsbM2sJ8UTxBMKkxkrW7ThQIFgmnvzPOaomzLY9qVFGuZmM+jUEFQiKckjzwWPHsWpbN++4LA+diZitIYzETGWH4yc1APDKup0ZxySMt93SUmltqmT3/ii7D+hfkF8LTYdRtWG27en3TRhjcjIZOa04VSAoyiHO3x89Br9P+NNrg1OFcqPPFggjNcoIoKUmTGtTJUvWdWYck0gUdnMeikmNVQAHtPPMt2Oaw6iaMNu6Uk1GufWETmoI+9WHoCiHNE3VYU6b2syfXi/MbBSzTUYjNQ/B4YRJDSxdtyvjZ40X0YfQ2lQJwLrOvQO2F+JDAKu89vbu3mT7z0SOGkIk6Ccc8KmGoCgKfPCYsWzo3M+yDbvyniPpVB7BGgLA8a0NbO/uZUNn+kJz8URxahmB5UMADvAjWC00C9MQEgZ22I7lXH0IkH+Bu5F9tRVFyZnzZowh5Pfxp9fyjzYqB6cywOw2y4+QyWxkjPelrx2qwgGaq8Os3zHYZFSYhtBSYyW9OY5lK1M5twkbKkOqISiKYiUmnXVEC4+8vilpdsiVctEQpo2qoSYcYGkGx3IxNQSwzEaDTUZWQb0CNIRaKznNcSybPBLd6iqCeRW4G9lXW1GUvLjo2HFs29PLy2szO1yHIpYY+VFGYEUQzZpUn1kgFKl0hcOkxsoDzFWFJKaB5UMAko5lNRkpilIQ5x41ioqgn4V5Rhv1xcrDZAQwpaWa9jQZw1C8BjkOkxor2bR7/4D+BQWbjKodDaFfIOQqYOor1GSkKIpNZSjA+6eP5rE3NifNP7ngaAgjOezUYXRthO7eGPv6DuwHUazSFQ5N1SGMga6e/ptvrtVJBxMK+KgOB5I39HxqI9VXqslIUZQULjp2HDv3RXlu5facj42WQWKaw6ga+4m6q/eAffECcwKyUROxisft6ekXRl6Uy6gI+dkftea0opZyIxL00xdL5OxDGvlXW1GUvDjz8GZaasLc8eSqnHMSyiXKCPqdsFu70lQeTRiK+RFqI1YSWGpDmkIT0wCqQn729sbt+XJ3UleE/ABD1nlKhwoERTlICQf8/NP7p7Fk3U7+9va2nI4tlygjgFGDwjRTKbbJqNYuNZ2qIRSamAZQEQr09zMwuZugIgHruvVEczMXjvyrrShK3lwxeyKHtVRx25/fSdYnckMsqSGM/FtE0mSURiAUs7gd9JuMBvgQCowyAktDGGAyylUgBC0NIdcOeiP/aiuKkjcBv48vn38kq7Z189DSdtfHObWMAkV8uvaK+sogIb9vQEE4h2KWv4ZMJqPCQ10rBpmMcv0MjkDYXwyBICJzRWSFiKwSka+m2R8WkQfs/YtFpM3e/ncislRE3rB/vy/lmBPs7atE5EdSTM+PohzCnD9jNMdPquf2v73LfpdtFR0NIRQY+c+MIkJLTZiOdE7lIoedpncqF+5DqAz5k9cqnzyEomkIIuIH7gAuAKYD80Vk+qBh1wI7jTFTgduB2+zt24EPGmNmAh8D7kk55qfAdcA0+2duTitXFMUVIsLX/v4otnb18qvn33N1TLSMNARwSkan8yEUr3QFQFUogE/6TUaO877QU1aFAuzt6zcZ5a4hFM+HMAdYZYxZY4zpAxYAFw8aczHwG/v1Q8C5IiLGmFeNMU5mzHIgYmsTY4FaY8wLxvoL3g3My2nliqK45sS2Rt5/1Gh+9tTqAe0ZM9HfQnPkawjgNJXJZDIq3nl9PqE6HEhqCE6Upydhp339JqNcVQRHQ+gtgsloPLAh5X27vS3tGGNMDNgNNA0a82HgVWNMrz0+1aCZbk4AROQ6EVkiIks6OjpcLFdRlHR8Ze4R7O2LcceTq7KOTYadjuAWmqmMqolkdCoXs5YRWJFGjg/BMw0hPCjKKMfjkyajIoSdplvL4KDmIceIyAwsM9I/5jCntdGYO40xs40xs1taWlwsV1GUdEwbXcMlx03g3hfXZY04iiXDTsvEZFQTZte+6AE282KbjABqIsGkycjREAr1IVQE/eyPxkkkDIbcncoVSR+C9yajdmBiyvsJwOACKckxIhIA6oBO+/0E4I/ANcaY1SnjJ2SZU1EUjzmxrYHeWILNuw80r6TimIyK6ZD1Eic5rWOQlpAohYYQCdCVNBlZEqHQU1aG+qOEEol8wk6tW7vbIAIHNwLhZWCaiEwWkRBwJbBw0JiFWE5jgEuBJ4wxRkTqgUeBrxljnncGG2M2A3tE5GQ7uuga4OGcVq4oSs60NlltH9M1h08lmjCE/L6iln3wkkzJafEC+xu7oSaSajKythXqQ6gMW9FL+/riGEzODXeKZjKyfQI3AI8DbwMPGmOWi8itInKRPeyXQJOIrAK+CDihqTcAU4GbRGSZ/TPK3vcZ4L+BVcBq4LGcVq4oSs44bR/X7tg75LhoLDHi22em0q8hDNR8EkVOTAOorUh1KnvjQ6i0b+j7+mJW2GnOmcr5mYwCbgYZYxYBiwZtuznldQ9wWZrjvgV8K8OcS4Cjc1msoiiFMaY2QijgO6Ax/GBiCVMWWcoOmTSEYpeuACs5LRl2am8rOFM57AiEeF55DeFk2KlmKiuKkgGfT5jUWMm6LBpCXzxRNg5lgKaqEH6fHFDxNF7kTGWwfAjdvTESCZPUEAqlItRvMgKTc5RROOBDpDhhp4qiHES0NlZm9SHE4omy0hB8PqG5OnRALkIiUfjTejZqK4IYA919MYxtoSnYhxAaaDLK1Q8iIkQC/uKUrlAU5eBhUlMl6zv3DVkSOxo3ZeVDgPS5CFbpiuKeN1ngbn/UOx9CKNVklLtTGaxII612qijKkLQ1VbGvL05H94GJXA7RMtMQwMpF2No1PD4EsOoZJQVCgeesTJqMYnl3YIsE/epDUBRlaCbZkUbrhzAbReOJsslSdhhVGz4wyiiP5jK5UpNS8dTRubxokAN22GmexfIqgn56YqohKIoyBK2NTuhpZoEQixuCgfIyGTVXh+nc20cipW1kaUpX9Fc8TSamFTin0/Fsf9JklDth1RAURcnGhIZKfALrh4g06osnCJSZhtBYFSJhGNBcPmGKn22d1BB6ot4lptkmI6cnQn4mI58KBEVRhiYU8DGuvoJ1Q+QixOJWpnI50VRtJad17u33I5QiMc1xKnf3xjxzKvt9QijgY180lncHtkjAT686lRVFyUZrU+WQJqNovLwylQGaq0IAbO/uL+8dL3L5a4DqcL/JyCsNAew2mgWYjCJBX1GqnSqKcpAxqbFqSJNRtMwylQEaqy2BsCNVIBS5YxpY0Twhv4+unqhnxe3AMhv1O5XzW5eajBRFyUpbUyU790XZnWJvTyUaK69MZYCmqgNNRqYE5a/BMht1p2gIXkQ2VYT8dthpfpFSlkBQk5GiKFlozRJ6GkuUXx5CQ6Xl3C21yQigOhIYmIfgiYbg769llMfx4YA6lRVFccGkRrsMdmd6s5GVqVxet4eA30dDZZAdKRpCKcJOwdIQ9vREPWuhCVYewb6+uNUxTU1GiqIUC0dDyFTTKFpmxe0cmuxcBCCZj1ASk1E4SHdvLFkOxAsZVBUOsN/uh5CPgAkHfZqYpihKdqrCAZqrwxlNRuWYqQxWLoJjMnLMN6XTEGKetdAEy4ewty9ml7/O/fhIwE9fLDEgUS8b5XfFFUXxBCv0NL3JqBwzlQGaq0PssGs0xT2qK+QGx4dgvPQhBK2wU5N3cTsr27k3By3BlUAQkbkiskJEVonIV9PsD4vIA/b+xSLSZm9vEpEnRaRbRH486Jin7DkHd1JTFKUEtNpVT9NRjpnKYGkI/SYja1uxE9PAKnDntQ/BcSrnX9wu9yY5Wa+4iPiBO4ALgOnAfBGZPmjYtcBOY8xU4HbgNnt7D3AT8C8Zpr/KGDPL/tnmetWKohRMa2MVm3f3pL1hxOKGUKD8BEJTVZid+6LE4ol+k1EJPkaN3SQnnvBOQ6gIBezEtPxMUMXSEOYAq4wxa4wxfcAC4OJBYy4GfmO/fgg4V0TEGLPXGPMclmBQFGUE4TiWN6TREqLxBIFSxGt6TLOdnNa5r6/fZFQCDaE6HCBhYG+f1VvZCx9CZchPXzxBLJ7IO1MZPNYQgPHAhpT37fa2tGOMMTFgN9DkYu5f2+aim6TYNWoVRRlApkgjY0zZ9VR2aEwmp/VXPS2FQEgtgQ2FVzuF/iY5e3tjeWkc4YB1fC7lK9xc8XRLGey2djNmMFcZY2YCZ9g/V6c9uch1IrJERJZ0dHRkXayiKO5obXJyEQYKhGjc+tctz7DT/vIVjvmm2KUroL/AnZP57Y0Pwa542hfP02TkaAjemozagYkp7ycAmzKNEZEAUAd0DjWpMWaj/XsPcD+WaSrduDuNMbONMbNbWlpcLFdRFDc0VAapCQdYNyjSKGZ7Y8tRQ2iyC9zt2NvX7+AtoUBwNAQv/PHJNpq9sfxMRo6G4LHJ6GVgmohMFpEQcCWwcNCYhcDH7NeXAk+YIRq2ikhARJrt10HgA8CbrletKErBiAiTmioPMBlFY9a/brllKkN/Cewd3b2elpHIRlIg9HjnQ3Ca5OSrIYSDuQuEQLYBxpiYiNwAPA74gV8ZY5aLyK3AEmPMQuCXwD0isgpLM7jSOV5E1gK1QEhE5gHnAeuAx21h4Af+BvzC9aoVRfGEtqYq3trcNWBb1NYQQmVoMqqvCOKTQSajEvoQvDUZ+ZOvCws7dW8yyioQAIwxi4BFg7bdnPK6B7gsw7FtGaY9wd0SFUUpFpOaKvnLW1uIxRNJjSAat24g5agh+HxCY1XINhmVsHTFIB+Cl07lfOfrDzv11mSkKMpBSmtjJdG4YfPu/sjwWNKpXJ63h6aqsGUyKmFimtMkp6sITuV854vkYTIqzyuuKIonTGy0Qk/bd+5PbuuLO07l8jMZAUkNIV7CxLSqUAARq68yeFf+2iG/WkbFiTJSFOUgpdGOytm5r7+HQNlrCHY9o3gJ8xB8PqE6HGD3fu+dytZ8uR+fj1O5PK+4oiie4AgEp/4PpPgQyjBTGaC2YmAp6lLkIYBVz6jfZFT4fKkmo7zyEFRDUBQlF+rtLmO79h0oEIJlWMsIrMYy+/viJS1dAQM7lHmhIaSa7PKZLeD3IQLxhAoERVFcEA74qQ4H6Nzb31s5malchtVOwRYI0XhJTUYAoRSB4IWGkPr3z1fABHxCTPshKIriloaq4CAfQnk7lStCfhKm31RSKpNR0O9LVhb1QkPw+SQpWPL9CH6fJAWjq3PmdxpFUQ4WGitDA3wIfWWchwD94Zb77MqjpXKFhAK+5NO4V+d0rkG+0wV8PtUQFEVxT0NVKG2UUahsBYK17r29tkAomYbQfx6vzFRBe+35zqcagqIoOdEwSEPoz1QuU5ORrSF091r2/FKUrgAIBQoLE01HUkvLcz7Lh6BOZUVRXNJQGWJnqkBIlHceQsUgk1GpfAipGpVXGoIT+ptPT2VQDUFRlBxprAqyty+erHkTjZW3UzmSbCzjhICW5ryhQEqYqGcagmMyyu94v0+SJkA3qEBQlEOcBjs5bdc+K/S0nPshQL+G4PgQSmYyKoqGYDuVNcpIUZRS0Fg5MFu5L+70QyhPDaHfh1Bak1GwCALB0dLyNRkFfJJM0HODCgRFOcRxNATHj+DkIZRrlJFTAygZdloqH0IgNZHMmzkdp3K+OYJ+TUxTFCUXkvWM7NDTcu6HAP2tIx0fQqkylQdqCN7M2V9PKl8NwUdcfQiKoriloXKghpAsXVGmJqNIyM5D6CutDyEcKLzUxGAcIVOID8FzDUFE5orIChFZJSJfTbM/LCIP2PsXi0ibvb1JRJ4UkW4R+fGgY04QkTfsY34kXv0FFUXJCafAnVPPKFncroxrGUFqYlppzlsMH0KhUUYBv3hb3E5E/MAdwAXAdGC+iEwfNOxaYKcxZipwO3Cbvb0HuAn4lzRT/xS4Dphm/8x1vWpFUTwj6PdREwkks5VjcYPfJyWzvXtNJDg8JqNUH4JXfzpHKBeSh+C1hjAHWGWMWWOM6QMWABcPGnMx8Bv79UPAuSIixpi9xpjnsARDEhEZC9QaY14wVtHyu4F5rletKIqnNFb1ZytH44my7YUAloAL+qXfZDQcTmVPuir3rz1fmRYoQtjpeGBDyvt2e1vaMcaYGLAbaMoyZ3uWOQEQketEZImILOno6HCxXEVRcqWhsr+eUTRuyjbCyCES9PebjIbBqex9YtrI0RDSrWTwGdyMyWu8MeZOY8xsY8zslpaWIaZUFCVfGqtSBUKibHMQHCqC/qTJaDg0BK/MbYUmBwZ8Ps81hHZgYsr7CcCmTGNEJADUAZ1Z5pyQZU5FUUqEVc+oP1O5XLOUHSpCVpMcKGH56wHVTr2ZM1CgychXBA3hZWCaiEwWkRBwJbBw0JiFwMfs15cCTxiTOT3OGLMZ2CMiJ9vRRdcAD7tetaIontJYFezPVI6ZshcIkZTKo8PhVPbKh+Bch3w/Q8AnJHIQCIFsA4wxMRG5AXgc8AO/MsYsF5FbgSXGmIXAL4F7RGQVlmZwpXO8iKwFaoGQiMwDzjPGvAV8BrgLqAAes38URRkGGqpC7I/G2d8XtzWE8jYZOQXuYLhKV3gzZyBZuiI/cvUhZBUIAMaYRcCiQdtuTnndA1yW4di2DNuXAEe7XaiiKMUjmZy2r8/2IZS3hlAR9D4nIBshv/eJaYUWt7OijLQfgqIoOdCQUuAuGi9/k5GTnAYlTEwrRh7CCIwyUhTlIMepZ+RoCOVuMqpINRmVqnRFETKVk+auEZSHoCjKQU5jlVW+Yue+KLGDQEOIBEvvQyhGtdNkLaO8M5V92iBHUZTcSC1w11fmmcow0GRUqjJpwaL4EAqsZaQagqIouVJXEUTE8iHE4okBT7vlyHBrCN5FGRVY7dSvPgRFUXIk4PdRVxG0fQjmoNIQSuVDGLEd0zTKSFGUXGmsDNlRRgdHprJDqQrrhwPeCwQn7NQMWQkoMz5RDUFRlDyorwymRBmV961hOExGxSxuF3f/kD/w+Bwzlcv7qiuK4hlWCewosYQp/7DTYTAZFSfKyBEI+UkE9SEoipIXVoG7PqKxgyBTOeR95dFsBAcUt/PmnM48udzUU9EoI0VR8sIpgd13EOQhOBpCKX3joSL4EBxzVy5mn4HH+7yvZaQoysFPQ1WI3lgC0xMte5NR2BYIpfIfwMBaRl6Xvy5EQ8iF8n4MUBTFMxrt5LS+WPk7lfs1hNIJBBHpDxP1ymTkaAiZuwkMSa4CsbyvuqIontFg1zMCDoqOaVBagQCWluDlKZ0n/Fz8AOmOd4sKBEVRgP56RkDZ91R28hBKaTICq+Kpl0LIb+ch5GsyUg1BUZS8qK9M0RBKVTO6SAyHUxksQerlOR25rBqCoiglpTFFIAQD5W0yigyDUxms5DQvi+k5GkK+AsGfo6bnarSIzBWRFSKySkS+mmZ/WEQesPcvFpG2lH1fs7evEJHzU7avFZE3RGSZiCzJadWKonhObUUw+XQbLHcNITQ8PoRwwOdRN2WLQn0IuSblZb3qIuIH7gAuAKYD80Vk+qBh1wI7jTFTgduB2+xjp2P1V54BzAV+Ys/ncI4xZpYxZnZOq1YUxXP8Pkmajco97DRi5wSUKinNIeSxD8GLxLSczudizBxglTFmjTGmD1gAXDxozMXAb+zXDwHniqU3XQwsMMb0GmPeA1bZ8ymKMgJpqLQcy+WeqRzw+wj6pWRlKxyCnvsQCk1M814gjAc2pLxvt7elHWOMiQG7gaYsxxrgLyKyVESuy3RyEblORJaIyJKOjg4Xy1UUJV+cVprlHmUElh+h1D4ErzWEpMkozzyEXMOH3Vz1dDMOXl2mMUMde5ox5ngsU9RnReTMdCc3xtxpjJltjJnd0tLiYrmKouSL0zmt3PMQwIo0KrGCQNAvnp7TV6gPoQgaQjswMeX9BGBTpjEiEgDqgM6hjjXGOL+3AX9ETUmKMuw4GkK5ZyqD5VguvYbg9zTKqNSJaW5qGb0MTBORyRMcA68AABIkSURBVMBGLCfxRwaNWQh8DHgBuBR4whhjRGQhcL+I/BAYB0wDXhKRKsBnjNljvz4PuDWnldtEo1Ha29vp6enJ53BlmIlEIkyYMIFgMJh9sFJ0DhanMlgaQl8sz0YCeRLyi6c+hEKdyv4co8WyCgRjTExEbgAeB/zAr4wxy0XkVmCJMWYh8EvgHhFZhaUZXGkfu1xEHgTeAmLAZ40xcREZDfzRlqQB4H5jzJ9zWrlNe3s7NTU1tLW1layZtuINxhh27NhBe3s7kydPHu7lKPRnKx8MGkIk6Ceab2eZPPHch+AvzKlcDA0BY8wiYNGgbTenvO4BLstw7LeBbw/atgY4NqeVZqCnp0eFQZkiIjQ1NaHBAiOHfh9C+QuEiqCffX2xkp7T+8S0QjWEQzBTWYVB+aLXbmSR9CGUuuZDEYgEvX1ad0OxitvlJw6KpCEoinJoMGdyI1edNIljJ9YP91IK5so5k9i5t6+k55x33HiOGFPj2XxHj6vjH05r4x9Ozc+kWl8Z4oxpzaxzOf6g0BCGm+rq6gHv77rrLm644YYhjzn77LNZsiR7xY4lS5Zw4403pt3X1tbG9u3bAfj2t7/NjBkzOOaYY5g1axaLFy/mm9/8Jl/72tcGHLNs2TKOOuqo5PEzZ85k5syZTJ8+nW984xv09vYOuR5jDO973/vo6urKuvZU3njjDT7+8Y/ndIxSemoiQb59yUyqwuX/rHj+jDFcOWdSSc952tRmPnnGYZ7N5/MJ//bBGUxqqszr+Onjarnn2pPcny+vsyglIRaLMXv2bH70ox8NOe6FF17gkUce4ZVXXuH111/nb3/7GxMnTmT+/Pk88MADA8YuWLCAj3ykP0jsySef5I033uCll15izZo1XHddxhxBABYtWsSxxx5LbW1tTp9l5syZtLe3s379+pyOUxSldJT/Y0AKt/xpOW9tyu3JNRvTx9Xybx+ckdexe/bs4ZhjjuHdd98lGAzS1dXFMcccw8qVKwG49957ufHGG+nq6uJXv/oVc+bM4Zvf/CabNm1i7dq1NDc3c9111/H973+fRx55hB07djB//nw6OjqYM2cOxs5e3Lx5M83NzYTDYQCam5uTa6ivr2fx4sWcdJL1lPDggw/y+OOPH7DW6upqfvaznzFx4kQ6Ozvp7e3liiuuoKuri1gsxk9/+lPOOOMM7rvvvgFC4+677+b73/8+IsIxxxzDPffcw+9+9ztuueUW/H4/dXV1PPPMMwB88IMfZMGCBXz5y1/O6++pKEpxUQ3BA/bv38+sWbOSPzffbAVg1dTUcPbZZ/Poo48C1tP5hz/84WTM/d69e/nf//1ffvKTn/CJT3wiOd/SpUt5+OGHuf/++wec55ZbbuH000/n1Vdf5aKLLko+bZ933nls2LCBww8/nOuvv56nn346ecz8+fNZsGABAC+++CJNTU1MmzYt7eeora1l8uTJrFy5kvvvv5/zzz+fZcuW8dprrzFr1iwAnn/+eU444QQAli9fzre//W2eeOIJXnvtNf7zP/8TgFtvvZXHH3+c1157jYULFybnnz17Ns8++2yef2VFUYrNQaUh5PskXygVFRUsW7Ys+f6uu+5K+gc++clP8t3vfpd58+bx61//ml/84hfJcfPnzwfgzDPPpKuri127dgFw0UUXUVFRccB5nnnmGf7whz8AcOGFF9LQ0ABYT/dLly7l2Wef5cknn+SKK67gO9/5Dh//+Me58sorOfXUU/nBD37AggULkufMhKN1nHjiiXziE58gGo0yb968pEDo7OykpsZymj3xxBNceumlSY2ksbERgNNOO42Pf/zjXH755XzoQx9Kzj1q1Cg2bRqc5K4oykhBNYQic9ppp7F27Vqefvpp4vE4Rx99dHLf4JBL531VVVXG+TKFafr9fs4++2xuueUWfvzjH/P73/8egIkTJ9LW1sbTTz/N73//ey6//PKMc+/Zs4e1a9dy+OGHc+aZZ/LMM88wfvx4rr76au6++24AAoEAiYSV7GOMSbuen/3sZ3zrW99iw4YNzJo1ix07dgBWzkg6QacoyshABUIJuOaaa5g/fz7/8A//MGC74/B97rnnqKuro66ubsh5zjzzTO677z4AHnvsMXbu3AnAihUrkn4JsCKJWltbk+/nz5/PF77wBaZMmcKECRPSzt3d3c3111/PvHnzaGhoYN26dYwaNYpPfepTXHvttbzyyisAHHHEEaxZswaAc889lwcffDB5w+/s7ARg9erVnHTSSdx66600NzezYYNV8Pbdd98dIBAVRRlZHFQmo5HKVVddxTe+8Y0DzDUNDQ2ceuqpSadyNv7t3/6N+fPnc/zxx3PWWWcxaZIVUtfd3c3nPvc5du3aRSAQYOrUqdx5553J4y677DI+//nP81//9V8HzHnOOedgjCGRSHDJJZdw0003AfDUU0/xve99j2AwSHV1dVJDuPDCC3nqqaeYOnUqM2bM4Otf/zpnnXUWfr+f4447jrvuuosvfelLrFy5EmMM5557LsceayWlP/nkk1x44YX5/REVRSk6YvKssz0czJ492wyO3X/77beTcfUjlYceeoiHH36Ye+65Z7iXUjCbN2/mmmuu4a9//WtOx/X29nLWWWfx3HPPEQgMfA4ph2uoKOWMiCx105lSNYQi87nPfY7HHnuMRYsWZR9cBowdO5ZPfepTdHV15ZSLsH79er7zne8cIAwURRk56H9nkUlnpil3hnJMZ2LatGkZw10VRRkZHBRO5XIyeykD0WunKCOHshcIkUiEHTt26I2lDHH6IUQikeFeiqIoHAQmowkTJtDe3q419csUp2OaoijDT9kLhGAwqN22FEVRPMCVyUhE5orIChFZJSJfTbM/LCIP2PsXi0hbyr6v2dtXiMj5budUFEVRSktWgSAifuAO4AJgOjBfRKYPGnYtsNMYMxW4HbjNPnY6Vn/lGcBc4Cci4nc5p6IoilJC3GgIc4BVxpg1xpg+YAFw8aAxFwO/sV8/BJwrVpGbi4EFxpheY8x7wCp7PjdzKoqiKCXEjQ9hPLAh5X07MLgFT3KMMSYmIruBJnv7i4OOHW+/zjYnACJyHeAU4O8RkeVDrLUO2D3E/mZg+xD7RzrZPl85nLPQ+XI9PpfxbsdmGzfU/nL/DkLpv4fFOF8hc+ZzrNffw1y/g62ZBqbiRiCkK685OMYz05hM29NpJmnjRo0xdwJ3AojIncaYjC29XOxf4iZ9e6SS7fOVwzkLnS/X43MZ73ZsId/Dcv8OQum/h8U4XyFz5nOs19/DQu+FmXBjMmoHJqa8nwAMLmqfHCMiASzp1DnEsW7mTMefCtxf7gzH5/P6nIXOl+vxuYx3O1a/h+V/vkLmzOdYr7+HRfkOZi1uZ9/g3wXOBTYCLwMfMcb8//bOPkaLq4rDz4+VDymVom0IbahLCbQCVarVhFoVDKVJUSwVcWNF0D+qLVTE4FeqkUD8olpjQvohLcEgpqIlSNB0JYS2fFgshbKwbGoV8aMl1RRpiym2S49/3DPZ4e27u+8u7+7sy54nmeydOzN3zp2975y59878TnNun4XAlWb2BUkNwE1mNlfSROAXpDmDi4FtwDhSz6HDMnuCc+HpLKhtog0GfZlOh4x8TmAR0AjUAWvMrFnScmCvmW0GHgDWSfozqWfQ4Mc2S9oAHAZagYVmdhqgXJnVr94b+GnnuwRBjxJtMOiz1JT8dRAEQdBz1LyWURAEQVAdwiEEQRAEQB9wCJLeLOlR/3oZSfMlPePL/HaO+YSkZkmvS+p0gk7SaEnbJbX4cYtz25ZJelbSU77ckNvWJdkNSQ9KCtH/GqfW2qSkQZIe8xdAgqD7mFmhC7AQWOzptwJH/O8IT48oc8w7gMuBR4CrKzjHKODdnj6f9IbTBF9fBiwtc8wE4AAwGBgD/IU0AV7n6cuAQb5PVtaHgNVFX9NY+leb9G3fBm4u+trFUttL4T0E4GbgN56+HthqZsfN7D/AVpIG0hmYWYuZPV3pCczsmJnt8/TLQAttX0y3R3dkN3YA0+NJreaptTYJsMntDoJuU6hDkDQIuMzMjnpWOZmMzn4kXT1nPXAVsCeXvUhSk6Q1kkZ0Yku7NprZ66Qf6buqaXPQe9RomwQ4BLy3mnYF/Y+iewgXAidy65XIZHQbScOAh4AvmdlLnn0PMBaYDBwDftSJLZ3Z+C/SR3hBbVKLbRJL3/e8Kun8atkW9D+KdgivAPn4id2VtOgUSQNJP7z1ZrYxyzez583stD/dr6atC95d2Y0hpHoFtUkttsmMwcCpatgW9E8KdQg+JlsnKfsBNgIzJI3wbvIMz6sISZdI2lYmX6SvqVvM7K6SbaNyq7NJXW+AzUCDUvCfMSTJjT+SZDbGSRrjwwsNvm/GeKA3vroOeoAabZNIehvwbzN7rVLbgqCUonsIAL8HrgUws+PACtJN9wlguech6f7sdT5JsyX9E5gC/FZS9gMdRZLIKOX9wDzgw2Ve5Vsp6aCkJmAasMRtaQYy2Y2HcdkNM2sFMtmNFmCD74ukkcArZnasWhcnKISaapN+zDTgd1W7AkG/pHDpCklXAV82s3lVKGsR8HdL+kq9jqQlwEtm9kAR5w+qQy22SUkbgW905U2nICil8NcjzWy/f6BTl3va6W5Zq6plVzc5Aawr2IbgLKm1NulDl5vCGQRnS+E9hCAIgqBv0BfmEIIgCII+QDiEIAiCAAiHEARBEDjhEIIgCAIgHEJQEJK+6NLP67t4XL2kT/WUXT2BpIsl/bqC/U62k3+jpAnVt6zj8wb9j3AIQVHcBtxgZl1V6KwHuuwQstgGRWBmz5nZnLMo4kaS9HWHhMpucLaEQwh6HUn3kuJJbJZ0hyt6PiFpv6SP+T71knZI2ufLNX7494EP+Je9SyQtkLQqV/YWSVM9fVLSckl7gCmS3qMU+OZJSY2ZRIT3Vg67uuiDHdh9UNIFSrwg6TOev07SdEl1ku70ujRJ+nyuLoc8PVTSBt/+S0l7lAuoI+k7kg5IelzSSK/3LOBOr/PYEpvWSrpL0nbgB5LeJ2m3X8vdki73/RZI2ijpYaVAPyvL1O9CSX+QNLNL/9Dg3KHogAyx9M8FOEpSFv0u8GnPu4AUKOY8YCgwxPPHAXs9PRXYkitnAbAqt74FmOppA+Z6eiCwG7jI1z8JrPH0c8DgzIYObL4XmAlMIslYrPb8Z4BhwC3ANz1vMLCXFMimHjjk+UuB+zw9iSRrcXXO3o96emWurLXAnHZsWut1zgLlvAV4k6enAw/lrtMRYDhJvO9vwGjfdhIYSZLfvq7othFLcUt0MYOimQHMkrTU14cAl5Ju0qskTQZOk0QDu8ppkpoopGhmk4CtSVeOOpK0NEATsF7SJlKgmfbYAXyQdDO9B7hF0iXAcTM7KWkG8E5J2fDQcJIz+1OujGuBnwCY2SHXK8p4lXRzB3gSuK7Cev7K2r6oHg78TCmUq5EcYcY2M3sRQNJh4O2k+AoDgW0kbaRHKzxncA4SDiEoGgEftxLZBUnLgOdJwYYG0L6scytnDn3mpatP5W6UAprNbEqZMmaSbvSzgG9JmmhJxLCUx0jhNS8F7iApkc4hOYrsHLeb2RlqqEoBcMjt0x6vmVkmHXCayn+f/82lVwDbzWy2n/eR3Lb/5dL58ltJDuh6IBxCPybmEIKiaQRulz+2u7AcpCfdY5ZiAswjPdEDvEyKQZxxFJgsaYCk0bTFDijlaeAiSVP8PAMlTZQ0gDR0sh34KmnYali5AszsH6RhrnFmdgTYSRoCyhxCI3CrUpwDJI2XdF5JMTuBub59AnBlu1emjTPqLGmRi+aVYzjwrKcXVFA2pJ7E54ArJH29wmOCc5BwCEHRrCANWTT5xOsKz78bmC/pcdJwUfYU3AS0+sTrEmAX8FfgIPBDYF+5k1iKfz2HNPF6AHgKuIbkaH4u6SCwH/ixmZ0oV4azh7YhoB2kEJY7ff1+kjT1Pq/LfbzxKf9ukmNqAr7m9Xmxg/NBitv9FZ8oHgtcAbzQzr4rge9J2kWbE+0U70k1ANMk3VbpccG5RYjbBUEv4q+/DjSzU35z3waMd4dVaRlbgJu6ckwQVELMIQRB7zIU2O7DSgJu7eqN3cw+0iOWBf2e6CEEQQmSPgssLsneZWYLi7AnCHqLcAhBEAQBEJPKQRAEgRMOIQiCIADCIQRBEAROOIQgCIIAgP8DKjQEgjqFH3AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "report_results('rank', scores_cold);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'YaMus': {'HybridSVDs(cs)': {'features_weight': 0.9, 'rank': 2000}}}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config_cold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### saving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(experiment_name+'_coldstart', config=config_cold, tuning=scores_cold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_cold = {}\n",
    "for label in track(data_labels):\n",
    "    models_cold = prepare_cold_start_recommender_models(label, data_models_cold, config_cold)\n",
    "    result_cold[label] = ee.run_cv_experiment(models_cold,\n",
    "                                              fold_experiment=ee.topk_test,\n",
    "                                              topk_list=topk_values,\n",
    "                                              ignore_feedback=True,\n",
    "                                              iterator=lambda x: track(x, label=f'{label} folds'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEWCAYAAACKSkfIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGZhJREFUeJzt3X+UlWXd7/H314FEQ9QYPKcEHU6giShYaD7pQZMT6dHESovJxMzCsyyPp1M9x+eUv1ha+lTHVWlLaan4i8CwlFW4XJk/SFMIDCEiE3lQ5sgpBRXxERP4nj/2ZhzGGdkDA3vger/WmjX3j+u+93ffM/sz91z3va8dmYkkqQy71bsASdKOY+hLUkEMfUkqiKEvSQUx9CWpIIa+JBXE0Jekghj62qVFxB0RcVO7ZcdFxKqIeO87bNcrIjIino+IhjbL31Xddv32rFvaXgx97er+O/BfI+JjABHRB/gp8PXMXFnD9q8CY9vMnwK82O1VSjuIoa9dWmauAi4AJkfEu4FLgWcyc0pE/FNEPB4RL0fEyoj4UUT0breL24AJbeYnALe2bRARLRFxfJv5KyJiSnV6z4iYWv3v4OWImBsRjd3/TKXaGPra5WXmz4H5wM+AicB51VXrgQuBRuAY4MQ26zb5BXBCRPSLiP7A0cCvuvDw5wB7AgOB/sD5wLqteybStutV7wKkHeQrwDPAtzLzOYDM/EOb9csiYjJwHHBtm+X/DtwLnAHsAfwSeKMLj/smlT8qQzJzETBvq5+B1A0MfRUhM/8WES8Cizcti4gPAD8APkTlbLwXMKeDzW+l0i20B5X/DLpiCvA+4M6I6Eelu+jbmemFYNWF3Tsq2Q3An6ichfcDLgGig3YPAgcC+2TmYx2sf43KH41N/uOmicz8R2ZelpmHAMcCnwTO7Kb6pS4z9FWyvYBXgNci4hDe3p8PQFbGHz8FOK2T/SwAxldv8zwK+NSmFRFxQkQMj4jdgDVUuns2dONzkLrE0FfJvg6cTeW2zBuA6Z01zMw/ZeafO1n9LeADwMvAxcDUNuveR+Vi8BoqXUv3U7mgLNVF+CEqklQOz/QlqSCGviQVxNCXpIIY+pJUkB735qzGxsZsamqqdxmStFOZP3/+i5k5YEvtelzoNzU1MW+e71SXpK6IiGdraWf3jiQVxNCXpIIY+pJUkB7Xp9+RN998k5aWFtatcxjynVWfPn0YOHAgvXu3/4wSSTvSThH6LS0t7LXXXjQ1NRHR0SCI6skyk1WrVtHS0sLgwYPrXY5UtJ2ie2fdunX079/fwN9JRQT9+/f3PzWpB9gpQh8w8Hdy/vyknmGnCX1J0rYz9HuwpqYmXnzxxW1uI0mb7BQXciVpe2m66Nf1LoHlV528wx7LM/1utnz5cj7wgQ/wpS99ieHDh3PmmWdy//33c8wxxzB06FDmzp3L6tWrOe200zj88MM5+uijWbhwIQCrVq1i7NixHHHEEZx33nm0/YCb22+/naOOOoqRI0dy3nnnsWGDn7gnqesM/e1g6dKlXHjhhSxcuJC//OUvTJ06lUceeYTvf//7fOc73+HSSy/liCOOYOHChXznO99hwoQJAFx++eUce+yx/PGPf+TUU0/lueeeA2DJkiVMnz6dRx99lAULFtDQ0MAdd9xRz6coaSdl9852MHjwYA477DAADj30UMaMGUNEcNhhh7F8+XKeffZZ7rrrLgBOOOEEVq1axSuvvMLs2bP5xS9+AcDJJ5/MvvvuC8Bvf/tb5s+fz5FHHgnA66+/zn777VeHZyZpZ2fobwe777576/Ruu+3WOr/bbruxfv16evV6+2HfdEtjR7c2ZiZnn3023/3ud7dTxZJKYfdOHYwePbq1e+ahhx6isbGRfv36bbb83nvv5aWXXgJgzJgxzJgxg7///e8ArF69mmefrWkUVUnajGf6dXDZZZdxzjnncPjhh7Pnnntyyy23AHDppZfS3NzMBz/4QY477jgOOOAAAIYNG8YVV1zB2LFj2bhxI7179+a6667jwAMPrOfTkLQTirZ3iPQEo0aNyvYforJkyRIOOeSQOlWk7uLPUT3RrnLLZkTMz8xRW2pn944kFcTuHalAu8rZrbrOM31JKoihL0kFMfQlqSCGviQVZKe8kNvdF6FquaDUt29f1q5d2zo/ZcoU5s2bx7XXXtvpNscffzzf//73GTXqne+imjdvHrfeeis/+tGP3rauqamJefPm0djYyJVXXsnUqVNpaGhgt91244YbbuDee+/ljTfe2OzdugsWLKC5uZklS5bQ1NTEXnvtBcCGDRv41Kc+xcUXX7zZu4bby0zGjBnD3XffTb9+/d6x9rYWLVrED37wA6ZMmVLzNpJ2LM/062z9+vWMGjWqw8Bv67HHHuNXv/oVTzzxBAsXLuT+++9n0KBBNDc3M3369M3aTps2jc997nOt8w8++CCLFi1i7ty5LFu2jIkTJ77jY82aNYsRI0Z0KfABDjvsMFpaWloHipPU8xj62+jVV19l8ODBvPnmmwCsWbOGpqam1vnbb7+dj3zkIwwfPpy5c+cClXfkTpw4kbFjxzJhwgQeeughTjnlFKDz4ZVXrlxJY2Nj6xl6Y2Mj73vf+zj44IPZZ599mDNnTmtNd955J+PHj39brX379uX666/n7rvvZvXq1axcuZLRo0czcuRIhg8fzu9+9zsA7rjjDsaNG9e63a233srhhx/OiBEjOOusswD4+c9/zvDhwxkxYgSjR49ubfuJT3yCadOmdc/BldTtDP0avf7664wcObL165JLLgFgr7324vjjj+fXv650OU2bNo1Pf/rT9O7dG4DXXnuN3//+9/zkJz/hi1/8Yuv+5s+fzz333MPUqVM3e5zOhlceO3YsK1as4KCDDuL888/n4Ycfbt2mubm5NWgff/xx+vfvz9ChQzt8Hv369WPw4ME8/fTTTJ06lY9//OMsWLCAJ598kpEjRwLw6KOP8qEPfQiAxYsXc+WVV/LAAw/w5JNP8sMf/hCASZMmcd999/Hkk08yc+bM1v2PGjWq9Y+HpJ7H0K/RHnvswYIFC1q/Jk2a1LruS1/6EjfffDMAN998M+ecc07ruubmZqAyyNqaNWt4+eWXATj11FPZY4893vY4s2fP5vOf/zyw+fDKffv2Zf78+UyePJkBAwbw2c9+trXvfPz48cyYMYONGzcybdq01sfszKb/Ho488khuvvlmLrvsMhYtWtTa97969erW6QceeIDTTz+dxsZGAN7znvcAcMwxx/CFL3yBn/70p5t9oMt+++3H888/v8XjKak+DP1ucMwxx7B8+XIefvhhNmzYwPDhw1vXtR8qedP8u9/97k7319HwygANDQ0cf/zxXH755Vx77bWtY/IPGjSIpqYmHn74Ye666y4+85nPdLrvV199leXLl3PQQQcxevRoZs+ezf77789ZZ53FrbfeCkCvXr3YuHEjUPkD0VE9119/PVdccQUrVqxg5MiRrFq1CoB169Z1+MdMUs9g6HeTCRMm0NzcvNlZPtB6kfWRRx5h7733Zu+9937H/XQ2vPJTTz3F008/3dpuwYIFm42y2dzczNe+9jXe//73M3DgwA73vXbtWs4//3xOO+009t13X5599ln2228/vvzlL3PuuefyxBNPAHDwwQezbNkyoDKs85133tka6qtXrwbgmWee4cMf/jCTJk2isbGRFStWAPDXv/51sz96knqWnfKWzZ44ZseZZ57Jt7/97bd1rey777585CMfYc2aNdx0001b3E9nwyuvXbuWCy64gJdffplevXoxZMgQJk+e3LrdGWecwYUXXsiPf/zjt+3zox/9KJnJxo0b+eQnP8nFF18MVMby/973vkfv3r3p27dv65n+ySefzEMPPcSQIUM49NBD+da3vsVxxx1HQ0MDRxxxBFOmTOGb3/wmTz/9dOvtnSNGjAAqdwqdfHLP+/lIqnBo5W4yY8YM7rnnHm677bZ6l7LNVq5cyYQJE/jNb37Tpe3eeOMNjjvuOB555JEOPx1sZ/g5lsIB196yqxyLWodW3inP9HuaCy64gHvvvZdZs2bVu5Ru8d73vpcvf/nLrFmzpkv36j/33HNcddVVHQa+pJ7BV2c36KhLZWf3TheDOzN06NBObxWV1DPUdCE3Ik6MiKciYmlEXNTB+t0jYnp1/ZyIaGq3/oCIWBsR39jaQntaN5S6xp+f1DNsMfQjogG4DjgJGAY0R8Swds3OBV7KzCHANcDV7dZfA9y7tUX26dOHVatWGRw7qcxk1apV9OnTp96lSMWrpXvnKGBpZi4DiIhpwDjgz23ajAMuq07PAK6NiMjMjIjTgGXAa1tb5MCBA2lpaeGFF17Y2l2ozvr06dPpraSSdpxaQn9/YEWb+Rbgw521ycz1EfEK0D8iXgf+F/AxoNOunYiYCEwEWm9RbKt3794MHjy4hlIlSe+klj79jt4e2r6fpbM2lwPXZObaDta/1TBzcmaOysxRAwYMqKEkSdLWqOVMvwUY1GZ+INB+cJVNbVoiohewN7Cayn8Ep0fEvwL7ABsjYl1mdj4IvSRpu6kl9P8ADI2IwcD/BcYDn2vXZiZwNvAYcDrwQFauuv7nTQ0i4jJgrYEvSfWzxdCv9tF/FbgPaABuyszFETEJmJeZM4EbgdsiYimVM/y3D+Yu1dmu8s5LaVvU9OaszJwFzGq37JI20+uAM7awj8u2oj5JUjdylE1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgpi6EtSQQx9SSqIoS9JBTH0Jakghr4kFcTQl6SCGPqSVBBDX5IKYuhLUkEMfUkqiKEvSQUx9CWpIIa+JBXE0Jekghj6klQQQ1+SCmLoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgpi6EtSQQx9SSqIoS9JBTH0Jakghr4kFaSm0I+IEyPiqYhYGhEXdbB+94iYXl0/JyKaqsuPiogF1a8nI+KT3Vu+JKkrthj6EdEAXAecBAwDmiNiWLtm5wIvZeYQ4Brg6uryPwGjMnMkcCJwQ0T06q7iJUldU8uZ/lHA0sxclpn/AKYB49q1GQfcUp2eAYyJiMjMf8/M9dXlfYDsjqIlSVunltDfH1jRZr6luqzDNtWQfwXoDxARH46IxcAi4L+1+SPQKiImRsS8iJj3wgsvdP1ZSJJqUkvoRwfL2p+xd9omM+dk5qHAkcC/RESftzXMnJyZozJz1IABA2ooSZK0NWoJ/RZgUJv5gcDznbWp9tnvDaxu2yAzlwCvAcO3tlhJ0rapJfT/AAyNiMER8S5gPDCzXZuZwNnV6dOBBzIzq9v0AoiIA4GDgeXdUrkkqcu2eCdNZq6PiK8C9wENwE2ZuTgiJgHzMnMmcCNwW0QspXKGP766+bHARRHxJrAROD8zX9weT0SStGU13T6ZmbOAWe2WXdJmeh1wRgfb3Qbcto01SpK6ie/IlaSCGPqSVBBDX5IKYuhLUkEMfUkqiKEvSQUx9CWpIIa+JBXE0Jekghj6klQQQ1+SCmLoS1JBDH1JKogfUr6La7ro1/UuAYDlV51c7xIk4Zm+JBXF0Jekghj6klQQQ1+SCmLoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgpi6EtSQQx9SSqIoS9JBTH0Jakghr4kFcTQl6SCGPqSVBBDX5IKYuhLUkFqCv2IODEinoqIpRFxUQfrd4+I6dX1cyKiqbr8YxExPyIWVb+f0L3lS5K6YouhHxENwHXAScAwoDkihrVrdi7wUmYOAa4Brq4ufxH4RGYeBpwN3NZdhUuSuq6WM/2jgKWZuSwz/wFMA8a1azMOuKU6PQMYExGRmX/MzOeryxcDfSJi9+4oXJLUdbWE/v7AijbzLdVlHbbJzPXAK0D/dm0+DfwxM9/YulIlSduqVw1tooNl2ZU2EXEolS6fsR0+QMREYCLAAQccUENJkqStUcuZfgswqM38QOD5ztpERC9gb2B1dX4g8EtgQmY+09EDZObkzByVmaMGDBjQtWcgSapZLaH/B2BoRAyOiHcB44GZ7drMpHKhFuB04IHMzIjYB/g18C+Z+Wh3FS1J2jpbDP1qH/1XgfuAJcCdmbk4IiZFxKnVZjcC/SNiKfA/gU23dX4VGAJcHBELql/7dfuzkCTVpJY+fTJzFjCr3bJL2kyvA87oYLsrgCu2sUZJUjfxHbmSVBBDX5IKYuhLUkEMfUkqiKEvSQUx9CWpIIa+JBXE0Jekghj6klQQQ1+SCmLoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgpi6EtSQQx9SSqIoS9JBTH0Jakghr4kFcTQl6SCGPqSVJBe9S5ge2i66Nf1LgGA5VedXO8SJGkznulLUkEMfUkqiKEvSQUx9CWpIIa+JBXE0Jekghj6klQQQ1+SCmLoS1JBDH1JKkhNoR8RJ0bEUxGxNCIu6mD97hExvbp+TkQ0VZf3j4gHI2JtRFzbvaVLkrpqi6EfEQ3AdcBJwDCgOSKGtWt2LvBSZg4BrgGuri5fB1wMfKPbKpYkbbVazvSPApZm5rLM/AcwDRjXrs044Jbq9AxgTEREZr6WmY9QCX9JUp3VEvr7AyvazLdUl3XYJjPXA68A/WstIiImRsS8iJj3wgsv1LqZJKmLagn96GBZbkWbTmXm5MwclZmjBgwYUOtmkqQuqiX0W4BBbeYHAs931iYiegF7A6u7o0BJUvepJfT/AAyNiMER8S5gPDCzXZuZwNnV6dOBBzKz5jN9SdKOscVPzsrM9RHxVeA+oAG4KTMXR8QkYF5mzgRuBG6LiKVUzvDHb9o+IpYD/YB3RcRpwNjM/HP3PxVJ0pbU9HGJmTkLmNVu2SVtptcBZ3SybdM21CdJ6ka+I1eSCmLoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgpi6EtSQQx9SSqIoS9JBTH0Jakghr4kFcTQl6SCGPqSVBBDX5IKYuhLUkEMfUkqiKEvSQUx9CWpIIa+JBXE0Jekghj6klQQQ1+SCmLoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgpi6EtSQQx9SSqIoS9JBakp9CPixIh4KiKWRsRFHazfPSKmV9fPiYimNuv+pbr8qYj4ePeVLknqqi2GfkQ0ANcBJwHDgOaIGNau2bnAS5k5BLgGuLq67TBgPHAocCLwk+r+JEl1UMuZ/lHA0sxclpn/AKYB49q1GQfcUp2eAYyJiKgun5aZb2TmvwFLq/uTJNVBrxra7A+saDPfAny4szaZuT4iXgH6V5c/3m7b/ds/QERMBCZWZ9dGxFM1Vb99NQIvbssO4upuqqT+PBZv2aZjsQsdB/BYtNUTjsWBtTSqJfSjg2VZY5tatiUzJwOTa6hlh4mIeZk5qt519AQei7d4LN7isXjLznQsauneaQEGtZkfCDzfWZuI6AXsDayucVtJ0g5SS+j/ARgaEYMj4l1ULszObNdmJnB2dfp04IHMzOry8dW7ewYDQ4G53VO6JKmrtti9U+2j/ypwH9AA3JSZiyNiEjAvM2cCNwK3RcRSKmf446vbLo6IO4E/A+uBr2Tmhu30XLpbj+puqjOPxVs8Fm/xWLxlpzkWUTkhlySVwHfkSlJBDH1JKoih305E3BQRf4+IP9W7lnqKiD4RMTcinoyIxRFxeb1r2tE6+l2IiPdExG8i4unq933rWeOOEBGDIuLBiFhS/V24sLq8xGPR4euieqPLnOqxmF696aVHMvTfbgqVISNK9wZwQmaOAEYCJ0bE0XWuaUebwtt/Fy4CfpuZQ4HfVud3deuBr2fmIcDRwFeqQ6yUeCw6e11cDVxTPRYvURmapkcy9NvJzNlU7kAqWlasrc72rn4VddW/k9+FtkOO3AKctkOLqoPMXJmZT1SnXwWWUHlnfYnHorPXxQlUhqCBHn4sDH11KiIaImIB8HfgN5k5p9419QD/ITNXQiUMgf3qXM8OVR1B9whgDoUei/avC+AZ4OXMXF9t0uFwMz2Foa9OZeaGzBxJ5Z3UR0XE8HrXpPqJiL7AXcD/yMw19a6nXtq/LoBDOmq2Y6uqnaGvLcrMl4GH8FoHwN8i4r0A1e9/r3M9O0RE9KYS+Hdk5i+qi4s8Fpu0eV0cDexTHYIGevhwM4a+OhQRAyJin+r0HsB/Af5S36p6hLZDjpwN3FPHWnaI6jDpNwJLMvP/tFlV4rHo6HWxBHiQyhA00MOPhe/IbScifgYcT2Wo1L8Bl2bmjXUtqg4i4nAqF6QaqJwc3JmZk+pb1Y7V0e8CcDdwJ3AA8BxwRmbu0hf+I+JY4HfAImBjdfH/ptKvX9qx6PB1ERH/icpnjbwH+CPw+cx8o36Vds7Ql6SC2L0jSQUx9CWpIIa+JBXE0Jekghj6klQQQ1/Fioh9IuL8etch7UiGvkq2D2Doqyhb/IxcaRd2FfD+6uBZv6kuO4nKuClXZOb0iDgemASsAg4GZgPnZ+bGtjuKiC8ApwJ7Au8HfpmZ/7wjnoTUFZ7pq2QXAc9UB896nMr46COovLX+e5vGlaEyqNbXgcOoBPqnOtnfSOCz1XafjYhB27F2aasY+lLFscDPqiMo/g14GDiyum5uZi7LzA3Az6ptO/LbzHwlM9cBfwYO3O5VS11k6EsV8Q7r2o9VkhHxyYhYUP0aVV3edqyVDdh9qh7I0FfJXgX2qk7PptIl0xARA4DRwNzquqOqn4G6G5Xum0cy85eZObL6NW/Hly5tHUNfxcrMVcCj1Q8+/ydgIfAk8ADwz5n5/6pNH6Ny0fdPwL8Bv6xDuVK3cJRN6R1U7975RmaeUu9apO7gmb4kFcQzfUkqiGf6klQQQ1+SCmLoS1JBDH1JKoihL0kF+f+PwZyXgnFJIAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "report_results('topn', result_cold, target_metric);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEWCAYAAACKSkfIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGetJREFUeJzt3Xt0lfW95/H3lyQCylUCZ1TAZAQvXEON6BEHKMyhOFhAC5VohVoVZ1Edp6enHTpWBRZandrjagtdSpdyUxoo3lgVDktFQKiCQQOISAkcLhmYCglXjyCB7/yxH+Jms0N2IGQn/D6vtbLyXH7Ps7/7SfLZT377eX7b3B0REQlDo3QXICIidUehLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPpyQTOzV8zspYRl/cyszMwuO8N2mWbmZrbLzDLill8UbVtxPusWOV8U+nKh+x/AfzOzfwIwsybAH4GfuvvuFLY/BAyKm78N2FvrVYrUEYW+XNDcvQx4GJhmZpcATwBb3H2Gmf2jmX1oZvvNbLeZ/c7MshJ2MRsYHTc/GpgV38DMSs2sf9z8ZDObEU1fbGZzov8O9pvZajPLrv1nKpIahb5c8Nz9z8Aa4E/AWODBaFUF8AiQDfQBBsetO+k1YICZtTCzNsBNwF9q8PD3AhcD7YE2wDjgyNk9E5Fzl5nuAkTqyI+BLcCj7r4DwN0/ilu/1cymAf2AKXHL/wNYBIwEmgKvA0dr8LjHiL2odHL39UDRWT8DkVqg0JcguPvfzWwvsOHkMjO7FvgNcD2xs/FMYFWSzWcR6xZqSuw/g5qYAVwOzDOzFsS6i37p7nojWNJC3TsSsheAT4mdhbcAHgcsSbv3gCuBVu7+QZL1XxJ70TjpP52ccPev3X2Cu18H3ALcDtxdS/WL1JhCX0LWHDgAfGlm13F6fz4AHht//DZgeBX7KQZGRZd59gbuOLnCzAaYWTczawQcJNbdc7wWn4NIjSj0JWQ/BcYQuyzzBWBuVQ3d/VN3/6yK1Y8C1wL7gceAOXHrLif2ZvBBYl1L7xB7Q1kkLUwfoiIiEg6d6YuIBEShLyISEIW+iEhAFPoiIgGpdzdnZWdne05OTrrLEBFpUNasWbPX3dtW167ehX5OTg5FRbpTXUSkJsxseyrt1L0jIhIQhb6ISEAU+iIiAal3ffrJHDt2jNLSUo4c0TDkDVWTJk1o3749WVmJn1EiInWpQYR+aWkpzZs3JycnB7NkgyBKfebulJWVUVpaSm5ubrrLEQlag+jeOXLkCG3atFHgN1BmRps2bfSfmkg90CBCH1DgN3D6+YnUDw0m9EVE5Nwp9OuxnJwc9u7de85tREROahBv5IqInC85499Kdwlse3pInT2WzvRr2bZt27j22mu5//776datG3fffTfvvPMOffr0oXPnzqxevZry8nKGDx9Ojx49uOmmm1i3bh0AZWVlDBo0iF69evHggw8S/wE3L7/8Mr179yYvL48HH3yQ48f1iXsiUnMK/fOgpKSERx55hHXr1vH5558zZ84cVqxYwbPPPstTTz3FE088Qa9evVi3bh1PPfUUo0ePBmDixInccsstfPLJJwwdOpQdO3YAsHHjRubOncvKlSspLi4mIyODV155JZ1PUUQaKHXvnAe5ubl0794dgK5duzJw4EDMjO7du7Nt2za2b9/Oq6++CsCAAQMoKyvjwIEDLF++nNdeew2AIUOG0Lp1awDeffdd1qxZww033ADAV199Rbt27dLwzESkoVPonweNGzeunG7UqFHlfKNGjaioqCAz8/TDfvKSxmSXNro7Y8aM4Ve/+tV5qlhEQqHunTTo27dvZffM0qVLyc7OpkWLFqcsX7RoEfv27QNg4MCBzJ8/ny+++AKA8vJytm9PaRRVEZFT6Ew/DSZMmMC9995Ljx49uPjii5k5cyYATzzxBAUFBXzrW9+iX79+dOzYEYAuXbowefJkBg0axIkTJ8jKymLq1KlceeWV6XwaItIAWfwVIvVBfn6+J36IysaNG7nuuuvSVJHUFv0cpT66UC7ZNLM17p5fXTt174iIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISkAZ5nX5tX2KVyuVSzZo14/Dhw5XzM2bMoKioiClTplS5Tf/+/Xn22WfJzz/zVVRFRUXMmjWL3/3ud6ety8nJoaioiOzsbJ588knmzJlDRkYGjRo14oUXXmDRokUcPXr0lLt1i4uLKSgoYOPGjeTk5NC8eXMAjh8/zh133MFjjz12yl3DidydgQMH8sYbb9CiRYsz1h5v/fr1/OY3v2HGjBkpbyMidUtn+mlWUVFBfn5+0sCP98EHH/CXv/yFjz/+mHXr1vHOO+/QoUMHCgoKmDt37iltCwsLueuuuyrn33vvPdavX8/q1avZunUrY8eOPeNjLVy4kJ49e9Yo8AG6d+9OaWlp5UBxIlL/KPTP0aFDh8jNzeXYsWMAHDx4kJycnMr5l19+mZtvvplu3bqxevVqIHZH7tixYxk0aBCjR49m6dKl3HbbbUDVwyvv3r2b7OzsyjP07OxsLr/8cq655hpatWrFqlWrKmuaN28eo0aNOq3WZs2a8fzzz/PGG29QXl7O7t276du3L3l5eXTr1o33338fgFdeeYVhw4ZVbjdr1ix69OhBz549ueeeewD485//TLdu3ejZsyd9+/atbPvd736XwsLC2jm4IlLrFPop+uqrr8jLy6v8evzxxwFo3rw5/fv35623Yl1OhYWFfO973yMrKwuAL7/8kr/+9a/84Q9/4Ec/+lHl/tasWcObb77JnDlzTnmcqoZXHjRoEDt37uTqq69m3LhxLFu2rHKbgoKCyqD98MMPadOmDZ07d076PFq0aEFubi6bN29mzpw5fOc736G4uJi1a9eSl5cHwMqVK7n++usB2LBhA08++SRLlixh7dq1/Pa3vwVg0qRJLF68mLVr17JgwYLK/efn51e+eIhI/aPQT1HTpk0pLi6u/Jo0aVLluvvvv5/p06cDMH36dO69997KdQUFBUBskLWDBw+yf/9+AIYOHUrTpk1Pe5zly5fzgx/8ADh1eOVmzZqxZs0apk2bRtu2bbnzzjsr+85HjRrF/PnzOXHiBIWFhZWPWZWT/z3ccMMNTJ8+nQkTJrB+/frKvv/y8vLK6SVLljBixAiys7MBuPTSSwHo06cPP/zhD/njH/94yge6tGvXjl27dlV7PEUkPVIKfTMbbGabzKzEzMYnWd/YzOZG61eZWU60PMvMZprZejPbaGa/qN3y64c+ffqwbds2li1bxvHjx+nWrVvlusShkk/OX3LJJVXuL9nwygAZGRn079+fiRMnMmXKlMox+Tt06EBOTg7Lli3j1Vdf5fvf/36V+z506BDbtm3j6quvpm/fvixfvpwrrriCe+65h1mzZgGQmZnJiRMngNgLRLJ6nn/+eSZPnszOnTvJy8ujrKwMgCNHjiR9MROR+qHa0DezDGAqcCvQBSgwsy4Jze4D9rl7J+A54Jlo+Uigsbt3B64HHjz5gnChGT16NAUFBaec5QOVb7KuWLGCli1b0rJlyzPup6rhlTdt2sTmzZsr2xUXF58yymZBQQE/+clPuOqqq2jfvn3SfR8+fJhx48YxfPhwWrduzfbt22nXrh0PPPAA9913Hx9//DEA11xzDVu3bgViwzrPmzevMtTLy8sB2LJlCzfeeCOTJk0iOzubnTt3AvC3v/3tlBc9EalfUrlkszdQ4u5bAcysEBgGfBbXZhgwIZqeD0yx2OmhA5eYWSbQFPgaOHiuRdflhwin6u677+aXv/zlaV0rrVu35uabb+bgwYO89NJL1e6nquGVDx8+zMMPP8z+/fvJzMykU6dOTJs2rXK7kSNH8sgjj/D73//+tH1++9vfxt05ceIEt99+O4899hgQG8v/17/+NVlZWTRr1qzyTH/IkCEsXbqUTp060bVrVx599FH69etHRkYGvXr1YsaMGfzsZz9j8+bNlZd39uzZE4hdKTRkSP37+YhITLVDK5vZCGCwu98fzd8D3OjuD8W1+TRqUxrNbwFuBA4As4GBwMXAT9x9WsJDYGZjgbEAHTt2vD7xA0IawpC88+fP580332T27NnpLuWc7d69m9GjR/P222/XaLujR4/Sr18/VqxYkfTTwRrCz1HCE9rQyqmc6SfrYE58paiqTW/gOHA50Bp438zeOflfQ2XD2AvBNIiNp59CTfXKww8/zKJFi1i4cGG6S6kVl112GQ888AAHDx6s0bX6O3bs4Omnn04a+CJSP6Ty11kKdIibbw8kXp5xsk1p1JXTEigH7gL+zd2PAV+Y2UogH9jKBSRZl0pDd6Y3g6vSuXPnKi8VFZH6IZWrdz4COptZrpldBIwCFiS0WQCMiaZHAEs81m+0AxhgMZcANwGfn02h9e0TvqRm9PMTqR+qDX13rwAeAhYDG4F57r7BzCaZ2dCo2YtAGzMrAf4ZOHlZ51SgGfApsReP6e6+rqZFNmnShLKyMgVHA+XulJWV0aRJk3SXIhK8lDpf3X0hsDBh2eNx00eIXZ6ZuN3hZMtrqn379pSWlrJnz55z3ZWkSZMmTaq8lFRE6k6DeMctKyuL3NzcdJchItLgaRgGEZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAZKa7ABGpeznj30p3CWx7eki6SwiSzvRFRAKi0BcRCUhKoW9mg81sk5mVmNn4JOsbm9ncaP0qM8uJW9fDzD4wsw1mtt7MmtRe+SIiUhPVhr6ZZQBTgVuBLkCBmXVJaHYfsM/dOwHPAc9E22YCLwP/3d27Av2BY7VWvYiI1EgqZ/q9gRJ33+ruXwOFwLCENsOAmdH0fGCgmRkwCFjn7msB3L3M3Y/XTukiIlJTqYT+FcDOuPnSaFnSNu5eARwA2gBXA25mi83sYzP7ebIHMLOxZlZkZkV79uyp6XMQEZEUpRL6lmSZp9gmE7gFuDv6fruZDTytofs0d8939/y2bdumUJKIiJyNVEK/FOgQN98e2FVVm6gfvyVQHi1f5u573f0/gIXAt861aBEROTuphP5HQGczyzWzi4BRwIKENguAMdH0CGCJuzuwGOhhZhdHLwb9gM9qp3QREampau/IdfcKM3uIWIBnAC+5+wYzmwQUufsC4EVgtpmVEDvDHxVtu8/M/pXYC4cDC909/bcCiogEKqVhGNx9IbGumfhlj8dNHwFGVrHty8Qu2xQRkTTTHbkiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFJKfTNbLCZbTKzEjMbn2R9YzObG61fZWY5Ces7mtlhM/uX2ilbRETORrWhb2YZwFTgVqALUGBmXRKa3Qfsc/dOwHPAMwnrnwMWnXu5IiJyLlI50+8NlLj7Vnf/GigEhiW0GQbMjKbnAwPNzADMbDiwFdhQOyWLiMjZSiX0rwB2xs2XRsuStnH3CuAA0MbMLgH+FzDxTA9gZmPNrMjMivbs2ZNq7SIiUkOphL4lWeYptpkIPOfuh8/0AO4+zd3z3T2/bdu2KZQkIiJnIzOFNqVAh7j59sCuKtqUmlkm0BIoB24ERpjZ/wFaASfM7Ii7TznnykVEpMZSCf2PgM5mlgv8X2AUcFdCmwXAGOADYASwxN0d+C8nG5jZBOCwAl9EJH2qDX13rzCzh4DFQAbwkrtvMLNJQJG7LwBeBGabWQmxM/xR57NoERE5O6mc6ePuC4GFCcsej5s+AoysZh8TzqI+ERGpRbojV0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkICndkStyIcgZ/1a6S2Db00PSXYIETmf6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEpCUQt/MBpvZJjMrMbPxSdY3NrO50fpVZpYTLf8nM1tjZuuj7wNqt3wREamJakPfzDKAqcCtQBegwMy6JDS7D9jn7p2A54BnouV7ge+6e3dgDDC7tgoXEZGaS+VMvzdQ4u5b3f1roBAYltBmGDAzmp4PDDQzc/dP3H1XtHwD0MTMGtdG4SIiUnOphP4VwM64+dJoWdI27l4BHADaJLT5HvCJux89u1JFRORcZabQxpIs85q0MbOuxLp8BiV9ALOxwFiAjh07plCSiIicjVTO9EuBDnHz7YFdVbUxs0ygJVAezbcHXgdGu/uWZA/g7tPcPd/d89u2bVuzZyAiIilLJfQ/AjqbWa6ZXQSMAhYktFlA7I1agBHAEnd3M2sFvAX8wt1X1lbRIiJydqoN/aiP/iFgMbARmOfuG8xskpkNjZq9CLQxsxLgn4GTl3U+BHQCHjOz4uirXa0/CxERSUkqffq4+0JgYcKyx+OmjwAjk2w3GZh8jjWKiEgt0R25IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEpCUxt6Rhitn/FvpLgGAbU8PSXcJIoLO9EVEgqLQFxEJiEJfRCQgCn0RkYAo9EVEAqLQFxEJiEJfRCQgCn0RkYAo9EVEAqLQFxEJiEJfRCQgCn0RkYAo9EVEAqLQFxEJyAU5tLKGExYRSU5n+iIiAVHoi4gERKEvIhIQhb6ISEAU+iIiAVHoi4gERKEvIhIQhb6ISEAU+iIiAVHoi4gERKEvIhKQlELfzAab2SYzKzGz8UnWNzazudH6VWaWE7fuF9HyTWb2ndorXUREaqra0DezDGAqcCvQBSgwsy4Jze4D9rl7J+A54Jlo2y7AKKArMBj4Q7Q/ERFJg1TO9HsDJe6+1d2/BgqBYQlthgEzo+n5wEAzs2h5obsfdfd/B0qi/YmISBqkMrTyFcDOuPlS4Maq2rh7hZkdANpEyz9M2PaKxAcws7HA2Gj2sJltSqn68ysb2HsuO7BnaqmS9NOx+MY5HYsL6DiAjkW8+nAsrkylUSqhb0mWeYptUtkWd58GTEuhljpjZkXunp/uOuoDHYtv6Fh8Q8fiGw3pWKTSvVMKdIibbw/sqqqNmWUCLYHyFLcVEZE6kkrofwR0NrNcM7uI2BuzCxLaLADGRNMjgCXu7tHyUdHVPblAZ2B17ZQuIiI1VW33TtRH/xCwGMgAXnL3DWY2CShy9wXAi8BsMyshdoY/Ktp2g5nNAz4DKoAfu/vx8/Rcalu96m5KMx2Lb+hYfEPH4hsN5lhY7IRcRERCoDtyRUQCotAXEQmIQj+Bmb1kZl+Y2afpriWdzKyJma02s7VmtsHMJqa7prqW7HfBzC41s7fNbHP0vXU6a6wLZtbBzN4zs43R78Ij0fIQj0XSv4voQpdV0bGYG130Ui8p9E83g9iQEaE7Cgxw955AHjDYzG5Kc011bQan/y6MB951987Au9H8ha4C+Km7XwfcBPw4GmIlxGNR1d/FM8Bz0bHYR2xomnpJoZ/A3ZcTuwIpaB5zOJrNir6Cete/it+F+CFHZgLD67SoNHD33e7+cTR9CNhI7M76EI9FVX8XA4gNQQP1/Fgo9KVKZpZhZsXAF8Db7r4q3TXVA//g7rshFoZAuzTXU6eiEXR7AasI9Fgk/l0AW4D97l4RNUk63Ex9odCXKrn7cXfPI3YndW8z65bumiR9zKwZ8CrwP939YLrrSZfEvwvgumTN6raq1Cn0pVruvh9Yit7rAPi7mV0GEH3/Is311AkzyyIW+K+4+2vR4iCPxUlxfxc3Aa2iIWigng83o9CXpMysrZm1iqabAv8V+Dy9VdUL8UOOjAHeTGMtdSIaJv1FYKO7/2vcqhCPRbK/i43Ae8SGoIF6fix0R24CM/sT0J/YUKl/B55w9xfTWlQamFkPYm9IZRA7OZjn7pPSW1XdSva7ALwBzAM6AjuAke5+Qb/xb2a3AO8D64ET0eL/TaxfP7RjkfTvwsz+M7HPGrkU+AT4gbsfTV+lVVPoi4gERN07IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUehLsMyslZmNS3cdInVJoS8hawUo9CUo1X5GrsgF7GngqmjwrLejZbcSGzdlsrvPNbP+wCSgDLgGWA6Mc/cT8Tsysx8CQ4GLgauA193953XxJERqQmf6ErLxwJZo8KwPiY2P3pPYrfW/PjmuDLFBtX4KdCcW6HdUsb884M6o3Z1m1uE81i5yVhT6IjG3AH+KRlD8O7AMuCFat9rdt7r7ceBPUdtk3nX3A+5+BPgMuPK8Vy1SQwp9kRg7w7rEsUrczG43s+LoKz9aHj/WynHUfSr1kEJfQnYIaB5NLyfWJZNhZm2BvsDqaF3v6DNQGxHrvlnh7q+7e170VVT3pYucHYW+BMvdy4CV0Qef/yOwDlgLLAF+7u7/L2r6AbE3fT8F/h14PQ3litQKjbIpcgbR1Tv/4u63pbsWkdqgM30RkYDoTF9EJCA60xcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCcj/B6MarCXLjjvaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "report_results('topn', result_cold, 'coverage');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### saving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(experiment_name+'_coldstart', cv=result_cold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}