FactorizationMachines_YaMus.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: MKL_NUM_THREADS=24\n",
      "env: OMP_NUM_THREADS=24\n"
     ]
    }
   ],
   "source": [
    "%env MKL_NUM_THREADS=24\n",
    "%env OMP_NUM_THREADS=24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import pandas as pd\n",
    "from ipypb import track\n",
    "\n",
    "from polara.evaluation import evaluation_engine as ee\n",
    "from polara.evaluation.pipelines import random_grid, find_optimal_config\n",
    "from polara.recommender.coldstart.models import ItemColdStartEvaluationMixin\n",
    "from polara.recommender.external.turi.turiwrapper import (TuriFactorizationRecommender,\n",
    "                                                          ColdStartRecommendationsMixin)\n",
    "\n",
    "from data_preprocessing import (get_yahoo_music_data,\n",
    "                                get_similarity_data,\n",
    "                                prepare_data_model,\n",
    "                                prepare_cold_start_data_model)\n",
    "from utils import (report_results, save_results,\n",
    "                   apply_config, print_data_stats,\n",
    "                   save_training_time, save_cv_training_time)\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_name = 'fm'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_labels = ['YaMus']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# according to https://apple.github.io/turicreate/docs/api/generated/turicreate.recommender.ranking_factorization_recommender.RankingFactorizationRecommender.html\n",
    "init_config = dict(with_data_feedback = False, # implicit case\n",
    "                   ranking_optimization = True,\n",
    "                   solver = 'adagrad',\n",
    "                   sgd_step_size = 0, # let Turi autotune it\n",
    "                   seed = seed,\n",
    "                   max_iterations = 25,\n",
    "                   other_tc_params = {}\n",
    "                   )\n",
    "fm_init_config = dict.fromkeys(data_labels, {'FM': init_config, # standard scenario\n",
    "                                             'FM(cs)': init_config}) # cold start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "          'regularization': [1e-10, 3e-10, 1e-9, 3e-9, 1e-8, 3e-8, 1e-7, 3e-7, 1e-6, 3e-6],\n",
    "          'linear_regularization': [1e-10, 3e-10, 1e-9, 3e-9, 1e-8, 3e-8, 1e-7, 3e-7, 1e-6, 3e-6],\n",
    "          'rank': [100] # for initial tuning (exploration)\n",
    "         }\n",
    "\n",
    "if init_config['solver'] == 'adagrad':\n",
    "    params.update({\n",
    "                   'adagrad_momentum_weighting': [0.9, 0.95, 0.99]\n",
    "                  })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranks_grid = [1, 10, 100, 500, 1000, 2000, 3000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "fm_ranks = {'YaMus': ranks_grid}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "topk_values = [1, 3, 10, 20, 30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_metric = 'mrr'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dict = dict.fromkeys(data_labels)\n",
    "meta_dict = dict.fromkeys(data_labels)\n",
    "similarities = dict.fromkeys(data_labels)\n",
    "sim_indices = dict.fromkeys(data_labels)\n",
    "feature_idx = dict.fromkeys(data_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = [data_dict, similarities, sim_indices, meta_dict]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Yahoo Music"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "lbl = 'YaMus'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dict[lbl], meta_dict[lbl] = get_yahoo_music_data('/data/recsys/yahoo_music/yamus_train0_rating5.gz',\n",
    "                                                      meta_path='/data/recsys/yahoo_music/yamus_attrs.gz',\n",
    "                                                      implicit=True,\n",
    "                                                      pcore=5,\n",
    "                                                      filter_data={'genreid': [0]}, # filter unknown genre\n",
    "                                                      filter_no_meta=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>albumid</th>\n",
       "      <th>artistid</th>\n",
       "      <th>genreid</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>songid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[12070]</td>\n",
       "      <td>[8490]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[19512]</td>\n",
       "      <td>[7975]</td>\n",
       "      <td>[134]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[18953]</td>\n",
       "      <td>[3492]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[695]</td>\n",
       "      <td>[2653]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[243]</td>\n",
       "      <td>[2282]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        albumid artistid genreid\n",
       "songid                          \n",
       "0       [12070]   [8490]      []\n",
       "1       [19512]   [7975]   [134]\n",
       "2       [18953]   [3492]      []\n",
       "3         [695]   [2653]      []\n",
       "4         [243]   [2282]      []"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_dict[lbl].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(meta_dict[lbl].applymap(len).sum(axis=1)==0).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "similarities[lbl], sim_indices[lbl], feature_idx[lbl] = get_similarity_data(meta_dict[lbl])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# conform with Turi input requirement for non-existent features\n",
    "meta_dict[lbl].loc[:, 'genreid'] = meta_dict[lbl]['genreid'].apply(lambda x: {i:1 for i in x} if x else {})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>albumid</th>\n",
       "      <th>artistid</th>\n",
       "      <th>genreid</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>songid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[12070]</td>\n",
       "      <td>[8490]</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[19512]</td>\n",
       "      <td>[7975]</td>\n",
       "      <td>{134: 1}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[18953]</td>\n",
       "      <td>[3492]</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[695]</td>\n",
       "      <td>[2653]</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[243]</td>\n",
       "      <td>[2282]</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        albumid artistid   genreid\n",
       "songid                            \n",
       "0       [12070]   [8490]        {}\n",
       "1       [19512]   [7975]  {134: 1}\n",
       "2       [18953]   [3492]        {}\n",
       "3         [695]   [2653]        {}\n",
       "4         [243]   [2282]        {}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_dict[lbl].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "YaMus\n",
      "{'userid': 183003, 'songid': 134059}\n",
      "density 0.09740952587383789\n",
      "similarity matrix density 0.4576464914574314\n"
     ]
    }
   ],
   "source": [
    "print_data_stats(data_labels, all_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Standard experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_recommender_models(data_label, data_models, config):\n",
    "    data_model = data_models[data_label]\n",
    "    fm = TuriFactorizationRecommender(data_model, item_side_info=meta_dict[data_label])\n",
    "    fm.method = 'FM'\n",
    "    models = [fm]\n",
    "    apply_config(models, config, data_label)\n",
    "    return models\n",
    "\n",
    "def fine_tune_fm(model, params, label, ntrials=60, record_time_as=None):\n",
    "    param_grid, param_names = random_grid(params, n=ntrials)\n",
    "    best_fm_config, fm_scores = find_optimal_config(model, param_grid, param_names,\n",
    "                                                    target_metric,\n",
    "                                                    return_scores=True,\n",
    "                                                    force_build=True,\n",
    "                                                    iterator=lambda x: track(x, label=label))\n",
    "    model_config = {model.method: dict(zip(param_names, best_fm_config))}\n",
    "    model_scores = {model.method: fm_scores}\n",
    "    try:\n",
    "        if record_time_as:\n",
    "            save_training_time(f'{experiment_name}_{record_time_as}', model, fm_scores.index, label)\n",
    "    finally:\n",
    "        return model_config, model_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {}\n",
    "scores = {}\n",
    "data_models = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
       "<progress style=\"width:60ex\" max=\"1\" value=\"1\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">1/1</span>\n",
       "<span class=\"Time-label\">[17:56:52<17:56:52, 64611.86s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[A\u001b[2K\r",
       " [████████████████████████████████████████████████████████████] 1/1 [17:56:52<17:56:52, 64611.86s/it]\u001b[B"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:15ex; max-width:15ex; vertical-align:middle; text-align:right\">YaMus</span>\n",
       "<progress style=\"width:45ex\" max=\"30\" value=\"30\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">30/30</span>\n",
       "<span class=\"Time-label\">[17:56:21<34:08, 2152.70s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[2K\r",
       "          YaMus [█████████████████████████████████████████████] 30/30 [17:56:21<34:08, 2152.70s/it]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for label in track(data_labels):\n",
    "    data_models[label] = prepare_data_model(label, *all_data, seed)\n",
    "    model, = prepare_recommender_models(label, data_models, fm_init_config)\n",
    "    config[label], scores[label] = fine_tune_fm(model, params, label, ntrials=30, record_time_as='param')\n",
    "del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/py36/lib/python3.6/site-packages/pandas/plotting/_core.py:1001: UserWarning: Attempted to set non-positive left xlim on a log-scaled axis.\n",
      "Invalid limit will be ignored.\n",
      "  ax.set_xlim(left, right)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAEXCAYAAAB/HzlmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xl4m+WZ6P/vbVmy5d2xncV2EjtxICQOWQiBbIWWUqBbSgstdNpS4Axtp/2dzrQzc+gsnU5POWdoZ4ZZ2g6lB0ph2qHQNW0ZKDtxEhLClsSExHLiJM5q2bET75b0/P54XzmyLNmyrM3O/bkuX5Feve+jR46lW892P2KMQSmllEqGrHRXQCml1PSlQUYppVTSaJBRSimVNBpklFJKJY0GGaWUUkmjQUYppVTSaJBRSimVNBpklIqBiPxERB4KO3aViLSLyJwxrssWESMix0XEEXLcZV/rS2a9lUo3DTJKxeZ/Au8XkWsBRCQX+CHwVWPMiRiuPwe8L+T+BwFvwmupVIbRIKNUDIwx7cD/BzwgIvnA3wHNxpiHRWStiLwiIp0ickJE/k1EnGFFPAp8JuT+Z4BHQk8QkVYRuTrk/rdE5GH7dp6I/NRu/XSKyE4RKU/8K1UqsTTIKBUjY8wTwGvAfwF3AZ+zH/IBXwbKgfXA9SGPBf0SeI+IFIlIGXAl8LsJPP3tQB5QDZQBfwL0x/dKlEqd7HRXQKkp5otAM/DXxpgjAMaYV0MePygiDwBXAd8NOd4L/DdwM+AGfgUMTOB5h7CCWJ0xZg+wK+5XoFQKaZBRagKMMadExAs0Bo+JyGLgn4DLsFob2cCOCJc/gtXN5sZq+UzEw0Al8LiIFGF1v/2NMUYnDqiMpt1lSk3eD4C9WK2MIuDrgEQ47wVgPlBijNke4fEerCAVNDt4wxgzaIz5hjHmEmADcCPwRwmqv1JJo0FGqckrBLqAHhG5hNHjMQAYa1+NDwIfiVLOm8At9rTnNcBHgw+IyHtEpF5EsoCzWN1n/gS+BqWSQoOMUpP3VeA2rGnKPwB+Fu1EY8xeY8zbUR7+a2Ax0An8LfDTkMcqsSYPnMXqqnsWawKCUhlNdNMypZRSyaItGaWUUkmjQUYppVTSaJBRSimVNBpklFJKJc20XYxZXl5uampq0l0NpZSaMl577TWvMaYikWVO2yBTU1PDrl2aeUMppWIlIocTXaZ2lymllEoaDTJKKaWSRoOMUkqppJm2YzKRDA0N0draSn//1N2GIzc3l+rqapzO8D2xlFIq81xQQaa1tZXCwkJqamoQiZQkN7MZY2hvb6e1tZXa2tp0V0cppcZ1QXWX9ff3U1ZWNiUDDICIUFZWNqVbYkqpC8sFFWSAKRtggqZ6/ZVSF5YLLsgopZRKHQ0yKeZwOFixYsXwT0tLCy+++CIiwoMPPjh83htvvIGI8I//+I9prK1SKt22N7dzw79uoW9wau5Rp0EmxdxuN2+++ebwTzD1zbJly/jZz87vdfXYY4+xfPnyNNVSKZUpGo93se/EWVrae9JdlbhokMkQ8+bNo7+/n1OnTmGM4amnnuKGG25Id7WUUmk24AsAcOxMX5prEp8LagpzqL//bSNvHz+b0DKXVBbxdx9aOuY5fX19rFixAoDa2lp+9atfDT9200038cQTT7By5UpWrVpFTk5OQuunlJp6BoasbrJjnRpkVAyC3WWRfPzjH+cTn/gE77zzDrfeeivbtm1Lce2UUplmuCWjQWZqGa/FkQ6zZ8/G6XTyzDPP8K//+q8aZJRSU767LO4xGRG5XkT2i4hHRO6O8HiOiPzMfnyHiNSEPPY1+/h+EbnOPjZXRF4QkX0i0igiXw45/xsickxE3rR/3h9vvTPdN7/5Te69914cDke6q6KUygD9dndZ64XUkhERB/A94FqgFXhVRDYbY94OOe1O4Iwxpk5EbgHuBT4hIkuAW4ClQCXwrIhcBPiArxpjXheRQuA1EXkmpMz7jDHTfj7vunXr0l0FpVQGmeotmXi7y9YAHmPMQQAReQzYBIQGmU3AN+zbPwe+K9Zy9U3AY8aYAeCQiHiANcaY7cAJAGPMORHZB1SFlTnldXd3jzp29dVXc/XVV486/o1vfCP5FVJKZbQBn9WS8XYP0D/kJ9c5tXo54u0uqwKOhtxvtY9FPMcY4wO6gLJYrrW71lYCO0IOf0lEdovIQyJSGme9lVJqSukfCgzfPtE19fIWxhtkIiXQMjGeM+a1IlIA/AL4U2NMcI7xfwALgRVYrZ1/ilgpkbtEZJeI7Gpraxv7FSil1BQw4POTZX9qTsUus3iDTCswN+R+NXA82jkikg0UAx1jXSsiTqwA8xNjzC+DJxhjThlj/MaYAPBDrO66UYwxDxhjVhtjVldUVESsuDHhsXBqmer1V0pNzMBQgOrSPACOdfamuTYTF2+QeRVYJCK1IuLCGsjfHHbOZuA2+/ZNwPPG+oTcDNxizz6rBRYBO+3xmgeBfcaYfw4tSETmhNy9EdgbT6Vzc3Npb2+fsh/Uwf1kcnNz010VpVSKDPgCzC/LI0umZksmroF/Y4xPRL4EPA04gIeMMY0i8k1glzFmM1bAeNQe2O/ACkTY5z2ONaDvA75ojPGLyAbg08AeEQmuVvwrY8yTwLdFZAVWt1oL8Ll46l1dXU1raytTuSstuDOmUurC0D/kpyDHzayi3Ck5jTnuxZj2h/+TYce+HnK7H7g5yrX3APeEHWsg8ngNxphPx1vPUE6nU3eUVEpNKQO+ALlOB1Ul7inZktEEmUoplcEGfH5ysrOoLHFPydQyGmSUUiqD9Q8FyMnOoqrUzcmufvyBqTWmrEFGKaUy2IDPP9xd5gsYTp2dWmtlNMgopVSGMsYw4DvfkgE4PsW6zDTIKKVUhhr0BzAGcpwOqkusIDPVxmU0yCilVIYKJscMbcm0TrEZZhpklFIqQw3YectynA7yXNmU5jm1JaOUUioxgnvJ5GRbH9VVpVNvrYwGGaWUylDB7rJgev+qKbhWRoOMUkplqOBeMsMtmZI8jp3pm1L5FzXIKKVUhgruJRPaXdY35OdM71A6qzUhGmSUSrHNbx3nsZ1H0l0NNQUEWzKh3WUwtbIxa5BRKsX+8en9/MuzTemuhpoCQqcwQ0iQmULjMhpklEqhI+29HOno5eTZftrODaS7OirDDQzPLrNbMqXjBxljDJ7T55JfuRhpkFEqhbZ4zu9l1Hi8K401UVPB+dll1kd1aZ4Tt9MxZnfZd57ez3v/+WX2n4wcaB7Z3sLh9p6E1zUaDTJKpVBDk5fyAhcAjcfPprk2KtOFLsYEEBFrrUyUbZifbjzJ919sBmDfidF/X97uAb7+m0b+8Q8HklTj0TTIKJUi/oBhW3M71yyeRU1ZHnuPaUtGja0/bAozRF8rc7Ctmz9//C2WVRXjyBI8p7tHndNsH/tD40nO9qdmhpoGGaVSZM+xLrr6htiwqJylVcXs0SCjxhFsyQRnl0HkVf89Az4+9+hrZDuE//jUKubPyKO5LUKQabO6yQZ8AX6/+0QSa36eBhmlUmTLgTZEYH1dOfWVxbSe6aOzdzDd1VIZLHwxJlgtmTO9Q/QO+gA42z/EX/58N81t3fz7rauoLs1j4cyCKEGmG7fTwcKKfH7xWmtKXkN2Sp5FKcUWj5ellUXMyHexrKoYsMZl1teVp7lmKlP1DwXIEsjOkuFj1fYMs397zsOeY53sONiBL2D4y+svZsMi629pYUUBL+4/jc8fINtxPkA1t3WzoCKfD1w6h28/tZ8Wbw815flJfQ3aklEqBXoGfLxx5Awb6ioAWFpZBKDjMmpMwV0xRc4HmfllVlC4/6VmTp0d4M6NtfziC2v5k6vrhs9ZWJHPkN9wNKxbrbmtm4UVBdy4sgoR+OUbx5L+GrQlo1QK7DjUzpDfsNH+plma76KqxM1enWGmxhDcFTPU8upi/t9nVlM3syBqK6RuZgEAntPd1Nrn9A/5aT3Tx02r5jKn2M2GunJ++Xorf3rNIrJCWkqJpi0ZpVJgS5OXnOwsLptfOnysvqqIRm3JqDH0D/mHF2IGiQjvXTJrzG6uBRVWkAkdl2lp78EYWDjTuu5jq6ppPdPHzpaOJNT8PA0ySqVAQ5OXNbUzRswSqq8s5qC3h3Mpmkqqpp4BX2B4IeZEFLudVBTmDE9ZBmg+bc0sW1BuBaDrls6mICc76RMANMgolWQnu/ppOt093FUWVG8P/u87kTkpQFRmGRgKjGrJxKquogBPSEumua0bEYa7z9wuB+9bMovn3jmdkLpGo0FGqSRr8HgBhgf9g5ZWWYP/ul5GRdPv85MTR0sGrG6x5tPdw3vPNLd1U1Xixu06H7RqyvPp6BkcniqdDBpklEqyhqY2ygtcLJ5dOOL4zMJcZhbm6LiMimpgKEDuJFoyZ/t9tHVbiViDM8tCVRTmANDenbz1WhpklEoiYwwNnnbW15VHnMGzrKqYvZooU0UxMKmWjD34f7qHQMDQfLpndJApsIJMMjOCa5BRKoneOXkOb/cAG6IsuFxaVYzndDd9g8nrrlBTV//Q6CnMsVoYMsPs5Nl++ob8wzPLgsrtloy3W4OMUlNSQ5M1HrNxUUXEx+sriwgY2HdS18uo0ayWTHzdZXOKc8lzOfCc7h6eyhytu0xbMkpNUVs8XupmFjC7ODfi48EZZjouoyKJtBgzViLCwgorh1lwKnN4kAluO5GRQUZErheR/SLiEZG7IzyeIyI/sx/fISI1IY99zT6+X0Sus4/NFZEXRGSfiDSKyJdDzp8hIs+ISJP9b2n48ymVafqH/Ow81B61qwysb5sz8l3sPaYtGTVa/ySmMIOVXqb5dDfNbT0U5WYPB5WgnGwHRbnZmdddJiIO4HvADcAS4FYRWRJ22p3AGWNMHXAfcK997RLgFmApcD3wfbs8H/BVY8wlwJXAF0PKvBt4zhizCHjOvq9URnv98Bn6hwKj1seEEhGWVhbpNGYVkZW7LP4Op7qZBRzv6mf3sS4WziwYkQMtqKIwZ3gGWjLEW/s1gMcYc9AYMwg8BmwKO2cT8GP79s+Ba8R6hZuAx4wxA8aYQ4AHWGOMOWGMeR3AGHMO2AdURSjrx8BH4qy3UimzxeMlO0u4YkHZmOfVVxVz4NS5pK5VUFOT1V02mZaM1T321tHOUV1lQRWFORnZXVYFHA2538r5gDDqHGOMD+gCymK51u5aWwnssA/NMsacsMs6AcyMVCkRuUtEdonIrra2tkinKJUyDU1eVs0rpSBn7Dy0y6qK8QUMB06O3v9DXbiMMQxOYkwGzifKhNHjMUEVhbkZGWQipew0MZ4z5rUiUgD8AvhTY8yEOqqNMQ8YY1YbY1ZXVESezaNUKpzpGWTv8a7h/T3GUl9pDf7rehkVasA3elfMiZpXlofDXp+1sCJyQs3yAhfeDFyM2QrMDblfDRyPdo6IZAPFQMdY14qIEyvA/MQY88uQc06JyBz7nDlAcpPtKDVJW5u9GENMQWbuDDeFudm6t4waIbj18mRaMjnZDubNyAPOL84MV1GYQ/eAb3inzUSLt/avAotEpFZEXFgD+ZvDztkM3Gbfvgl43lhJdDYDt9izz2qBRcBOe7zmQWCfMeafxyjrNuA3cdZbqZRoaPJSmJvNpfYU5bGICPWVxbq3jBpheOvlSQz8g9VNlp0lw8EmXHDVv/dcclozcW1aZozxiciXgKcBB/CQMaZRRL4J7DLGbMYKGI+KiAerBXOLfW2jiDwOvI01o+yLxhi/iGwAPg3sEZE37af6K2PMk8A/AI+LyJ3AEeDmeF+wUslmjGFLk5d1C8tGbH07lvqqIn68/TBD/gDOGK9R01u/3ZKJN3dZ0M2rq6mbWRD17yq46j9ZM8zi3hnT/vB/MuzY10Nu9xMlGBhj7gHuCTvWQOTxGowx7cA18dZVqVRqae/lWGcfn796YczX1FcVM+gL0NzWzeLZRUmsnZoqEtWSuW7pbK5bOjvq48nOX6ZfmZRKsIYma2bjxjEWYYZbag/+72nVcRllCQ78T2YKcyxmJrklo0FGqQTb0uSlutTN/LLIfeCR1Jbnk+dy0KjjMhmr8XgXT+w6Ov6JCdI/ZLVkJrMYMxYz8l2IaEtGqSnB5w+wvbmdjYvKI66ujsaRZa381xlmmevBhkN887dvp+z5UtWSyXZkMSPPlbTUMhpklEqgt1o7OTfgG7ULZiyWVhbz9omz+APhS85UJjjc3kvfUOqyMgyPyUxiCnOskrnqX4OMUgm0pcmLCKxbOHYqmUjqq4rpHfRzyNuThJqpyTrc3osvYBjyB1LyfMOzyyaxGDNWGmSUmiIamrwsqyqmNN81/slh6qusWWWNuvI/4/QM+Ia7k1LVmkllS6a8IEe7y5TKdOf6h3jjaOeYqf3HUldRQE52lo7LZKDD7b3Dt/tTtIvp8Ir/JA/8g7ZklJoSXjnYgT9gYkolE0m2I4vFczTtfyY60nG+CzNVLZnh2WVJHvgHa61McKJBommQUSpBGpracDsdXDY//j316iuLaDx2loAO/meUES2ZodSMyQzPLktBS6a8cOLdu7HSIKNUgmzxeFlTO2NSU07rq4o5N+Dj6Jne8U9WKdMSEmRSNyaTminMABUFkbcHTwQNMkolwPHOPg629Yy5C2YsltkJNXU75sxypKOH4LKnvhSNyfQP+XE6ZDhVfzJV2Kv+k0GDjFIJ0NDkBWJL7T+WRbMKcDpE95bJMIfbe5lvZzHuT2FLJhWtGNAgo1TG2+LxUlGYw8WzCidVTk62g4tmFeoMswwy6AtwvLOPi+z/21ROYU7F9GWAErczaS0mDTJKTVIgYNjq8bKxbmKpZKKpryym8fhZrO2XVLq1nuklYGDxHGsdU6paMv1DgZQsxATIyhLKC5Iz+K9BRqlJevvEWTp6BifdVRZUX1VER88gJ7r6E1KempzDHdag/+LZyWnJBBddjj4eSFlLBpLXZaZBRqlJavDY4zFxLsIMt9Qe/Nf1MpnhiD2z7OJgkEnQwH8gYPjrX+1h9beejdg6Ghjy40phkCkv0CCjVEZqaPJy8axCZhYlZhroJbOLyBJo1CCTEVrae8hzOZhbmriBf2MMX9+8l5/sOMK5fl/ElC79vgA5Keoug/OblyWaBhmlJqF/yM/Olo6EdZUBuF0OFs0sZK/uLZMRjrT3Mm9GHq7sLBxZMunuMmMMf7e5kf985Qgr5pYA0Nk7NOq8gSE/udpdptSF7dWWDgZ9gYQGGYClVbq3TKY43NE7vAGd2+mgbzD+Ff/GGP7+t2/zyPbD/PHGWu6+YTEAXX0RgkyqWzIaZJTKPA1NXlyOLK6onZHQcusrizl9boDTZ6fW4P8hbw/v+vYL/PeeE+muSkIEAoYjHb3UlOUDVtr9/igD9eMxxvCt3+/j4W0t3LG+lr96/yWU5lkzuiK1ZPqHUjeFGXRMRqmMtKXJy6r5JeS5shNabr09+D+VtmM+0zPIHQ+/ypGOXn63e3oEmZNn+xn0BZgXbMm4suLOwvzC/tM82HCIz66r4W8/eAkiQkmeE4DOvsFR5w/6UjeFGbQlo1TG8XYP8PaJs2xcNPFdMMezpNJakzFVuswGfH4+95+vcayzj+XVxWxr9k6LJJ/BxJjzZ1gtGbfTEfeYzLP7TlOQk81fvf+S4fVUxW47yEQak0nxFOaV80qSUq4GGaXitDXBU5dDFeRks6A8f0qklzHG8LVf7GHnoQ7+8ebl3LauhjO9Q+w7OXVaYdEcbrdS/AfHZHLjDDLGGF7a38a6hWUjpiXnOh3kOrMijsmkurssWSlsNMgoFaeGJi/Fbudw11aiLa0qnhKJMv/9eQ+/fOMYX732Ij68vJL1dtDd5mlPc80m73BHL06HMKfYmp6e63TEtU6mua2HY519XHXx6FZvidtFZ+/o7rKBFHeXJYsGGaXiYIyhweNlfV1Z0nI+1VcWcayzj46e0R9AmeI3bx7jn585wMdWVfOl99QBMKsol4UV+Wxt9qa5dpN3pL2X6tI8sh3WR6Xb6aA/js29XjrQBsC7InStluQ5o3SXpbYlkyxT/xUolQbNbT2c6OpnQ13ix2OClg0P/mdml9mulg7+4ondXFE7g//70WUj8ratrytn5yFrevdU1tLewzw7+zLYQSaOlsxLB9pYWJHP3JCygordTjrDusv8AcOQ36QsC3MyaZBRKg4NTdY308nuHzOWpZWZu7fM4fYe7nr0NapK3dz/qctGpT9Zt7Cc3kE/b7V2pqmGk2eM4Uh7LzVlIUHGNfExmf4hPzsOtnPVRTMjPl6a56IrrCUTzGeWm4JdMZNt6r8CpdJgS5OX+WV5Eb+ZJkpxnpO5M9wZN/jf1TvE7Q+/SsAYHvrs5ZTmj87eu3ZBGVlyfnLEVHSmd4hzAz7m2WtkIL6B/1cOtjPgC0QcjwG7uyxsCvPAUHBXzKn/ET31X4FSKTbkD/DKwfakzCoLV19ZnFE5zAZ9AT73n7to7ejjgU+vprY8P+J5xXnWhIipPPg/PLMs5ItErnPi62ReOtBGTnb0BbvFeU7O9A6N2NohuOAzlSv+k0WDjFIT9MaRTnoG/UntKguqryqmpb2Xs/2jB4ZTzRjDX/1qD68c7ODem5axZpwsB+sWlvP6kTP0DPjier7eQR/vu+8l7npkF68dPhNXGZMRXCNTUz5yTGaiLZmXDrSxdmFZ1JliJW4Xg74A/UPnx6+CLRntLlPqAtTQ1EaWwNqFyQ8yS+1FmY0ZMC7z/Reb+flrrXz5mkXcuLJ63PPX15XhCxh2tnTE9Xwv7W/jwKlutjR5+dh/bOPj92/n+XdOpWwzt8PtvYhAdenIIOMLGIb8sU1oONrRy8G2Hq66KPoEkUir/gd8we6yC7glIyLXi8h+EfGIyN0RHs8RkZ/Zj+8QkZqQx75mH98vIteFHH9IRE6LyN6wsr4hIsdE5E375/3x1lupydri8XJpdcnwau1kCg7+p3uG2e92H+c7T+/nIysq+dP3LorpmtXzZ+ByZLEtznGZpxpPUprnZMdfX8PXP7iEY5193PHwLq7/ly384rXWmD/o43W4o4fZRbkjWiBul3U71nT/wanLYwaZCKv+gwP/F+yYjIg4gO8BNwBLgFtFZEnYaXcCZ4wxdcB9wL32tUuAW4ClwPXA9+3yAB62j0VynzFmhf3zZDz1VmqyuvqGeOtoZ0q6ysDKJzW7KDet6WUOnDrHVx5/i8trSrn3pktj3mLa7XKwan4JW+MYlxnw+Xl+32muXTKLolwnd2yo5cW/uJr7PrEcEfjqE29x1bdf4MGGQ/QOxtcdN57D7eezLwcFA06sXWYvHWhj7gx31LErsMZkYGSQ6R/uLrtwWzJrAI8x5qAxZhB4DNgUds4m4Mf27Z8D14j117kJeMwYM2CMOQR47PIwxrwMxNe2VioFtje3EzDJSSUTTX1Vcdr2ljHG8De/2ku+y8H9n7pswt036xeWD29PPRHbPO2cG/BxQ/2c4WNORxY3rqzmv7+8kR999nLmzsjjf//ubf7iid0TKjtWh9t7h3OWBQU/9PtjSPc/6AuwzePlqosqxgzMJW5rdl7XiO6yC7wlA1QBR0Put9rHIp5jjPEBXUBZjNdG8iUR2W13qZVGOkFE7hKRXSKyq62tLbZXotQENHjayHM5WDkv4p9gUtRXFdHc1p20b+xj+eXrx9jZ0sHdNyymLI5U8OvsYLy9eWKtmaf2nqQgJ5t1dWWjHhMR3r14Jj/73Fo+u66GZ94+FTH312T0DFi7Vc4La8m4J9CSee3wGXoG/VHXxwSVRGjJnJ/CfOG2ZCKF5fDRuGjnxHJtuP8AFgIrgBPAP0U6yRjzgDFmtTFmdUVF8lZiqwtXQ5OXKxeUpXTv9frKYoyBfSdS25rp7B3k/zy5j1XzSrj5srlxlbG8upiCnOwJpZjx+QM8s+8U71k8c9wP2U0rKhn0B3j27VNx1S+a4ZllZSNbMm6X9f8ey5jMiwdO43QIaxeODpShzg/8h3SX6WJMWoHQv7pq4Hi0c0QkGyjG6gqL5doRjDGnjDF+Y0wA+CF295pSqXS0o5eW9t6UdpXB+b1lUr3y/ztP76ezb4h7blxGVpz52bLtDd0mMvj/assZOnoGub5+9rjnrphbQlWJmycTvEnakY6R2ZeDJjIm89L+NlbPn0FBzth7DbmdDlyOLG3JhHkVWCQitSLiwhrI3xx2zmbgNvv2TcDzxpp7uBm4xZ59VgssAnaO9WQiMifk7o3A3mjnKpUsDfYHZaoG/YNmFeVQXuBK6eD/m0c7+enOI3x2XQ2XzCmaVFnr6sppae/lWGdfTOc/3XiSnOysMWdkBYkIH7h0Di83tSW0yyzYkom3u+zU2X7eOXku6ir/UCJCcZ4zbEzGDjIXakvGHmP5EvA0sA943BjTKCLfFJEP26c9CJSJiAf4CnC3fW0j8DjwNvAU8EVjjB9ARP4L2A5cLCKtInKnXda3RWSPiOwG3g38WTz1VmoyGpq8zCrKoW5mQUqfV0RYWlnMnhQFGX/A8De/3sPMwpyYpyuPZb09rhJLiplAwPDU3pNcdVEF+eO0AILev2wOQ37DMwnsMmtp72VGvoui3JHT1IenMI+z6n+b3T0YKetyJCVuZ9jsMru7bBq0ZOLeM9aeRvxk2LGvh9zuB26Ocu09wD0Rjt8a5fxPx1tPpRLBHzBsbfZyzeJZMU/hTaT6qiIaPF76h/xJn9b6kx2H2XvsLN/95EoKcye/FujiWYWUF7jY5vHy8dVjj+281drJybP9/GX9xTGXv7y6mKoSN7/ffZybLht/kWgsjnSMzL4cFPzQH68l03ZuABjd3RZNeLr/C74lo9SFpvF4F529QynvKguqryzGHzDsP3kuqc9z+lw/33lqPxsXlfOBZXPGvyAGIsLaheVsbW4fd7X+U40nyc4Srlk8a0Llf/DSOTR4vKOyGUfT4u3h+n95md+8eSzi45HWyMD5lsx4QaZ7wHrcHeMXgmK3a8TAf3AKs8sx9T+ip/4rUCoFtjRZ3R/rUzzoHzQ8+J/klf//5/f7GPAF+PsPL01oi239wjLazg3gOd0d9RxjDE/vPcnahWXDCxRjFewy+8PbJ2M6//6XmnmYd8biAAAgAElEQVTn5Dm+/Nib/PtzTSOC36AvwPHOvhGJMYOG18kMjb1OpnfAR77LEfOEiZI8J10hu2P2DwVwZWfFPeEik2iQUSoGDU1eFs8upKJw4mtFEqG61E2x25nUGWbbmr38+s3jfP7qhSyoSOy4UzA4jzUus//UOVrae2OaVRbu0upiqkvd/D6GWWanz/Xzy9ePcfNl1Xx0ZRX/9MwB/vyJ3cMbrLWe6SVgYH7Z6FX6bmdsaWV6Bv3kxTimBPaYTFhLZjosxAQNMkqNq2/Qz2uHz6StqwysLqH6qqKk5TAb9AX421/vZd6MPP7k6oUJL3/ujDzmznCzdYxFmU/tPYkIvG/JxINMcJZZQ5OXzt6xsws8su0wQ4EAf/LuOv7p48v5s/dexC9eb+UzD+2gq3eIwx3WzLJI3WVOh+DIEvrGGfjvHbRaMrEqyXPSO+gf7iYb8AWmxfRl0CCj1Lh2HGpn0B9gQ4wzhZKlvrKYd06cS0piyP/XcJDmth7+ftPSpE0sWL+wnFcOtuOLUv+n9p7k8vkz4m4tfnBZJb6A4Q9jzDLrHfTx6CuHed+SWdSW5yMifPm9i7jvE8t5/XAnN/7HVhrsrtHw6ctgBbPc7Kxxx2R6BnzkuWJvyRTnBVPLWK0Za4LH9Ph4nh6vIgJfwODtHojrp6NnMGXpxFXma2jy4nJksaZm7P1Tkm1pVTGD/gAHTiV28P9oRy//9lwT1y+dzbsvHjsFymSsqyvnXL8vYh62Fm8P75w8x3VxdJUF1VcVMXeGm9/vjt5l9virR+nqG+Kudy0YcfzGldU8eucaOnoGebDhEHkuBxVR0ujEsgVzz4B/3EWYoUrDUstYLZnp8fEc9xTmTLfvxFlWf+vZuK+vLM5lfV05GxaVs25hedr64lX6NXi8rK4pHZ5ZlC71IXvLBLcASIS//+3bZInw9Q+FJ1JPrHULz6+XWTG3ZMRjTzVaA/bXLY19Vlk4EeEDyyr5f1sOcqZncNS20D5/gAe3HuKy+aVcNn/0F4YrFpTxyy+s446HX2V2cW7UiQ+5Tse462R6B30Rt6WOJpgkczjIDE2f7rJpG2QqS9x8c9PSuK4d8AV47fAZ/vD2KZ54rRWAxbML2bionPV15VxRW5b2DxyVGqfPWSu3//L62NdtJEtNWT4FOdnsPd7Fx4kvl1i4Z94+xbP7TvG1GxZTWeJOSJnRlBfksHh2IduavXzx3XUjHntq70mWVRWP2CAsHh+8dA73v9TMH94+yScunzfyORpPcrSjj79+f/RguqCigGe/chWDY3RJup2O4dxi0fQM+qkuncDA/3BLxhpPGvBNn+6yaRtkyvJdfHptTdzX/4+N1gK8vce6aPB4aWjy8uNth/nhlkO4HFmsml/CxkUVrK8rZ1lVMY5pMNVQjRacDbWxLv0JV7OyhCWVRQlLL9M36Ocbmxu5aFYBd2yoTUiZ41m3sJyf7Dg8YlHpia4+3jzayV9cN/lAvrSyiHkz8vj9npFBxhjDD18+SG15PtcuGbu1lO3IInuM9Slul2PcgX9rTCb2L6LBDfCCM8y0JXOBcGQJy+eWsHxuCV98dx19g352tnSw1eNlS5OX7zy9n+88vZ+i3GzWLbS61jbUlTO/LC8tq8JV4m1p8lKa5xzeBjnd6iuL+enOw/gDZtJfbL77QhPHOvt4/HNrcaZo0d/6ujIe2nqI1w+fGd4G4A+N1kB9PFOXwwVnmT3w8sgusx2HOnirtYt7bqyf9O8t1xnLmIwv5rQ4cL4l0zU8JuOfUHdbJtMgMwFul4OrLqoYTtzn7R5gq8fLVrulE+xXri51syFkPGfGNPljudAYY2ho8rKurjxjFsXVVxXRPxTgYFs3i2YVxl2O5/Q5Hnj5IB9bVc2a2tRNaFhTOwNHlrC12TscZJ7ae5JFMwtYmKC1OR9YNof/eLGZpxtPcssaqzXzwMsHKct38bFVk087k+t0jJmM0xhD76Cf/JzYWyIFOdk4soROO0lm/1BgWuQtAw0yk1JekMOmFVVsWlGFMYZD3p7hVs7vd5/gsVePImI14dfXlbOxroLVNaXTYkvVC0HT6W5OnxtgY5pW+UcSuvI/3iBjjOFvf92I2+nga+9fnMjqjasw18ny6mK2etr5i+ugvXuAHYfaR43RTMbSyiLml+Xx+z0nuGXNPJpOneP5d07zZ++9KCHvPbczi1Nd0Vsyg/4AvoCZ0BRmERmRJHPA558WectAg0zCiAgLKgpYUFHAp9fW4PMH2H2si4YmLw0eLw81HOIHLx3ElW1NhV1fV87GReUsmVOUMd+S1UjB9RIb0rgIM9yC8nxynVnsPXaWG1fGV8Zv3jzO9oPtfOsj9ZTHsdvlZK2vK+d7L3g42z/Es/tOETBw3dLJd5UFWbPM5vCDlw/S0TPID7ccJNeZxafXzk9I+eMN/PfYecsmshgToDjv/Kp/ncKsxpXtyGLVvFJWzSvlf16ziJ4BHzsPdbClyepeu/epd7j3KWt+/Lo6ayxnQ105cyPkS1Lp0eDxUlOWN+kZT4mU7cjikjlFcaX939PaxQ9ebubJPSdYPreEW9fMG/+iJFi3sJx/f97DjoMdPLX3JHNnuBM+5vWBS+fw/RebeXT7YX79xnE+cfnchHVbjzfw3zNgbZM9kTEZsFLLBMdkUpFtO1U0yKRIfk427148k3cvtha7nT7bz9Zm73DQCS4gm1+WNxxw1i0sn3CiQJUYQ/4Arxxs56OrqtJdlVHqK4v51RvHCATMuK1gYwwvHmjjgZcOsv1gO4U52fzxxgXc9a4FaZsRuWp+CbnOLJ5uPMlWTzu3rZuf8IkyS+YUUVuez78+dwAD3JnA2XPjDfz32gFowkEmz8Xpc/2AtmRUAswsyuXGldXcuLIaYwzNbd3DAefXbxzjJzuOkCWwrKqYDfb6nMvml06baY2Z7o0jnfQO+lO+1XIs6quKePSVwxzu6KW2fHQSR7Bykf3mzWP8cMtBDpzqZnZRLn/1/sXcsmbeqI24Ui0n28HlNTP4xeutGJOYWWXhRIT3L5vN915o5ob62dRE+T3FI9fpGDNBZs+g1ZKZyBRmsFoywWwO0yl3mQaZDCAi1M0spG5mIbevr2XIH+Cto53DQef+lw7yvReayXVmsaa2jI11VtBZPLtQx3OSpMHjJUtg7YJMDDL24P+xrlFB5mz/ED/dcYQfbT3EqbMDLJ5dyD/dvJwPLa/ElUHfjNctLGdLk5eKwhxWzi1NynN8bFU1v37jeEInFYA1JjPkNwz5AxGnfsfbXVacZ3WXDfkD+ANGF2Oq5HE6slhdM4PVNTP4s2sv4lz/EDsOdliLQj1e7nlyHwDlBa4R63OSvWL7QtLQ1May6pKM7K5cNLMQlyOLvce7+NDySsBa0PijrS38dMcRugd8rFtYxr0fu5SrLqrIyDVbwS2Zr1s6K2lflBZUFLD17vckvNzQdP+Rg0xw4H+iYzIuzg34hoOUtmRUyhTmOnnvklm8116pfKKrj62edhqa2mjwtLP5reMALKjIHx7PuXJhWdq7Raaqs/1DvNXaxReuSnzK+0RwZWdx8exCGo+d5Z2TZ3ng5YNsfvM4Bmvzrs+9a8FwaydT1VcW85VrL+LGlZk35jWeXNf5jcsKc0c/3jsYbMlMsLvM/kIT3LpZpzCrtJlT7Oamy6q56TJrPGf/qXPDU6Wf2NXKI9sPW9kKqovtRaEVrJhbklHdJZnsleZ2/AGTUVOXw9VXFfGzV49y/b9swe108Kkr53PnhtopMzsxK0v4n9csSnc14jLexmXBlshE1snA+SBz8qw1+K+LMVVGEBEWzy5i8ewi/sfGBQz6Arx+5MzwotDvvuDh3573kOdycEXtDDYsqmBDXTkXzSrIyG6UTLDV48XtdLByXsn4J6fJ+5bMZsfBDj66qopPXTmfkjzNKpEqwSATbYZZz/Dssgmuk7Hzl506qy0ZlcFc2VlcuaCMKxeU8dX3XUxX3xDbm9ut1DceLy/sfxuAisKc4a61DYvKmVUUod1/gdri8XLFghkZ3SceOh1epVZwQD7aWpneAR8i54NRrIJfFE7ZLRmdwqymhGK3k+vrZw9PE20902sHnHZeOtDGr944BsCimQXDEwiuWFA2oQ2XppPjnX0cbOvhk2laqKgyXywtmXxX9oR7CkqGWzJ2kNHFmGoqqi7N4xOXz+MTl88jEDDsO3l2eDzHmvraQnaWsHJeCR9eUcWnr0xMKo6posGTealkVGY5P/AffUxmomtkIGRMpktbMmqayMoSllYWs7SymM9dtZD+IT+vHz5Dg8fL8++c5m9/vZf6yiJWzkvOOoZMtNXjpbwgh4snkeFYTW/jDvwP+ie8RgasWaQicCo4uyyDu2snYnqESpUQuU4H6+rK+cvrF/PzL6yjMCebH21tSXe1UiYQMGz1eNlQV6aTIlRU43WX9Q74JjzoD9b+VcVuJ6fslsx0WYw5PV6FSriCnGw+fvlcntxzYrj5Pt3tP3UOb/cg6zMwlYzKHMHElX2Dkbdo7hn0TXj6clCJ20lbt7Zk1AXis+tqCBjDo6+0pLsqKZGJqf1V5hl34H/AP+E0/0HFeS78AQNMnzGZ6fEqVFLMnZHHtUtm8dMdR8bd03w62OLxsrAinznFmp5HRZfrsj42o4/J+MiLc3ZmcIYZMG1S/WuQUWO6fX0tZ3qH+PWbx9JdlaQa8PnZeaidjYsq0l0VleFcjiyyJHqQ6R3wUxBvd1lIrrzpshgz7lchIteLyH4R8YjI3REezxGRn9mP7xCRmpDHvmYf3y8i14Ucf0hETovI3rCyZojIMyLSZP974Ux3SrMramewZE4RP9p6CGNMuquTNK8dPkP/UEDHY9S4RAS3M/rGZVZLJr5WSGhL5oLuLhMRB/A94AZgCXCriCwJO+1O4Iwxpg64D7jXvnYJcAuwFLge+L5dHsDD9rFwdwPPGWMWAc/Z91UKiAh3bKjlwKlutnra012dpNnq8eLIEq5cMCPdVVFTgNsVeeMyYww9A74JZ2AOKrZX/YtYLabpIN5XsQbwGGMOGmMGgceATWHnbAJ+bN/+OXCNWPNCNwGPGWMGjDGHAI9dHsaYl4GOCM8XWtaPgY/EWW8Vhw8tn0N5gYuHth5Kd1WSpqHJy4q5JRRq5moVg5zsyEFmwBcgYJh0SyYnO2vaTKOPN8hUAUdD7rfaxyKeY4zxAV1AWYzXhptljDlhl3UC0KRNKZST7eCPrpjP8++c5mBbd7qrk3BdvUPsPtaVkbtgqszkdkXeHTOYgTnetEzBMZnpMn0Z4g8ykUJseId9tHNiuTYuInKXiOwSkV1tbW2JKFLZ/ujKebgcWfx4W0u6q5Jw25q9GKNTl1Xs3E4H/UOj18n02uM0ca+TsYPMdFmICfEHmVZgbsj9auB4tHNEJBsoxuoKi+XacKdEZI5d1hzgdKSTjDEPGGNWG2NWV1ToLKFEmlmYy4eWV/LEa6109Q2luzoJ1eDxku9ysGJu5qb2V5kl2sB/d3Dr5XjXybitMRltycCrwCIRqRURF9ZA/uawczYDt9m3bwKeN9b0pM3ALfbss1pgEbBznOcLLes24Ddx1ltNwu3ra+gd9PP4q0fHP3kKafB4uXJBWcStdJWKJDfKwH9wV8y418nknR+TmS7ieiX2GMuXgKeBfcDjxphGEfmmiHzYPu1BoExEPMBXsGeEGWMagceBt4GngC8aY/wAIvJfwHbgYhFpFZE77bL+AbhWRJqAa+37KsXqq4q5onYGD29rweePnFJjqjna0cvh9l7tKlMTkpudFWVMxjpWMNmB/2nUXRZ3FmZjzJPAk2HHvh5yux+4Ocq19wD3RDh+a5Tz24Fr4q2rSpzb19fy+f98jWf3neL6+jnprs6kBVP7b9QgoyYg2hTm4ZZMvFOY7SAzXbZeBl3xrybo2iWzqC5181BDS7qrkhANHi+zinJYWFGQ7qqoKST6mIy99XKcQSbbkUVhTva0aslMn1eiUsKRJXx2XQ07WzrY09qV7upMSiBg2ObxsqGuYtqsSVCpkeuMPIX5/JhM/C2R4jynDvyrC9vHL59LvsvBj6b44sy3T5zlTO8QGxaVpbsqaoqx1smMHpc8PyYT/36Q771kFmsXTJ+/SQ0yasKKcp3cvHouv919nNPnpu5eM1vs1P6ar0xNlNvpYNAfGDUBpnfQR5ZMbnbYNz68lD9+14LJVjFjaJBRcbltXQ2+gOE/XzmS7qrEbavHy8WzCplZmJvuqqgpJrhYst83Msh023nLtPv1PA0yKi615flcs3gmP3nlcNSU55msf8jPzpYOnbqs4jK8cVnY4H/vgH9S4zHTkQYZFbc71tfS3jPI5rfGS9iQeXa1nGHQF9B8ZSouwQ3Fwr9g9Qz6yJ/EeMx0pEFGxW3twjIunlXIj7a2TLm9ZrZ42nA6hDW1mtpfTZzbFTnI9A76456+PF1pkFFxs/aaqWHfibO8cjDSDg2Za6vHy8p5pfqtU8VluLssLMh0D/jIizNv2XSlQUZNyqYVVczIn1p7zXT0DNJ4/CwbtatMxSk32piMdpeNokFGTUqu08EfXTGPZ/ed4nB7T7qrE5OtHiu1/3od9Fdxyo3Skukd8GuQCaNBRk3ap66cj0OEh6fIXjNbPV4Kc7O5tKo43VVRU5Q7ysC/NYVZu8tCaZBRkzarKJcPXjqHJ3a1cq4/s/eaMcawpcnL2gVlZGtqfxWn8wP/4Ysx/XEnx5yu9F2mEuKODbV0D/h4YldruqsypsPtvRzr7NOsy2pSIg38G2PsKczakgmlQUYlxKXVJayeX8rD21rwBzJ3OvMWj6aSUZMXaTFm/1AAY9AxmTAaZFTC3L6+liMdvTy371S6qxLV1iYvVSVuasvz010VNYUFU/GHtmQmu/XydKVBRiXMdUtnUVmcy4+2tqS7KhH5A4ZtzV7W15Vpbik1KTnZWYiMHPif7IZl05UGGZUw2Y4sbltXw/aD7bx9/Gy6qzPKnmNdnO33sWFRRbqroqY4EcEdtqdMMM2/jsmMpEFGJdQtl8/D7czMvWYamtoAWLdw+uzVodLH7Ry5BXOwJaNjMiNpkFEJVZzn5GOXVfGbN4/j7R5Id3VGaPB4WTKniPKCnHRXRU0DuU4HfYPnpzAHx2S0u2wkDTIq4T67rpZBf4Cf7sicvWZ6B328dviMTl1WCZPrzAobk9Huskg0yKiEq5tZwNUXV/DoK4cZ8GXGXjM7D3Uw5Dc6dVkljNs1srusZ3h2mbZkQmmQUUlxx/pa2s4N8PvdJ9JdFQAamry4srM0tb9KGLfTMWKdzPmWjAaZUBpkVFJsXFRO3cwCHtp6KCP2mmnweFk9v3Q4saFSk5XrdNDvG71ORlP9j6RBRiWFiHD7+hr2HjvLrsNn0lqXtnMDvHPynG61rBJqdEvGhyNLyMnWj9VQ+ttQSfPRldUUu5081JDe6czbmq1UMrrVskokt2v0Opk8l0MX+obRIKOSxu1y8Mkr5vF040mOdvSmrR5bmryU5DlZWqmp/VXi5GaPXidToOMxo2iQUUn1mbXzEREe2d6Sluc3xrDV42XdwjIcWfoNUyWO2zWyuyzYklEjaZBRSTWn2M0N9bN57NWjw1M8U6m5rYcTXf1sqNNUMiqxrIH/84sxe3Tr5Yg0yKiku2NDLef6ffzi9dTvNbPVo+MxKjncTgeDvsDw1ha92pKJSIOMSrpV80pZMbeEH21tIZDivWa2NHmZNyOPeWV5KX1eNf25XdbHZ3Dwv0fHZCKKO8iIyPUisl9EPCJyd4THc0TkZ/bjO0SkJuSxr9nH94vIdeOVKSIPi8ghEXnT/lkRb71VetyxoZZD3h5ePHA6Zc855A/wysF2XeWvkiI3bHfMngGf5i2LIK4gIyIO4HvADcAS4FYRWRJ22p3AGWNMHXAfcK997RLgFmApcD3wfRFxxFDmXxhjVtg/b8ZTb5U+N9TPZnZRavea2d3aSfeAT/OVqaTIDdsds2fQr3nLIoi3JbMG8BhjDhpjBoHHgE1h52wCfmzf/jlwjVgTyDcBjxljBowxhwCPXV4sZaopyunI4tNr57OlycuBU+dS8pxbmryIwNoFmtpfJV5wC+Zgd1mvtmQiijfIVAFHQ+632scinmOM8QFdQNkY145X5j0isltE7hORiLnaReQuEdklIrva2tom/qpUUn1yzTxysrNSttfMVo+XZVXFlOa7UvJ86sJyPsgECAQMvUN+nV0WQbxBJtKCg/AR3WjnTPQ4wNeAxcDlwAzgf0WqlDHmAWPMamPM6ooKnbKaaUrzXXx0VTW/fP0YHT2DSX2u7gEfbxzp1PEYlTRu1/kxmb4hP8ZAvs4uGyXeINMKzA25Xw0cj3aOiGQDxUDHGNdGLdMYc8JYBoAfYXWtqSno9vU1DPgC/NfO5O41s+NgO76AYaMGGZUkoQP/PfaumHnakhkl3iDzKrBIRGpFxIU1kL857JzNwG327ZuA542VjnczcIs9+6wWWATsHKtMEZlj/yvAR4C9cdZbpdlFswrZuKicR7a3MOQPjHt+vLY0ecnJzmLV/NKkPYe6sOU6rY/PvkE/vQN2mn9tyYwSV5Cxx1i+BDwN7AMeN8Y0isg3ReTD9mkPAmUi4gG+AtxtX9sIPA68DTwFfNEY449Wpl3WT0RkD7AHKAe+FU+9VWa4Y30tp84O8OSe5O01s9XjZU3tDE3tr5ImdOA/2JLRMZnR4v6NGGOeBJ4MO/b1kNv9wM1Rrr0HuCeWMu3j74m3nirzXHVRBQvK83loawubVoTPF5m8k139NJ3u5qbLqhNetlJBwTGZ/iE/PcMtGQ0y4XTFv0q5rCzhs+treOtoJ68fSfxeM8OpZHR9jEoid8QxGW05h9Mgo9LiY6uqKczNTspeMw0eL2X5Li6ZXZTwspUKCh3479WWTFQaZFRa5Odkc+uaefz33pMc7+xLWLnGGBo8XtbVlZOlqf1VEuVkZyEC/YP+4QzjuuJ/NA0yKm0+s3Y+xhge2X44YWUeONVN27kBnbqskk5EhjcuGx7415bMKBpkVNpUl+Zx3dLZ/NfOI/QOJmavmQZ7PGa9jseoFHC7rCDTa+cv0zGZ0TTIqLS6Y0MtXX1D/OqNYwkpr6GpjQXl+VSVuBNSnlJjcTsd9A8F6BnwkZ0luBz6kRpOfyMqrVbPL2VZVTEPNRya9F4zg74AOw51aCoZlTK5ziyru2zA2hXTWi+uQmmQUWklItyxoYbmth622F1d8XrjyBl6B/06dVmljNvlsAb+B/262j8KDTIq7T6wrJKKwpxJT2fe6vGSJXClpvZXKRIc+O8d9Gnesig0yKi0c2Vn8ekr5/PSgTY8p7vjLmeLx8vyuSUUu50JrJ1S0QUH/nsGtCUTjQYZlRE+ecU8XNlZPLwtvtbM2f4h3jrayQYdj1EplBsy8K95yyLTIKMyQnlBDh9ZUckvXjtGZ+/E95rZ3txOwKBBRqWUNbvMGpPRXTEj0yCjMsbt62vpG/Lz2KtHxz85TEOTlzyXg5XzNLW/Sh2302Gl+h/06Wr/KDTIqIxxyZwi1i4o45FtLfgmuNfMVo+XK2pn4MrWP2mVOqFjMtqSiUzfkSqj3LGhluNd/TzdeCrma4519nHQ26PrY1TK5YSskynQlkxEGmRURnnP4pnML8vjoa2xTwDY2mStr9m4qCJZ1VIqIrfTwaAvQN+QtmSi0SCjMoojS/jsuhpeO3yGt452xnTNFo+XisIcLppVkOTaKTWSO2TnVR2TiUyDjMo4N11WTUFONj+KoTUTCBi2ebxsqCvXlB4q5dwha2O0JROZBhmVcQpznXx89Vx+t/sEp872j3nuvpNnae8Z1PEYlRa5IS2ZAl0nE5EGGZWRPruuBr8xPDrOXjPDWy1rkFFpEBpk8nTFf0QaZFRGmleWx7WXzOInOw7TP+SPet6WJi+LZhYwuzg3hbVTyjJyTEZbMpFokFEZ6/b1tZzpHeI3b0bea6Z/yM+rLZraX6WPW1sy49IgozLWlQtmcMmcIh5qaMGY0XvNvH74DP1DATZqan+VJm7X+Y9QHZOJTIOMylgiwh3ra9h/6hzbmttHPd7g8ZKdJVyhqf1VmowYk9EgE5EGGZXRPrS8kvICV8S9Zho8XlbOK9FvkCptRozJaHdZRBpkVEbLdTr45BXzeX7/aQ55e4aPd/YOsudYl47HqLQaObtMv+xEokFGZbxPXTmP7Czhx9taho9ta27HGHQ8RqVVsCXjcmRpctYo9LeiMt7Mwlw+tLySx3cdpatvCLC6ygpysrm0uiTNtVMXsuCK/zxNKROVBhk1JdyxvpbeQT9P7LL2mmlo8nLlgjKcDv0TVumTY7de8rWrLCp9h6opob6qmDU1M3h4Wwst3h6OdPSyoU5nlan0EhHcToeukRmDBhk1ZdyxoYbWM31847eNAGzQ1P4qA+Q6s3S1/xjiDjIicr2I7BcRj4jcHeHxHBH5mf34DhGpCXnsa/bx/SJy3XhlikitXUaTXaYr3nqrqevaJbOpLnXz4v42ZhflsrAiP91VUgq306Fp/scQV5AREQfwPeAGYAlwq4gsCTvtTuCMMaYOuA+41752CXALsBS4Hvi+iDjGKfNe4D5jzCLgjF22usAE95oB2LBIU/urzJCXk61jMmOItyWzBvAYYw4aYwaBx4BNYedsAn5s3/45cI1YnwqbgMeMMQPGmEOAxy4vYpn2Ne+xy8Au8yNx1ltNcR+/fC7L55bw0ZVV6a6KUgD87QeX8KX31KW7Ghkr3vBbBRwNud8KXBHtHGOMT0S6gDL7+Cth1wY/MSKVWQZ0GmN8Ec4fQUTuAu6y73aLyP4IpxUDXVFf2XnlgDeG86azWH9XKbc5tXVLxnMloszJlBHPtRO5Rt9nscuk99n8RBcYb5CJ1E8RnsEw2jnRjkdqVY11/uiDxjwAPBDpseFKiTxgjLlrrGBymRwAABG2SURBVHPs83YZY1aPd950FuvvKh1SWbdkPFciypxMGfFcO5Fr9H0Wu0x+nyVCvN1lrcDckPvVwPFo54hINla07hjj2mjHvUCJXUa055qI307i2gtNJv+uUlm3ZDxXIsqcTBnxXDuRazL5byfTTOvflURKoT7uRdYH/gHgGuAY8CrwSWNMY8g5XwSWGWM+LyK3AB81xnxcRJYCP8Uag6kEngMWYbVYIpYpIk8AvzDGPCYi9wO7jTHfj/tVx/YaL/hvWEolm77Ppr+4usvsMZYvAU8DDuAhOxh8E9hljNkMPAg8KiIerBbMLfa1jSLyOPA24AO+aIzxA0Qq037K/wU8JiLfAt6wy062MbvdlFIJoe+zaS6uloxSSikVC13xr5RSKmk0yCillEqacYOMiLhF5CV7RT4i8pSIdIrI7+J5wjFSx7xHRF4Xkb0i8uOQ2WTRypkvIq+JyJsi0igin4+jLrfZqWqaROS2kOO3isgeEdltv94xNy0RkcUisl1EBkTkz2N8vRNOlRMtHU/YORF/jyJSKiK/sl/TThGpt4+7ROTl8X7fKnWS8J57SEROi8jesOMzROQZ+2/wGREpHaecRLznIr6Wib4fRKRMRF4QkW4R+W7YY5fZ71+PiPybiJUaYqKv174m4mdE2DnL7ff/HhH5rYgU2ceXicjD4z3HtGeMGfMH+CLw5ZD71wAfAn433rURynIAzcACwAW8hZVCJgtrIeZF9nnfBO4cpywXkGPfLgBagMoJ1GUGcND+t9S+XYo1GeI0UG6f923gG+OUNRO4HLgH+PPxXq/92OPALfbt+4EvjPMcS+zrc4Bau1xH2DlRf4/Ad4C/s28vBp4Lue7vgD+a6P+n/iTnJ5HvOfv6dwGrgL1hx78N3G3fvhu4d5xyJvWeG+u1xPF+yAc2AJ8Hvhv22E5gLdaM1f8Gbojz9Ub8jIhw3qvAVfbtO4D/HfLYs8C8dP9NpfMnlu6yPwJ+E7xjjHkOOBd+kv3t4SX7m87TIjInQlnR0tGUAQPGmAP2ec8AHxurUsaYQWPMgH03h5BWmYi8z/5m8bqIPCEiBRGKuA54xhjTYYw5Yz/n9Vh/mALk29+AihhnXY4x5rQx5lVgKJbXa5c70VQ50dLxhBrr97gEa7o4xph3gBoRmWU/9mus/2eVGRL5nsMY8zLWDM9woamfxv0bTMB7LuJrief9YIzpMcY0AP1hZc0Biowx2431Kf9ISFkTer1E/4wIdzHwsn07/LPrt9gzay9UYwYZu8m6wBjTMs55TuDfgZuMMZcBD2F9qw8XKR1NFdaCS6eIBOfL38TIhZnRnneuiOy2y7zXGHPc7tr6G+C9xphVwC7gK7HWxRgzBHwB2IMVXJYQ/5TpaK835lQ5MZQVaqzf41vARwFEZA1W+ohq+7G9WC0xlWZJeM+NZZYx5gSA/e/MGOo3mfdcNPG8H6Kpsq8PCi1roq83lvccWO+fD9u3b2bkZ9cuYGNMNZ+mxuuHLwc6YyjnYqAeeMbu/nQAJyKcFzFFjDHGiLVg8z4RyQH+gLWGZkzGmKPApSJSCfxaRH6O9WG5BNhq18UFbI+1Lvab9wvASqzm8b8DXwO+NV59Yn2OMY7HU9b5O2P/Hv8B+FcReRMrgL4RfMwY4xeRQREpNMaM+sasUirR77mEmuR7Lpp43g+ZVNYdwL+JyNeBzcBgyGOnsRadX7DGCzJ9QG4M5QjQaIxZO+KgyFzOp0y4H+vbdMR0NMaY7dgRX0TeB1wUw/NiX3tcRBrt6wewmri3htXlCuAH9t2vY30ruTqsLi8CK+wym+3rHsfqv43HuKly7G9vsaTKiSWVT9TfozHmLHC7fVyAQ/ZPUA5hXQ8qLRL6njPG3D9GGadEZI4x5oTdzXQ61krG854z1iLtSOJ5P0TTyvkWOmFlTfT1RvuMGMHufn4fgIhcBHwg5OFcrP/TC9aY3WV2P6RDRMb7o98PVIjIWrCa8iKy1Bhz1Bizwv65H2uAbJE9k8SF1Ve52b5mpv1vDtYK//vt+2tE5JHwJxSRahFx27dLgfV2PV4B1otInf1YnohcZIzZEVKXzViZBd4n1qyrUqw/kqexUtosEZHgtovXAvvssr4kVlaCWEV8vXZf8QtY3VkAt2H3wYvIjSLyfyOUtRm4RazN4GqxUvHsjPB7ifZ7LJHzM3b+B/CyHXgQkTKgze4qVGmUhPfcWDZj/e3ByL/BZL3nor3meN4P0co6AZwTkSvtL1Of4fz41oReL9E/I0YIec9lYXUbhv7eL8LqTrtwjTczAGs84r0h97cAbVjRuRW4zj6+Amvw6y2gEfjjKOW9HytHWTPw1yHHv4P1Yb4f+NOQ4zcBP4hQzrXAbvv5dgN3hTz2HqwP+N32z4ej1OUOrAF0D3B7yPHP23X5/9s792CrqjqOf75ICoHhCE6a0QDGeOuPfGEpkV1HIzXNrNQcJa9aTr4fU2PWTIMxo6blo8wH8QeUyCiKjWOKDiSCiA8E4lEkKRdzxjGncUjwhfDrj99v37vvuXufc+6Fw+Fe1mfmzN1n3fX8/dbaa+219vn9VuKrwuERfgdwVkE++4cs/odvdbyOHz5Wa+8YfJL4FzCbzrd2fgxcW1Lfn0c+/yTemInwx4i3fKrI8WhgHbAWmEPuLZmQ8W9q9YX02TmfBoy5WfhW2pZIn71xOBx/GWRd/N031x8aNebK2tKb8dCOv9CwKfLK3twch9/YX4kxq960N/5Xdo+YBoyL6ytijL+Mb0srF+8O4JRm96lmfmqalZF0GHC1mU2qGrFBSLoZ+JOZrWxG+RV1eRQ39Plhzci9L+Ne4Coze6tRZRSUOQcfyEX+dxI7mTTmutSl4eOhUe2N3YSngQnW+VLDbkddtssknQ/MsDBkmeg/ZNt4Zla0XZBoEmnM9X0kjcXfWF3Q7Lo0k2QgM5FIJBINI9kuSyQSiUTDSJNMIpFIJBpGmmQSiUQi0TDSJJNIJBKJhrHbTzKSNvUizWOS9ulFuislfXx78ynId7LCxYCkX0o6fnvz3NXoD3raEfRGDiX5tKnCRH6jkdSqXroraCYhq13SNIykH0n6fo04pbqW9LOK78/uyPpBH5xk5DSl3lnZZnaSmdVjX6qSK4GOm9d25FOKmf3CzObtyDzzKHyc1IjTNB3ly9+V9aR+4r+n2breSbSxi9ofM7O7t/PnB10mGTMbv51V6kaf6BySRkn6h6Q7gWXAJBWYFZd0kqS1kp6ROyt6NMI7VvrxfbWkURVlDJU0P/JcJenUkrJHSmqXNCJWESvis17SU5HmLklL5Y6drouwy/GO+lQuXrvCIZqkq6NeqyVdWVH2HyKvJxVmParIarqk7+byvy7XppYIHyJ3ZPWipOUVbV0U8ZdJGh/hrXIHUffhxjXr0dFIlZh/3131FLq5JfL9ldycybOhg2clHRzx2iTNkTv4WifppoK8RoRsv1Hwvz/Lzf+vkXRhLvw8SS9Leho3CZOFnyJ3GLZc0jyFCwhJ+8mdey2TdI+kDVFukay7yTLyOCHTNWEFvIzQ/4yQX7ukb0u6KfQ8V268FknHRV1XRT/eK6en60MuSyUdLneB8IpyDtYk/UTe91fm9F6oQ/lYGgfMjP4zuKI/jJO0oCf1L2j3F+U/iEbSqZLekzsTHCTp1Qg/KPJ4ST5GW3JlZrsYR0ablki6WV2d1H2qsj9JuhEYHO2aGWGb4m+rpAWSHgz9zZQ6nL8Vjt9Smm1yoJ4PMArYBhyFW6ldCAyJ/12DG7wchJvlHh3hswjHSMBkujoTWw2MiutN8XcgnaZgRuBmJJQvO5e+nXBqFt8/hpvLOCW+Z+Yq9sAN6n2hJF17lHUEfvMegjuDWoNbgR6FW0o+NOI/AJxTIJ+O9gHTcfPvWf6XxfXFwLS4vj7LB9gHN4cxBF+9D4rwscDSuG4FNmeyraWjnAyTnrrKaDrwKOFsDvdVNDCujwceius23AL4sJDXBmBkJgfgk8DzwNdKdJG1a3DIcDhwAPAasB9uJXkx4ewLd8iV/WbuB4SJIdwkyrVxfQJugXhEiay7yTKn67Ghoweo4ngt9P9M6OkQ4F06HY49jPt/yfLMHPP9kTCfFHq6KK5vxc3b7B1t/k+ETwSmRn0GhD6OqabDaM+4on6FT0AL6q1/SbsHAuvj+te4eZ4vA18FZkX4fGBsXH8J+GvB2F8NjI/rGwknddToTxV1ycZZK7ARNwo6ALeqPYEq47fs05ce2TeY2XOSTqbYrHgL8Kq5Qy/wxl9YmFMxAq6XdAw+gA7EB3NH2VXS3o4rPbN+e0asIAfig/vzeIcvYwLwsJlthg4zL1/BDfqtN7MVEe8lfDD0hDm5tNlKciLwTXU+NQwCPoNbq71D0qHAVrpawn4hJ9sy8nI6iqSnImZb56/4hwEz5L8MN/zmlDHfzDZGOX/H/f/8O+LMBy4xs6dLyrhc0mlxPRK/ye+P3wzfijzvp1O/nwbul1sm3pNO69wTgNMAzGyupLdzZVTKukiWA0Iu66LMe6mt68fNbIukVfiENTfCV+EyPTjyzBzzzcA9id4W3x/JxR9q7rriHUnvy8/VJsZnecQbGvJ5je0fa/XUvxtm9pHcVfTncEeEt+AT3x7AIvkOwHhgdowlcKvpHUTb9jaz7EzlPuDkXJSy/lSNF8zs9UizIuq/iR6O3740yWyOv6LYrPhhVdJ+RNetwSILt2fjK54jopO05+JtLoiflduGK+zS+D4aN+p3pJm9LffxXcuibpHfiowPctdb8dVpT8jSb6VT3wK+YxW2yiRNBt7EV2ED6Gr6v1QGJXGSnorJ13EK8JSZnSbfFlxQJb9Mdx/hN8Cv43axulZQasWfio42s3djKydrV5l5j98Bt5jZI5F+cpZdSfwu7aghy56aFPkAwMy2SdpisVzGFxQDa9SpI33Ez8swn/4GM7snnyjkX68O8/20ss/Uqn8Zi4ATcSOm8/Cn3j1wuQ7AnbodWiV9vXKBrv2pp2lqldONPnEmU0GhWXHcuvAYde7hn5lL0477OEfS4cDognyH4Y/UWyQdi9+QqiLpCLwTnGNm2yL4E/gA3Cjf2z4xl+Qd/PG9koXAt6ItQ/DV46IaZffU7UCeJ4DLcnus2Y1/GPBGtGUS3sl7S9ITNfU0DHctAb6lUQ+GWwZukdTh50jS2lyeb8cE04I/UYJvr7VKGh5nA6eX1OPcXPgzwBmR/0R8W62IMlmuBUZLOii+n1WUuIdkrsM/G98nUTDZVuEJ4Hx1ng8eqDDVX4XK/tCOb51CDTfxPWAh/sLJknjaHI4/9a8xd8mxXtLpUWdJOiSf2NxFxDuSMn3X6/J5i0rOikqoNn4L6XOTTCigDZgldwP7HNBiZu/h5w5z5YeMb+J7igAPAfvGI99F+BlEJTOBcZKW4qvltQVxKrkU2Bc/JF4haZqZ/Q1/FF+Du8RdnIs/FXhccaCca9MyfOXyAn4zmGZmy6lOC/DfOupYxBR822VlHA5OifA7gXMlPYdvpdTz9FJI0lMH1fR0E3CDpMX0YEKP7bbvAcdKulh+CJ2tMOcCA0PmU3C5Y+5nZTK+ZTkPP7DPmIxvxSzCHYhlXIf7U1mGTxxv4DfcyvoUytLM3se3Uv4Sut5QbxurtP193Pne7NiS2kZX/y210j+JbyUtifQPUrygyDMduDv6zmBcLreHvHaUAdPn8W3fhfF9JbAy9yR0NnCBpMytw6kFeVwATJW0BO8PGwviVDIVvw/MrKeSNcZvIf3KQKakoWa2KVbovwfWmdmtza5XI9BOcDvQKJKedngZJwNjzOy3OzjfvYCtcWZwNHBXjS2bRBPJxlVc/xQ4wMyuaFQ59Y7f/jbJXIU/7u+Jr6x+aGbvNrdWiUqSnvoG8hcSHsB3PD4ELjazF5tbq0QZks4ErsXPTjYAbdYAPzw9Hb/9apJJNB65q+b5Bf86zsx6u32X2I2QdB7uTTLPYjO7pBn12ZlIepjuZ43XmFk3t879hTTJJBKJRKJh9LmD/0QikUj0HdIkk0gkEomGkSaZRCKRSDSMNMkkEolEomH8HzkuDgIzqfyQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "report_results('tuning', scores);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'YaMus': {'FM': {'regularization': 1e-09,\n",
       "   'linear_regularization': 1e-10,\n",
       "   'rank': 100,\n",
       "   'adagrad_momentum_weighting': 0.9}}}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### saving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(f'{experiment_name}_param', config=config, tuning=scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## rank estimation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "rank_config = {}\n",
    "rank_scores = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
       "<progress style=\"width:60ex\" max=\"1\" value=\"1\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">1/1</span>\n",
       "<span class=\"Time-label\">[16:54:07<16:54:07, 60847.11s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[A\u001b[2K\r",
       " [████████████████████████████████████████████████████████████] 1/1 [16:54:07<16:54:07, 60847.11s/it]\u001b[B"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:15ex; max-width:15ex; vertical-align:middle; text-align:right\">YaMus</span>\n",
       "<progress style=\"width:45ex\" max=\"7\" value=\"7\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">7/7</span>\n",
       "<span class=\"Time-label\">[16:54:07<01:35:37, 8692.44s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[2K\r",
       "          YaMus [█████████████████████████████████████████████] 7/7 [16:54:07<01:35:37, 8692.44s/it]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for label in track(data_labels):\n",
    "    model, = prepare_recommender_models(label, data_models,\n",
    "                                        [fm_init_config, config]) # initiate with optimal config\n",
    "    rank_config[label], rank_scores[label] = fine_tune_fm(model, {'rank': fm_ranks[label]},\n",
    "                                                          label, ntrials=0, record_time_as='rank')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAEZCAYAAABFFVgWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8FdX9//HXhxAIsggERDAgqygWQYy4lYpai1oqLqhYrdhS0a97q/ar9Wur/tpv1daibdEWBUG+Kot1wapY9w0FwqasEgEhimyBAAGSkHx+f9yJvYYbcgmZ3IX38/HIg3tnzpx7ZiC875yZOcfcHRERkTA0SHQDREQkfSlkREQkNAoZEREJjUJGRERCo5AREZHQKGRERCQ0ChkREQmNQkakCjN7yszGVVl2qpltMrP2e9muoZm5mX1lZhlRyxsF2+4Os90iyUghI7KnG4FzzOxMADPLAh4DbnH3tXFsvw34QdT7wcDGOm+lSApQyIhU4e6bgBuAMWbWFPgt8Lm7jzezk8zsYzPbYmZrzewvZpZZpYqJwBVR768AnowuYGYFZjYw6v3vzGx88PogM3s6OPvZYmazzKxN3e+pSPgUMiIxuPtUYA7wDDASuDpYtRu4CWgDnAKcFbWu0nPA6WbWwsyygROBf+3Dx/8UOAjIAbKBa4FdtdsTkcRqmOgGiCSx64DPgTvdfTWAu8+OWr/CzMYApwJ/i1q+A3gVuAhoAjwPlOzD55YRCbHu7v4pkFfrPRBJMIWMSDXcfZ2ZbQQWVS4zsyOBB4HjiJxtNARmxtj8SSLdbE2InPnsi/FAB2CKmbUg0v32P+6uGwck5ai7TGTf/ANYSOQsowXwG8BilHsbOBxo6e4fxVhfTCSkKh1a+cLdS939bnc/CvgucD5wWR21X6ReKWRE9k1zoAgoNrOj2PN6DAAemUNjMHBeNfXMB4YFtz33By6oXGFmp5vZd8ysAbCVSPdZeR3ug0i9UciI7JtbgOFEblP+BzC5uoLuvtDdF1ez+k7gSGALcBfwdNS6DkRuHthKpKvuDSI3IIikHNOkZSIiEhadyYiISGgUMiIiEhqFjIiIhEYhIyIioVHIiIhIaNLiif82bdp4586dE90MEZGUMmfOnI3u3jbMz0iLkOncuTN5eRreSURkX5jZF2F/hrrLREQkNAoZEREJjUJGRERCkxbXZEREwlBWVkZBQQG7dqX2nHFZWVnk5OSQmVl1EtfwKWRERKpRUFBA8+bN6dy5M2axZnRIfu7Opk2bKCgooEuXLvX++eouExGpxq5du8jOzk7ZgAEwM7KzsxN2NqaQERHZi1QOmEqJ3AeFjIhIEsvIyKBv377f/KxatYp33nkHM2Ps2LHflJs3bx5mxp/+9KcEtnZPChkRkSTWpEkT5s+f/81P5egmvXv3ZvLk/8yZN2nSJPr06ZOgVlZPISMikoI6derErl27WLduHe7O9OnTOfvssxPdrD3EFTJmdpaZLTOzfDO7Pcb6xmY2OVg/08w6R627I1i+zMwGBcs6mtnbZrbEzBaZ2U1R5e82sy/NbH7wc87+76aISGrauXPnN11l559//rfWDR06lKlTpzJjxgz69etH48aNE9TK6tV4C7OZZQCjgTOBAmC2mU2rMnf5CGCzu3c3s2HA/cAlZtYLGAYcTWTe8jfM7AhgN3CLu881s+bAHDN7ParOUe6eXB2LInJAu+elRSz+amud1tmrQwt++6Oj91qmsrsslosvvphLLrmEpUuXcumllzJjxow6bV9diOdMpj+Q7+4r3L0UmAQMqVJmCDAheP0scIZFbmcYAkxy9xJ3XwnkA/3dfa27zwVw923AEuCw/d8dEZEDx6GHHkpmZiavv/46Z5xxRqKbE1M8D2MeBqyJel8AnFBdGXffbWZFQHaw/OMq234rTIKutWOBmVGLrzezK4A8Imc8m+Nop4hIaGo640iUe++9l/Xr15ORkZHopsQUz5lMrBusPc4ye93WzJoB/wRudvfK89BHgW5AX2At8GDMRpmNNLM8M8vbsGHD3vdARCRNnXzyyZx33nmJbka14jmTKQA6Rr3PAb6qpkyBmTUEDgYK97atmWUSCZin3P25ygLuvq7ytZk9BvwrVqPcfQwwBiA3N7dq6ImIpIXt27fvsWzgwIEMHDhwj+V33313+A3aR/GcycwGephZFzNrRORC/rQqZaYBw4PXQ4G33N2D5cOCu8+6AD2AWcH1mrHAEnf/c3RFZtY+6u35wMJ93SkREUkONZ7JBNdYrgdeAzKAce6+yMzuBfLcfRqRwJhoZvlEzmCGBdsuMrMpwGIid5Rd5+7lZvZd4CfAp2ZWedvEr939FeABM+tLpFttFXB1He6viIjUo7hGYQ7+83+lyrLfRL3eBVxUzba/B35fZdkHxL5eg7v/JJ42iYhI8tMT/yIiexHp+U9tidwHhYyISDWysrLYtGlTSgdN5XwyWVlZCfl8TVomIlKNnJwcCgoKSPXHJCpnxkwEhYyISDUyMzMTMptkOlF3mYiIhEYhIyIioVHIiIhIaBQyIiISGoWMiIiERiEjIiKhUciIiEhoFDIiIhIahYyIiIRGISMiIqFRyIiISGgUMiIiEpq0GCDzs3XbOOPBd2q9fdPGDclu2ojWTRvTplkjsps1Irtp42/92bppI7IyM+qu0SIiB4C0CJmszAyObN+idhs7bC/ZzcbtpSz7ehsbi0sp3V0Rs2jzxg2/CZzsZkEgRYVQm2b/CaZWB2XSMEMniiKJUF7hvLboa07p3oaDm2QmujkHtLQImU6tD2L0j/vVSV3uzvaS3WzaXsqm4lI2bS+p8mcpm4pLWFO4g3mrt7B5RynlFXtOaGQGLZtkkt2sMdlBAEXCKQioIKgi4dSIFlmZNGgQc0ZqEdkHJbvLuemZ+Uxf9DU5rZow+sf96NOxZaKbdcBKi5CpS2ZG86xMmmdl0rlN0xrLV1Q4RTvL2FRcwsbtkRAqrHxdXPJNWC39eiubikvZsqMsZj0NGxitm+55RhT5MxJI2c0a0SZYdlCjDMwUSiLRtpfs5uqJeXyYv4mR3+vKy5+sZejfZ/Drc47iypM763cmARQy+6lBA6NV00a0atqI7ofUXL6svILNxaXfOivaGATTpu2l34TT6tU7KCwuZXvJ7pj1NG7YICqMdD1JZHNxKVeOn83CL4t48KI+XHhcDtcO7MatUz/hnpcW8/GKTTwwtI+6z+qZpfLc1ZVyc3M9Ly8v0c0Ixa6y8j2764LXG7eXUBi1rDbXkyq78HQ9SVLZ10W7+MnYmXxRuIPRP+7Hmb3afbPO3Rn7wUrue3Up7Vtm8bdL1X1WyczmuHtuqJ+hkEkfldeTCouDM6IgjAqDQNpUpQuvsLj215Oiw0rXkySRVm4s5vLHZ1K0s4zHh+dyYtfsmOXmrt7MDU/PY/22Xdx5zlEMV/eZQiZeCpnaqXo9qTA4Y6rssqsaVnVxPal1s0Y01fUkqSMLvyziyidm4Q4Tftaf7xx28F7Lb9lRyq1TF/DGkvWcdfSh3D/0mAO6+0whEyeFTP0oK69g847K7rnos6JvX0+q7MLT9SQJ06yVhYwYP5sWTTKZOKI/Xds2i2u7qt1no3/cj2NyDszuM4VMnBQyyWlv15O+FVJxXE9qHXVWpOtJ8uaSdVz71FxyWjVh4ogT6NCyyT7XMeeLzdzw9Fw2bC85YLvPFDJxUsikPnenuLT8P911dXQ96VtnR80a061NU07qln3A/WeSTl6Y9yW3TF3A0R1aMP6n/WndtFGt69qyo5RbpizgzaXrOfs7ke6zFlkHTveZQiZOCpkDT/T1pOgHZ7+5tlRcUu31pGM7teS/zzqy2gvEkrzGf7iSu19azMndshlzRS7NGu//UxjuzuPvr+T+6Uvp0DLy8GbvnL1f20kXCpk4KWSkJpXXk95csp6H31jO11t3ceoRbbltUM8aLxZL4rk7D72xnIffXM4PerXjL5ceW+fX6iq7zzZuL+XOHx7FFScdnvZnvAqZOClkZF/sKivnyY9WMfrtzynaWcaP+nTgljOPiGuEB6l/FRXOPS8tYsJHX3DRcTn84YLeoV1321xcyi1TF/DWAdJ9ppCJk0JGaqNoZxmPvbeCsR+spKy8gkuO78iNZ/SgXYusRDdNAmXlFdw2dQEvzP+KqwZ04dfnHBX62UVFhfP4Byu4f/oyDkvz7rP6CJm4vg6Y2VlmtszM8s3s9hjrG5vZ5GD9TDPrHLXujmD5MjMbFCzraGZvm9kSM1tkZjdFlW9tZq+b2fLgz1b7v5siezq4SSa3DurJu78ayI9P6MTk2Ws49Y9vc//0pRRV80yQ1J+dpeVcPXEOL8z/il+d1bNeAgYiQ0WN/F43plx9IrvLK7jw0Rk8+dEq0uELeSLUeCZjZhnAZ8CZQAEwG7jU3RdHlbkWOMbdrzGzYcD57n6JmfUCngH6Ax2AN4AjgEOA9u4+18yaA3OA89x9sZk9ABS6+31BoLVy9//eWxt1JiN1YfWmHfz59WW8uOArmjduyH8N7M6VJ3emSSM9p1PfinaW8fMJs8n7YjO/P683Pz6hU0LaEd19dk7vQ7nvwvTqPkuWM5n+QL67r3D3UmASMKRKmSHAhOD1s8AZFvnKMQSY5O4l7r4SyAf6u/tad58L4O7bgCXAYTHqmgCcV7tdE9k3nbIP4qFhx/LyDQPI7dya+6cv5dQ/vs1TM7+grDz2MzxS9zZsK2HYmI+Zv2YLf7302IQFDECrpo14/Ipc7jj7SF5btI7Bf/mAhV8WJaw9qSiekDkMWBP1voD/BMIeZdx9N1AEZMezbdC1diwwM1jUzt3XBnWtJXLWI1JvenVowbgrj2fK1SfRqfVB3Pn8Qs7887u8tOArKmI8myN1Z03hDi76+wxWbSzm8eHHM/iYDoluEg0aGFefGuk+Kyuv4IJHZjDxI3WfxSuekInVCVr16FZXZq/bmlkz4J/Aze6+NY62/OcDzUaaWZ6Z5W3YsGFfNhWJS/8urZl6zUmMHZ5L44YZ3PDMPH70tw9497MN+g8mBJ+t28bQv89g844y/u/nJ3DqEW0T3aRvOe7w1rxy4wBO6Z7NXS8u4rqn57J1l67d1SSekCkAOka9zwG+qq6MmTUEDgYK97atmWUSCZin3P25qDLrzKx9UKY9sD5Wo9x9jLvnuntu27bJ9Y9R0oeZccZR7XjlpgGMuqQPRTvLGD5uFpc+9jFzV29OdPPSxrzVm7n4Hx/hDlOuPonjDk/O+31aNW3E2OHHq/tsH8QTMrOBHmbWxcwaAcOAaVXKTAOGB6+HAm955KveNGBYcPdZF6AHMCu4XjMWWOLuf95LXcOBF/d1p0TqWkYD4/xjc3jrloHcc+7R5K/fzgWPzGDkk3l8tm5bopuX0j5YvpHLHp/JwU0y+ed/nUzPQ5snukl7Vdl9Nnmkus/iEddzMmZ2DvAQkAGMc/ffm9m9QJ67TzOzLGAikWsrhcAwd18RbHsn8DNgN5FusVfN7LvA+8CnQOUV1V+7+ytmlg1MAToBq4GL3L1wb+3T3WVS34pLdjPug5WMeW8FxaW7uaBfDjd/vwc5rQ5KdNNSyqufruWmSfPp2rYpT47ozyHNU+sZpcLiUm6ZMp+3l23gh73b84cLe6fU3Wd6GDNOChlJlMLiUh59J58JH30BDpefeDjXndaN7GaNE920pDdp1mp+/fyn9OvUirHDj+fgg1LnP+doFRXOmPdX8MfXlpHTKvLwZqoMVaSQiZNCRhLtqy07efiN5Uyds4YmmRlc9b2u/HxA1zoZwDEd/f3dz7nv1aUM7NmWRy87Li2eRcpbVcj1T8+jsLiUuwYfxeUnJv/YZwqZOClkJFnkr9/Og/9exqsLv6Z100Zcd1p3Lj+xE40bpv5/onXB3blv+lL+8e4Kzu3TgT9d1IdGDdNn/p/C4lJ+OWU+7yzbwA+Pac99F/SmeRJ3nylk4qSQkWSzYM0WHnhtKR/mb+Kwlk24+fs9uKBfDhkNkvubbZjKK5w7n/+USbPX8JMTD+eec4+mQRoej4oK5x/vreBP/07+7jOFTJwUMpKsPli+kQdeW8onBUX0OKQZtw7qyQ96tUv6bpS6VrK7nJsnzefVhV9z4+nd+cWZR6T9MZi9qpAbKrvPftSLy0/olHT7rJCJk0JGkpm7M33h1/zx38tYsaGYvh0jk6ad1O3AmDStuGQ3V0+cwwf5G7lrcC9GfLdLoptUbwqLS/nF5Pm8+1lydp8pZOKkkJFUsLu8gn/OLWDU65FJ0753RFt+leaTpm0uLuXK8bNZ+GURD1x4DBcel5PoJtW7igrn7+99zoP//oyOrZrwtyTqPlPIxEkhI6lkV1k5Ez/6gtHv5LNlRxmDj2nPLT/oSZc0mzTt66Jd/GTsTL4o3MHoH/fjzF7tEt2khJq1spAbn5lH4Y5SfjO4F5clQfeZQiZOChlJRVt3RSZNe/z9lZQGk6bdlCaTpq3cWMzlj8+kaGcZjw/P5cSuB0bXYE02bS/hl1MW8O5nGzi3TwdGXdI3oTeDKGTipJCRVLZ+2y5Gv5XP07NWk9HAuPLkLvzXqd1S9uHEhV8WceUTs3CHCT/rnzRdQ8miosL561v5jHrjM24b1JPrTuuesLYoZOKkkJF0sHrTDka98RkvzP+S5o0bcs3Abvz05C4p9aDirJWFjBg/m+ZZDZn48xPo1rZZopuUlNydG56Zx/SFX/P8tackbHpnhUycFDKSTpas3cqfXlvGm0vXc0jzxtx4Rg8uOb4jmRnJ/dDim0vWce1Tc8lp1YSJI06gQ8smiW5SUivaUcagh97joMYZvHzDgIR8mUiWmTFFpB4d1b4FY688nqnXRCZN+58XFvL9P7/LtCSeNO2FeV8ycuIceh7anKnXnKyAicPBB2Xy4MV9WLGhmP99ZUmimxMahYxIkjq+c2TStHFX5tIkM4Mbn5nH4L9+wDvL1ifVsPLjP1zJzZPn079za56+6kRaN22U6CaljFO6t2HEd7sw8eMveHtpzKmzUp5CRiSJmRmnH9mOV24cwEOX9GVbSRlXPjGbYWM+Zs4XiZ00zd0Z9fpn3P3SYn7Qqx1P/PR4DQhaC7cN6knPds257dlP2LS9JNHNqXMKGZEU0KCBcd6xh/HmLwdy75Cj+XxDMRc+OoOrEjRpWkWFc89Li3n4zeVcdFwOj1zWj6zM1LlBIZlkZWbw0LC+bN1Zxu3PfZpUZ6l1QSEjkkIaNWzAFSd15t3bBnLrD47g4883Meih97hlygIKNu+olzaUlVfwyynzGT9jFVcN6MIDQ4+hYZLflJDsjmrfgtsG9eT1xeuYkrcm0c2pU7q7TCSFbS4u5dF3P2f8jFXgcNmJnbjutO60CWnStF1l5Vz71FzeWrqe2wb15NqB3RL+1Hq6qKhwLnt8JgsKtvDKjQPoXA8jQOgW5jgpZORAt7YoMmnalLzIpGk/H9CVnw/oUqeDMRbtLOOqCXnM/qKQ3533HS474fA6q1sivtqyk7Meeo+ubZvx7DUnhX6GqFuYRSQu7Q9uwn0XHsO/f3Eqp/Zsy8NvLufUP77D2A9WsqusfL/r37CthEvHfMy8NZv566XHKmBC0qFlE353fm/mr9nC6Lc/T3Rz6oRCRiSNdD+kGY9cdhzTrj+Fozu04P/9azFnPPguU/PWUF7LZ2zWFO7gor/PYOXGYh4ffjyDj+lQx62WaOf26cB5fTvwl7eWM291Yu8grAvqLhNJYx/mb+SB6UtZUMtJ05av28blY2eys7ScJ37an+MObxVyiwUiXZPnPPw+mRnGyzcOoGlIt4aru0xE9ssp3dvwwnWn8Ohl/Sh35+qJczj/kRl89PmmGredt3ozF/3jI9xhyjUnKWDq0cFNIqMBfFG4g9+9nNqjAShkRNKcmXF27/b8++bvcf+FvVm3dReXPvYxV4ybxcIvi2Ju88HyjVz2+ExaZGXy7DUnc+ShLeq51XJi12xGfq8rz8xazeuL1yW6ObWm7jKRA0xNk6a9+ulabpo0n65tm/Lkz/pzSBrMb5OqSnaXc/7oGazbuovpN3+Pts3r9tZ03cIcJ4WMyL6rOmnaxbkd6da2Kf/7yhKO7dSKccOPT9k5bdLJZ+u2MfivH/Dd7m0YOzy3Tp9L0jUZEQlNi6xMbvlBT9771WlcfkInnp2zht+9vIQBPdoycUR/BUySOKJdc+44+0jeWrqep2auTnRz9pnOZEQEiNyq/GH+Ri7ol0Ojhvr+mUwqKpzhT8xi9qpCXr5xQJ1NBqczGRGpNx1bH8Sw/p0UMEmoQQPjTxf1ISszg19Mnk9ZeUWimxQ3/WsSEUkB7Vpk8b/n9+aTgiL+8ubyRDcnbgoZEZEUcU7v9gw9LofRb+cz54vCRDcnLgoZEZEU8tsf9aJDyybcPHk+20t2J7o5NYorZMzsLDNbZmb5ZnZ7jPWNzWxysH6mmXWOWndHsHyZmQ2KWj7OzNab2cIqdd1tZl+a2fzg55za756ISHppnpXJqEv68uXmndwzbVGim1OjGkPGzDKA0cDZQC/gUjPrVaXYCGCzu3cHRgH3B9v2AoYBRwNnAY8E9QGMD5bFMsrd+wY/r+zbLomIpLfjO7fm2oHdmTqngOkL1ya6OXsVz5lMfyDf3Ve4eykwCRhSpcwQYELw+lngDIs8MTQEmOTuJe6+EsgP6sPd3wNSo1NRRCTJ3PT9HvQ+7GBuf+5T1m3dlejmVCuekDkMiJ4PtCBYFrOMu+8GioDsOLeN5Xoz+yToUtOofCIiVWRmNGDUJX3ZVVbOrVMXUFHLqRzCFk/IxBrDoOreVFcmnm2rehToBvQF1gIPxmyU2UgzyzOzvA0bNtRQpYhI+ul+SDPu/GEv3l++kSc/WpXo5sQUT8gUAB2j3ucAX1VXxswaAgcT6QqLZ9tvcfd17l7u7hXAYwTdazHKjXH3XHfPbdu2bRy7ISKSfi4/oROn9WzLH15dyvJ12xLdnD3EEzKzgR5m1sXMGhG5kD+tSplpwPDg9VDgLY+MVzMNGBbcfdYF6AHM2tuHmVn7qLfnAwurKysicqAzM+4fegxNGzfkpknzKd2dXKMB1BgywTWW64HXgCXAFHdfZGb3mtm5QbGxQLaZ5QO/BG4Ptl0ETAEWA9OB69y9HMDMngE+AnqaWYGZjQjqesDMPjWzT4DTgF/U0b6KiKSlQ5pncd8FvVm8dit/fv2zRDfnWzRApohImrj9n58wOW8Nk646kRO6ZtdYXgNkiohI3O4a3ItOrQ/il1MWsHVXWaKbAyhkRETSRtPGDRl1SV++3rqL376YHKMBKGRERNJIv06tuP607jw/70teWrDXm3nrhUJGRCTNXH96d/p2bMmdz3/K2qKdCW2LQkZEJM1Ujgawu8ITPhqAQkZEJA11adOUuwb34sP8TYz7cGXC2qGQERFJU8OO78j3j2rHA9OXsfTrrQlpg0JGRCRNmRn3X9ibFk0yuXnSfHaVldd7GxQyIiJpLLtZYx4Y2pulX2/jwX8vq/fPV8iIiKS5049sx+UnduKx91cyI39jvX62QkZE5ABw5zm96NqmKbdMXUDRjvobDUAhIyJyAGjSKIOHhvVlw7YS7nzhU+pr3EqFjIjIAeKYnJbc/P0e/OuTtbw4v35GA1DIiIgcQK45tRvHHd6Ku16on6m6FDIiIgeQhhkNGHVxXyrUXSYiImHolH0Q/7z25Hr5LIWMiMgB6MhDW9TL5yhkREQkNAoZEREJjUJGRERCo5AREZHQKGRERCQ0ChkREQmNQkZEREKjkBERkdAoZEREJDQKGRERCY1CRkREQqOQERGR0ChkREQkNAoZEREJjUJGRERCE1fImNlZZrbMzPLN7PYY6xub2eRg/Uwz6xy17o5g+TIzGxS1fJyZrTezhVXqam1mr5vZ8uDPVrXfPRERSaQaQ8bMMoDRwNlAL+BSM+tVpdgIYLO7dwdGAfcH2/YChgFHA2cBjwT1AYwPllV1O/Cmu/cA3gzei4hICornTKY/kO/uK9y9FJgEDKlSZggwIXj9LHCGmVmwfJK7l7j7SiA/qA93fw8ojPF50XVNAM7bh/0REZEkEk/IHAasiXpfECyLWcbddwNFQHac21bVzt3XBnWtBQ6Jo40iIpKE4gkZi7HM4ywTz7a1YmYjzSzPzPI2bNhQF1WKiEgdiydkCoCOUe9zgK+qK2NmDYGDiXSFxbNtVevMrH1QV3tgfaxC7j7G3XPdPbdt27Zx7IaIiNS3eEJmNtDDzLqYWSMiF/KnVSkzDRgevB4KvOXuHiwfFtx91gXoAcyq4fOi6xoOvBhHG0VEJAnVGDLBNZbrgdeAJcAUd19kZvea2blBsbFAtpnlA78kuCPM3RcBU4DFwHTgOncvBzCzZ4CPgJ5mVmBmI4K67gPONLPlwJnBexERSUEWOeFIbbm5uZ6Xl5foZoiIpBQzm+PuuWF+hp74FxGR0ChkREQkNAoZEREJjUJGRERCo5AREZHQKGRERCQ0ChkREQmNQkZEREKjkBERkdAoZEREJDQKGRERCY1CRkREQqOQERGR0ChkREQkNAoZEREJjUJGRERCo5AREZHQKGRERCQ0ChkREQmNQkZEREKjkBERkdAoZEREJDQKGRERCY1CRkREQqOQERGR0ChkREQkNAoZEREJjUJGRERCo5AREZHQKGRERCQ0ChkREQlNXCFjZmeZ2TIzyzez22Osb2xmk4P1M82sc9S6O4Lly8xsUE11mtl4M1tpZvODn777t4siIpIoDWsqYGYZwGjgTKAAmG1m09x9cVSxEcBmd+9uZsOA+4FLzKwXMAw4GugAvGFmRwTb7K3O29z92TrYPxERSaB4zmT6A/nuvsLdS4FJwJAqZYYAE4LXzwJnmJkFyye5e4m7rwTyg/riqVNERFJcPCFzGLAm6n1BsCxmGXffDRQB2XvZtqY6f29mn5jZKDNrHEcbRUQkCcUTMhZjmcdZZl+XA9wBHAkcD7QG/jtmo8xGmlmemeVt2LAhVhEREUmweEKmAOgY9T4H+Kq6MmbWEDgYKNzLttXW6e5rPaIEeIJI19oe3H2Mu+e6e27btm3j2A0REalv8YTMbKCHmXUxs0ZELuRPq1KVL54fAAAIlUlEQVRmGjA8eD0UeMvdPVg+LLj7rAvQA5i1tzrNrH3wpwHnAQv3ZwdFRCRxary7zN13m9n1wGtABjDO3ReZ2b1AnrtPA8YCE80sn8gZzLBg20VmNgVYDOwGrnP3coBYdQYf+ZSZtSXSpTYfuKbudldEROqTRU44Ultubq7n5eUluhkiIinFzOa4e26Yn6En/kVEJDQKGRERCY1CRkREQqOQERGR0ChkREQkNAoZEREJjUJGRERCo5AREZHQKGRERCQ0ChkREQmNQkZEREKjkBERkdAoZEREJDQKGRERCY1CRkREQqOQERGR0ChkREQkNAoZEREJjUJGRERCo5AREZHQKGRERCQ0ChkREQmNQkZEREKjkBERkdAoZEREJDQKGRERCY1CRkREQqOQERGR0ChkREQkNAoZEREJjUJGRERCo5AREZHQxBUyZnaWmS0zs3wzuz3G+sZmNjlYP9PMOketuyNYvszMBtVUp5l1CepYHtTZaP92UUREEqXGkDGzDGA0cDbQC7jUzHpVKTYC2Ozu3YFRwP3Btr2AYcDRwFnAI2aWUUOd9wOj3L0HsDmoW0REUlA8ZzL9gXx3X+HupcAkYEiVMkOACcHrZ4EzzMyC5ZPcvcTdVwL5QX0x6wy2OT2og6DO82q/eyIikkjxhMxhwJqo9wXBsphl3H03UARk72Xb6pZnA1uCOqr7LBERSREN4yhjMZZ5nGWqWx4r3PZWfs9GmY0ERgZvt5vZsljl4tQG2Lgf20vqOJjIlyCJLd2OTzLvTzK0rUfYHxBPyBQAHaPe5wBfVVOmwMwaEjl4hTVsG2v5RqClmTUMzmZifRYA7j4GGBNH+2tkZnnunlsXdUlyM7Mx7j6y5pIHpnQ7Psm8P8nQNjOrk/9D9yae7rLZQI/grq9GRC7kT6tSZhowPHg9FHjL3T1YPiy4+6wLkdScVV2dwTZvB3UQ1Pli7XdPZA8vJboBSS7djk8y708ytC30Nljk//UaCpmdAzwEZADj3P33ZnYvkOfu08wsC5gIHEvkDGaYu68Itr0T+BmwG7jZ3V+trs5geVciNwK0BuYBl7t7SR3uc6z905mMiEgI4gqZdGdmI4PuNxERqUMKGRERCY2GlRERkdCkdciYWRMzezcYZWC6mW0xs39VKVPjMDZmNtjM7qm/lkuyMbOmZjbBzB4zs8sS3Z5kk07HJ532JQz7enzSOmSI3HDwnLuXA38EfhKjTDzD2LwMnGtmB4XWUqmRmXU0s7fNbImZLTKzm/ajrnFmtt7MFsZYF2tcvQuAZ939KuDc2n5umMwsy8xmmdmC4PjU+otRshyf4AvivKpfDvexjqTYl7pmZi3N7FkzWxr8TpxUy3pCPT7pHjKXEdwC7e5vAtuiV8Y7jE1wa/U7wOAQ2yo12w3c4u5HAScC11UdR8/MDjGz5lWWdY9R13gi4+l9y17G1cvhP6NUlO/nfoSlBDjd3fsAfYGzzOzE6AIpeHxuApbEWpGC+1LXHgamu/uRQB+qHKdkOT5pGzJBt1dXd1+1l2L7MoxNHjCg7loo+8rd17r73OD1NiK/VFX/vk4FXgxuq8fMrgL+EqOu94jcbl9VdWP1FRD55YIk/b3xiO3B28zgp+qdPSlzfMwsB/gh8Hg1RVJmX+qambUAvgeMBXD3UnffUqVYUhyfpDyAdaQNUPWgVxX3MDbAeqDDfrVI6oxFppM4FpgZvdzdpwLTgUlBf/HPgIv3oerqxtV7DrjQzB4lOR6iiynoXppP5N/r6+6eysfnIeBXQEWslSm2L3WtK7ABeCLoTnzczJpGF0iW4xPPsDKpaieQVUOZuIexCeraWYftk1oys2bAP4k83Lu16np3f8DMJgGPAt2ivt3HVX2MZe7uxcBPa9XgehRcf+xrZi2B583sO+6+sEqZpD8+ZjYYWO/uc8xsYHXlUmFfQtIQ6Afc4O4zzexh4HbgruhCyXB80vZMxt03AxmVp4rVlKl2GBszO9/M/hBV/AhgjwtjUr/MLJNIwDzl7s9VU2YA8B3geeC3+/gR8YzVl/SCrpN3iN3XngrH5xQiN9usItJNc7qZ/V/VQimyL2EoAAqizlSfJRI635IUx8fd0/aHSH/l94PX7xM5vdwZHLxBwfKuRMZTywemAo2D5bcCd0TV9S+gd6L36UD+IfLN6kngob2UORZYCnQj8iXqaeB31ZTtDCyssqwhsALoAjQCFgBHJ3rf4zw+bYGWwesmwb/5wal+fICBwL/0d73HPr0P9Axe3w38MRmPT8IPVMh/CccCE2u57f8BbYPX7YA3E70/B/oP8F0i18w+AeYHP+dUKXNK9JcBIhe/r4pR1zPAWqCMyJeOEVHrzgE+Az4H7kz0fu/D8TmGyHh/nxA56/5NjDIpd3z2EjIpty91fFz6Erkh6RPgBaBVMh6ftB9Wxsx+BkzwSF91bes4Hihz9/l11zIRkfSX9iEjIiKJk7YX/kVEJPEUMiIiEhqFjIiIhEYhIyIioVHIiIhIaBQyIglgZneb2a2JbodI2BQyIvvJIvS7JBKDfjFEasHMOgcTRT0CzAXGmlle1cnCzGyVmd1jZnPN7FMzOzJGXVeZ2atm1qQ+90GkPihkRGqvJ/Ckux9LZDK1XCJDu5xqZsdEldvo7v2IjIT7rS4yM7se+BFwnrtrlG9JOwoZkdr7wt0/Dl5fbGZziYwddjSRmQQrVY4WPYfIQISVfkJk1sEL3b0k5LaKJIRCRqT2igHMrAuRM5Qz3P0Y4GW+PZdRZYCU8+05nBYSCZ0cRNKUQkZk/7UgEjhFZtaOyNlJPOYBVwPTzEyzrkpaUsiI7Cd3X0AkMBYB44AP92HbD4icBb1sZm3CaaFI4mgUZhERCY3OZEREJDQKGRERCY1CRkREQqOQERGR0ChkREQkNAoZEREJjUJGRERCo5AREZHQ/H+ZJMPOk6Ko3wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "report_results('rank', {lbl: v.sort_index() for lbl, scr in rank_scores.items() for k, v in scr.items()});"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'YaMus': {'FM': {'rank': 1}}}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rank_config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### saving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(f'{experiment_name}_rank', config=rank_config, tuning=rank_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
       "<progress style=\"width:60ex\" max=\"1\" value=\"1\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">1/1</span>\n",
       "<span class=\"Time-label\">[58:14<58:14, 3493.68s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[A\u001b[2K\r",
       " [████████████████████████████████████████████████████████████] 1/1 [58:14<58:14, 3493.68s/it]\u001b[B"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:15ex; max-width:15ex; vertical-align:middle; text-align:right\">YaMus</span>\n",
       "<progress style=\"width:45ex\" max=\"5\" value=\"5\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">5/5</span>\n",
       "<span class=\"Time-label\">[58:14<12:01, 698.73s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[2K\r",
       "          YaMus [█████████████████████████████████████████████] 5/5 [58:14<12:01, 698.73s/it]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "result = {}\n",
    "for label in track(data_labels):\n",
    "    models = prepare_recommender_models(label, data_models, [fm_init_config, config, rank_config])\n",
    "    result[label] = ee.run_cv_experiment(models,\n",
    "                                         fold_experiment=ee.topk_test,\n",
    "                                         topk_list=topk_values,\n",
    "                                         ignore_feedback=True,\n",
    "                                         iterator=lambda x: track(x, label=label))\n",
    "    save_cv_training_time(experiment_name, models, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHJVJREFUeJzt3X2Q1dWd5/H3h+ZJNz5imxBA6Y2dUfABXCROYiUGoqKmgpnSpKlswqZI4VRw19SamWBm1wdKjEwerNoanBoSGRljBERNuiYklqKuayoCTSSEB6l0ALVHFhEQdVYwjd/94x7xern39K8f4PLweVV18fud3znnnnOL5sPv4Z6riMDMzKyWfvUegJmZHd4cFGZmluWgMDOzLAeFmZllOSjMzCzLQWFmZlkOCjMzy3JQmFWQ9ICk+RVln5G0Q9LQTLv+kkLSK5IaysoHpradB3PcZgeLg8LsQP8NuErSZQCSBgM/Bm6KiK0F2r8JXF62/3ngtT4fpdkh4qAwqxARO4D/CsyT9B+AW4E/RcR9kv5S0nOSXpe0VdL/kjSgoov7ga+V7X8N+JfyCpI6JF1atn+HpPvS9vGSfpbOQl6XtELSaX0/U7NiHBRmVUTEQ8Aq4EFgOnB9OtQJ3AicBnwKmFR27D2PABMknShpCHAx8K/dePmvA8cDw4EhwDeBPT2biVnv9a/3AMwOYzOAPwF/FxEvAUTEyrLjmyTNAz4D/ENZ+f8DfgVcBxwHPArs7cbr/plSEJ0VEX8A2no8A7M+4KAwqyEitkl6DVj3Xpmks4EfAv+J0v/6+wPLqzT/F0qXrI6jdAbSHfcBHwUWSzqR0qWs/xERvhludeFLT2bd80/AWkr/2z8RuAVQlXpPAWcCJ0fEb6sc/3dKQfOej7y3ERHvRMRtEXEOcAnwReArfTR+s25zUJh1zwnAbuDfJZ3DgfcnAIjS+v2fB66p0c9qoCU9Ujse+Kv3DkiaIOlcSf2ANyhditrXh3Mw6xYHhVn33ARMpfQI7D8Bi2pVjIi1EbG+xuG/A84GXgf+J/CzsmMfpXRD/A1Kl72eoHRT3awu5C8uMjOzHJ9RmJlZloPCzMyyCgWFpEmSNkpqlzSzyvFBkhal48sljSw7dnMq3yjpilQ2OH3a9PeS1km6vaz+fZI2S1qdfsb0fppmZtZTXX6OIi1uNhe4DOgAVkpqrbhJNw3YFRFnSWoB5gBfljQKaAFGU7pB94Skj1P68NGEiHgrLX/wrKRfRcRzqb+/iYglfTVJMzPruSIfuBsPtEfEJgBJC4HJQHlQTAZuS9tLgH+QpFS+MCL2ApsltQPj03Plb6X6A9JPj++qn3baaTFy5MieNjczOyatWrXqtYho7KpekaAYBrxctt8BfKJWnYjolLSb0ho1w4DnKtoOg/1nKquAs4C5EVH+6dbZkm4BlgEzU9B8gKTplNbg4YwzzqCtzascmJl1h6QXi9Qrco+i2qdOK//3X6tOzbYRsS8ixlBa+Gy8pHPT8ZspPV9+EXAq8J1qg4qIeRExLiLGNTZ2GYhmZtZDRYKiAxhRtj8ceKVWHUn9gZOAnUXaRsTrwNOUVuEkIrZGyV7gnyld+jIzszopEhQrgWZJTZIGUro53VpRp5XSp1UBrgWeTEsYtFJapmCQpCagGVghqVHSyQCSjgM+B7yQ9oemP0Vp+YO1vZmgmZn1Tpf3KNI9hxuAx4AGYH5ErJM0C2iLiFbgXuD+dLN6J6UwIdVbTOnGdycwIyL2pTBYkO5T9AMWR8R76/U/IKmR0mWr1cBf92Rif/7zn+no6GDPniN3Gf/BgwczfPhwBgyo/F4cM7ND56hYwmPcuHFReTN78+bNnHDCCQwZMoTSycmRJSLYsWMHb775Jk1NTfUejpkdhSStiohxXdU7aj+ZvWfPniM2JAAkMWTIkCP6jMjMjg5HbVAAR2xIvOdIH7+ZHR2O6qAwM7Pec1D0sZEjR/Laa6/1uo6Z2eHC35ltZtZNI2f+st5DYMtdVx+y1/IZBbBlyxbOPvtsvvGNb3Duuefyla98hSeeeIJPfepTNDc3s2LFCnbu3Mk111zD+eefz8UXX8yaNWsA2LFjB5dffjljx47l+uuvp/wpsp/+9KeMHz+eMWPGcP3117Nvn7/N0syOPA6KpL29nRtvvJE1a9bwwgsv8LOf/Yxnn32WH/zgB9x5553ceuutjB07ljVr1nDnnXfyta99DYDbb7+dSy65hOeff54vfOELvPTSSwBs2LCBRYsW8Zvf/IbVq1fT0NDAAw88UM8pmpn1iC89JU1NTZx33nkAjB49mokTJyKJ8847jy1btvDiiy/y8MMPAzBhwgR27NjB7t27eeaZZ3jkkUcAuPrqqznllFMAWLZsGatWreKiiy4C4O233+b000+vw8zMzHrHQZEMGjRo/3a/fv327/fr14/Ozk769z/wrXrv8dVqj7FGBFOnTuV73/veQRqxmdmh4UtPBX3605/ef+no6aef5rTTTuPEE0/8QPmvfvUrdu3aBcDEiRNZsmQJr776KgA7d+7kxRcLrehrZnZY8RlFQbfddhtf//rXOf/88zn++ONZsGABALfeeitTpkzhwgsv5DOf+QxnnHEGAKNGjeKOO+7g8ssv591332XAgAHMnTuXM888s57TMDPrtqN2racNGzZwzjnn1GlEfedomYfZ0eRoeTz2mF/ryczM+oaDwszMshwUZmaW5aAwM7MsB4WZmWU5KMzMLOuY+RxFXz/OVuTRtIaGhv3LggD8/Oc/Z8uWLXz2s5/lJz/5CdOmTQPg+eef58ILL+T73/8+3/72t/t0nGZmveUzioPouOOOY/Xq1ft/Ro4cCcB5553HokWL9tdbuHAhF1xwQZ1GaWaWd8ycURxOzjjjDN544w22bdvG6aefzq9//Wuuuuqqeg/LLOto+ZCZdZ+D4iB6++23GTNmDFBanfbRRx/df+zaa6/loYceYuzYsVx44YUfWJTQzOxwUujSk6RJkjZKapc0s8rxQZIWpePLJY0sO3ZzKt8o6YpUNljSCkm/l7RO0u1l9ZtSH39MfQ7s/TTro/zSU3lIAHzpS1/ioYce4sEHH2TKlCl1GqGZWde6DApJDcBc4EpgFDBF0qiKatOAXRFxFnA3MCe1HQW0AKOBScA9qb+9wISIuAAYA0ySdHHqaw5wd0Q0A7tS30edj3zkIwwYMIDHH3+ciRMn1ns4ZmY1FTmjGA+0R8SmiHgHWAhMrqgzGViQtpcAE1X6kobJwMKI2BsRm4F2YHyUvJXqD0g/kdpMSH2Q+rymh3M77M2aNYs5c+bQ0NBQ76GYmdVU5B7FMODlsv0O4BO16kREp6TdwJBU/lxF22Gw/0xlFXAWMDcilks6DXg9Ijor61eSNB2YDuxf2jvncLwJ9slPfrLeQzAz61KRoDjw69ugcm3yWnVqto2IfcAYSScDj0o6F9hW4LVI7ecB86C0zHj1odfXW2+9dUDZpZdeyqWXXnpA+W233XbwB2Rm1gNFLj11ACPK9ocDr9SqI6k/cBKws0jbiHgdeJrSPYzXgJNTH7Vey8zMDqEiQbESaE5PIw2kdHO6taJOKzA1bV8LPBmlb0RqBVrSU1FNQDOwQlJjOpNA0nHA54AXUpunUh+kPn/R8+mZmVlvdXnpKd1zuAF4DGgA5kfEOkmzgLaIaAXuBe6X1E7pTKIltV0naTGwHugEZkTEPklDgQXpPkU/YHFE/Gt6ye8ACyXdATyf+u6RiKB0f/zIdDR8+6CZHfkKfeAuIpYCSyvKbinb3gNcV6PtbGB2RdkaYGyN+psoPWnVK4MHD2bHjh0MGTLkiAyLiGDHjh0MHjy43kMxs2PcUfvJ7OHDh9PR0cH27dvrPZQeGzx4MMOHD6/3MMzsGHfUBsWAAQNoamqq9zDMzI54Xj3WzMyyHBRmZpbloDAzsywHhZmZZTkozMwsy0FhZmZZDgozM8tyUJiZWZaDwszMshwUZmaW5aAwM7MsB4WZmWU5KMzMLMtBYWZmWQ4KMzPLclCYmVmWg8LMzLKO2m+4M+sLI2f+st5DYMtdV9d7CHaM8xmFmZllOSjMzCyrUFBImiRpo6R2STOrHB8kaVE6vlzSyLJjN6fyjZKuSGUjJD0laYOkdZJuLKt/m6R/k7Q6/VzV+2mamVlPdXmPQlIDMBe4DOgAVkpqjYj1ZdWmAbsi4ixJLcAc4MuSRgEtwGjgo8ATkj4OdAI3RcTvJJ0ArJL0eFmfd0fED/pqkmZm1nNFzijGA+0RsSki3gEWApMr6kwGFqTtJcBESUrlCyNib0RsBtqB8RGxNSJ+BxARbwIbgGG9n46ZmfW1IkExDHi5bL+DA/9R318nIjqB3cCQIm3TZaqxwPKy4hskrZE0X9Ip1QYlabqkNklt27dvLzANMzPriSJBoSplUbBOtq2kDwEPA9+KiDdS8T8CHwPGAFuBH1YbVETMi4hxETGusbExPwMzM+uxIkHRAYwo2x8OvFKrjqT+wEnAzlxbSQMohcQDEfHIexUiYltE7IuId4EfU7r0ZWZmdVIkKFYCzZKaJA2kdHO6taJOKzA1bV8LPBkRkcpb0lNRTUAzsCLdv7gX2BARPyrvSNLQst0vAmu7OykzM+s7XT71FBGdkm4AHgMagPkRsU7SLKAtIlop/aN/v6R2SmcSLantOkmLgfWUnnSaERH7JF0CfBX4g6TV6aW+GxFLgb+XNIbSJaotwPV9OF8zM+umQkt4pH/Al1aU3VK2vQe4rkbb2cDsirJnqX7/goj4apExmZnZoeFPZpuZWZaDwszMshwUZmaW5aAwM7MsB4WZmWU5KMzMLMtBYWZmWQ4KMzPLclCYmVmWg8LMzLIcFGZmluWgMDOzLAeFmZllOSjMzCzLQWFmZlkOCjMzy3JQmJlZloPCzMyyHBRmZpbloDAzsywHhZmZZTkozMwsq1BQSJokaaOkdkkzqxwfJGlROr5c0siyYzen8o2SrkhlIyQ9JWmDpHWSbiyrf6qkxyX9Mf15Su+naWZmPdVlUEhqAOYCVwKjgCmSRlVUmwbsioizgLuBOantKKAFGA1MAu5J/XUCN0XEOcDFwIyyPmcCyyKiGViW9s3MrE6KnFGMB9ojYlNEvAMsBCZX1JkMLEjbS4CJkpTKF0bE3ojYDLQD4yNia0T8DiAi3gQ2AMOq9LUAuKZnUzMzs75QJCiGAS+X7Xfw/j/qB9SJiE5gNzCkSNt0mWossDwVfTgitqa+tgKnVxuUpOmS2iS1bd++vcA0zMysJ4oEhaqURcE62baSPgQ8DHwrIt4oMJb3O4mYFxHjImJcY2Njd5qamVk3FAmKDmBE2f5w4JVadST1B04CdubaShpAKSQeiIhHyupskzQ01RkKvFp0MmZm1veKBMVKoFlSk6SBlG5Ot1bUaQWmpu1rgScjIlJ5S3oqqgloBlak+xf3Ahsi4keZvqYCv+jupMzMrO/076pCRHRKugF4DGgA5kfEOkmzgLaIaKX0j/79ktopnUm0pLbrJC0G1lN60mlGROyTdAnwVeAPklanl/puRCwF7gIWS5oGvARc15cTNjOz7ukyKADSP+BLK8puKdveQ41/0CNiNjC7ouxZqt+/ICJ2ABOLjMvMzA4+fzLbzMyyHBRmZpbloDAzsywHhZmZZTkozMwsy0FhZmZZDgozM8tyUJiZWZaDwszMshwUZmaW5aAwM7MsB4WZmWU5KMzMLKvQ6rF2bBk585f1HgJb7rq63kMws8RnFGZmluWgMDOzLAeFmZllOSjMzCzLQWFmZlkOCjMzy3JQmJlZloPCzMyyCgWFpEmSNkpqlzSzyvFBkhal48sljSw7dnMq3yjpirLy+ZJelbS2oq/bJP2bpNXp56qeT8/MzHqry6CQ1ADMBa4ERgFTJI2qqDYN2BURZwF3A3NS21FACzAamATck/oDuC+VVXN3RIxJP0u7NyUzM+tLRc4oxgPtEbEpIt4BFgKTK+pMBhak7SXARElK5QsjYm9EbAbaU39ExDPAzj6Yg5mZHURFgmIY8HLZfkcqq1onIjqB3cCQgm2ruUHSmnR56pRqFSRNl9QmqW379u0FujQzs54oEhSqUhYF6xRpW+kfgY8BY4CtwA+rVYqIeRExLiLGNTY2dtGlmZn1VJGg6ABGlO0PB16pVUdSf+AkSpeVirT9gIjYFhH7IuJd4MekS1VmZlYfRYJiJdAsqUnSQEo3p1sr6rQCU9P2tcCTERGpvCU9FdUENAMrci8maWjZ7heBtbXqmpnZwdfl91FERKekG4DHgAZgfkSskzQLaIuIVuBe4H5J7ZTOJFpS23WSFgPrgU5gRkTsA5D0IHApcJqkDuDWiLgX+HtJYyhdotoCXN+XEzYzs+4p9MVF6RHVpRVlt5Rt7wGuq9F2NjC7SvmUGvW/WmRMZmZ2aPiT2WZmluWgMDOzLAeFmZllOSjMzCzLQWFmZlkOCjMzy3JQmJlZloPCzMyyHBRmZpbloDAzsywHhZmZZTkozMwsy0FhZmZZDgozM8tyUJiZWZaDwszMshwUZmaW5aAwM7MsB4WZmWU5KMzMLMtBYWZmWQ4KMzPLKhQUkiZJ2iipXdLMKscHSVqUji+XNLLs2M2pfKOkK8rK50t6VdLair5OlfS4pD+mP0/p+fTMzKy3ugwKSQ3AXOBKYBQwRdKoimrTgF0RcRZwNzAntR0FtACjgUnAPak/gPtSWaWZwLKIaAaWpX0zM6uTImcU44H2iNgUEe8AC4HJFXUmAwvS9hJgoiSl8oURsTciNgPtqT8i4hlgZ5XXK+9rAXBNN+ZjZmZ9rEhQDANeLtvvSGVV60REJ7AbGFKwbaUPR8TW1NdW4PRqlSRNl9QmqW379u0FpmFmZj1RJChUpSwK1inStkciYl5EjIuIcY2NjX3RpZmZVdG/QJ0OYETZ/nDglRp1OiT1B06idFmpSNtK2yQNjYitkoYCrxYYY6+NnPnLQ/EyWVvuurreQzAzO0CRM4qVQLOkJkkDKd2cbq2o0wpMTdvXAk9GRKTylvRUVBPQDKzo4vXK+5oK/KLAGM3M7CDpMijSPYcbgMeADcDiiFgnaZakL6Rq9wJDJLUD/530pFJErAMWA+uBXwMzImIfgKQHgd8CfyGpQ9K01NddwGWS/ghclvbNzKxOilx6IiKWAksrym4p294DXFej7WxgdpXyKTXq7wAmFhmXmZkdfP5ktpmZZTkozMwsy0FhZmZZDgozM8tyUJiZWZaDwszMshwUZmaW5aAwM7MsB4WZmWU5KMzMLMtBYWZmWQ4KMzPLclCYmVmWg8LMzLIcFGZmluWgMDOzLAeFmZllOSjMzCzLQWFmZlkOCjMzy3JQmJlZloPCzMyyCgWFpEmSNkpqlzSzyvFBkhal48sljSw7dnMq3yjpiq76lHSfpM2SVqefMb2bopmZ9Ub/ripIagDmApcBHcBKSa0Rsb6s2jRgV0ScJakFmAN8WdIooAUYDXwUeELSx1ObXJ9/ExFL+mB+ZmbWS0XOKMYD7RGxKSLeARYCkyvqTAYWpO0lwERJSuULI2JvRGwG2lN/Rfo0M7PDQJGgGAa8XLbfkcqq1omITmA3MCTTtqs+Z0taI+luSYOqDUrSdEltktq2b99eYBpmZtYTRYJCVcqiYJ3ulgPcDJwNXAScCnyn2qAiYl5EjIuIcY2NjdWqmJlZHygSFB3AiLL94cArtepI6g+cBOzMtK3ZZ0RsjZK9wD9TukxlZmZ1UiQoVgLNkpokDaR0c7q1ok4rMDVtXws8GRGRylvSU1FNQDOwItenpKHpTwHXAGt7M0EzM+udLp96iohOSTcAjwENwPyIWCdpFtAWEa3AvcD9ktopnUm0pLbrJC0G1gOdwIyI2AdQrc/0kg9IaqR0eWo18Nd9N10zM+uuLoMCICKWAksrym4p294DXFej7WxgdpE+U/mEImMyM7NDw5/MNjOzLAeFmZllOSjMzCzLQWFmZlkOCjMzy3JQmJlZloPCzMyyHBRmZpbloDAzsywHhZmZZTkozMwsy0FhZmZZDgozM8tyUJiZWZaDwszMshwUZmaW5aAwM7MsB4WZmWU5KMzMLMtBYWZmWQ4KMzPLclCYmVlWoaCQNEnSRkntkmZWOT5I0qJ0fLmkkWXHbk7lGyVd0VWfkppSH39MfQ7s3RTNzKw3ugwKSQ3AXOBKYBQwRdKoimrTgF0RcRZwNzAntR0FtACjgUnAPZIauuhzDnB3RDQDu1LfZmZWJ0XOKMYD7RGxKSLeARYCkyvqTAYWpO0lwERJSuULI2JvRGwG2lN/VftMbSakPkh9XtPz6ZmZWW/1L1BnGPBy2X4H8IladSKiU9JuYEgqf66i7bC0Xa3PIcDrEdFZpf4HSJoOTE+7b0naWGAuB9NpwGu96UBz+mgk9ef34n1+L97n9+J9h8t7cWaRSkWCQlXKomCdWuXVzmRy9Q8sjJgHzKt2rB4ktUXEuHqP43Dg9+J9fi/e5/fifUfae1Hk0lMHMKJsfzjwSq06kvoDJwE7M21rlb8GnJz6qPVaZmZ2CBUJipVAc3oaaSClm9OtFXVagalp+1rgyYiIVN6SnopqApqBFbX6TG2eSn2Q+vxFz6dnZma91eWlp3TP4QbgMaABmB8R6yTNAtoiohW4F7hfUjulM4mW1HadpMXAeqATmBER+wCq9Zle8jvAQkl3AM+nvo8Eh81lsMOA34v3+b14n9+L9x1R74VK/4k3MzOrzp/MNjOzLAeFmZllOSh6SdJ8Sa9KWlvvsdSbpMGSVkj6vaR1km6v95gOpWp/FySdKunxtCTN45JOqecYDxVJIyQ9JWlD+rtwYyo/5t6PWr8XR9JyRQ6K3ruP0vIkBnuBCRFxATAGmCTp4jqP6VC6jwP/LswElqUlaZal/WNBJ3BTRJwDXAzMSMv0HIvvR63fiyNmuSIHRS9FxDOUnvQ65kXJW2l3QPo5Zp6WqPF3oXx5m2NmSZqI2BoRv0vbbwIbKK2ycMy9H5nfiyNmuSIHhfWptOjjauBV4PGIWF7vMdXZhyNiK5T+8QROr/N4Drm0mvRYYDnH6PtR+XsB/ImCyxUdDhwU1qciYl9EjKH0qfrxks6t95isfiR9CHgY+FZEvFHv8dRL5e8FcE61aod2VMU5KOygiIjXgafx/ZttkoYCpD9frfN4DhlJAyiFxAMR8UgqPmbfD/jA78XFHEHLFTkorM9IapR0cto+Dvgc8EJ9R1V35cvbHDNL0qSvDLgX2BARPyo7dMy9HzV+LzZwBC1X5E9m95KkB4FLKS0bvA24NSKOlGVH+pSk8yndlGug9J+QxRExq76jOnSq/V0Afg4sBs4AXgKui4ij/uEHSZcA/wf4A/BuKv4upfsUx9T7Uev3QtJ/pPRdPKdSWq7oP0fE3vqNtDYHhZmZZfnSk5mZZTkozMwsy0FhZmZZDgozM8tyUJiZWZaDwqwbJJ0s6Zv1HofZoeSgMOuekwEHhR1TuvzObDP7gLuAj6UF3h5PZVdSWqfnjohYJOlSYBawA/gL4BngmxHxbnlHkv4L8AXgeOBjwKMR8beHYhJm3eEzCrPumQn8KS3w9hyl7xe4gNKyDN9/bx0jSgu/3QScRykE/qpGf2OAL6d6X5Y04iCO3axHHBRmPXcJ8GBaGXQb8L+Bi9KxFRGxKSL2AQ+mutUsi4jdEbEHWA+cedBHbdZNDgqznlPmWOXaOCHpi5JWp59xqbx8bZ99+HKwHYYcFGbd8yZwQtp+htLlogZJjcCngRXp2Pj0ncj9KF1aejYiHo2IMemn7dAP3axnHBRm3RARO4DfSFoL/CWwBvg98CTwtxHxf1PV31K68b0W2Aw8WofhmvUJrx5r1sfSU0/fjojP13ssZn3BZxRmZpblMwozM8vyGYWZmWU5KMzMLMtBYWZmWQ4KMzPLclCYmVnW/wcWzk4r7FAUqgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "report_results('topn', result, target_metric);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### saving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(experiment_name, cv=result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cold start"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "import gc\n",
    "del data_models, models\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TuriRecommenderColdStart(ItemColdStartEvaluationMixin,\n",
    "                               ColdStartRecommendationsMixin,\n",
    "                               TuriFactorizationRecommender): pass\n",
    "\n",
    "def prepare_cold_start_recommender_models(data_label, data_models, config):\n",
    "    data_model = data_models[data_label]\n",
    "    fm = TuriRecommenderColdStart(data_model, item_side_info=meta_dict[data_label])\n",
    "    fm.method = 'FM(cs)'\n",
    "    models = [fm]\n",
    "    apply_config(models, config, data_label)\n",
    "    return models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config_cold = {}\n",
    "scores_cold = {}\n",
    "data_models_cold = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for label in track(data_labels):\n",
    "    data_models_cold[label] = prepare_cold_start_data_model(label, *all_data, seed)\n",
    "    model, =  prepare_cold_start_recommender_models(label, data_models_cold, fm_init_config)\n",
    "    config_cold[label], scores_cold[label] = fine_tune_fm(model, params, label, ntrials=30)\n",
    "del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "report_results('tuning', scores_cold);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config_cold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### saving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(f'{experiment_name}_coldstart_param', config=config_cold, tuning=scores_cold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## rank estimation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rank_config_cold = {}\n",
    "rank_scores_cold = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for label in track(data_labels):\n",
    "    model, = prepare_cold_start_recommender_models(label, data_models_cold,\n",
    "                                                   [fm_init_config, config_cold]) # initiate with optimal config\n",
    "    rank_config_cold[label], rank_scores_cold[label] = fine_tune_fm(model, {'rank': fm_ranks[label]},\n",
    "                                                                    label, ntrials=0)\n",
    "del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "report_results('rank', {lbl: v.sort_index() for lbl, scr in rank_scores_cold.items() for k, v in scr.items()});"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rank_config_cold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### saving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(f'{experiment_name}_coldstart_rank', config=rank_config_cold, tuning=rank_scores_cold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_cold = {}\n",
    "for label in track(data_labels):\n",
    "    models_cold = prepare_cold_start_recommender_models(label, data_models_cold,\n",
    "                                                        [fm_init_config, config_cold, rank_config_cold])\n",
    "    result_cold[label] = ee.run_cv_experiment(models_cold,\n",
    "                                              fold_experiment=ee.topk_test,\n",
    "                                              topk_list=topk_values,\n",
    "                                              ignore_feedback=True,\n",
    "                                              iterator=lambda x: track(x, label=label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "report_results('topn', result_cold, target_metric);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "report_results('topn', result_cold, 'coverage');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### saving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(f'{experiment_name}_coldstart', cv=result_cold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}