plot_figures.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "# load libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import joblib\n",
    "import time\n",
    "import socket\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from ranking_cv import ndcg_from_file\n",
    "from ranking_cv import prec_at_k_from_file\n",
    "\n",
    "%pylab inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "exponential_scaling = False\n",
    "\n",
    "gene_features = ['paccmann']\n",
    "    \n",
    "losses = ['approx_ndcg',\n",
    "          'mse']\n",
    "\n",
    "models = ['paccmann',\n",
    "         'nn_baseline']\n",
    "\n",
    "pred_dir = 'results/preds/'\n",
    "cv_split_dir = 'cv_splits/'\n",
    "\n",
    "suffix = '20210617'\n",
    "\n",
    "cv = 5\n",
    "ks = [1,3,5,10,15,30]\n",
    "\n",
    "data_dict = dict()\n",
    "data_dict['k'] = []\n",
    "data_dict['model'] = []\n",
    "data_dict['ndcg'] = []\n",
    "data_dict['ndcg_std'] = []\n",
    "data_dict['ndcg_stderror'] = []\n",
    "data_dict['prec'] = []\n",
    "data_dict['prec_std'] = []\n",
    "data_dict['prec_stderror'] = []\n",
    "\n",
    "data_list_dict = dict()\n",
    "for loss in losses:\n",
    "    for model in models:        \n",
    "        for gene_feature in gene_features:\n",
    "            ndcg_cv = dict()\n",
    "            prec_cv = dict()\n",
    "            for split_nr in range(cv):\n",
    "                predict_load_path = pred_dir + 'pred_test_' +\\\n",
    "                        str(model) + '_' + str(loss) + '_' +\\\n",
    "                        str(gene_feature)  + '_max_conc_cell_wise' + suffix +\\\n",
    "                        '_' +\\\n",
    "                        str(split_nr) + '.npy'\n",
    "\n",
    "                predict_csv_path = pred_dir + 'pred_test_' +\\\n",
    "                        str(model) + '_' + str(loss) + '_' +\\\n",
    "                        str(gene_feature) +  '_max_conc_cell_wise' + suffix +\\\n",
    "                        '_' +\\\n",
    "                        str(split_nr) + '.csv'\n",
    "                \n",
    "                if not os.path.exists(predict_load_path):\n",
    "                    print('error: file ' + str(predict_load_path) + ' does not exist')\n",
    "                    continue\n",
    "\n",
    "                result_data = np.load(predict_load_path,allow_pickle=True)\n",
    "                result_data_pd = pd.read_csv(predict_csv_path)\n",
    "                \n",
    "                train_df_path = cv_split_dir + '/cv_' + str(cv) + '/train_cv_' + str(cv) +\\\n",
    "                        '_fold_' + str(split_nr) + '_max_conc.csv' \n",
    "                test_df_path  = cv_split_dir + '/cv_' + str(cv) + '/test_cv_' + str(cv) +\\\n",
    "                        '_fold_' + str(split_nr) + '_max_conc.csv'\n",
    "                \n",
    "\n",
    "                model_name = model + '_' + str(loss) + '_' + str(gene_feature)\n",
    "                if model_name not in data_list_dict:\n",
    "                    data_list_dict[model_name] = dict()\n",
    "                \n",
    "                for k in ks:\n",
    "                    if k not in data_list_dict[model_name]:\n",
    "                        data_list_dict[model_name][k] = dict()\n",
    "                    if k not in prec_cv:\n",
    "                        prec_cv[k] = []\n",
    "                        ndcg_cv[k] = []\n",
    "                    ndcg = ndcg_from_file(pred_path=predict_csv_path,\n",
    "                                          gt_test_path=test_df_path,\n",
    "                                          gt_train_path=train_df_path, \n",
    "                                          k=k, exponential_scaling=exponential_scaling)\n",
    "                    if 'ndcg' not in data_list_dict[model_name][k]:\n",
    "                        data_list_dict[model_name][k]['ndcg'] = []\n",
    "                        data_list_dict[model_name][k]['prec'] = []\n",
    "                    data_list_dict[model_name][k]['ndcg'].append(np.mean(ndcg))\n",
    "                    \n",
    "                    prec = prec_at_k_from_file(pred_path=predict_csv_path,\n",
    "                                               gt_test_path=test_df_path,\n",
    "                                               gt_train_path=train_df_path,\n",
    "                                               k=k)\n",
    "                    data_list_dict[model_name][k]['prec'].append(np.mean(prec))\n",
    "                    prec_cv[k].append(np.mean(prec))\n",
    "                    ndcg_cv[k].append(np.mean(ndcg))\n",
    "\n",
    "\n",
    "            for k in ks:\n",
    "                if k in ndcg_cv:\n",
    "                    data_dict['k'].append(k)\n",
    "                    data_dict['model'].append(model_name)\n",
    "                    data_dict['ndcg'].append(np.mean(ndcg_cv[k]))\n",
    "                    data_dict['ndcg_stderror'].append(np.std(ndcg_cv[k]) / np.sqrt(len(ndcg_cv[k])))\n",
    "                    data_dict['ndcg_std'].append(np.std(ndcg_cv[k]))\n",
    "                    data_dict['prec'].append(np.mean(prec_cv[k]))\n",
    "                    data_dict['prec_stderror'].append(np.std(prec_cv[k]) / np.sqrt(len(prec_cv[k])))\n",
    "                    data_dict['prec_std'].append(np.std(prec_cv[k]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k</th>\n",
       "      <th>model</th>\n",
       "      <th>ndcg</th>\n",
       "      <th>ndcg_std</th>\n",
       "      <th>ndcg_stderror</th>\n",
       "      <th>prec</th>\n",
       "      <th>prec_std</th>\n",
       "      <th>prec_stderror</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>paccmann_approx_ndcg_paccmann</td>\n",
       "      <td>0.888598</td>\n",
       "      <td>0.016082</td>\n",
       "      <td>0.007192</td>\n",
       "      <td>0.493755</td>\n",
       "      <td>0.046381</td>\n",
       "      <td>0.020742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>paccmann_approx_ndcg_paccmann</td>\n",
       "      <td>0.869132</td>\n",
       "      <td>0.008459</td>\n",
       "      <td>0.003783</td>\n",
       "      <td>0.397337</td>\n",
       "      <td>0.018369</td>\n",
       "      <td>0.008215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>paccmann_approx_ndcg_paccmann</td>\n",
       "      <td>0.861589</td>\n",
       "      <td>0.011362</td>\n",
       "      <td>0.005081</td>\n",
       "      <td>0.421913</td>\n",
       "      <td>0.018178</td>\n",
       "      <td>0.008130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10</td>\n",
       "      <td>paccmann_approx_ndcg_paccmann</td>\n",
       "      <td>0.869598</td>\n",
       "      <td>0.006215</td>\n",
       "      <td>0.002780</td>\n",
       "      <td>0.475475</td>\n",
       "      <td>0.009141</td>\n",
       "      <td>0.004088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15</td>\n",
       "      <td>paccmann_approx_ndcg_paccmann</td>\n",
       "      <td>0.878439</td>\n",
       "      <td>0.008366</td>\n",
       "      <td>0.003741</td>\n",
       "      <td>0.526846</td>\n",
       "      <td>0.008886</td>\n",
       "      <td>0.003974</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k                          model      ndcg  ndcg_std  ndcg_stderror  \\\n",
       "0   1  paccmann_approx_ndcg_paccmann  0.888598  0.016082       0.007192   \n",
       "1   3  paccmann_approx_ndcg_paccmann  0.869132  0.008459       0.003783   \n",
       "2   5  paccmann_approx_ndcg_paccmann  0.861589  0.011362       0.005081   \n",
       "3  10  paccmann_approx_ndcg_paccmann  0.869598  0.006215       0.002780   \n",
       "4  15  paccmann_approx_ndcg_paccmann  0.878439  0.008366       0.003741   \n",
       "\n",
       "       prec  prec_std  prec_stderror  \n",
       "0  0.493755  0.046381       0.020742  \n",
       "1  0.397337  0.018369       0.008215  \n",
       "2  0.421913  0.018178       0.008130  \n",
       "3  0.475475  0.009141       0.004088  \n",
       "4  0.526846  0.008886       0.003974  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_df = pd.DataFrame(data_dict)\n",
    "data_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAUf0lEQVR4nO3dfZBdd33f8feHNSbg4TESgZEtSyZKbBpj11lEGGgQNCayIRUU0gonxZCkihmUTh46jcLkAcK0NVBongxCGAU1TOLSBBwliAji4NBMeJCc2gj5IVFVPyzGWDYEx4ZihL/5416111d3d+9q99y7d8/7NbOz9/zO7979nrl797PnnN/5nVQVkqT2esy4C5AkjZdBIEktZxBIUssZBJLUcgaBJLXcaeMuYKFWrVpV69atG3cZkjRRbrjhhvuqavWgdRMXBOvWrePgwYPjLkOSJkqSO2Zb56EhSWo5g0CSWs4gkKSWMwgkqeUMAklqOYNAklrOIJCkljMIJKnlDAJJajmDQJJGYNOmTWzatGncZQw0cVNMSNIkOH/P+Y9aPnrP0YHthy4/NLKaZmMQSNIInPNL54y7hFl5aEiSWs4gkKSWMwgkqeUMAklquUaDIMnmJLclOZJkxyx9NiW5McnhJH/ZZD2SpJM1NmooyRRwFXAxMAMcSLK3qm7u6fMU4N3A5qq6M8nTm6pHkjRYk3sEG4EjVXW0qh4GrgG29PW5DPhwVd0JUFX3NliPJGmAJoNgDXBXz/JMt63X9wBPTXJ9khuSvHbQCyXZluRgkoPHjh1rqFxJaqcmgyAD2qpv+TTg+4GXAT8M/EqS7znpSVW7qmq6qqZXr1699JVKUos1eWXxDHBWz/KZwN0D+txXVQ8BDyX5FHAB8LcN1iVJ6tHkHsEBYEOS9UlOB7YCe/v6/DHwz5KcluQJwPOAWxqsSZLUp7E9gqo6nmQ7sB+YAnZX1eEkV3TX76yqW5L8GfB54BHg6qr6QlM1SZJO1uikc1W1D9jX17azb/kdwDuarEOSNDuvLJakljMIJKnlDAJJajmDQJJaziCQpJYzCCSpz3K+0XwTDAJJajlvXi+p9dbt+Oijlu85ev/A9tuvfNnIaholg0CS+jzjsivHXcJIeWhIklrOIJCkljMIJKnlDAJJajmDQJKWmVFfx2AQSFLLOXxUksbolnPPO6nt63feMXDdebc2cwNHg0CSlpk9a88e6c/z0JCkZalt8/2Mk3sEksbu/D3nn9R29J6jA9cduvzQSGpqE/cIJKnl3COQtCyd80vnjLuE1nCPQJJaziBogCe5JE2S1geBf7QltV2jQZBkc5LbkhxJsmPA+k1Jvpbkxu7XrzZZjyTpZI2dLE4yBVwFXAzMAAeS7K2qm/u6/s+qenlTdUiS5tbkHsFG4EhVHa2qh4FrgC0N/jxJ0iloMgjWAHf1LM902/o9P8lNST6W5J8MeqEk25IcTHLw2LFjTdQqSa3VZBBkQFv1Lf8NcHZVXQD8NnDtoBeqql1VNV1V06tXr17aKiWp5Zq8oGwGOKtn+Uzg7t4OVfVAz+N9Sd6dZFVV3ddgXUvKS+MlTbomg+AAsCHJeuCLwFbgst4OSZ4BfLmqKslGOnso9zdV0LodHz2p7Z6j9w9cd/uVL2uqDEkryInh59dff/1Y61iMxoKgqo4n2Q7sB6aA3VV1OMkV3fU7gVcDb0hyHPgGsLWq+g8fSZIa1OhcQ1W1D9jX17az5/HvAL/TZA2SmrcS/itus9ZfWSxJbefso9IKMMr/yD3XtvIYBNIE6h+R5kg1LYZB0ADnUVfbPOOyK8ddghbBIJBWAP/50GK0Pgj8T0ZS2zlqSJJaziCQpJYzCCSp5QwCSWo5g0AaMe+TreWm9aOGpKb1X23rVbgT7M1PPrnt9ocGr1u/tvl6loh7BJLUcu4RSCPmtStabtwjkKSWMwgkqeUMAklquaGCIMn3D2j7kaUvRxoNh3CuHL6XizfsHsH7kvy/ic6TvAb45WZKkiSN0rCjhl4N/GGSHwNeCLwWeGljVUnSLG4597xHLX/9zjsGtp936y0jq2nSDRUEVXU0yVbgWuAu4KVV9Y0mC5NOWOxtGPvv2gXe0UvqNWcQJDkEVE/T04Ap4LNJqKrnNFmcJM1nz9qzx13CxJtvj+DlI6lC6hrVjdG9o5f0/80ZBFV1x6gKkSSNR6PXESTZnOS2JEeS7Jij33OTfDvJq5uspw0cSidpoRqbayjJFHAVcDEwAxxIsreqbh7Q723A/qZq0eIs9mTtYjk3j9SsBQVBku8EvlpVjwzRfSNwpKqOdp97DbAFuLmv388AfwQ8dyG1SNJycP3rzhh3CYs2bxAkeSrwVuB84EvA05LMAD9TVQ/N8dQ1dIaanjADPK/vtdcArwRewhxBkGQbsA1g7drJmeNbkibBfMNHnwLsA95UVdt72l8MXJnkvwM3VtWDg54+oK36ln8D+MWq+nYyqHv3SVW7gF0A09PT/a/Rak3c9KR/bL1j7qWVbb49gl8B/ktVfTLJ7wE/ANwHrAIO0TnZ/KbuV78Z4Kye5TOBu/v6TAPXdENgFXBpkuNVde0Ct0PSBBj3+SYNNl8QvKiqfqH7+JvAa6rqYJKLgDcAfwX851meewDYkGQ98EVgK3BZb4eqWn/icZIPAH9qCCw/jrmXVrb5guBxSVJVBfxT4KZu+xeAi6rqkSRPGPTEqjqeZDud0UBTwO6qOpzkiu76nUuzCZKkxZgvCD4H/HPgz4H3AB9P8mng+cB7kzwXODzbk6tqH51zDL1tAwOgql43fNmajUMtJS3UfEHwH4EPJXlZVV2d5FrgHOBdwOPoDPu8vNkSJUlNmm+KiaNJ3gjsTfJx4DPAt+nMQfQK4I1VdVvjVUqSGjPvdQRV9dkkz6dziOgCOsNC/xr49ao63nB9kpaYI3fUb9j7ETwCfKL7JUlaQbx5vSS1XGOTzkkav/7bN4K3dtTJ3COQpJYbao8gyQuANwNnd58ToKrKS04lacINe2jo/cDPATfQGT4qaUJ5j1/1GzYIvlZVH2u0EknSWAwbBJ9M8g7gw3QmnwOgqv6mkaokrQxvfvKjl29/aHD7eu8zMk7DBsGJG8pM97QVnRvKSGqIF39pFIa9oOzFTRciSRqPoYaPJnlyknclOdj9emeSJ8//TEnScjfsdQS7gX8A/lX36wHgd5sqSpI0OsOeI3hWVb2qZ/ktSW5soB5J0ogNu0fwjSQvPLHQvcDsG82UJEkapWH3CN4A7OmeFwjwFeB1TRUlSRqdYUcN3QhckORJ3eUHmixKkjQ6cwZBkh+vqg8m+fm+dgCq6l0N1iZJGoH59gjO6H5/YtOFSJLGY757Fr+3+/0toylHkjRqw15Q9vYkT0ry2CTXJbkvyY83XZwkqXnDjhp6aVX9hySvBGaAHwU+CXxwricl2Qz8JjAFXF1VV/at3wK8FXgEOA78bFX91cI2QepoYl6ekc710z8RG6y4Sdquf90Z83fSyA0bBI/tfr8U+IOq+sqJE8azSTIFXAVcTCc8DiTZW1U393S7DthbVZXkOcCHgHMXsgGSpMUZ9oKyP0lyK53ZR69Lshr4v/M8ZyNwpKqOVtXDwDXAlt4OVfVgVVV38Qw6M5pKkkZoqCCoqh3A84HpqvoW8BB9f9QHWAPc1bM80217lCSv7IbMR4GfGKYeSdLSme86gpdU1V8k+Zc9bb1dPjzX0we0nfQff1V9BPhIkh+kc77ghwbUsQ3YBrB27WQeG9VkuOXc8x61/PU77xjYft6tt4ysJqlp850jeBHwF8CPDFhXzB0EM8BZPctnAnfP1rmqPpXkWUlWVdV9fet2AbsApqenPXwkSUtovusIfq37/fWn8NoHgA1J1gNfBLYCl/V2SPLdwP/uniy+CDgduP8UfpaWCe+oJU2eoUYNJflPwNur6u+7y08FfqGqfnm251TV8STbgf10ho/urqrDSa7ort8JvAp4bZJv0ZnN9F/3nDxe8fyjufztWXv2uEuQGjfs8NFLqupNJxaq6qtJLgVmDYJuv33Avr62nT2P3wa8bfhyJUlLbdjho1NJHndiIcnjgcfN0V+SNCGG3SP4IJ3rB36XzkninwD2NFaVRmKxh6b6R9KAo2ykSTTs/QjenuTzdIZ2BnhrVe1vtDJJ0kgMu0cAcAtwvKr+PMkTkjyxqv6hqcKkefXPv7PC5uUB5+bRaAw7++i/Bf4QeG+3aQ1wbUM1SZJGaNiTxW8EXgA8AFBVfwc8vamiJEmjM+yhoW9W1cMnppdIchotmyDOMf/Dcdy9NHmG3SP4yyRvAh6f5GLgfwB/0lxZkqRRGTYIfhE4BhwCfprORWJzXkwmSZoM8x4aSvIY4PNV9X3A+5ovSZI0SvPuEVTVI8BNSSZ3DJ4kaVbDnix+JnA4yefo3JQGgKr6F41UJUkamWGD4C2NViFJGpv57lD2HcAVwHfTOVH8/qo6PorCVqKx3v2q/2pbWJFX4kpauPnOEeyhc8P6Q8AlwDsbr0iSNFLzHRp6dlWdD5Dk/cDnmi9JOjXOyyOdmvn2CL514oGHhCRpZZpvj+CCJA90H4fOlcUPdB9XVT2p0eokSY2b7+b1U6MqRJI0HsNOMSFJWqEMAklqOYNAklrOIJCkljMIJKnlFnLz+gVLshn4TWAKuLqqruxb/2N07nUA8CDwhqq6qcmahtaCG6NLEjQYBEmmgKuAi4EZ4ECSvVV1c0+3/wO8qKq+muQSYBfwvKZq0qN5Ja4kaPbQ0EbgSFUdraqHgWuALb0dquqvq+qr3cXPAGc2WI8kaYAmg2ANcFfP8ky3bTY/CXxs0Iok25IcTHLw2LFjS1iiJKnJcwQZ0FYDOyYvphMELxy0vqp20TlsxPT09MDXmER71p497hIkqdEgmAHO6lk+E7i7v1OS5wBXA5dU1f0N1iNJGqDJQ0MHgA1J1ic5HdgK7O3t0L0P8oeBf1NVf9tgLZKkWTS2R1BVx5NsB/bTGT66u6oOJ7miu34n8KvAdwLvTgJwvKqmm6pJknSyRq8jqKp9wL6+tp09j38K+Kkma5Akzc0riyWp5QwCSWo5g0CSWs4gkKSWMwgkqeUMAklqOYNAklrOIJCkljMIJKnlGr2yeCXxJi6SVir3CCSp5QwCSWo5g0CSWs4gkKSWMwgkqeUMAklqOYNAklrOIJCkljMIJKnlDAJJajmDQJJaziCQpJYzCCSp5QwCSWq5RoMgyeYktyU5kmTHgPXnJvl0km8m+fdN1iJJGqyx+xEkmQKuAi4GZoADSfZW1c093b4C/DvgFU3VIUmaW5N7BBuBI1V1tKoeBq4BtvR2qKp7q+oA8K0G65AkzaHJIFgD3NWzPNNtW7Ak25IcTHLw2LFjS1KcJKmjySDIgLY6lReqql1VNV1V06tXr15kWZKkXk0GwQxwVs/ymcDdDf48SdIpaDIIDgAbkqxPcjqwFdjb4M+TJJ2CxkYNVdXxJNuB/cAUsLuqDie5ort+Z5JnAAeBJwGPJPlZ4NlV9UBTdUmSHq2xIACoqn3Avr62nT2P76FzyEiSNCZeWSxJLWcQSFLLGQSS1HIGgSS1nEEgSS1nEEhSyxkEktRyBoEktZxBIEktZxBIUssZBJLUcgaBJLWcQSBJLWcQSFLLGQSS1HIGgSS1nEEgSS1nEEhSyxkEktRyBoEktZxBIEktZxBIUssZBJLUcgaBJLVco0GQZHOS25IcSbJjwPok+a3u+s8nuajJeiRJJ2ssCJJMAVcBlwDPBl6T5Nl93S4BNnS/tgHvaaoeSdJgTe4RbASOVNXRqnoYuAbY0tdnC/DfquMzwFOSPLPBmiRJfU5r8LXXAHf1LM8AzxuizxrgS72dkmyjs8cA8GCS25a21JNl6J5fmG3FKuC+3ob+3aHZf/jwP33ol1xQ74HbdOrbA2PeJt+j4QrwPZr3JRfUe9m9R2fPtqLJIBhUcZ1CH6pqF7BrKYoalSQHq2p63HUslZW2PbDytmmlbQ+svG1artvT5KGhGeCsnuUzgbtPoY8kqUFNBsEBYEOS9UlOB7YCe/v67AVe2x099APA16rqS/0vJElqTmOHhqrqeJLtwH5gCthdVYeTXNFdvxPYB1wKHAG+Dry+qXrGYKIOZQ1hpW0PrLxtWmnbAytvm5bl9qTqpEPykqQW8cpiSWo5g0CSWs4gWGJJdie5N8msA6MnSZLvSPK5JDclOZzkLeOuabGS3J7kUJIbkxwcdz2nYtDvWZKnJflEkr/rfn/qOGtciFm2581Jvth9n25Mcuk4a1yI2T43y/U9MgiW3geAzeMuYgl9E3hJVV0AXAhs7o7wmnQvrqoLl+OY7iF9gJN/z3YA11XVBuC67vKk+ACDPzf/tfs+XVhV+0Zc02LM9rlZlu+RQbDEqupTwFfGXcdS6U7/8WB38bHdL0cYjNksv2dbgD3dx3uAV4yypsVo0edmWb5HBoHmlWQqyY3AvcAnquqzYy5psQr4eJIbutOXrBTfdeI6nO73p4+5nqWwvTsz8e7lchhlWLN8bpble2QQaF5V9e2qupDOld8bk3zfmEtarBdU1UV0Zr99Y5IfHHdBGug9wLPoHFr5EvDOsVazQJP0uTEINLSq+nvgeib8HEhV3d39fi/wEToz5a4EXz4xe2/3+71jrmdRqurL3T+mjwDvY0Lfp77PzbJ8jwwCzSnJ6iRP6T5+PPBDwK1jLWoRkpyR5IknHgMvZY6pLyfMXuDy7uPLgT8eYy2L1jcl/SuZoPdpjs/NsnyPmpx9tJWS/AGwCViVZAb4tap6/3irWpRnAnu6Nxp6DPChqvrTMde0GN8FfCSd6XxPA36/qv5svCUt3KDfM+BK4ENJfhK4E/jR8VW4MLNsz6YkF9I5p3M78NPjqu8UDPzcJPk0y/A9cooJSWo5Dw1JUssZBJLUcgaBJLWcQSBJLWcQSFLLGQTSEkiybqXMOKv2MQgkqeUMAmmJJTknyf9K8txx1yINwyCQllCS7wX+CHh9VR0Ydz3SMJxiQlo6q+nMHfOqqjo87mKkYblHIC2drwF3AS8YdyHSQrhHIC2dh+nccWp/kger6vfHXI80FINAWkJV9VCSlwOfSPJQVS2LaYaluTj7qCS1nOcIJKnlDAJJajmDQJJaziCQpJYzCCSp5QwCSWo5g0CSWu4fAYfdD5ONPqmKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnh0lEQVR4nO3deXgUVdbH8e8JCYYAomzqAApqEFCCkoAgLggqoCxC0AGcEUQGwUFxXhVxxHUGxFEQGFxGZzDIqICyqKjs4gCKQGIEYgBBokaUTY2yCnjfP9LELJ10B9JJOvX7PA8P3VW3bp+bTud03ao6Zc45RETEuyLKOgARESlbSgQiIh6nRCAi4nFKBCIiHqdEICLicZFlHUBx1a5d2zVs2LCswxARCSvJycm7nXN1/K0Lu0TQsGFD1q5dW9ZhiIiEFTP7srB1mhoSEfE4JQIREY9TIhAR8TglAhERj1MiEBHxOCUCERGPUyIQEfE4JQIREY9TIhAR8TglghBo37497du3L+swRKQcKc9/F8KuxERJO/bGLFu27Li2bz61eYFlX3z3hd916/uvP67XEJHwU+Bvwy3+l5eHvwueSgQNR75TYNl3X+zxuy5j7HWlEpOISFnzVCLw5/R+Y0u8z7PvP7vE+xTxmhPdW5fgeT4RiEjZ0xRr2VIikID0zaz8q4jvkfasS48SgRSQ/xtYRfxmFu5/OMPpQGQ4yH+M8LtXRwIFp44r6rFDJQIJSN/MxGtCceywPFMikApP888ly+/Zdx77Bl3RKBGIJ5XmXo4Xph289g26olEiqGDCfe7bn4o2Jv3RlEBK+3deiSDMFdhNb3Ov3+Xh8m3T37RDuI9JpCjpTZoWWLb/qy/9rmu6MT0kMSgRiIiUM1PPPKtUX09F50REPE6JQETCWnmu6hkulAhERDxOxwhEPCbcz8LKfwC1tA+sVkRKBCIS1kr7wGpFpEQgUoGVh1MTpfwL6TECM+tsZpvMbIuZjfSzvoaZvW1mn5pZmpndEsp4RIqrIh6InHrmWfoWXYIqwu9IyBKBmVUCngG6AM2AvmbWLF+zPwOfOedaAO2BcWZWOVQxiYhIQaGcGmoNbHHOfQFgZtOBHsBnudo4oLqZGVAN+B44EsKYJMR0IFIk/IRyaqge8HWu55m+ZblNBpoC24H1wHDn3K/5OzKzwWa21szW7tq1K1TxlrqKsEspUhz6nS+fQrlHYH6WuXzPOwGpQAfgHGCRmS13zv2UZyPnXgBeAEhISMjfh5QRLxyI1Fy6eEEo9wgygQa5ntcn+5t/brcAs122LcA2oEkIYxIJK/oGLaUhlHsEa4BYM2sEfAP0Afrla/MV0BFYbmanAecBX4QwJgkxfYMWCT8hSwTOuSNmNgxYAFQCpjjn0sxsiG/988DfgCQzW0/2VNJ9zrndoYqprOlAZGiF+4FqkbIS0gvKnHPvAu/mW/Z8rsfbgWtCGYOISIl5pEbBZRn7/K9rdGbo4ykhKjrnYZp/FhFQIgia/miKSEWlRCAi4nEqOleGdIaNVHj5580rwHx6RaQ9AhERj9MegUh5UUHPSJHyT3sEIiIep0QgIuJxmhqS8KUDkVIOLBtQtaxDOGFKBF6h+WcRKYSmhkREPE57BIXRtIOIeIQSgYiUmoown14RaWpIRMTjtEcgUo7pG7SUBu0RiIh4nBKBiIjHaWpIKgxNo4gcH+0RiIh4nPYIPEzfoEUEtEcgIuJ52iMIkr49i0hFpT0CERGPUyIQEfE4JQIREY9TIhAR8TglAhERj1MiEBHxOCUCERGPUyIQEfE4JQIREY9TIhAR8TglAhERj1MiEBHxOCUCERGPUyIQEfE4JQIREY8LaSIws85mtsnMtpjZyELatDezVDNLM7MPQhmPiIgUFLIb05hZJeAZ4GogE1hjZm855z7L1eYU4Fmgs3PuKzOrG6p4RETEv1DuEbQGtjjnvnDO/QJMB3rka9MPmO2c+wrAObczhPGIiIgfoUwE9YCvcz3P9C3LrTFwqpktM7NkM7vZX0dmNtjM1prZ2l27doUoXBERbwplIjA/y1y+55FAPHAd0Al40MwaF9jIuReccwnOuYQ6deqUfKQiIh4WypvXZwINcj2vD2z302a3c24fsM/M/ge0ADaHMC4REckllHsEa4BYM2tkZpWBPsBb+dq8CVxmZpFmFgNcDKSHMCYREcknZHsEzrkjZjYMWABUAqY459LMbIhv/fPOuXQzmw+sA34F/u2c2xCqmEREpKBQTg3hnHsXeDffsufzPX8SeDKUcYiISOGCmhoysy5+lg0p+XBERKS0BXuM4EEz63DsiZndR8FrAkREJAwFOzXUHZhnZvcCnYEmvmUiIhLmgkoEzrndZtYdWAwkA72dc/mvCRARkTBUZCIws5/JexFYZeBsoLeZOefcyaEMTkREQq/IROCcq15agYiISNnQ/QhERDxOiUBExOOUCEREPC7oRGBmNc0s3swaBG4tIiLhIuDpo2bWCBgPHAW2AHXNrA4w0DmnmwOIiIS5QKeP1gdmAH9wzm3OtfwC4B9m9jqwRglBRCR8BZoaeggY6ZzbbGZvmFmWmX0ErCC7ouh3vjYiIhKmAiWCls65pb7HDmjunGsLxAHRzrkUICGUAYqISGgFSgRRZnZs+uhs4Aff4x99zwFOCkFcIiJSSgIdLH6f7Cqjs4CHgSVmtpXsJPCYmXUEPg5tiCIiEkqBEsEYYL6ZbXTOzTOzd4HawG6y7y38IqpCKiIS1gLVGtppZjcAz5rZTmAV2aeRXgKcRfbZRPlvSC8iImEk4HUEzrmtQCcziyV7L8CAMc65jaEOTkREQi/oexY75z4HPg9hLCIiUgZUa0hExOOUCEREPE6JQETE44pMBGZ2q++G9ceef2NmP5nZz2Y2NPThiYhIqAXaIxgCTMn1fKfvPsV1gL4hi0pEREpNoEQQ4Zzbk+v56wDOuYNAlZBFJSIipSZQIqiR+4lzbgyAmUUAtUIVlIiIlJ5AiWChmf3dz/LHgIUhiEdEREpZoAvK7gX+bWZbgE99y1oAa4FBoQxMRERKR6BaQ/uAvmZ2NnC+b/FnvrITIiJSAQS6VWUnoLpz7g3gi1zLbyL7DKJFIY5PRERCLNAxgkeBD/wsX0L2cQIREQlzgRJBjL8b0zvnvgOqhiYkEREpTYESQXSuW1XmMLModB2BiEiFECgRzAZeNLOcb/++x8/71omISJgLlAhGATuAL80s2cxSgAxgl2+diIiEuUCnjx4BRprZo8C5vsVbnHMHQh6ZiIiUioBlqM2sFtkXjw3x/bvVtywgM+tsZpvMbIuZjSyiXSszO2pmvYMNXERESkagMtRNgQ1APLCZ7FtVtgLWm1mTANtWAp4BugDNyL4wrVkh7Z4AFhzPAERE5MQEKjHxN2C4c25m7oVmlgiMBhKL2LY12dNIX/i2mQ70AD7L1+4OYBbZCUZEREpZoKmh5vmTAIBzbhZwQYBt6wFf53qe6VuWw8zqAT3JPgupUGY22MzWmtnaXbsKXNYgIiInIFAi2Hec6wDMzzKX7/kE4D7n3NGiOnLOveCcS3DOJdSpUyfAy4qISHEEmhqqa2b/52e5kX2XsqJkAg1yPa8PbM/XJgGYbmYAtYFrzeyIc25ugL5FRKSEBEoELwLVC1n37wDbrgFizawR8A3QB+iXu4FzrtGxx2aWBMxTEhARKV2BriN49Hg7ds4dMbNhZJ8NVAmY4pxLM7MhvvVFHhcQEZHSEagM9UNFrHbOub8Vtb1z7l3g3XzL/CYA59yAovoSEZHQCDQ15O+AcFXgVrLvWVxkIhARkfIv0NTQuGOPzaw6MBy4BZgOjCtsOxERCR+B9ggws5rA/wE3AVOBls65H0IdmIiIlI5AxwieBHoBL5B9cdneUolKRERKTaALyu4Gfkd2yentZvaT79/PZvZT6MMTEZFQC3SMIGB1UhERCW/6Qy8i4nFKBCIiHqdEICLicUoEIiIep0QgIuJxSgQiIh6nRCAi4nFKBCIiHqdEICLicUoEIiIep0QgIuJxSgQiIh6nRCAi4nEBb0wjIv4dPnyYzMxMDh48WGS7F7ufEVR/6TYz6NeeEBncR/fwM0F3SXp6elDtgh0PBD+mYMcDwY8p2PFAxXqPoqOjqV+/PlFRUUH3q0QgcpwyMzOpXr06DRs2xMwKbXc488eg+msaUXgf+f1auXJQ7c7+zgXdZ5WmTYNqF+x4IPgxBTseCH5MwY4HKs575Jxjz549ZGZm0qhRo6D71dSQyHE6ePAgtWrVKjIJiJQmM6NWrVoB91LzUyIQOQFKAlLeHM/vpBKBiIjH6RiBSAlpOPKdEu0v487fFXubh+56iP5D+3POeeec8OtXq1aNvXv3Bt2+S9s4mja/kPEvvAzAonfeZPySWSRNeJSkGW8x8O5HSV34GnHNGgNwQYcbmDd1Ig0bFG+cj02ezKXx8XRo25bJ06YxsHdvYqpUAaBO69bsWr26yO2TkpIYOHAgqampxMXFAdCrY1v+mTSDeg3OpEvbOGKqVgPg119/pWPnrgwefi+VTzoJgM1bv+SuR55i8xdfEhUZSfMmsfzz7yM4rU4tVn+ygRGjJ/LNtzuJql6N2qfV5i8P/oXGvjHndk3La5ixaAZwSrHGHwraIxCpQB6b8FiJJIHj9dm6T9iyyf+ZLfXPOI3Rk/5zwq/x0LBhdGjbFoDJ//0v+4s5Hw5Qv359Ro8eXej6f898m1mLP+SVtxeT+VUGj428C4CDBw9x3c13MvSPvdmy8i3SP5jN0Jt7s2vPD+zYtYcbh9zHmPuG8fnKN3l96ev8afif+Drj6+MaZ2lSIhAJQ/v37Wdo36H0at+L6y+7nvfmvAfAgB4D2JC6AYBWZ7Vi1PjxXHLjjVw3aBBr1q+n0y230KxzZ+a9/z4A0+bO5YY77qD7kCGcd955PProo35f78knn6RVq1bExcXx7LjHC43r5tvu4D+Tx/td1/Wqy0jb/AWbtmQUuv36lPUMHzAcgKXvLSW+QTyHfznMoYOH6JzQGYDBDzzAnIULefaVV/h25066DBxI54EDc/p4eNIkLk5MpE2bNuzYscN/LF27kpaWxqZNmwqNBSCmajVGPT6e9xe8Q9YPP/Dq3PdoGx9Ht2uuyGlzZbtWXNDkXCa/NIP+N3TlklYtcta1bNOSjtd2LPI1ACZNnUpCz54k9OzJ5GnTANi3fz89b7+dixMTSejZkzfmzwdg5MiRNGvWjLi4OO65556AfQdDiUAkDK1cupK6p9dl9rLZzF0+l0s7XlqgzYH9B7isVSs+nDmTalWr8tg//8m8F15g+sSJ/O2Z305eX7thAy+NHUtqaiqvv/46a9euzdPPwoUL+fzzz1m9ejWpqal8tj6V5FUr/cbVqev1pG9Yx1fbviiwLiLCGDG0P2P+OaXQcTWNa8rG9RsBSF6VzLlNzmX9J+tZl7yO5i2b52l7+003cUbdurw3ZQrzp2T3ue/AAVrHxfHxrFlcfvnlvPjii35fJyIighEjRjBmzJhCYzmmWvWTqdfgLL7M2MqGjVuJj/N/Cmfa5q20bB78KavHpKSlMW3uXD545RWWvfIKL82aRWp6OotWruSMunX5eNYs1s6Zw9Xt2vH9998zZ84c0tLSWLduHaNGjSr26/mjRCAShmKbxbLqf6sY/9h4kj9KpvrJ1Qu0iaocxTWXZieI82NjuTQhgaioKC6IjeWr7dtz2nVs25Zap5xClSpV6NWrFytWrMjTz8KFC1m4cCEXXXQRLVu2JGPL53yZUfAPPUBEpUr0v+0O/vPM037X9+vZmVUp69n21Td+10dGRtKgUQO2bt7KhpQN9B/an+SPkklZlUJ8m/iAP5fKUVFce0X2t/X4+HgyMjIKbduvXz9WrVrFtm3bAvbrXPDn+h/Tt1Nful3Sjcf/WvgeFMBHn3xCt44dqRoTQ7WYGLp37MiHKSmcHxvL+6tWMWr8eFYmJ1OjenVOPvlkoqOjGTRoELNnzyYmJqbYcfmjRCAShhqe05CZi2cS2zSWCX+fwHNPPVegTWRkZM6phBEREZzku8ApIiKCI0eO5LTLf7ph/ufOOe6//35SU1NJTU1l3ooUevX5Y6GxdU38PSkff8i332T6jenu2/7AE88kFbp9fJt4VixZQWRUJG0ub0PKxymkfJxCfNvAiSAq15grVaqUZ5x+Y7n7bp544oki+9y392e2Z35Fw0bncv5555C8zv8xkPMbn0PK+t/WvbbgNe4YeQd7f9rL0aNHSWyfSGL7RCaPnZxnu8KSTGzDhqycMYPzY2N5aOJExjz3HJGRkaxevZrExETmzp1L586di4w9WEoEImFo53c7ia4STbcbujHgzwNIL+SPUzCWfPQR32dlceDAAebOnUu7du3yrO/UqRNTpkzJOYNox7fb2bN7V6H9RUVF8YdBQ3nlPwWTE8CAG7uzeMXH7Nrzg9/18W3jmfavabRIaEHN2jXJ+iGLbVu2cW6Tcwu0rR4Tw959+4IdasFYBgxg8eLF/PD9Hr/r9+/by+gH7uHKTtdx8imn0O/6znyY/CnvLF6e02b++ytZn/45fx5wI0kz3+bDNZ/mrDtw4ACQnZRmLZvFrGWzGDZyWJ7XaBcfz7ylS9l/4AD79u/n7aVLuaRlS7bv3ElMdDR9u3VjeP/+pKans3fvXrKysrj22muZMGECqampxz323HT6qEgJyRh7nd/l64IsXxAXEXiK4pjPP/ucpx59igiLIDIqkgeffDDobfO75KKLuPX++9m2Ywf9+vUjISEhz/prrrmG9PR02vrO1ImoXIUxE/9Frdp1Cu2zZ58/8uKkcX7XVa4cxZ0D+zL8oSf9ro9rGceeXXty9gAaN2tMzdo1/V4oNbB3b64fOpTT69TJOU5QHJUrV+bOO+9k+PDheZYPurEbzjmcc3TodB2Dh98LQJUq0cybOpG7Hn6Kux5+iqioSOKaxjLxsXs5rU4tZjw3lvvGTOKb73ZSrU4tTq15KkPuGVJkDBc1a8YfevTg8n79ABjQqxcXNm3KopUreWDcOCwigqjISCY++CA///wzPXr04ODBgzjnePpp/1NwxWXHM/dVlhISElz+g1nBKs553hnR/YJq17zRmUH3OfPxwndTc2u6Mfhvd8GOKdjxQPBjCnY8EPyYwuk9Sk9Pp2kQ9WxCkQjSSqiOzbS5c0lJS+PpBx6gygUXBNVnsOOB4McU7HigGLWGghwPlO/3KLdgx+Tvd9PMkp1zCf7aa2pIRMTjNDUk4mF/vP56/nj99WUdhpQx7RGIiHicEoGIiMeFNBGYWWcz22RmW8xspJ/1N5nZOt+/D82shb9+REQkdEKWCMysEvAM0AVoBvQ1s2b5mm0DrnDOxQF/A14IVTwiIuJfKA8Wtwa2OOe+ADCz6UAP4LNjDZxzH+ZqvwqoH8J4RELrkRp+F8cdb3+DlxV7Ey+UoZaSF8qpoXpA7vqrmb5lhbkVeC+E8YhUeF4oQy0lL5SJwN/90vxeOWFmV5KdCO4rZP1gM1trZmt37Sr80nYRr6ioZailbIQyEWQCDXI9rw9sz9/IzOKAfwM9nHN+C344515wziU45xLq1Cn8snYRr6ioZailbIQyEawBYs2skZlVBvoAb+VuYGZnArOBPzrnNocwFpEKpaKWoZayEbJE4Jw7AgwDFgDpwEznXJqZDTGzY1WYHgJqAc+aWaqZHV8RIRGPqchlqKX0hfQ6Aufcu865xs65c5xzo33LnnfOPe97PMg5d6pz7kLfP78FkUQkr4pchlpKn2oNiZSUR7L8LlYZ6rwClaGW0qdEIBKG2nVoR7sO7QosT3ozKefxmi/XgK/E8ajbb8/Tbtfq1TmP69SsyX/9lKHOfQ3B8OHDc2r2F5bY3vtoXc7jyiedxOLk9JzkNuD33Rnw++456++8tS933tq3qCFKKVKtIRERj9MegYiHqQy1gPYIREQ8T4lARMTjlAhERDxOiUBExON0sFikhDSf2rxE+1t/9ctFro87LY7YprEcPXqUs2PPZvTk0VSJqXLCr5uUlMQtt9zC4sWL6dixIwBz5syhV69evP766zRuc9UJv8bxmjx2Ml3Pa0mHtm2ZPG0aA3v3JqZK9pjrtG6d57RYf5KSkhg4cCCpqanExWUXCL/gggt48sVXqdfgTLq0jSOmajUAfv31Vzp27srg4fdS+aSTANi89UvueuQpNn/xJVGRkTRvEss//z6C0+rUYvUnGxgxeiLffLuT6tWqUu30Ovzlwb/Q2Fd2O7drWl7DjEUzOLXWqSX54zlu2iMQCVMnRZ/ErGWzmLt8LlGVo5g5dWaJ9d28eXNee+21nOfTp0+nRYuyv4HgsJHD6OC7sG3yf//L/oMHi91H/fr1GT16dKHr/z3zbWYt/pBX3l5M5lcZPDbyLgAOHTzIdTffydA/9mbLyrdI/2A2Q2/uza49P7Bj1x5uHHIfY+4bxucr3yRlwav8afif+Drj60JfpzxRIhCpAFq2aclX275i2YJl9O3Ul95X9mZQ4iB27N4NwN79+xk8ahStevakda9ezF20CICFK1bQ9sYbuTgxMefbP8Bll13G6tWrOXz4MHv37mXLli1ceOGFOeufn/AP+l3XgV4d2/LYfXfhXPaFa7fe0JWnxzxMv64d6XZ5Ass/TgEgacZb9Bp0N51v+jOx7Xow4u8TCoxhfcp6hg/Ivmht6XtLiW8Qz+FfDnPo4CE6J3QG4IFhDzBn4UKefeUVvt25ky4DB9J54MCcPh6eNImLExO54qab2LFjh9+fVdeuXUlLS2PTpk1F/kxjqlZj1OPjeX/BO2T98APvzn2DtvFxdLvmipw2V7ZrxQVNzmXySzPof0NXLmn1W7Js2aYlHa/t6K/rPCZNnUpCz54k9OzJ5GnTANi3fz89b7+dixMTSejZkzfmzwdg5MiRNGvWjLi4OO65556AfQdLU0MiYe7IkSOsWLKCdh3acdHFF/Hq/FcxM96Y9gZPv/QSY++9l7HPP0+N6tVZM2cOAD9kZbHr++/58yOPsCgpiYb163Pgd7/dKczMuOqqq1iwYAFZWVl0796dbdt+K4HRt/+fGHLXCAD+Ovw2Plg8n/ZXdwHg6JGjvDpvCcuXLuTR8eNYPON5AFLTNvPJglc5qXJlzru8J3fc0ocG9U7P6bNpXFM2rt8IQPKqZM5tci7rP1nP0SNHad4y77Tb7TfdxKSXX+a9KVOofWr29Mq+AwdoHRfHo3feyQPjx/Piiy8yatSoAj+viIgIRowYwZgxY5g6dWqRP9tq1U+mXoOz+DJjK1s2pZMQ19Rvu7TNW+l/Q7ci+/K73adpTJs7lw9eeQUHXNGvH5cmJJCRmckZdesy59lnAcj6+We+z8pizpw5bNy4ETPjxx9/LPbrFUZ7BCJh6tDBQyS2T+T3V/2eM+qdQeJNiezYvoPBNw6m5+U9SXomic+2bgVg6apV3NanT862p9aowep167g0Pp6G9bPvEFuzZs08/ffp04fp06czffp0+vbNWw5izUfLuanbVSRedQmrVy5n6+aNOes6dukKQLPmF5KRmavc9aWtqXFydaKjT6JZ47P58ptv8/QZGRlJg0YN2Lp5KxtSNtB/aH+SP0omZVUK8W3iA/48KkdFce0V2d/WL2rWjIyMjELb9uvXj1WrVuVJboU5trdTHBd3vZlul3Tj8b8WfhMfgJRVKXTr2JGqMTFUi4mhe8eOfJiSwvmxsby/ahWjxo9nZXIyNapX5+SqVYmOjmbQoEHMnj2bmJiYYsdVGCUCkTB17BjBrGWz+OvYvxJVOYox94+h3639mPO/OTw07iEOHToEZN8a0F956fzLcmvdujUbNmxg9+7dNG782wHPQwcPMvqBexj3ryRmLf6QxH4384vvdQAqV84+sBpRqRJHjhz9Ld7KUTmPK0VE5Fl3THybeFYsWUFkVCRtLm9DyscppHycQnzbwIkgKlfZ7Ur5Sm3nFxkZyd13380TTzxRZJ/79v7M9syvaNjoXM45rwnJhVR5Pb/xOaSs/23dx/Ne5o6Rd7D3p70cPXqUxPaJJLZPZPLYyXk3LCTHxDZsyMoZMzg/NpaHJk5kzHPPERkZyerVq0lMTGTu3Ll07ty5yNiLQ4lApALZ+9Ne6p5RF4C3pv92H6ir2rbl+VwHf3/IyuLiFi1YvnYtGZnZ9w34/vvvC/T3+OOPM2bMmDzLjiWXU06txf59e1n0zpslFn9823im/WsaLRJaULN2TbJ+yGLblm2c2+TcAm2rx8Swd9++436tAQMGsHjxYgq7/e3+fXsZ/cA9XNnpOk4+5RSu7dGbD5M/5Z3Fy3PazH9/JevTP+fPA24kaebbfLjm05x1Bw4cAKBSpUo5CXvYyGEFxjtv6VL2HzjAvv37eXvpUi5p2ZLtO3cSEx1N327dGN6/P6np6ezdv5+srCyuvfZaJkyYQGpq6nGPPT8dIxApIev7r/e7PBRlqAtz+4jbufvWu6l7el3iEuLYsyX7j/x9t93GX0aPJqFnTypFRHD/0KFcf9VVTH74Yfr85S+4X3/ltDPPZJHvIPIxXbp0KfAaJ9eoQWLfm+l9dTt+1+BMzm/R8oTjPiauZRx7du3J2QNo3KwxNWvX9LvnMrB3b64fOpTT69Rh/pTi3/6ycuXK3HnnnTlVVY8ZdGM3nHM45+jQ6ToGD78XgOgqVZg3dSJ3PfwUdz38FFFRkcQ1jWXiY/dyWp1azHhuLPeNmcQ33+2kbq2anFS7JkPuGeLvpXM0a9GMP/ToweX9+gEwoFcvLmzalEUrV/LAuHFYRARRkZFMfPBBft63jz5du3Lw4EGcczz9tP+7wB0PO575r7KUkJDg8t9TNVgNR74TdNuM6H5BtWve6Myg+5z5eOG7qrk13Rj8TUaCHVOw44HgxxTseCD4MYXTe5Senk7Tpv4PHuYWikSQ5rvbWCBnfxf85zt/GerCBDseCH5MwY4Hgh9TsOOBivce+fvdNLPkwm7+pakhERGPUyIQEfE4JQIREY9TIhAR8TglAhERj1MiEBHxOF1HIFJC0pv4P5U0yu9SP9vne9506asBtymLUtRPPZ/E1df1OOHXkPJDewQiYcyLpail5GmPQKSCaNmmJZs/28yyBcv41/h/cfiXw5xRtQZTxo7ltNq12bt/P/83ZgyfpKVhZvx16FCuv/pqFq5YwcOTJuGioqhduzZLliwBsktRL1++nMOHD3Po0CG/paj/t2g+Bw8e4MKEi3lw7NOYGbfe0JULLopnzYcr+OWn3fxn3ENcdnFLkma8xVuLPmD/gYNszcikZ5cr+ceou8rmhyV5aI9ApAI4Voo6tmlsTinqN95/g95duvD0Sy8B5ClFvXr2bK5o3TqnFPVr48fz6aef8vrrr+f0mbsU9Ztvvkn37t3zvGbf/n/i1XeWMnvJRxw8eIAPFs/PWXesFPWER+/h0fEv5CxPTdvMjOfGsn7JTGa8tZCvv/kuxD8ZCYb2CETC2LFS1JBduTPxpkS2bdnGPX+6h907dsOBw5xVrx6QXYr65SefzNn21Bo1eGfZsoClqCdNmkRWVhbjxo3LU4BuzUfLeem5SRw8sJ+sH3/knMZNcu5JcKwUdXxcU7+lqIGcUtS570kgZUOJQCSMHTtGkNuY+8fQf2h/rux8JZlvr2a07+YmJ1KKukqVKn5LUb/2zlJO/119nhs/1m8p6kqVIopdilpKn6aGRCqY3KWo//vmbyWiw6UUtZQ+7RGIlJDCKqyWZhlqyFuK+vKmcXz5zTdA4FLU3HcfdevWLfNS1FL6lAhEwtiaL9cUWNahSwc6dOkA5C1xXC0mhhdHjy7QvtNll9HpssvylDgeMGAAAwYMKNA2KSkpJ7ENGzGKYSMK3hP4P6/Py3lcu+apZHycXVp8wO+7M+D3vx1wnvfypACjk9KiqSEREY9TIhAR8TglApETEG53+JOK73h+J5UIRI5TdHQ0e/bsUTKQcsM5x549e4iOji7WdjpYLHKc6tevT2ZmJrt27Sqy3Y4fDgTVX7oV3U9u30UG99E9+lPQXRJVqVJQ7YIdDwQ/pmDHA8GPKdjxQMV6j6Kjo6nvu0AwWEoEIscpKiqKRo0aBWzXZeQ7QfWXEd0v6Ne+sdGZQbWb+fiRoPss7PTX/IIdDwQ/pmDHA8GPKdjxQMV7j4orpFNDZtbZzDaZ2RYzG+lnvZnZJN/6dWamk5FFREpZyBKBmVUCngG6AM2AvmbWLF+zLkCs799g4LlQxSMiIv6Fco+gNbDFOfeFc+4XYDqQ/24WPYCXXbZVwClmdkYIYxIRkXxCeYygHvB1rueZwMVBtKkHfJu7kZkNJnuPAWCvmW0q2VALKrwMV34bCltRG9ide0H+3aHCXzz4Vw+6y2K19jum4x8PlPGY9B4FF4Deo4BdFqt1uXuPzipsRSgTgb+I859nF0wbnHMvAC/4aVtumdla51xCWcdRUiraeKDijamijQcq3pjK63hCOTWUCTTI9bw+sP042oiISAiFMhGsAWLNrJGZVQb6AG/la/MWcLPv7KE2QJZz7tv8HYmISOiEbGrIOXfEzIYBC4BKwBTnXJqZDfGtfx54F7gW2ALsB24JVTxlIKymsoJQ0cYDFW9MFW08UPHGVC7HY7o8XkTE21RrSETE45QIREQ8TomghJnZFDPbaWaFnhgdTsws2sxWm9mnZpZmZo+WdUwnyswyzGy9maWa2dqyjud4+Ps9M7OaZrbIzD73/X9qWcZYHIWM5xEz+8b3PqWa2bVlGWNxFPa5Ka/vkRJByUsCOpd1ECXoENDBOdcCuBDo7DvDK9xd6Zy7sDye0x2kJAr+no0EljjnYoElvufhIgn/n5unfe/Thc65d0s5phNR2OemXL5HSgQlzDn3P+D7so6jpPjKf+z1PY3y/dMZBmWskN+zHsBU3+OpwPWlGdOJ8NDnply+R0oEEpCZVTKzVGAnsMg593EZh3SiHLDQzJJ95UsqitOOXYfj+79uGcdTEob5KhNPKS/TKMEq5HNTLt8jJQIJyDl31Dl3IdlXfrc2swvKOKQT1c4515Ls6rd/NrPLyzog8es54Byyp1a+BcaVaTTFFE6fGyUCCZpz7kdgGWF+DMQ5t933/05gDtmVciuCHceq9/r+31nG8ZwQ59wO3x/TX4EXCdP3Kd/nply+R0oEUiQzq2Nmp/geVwGuAjaWaVAnwMyqmln1Y4+Bayii9GWYeQvo73vcH3izDGM5YflK0vckjN6nIj435fI90q0qS5iZvQa0B2qbWSbwsHPuP2Ub1Qk5A5jqu9FQBDDTOTevjGM6EacBcyy7nG8k8Kpzbn7ZhlR8/n7PgLHATDO7FfgKuKHsIiyeQsbT3swuJPuYTgZwW1nFdxz8fm7M7CPK4XukEhMiIh6nqSEREY9TIhAR8TglAhERj1MiEBHxOCUCERGPUyIQKQFm1rCiVJwV71EiEBHxOCUCkRJmZmeb2Sdm1qqsYxEJhhKBSAkys/OAWcAtzrk1ZR2PSDBUYkKk5NQhu3ZMonMurayDEQmW9ghESk4W8DXQrqwDESkO7RGIlJxfyL7j1AIz2+uce7WM4xEJihKBSAlyzu0zs67AIjPb55wrF2WGRYqi6qMiIh6nYwQiIh6nRCAi4nFKBCIiHqdEICLicUoEIiIep0QgIuJxSgQiIh73/54DMjB6qRF1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def get_data_for_model(data_df,model):\n",
    "    tmp_df = data_df[data_df['model'] == model]\n",
    "    tmp_df = tmp_df.sort_values('k', axis=0, ascending=True)\n",
    "    return tmp_df\n",
    "\n",
    "\n",
    "legend_label_mapping = {'krl genes: all_gene':'KRL',\n",
    "                       'krl genes: paccmann':'KRL with network propagation',\n",
    "                       'nn_baseline_approx_ndcg_paccmann':'simple NN with NDCG-loss',\n",
    "                       'nn_baseline_mse_paccmann':'simple NN',\n",
    "                       'paccmann_approx_ndcg_paccmann':'PaccMann with NDCG-loss',\n",
    "                       'paccmann_mse_paccmann':'PaccMann'}\n",
    "\n",
    "\n",
    "unique_models = list(np.unique(np.unique(data_df['model'])))\n",
    "num_bars = len(unique_models)\n",
    "ks = list(np.unique(data_df['k']))\n",
    "\n",
    "ind = np.arange(len(ks))  # the x locations for the groups\n",
    "width = 1/(num_bars+1)  # the width of the bars\n",
    "start_offset = width * (num_bars/2)\n",
    "\n",
    "\n",
    "fig, ax = plt.subplots()#figsize=(15,15))\n",
    "rects = []\n",
    "for i in range(num_bars):\n",
    "    cur_model_name = unique_models[i]\n",
    "    cur_data = get_data_for_model(data_df,cur_model_name)\n",
    "    cur_means = list(cur_data['prec'])\n",
    "    cur_stderror = list(cur_data['prec_stderror'])\n",
    "    legend_name = legend_label_mapping[cur_model_name]\n",
    "    rects.append(ax.bar(ind - start_offset + (i*width), cur_means, width, yerr=cur_stderror,\n",
    "                label=legend_name))\n",
    "    #print(cur_model_name)\n",
    "\n",
    "# Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "ax.set_ylabel('Precision @ k')\n",
    "ax.set_xlabel('k')\n",
    "ax.set_xticks(ind)\n",
    "ax.set_xticklabels(ks)\n",
    "#ax.legend(bbox_to_anchor=(1.05, 1), loc='upper left')\n",
    "#ax.legend()\n",
    "plt.savefig('prec_at_k_max_conc.pdf',dpi=500)\n",
    "plt.show()\n",
    "\n",
    "fig, ax = plt.subplots()#figsize=(15,15))\n",
    "rects = []\n",
    "for i in range(num_bars):\n",
    "    cur_model_name = unique_models[i]\n",
    "    cur_data = get_data_for_model(data_df,cur_model_name)\n",
    "    cur_means = list(cur_data['ndcg'])\n",
    "    cur_stderror = list(cur_data['ndcg_stderror'])\n",
    "    legend_name = legend_label_mapping[cur_model_name]\n",
    "    rects.append(ax.bar(ind - start_offset + (i*width), cur_means, width, yerr=cur_stderror,\n",
    "                label=legend_name))\n",
    "    #print(cur_model_name)\n",
    "\n",
    "# Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "ax.set_ylabel('NDCG @ k')\n",
    "ax.set_xlabel('k')\n",
    "ax.set_xticks(ind)\n",
    "ax.set_xticklabels(ks)\n",
    "ax.legend(loc='lower right')\n",
    "#ax.legend(bbox_to_anchor=(1.05, 1), loc='upper left')\n",
    "plt.savefig('ndcg_at_k_max_conc.pdf',dpi=500)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the text for the table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k: 1\n",
      " & 0.491$\\pm$0.022 & 0.885$\\pm$0.006 & simple NN with NDCG-loss & 0.141 & 0.602 \\\\\n",
      " & 0.211$\\pm$0.055 & 0.726$\\pm$0.043 & simple NN & 0.02 & 0.009 \\\\\n",
      " & 0.494$\\pm$0.021 & 0.889$\\pm$0.007 & PaccMann with NDCG-loss & - & - \\\\\n",
      " & 0.245$\\pm$0.046 & 0.78$\\pm$0.02 & PaccMann & 0.002 & 0.009 \\\\\n",
      "k: 3\n",
      " & 0.397$\\pm$0.011 & 0.849$\\pm$0.003 & simple NN with NDCG-loss & 0.003 & 0.009 \\\\\n",
      " & 0.244$\\pm$0.037 & 0.734$\\pm$0.042 & simple NN & 0.038 & 0.009 \\\\\n",
      " & 0.397$\\pm$0.008 & 0.869$\\pm$0.004 & PaccMann with NDCG-loss & - & - \\\\\n",
      " & 0.279$\\pm$0.014 & 0.795$\\pm$0.01 & PaccMann & 0.001 & 0.009 \\\\\n",
      "k: 5\n",
      " & 0.414$\\pm$0.008 & 0.848$\\pm$0.003 & simple NN with NDCG-loss & 0.018 & 0.047 \\\\\n",
      " & 0.274$\\pm$0.036 & 0.74$\\pm$0.041 & simple NN & 0.043 & 0.009 \\\\\n",
      " & 0.422$\\pm$0.008 & 0.862$\\pm$0.005 & PaccMann with NDCG-loss & - & - \\\\\n",
      " & 0.299$\\pm$0.011 & 0.803$\\pm$0.008 & PaccMann & 0.001 & 0.009 \\\\\n",
      "k: 10\n",
      " & 0.448$\\pm$0.005 & 0.854$\\pm$0.003 & simple NN with NDCG-loss & 0.001 & 0.028 \\\\\n",
      " & 0.311$\\pm$0.038 & 0.752$\\pm$0.039 & simple NN & 0.05 & 0.009 \\\\\n",
      " & 0.475$\\pm$0.004 & 0.87$\\pm$0.003 & PaccMann with NDCG-loss & - & - \\\\\n",
      " & 0.366$\\pm$0.01 & 0.817$\\pm$0.007 & PaccMann & 0.001 & 0.009 \\\\\n",
      "k: 15\n",
      " & 0.48$\\pm$0.006 & 0.858$\\pm$0.004 & simple NN with NDCG-loss & 0.005 & 0.028 \\\\\n",
      " & 0.351$\\pm$0.041 & 0.761$\\pm$0.039 & simple NN & 0.05 & 0.009 \\\\\n",
      " & 0.527$\\pm$0.004 & 0.878$\\pm$0.004 & PaccMann with NDCG-loss & - & - \\\\\n",
      " & 0.424$\\pm$0.006 & 0.827$\\pm$0.006 & PaccMann & 0.0 & 0.009 \\\\\n",
      "k: 30\n",
      " & 0.562$\\pm$0.005 & 0.867$\\pm$0.003 & simple NN with NDCG-loss & 0.001 & 0.009 \\\\\n",
      " & 0.447$\\pm$0.045 & 0.776$\\pm$0.036 & simple NN & 0.04 & 0.009 \\\\\n",
      " & 0.629$\\pm$0.004 & 0.891$\\pm$0.003 & PaccMann with NDCG-loss & - & - \\\\\n",
      " & 0.538$\\pm$0.004 & 0.845$\\pm$0.004 & PaccMann & 0.0 & 0.009 \\\\\n"
     ]
    }
   ],
   "source": [
    "legend_label_mapping = {'krl genes: all_gene':'KRL',\n",
    "                       'krl genes: paccmann':'KRL with network propagation',\n",
    "                       'nn_baseline_approx_ndcg_paccmann':'simple NN with NDCG-loss',\n",
    "                       'nn_baseline_mse_paccmann':'simple NN',\n",
    "                       'paccmann_approx_ndcg_paccmann':'PaccMann with NDCG-loss',\n",
    "                       'paccmann_mse_paccmann':'PaccMann'}\n",
    "\n",
    "# p-values for KRL vs. paccmann-ndcg\n",
    "p_models = ['krl genes: all_gene','paccmann_approx_ndcg_paccmann']\n",
    "\n",
    "from scipy import stats\n",
    "from scipy.stats import ranksums\n",
    "\n",
    "\n",
    "for j in range(len(ks)):\n",
    "    k = ks[j]\n",
    "    print('k: ' + str(k))\n",
    "    \n",
    "    for i in range(len(unique_models)):\n",
    "        \n",
    "        cur_print_string = ''\n",
    "        cur_model_name = unique_models[i]\n",
    "        # investigate models for ttest and wilcoxon ranksum test\n",
    "        ndcg_list = [data_list_dict['paccmann_approx_ndcg_paccmann'][k]['ndcg']]\n",
    "        prec_list = [data_list_dict['paccmann_approx_ndcg_paccmann'][k]['prec']]\n",
    "        ndcg_list.append(data_list_dict[cur_model_name][k]['ndcg'])\n",
    "        prec_list.append(data_list_dict[cur_model_name][k]['prec'])\n",
    "        \n",
    "        \n",
    "        min_length = np.min([len(ndcg_list[0]),len(ndcg_list[1])])\n",
    "        if cur_model_name != 'paccmann_approx_ndcg_paccmann':\n",
    "            p_value_ttest  = stats.ttest_rel(ndcg_list[0][0:min_length], ndcg_list[1][0:min_length])\n",
    "            p_value_wilcox =  stats.ranksums(ndcg_list[0][0:min_length], ndcg_list[1][0:min_length])\n",
    "        #print(cur_model_name)\n",
    "        #print(p_value_wilcox)\n",
    "        #print(ndcg_list[0][0:min_length], ndcg_list[1][0:min_length])\n",
    "        \n",
    "        cur_data = get_data_for_model(data_df,cur_model_name)\n",
    "        # prec at k\n",
    "        cur_means = list(cur_data['prec'])[j]\n",
    "        cur_stderror = list(cur_data['prec_stderror'])[j]\n",
    "        cur_print_string += ' & ' + str(np.round(cur_means,decimals=3)) +\\\n",
    "            '$\\pm$' + str(np.round(cur_stderror,decimals=3))\n",
    "        # ndcg at k\n",
    "        cur_means = list(cur_data['ndcg'])[j]\n",
    "        cur_stderror = list(cur_data['ndcg_stderror'])[j]\n",
    "        cur_print_string += ' & ' + str(np.round(cur_means,decimals=3)) +\\\n",
    "            '$\\pm$' + str(np.round(cur_stderror,decimals=3)) \n",
    "\n",
    "        legend_name = legend_label_mapping[cur_model_name]\n",
    "        # model\n",
    "        if cur_model_name != 'paccmann_approx_ndcg_paccmann':\n",
    "            cur_print_string += ' & ' + legend_name + ' & ' +\\\n",
    "            str(np.round(p_value_ttest.pvalue,decimals=3)) + ' & ' +\\\n",
    "            str(np.round(p_value_wilcox.pvalue,decimals=3)) + ' \\\\\\\\'            \n",
    "        else:\n",
    "            cur_print_string += ' & ' + legend_name + ' & - & - \\\\\\\\'\n",
    "\n",
    "        print(cur_print_string)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}