code/scripts/mimic-omop_sequentialNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2049a6df-a525-4037-8d57-eb6ebb57400f",
   "metadata": {},
   "source": [
    "Reproducible conda environment used: file tf_cpu-linux-64.lock "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d9e9bfdc-b909-499a-8c89-f2ad24f154df",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-08 17:36:24.049074: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-09-08 17:36:24.050864: I tensorflow/core/common_runtime/process_util.cc:146] Creating new thread pool with default inter op setting: \n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "from pathlib import Path\n",
    "import os\n",
    "sys.path.insert(0,str(Path(os.path.abspath('')).parent))\n",
    "from proj_config import *\n",
    "import scipy as sp\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas_utils\n",
    "import timeit\n",
    "import hot_encoding as oh\n",
    "from importlib import reload\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "import keras_tuner as kt\n",
    "import keras_utils\n",
    "from keras_utils import metrics as kumet\n",
    "from sklearn import metrics as skmet\n",
    "from skmet_wrappers import *\n",
    "import json\n",
    "import gc\n",
    "\n",
    "models_dir = get_modelsdir() / \"mimic-omop/sequentialNN_3/\"\n",
    "datadir = get_datadir() / \"mimic-omop/pre-processed/\"\n",
    "plots_dir = get_project_root()/\"docs/latex/overleaf/gfx/\"\n",
    "tables_dir = get_project_root()/\"docs/latex/overleaf/tables/\"\n",
    "\n",
    "# Change matplotlib default style for publication style figures with correct\n",
    "# fontsize for a twocolumn LaTeX layout\n",
    "fig_width_pt = 252.0  # Get this from LaTeX using \\showthe\\columnwidth\n",
    "inches_per_pt = 1.0/72.27               # Convert pt to inches\n",
    "golden_mean = (np.sqrt(5)-1.0)/2.0         # Aesthetic ratio\n",
    "fig_width = fig_width_pt*inches_per_pt  # width in inches\n",
    "fig_height =fig_width*golden_mean       # height in inches\n",
    "fig_size = [fig_width,fig_height]\n",
    "single_col_fig_size = fig_size\n",
    "dual_col_fig_size = tuple(np.asarray(single_col_fig_size)*2)\n",
    "\n",
    "params = {'backend': 'pdf',\n",
    "          'axes.labelsize': 8.0,\n",
    "          'axes.labelpad': 3.0,\n",
    "          #'text.fontsize': 10,\n",
    "          'legend.fontsize': 7.0,\n",
    "          'legend.labelspacing': 0.2,\n",
    "          'legend.handletextpad': 0.4,\n",
    "          'xtick.labelsize': 7.0,\n",
    "          'ytick.labelsize': 7.0,\n",
    "          'text.usetex': False,\n",
    "          'figure.figsize': fig_size,\n",
    "          'savefig.bbox': \"tight\"}\n",
    "plt.rcParams.update(params)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b21c8668-4d43-481b-b157-c597376bc7d9",
   "metadata": {},
   "source": [
    "# Analyze results\n",
    "## MIMIC stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "9f9db7ac-a203-4fa9-851a-bf225df6c228",
   "metadata": {},
   "outputs": [],
   "source": [
    "oh_diagnoses_df = pd.read_parquet(datadir/\"oh_diagnoses_rollup.parquet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "44199ced-5c4a-408c-a8a5-9a24b06c2064",
   "metadata": {},
   "outputs": [],
   "source": [
    "icd9_concepts = pd.read_csv(datadir / \"icd9_concepts.csv.gz\")\n",
    "icd9_allcodes = icd9_concepts.concept_code\n",
    "icd9_chapters = icd9_concepts.query('concept_class_id==\"Chapter\"').concept_code\n",
    "icd9_subchapters = (icd9_concepts.query('concept_class_id==\"SubChapter\"').concept_code)\n",
    "icd9_3dig = (icd9_concepts.query('concept_class_id.str.startswith(\"3-dig\",na=False)').concept_code)\n",
    "icd9_4dig = (icd9_concepts.query('concept_class_id.str.startswith(\"4-dig\",na=False)').concept_code)\n",
    "icd9_5dig = (icd9_concepts.query('concept_class_id.str.startswith(\"5-dig\",na=False)').concept_code)\n",
    "icd9_billing = (icd9_concepts.query('concept_class_id.str.endswith(\"-dig billing code\",na=False)').concept_code)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b4c38bd",
   "metadata": {},
   "source": [
    "### Table 2: ICD Codes per stay in MIMIC-III "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "e6b00e0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build number of codes per stay table\n",
    "n_codes_df = pd.DataFrame()\n",
    "for code_set,name in zip([icd9_chapters,icd9_subchapters,icd9_3dig,icd9_4dig,icd9_5dig,icd9_billing,icd9_allcodes],\n",
    "                            ['Chapters','Subchapters','3 digits','4 digits','5 digits','Billing','All levels']):\n",
    "    tmp = oh_diagnoses_df.loc[:,oh_diagnoses_df.columns[oh_diagnoses_df.columns.isin(code_set)]].sum(axis=1).squeeze()\n",
    "    tmp_df = pd.DataFrame({'Min':tmp.min(),'Max':tmp.max(),'Median':tmp.median(),'Mean':tmp.mean(),'SD':tmp.std()},index=[name])\n",
    "    n_codes_df = pd.concat([n_codes_df,tmp_df],axis=0)\n",
    "n_codes_df.Median = n_codes_df.Median.astype(int)\n",
    "n_codes_df.style.format(precision=2).to_latex(buf=tables_dir/\"codes_per_stay_AllMIMIC.tex\",\n",
    "                environment='void',hrules=True,clines=\"skip-last;index\",column_format='lrrrrr')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec05d88f",
   "metadata": {},
   "source": [
    "### Table S2: ICD codes per stay in the cherry-picked subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "29c4e7b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build the same table for cherry picked codes (Sagi style)\n",
    "from project_miscellaneous import filter_Sagi_diagnoses\n",
    "sagi_codes = filter_Sagi_diagnoses(oh_diagnoses_df=oh_diagnoses_df)\n",
    "oh_diagnoses_sagi_df = oh_diagnoses_df.loc[:, ~oh_diagnoses_df.columns.isin(sagi_codes)]\n",
    "n_codes_df = pd.DataFrame()\n",
    "for code_set,name in zip([icd9_chapters,icd9_subchapters,icd9_3dig,icd9_4dig,icd9_5dig,icd9_billing,icd9_allcodes],\n",
    "                            ['Chapters','Subchapters','3 digits','4 digits','5 digits','Billing','All levels']):\n",
    "    code_set = code_set[code_set.isin(oh_diagnoses_sagi_df.columns)]\n",
    "    tmp = oh_diagnoses_sagi_df.loc[:,code_set].sum(axis=1).squeeze()\n",
    "    tmp_df = pd.DataFrame({'Min':tmp.min(),'Max':tmp.max(),'Median':tmp.median(),'Mean':tmp.mean(),'SD':tmp.std()},index=[name])\n",
    "    n_codes_df = pd.concat([n_codes_df,tmp_df],axis=0)\n",
    "n_codes_df.Median = n_codes_df.Median.astype(int)\n",
    "n_codes_df.style.format(precision=2).to_latex(buf=tables_dir/\"codes_per_stay_Sagi.tex\",\n",
    "                environment='void',hrules=True,clines=\"skip-last;index\",column_format='lrrrrr')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07cd4785",
   "metadata": {},
   "source": [
    "### Table 1: Unique codes per ICD9 level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "1296f798",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build the table containing the number of unique codes\n",
    "n_codes_df = pd.DataFrame()\n",
    "for code_set,name in zip([icd9_chapters,icd9_subchapters,icd9_3dig,icd9_4dig,icd9_5dig,icd9_billing,icd9_allcodes],\n",
    "                            ['Chapters','Subchapters','3 digits','4 digits','5 digits','Billing','All levels']):\n",
    "    tmp_df = pd.DataFrame({'ICD9-CM':len(code_set),\n",
    "                            'MIMIC III':sum(oh_diagnoses_df.columns.isin(code_set)),\n",
    "                            'Cherry picked':sum(oh_diagnoses_sagi_df.columns.isin(code_set))},index=[name])\n",
    "    n_codes_df = pd.concat([n_codes_df,tmp_df],axis=0)\n",
    "n_codes_df.style.format(precision=2).to_latex(buf=tables_dir/\"unique_codes_perlevel.tex\",\n",
    "                environment='void',hrules=True,clines=\"skip-last;index\",column_format='lrrr')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7a3f490",
   "metadata": {},
   "source": [
    "### Fig. S1 Zipf plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "1838bf02-0a6b-4a4e-99e3-2e930b69f2c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAACmCAYAAAD3VUKMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAA6mUlEQVR4nO2dd3hURdfAf7MlvZNGKAlJIPTeQksoIjWAlSLFT0XgFRsWRFFBfQW7oGJB5bWLiFQRaZHeAoReAiGQQhpppO/ufH9sEgKmbJJN5f6eZ5+9e+/cmbM3OXumnHNGSClRUFBomKhqWwAFBYXqQ1FwBYUGjKLgCgoNGEXBFRQaMIqCKyg0YBQFV1BowCgKrqDQgFEUXEGhAaOpbQGKI4ToAQwHNFLKV2tbHgWF+k6NKLgQwgvYALQF7KSUuoLzHwLdgSNSyqeAMcB84AkhRGMpZVxpdbq6ukofH59ql11Boa4TFhaWJKV0K+laTVnw68Bg4I/CE0KIroCtlLK/EGJZgfUWplbo4+PD4cOHzS+pgkI9QwgRVdq1GlFwKWUOkCPELfobCGwtON4K9AbWYbTgFiVZbyHEdGA6QPPmzatTZAWFBkFtjsGdgIsFx2lAOynlAeBAaTdIKb8UQsQBoy0sLLpVv4gKCvWb2pxFTwUcCo4dCj6Xi5RyvZRyuqOjY5nlEmIiuRpxgrzcnKrIqKBQr6lNC74PeBxYCQwBVphykxBiNDDa39+/zHIXNy0hMPob9FJwTbiSbOFFpm1T9E4+WLj6Yt+4JR7erXFwdkOolNVChYZJTc2ia4FNQCdgsxBinpTygBAiRwixCwiXUh40Z5te/ady6Iw/uuRItGmXscuOwTdlD64pGyHyZrl0bEhQNybdugm59s1RubRAY+uMxsoerbU9Wms7LG3ssbJ1xNLGHls7BzRaC3OKqqBQbdTUJFs+Rit9+/mnqqtN74DOeAd0/tf5rBtpxEedJTXmArmJFxEpl7G+cRW3zAg8MvZgEacvs948qWGLQwj5/Z8juHMANhZ1ypVAQeEWRH3N6NK9e3dZ1jJZnj4PnUGHSqhQCRVCCFTcPC4JvU5HYtxlcm6kkpuZTn52Bvk5N9AXvGReJlbXz9IheROZ0oov5ViutZ7KyK6+9PJ1wVKjRq0yeaVPQcEsCCHCpJTdS7xW3xS82Bj8sQsXLpRabsmRJXx14qtSr6uEChUFil/4I4BALdQ4WDrQyLoRLlYuNLJqRCPrRkXvXrZetNWpSN/wMs4xO4jFlXfyHmCtoQ8SFWqVQKsW2FlqaO3pQLsmDnRo4oivqx3uDpa42FigUn4EFMxIg1LwQsqz4GHxYRxPPI5BGpBIDNJgPJYSA8WOpQEDN4/1Uk9qbirJ2ckk5yRzPfs6KbkpGKShqO67fe5mQZ8F2EaHYdj8Cqpr4aRZNeGqYw+i7Ltwya4LV3VOnI5L59y1DPL1N5+xRiXo5u3M5EBv7m7niVatTPApVI0GpeCmWvCsyChkehpWfn6o7Wyr1KbeUKD0OcmEXg3l02Of4uPgw4fBH+Lr4AOnVsPJ3yFqD+SkGW9ybAbOPujtm5CscSdeuBInXYnUufDrBcGlVD2udhZ0aupE68b2NHexwcPBisaO1ng6WOFgrSl1KKGgUJwGpeCFlGfBd73yA2djbHBPPIqXPgp3Lwss/Xyx9PPH0s8XCz8/NM7OlWr7YNxBnt/5PNm6bBb2Xcgwn2HGCwY9xJ8yKnr0YUiLNr4y4kDenLyTQkW2bVMipRcndE3Yl+nFVl1nMrEuKmOlVdHY0RoPB0vaezkyvIMn3bxdKiWvQsPmjlTwyN0XOLH9CjFxEoNUYaXPwD0hDPfYA9hnXEEAahcXLH19sfD3w9LXDws/Xyz9/dG4u5drPeMz45nzzxzCE8N5qM1DPNvtWbRqbcmF9TqjkqddhdQrcP0SJJ2HxHPGd4OOfEcfLgxYykWNP/HpOVxLyyEuPYe41GxOxqSTpzcw565WzAj2U7r1CrfQoBTc1C56IXnZOiLDE7kQlsDV09cx6CV2dtDMPo3GOeexuhxO3qVLGNLSiu5R2dkZld3Xr8jaW/r5oW3SBKFWF5XL1+fzQdgH/HDmB2y1tmhV2qLJOo1Kg6u1K27WbrjZuOFm7YarjSvu1u5F7y5WLqgNeojaDWufgMxEaDkUnLyhkS807gxNupGZp2f+mpOsPhqDRiVwtNZib6XBy8maJk7WtHCzpb2XI02crfF1tVW69ncYDUrBCynPgpdETmY+l44lEhGWQPTZFKRB4uRhg383d3z9LbBJjyH30kXyIi6Se+kSeRcvoktMLLpfWFpi4e2N2sUFtb09KkcH1A6OXJaJXMqLQ69Vo9cI9Fo1uSo9KTKT6/p0kvTpXNGmk+QAstgMuoXKgqE+Q3kw4EE62TRFbH0VYsKMVj4/y1jINxhGfYh0bsGmk9c4GZNGek4+KVn5xKVmE5uaw7X0m+647vaW9PZtRM8WLvi729HKwx4na60yc9+AURS8BLIz8rh4NJGIsHhizqeCBBcvW/y7uWPnbIVaI1CpVZCXjSEhHkN8HLq4GAxxsahuXEeVnoQ6NRGZnorMMdHf3UKLbOJJnosdWTZqEq3z2eJwlaNeuahdXGjr2pZenr0IbhpEC2EJp9dC6Nvg1Rmmri+12qQbuVxMuMGlpEz2Xkxm38Vkkm7kFl33cLAkqJUbHZo60ampIwGe9lhq1KXWp1C/UBS8HDLTcrl4xKjscRFp5d9QDI1WhaWNBvdmNvQd7oG1Vo/My8OQm4vMzUPm5SFzc8iPjycvKoq8y1HoEhPRp6aiS0go+nGIa+/B+n6WbHWOBcDHwYdhLYbxeOoNNKH/hSePgUsLk2SSUhKTmk1Ewg0iEm6w/9J1wqKuk5KVD4BWLejt24h37utIY0frcmpTqOs0KAWv6Bi8ouRk5pOXrcOgl+h1BuO73oBBJzHoDejyDeTl6MjN1JGbpSM3W0dORh4XwhKwsFIzaEobfDq4mtSW1OnIOX2aG7t2kfLTz+iTk7F86H4O3teW7Ve2sy9uH/f5DOfV0K8Q/Z6FwfMr/b2klESnZHMiJo1jV1P5cX8UAZ72rJ7Vt9J1KtQNGpSCF2JOC24OkmNvsOXrUyTHZNJxYFMC7/FDozW9G2zIziZ+0WJSf/2Vxm+9hdO99/DxkY9ZfmI57bGicW4Wru3uxdW+KYOaDcLfuexouvJYsSeS19efZlBrd/zd7RgY4E6gX6Mq1alQOygKXkPo8vXsW32R4zuisW9khUMjK1Rq41je2l6Lo5s1jm42NAlwxsbh3xFpUqfjymOPkX04DO/vv8OqUyc+D/+cQ1FbSUo8TZJGQ4YAK7UV09pPw8nSiQDnALp7lvi3LZPMXB2vrztFeHQq5+NvADC8vSd9/F3p5+9KC9eqOQcp1ByKgtcwl08kcXxHNLo8PQa9xKCXZKXlkpmWB4DWUk33ET50GtQMtfbWNW1dSgqX738AmZuLz6pVaD3cjRei9sKKUSQGzuB5Qyxh8WEACARLBy0lqFlQpeVNzMjltXUnCYtKIT49Fzd7S/a/NFgJnKkn1AsFF0J0BBYCc6WUZ8srX5cVvDTy8/SkxGVy+M/LRIYn4eBmTd97/WnRyfWWteucc+e5PGECWk9PbLp1Q+PpgWWrVthEfIhGl4D8z0Gy9Tlk6bKYtXUW51LO0dmtM0O8h/BQm4cqvQ4upeS3w9G88PtxOjdzYkxnLx7q7a041tRxalTBK5AiuaR7pwH7G6qCF+fq6evs+u0CKXGZNGvjTOs+jWnkZYdzY1tUKkFGaCgJ772HPjUNfVKS8SYBjt5ZOD7zHmqfjqidncmwU/PTuZ/ZGb2TM9fPMMp3FE93fRoPW49KyaXTG3h38zn+OZ/I2WsZ+Lvb8exdrejr54qjTSmeegq1Sk0ruBVgjTFF8hAppa4gRfIMKeV0IcQy4BvABphZ7NbZwEjuEAUH0OsNnNoZw8H1keRm6QCwcbSgZTcPOg1phr2LFWCcgMs5c4aMDWtI+WUl0nDTQms8PWn8xhtYde7E15d+4pNjnwDQxqUNMzrNwN3GnRaOLbDVVnxM/fepayz66yyXEjMBaOpszbQ+PgT6NaKVh71i2esItdJFF0KEclPB/wMkSilXCiHuBbyklEtvK+8NvA1cBhZIKXNLqLN42uRuUVGlpoOuV+h1BlLjs0i8msGlo4lEnUpGCEHXoc3pcrc3Woti7rHfTiPv4Eb0Le9H59GX6yv+R/7VqwBovZuT29qHeNt8tulPccw1k0RHsHZsxHtB79HDs0eFZdPpDey9mMyp2HRCzyVwIPI6AL5utgS3cmdEB0+6+yhBMLVJXVDwl4EwKeVfQoghQB8p5cJK1lut6+B1gYzrOexdHUHE4QTsXCxp1cMTZ08b/Lq5o9UAP4+HmCPwfAT6zEwy9+wlLyqKrMOHyIu4SH5CAuiMPQIpBEmNNPzVGWYv2kxju8aVlktKybn4DE7GpPP9vsucicsgT29gZMfGLB3fRXGHrSXqgoIXt+D3AE2llEuqUn9D6KKXR+yFVPaujiAxKgODQWJpo8HRzRp7EYd/6mf4z/0SnP69AYQ0GMiLjCT3/HlyL0SQsmcn+vCTXOroSs8R/0ejCRNRWVpWWb60rHze33KO7/ZFMby9JzOD/ejY1KnK9SpUjLqg4F2Bx6WUjwshPgNWVDaL6p1gwW9HGiRxF9M4vSeWrPQ8UmJSuJEmGTI0m4B7Rppwv4FtL07FZvthnDPBbshgPJ5/Hgtv76rLJiXvbjYquZSS7c8F4+FgVeV6FUynpifZClMkdwOOAIUpkj8GumJMkfxEFeq/4xT8dnTZ2ax78Rvi8trQqIktjf2d6D3GF8syZrmllNy3/j7abzzH/bsNqNQaGk2ejPszzyA0Vc8Meyo2jTGf7MFKqybQrxHzRrRRnGVqiHqxDm4qioIb0S0bzOm0Ply2vofoc6lIg0SlEji6W9OsrQvdR/hgbXert1zsjVhWnV/FrmNrGL0mnh7nDahsbXG6714c770Xq1atqiTTqdg0ftgfxaqwaPL1kimB3rwwrDV2lkpq6eqkQSl4IXfCGLxM1j4B5zfD8xdIiErn8vEk9DoDSdE3iD6XgtZSjUMja1r19KDToGaIYhNg6XnpvLRzLtk7/mHaQTvcrqYjLCzwWrwIh+HDqyzambh0vtp1idVHYnCxtWBSr+bMCPLDVlH0aqFBKbhiwQvY/DIc/hZejv3XpWuX0jjxTzTpiTlcu5RGq14edBvmg7OnTZGXW44uh9f2vsafkX/SK8Od575NQ2ZmYtO7N54vz8OyZcsqi7jjbAIfbT1PeHQaTjZaHh/gx4wgXyXjjJlpUApeyB1vwbe9Abs/gFevQykKI6Xk0IZIDv95GSnB3ceB3mN88WrphFpjdFJZdX4VC/Yt4OO+79FxayTJ367AkJmJ21NP4jJ5Miqrqk2YSSk5HJXCpzsiCD2XSHCAG4vv7ahMxJmRBqXgigUvYOe7sP1NeCURNGXvlZaelM3lE8kc3nSZ7PQ8NBYqvFo606yNMx7+9ozfPxYHSwfeD34fX+lK9JNPkn04DKv27XF76kns+vevsrhSSj7ceoHPdkTg6WjF/FFtGdLGQwloMQMNSsELueMt+N5P4O+XYe4VsCp7K+VC8nJ0RJ9NIfrMda6eTSE13pj3zcLNwMbG3xBpd4pAr0Dm9ZyH7W9bSVy6FJmTg3WXLrhMm4b90Luq3L0+eiWF2T8fJTolm8aOVjzQvRmP9m+BvZXi515ZFAVviBxaDhvnwJzzYF+5wJIbKTlcPpHMnt8uoMs3YOgby4/qT2jh0IKv7/4ay1zJ9e/+R9ofa8i/ehW7gQNxHDcWu6CgKjnK6PQGNp+K54f9Uey7lExrT3t+fLQXjeyq7nxzJ1KWgpsULSCEmCSEqBNPXwgxWgjxZVpaxXKnNTg0BbnUdNmVrsLO2Yr2A5rw0JuBePo6YnGwKc87vMmZ5DM8uOFBjtw4jdusWfht+hO3Z54h88ABYp58isiQMaStX480GMpvpCTR1SpGdmzMz9N78+nErlxKzGTohztZsSeSPF3l6lQoGVPDgXyBPUKIJQVx27WGlHK9lHK6o6Np3dIGi7ZgkirfxIyuZWDraMmwx9tj72rNtQ0qno7+kKaXO/DUuudYdHAROpXE9fHpBOzfR9NPP0Hm5xP7/AtEz/oPN3btqrSiA4zs2Jj1s/vh52bH6+tPc8+yPWw8Hke+XlF0c1ChLroQYiDwBNAU+BL4oaSor5rgju+in/0TfpkA00PBq4tZqszL0XFqVyynd8cWjc/TLZPxaGXLAxMHYeds/FGRej1xr8wnbc0akBLrzp1xnjQJx9GjKt22lJKNJ+J4+8+zxKRm4+9ux/N3B3B3O09zfLUGjVnG4AX+5I8BnYCVgARGSymHmEvQinDHK/jF7fD9OHj4L/AONHv1qfFZRB5PZHXoZjyTjQke3Zrb49vZFf9uHjh52KC/kUnamjVc/+478q9codlXX2HXv1+V2s3XG9hyOp4PtpwnIuEGQ9t68FpIO5o4KemdS6PKCi6E2AecB76SUu4udn6RlHKu2SQ1AWWZrICoffDtMJj8B/gNqrZmVp1fxUfbl3GvdirNk9ty7VI6AM3budBxUDOat3XBkJnFxSFD0KelYRvYG6fx43EYOrRK7ebrDSzfFcnH285jkHB/t6bMGRqAi23ZS4J3IuZQcCcpZaq5BasKd7wFjz0KXwbD+J+h9Yhqa8YgDYxbO458Qz5/3vMnN1JyObM3lpP/xJCVnodHCwd8u7jRwluFbss6Un/9FV1iIvZ3302T99+rciBLdEoWn+6IYOXhaFztLHhpeBvGdPZSvOGKUeVZdOBTUfBEhZEfzSadQuUwwyy6KaiEijH+Y7iacZWd0Tuxc7akx8gWTPlvH/o90JKcG/nsW32RVcsukTtkAr5/b8Z54gQyNm8mbV3p2y2ZSlNnG96+pyPrnuiLs40FT/96jIUbTpOTry//ZgWTFbyxLDD1Be+VTwuiYB7MOIteHhNaT6Clc0tmb59NQlYCAGqNik6DmjFpYW/Gz++Jpa2WtR8d4/cPT2L5+BwsW7cm/u23yTpy1CwytPNy5M8n+/NQ7+Z8u+cyE77aT1xa9f64NQRMVXAhhAgoOAiowH0mI4QYIoR4qSBuXKE8asiCA1hrrHmhxwsYpIFvTn5zyzUhBI2a2DF+fk+CJgaQGp/FLwsOEjVyPnkO7lyZNo2UX36t0lJaISqV4M2xHfhsUlfOX8tg1JLdbDgeS3111qoJTFXUp4BvhRDnga+BJ0srKITwEkIcEULkCCE0xc5/KITYVZoCSym3SinfBpRtL02hBi04QO/GvQlsHFi04cLtWFhpaD+gCZMW9KZtfy/OHk1jb9s5nOv5H04u+Y2IIUPIOnTILLKM6NCYtU/0w9PRiid+OsrUbw8Rk6pY85IwScGllMellH2klK2klP2klMfLKH4dGAzsLzxRsMRmK6XsD1gIIXoIIYKEEL8Ue7kJIZ4BfqjKF7pjqEELXkjrRq25kHKB2Bv/DlEtxNbJkqAJAUxc0JuWPTyJtw3geMdZnHEfxuU5L6FLSTGLLP7udqx7oh+vjW7LocjrjPlkD1evZ5ml7oaEqa6qfYUQK4UQ24QQ24UQ20srK6XMkVLe/lcMBLYWHG8Feksp/5FSji98AfcAPYDehRN6JcgxXQhxWAhxODEx0RTRGy5qLQhVjVlwgPtb3Y9aqPni+BfllnV0s2bQlDY88l5//Lq6c9WpO7taPsWB/3uVnNOnzSKPWiV4uG8LVs0MJE+nZ8yne9hxLsEsdTcUTO2iLwM+BKYAkwteFcEJSC84TgOcby8gpfxCSjlRSvmRLGVQJaX8ElgAHLGwuMPXQ4UwWnFdzSl4M/tm9G/an9UXVnP2erl7UwCg1qoYNr09Y5/tgpWzHcc8xrH+lQ2cn/8eUm+emfB2Xo6sntUXR2stD397iDc3nFZ82gswVcHPSyn3SSljCl8VbCcVcCg4dij4XCkUX/RiaK0hv2bHnq8GvopGaPgz8s8K3deklTMPLQqiy0BPkty7sjW+AxtnfEnM6QSzTJL5u9ux6an+TO7tzfLdkYR8spvTsenl39jAMVXBdUKI1UKIl4UQ84QQ8yrYzj6M43KAIRQbn1cUJZqsGNqateAALlYuBHoF8seFP4i7EVehe9UaFX0ebMv4BYF4N8riivBnzZKTrH7nMFdPX6+ybFZaNQvHtOOTiV2IT89h1NJdLNl24Y6eZTdVwTcBa4FoIKbgVSJCCK0QYitGn/XNQoheUsojQI4QYhdgqGxOdIXb0FjVuAUHeL7H8+gMOqb9NY1z189V+H5nD1tGvj2O+0fk0+rCSlIuJ7FuyTF2/HiW/NyqdduFEIzq6EXocwMJ6eTFB1vOM/vno+Tq7kzHGFNn0f8HrAOOFhx/X0bZfCnlECmls5RysJTyQMH5p6SU/auSE13hNrRWNW7BAVo4tmDpoKVcz7nOZ8c+q3Q9bmNH0CmkDb3/eZGWHhmc3hXLyv8eIupkcpVldLTR8sEDnXlqcEs2HI/jPz8euSOV3NRZ9P/DuCXw9wVr2+uqVaoyUMbgxdDU/Bi8kO6e3RnqM5TtV7dzKe1SpetxmzUL+x5dafbrXHo1i8WgN7Dx03D+/voU1+MyqySjSiV45q5WPDm4JVvPJPDo/w6TkplXpTrrG6Z20R8B+gPXC/b7tqk+kcpGGYMXo5YseCETW08E4HRy5Ze9hFZL048/wm7gQGy/f4sguZn2A7yIDE/k54UH2PLNKfKydVWS89m7WvHWuPYcuHSdUUvvrMk3UxVcL6U0YIwBh1r0NlMseDFq0YKDsauuFmq2XynVLcIk1E5ONFv2GY0efYTM31fiE/ohk1/vQefBzTh/MJ7f3w0jOfZGldqY1MubXx/vTb7ewPgv97HvYtWHAfUBUxV8pRBiA+AjhFgF/FKNMpWJYsGLUcsW3EZrw7AWw9gXu88sM9Xuzz2H25xnyfxnJxlff0afe/0ZMasj2Rl5rH4njIiwqi2pdWnuzC/Te+Nqb8nUbw6yLrx0j7yGgqmTbJ8AzwEvAK9JKZdVq1Rly6JY8EJq2YID9PLsxY38G5xPOW+W+ho9+igOI0Zw/etvSPzgA1p0dOX+l3pg72rN5q9OsvajY8RHVr6L7etmx+8z+hDgac+TPx/l0x0RDXoZzdRJtgGAO5AANCr4fMfh7+/PL7/c7LwEBwej0+l4/fXX2bp16y1lC69VhXLrqGULDtC2UVsAIlIjzFKfEAKvxYtwHBNC8lfLydi2DXsXK+6f253AcX4kXc1g1eLD/LnsOFnplZswc7a14NfHezOigyfvbj7HC6uON1jPN1O76IXuqVOAD4AXq02icqitLnp4eDj9+/dn/fqqJzEwGxrrGvVFLwlvB2/stfbsjN5ptjqFVovnwoVYtGhB3Guvkx8Tg1qjouvd3jz0RiC9QloQdSqZX944QOyFygWv2Fho+GRCV2YE+fFbWDQzfwhD1wAzuZraRX+s4PUo0AujP3mtUFtd9NWrVzNr1iyysrLIzTU9kWxiYiIhISEMHDiQWbNmARAUFIShID76/vvv59q1axw4cIDg4GD69u3Lt99+e0sdn376Kb1792bgwIEcOXLk5gWnZuDqX/UvVwWsNFb09urNyaSTZq1XZWlJk48+wpCezuXJkzFkGSPFrGy1dB/Rgvte6I40wB8fHCV829VKdbNVKsHc4a15cVhrtp1N4JH/HW5wlrwyCbNcgfbmFsScLFh/qspLIW29HHhtdLuiz0eOHGHBggUMGzaMrVu3MnLkSJPqWbRoES+99BKBgYG8+OKL7Nu3jz59+rBnzx66detGZmYmnp6eTJ06lXXr1mFvb89dd93FpEmTiupYu3YtO3bswNra+tZ/5MD/GF+1TEunlmyN2kq2Lhtrjfmyn1oFtMLz9deJmzeP2Hkv0+SD9xEqo01ya27P+Pk92bz8JLt/u0BEWDyDp7bFyaPiK7gzg/3I1en5aOsFHvr6AN9M69Fg9jQ36VsIIS5gXCITGANF3qxGmeocFy9e5OTJkwwbNozc3FxatWplsoKfOXOGuXPnIoTgxo0b9OzZk/vuu4/vvvuOhIQEhhfsxx0eHk5ISAgASUlJFA+HXbBgATNnzsTCwoI33ngDD4/KbVVUXfg5+SGRXEy9SHtX8/72O44bS15UFMlffEFC48Z4vPhC0TVbJ0vGPtOFU7ti2b/mIr+8eZDhMzrg3a5Rhdt5ekgrmjhZ8+Lvx3lkxSFWPNwTa4sGkHtESlmvXsBo4Et/f39ZUyxevFhu3bq16PPo0aOlTqeTQUFBMj8/X7722mtyy5Ytt9xTeO3pp5+Whw8fLjqfn58vpZRywIABcsKECTI6OlpKKeWwYcPkjRs3pJRS5uXl3VJHVlaWlFLKH3/8US5atKj6vmglSchMkF2+6yLfPvB2tdRvMBhk7KuvydMBrWXK6j9KLJOakCm/n79XfvL4NnloY2Sl2/rjSLT0fnGDvOezPTIzN7/S9dQkwGFZir6YasFfLeMHYqF5fmpMQ0q5HljfvXv3x2qqzY0bNzJ79uyiz23btmX37t1l3HGTefPmMX36dNLS0lCpVHz11Ve0aNGCXr16sX//fpo0aQIYrXRISAgGgwEXFxd+//33ojpmzJhBZGQkubm5/xqf1wXcbNxo49KGM8lnqqV+IQSe818hNyKCuJdewjawN1rPW3c8cXSz4YGXerB5+SkOrLtEXraOwHF+iApuTzy2SxMkkmdXhvPIisN8M61HvbbkpuZF/x+QCxzAmHXFGvgbQEpZKymU7/i86HWM9w69x/dnvmfHAztwsXKpljZyzpwhctw9uM6aiduTJacF1OsM7PjhLOf2X6NFJ1eGTW+PSl3xHKE/HbjCy2tOcHdbT5Y91LVO52E3R150FynldCnl11LKGYCrlPLH2lJuhbrHcN/hGKSBXdG7qq0NqzZtsB82jKTPlpWajlmtUTF4ahs639WcyPAkVi0OIyczv8JtTezVnKcGt+SvU9eY/PXBeusMY6qCuwshehZsetAT40y6WRFC9BJCPCeEuKMm8BoKbV3a4mLlwqFr5smcWhruzz6DsLQkauJEdElJJZYRQtD3Xn96jfEl8UoG37+8lyunK+57/tTglgxp487uiCRm/3y0Xiq5qQo+DZgHnCx4/7/SClYhbfIBjLP0ddIHddu2bQQFBTFgwADGjRvH6NGjiYionPdWaGgoly5VPsSyLiKEwMPGg+s5Vc/MUhYWzZvjtehtABKXLC2zbPfhPgyf0QGhEqxfEs7JnRXLNCaE4MvJ3RnQyo0Nx+NYuME8ySJrElMdXc5IKccCA6SUY6WUp8ooXum0yVLKdzFmjalTJCUlsXDhQjZs2MDOnTtZvHgxeXmVjyuuiIIbzLBhQE3hbuPOmetnyMqv3vTFDsOHY3/XEFJXrkSfmlpmWd/Obkx+qw9Wdlr++elchZVcpRL87+EeNHGy5ts9l1l7rKLpCGsXU33RRwghjgE7hRCagkm3EpGVT5vcXwjxClDqhtC1lTZ548aNTJ48GXt7ewBatWpF48aNWbp0Kf369WPBggUAfPvttwQHB9O9e3f+/vtvAKZNm8asWbMYMGAAr776Knl5eaxYsYI5c+YwZ84csrOzmTBhAoMGDeLBBx8kPz+fFStW8OCDDzJy5EiOHTtGcHAwAwcO5MlSJpbqCve2vJek7CQOxB2o9rZc/s/YiYx+5plyy1paa3jw5R7YN7Lin5/OcXjT5Qq1JYRg7RN9AXjql2OcvVZ/4slN7aK/jFFJE6Qx4UOzCrbjRPlpk1dLKd+UUpb6F5O1lDY5Li6Oxo3/vR1bcHAwu3fv5s8/jRlGH3zwQUJDQ9m2bRvvvfdeUbm+ffuyc+dOjhw5QmJiItOmTeP999/n/fffZ/ny5YSEhLB9+3aCg4NZtWoVAE5OTmzcuJHr168THBzMjh07+Pjjur2rU1ePrmiEhj2xe6q9LZsuXbAfPoysffvJPnGi3PJ2zlbc92J37BtZcWDtJQ5tjKxQe652lqz9T1+0asHUbw6SnlPxibvawFR/vHwpZbYQonCWoaJrBqmYMW0y5a2Db5oL18r/o5eJZwcYvgiAxo0bExv779jh9u2NXlvW1kb3zM2bN/Pxxx8jpSQh4WYC/i5dugDQoUMHIiNv/cc6c+YMYWFhfPHFF+Tk5DBhwgQcHR3p1q0bYPRb37lzJxMnTmT48OFMnlzRlPQ1h6OlI4O9B7PtyjZe7vVytS8tuT/1FBmb/iJ69pO0DN1RbnkbBwsmvtaLb17YzcH1kcRfTmfEzI6oTFwr79TMiS8nd+fhFYfo8eZWwubfVeddWk214HuFEJ8BnkKID4GKroXU67TJI0aM4IcffiAjIwOAiIgI4uLi/vUP/Pbbb7Np0ybWrl2LSnXz0YaHhwNw8uRJfHx80Gq16AuS/gcEBPDCCy8QGhrK/v37iwJSCu/X6/UsXLiQn376iffff7/av2tV6e7RnaTsJOIyK5ZSuTJY+Phgf9cQdNeukb5li0n3aCzUPPxOPxr7OxJ1Ipkf5u9DGkyfHR/Y2p0ZQX7k6gyMWlJ9S4LmotyfHyGECvgLo3PLJeCUlHJTGeW1GNMsF6ZNnielPFAwq74LCJfVnTa5wPKaCzc3N+bPn8+oUaOQUuLi4kJJQ4RRo0YxYMAAevbsiZOTU9H5f/75h88++4ygoCCaNm1KcHAw8+bN48CBAzz//PM89thjfPbZZ0gpefvtt2+p8+DBg8ybN4/8/HyGDBli1u9VHXRxN/ZWtkRtYWq7qdXenscrr5CxZSsxs5/E9sB+1CZEGWot1Iyb05Xv5u0lIzmH5c/uZOqivlhYmWaN5w5vzd+nr3EpMZNHVhzi62k9qvo1qg1TPdnWSSlDakCechFCjAZG+/v7P3bhwoXaFqdcpk2bxiuvvIK/f+2GddYkkzYaI+F+HFkzflCpq1YR98p87AYNotlnn5p8n8EgWbXoMIlXMlBpBDOWBJvs2pqTr6f1/L8AeOe+jjzQvaLTUubDHJ5sMUKIF4UQ/YUQfYQQfcwoX4WQSsqmOo+Pow8J2TW3CaDTffehdnbmxvbt5EaaPnmmUgkemNcDjxYOGHSSz2eHmuz1ZqVVc/Bl46jzhVXH62ym1jIVXAgxp+DQChiK0cHlMeDRaparLJnqVdLFFStW3FHWG8DV2pWk7CQMsubW8JsWWO6oiZPKKflv7nm+G27N7THoJV/P2WVyBld3eyveGGucaB2xZBexdXCP8vIs+EgAKeXDgEpK+XDBq1RPtupGseB1H2dLZ3QGXbU7vBTHpksXLFu2RJ+SQsrKlRW6t9CSdwhuCsAvCw+arOSTe3szI8gPgD6LtpOTX7d2T6lImE2dcMStDQuenp7OyJEjCQ4OJjAwkJKi2C5fvsxDDz1Ubl0lJWg0lWPHjt2asqmOYqmxBCBXb3pqK3Pg/YNxR63U31ZV6v4B41vR9W5vwKjkSdEZJt03d3hr+rc0hmeM+aT6fQAqQnkK3lkI8bcQYkvxYyHE3zUhXEnUhgX/7rvvuOeeewgNDWXXrl0EBATUWNvFqYiC16aLq4XKuMKQb6hZZxC1oyPWXbqQc+IEspLfP3CcH33uMQ6pfn3zEIlXTFPy7x/phZVWxbn4DP6393Kl2q4OylPwLtwccxc/rrFkC3UBGxsb9u3bR1JSEhqNhrCwMF555RXAOMZesWIFADExMYwZM4bevXsXObQsW7asKGHiuXPGnTh/+eUXhgwZwqOPGqcy/vrrL4KCgujevTvfffcdYLT006ZNY8iQIfxfgVvml19+ybvvvsukSZOQUjJz5kwGDRrEyJEjSUlJITQ0lJCQEEaPHs369esZNWoUAwcO5IEHHqjJx4WF2qjgNW3BAWz7GV1K0zeVupJbLl2GNidwnLHbvfK/h8hMNe17bJ8TDMBr604Rn1672W4LKXPhT0oZVVOCmEqxZbJSyyw+uJiz189WqZ3WLq15sacxO/TkyZOJjo5m4MCBeHh4FCnm7Vy7do0tW7YQFhbG4sWLWbhwIb/99ht79uxBrVYXWdV27dqxfPlyhg4dSmpqKgMGDGDYsGHodDqCg4OZMmUKAC1btmTFihXMnDmT/fv3M336dHQ6HY8++ijr16+nefPmLFu2jE2bNvH5558TGBhIXl4ef/31FxEREbi6urJhw4YaD3O0VBu76Nm6mp90cn7wQZKWfkL8G2/iaGLevJLoerc3eTk6wjZFsWLuHqYt7outo2WZ93g5WfPqqLYs3HCaXv/dxoW3hqOtRLIJc1K7rVeC2uiia7VaXn31VU6cOMEjjzzCF198UVyeouMOHTqg0Wjo3LkzERERREZG0rVrV9RqY8qfQu+0QhdXLy8v0tLSCAsLY8iQIQwePJhTp24G6hW6uBbWV5wzZ87wyy+/EBwczFtvvcX168Ywza5duwLGTRo6dOjApEmT+PDDD839SMqkhWMLALPtdlIRNK6uxsm21FTS1m+oUl29x/gR0NsY+7TixT3k5ZS/kcX/9WtBOy+jV/bsn0pOSlGT1G1H2kpSaHnNRVRUFF5eXmi1Wtzd3dFqtcTFGV0xT5w4QceOHQGjK6peryc8PBw/Pz98fX05evQoBoMBlUpVZMGLu7hKKXnnnXdYvnw5TZo0oWXLlkXXwsPDGTFiBOHh4UyePJlLly4V5WQPCAhgypQpzJljXMnMz89nz549RT8iubm5PPPMM6hUKoYOHcqkSZNqLBurr6MvANEZtRP52+yrL4kIHkj8W2/hOHpUleoaMq0t2Rn5XDmVzM8LDjDlrT7lOsOsntWHgFf+4q9T11gXHktIJ68qyVAV6p0Frw2OHTtGv379CA4OZtGiRSxfvpzY2FhGjBhxS3pjd3d3xo4dy5NPPskLL7yAm5sb9957L3369GHgwIGU5nk3btw4xowZw6OPPoqz881AuzNnzjB48GCysrIIDAykd+/e/PDDD8yePZuQkBAuX77MoEGDGDRoEJtuG3NGRUURFBREYGAgbm5uuLu7V8/DKQG1So291p70vNpx/tB6emLdtSv61FT0GaZNkpXFyP90RKNVcSMll10ry/eetNSo+fPJ/gA8+fNRMnOrtoVVlSgt3WpdfVELaZNrg5JSMdcn7l51t5y7c26ttZ/y+2p5OqC1jP/wQ7PUl5mWKz95fJv85PFt8uqZZJPumbPymPR+cYN8aPl+s8hQGpSRNrneWXCpOLrUCxwsHNgfV+mgwaq3P3wYAMmff4Gs4iaQYAw1HfOMcU5k7UfHuHi0fFfc/47rAMCuC0msCqud4Uq9U/A7hddff71eRI+VhoXaolZm0QtRWVvjMGIEAGkbqjbZVkjTAOei5bO/vjhZbiJHC42KVTMCAXjut/Ba8VdXFFyhWhjcfDCZ+ZkkZ1c8m6m5cHv2WQBSvvvebHV2vdubPvcal2jXLwknKbpsl9buPi5FlnzKNwdq3JW1Tim4EOI+IcSy2pbjdk6ePEmfPn3o378/Dz/8cJnrysHBwYDRASYsLKzEMteuXeOtt94C4JtvvjG7vHUBDxvjjP2xhGO1JoNF0yZomzQh57x5l+u63NWcdgOMO9Js/fYUhnISRkzs1Rx/dzuSbuSx4Xj1J8IojtkVvLJpk4UQHYBManFr4tIICAhg79697NplzOBhyo4q06ZNK0q7dDuenp68/PLLQMNV8J6NewLUSH62snAYMRx0OrJPlZUIuOIETWhF87YuJMdksuOH8p2qfn6sN2Dsqmfn1ZwVrw4LXqm0ycA9QADQXQhRewuHJaDVaouOLS0tadbs1uD+Q4cO0bVrVyZOnEh6unGcVRhUkpubS0hICMOGDWP8+PGsWLGiKDBl3bp1nDhxguDgYLZs2cLUqVMJCgpi4MCB9Spdckm4WhuDLwq92moLu4EDAWNSCHMihCBoojEm4eqpZI5sjiqzZ+dmb8nogvXw+WtPlmv1zYXZHV2klDlAzm35ykpKm7wU+Of2+4UQnlLKf2c4NF6bDkwHaN68eakyXPvvf8k9UzVXVcs2rfGcN6/o87p165g3bx6tWrWiUaNbt6dduHAha9aswcXFBW9v71uurVmzhj59+jB37lxmzpx5y7WQkBA6dOhAaGgo+fn5LFq0iH/++QcpZZ3eC8tUnC2dazzg5HasWrcGIPe0+TdGdHC1pleIL2Gbo9j3x0UCenuW6c46K9iP9eGxrAqLZkaQL/7u9maX6XZqagzuRDlpkwuRUs4t41qtpE0GozKePHmSJk2asOG2WdnU1FSaN2+OnZ0drVq1uuVaZGRkkadb586dS61fq9UydepUHnroIV555ZV6b8EBNCoNOkMtOnkAKhsb7IcOJTs8nLwrV8xef/cRPgx6yPgj8tcXJ7h2qfQRZpvGDqx42Ji/7cXfTxCdUv3x8jXlqppKDaZNLm55zUFubi6WlsZfZgcHh6I0yYU4OjoSHR2Ns7Pzv7zVWrRowYkTJxgxYgTHjx+nR49bE/QVWmq9Xs+ECROYMmUK06dP59ChQ/Tq1cus36Om0ag0tW7BAWy6dyfj77/JOnQYizJ6fpWlsb8jzds14sqpZKJOJuPpW7qPRsemTvT1b8SeiGTColJo6mxjdnmKU1MWvF6nTS4M5wwKCiI+Pp6hQ4fecn3+/PmEhITwyCOP/GvoMHbsWPbs2cPdd9/NtWvXbhnPA/Ts2ZOxY8eyd+9eBg8eTN++fbl69SodOnSo9u9V3dQFCw7gGDIagNQKZnoxFTtnK0bP7oTWSs2V09c5sjmq1DG2i60F79/fGYA1R2M4dLl693IzKatqhSq8mTa5G3AEKEyb/DHQFWPa5CeqUH+9yqoKoNPp0Gg0zJw5kylTphAYGFjbItUIIWtCaOnUkveDazefu9TrOdvOGMHX+ng4opqGd+s+Pkr0uVSkQfLAvB64NS95jJ2nM3DXh/8QlZxFH79G/FQww15ZzJFV1WSklPlSyiFSSmcp5WBp3DUUKeVTUsr+VVHu+srIkSPp27dvUdDInUJdseBCrcbjJePUjiG7+rzrQp7qwujZnQDIzy19KcxCo+Kf5wcyoJUbWdW8ZFanHF1MoT76om/evJk9e/bwv/+Vumdjg0QjNOhk7Ss4gLAxjnWTv/m2WtvRWBhj/w9uuMQ/P51Dn1/6ZKm1VsWlxBs88+sxzl2retRbSdQ7Ba+ttMmhoaF4e3sTHBxM3759OXPmDE8//TR6vb5ozTs0NLQoldPs2bNrVL66iFatJV9f+5NsANYdjZY1bfXqam3H2dMGd297Uq5lcXJnDNfjMkstG9TKnUZ2lvxxNIZNJ6vHw63eKXhtWvDJkycTGhrKu+++y+eff85HH31UlK3ldpYuLXtz+juBumTBrQJa4TRhvFkiy8psx1bL/S/1YNCUNgDodaVb8Im9mrPjuWA0KkFeGeWqQr1T8LpAeno6Dg4OBAcHoyvlH6Zfv36A0Tf9pZdeokePHnz99deA0WmmW7duTJ8+vahcQ0Sr0taJMXghKgtLZF5ejbSl1hRsHllGF70QC42q2hS83qVsMiXp4q6V50m6alri+tJwbWZH/wdudVr5/vvv2blzJxcuXODvv/8u8k0vjwceeIDXX3+du+66i0ceeYTFixezc+dOUlNTi4JTGiIalYYsXc1tflAewsICQ2Ym17//AZfJ5eewrwqFCn50yxXOH45Ha6Gm56gWWFj/W+UsNCp2XkjkpdUn8He345F+LcwmR72z4LXdRd+5cyfHjh1jXgWcadq3b4+lpWVRvjS1Wo2trS1NmjTB1dW1usStderKLHoh1l06AxBfEMlXnTi6WePkYUPilQwuHkkgfNtVYiNSSyzbv6UbKVn5rD0WwxsbTps1C26DtOC3W15zY29vT3p6usn+4reXMxgMZGVlkZqaSlJSUnWIWCeoK55shdgPGoTrrJkkLfu82tuycbBg0gLj+nbilQxW/vdQqfuQL51gzBSzZNsFPthyHoMEtZlCERQLXgG+//57goODGTRoEM8//3yl63nhhRcYMGAAr776ao1lOq0N6poFB0CtBikrvfNJZSjMwlpeBJm6oFy+3nyy1TsLXlsEBwcTFXXrPhCjRhlT8r7++uu3lAPYvXs3YFxeK6TweMSIEYSEhBATE8P06dOrTebaRqvS1ikLDiDUxn95qdNVm0fb7agKFVxftoJrCsrpzRhKqih4LbBq1SqWLVtGZmYmS5YsqW1xqo26aMGFtuBfXqeDmlLwgv52aV30QjQFu6Do7mQFN2UMXtcZP34848ePr20xqp26qOAU+C1Ifc1lVTG1i65YcEwLF1WoG2iEhuScZLLys7DRVm9YpKkUdtFjnn6G5l8vr5E2Cy348e3RRIbfnFS1stEwcHKbos+FY/A5K49hqTH+EDWys+CtcZWPLKx3Cq5Qfwj0CiQsIQy9rNlMomVh3bkzVm3boq9BV2cbBwuatXEmMy2P1PibfgHW9rcOEbo0d6J9EwdiU2/uTJpVxSysZg8XrSm6d+8uTUl+qKDQ0CkrXLTOWHAhxNOALXBGSlm9EQEKCncIdSZtMpBSII+VuWVSULhTqQ4LXpg2+Y/CE8XTJgshlgkhegA2QPE0o7OllIlCiHeEED/L+jp2UFCoQ9SZtMkFcd4dgbzSlLt42mTghhDiHODIzc0SCo8L312ByviCFq/T1Gu3ny/r8+1yFj9XGZnLkreyMpd3XJ3PuLTrlXnG5pK5Op5xeXKaKq93qVdK23a0qi8gFNAUHL8MDCs4HgK8asZ2vrz9uNh7qduqmlqnqdduP1/W59vlrKrMZclbWZnLO67OZ1za9co8Y3PJXB3P2AQ5q/SMpay57YNTMVPa5BJYX8Lx+pIKVrJOU6/dfr6szyXJWRWZy7u3MjKXd1ydz7i065V5xsWP69ozvv3z7XJW9RlX3zKZECIUGCKl1BWMwR+XUj4uhPgMWCGlPFgtDd8qw2FZyvJBXaW+yVzf5IX6J3NV5K2OWXStEGIr0AnYLIToJaU8gnFcvgsw1IRyF/BlDbVjTuqbzPVNXqh/Mlda3nrr6KKgoFA+9S4eXEFBwXQUBVdQaMAoCq6g0IC5oxRcCNFDCPGqEGJhbctiCkKIjkKINUKI1rUti6kIIYYIIV4qwyW5TiGE6CWEeE4I8WZty1IRhBD3CSGWlVeuQSh4BfzfxwBvAIlCiMa1IiymyyulPA6sqS05i1MBmbdKKd8GSt4RooaogLwHAIHRa6xWMVVmIUQHIJOyPeuABqLg3PR/L9qWuLj/O2BR4P9uplyVVcZUeesSJssshHgG+KFWpLyJyfJKKd8FomtFylsxVeZBQADQXQjhVVaFdSZctCpIE/3fgXXAfMBCSlk9m0GZgKnyCiESgKFAKyHEAillbs1KepMKyNwV6AFIIcQBWUvrsBWQtxnQFmhUsxL+G1NlllIWWnJPKWVsWXU2CAUvBSfgYsFxGtCuoDt2oNYkKhsn/i1vFDCx1iQqHyf+LfNC4Itak6hsnPi3vEuBupx/wInbZC68IKWcW97NDaWLXhKpVJ//e3WQSv2SF+qfzKnUL3mhijI3ZAXfh3E8A8YItv1llK0L1Dd5of7JXN/khSrK3CAUvI75v5dLfZMX6p/M9U1eqB6ZFV90BYUGTIOw4AoKCiWjKLiCQgNGUXAFhQaMouAKCg0YRcEVFBowioIr/AshhI8QIkUIsUMIcaBg15mK1hFRDaIpVJCG7KqqUDXCpJRDhBBq4LQQ4ispZWZtC6VQMRQLrlAeNoAF0EgIsU0IESqE2COEaAUghFghhPhECLFRCLFfCOFe/GYhRD8hxCYhhGttCH+noyi4Qml0K/CeigOWFLwPl1IGA28CxQMdIqSUIzFG6z1QeFIIMQ54GrhXSlmZ3U8UqojSRVcojcIuelfgLYzx3Z8KITwxWvSM4mUL3q8AfgXHAngPY278LBRqBcWCK5RJgS90HMY94Y5KKQcAC7k1eUZxf2dR7NxI4HshROl7ZylUK4oFVzCFD4HtQKwQoj9w2pSbpJRnhRDTgB+FEFOllBfLu0fBvCjBJgoKDRili66g0IBRFFxBoQGjKLiCQgNGUXAFhQaMouAKCg0YRcEVFBowioIrKDRgFAVXUGjA/D/lu8UlxJKt1wAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 251.059x155.163 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Make a Zipf rank/frequency plot\n",
    "for code_subset,code_subset_name in zip([icd9_allcodes, icd9_chapters, icd9_subchapters, icd9_3dig, icd9_billing],\n",
    "                                       ['All levels', 'Chapters', 'Subchapters', '3 digits', 'Billing']\n",
    "                                       ):\n",
    "    code_subset = code_subset[code_subset.isin(oh_diagnoses_df.columns)]\n",
    "    freqs = (oh_diagnoses_df.loc[:,code_subset].sum(axis=0)/oh_diagnoses_df.shape[0]).sort_values(ascending=False)\n",
    "    plt.plot(range(1,len(freqs)+1),freqs, label=code_subset_name)\n",
    "plt.yscale('log')\n",
    "plt.xscale('log')\n",
    "plt.xlabel('Rank')\n",
    "plt.ylabel(\"Frequency\")\n",
    "plt.legend(frameon=False)\n",
    "plt.savefig(plots_dir/\"zipf_plot.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "02e76081-988f-40ea-8fef-dd48984584f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPwAAACfCAYAAADdy7STAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkyElEQVR4nO2deXhU1dnAfy8QSMRAREANikCRxRKLsu9DQKCiAStQXFgUFKifVu1nRUCNUREXioogKmrY/HhaXIpQiUIIGIILIAg1ttICiriASFgDWd7vj3tnmEySyU0yk5lkzu955pk755573vfOzHvP9p73iKpiMBgig1qhVsBgMFQdxuANhgjCGLzBEEEYgzcYIghj8AZDBGEM3mCIIOqEWoHK0LhxY23RokWo1TAYQsrWrVsPqWoTJ3mrtcG3aNGCLVu2hFoNgyGkiMg+p3lNk95giCCMwRsMEYQxeIMhgohcg5+TYL0MhgiiWg/aVYqcb0KtgcFQ5URuDW+IKFq3bs3y5cs9n10uF/n5+SQnJ7N27doied3nKkMgyggGkVvDG8KSXrPS+e7IqQpf3ywuhk1TE4uk7dixgz59+vDee+8xevToyqpYrTEGbwgrvjtyir2zhlb4+hZTVxdLe/vtt/nDH/7AzJkzOX36NPXq1XNU1sGDB5kwYQLHjh2jffv2zJ8/n379+rF+/Xpq1arFyJEjmTt3Lvv27eOBBx4gLy+PiRMncuutt3rKmDdvHkuWLCEmJobZs2dz1VVXVfjeAkFkN+kbNofkhmbwroazbds2unTpwpAhQ4o13/0xa9YsHnzwQdavX09sbCybN2+mZ8+ebNq0iZMnT3LixAkuvPBCHn74YVauXElmZibLli3jzJkznjL+/ve/s379etavX8+VV14ZjNsrF5Fdw9+703pPbhhaPQxB4z//+Q+7du1iyJAhnD59mjZt2jB0qLMWRHZ2NlOnTkVEOH78OF27dmXEiBEsXryYn376id/+9reA1WVISkoC4NChQxw8eNBTxqOPPsqUKVOoW7cujz32GBdccEHgb7IcRLbBG2o8b731FgsXLmTAgAEAJCUlUVBQ4Ojatm3bcsstt9CpUycA8vPzqVOnDvfddx8HDx7kmWeeAeDKK69kxYoV1K9fn7y8PKKiojxldOzYkdTUVN58801SU1N54IEHAnyH5aNMgxeR1qq6W0QaAGOAVarq2HfXYAglq1ev5q677vJ8vvzyy8nMzHR07bRp07jjjjvIycmhVq1avPrqq7Rs2ZJu3brx8ccf06xZM8CqxZOSkigsLKRRo0a89dZbnjImT57Mnj17OH36NG+88UZgb64CSFlBLEUkXVUTRWQOcAi4WlVdVaFcWXTu3FkrvHgmuSEk5xQ/NoSUYIzS13REZKuqdnaS10mTXuz3xqp6r4gMrLhqYcKcBGvAzhB2RJqxVjVODP6UiCwFtpbjmvAm5xtToxsiEifGewPQTlU/F5F6wN1B1slgMAQJJ/PwfwcuE5EoVT2tqp8HW6lQsTtxANnt2rM7cUCoVTEEkHXr1tGvXz/69u3L9ddfz3XXXcfu3bsrVFZGRgb//e9/A6xh1eGkhp8I3Ar8WUQ2AK+q6lfBVSs05B04QPuvsslu1z7UqkQucxIqt7CpYfOz/hVY8+IpKSmsWrWK2NhY/v3vfxcZtS8vGRkZ9O7dm1atWpWZt7CwkFq1wsu3rUyDV9VvgEdF5GlgPrBDRD4CHlfVjCDrZ4g0Kju+4uNEtXr1asaMGUNsbCwAbdq04aKLLmLu3Lls3bqVq6++mkceeYQ33niDRYsWcfz4cWbOnMmgQYMYP34855xzDrt27cLlcjFjxgxSU1N55513GDhwII8//ji33XYbP/74I02aNGHp0qUsW7aM999/n+PHj/PYY49x3333ISIkJCTwwgsvVOabCQhlPn5E5DciMhdYD3wJXAzcDDwZZN0Mhkrz/fffc9FFFxVLd7lcZGZm8o9//AOA3//+92RkZLBu3TqeffZZT75evXqxceNGtm3bxsGDBxk/fjyzZ89m9uzZLFy4kKSkJNLT03G5XKxYsQKAuLg4Vq9ezeHDh3G5XKxfv57nn3++am64DJy0Nx4E3lHV7qr6jKoeVNUfgYeCrJvBUGkuuugiDhw4UCy9Q4cOAMTExACQlpaGy+UiKSmJb7/91pPP7f+ekJDAnj17ipSRnZ3Nc889h8vlYtGiRfz0008AHs+8fv36UVhYyE033cTSpUsDf3MVwEmTfrSI1BaRi7Dn5FX1gKo6X4UQDrgXyHj17ww1n2uuuYYRI0YwevRoYmNj2b17N99//z0iUiTfk08+yYYNGzh9+jS9evXypO/YsYPLL7+cXbt2ceeddxIVFeVxzW3bti0DBgzghhtuACAvL49ly5Z5+u0FBQWkpKQAlovtmDFjquKW/eLEtXYicD/QBDiK5W3n16tHRMYC44DaWM3/m4BhwD5gvKrmicjNwJ3AYeAmVT0qIonAE0AuMEZV91f0xophItxUD9wrGCtzvRdNmjThoYce4tprr0VVadSoEXXr1i122bXXXkvfvn3p2rUrcXFxnvQNGzZ4lsVefPHFuFwupk2bxieffML999/P7bffzvz581FVnnyyaC/3008/Zdq0aeTl5TFwYJj4q6mq3xewBagLZAD1gNfKyN/MOw/Wg+If9vEDwEggCvgI64Hze+B++/x6IBboBswrS7dOnTqpYx5pYL3cx978pYPqXzrol23bqap63g2Rzbhx4/Trr78OtRplAmzRMmzF/XLSh89R1TNY/f0zQLsy8g8GaovIOnuwr6v9sABYC3QH2gA7VTXfnSYi5wCnVPWYqn4CXO5At4Aw+JJ4BseGXzgigyHQODH4z0QkBlgJ7ALKamZfANRV1QHASSAOqysAkAOcV0raeV5pYHUHiiEid4jIFhHZ4r3uuDIcOHGAA1HV32PYEFhSU1Np3bp1qNUIKE4M/nlVPaWqzwK9KNu1NgfYYB+nAy2ABvbnBsAR++Wb9otXGkBhSYWr6iuq2llVOzdp4mg7rbPY/cPBzZszeMXg8l1rMNQAnFRry4BEAFU9IiJ/BUb5yZ8F3G4fdwS+xeqnPw0MBD4G/g10EJHa7jRVPSkiMSJyLlZz/svy304Z2CP0BxYlwIniUzWG0DN4xWAOVOK3ia8fT9qINM/no0ePcuONN3LixAlOnz7N3Llz6dy56Jjz3r17mTFjRplTZ8nJyfTu3btCA3Dbt2+nsLAw5DHtSjV4EekD9AUuFZFpdnIU1qBcqajqdhE5JSIZWCP6NwEXiUgm8A3wnFqj9K9iDdz9YucBa4T+Q6xR+nEVviuHuP9c8fXjuWH1RLJ61cY41YaWAycOsHNcxadOExYVjU+4ePFifve73zFhwgTy8/M5daria+0rw/bt28nPz3dk8MF0yfVX6lGs/vpx4Dv79W/gxrIKVdX/VVWXqo5Q1TOq+pSq9lbVm+wBQFR1iar2VNWhqppjp61V1R6q2l8tl96g4v5zpY1II6/e+eRGxQVbpKGKOeecc9i8eTOHDh2iTp06bN26lRkzZgBWHz01NRWA7777jmHDhtG9e3ePg81LL71E9+7d6d+/P//6178AWL58OQMHDmTixIkArFmzhn79+tG5c2cWL14MWC2B8ePHM3DgQG677TYAXnnlFZ555hluvvlmVJUpU6aQmJjI0KFD+eWXX8jIyCApKYnrrruO9957j2uvvZb+/fszapS/xnT5KbWGV9UdWH7zb6vqMRGJAgZh9dGrPfH140OtgqEKGDNmDPv376d///5ccMEFHkP15YcffuDDDz9k69atPPXUU6SkpPC3v/2NTZs2Ubt2bQoLrSGlX//61yxcuJBBgwZx5MgR+vbty5AhQ8jPz8flcjF27FgALrvsMlJTU5kyZQoff/wxd9xxB/n5+UycOJH33nuP5s2b89JLL/H++++zYMECevTowZkzZ1izZg27d++mcePGrFq1yj21HTCcLo8FSAaGA8tLzVmNSBuRVqSvZ6iZREVF8fDDD7Nz504mTJjAyy+/7DnnbUwJCQnUqVOHjh07snv3bvbs2cNVV11F7drWZJG7ie12yY2PjycnJ4etW7cycOBABgwYwD//+U9PeW6XXHd53mRnZ7N8+XJcLhdPPPEEhw8fBvA091u3bk1CQgI333wzc+bMCej34cTg3d/KJap6OxAdUA0MhiCyb98+8vLyAGjatClRUVF8//33AOzceXasYNeuXRQUFLBjxw5+9atf0apVKz7//HNPze5+93bJVVWefvppFi5cyNq1a2nY8KyH4I4dOzzvv/rVr4q55I4dO5aMjAwyMzOZOXMmcPahcvr0ae69916WLVvGmjVr+PHHHwP2fTgZpa8jIo8D/ynHNQZDhYivH19s4K2813uzfft2Ro0aRUxMDFFRUbz22mtMmjSJa665hvPPP9+Tr2nTpgwfPpyDBw+ybNkymjRpwg033EDPnj2JiYlhwYIFJcq7/vrrGTZsGB07duS8887zpGdnZzNgwAAuueQSevToQdOmTRk/fjy7du3ihRde4O677yYx0Yrfd88999CgwdkZ6X379nkGGVu1akXTpk0r/H0UoyxXPCxHmuux3GtjgN86deML9qvcrrU2HVI7FDv94qR1+uKkdapqXGsNleORRx7RDz/8sMrkUQ7XWier5X4E3vFKej9wjxuDwVCVmOY5sHhaFtG5P5MbfX7ZmQ2GMkhOTg61CqUSXgG3QsSxw7n0/PjhUKthCAK7du2iZ8+e9OnTh1tvvdXvNJfL5QKs+fmtW7eWmOeHH37giSeeAOD1118PuL7Bxp+n3SOcHaEHOAVsU9V1QdfKELHsThxAXgkRapwSFR9P6/Szf9G2bduSlZUFwK233sqWLVvo0qWL3zLGjx9f6rkLL7yQ6dOnA5bBux1rqgv+mvS+cXyjgVtFZKCqPhhEnQwRjDtycEXxjTjsvbFjvXr1uOSSS4qc/+yzz5g0aRLt2rXj6FFrsabbZ75Pnz6MHDmSM2fOEBcXx5AhQzzBLEeNGsXOnTtxuVxMnz6dpUuXsnfvXmrVqsW6devCLlqtG3+edstKSH5NRDYGUZ+g4u07b4gcVq5cybRp02jTpk2RqTiAlJQU3n33XRo1asSll15a5Ny7775Lz549mTp1KlOmTClyLikpiYSEBDIyMsjLy2PWrFls2LABVS0WPiucCM/HUJDw9p03RA5JSUns2rWLZs2asWrVqiLnjhw5QvPmzTn33HNp06ZNkXN79uzhiiuuACyPudKIiopi3Lhx3HLLLcyYMcPjpBOOlGrwInKTz+s2EVkEfFKF+hkMleL06dOe4wYNGnii1Lpp2LAh+/fv58SJE3z99ddFzrVs2dLjjffFF18UK9tdkxcUFHDjjTeydOlSDh48yGeffRbo2wgY/vrwl/l8PgUsU9UPgqiPwRBQ1qxZw1/+8hfAWtAyaNCgIucfeughkpKSaNOmDc2bFw2AOXz4cEaOHMngwYM599xzi4wHAHTt2pXhw4fzpz/9ienTp1NQUECDBg1ISKi4p2Cw8Wfwi7FCVf3LnSAibUXkUlXdF3zVqpaoeNOvDwei4uMrtdWX7+84bNgwhg0bVmr+bt26sW3btiJp3vPob7/9NnXq1GHKlCm0atWKFi1aeAJlzJ4925Nv48bqMbTlz+Cfxwoj7c0JrO2mkoKmUYhonb6OFX/8G/MmpxPdPcUEwggR3lNq4cDQoUM5fvw4rVu3pkePHqFWp9L4M/iGqvqtd4Kq7heRSgQND2+WXZXCznE7mTc5PdSqGMKEtLSaNcBbkVH68J1zMBgMfvFn8OtE5CkRqQcgIvXsHWTDq81VCRZPy7Ka8Lk/h1oVQ5DIyMjg0ksvxeVy0atXL7Kzs7nnnnsoKCggOTmZtWvXkpGR4Ql7VZmtpKsD/pr0T2BtMbXNjkt/Emsgb7afa6oVxw7ncueCRHuQaGSo1TFgPYSPHc6t8PWxjaIZO7NnkbQxY8bw+OOPk5WVxYIFC/zu5Dp37twKy64O+PO0KwBm2S+DoUpwP4Qrir/xl6NHj9KgQQNcLhdr15a8F2rv3r3JzMzE5XLRo0cP1q5dy+TJk5kwYQIrV67k0UcfpVOnTnz55ZdkZmZWWM9Q4W/xzKsUXTzjQVXvCJpGBkOAWbJkCRs3buTrr7/mgw8+4KOPPnJ03ahRo0hOTubqq69mwoQJPPXUU2zcuJEjR454VtZVN/w16Su1obWI3Af8TlV7i8j9hHr3WEPE4m7S//jjj6VGrS2JDh06EBUV5VkIU7t2berXr0/9+vVp3LhxsNQNKqUO2qnqhtJeZRVqD/T9xj5uAvRX1d7AF8BwO+T1ZKyNLpYAk+xLH8IKhT0VqJIVednt2nucNeLz8isVT80Q3sTGxnpWxDnBdxFMYWEhJ0+e5MCBAxw6dCjQ6lUJwYp4MxFYBKRQfPfYm7C2kdqpqvkishZ4xXv3WOATEamSsQPvpZhp+w9Acg7zNpt5+FAR2yi6Un4QsY2KB1VesmQJmZmZ5ObmMmPGDJ599tkKlf3nP/+Zvn370rFjRy644IIK6xhSSgt2B1zsNDCez3VRwF/t40zgZmCy/bk18DrWppSz7LQ6WJtONgOWe5WzsZTy78Das35L8+bNnUf6e6RBkeCVix7cpAvH/bVYHlX1BLM0GLzJy8tTVdX9+/frNddcE2JtzkKA9odfBCAiK8v5DBkDvOn1+QjhsnusF8cO59Lvm5JDDxsMJbFixQr69evHsGHDPFFvqht+m/Qi8gbQSURe8U5X/6P0bYGOIjIZ+DXQGatZH/rdY30o5rdtbyddNEivwWAxevRoRo8eHWo1KoU/gx8KXIm15XNJ0W9KRFUfcB+LSKaqPioiD4Tj7rHFsLeTxvjSG2oo/hxvcoHNItIJOA9oCexRVcd+qGqNzKOqTwFP+ZxbgjVC7522FmtgL6jsThwAbapnk8xgqAxOFs+MxfKf/yOwVkSqV5jOEqhMVFSDoTrjZFpuEtDFbobXBTZijbQbDIZqhpMaXjg7Yl6IWR5rMFRbnNTwrwFbReQL4AqsiDcGg6Ea4mQzyVdF5B2gFdag3cHgq2UwGIKBI9daVT0EVE/n4RLI6p5SogumwVDTiaiNKNzkRp9fLEiCL4NXDK4ibQyGqsOvwYtILRH5fVUpE04cOGGm7gw1D78Gr6qFwI1VpEtYEV9ganlDzcNJk/4XEXlJRMa4t50KulZB4sX5+Y43OUj75htTyxtqHE4Mfi/wA9Yo/WUU34Iq/JmTAA2b0zQHR1sRR+f+zOKfXq4CxQyGqsXJtNyjACLSSFUPB1+lIJDzDSTnwMvOavd+3ywgzfjaG2ogZdbwInKNiGwHPhKROvYOsjUa97LZF+fnh1gTgyGwOGnSTwd6AD+paj5wSXBVCh+a5oRaA4MhsDgx+DOqeoqzIasjwpc+ttZPfNQjxVpKazDUEJwY/GYReQm4UETmYAWtqJZkdU9h3uR0R152Y5tOIq/e+WYpraFG4WTQbpqIDAb+A+xS1TXBVys45EafX6ldTQyG6o5T19p6WE35euIbrLsGE9somqzuKaFWw2AIGE5G6Rdjxbc7DPwWn7BUNZmxM3uSG30+2e3am768oUbgZLVcc1V12ceviUhG8NQJPIunZcGRlxlbiTLaf5Xt2EPPYAhnnDTps0Skq1h0wQozXW04djiXY4VNQ62GwRAW+Ns99musqTjBWkDjPs7D2vstIjgZncO8yelE9UwhKnFA8Vj2BkM1wt9mkpepahv7vaWqtrLf2/grUES6iUiWiHxkT+MhIveLSKaILLM3kkREbrbzrRKRBnZaoohsFpH1InJxIG+0otz/3PXcuSCRvLpmis5Q/XEyaHehiPxRRB4VkRQRKWvYeh+QqKp9gKYi0ocw3j22VBo2txbdGAw1CCd9+Pfs9y+AHfarVFT1B3sTC4B8rMCXGfbntUB3oA327rHuNO/dY1X1E6ztpkLHvTutRTc2x+r9TFavJ8xovaFa42SU/rCqPl/egkXkCqAx1kaRBXZyDtYuNnHAUZ+087zSAGqXUu4dWDvI0rx5c8f6LJ6WRXSu401zirHsqhQmb37eNOsN1RonNfxCEZkjIuNEZKyIlDnDJSKNgBeBCYTJ7rHHDufS8+OHHec3GGoiTgx+ChANXIy1Us7vajkRqQMsBe5X1R+Az4B+9mm/u8cCMSJyroh0pQp2jy0P8fXjORmdYzzvDNUaJ036XFWdUo4yRwJdgKdsL9wHgY2h3D02ttZPHIgqsYfgmLQRaSScSGDy5ufJbteeqPh4M0VnqHY4Mfg9IvI/wOfYS2RVNau0zKr6f8D/+SRvJoS7x45tOomEls0ZlhaY8oznnaG64sTgzwE62S+wjL5UgzeUn16z0vnuyCkAmsXFsGmqWdFnCA5OlsfeWhWKVBdiG0Uzb3I60d1TCEQd32tWOgB7Zw0FoMXU1QEo1WAoGSeON2+IyOver6pQLNS4DdGb+PrxLLvqUe5ckEhu9PkBmZP/7sipIjV6s7iYEmUbDIHA0bQc1g6yr2M53RwPqkZhgruJ7U3aiLQiseorOyffa1Y6zeJiiqS5jd8YvSEYOGnSb/L6mCki75Wa2eAId5+9tP76pqmJpmlvCAplGrzPTjOXAM69XWow7mg45enHu2vt746c8vTZDYaqxEmT3r3bTGvgBJAUVI3CiP3auNQFNO5oOE768b1mpRepsX2b8SXRLC6GFlNXm6a9IaA43nkmEul9+gX25vjfSs9JP74iNbq7qW+a9oZA4iQARjHKWhMfjrw4P5+o+PhKlxNfP56ERQnE148niQeIio8nu117DtVvxJirp3lqb+95dYMhXHASAKONbeB/Bn4GAuSvVrU0zaFSrrC9ZqXTa1Y6aSPS2DluJwdOHOB4bXir/XTaf5VN4xOHi9Tie2cN9XyujNGbaTpDIHEyD58kIpuBAcAIVb0r+GqFH+4a29v4Xoo9xbkFRfNtmppYZOTd93N52TQ1scQpQoOhIpRq8CJynYhkYUWguUFV71LV76pOtfDD1/iaxcVwvDbMm5zORz0eD1pwDFPLGwKFv0G7vwNfAW2BVHvlmwCqqoOqQLewxD16HtueIjX3vMnpQQuO4T0v7zZ8429vqAj+DL5llWlRjXAb2uAVLzB4xWDSRlTNkIb7QdMsLsY08Q0VplSDV9V9ValIdSNtRBoJi4rO0UfFx7M7SKGsvWt033l9X9wefGYVnsEXJ8tjI5ZmcTFWKI5SiK8f76nlLc+7x+jy9oSg61WW4bofCM3iYjwzBSU5/5gHQOQREQY/+OJ4nisjT0l9401TE9n/SGMuTm5IZr3GWFvsncW7lh87syeLp2WV2902GJTmn+9NSa0E8xCo+USEwR+IqoMVMbt0SusX9z79AntnDeXi5IZlyhk7syfzJqcHrVkfSEoybNMKqPlEhMFXBvdg2d7oks97e965m/YbWt5JXjWMe+ekFQDmQVCdiQiDr4xbreePnVzyefcovW/TPt01j+i8I1ANavvSKM2oTUug+hIRBt80B1p/UrbReU99FcO99dS9O0u81nsAb+zMnoC1+cUG7qzWRl8STlsCUNSt2DwUQk9EGLwvpTmv+F2hdu9O8NOPTxuRxuAVg4vMzbv79ECNDm3tz5C9PQRNqyD0hJ3B2zvOdga2qeofgyGjLMeVii52KcnoYxtFk9ZmOrSh2jfxK4LTboE35mEQPMLK4EXkKqC+qvYRkZdEpIuqfhYMWW7/dLeDSqCanm6j9wzkzTzribd4WhZpUdNJs2v96Nyf6ffNgoh6ALgpq1UQiDgA5sFRnLAyeKAHZzeicO80W2mD/9nesc43lpy3g0og/xju2t1t+G7ir4ov4oq7eFoWadFnHwDexDaK9owFRBqB+i0C9eDwR0nxD3y9IiF8xi9EtcQYFyFBRKYDW1V1jYgMBHqqaopPHs/usVgLe/5VRrGNgUMBV9bIMXLCR86lquoo1mS41fBHKL6rbBFU9RXgFacFisgWVe0cCOWMHCOnustxEsSyKnEH2oCzO80aDIYAEVYGr6rbgFwR+QgoVNVPQ62TwVCTCLcmPUGYinPc/DdyjJyaLiesBu0MBkNwCasmvcFgCC411uBFZI6IfCQizweh7G4ikmWXP8dOu19EMkVkmYhEBVjefSKSGUw5IjJWRNaJSIaINAuGHBE5R0RW2zL+LiL1AilHROJFZJuI5IpIHTutWPkicrP9+60SkQb+Sy1bjoi0tP8LG0XkTRGpHQw5Xuk3iMi3Xp8dy6mRBu/tsQfUFZEuARaxD0i0y28qIn2A/qraG/gCGB4oQSJSD/iNfdwkGHJEpBnQT1UHqKoLOBMMOcAQ4BNbxqfA6ADLOYw1y/MxlPx92UY/GegLLAEmVVYO1vTxdaraF9gDXBMkOW5GAN8ClFdOjTR4SvbYCxiq+oOquoNf5QNXABlBkjcRWGQfdw2SnMFAbbuGnxtEOf8B6tnHcUCLQMpR1VxV/cUrqaT7aAPsVNX8isr0laOqv6jqEftjPlAQDDkAIjIU+BAotJPKJaemGnwccNQ+zgHOC4YQEbkCy+PpSDDk2U/vfqrq9r2NC4Yc4AKgrqoOAE4GUc7XQDcR+SfWAqndQZLjJq6E8ktKCwgiEo/lP/JBEOWMA5Z6fS6XnJpq8Ecow2OvsohII+BFYEIQ5Y0B3vT6HCw5OcAG+zgdq+YNhpxxQJqq/hpYjTUtHMzf6UgJ5ZeUVmnsrtci4Ha7tg24HBFJBLJU9YxXcrnk1FSDD6rHnj2AshS4X1V/wFrg0y8I8toCU0RkDfBrrFoxGHKysLolAB2x+ofBkCNY/VKw/MBbBEmOm5J+l38DHeyBtUDKfAWYr6pf2p+DIacDkOT+P4jI4+WWo6o18gU8D3wEvBiEsm8EDmL1DzOwxgweADKxauS6QZCZab8HRQ7wrH0vK4C6wZCD1fxMs+V8CDQKpBwgCqsf+wuwDuhWUvlYLacsrFZGwwDI6Qcc8/o/XB8kOd18/w/llWMcbwyGCKKmNukNBkMJGIM3GCIIY/AGQwRhDN5giCCMwZcTEWkhIioiw7zSdgeg3EqX4afsGBFJF5H1ItLcK90lIgvt4zoi8ojtD77R9stuJSLjRWSP7f/+iYgsEJGy991yrlvQ7rsyiMhw7++qpmAMvmJ8BTwoIhJqRdwLNcqgI/CtqvZX1W9KyfO/wLlYnn19gTuxpucAXlNVl6p2w4ohGPAFSVWJiDj53w8HjMEbAPgO2AYM804UkWQRucU+7i0iqfZxqojMF5H37Vp2lIh8ICJbbXdM9/VPisgGEVnq/lN6pW0WkWu95KSKyEpglI8Ot9s18Scicpv9QHgJGCQiq/zc01jgYbXnaVV1n6p+5ZtJVecAfXyNRkR+Y7cCMkTk/+y07vYqrkyxwo6LiNSy72+DiMzyur6hiPzV9udPF5HWdv43xVqJtl5E+vrIdIlImoi8JSLbRWSknX6JWKvy0u33Jnb6bhGZCawTkXN8yvnUlvGGiFyOtdBnroj8zc6TZt/bpyLSw24RbReRuvb5sSLykJ/vNzwItINITX9heYetBS7G8moSYLd9Lhm4xT7uDaTax6nA3fbxAuB5+/ge4F77eC/Qwz5+FauGGQIssNPOAXbY8pKBl0vQrYmdp6792mGnuYCFJeT3pAN7Srnf8cAMn7SPgaY+aZuBy+3j2vb7FqCVffw6kARc79Yd6AXstY9nAaPt499gOQCdD2zibKCWWiXovwvLQaUBltdZLWA50N3OMwx41vc79innBWCQtwz7N+vtlae+/d4eSLePHwNG2MdrgYtD/f8s6xV2Ia6qC6q6X0S2UnRJp7cXk29z/3P7fT9WC8F9/Buva90x/D7BcqstBPqJSIadXg/LCMDyrPKlFdbKqTMAIrITaOnsjjglIjGq6n9bHosmFA/F3Fhtt1JVLbDTGqrqf730bYf1vXjfp/s7S8C618n253xV/VlEXgWWiMhJIAXrO/Pmc1XNA/JE5CdbtwRglt3jqoO1SAesVWwluZ4+AzwgIuOw1hK85n1SRGKA50WkrV1GM/vUQmC+iHwOnFRVX93CDtOkrxxPYrluujmMVfMDdPLJq6Uci9e7O+xwF6za6p/AB2r1n13AFarqNrQCirMHuEJE6tpNzQQ7zQmLgRT3uITdLG7nm0lE7gY2qWqhz6mD7vxezf0cEWllH/fE6v9/7XOf7vv/J/C0172615QvVdVbgI3AvSXo3dFuXsdirfo7ZJd1r11Wb87uY6BqV8c+/Kyq/wPcAkwVK4jEGc7GfBwCFKgV/+APbp1VdR/Wb/kIPg+JcMXU8JXAruW3YP0hAP4KrBQrIIZTQ3OTD9wgIk9jtQBWqmqB3V/MwPpj7cfymy5Nn59EZD6W7zhY6wgOOhxbfBaYAWwUEcVacnmXfW6CWBuDxGAFkri7hOunAC/b136Ptd7gbmCZiBRgGeFKrEpmhIhswKrh8+3rnwAWiMhdWAa1Cqtpvty+vm4pcg8Af8Nqycywv7M/AfNE5Fw7z+sUXVLqy30iMsjW7UNVPWqPd6SISDaWQT8oImuxuhjevAbMB27zU37YYHzpDdUWEXFhjZlMDKEOw4Euqjo9VDqUB1PDGwwVRETuw5olGVZW3nDB1PAGQwRhBu0MhgjCGLzBEEEYgzcYIghj8AZDBGEM3mCIIIzBGwwRxP8D2WULA1wJipIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 251.059x155.163 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot number of codes per stay distribution\n",
    "for code_subset,code_subset_name in zip([icd9_allcodes, icd9_chapters, icd9_subchapters, icd9_3dig, icd9_billing],\n",
    "                                       ['All levels', 'Chapters', 'Subchapters', '3 digits', 'Billing']):\n",
    "    code_subset = code_subset[code_subset.isin(oh_diagnoses_df.columns)]\n",
    "    n_codes = oh_diagnoses_df.loc[:,code_subset].sum(axis=1)\n",
    "    plt.hist(n_codes,bins=range(0,max(n_codes)), label=code_subset_name, histtype='step')\n",
    "plt.xlabel('Number of ICD codes per stay')\n",
    "plt.ylabel(\"Number of ICU stays\")\n",
    "plt.legend(frameon=False)\n",
    "plt.savefig(plots_dir/\"codes_per_stay_distribution.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9ebf3188-000e-4d45-8f28-a045df28df20",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True,  True, False,  True, False,  True,  True, False, False,\n",
       "       False,  True, False,  True, False,  True])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "icd_concepts_subset = icd9_concepts.concept_class_id\n",
    "value_counts = icd_concepts_subset.value_counts()\n",
    "for pattern, code_level_name in zip(['Chapter','SubChapter','3-dig','billing'],\n",
    "                                   ['Chapter','Subchapter','3 digits','Billing']):\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ca89c670-3f58-4840-ad0f-1f4c5a190335",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-dig billing code      8212\n",
       "4-dig billing code      3863\n",
       "4-dig nonbill code      1730\n",
       "4-dig billing E code    1247\n",
       "3-dig nonbill code       836\n",
       "5-dig billing V code     634\n",
       "4-dig billing V code     474\n",
       "3-dig nonbill E code     172\n",
       "4-dig nonbill V code     159\n",
       "SubChapter               154\n",
       "3-dig billing code        92\n",
       "3-dig nonbill V code      89\n",
       "3-dig billing E code      44\n",
       "Chapter                   19\n",
       "3-dig billing V code       1\n",
       "Name: concept_class_id, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "icd9_concepts.concept_class_id.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e701eb7-9b77-48ad-ab04-ad604ddc086e",
   "metadata": {},
   "source": [
    "## Performance metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "1503caa7-a259-46a1-8079-cece23fa1b3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def listdir_only(path):\n",
    "    \"\"\"List directories in a directory\"\"\"\n",
    "    f_list = os.listdir(path)\n",
    "    d_list = []\n",
    "    for i in f_list:\n",
    "        if os.path.isdir(path/i):\n",
    "            d_list.append(i)\n",
    "    return d_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "2889a647-7380-43fb-b432-aaabe39f5a2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_best_model_json_file(path, filename, n):\n",
    "    with open(path/\"best_models\"/str(n)/filename, \"r\") as json_file:\n",
    "        res = json.load(json_file)\n",
    "    return res\n",
    "\n",
    "def get_best_models_perfs(path, n, dataset=\"test\"):\n",
    "    \"\"\"Read evaluate json file for the Nth best model.\"\"\"\n",
    "    filename = \"evaluate_\"+dataset+\".json\"\n",
    "    return read_best_model_json_file(path,filename,n)\n",
    "\n",
    "\n",
    "def get_best_model_hyperparms(path, n):\n",
    "    return read_best_model_json_file(path,\"model_hyperparameters.json\",n)\n",
    "\n",
    "def process_model_str(df,sep=' '):\n",
    "    df = df.copy()\n",
    "    df['input_data'] = ''\n",
    "    for pattern,datastr in zip(['drugSexAge',\"drug\"],\n",
    "                            [\"Drugs, Sex, Age\",'Drugs']):\n",
    "        df.loc[df.model_str.str.startswith(pattern),'input_data'] = datastr\n",
    "        df.model_str = df.model_str.str.removeprefix(pattern).str.strip(sep)\n",
    "    df['code_subset'] = ''\n",
    "    \n",
    "    # Check if the last characters are numeric and process the index\n",
    "    model_suffix = df.model_str.str.rpartition(sep='_')\n",
    "    isnum = model_suffix.loc[:,2].str.isnumeric()\n",
    "    df.loc[isnum,'model_str'] = model_suffix.loc[isnum,0]\n",
    "    # set index to 0 for models without index\n",
    "    model_suffix.loc[~isnum,2] = '0'\n",
    "    # check if there is a second numeric character (my weird naming convention is \n",
    "    # inputvars_methodname_codesubset_crossvalid_modelid)\n",
    "    model_sec_suffix = df.model_str.str.rpartition(sep='_')\n",
    "    issecnum = model_sec_suffix.loc[:,2].str.isnumeric()\n",
    "    df.loc[issecnum,'model_str'] = model_sec_suffix.loc[issecnum,0]\n",
    "    model_sec_suffix.loc[~issecnum,2] = '0'\n",
    "\n",
    "    # folders named blabla_crossvalid_modelid\n",
    "    df.loc[issecnum,'model_id'] = model_suffix.loc[issecnum,2].astype('int')\n",
    "    df.loc[issecnum,'crossval_id'] = model_sec_suffix.loc[issecnum,2].astype('int')\n",
    "    # folders named blabla_crossvalid or just blabla\n",
    "    df.loc[~issecnum,'crossval_id'] = model_suffix.loc[~issecnum,2].astype('int')\n",
    "    df.loc[~issecnum,'model_id'] = model_sec_suffix.loc[~issecnum,2].astype('int')\n",
    "    \n",
    "    for pattern,code_subset in zip(['Sagi',\"AllCodes\"],\n",
    "                        [\"Sagi\",'All codes']):\n",
    "        df.loc[df.model_str.str.endswith(pattern),'code_subset'] = code_subset\n",
    "        df.model_str = df.model_str.str.removesuffix(pattern).str.strip(sep)\n",
    "        \n",
    "    df.loc[df.model_str==\"\",'model_str'] = \"Vanilla\"\n",
    "    df.dropna(axis=0, subset=\"metric_value\", inplace=True)\n",
    "    df.rename(columns={'model_str':'model'},inplace=True)\n",
    "    df.loc[df.model.isin(['Chapters','Subchapters','3dig','billing']),'model'] = \"Per level\"\n",
    "    return df\n",
    "\n",
    "def cleanup_per_level_entries(df):\n",
    "    df = df.copy()\n",
    "    code_levels = ['All', 'Chapter', 'Subchapter', '3 digits', 'Billing']\n",
    "    for pattern, code_level in zip(['Chapters','Subchapters','3dig','billing'],\n",
    "                                  code_levels[1:]):\n",
    "        exclude_code_set = set(code_levels) - set([code_level])\n",
    "        df = df.loc[~(df.code_level.isin(exclude_code_set) & df.model_str.str.contains(pattern)), :]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a50c5e6c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      model  metric_value       input_data code_subset  model_id  crossval_id\n",
      "0  _HierL2_           0.0  Drugs, Sex, Age   All codes       0.0          2.0\n",
      "1  _HierL2_           0.0  Drugs, Sex, Age   All codes       0.0          0.0\n",
      "2  _HierL2_           0.0  Drugs, Sex, Age   All codes       2.0          0.0\n"
     ]
    }
   ],
   "source": [
    "# Small test of the above model_str parser\n",
    "parser_test_df = process_model_str(pd.DataFrame({'model_str':['drugSexAge_HierL2_AllCodes_2',\n",
    "                                                            'drugSexAge_HierL2_AllCodes',\n",
    "                                                            'drugSexAge_HierL2_AllCodes_0_2'],\n",
    "                                'metric_value':0.0\n",
    "}))\n",
    "try:\n",
    "    assert (parser_test_df.model == \"_HierL2_\").all()\n",
    "    assert (parser_test_df.model_id == np.array([0,0,2])).all()\n",
    "    assert (parser_test_df.crossval_id == np.array([2,0,0])).all()\n",
    "except:\n",
    "    print(parser_test_df)\n",
    "    raise AssertionError\n",
    "\n",
    "print(parser_test_df)\n",
    "del parser_test_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b4a6c5d9-05ca-4b42-9401-55722dc73dcd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore', 'No positive class found in y_true, recall is set to one for all thresholds.', )\n",
    "warnings.filterwarnings('ignore', 'No positive samples in y_true, true positive value should be meaningless')\n",
    "def per_sample_auc_PR(y_true,probas_pred,pos_label=None):\n",
    "    prec = {}\n",
    "    rec = {}\n",
    "    thresh = {}\n",
    "    auc = {}\n",
    "    y_true = np.asarray(y_true)\n",
    "    probas_pred = np.asarray(probas_pred)\n",
    "    for row in range(0,probas_pred.shape[0]):\n",
    "        prec[row],rec[row],thresh[row] = skmet.precision_recall_curve(y_true=y_true[row,:],\n",
    "                                                                      probas_pred=probas_pred[row,:],\n",
    "                                                                      pos_label=pos_label)\n",
    "        auc[row] = skmet.auc(rec[row],prec[row])\n",
    "    return np.array(list(auc.values()))\n",
    "\n",
    "def per_sample_f1(y_true, y_pred, *args, **kwargs):\n",
    "    sample_f1 = skmet.f1_score(y_true=y_true.transpose(),\n",
    "                          y_pred=y_pred.transpose(),\n",
    "                          average=None,\n",
    "                          *args,**kwargs)\n",
    "    return sample_f1\n",
    "\n",
    "def per_sample_average_precision(y_true, y_score, *args, **kwargs):\n",
    "    sample_AP = skmet.average_precision_score(y_true=y_true.transpose(),\n",
    "                          y_score=y_score.transpose(),\n",
    "                          average=None,\n",
    "                          *args,**kwargs)\n",
    "    return sample_AP\n",
    "\n",
    "def per_sample_AUROC(y_true, y_score, **kwargs):\n",
    "    # Filter out samples without any code\n",
    "    #sample_auroc = skmet.roc_auc_score(y_true=y_true.transpose(),\n",
    "    #                  y_score=y_score.transpose(),\n",
    "    #                  average=None,\n",
    "    #                  *args,**kwargs)\n",
    "    #return sample_auroc\n",
    "    fpr = {}\n",
    "    tpr = {}\n",
    "    thresh = {}\n",
    "    auc = {}\n",
    "    y_true = np.asarray(y_true)\n",
    "    probas_pred = np.asarray(y_score)\n",
    "    for row in range(0,probas_pred.shape[0]):\n",
    "        fpr[row],tpr[row],thresh[row] = skmet.roc_curve(y_true=y_true[row,:],\n",
    "                                                        y_score=probas_pred[row,:],\n",
    "                                                        **kwargs)\n",
    "        auc[row] = skmet.auc(fpr[row],tpr[row])\n",
    "    return np.array(list(auc.values()))\n",
    "\n",
    "def per_sample_metric(metric_fn,y_true,y_score,*args,**kwargs):\n",
    "    y_true = np.asarray(y_true)\n",
    "    y_score = np.asarray(y_score)\n",
    "    sample_metric = np.zeros(y_score.shape[0])\n",
    "    for row in range(0,y_score.shape[0]):\n",
    "        sample_metric[row] = metric_fn(y_true[row:row+1,:],y_score[row:row+1,:],*args,**kwargs)\n",
    "    return sample_metric\n",
    "\n",
    "def per_sample_LRAP(y_true,y_score,*args,**kwargs):\n",
    "    return per_sample_metric(skmet.label_ranking_average_precision_score,y_true,y_score,*args,**kwargs)\n",
    "\n",
    "def per_sample_NDCG(y_true,y_score,*args,**kwargs):\n",
    "    return per_sample_metric(skmet.ndcg_score,y_true,y_score,*args,**kwargs)\n",
    "\n",
    "def sample_averaged_auc_PR(y_true,probas_pred,pos_label=None):\n",
    "    return per_sample_auc_PR(y_true,probas_pred,pos_label).mean()\n",
    "\n",
    "def sample_averaged_AUROC(y_true,probas_pred,pos_label=None):\n",
    "    return per_sample_AUROC(y_true,probas_pred,pos_label=pos_label).mean()\n",
    "\n",
    "def micro_averaged_auc_PR(y_true,probas_pred,pos_label=None):\n",
    "    y_true = np.asarray(y_true)\n",
    "    probas_pred = np.asarray(probas_pred)\n",
    "    prec,rec,thresh = skmet.precision_recall_curve(y_true=y_true.ravel(),\n",
    "                                                   probas_pred=probas_pred.ravel(),\n",
    "                                                   pos_label=pos_label)\n",
    "    return skmet.auc(rec,prec)\n",
    "\n",
    "def macro_averaged_auc_PR(y_true,probas_pred,pos_label=None):\n",
    "    prec = {}\n",
    "    rec = {}\n",
    "    thresh = {}\n",
    "    auc = {}\n",
    "    y_true = np.asarray(y_true)\n",
    "    probas_pred = np.asarray(probas_pred)\n",
    "    for col in range(0,probas_pred.shape[1]):\n",
    "        prec[col],rec[col],thresh[col] = skmet.precision_recall_curve(y_true=y_true[:,col],\n",
    "                                                                      probas_pred=probas_pred[:,col],\n",
    "                                                                      pos_label=pos_label)\n",
    "        auc[col] = skmet.auc(rec[col],prec[col])\n",
    "    return np.array(list(auc.values())).mean()\n",
    "\n",
    "def per_code_AUROC(y_true,probas_pred,pos_label=None):\n",
    "    fpr = {}\n",
    "    tpr = {}\n",
    "    thresh = {}\n",
    "    auc = {}\n",
    "    y_true = np.asarray(y_true)\n",
    "    probas_pred = np.asarray(probas_pred)\n",
    "    for col in range(0,probas_pred.shape[1]):\n",
    "        fpr[col],tpr[col],thresh[col] = skmet.roc_curve(y_true=y_true[:,col],\n",
    "                                                         y_score=probas_pred[:,col],\n",
    "                                                         pos_label=pos_label)\n",
    "        auc[col] = skmet.auc(fpr[col],tpr[col])\n",
    "    return np.array(list(auc.values()))\n",
    "\n",
    "def macro_averaged_AUROC(y_true,probas_pred,pos_label=None):\n",
    "    return per_code_AUROC(y_true,probas_pred,pos_label=pos_label).mean()\n",
    "\n",
    "def filter_no_positive_sample(y_true,y_score):\n",
    "    y_true = np.asarray(y_true)\n",
    "    y_score = np.asarray(y_score)\n",
    "    not_all_neg = y_true.any(axis=1)\n",
    "    return y_true[not_all_neg, :],y_score[not_all_neg, :]\n",
    "\n",
    "def filter_no_positive_label(y_true,y_score):\n",
    "    y_true = np.asarray(y_true)\n",
    "    y_score = np.asarray(y_score)\n",
    "    not_all_neg = y_true.any(axis=0)\n",
    "    return y_true[:, not_all_neg],y_score[:, not_all_neg]\n",
    "\n",
    "def one_error(y_true, y_score, pos_label=1):\n",
    "    y_true = np.asarray(y_true)\n",
    "    y_score = np.asarray(y_score)\n",
    "    top_pred = np.full_like(y_score,fill_value=np.nan)\n",
    "    top_pred[np.arange(len(y_score)), y_score.argmax(1)] = 1\n",
    "    one_error = (y_true==top_pred).sum(axis=1)\n",
    "    return one_error\n",
    "\n",
    "def average_relevance_position(y_true, y_score, pos_label=1):\n",
    "    y_true = np.asarray(y_true)\n",
    "    y_score = np.asarray(y_score)\n",
    "    ranks = (-y_score).argsort(axis=1).argsort(axis=1).astype(float)\n",
    "    #print(y_score[1,ranks.astype(int)[1,0:15]])\n",
    "    ranks[y_true!=pos_label]=np.nan\n",
    "    #print(ranks[0,:][~np.isnan(ranks[0,:])])\n",
    "    return np.nanmean(ranks+1.0,axis=1)\n",
    "\n",
    "def sample_averaged_ARP(y_true, y_score,*args,**kwargs):\n",
    "    return average_relevance_position(y_true, y_score, *args, **kwargs).nanmean()\n",
    "\n",
    "def precision_recall_at_K(y_true, y_score, k, pos_label=1, rectified_precision = True):\n",
    "    y_score = np.asarray(y_score)\n",
    "    y_true = np.asarray(y_true)==pos_label\n",
    "    arg_sorted_scores = (-y_score).argsort(axis=1)\n",
    "    sorted_truth = np.take_along_axis(y_true, arg_sorted_scores,axis=1)\n",
    "    summed_truth = np.cumsum(sorted_truth, axis=1, dtype=np.float64)\n",
    "    n_codes = np.sum(y_true,axis=1).reshape([-1,1])\n",
    "    n_codes = np.where(n_codes==0,np.nan,n_codes)\n",
    "    recall = np.divide(summed_truth,n_codes)\n",
    "    n_codes_seen = np.arange(1,y_true.shape[1]+1,1,dtype=np.float32).reshape([1,-1])\n",
    "    if rectified_precision:\n",
    "        def get_last_non_zero_per_row(x):\n",
    "            return np.where(np.count_nonzero(x, axis=1)==0, np.nan, (x.shape[1]-1) - np.argmin(x[:,::-1]==0, axis=1))\n",
    "        last_relevant_rank = get_last_non_zero_per_row(sorted_truth).reshape([-1,1])+1\n",
    "        precision = summed_truth/np.where(last_relevant_rank<=n_codes_seen,last_relevant_rank,n_codes_seen)\n",
    "    else:\n",
    "        precision = summed_truth/n_codes_seen\n",
    "    if k is None:\n",
    "        return (precision,recall)\n",
    "    else:\n",
    "        return (precision[:,k],recall[:,k])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "96f5a280-686c-4f66-a2e9-f2de804ab841",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert csv.gz files to feather format with half (float16) precision for faster reading (30x faster than csv.gz), half storage space.\n",
    "raise RuntimeError # This is a long computation uncomment this error to proceed\n",
    "models_dir = get_modelsdir() / \"mimic-omop/sequentialNN_3/\"\n",
    "met_dirs = listdir_only(models_dir)\n",
    "for metric in met_dirs:\n",
    "    exp_dirs = listdir_only(models_dir/metric)\n",
    "    for exp in exp_dirs:\n",
    "        exp_models_dir = models_dir/metric/exp/\"best_models\"\n",
    "        for n in range(1,2):\n",
    "            best_model_dir = exp_models_dir/str(n)\n",
    "            if not os.path.exists(best_model_dir/\"predictions_test.feather\"):\n",
    "                try:\n",
    "                    tmp = pd.read_csv(best_model_dir/\"predictions_test.csv.gz\")\n",
    "                    tmp = tmp.astype({'visit_occurrence_id':'int64'})\n",
    "                    dtype_dict = dict([(col,'float16') for col in tmp.select_dtypes(include='float64').columns])\n",
    "                    tmp = tmp.astype(dtype_dict)\n",
    "                    # Parquet Takes 2x more space than the csv.gz equivalent file with full precision and does not support half precision\n",
    "                    #tmp.to_parquet(best_model_dir/\"predictions_test.parquet\",index=False,compression='brotli') \n",
    "                    tmp.to_feather(best_model_dir/\"predictions_test.feather\")\n",
    "                    del tmp\n",
    "                except:\n",
    "                    continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "77ff763f-ef35-4fbf-9cd1-5f15cddd17b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ReCompute evaluation metrics using sklearn using test prediction files\n",
    "test_visits = pd.read_csv(datadir/'test_visits_id.csv.gz').squeeze()\n",
    "train_visits = pd.read_csv(datadir/'train_visits_id.csv.gz').squeeze()\n",
    "oh_diagnoses_df = pd.read_parquet(datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "test_oh_diagnoses_df = oh_diagnoses_df.loc[test_visits,:]\n",
    "train_oh_diagnoses_df = oh_diagnoses_df.loc[train_visits,:]\n",
    "test_oh_diagnoses_df.sort_index(ascending=True,inplace=True)\n",
    "icd9_concepts = pd.read_csv(datadir / \"icd9_concepts.csv.gz\")\n",
    "\n",
    "# Get frequent, seldom, few shot and zero shot labels\n",
    "n_train_oh_diagnoses_df = train_oh_diagnoses_df.sum(axis=0).squeeze().to_numpy()\n",
    "n_test_oh_diagnoses_df = test_oh_diagnoses_df.sum(axis=0).squeeze().to_numpy()\n",
    "frequent_icd9_codes = train_oh_diagnoses_df.columns[(n_train_oh_diagnoses_df/float(len(train_oh_diagnoses_df))>.01) & (n_test_oh_diagnoses_df>=1)]\n",
    "seldom_icd9_codes = train_oh_diagnoses_df.columns[((n_train_oh_diagnoses_df/float(len(train_oh_diagnoses_df))<=.01) &\n",
    "                                                  (n_test_oh_diagnoses_df>=1) &\n",
    "                                                  (n_train_oh_diagnoses_df>5))]\n",
    "few_shot_icd9_codes = train_oh_diagnoses_df.columns[((n_train_oh_diagnoses_df<=5) &\n",
    "                                                  (n_test_oh_diagnoses_df>=1) &\n",
    "                                                  (n_train_oh_diagnoses_df>=1))]\n",
    "zero_shot_icd9_codes = train_oh_diagnoses_df.columns[((n_train_oh_diagnoses_df==0) &\n",
    "                                                  (n_test_oh_diagnoses_df>=1))]\n",
    "top_50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "65e8644b-f638-4acc-ac4b-15caef58e14a",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_visits = pd.read_csv(datadir/'dev_visits_id.csv.gz').squeeze()\n",
    "dev_oh_diagnoses_df = oh_diagnoses_df.loc[dev_visits,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "42293fe1-b73b-4916-af87-ae8138cdc533",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_dev_oh_diagnoses_df = dev_oh_diagnoses_df.sum(axis=0).squeeze().to_numpy()\n",
    "zero_shot_icd9_codes = train_oh_diagnoses_df.columns[((n_train_oh_diagnoses_df==0) &\n",
    "                                                  (n_dev_oh_diagnoses_df>=1))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b164042-cb01-4b9d-9b4c-27cd8046c82e",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "raise RuntimeError # This is a long computation uncomment this error to proceed\n",
    "import multiprocessing\n",
    "# ReCompute evaluation metrics using sklearn using test prediction files\n",
    "oh_diagnoses_df = pd.read_parquet(datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "\n",
    "icd9_concepts = pd.read_csv(datadir / \"icd9_concepts.csv.gz\")\n",
    "\n",
    "\n",
    "#del oh_diagnoses_df\n",
    "gc.collect()\n",
    "\n",
    "metrics_func_dict = {\n",
    "    'micro_F1':lambda y_true,y_score: skmet.f1_score(y_true=y_true, y_pred=y_score>.5, average='micro'),\n",
    "    'macro_F1':lambda y_true,y_score: skmet.f1_score(*filter_no_positive_label(y_true, y_score>.5), average='macro'),\n",
    "    'samples_F1':lambda y_true,y_score: skmet.f1_score(*filter_no_positive_sample(y_true, y_score>.5), average='samples'),\n",
    "    \n",
    "    'micro_AUROC':lambda y_true,y_score: skmet.roc_auc_score(y_true=y_true, y_score=y_score, average='micro'),\n",
    "    'macro_AUROC':lambda y_true,y_score: macro_averaged_AUROC(*filter_no_positive_label(y_true, y_score),pos_label=1),\n",
    "    'samples_AUROC':lambda y_true,y_score: sample_averaged_AUROC(*filter_no_positive_sample(y_true, y_score),pos_label=1),\n",
    "    \n",
    "    'micro_AUPRC': lambda y_true,y_score: micro_averaged_auc_PR(y_true=y_true, probas_pred=y_score, pos_label=1),\n",
    "    'macro_AUPRC': lambda y_true,y_score: macro_averaged_auc_PR(*filter_no_positive_label(y_true, y_score), pos_label=1),\n",
    "    'samples_AUPRC': lambda y_true,y_score: sample_averaged_auc_PR(*filter_no_positive_sample(y_true, y_score), pos_label=1),\n",
    "    \n",
    "    'LRAP':  lambda y_true,y_score: skmet.label_ranking_average_precision_score(y_true=y_true,y_score=y_score),\n",
    "    \n",
    "    #'micro_AP': lambda y_true,y_score: skmet.average_precision_score(y_true=y_true,y_score=y_score, pos_label=1, average='micro'),\n",
    "    #'macro_AP': lambda y_true,y_score: skmet.average_precision_score(y_true=y_true,y_score=y_score, pos_label=1, average='macro'),\n",
    "    #'samples_AP': lambda y_true,y_score: skmet.average_precision_score(y_true=y_true,y_score=y_score, pos_label=1, average='samples'),\n",
    "    \n",
    "    'NDCG': lambda y_true,y_score: skmet.ndcg_score(y_true=y_true,y_score=y_score, k=None),\n",
    "    \n",
    "    'micro_precision': lambda y_true,y_score: skmet.precision_score(y_true=y_true,y_pred=y_score>.5, pos_label=1, average='micro'),\n",
    "    'macro_precision': lambda y_true,y_score: skmet.precision_score(*filter_no_positive_label(y_true, y_score>.5), pos_label=1, average='macro'),\n",
    "    'samples_precision': lambda y_true,y_score: skmet.precision_score(*filter_no_positive_sample(y_true, y_score>.5), pos_label=1, average='samples'),\n",
    "    \n",
    "    'micro_recall': lambda y_true,y_score: skmet.recall_score(y_true=y_true,y_pred=y_score>.5, pos_label=1, average='micro'),\n",
    "    'macro_recall': lambda y_true,y_score: skmet.recall_score(*filter_no_positive_label(y_true, y_score>.5), pos_label=1, average='macro'),\n",
    "    'samples_recall': lambda y_true,y_score: skmet.recall_score(*filter_no_positive_sample(y_true, y_score>.5), pos_label=1, average='samples'),\n",
    "    \n",
    "    'coverage':  lambda y_true,y_score: skmet.coverage_error(y_true=y_true,y_score=y_score, sample_weight=None)\n",
    "}\n",
    "\n",
    "for p in [10,25,33,50,66,75,90,100]:\n",
    "    metrics_func_dict['RE@'+str(p)] = lambda y_true,y_score,q=p: np.nanmean(kumet.rank_errors_at_percentile(y_true=y_true, y_pred=y_score, q=q, no_true_label_value=None).numpy())\n",
    "\n",
    "def read_preds_and_compute_metrics(best_models_dir, compute_missing_only = False, cross_val_set=0):\n",
    "    cross_val_str = '' if cross_val_set==0 else '_'+str(int(cross_val_set))\n",
    "    test_visits = pd.read_csv(datadir/('test_visits_id'+cross_val_str+'.csv.gz')).squeeze()\n",
    "    train_visits = pd.read_csv(datadir/('train_visits_id'+cross_val_str+'.csv.gz')).squeeze()\n",
    "\n",
    "    test_oh_diagnoses_df = oh_diagnoses_df.loc[test_visits,:]\n",
    "    train_oh_diagnoses_df = oh_diagnoses_df.loc[train_visits,:]\n",
    "    test_oh_diagnoses_df.sort_index(ascending=True,inplace=True)\n",
    "\n",
    "    # Get frequent, seldom, few shot and zero shot labels\n",
    "    n_train_oh_diagnoses_df = train_oh_diagnoses_df.sum(axis=0).squeeze().to_numpy()\n",
    "    n_test_oh_diagnoses_df = test_oh_diagnoses_df.sum(axis=0).squeeze().to_numpy()\n",
    "    frequent_icd9_codes = train_oh_diagnoses_df.columns[(n_train_oh_diagnoses_df/float(len(train_oh_diagnoses_df))>.01) & (n_test_oh_diagnoses_df>=1)]\n",
    "    seldom_icd9_codes = train_oh_diagnoses_df.columns[((n_train_oh_diagnoses_df/float(len(train_oh_diagnoses_df))<=.01) &\n",
    "                                                    (n_test_oh_diagnoses_df>=1) &\n",
    "                                                    (n_train_oh_diagnoses_df>5))]\n",
    "    few_shot_icd9_codes = train_oh_diagnoses_df.columns[((n_train_oh_diagnoses_df<=5) &\n",
    "                                                    (n_test_oh_diagnoses_df>=1) &\n",
    "                                                    (n_train_oh_diagnoses_df>=1))]\n",
    "    zero_shot_icd9_codes = train_oh_diagnoses_df.columns[((n_train_oh_diagnoses_df==0) &\n",
    "                                                    (n_test_oh_diagnoses_df>=1))]\n",
    "    for n in range(1,2):\n",
    "        \n",
    "        if compute_missing_only and os.path.exists(best_models_dir/str(n)/\"test_sklearn_metrics.csv\"):\n",
    "            continue\n",
    "            \n",
    "        try:\n",
    "            if os.path.exists(best_models_dir/str(n)/\"predictions_test.feather\"):\n",
    "                predictions = pd.read_feather(best_models_dir/str(n)/\"predictions_test.feather\").set_index('visit_occurrence_id')\n",
    "            else:\n",
    "                predictions = pd.read_csv(best_models_dir/str(n)/\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "        except :\n",
    "            print(str(best_models_dir) + \" directory skipped\")\n",
    "            #raise RuntimeError\n",
    "            continue\n",
    "        predictions.sort_index(ascending=True,inplace=True)\n",
    "\n",
    "        icd9_allcodes = icd9_concepts[icd9_concepts.concept_code.isin(predictions.columns)].concept_code\n",
    "        icd9_chapters = icd9_concepts[icd9_concepts.concept_code.isin(predictions.columns)].query('concept_class_id==\"Chapter\"').concept_code\n",
    "        icd9_subchapters = (icd9_concepts[icd9_concepts.concept_code.isin(predictions.columns)].query('concept_class_id==\"SubChapter\"')\n",
    "                            .concept_code)\n",
    "        icd9_3dig = (icd9_concepts[icd9_concepts.concept_code.isin(predictions.columns)].query('concept_class_id.str.startswith(\"3-dig\",'\n",
    "                                         'na=False)').concept_code)\n",
    "        icd9_4dig = (icd9_concepts[icd9_concepts.concept_code.isin(predictions.columns)].query('concept_class_id.str.startswith(\"4-dig\",'\n",
    "                                         'na=False)').concept_code)\n",
    "        icd9_5dig = (icd9_concepts[icd9_concepts.concept_code.isin(predictions.columns)].query('concept_class_id.str.startswith(\"5-dig\",'\n",
    "                                         'na=False)').concept_code)\n",
    "        icd9_billing = (\n",
    "            icd9_concepts[icd9_concepts.concept_code.isin(predictions.columns)].query('concept_class_id.str.endswith(\"-dig billing code\",'\n",
    "                                'na=False)').concept_code)\n",
    "        metrics_df = pd.DataFrame()\n",
    "        for code_subset, code_subset_name in zip([icd9_allcodes, icd9_chapters, icd9_subchapters, icd9_3dig, icd9_billing],\n",
    "                                                 ['All', 'Chapter', 'Subchapter', '3 digits', 'Billing']):\n",
    "            if len(code_subset)==0:\n",
    "                continue\n",
    "            code_subset_dict = {'metric_name':[], 'metric_value':[]}\n",
    "            #print(code_subset_name)\n",
    "            for metric_name, metric_func in metrics_func_dict.items():\n",
    "                #print(metric_name)\n",
    "                code_subset_dict['metric_name'].append(metric_name)\n",
    "                code_subset_dict['metric_value'].append(metric_func(test_oh_diagnoses_df.loc[:,code_subset].to_numpy(),\n",
    "                                                                    predictions.loc[:,code_subset].to_numpy()))\n",
    "            code_subset_dict['code_level'] = code_subset_name\n",
    "            metrics_df = pd.concat([metrics_df, pd.DataFrame(code_subset_dict)])\n",
    "        metrics_df.to_csv(best_models_dir/str(n)/\"test_sklearn_metrics.csv\",index=False,header=True)\n",
    "    \n",
    "models_dir = get_modelsdir() / \"mimic-omop/sequentialNN_3/\"\n",
    "met_dirs = listdir_only(models_dir)\n",
    "for metric in met_dirs:\n",
    "    exp_runs = listdir_only(models_dir/metric)\n",
    "    #exp_dirs = [models_dir/metric/exp/'best_models' for exp in exp_dirs]\n",
    "    for exp in exp_runs:\n",
    "        # process model_str\n",
    "        tmp = process_model_str(pd.DataFrame({'model_str':[exp],\n",
    "                                'metric_value':[0.0]\n",
    "            }))\n",
    "        exp_dir = models_dir/metric/exp/'best_models'\n",
    "        print(exp_dir)\n",
    "        #exp_models_dir = models_dir/metric/exp/\"best_models\"\n",
    "        read_preds_and_compute_metrics(exp_dir, compute_missing_only=False, cross_val_set=tmp.crossval_id[0])\n",
    "        #read_preds_and_compute_metrics(Path('/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_micro_aucPR/drugSexAge_billing_AllCodes/best_models'),\n",
    "        #                               test_oh_diagnoses_df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f9b04198-4de1-480c-a2e5-30ee298f9172",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "overall_macro_AUROC\n",
      "overall_macro_aucPR\n",
      "overall_micro_F1\n",
      "overall_NDCG\n",
      "overall_micro_AUROC\n",
      "overall_macro_F1\n",
      "overall_micro_aucPR\n"
     ]
    }
   ],
   "source": [
    "# Read sklearn computed metrics\n",
    "models_dir = get_modelsdir() / \"mimic-omop/sequentialNN_3/\"\n",
    "tables_dir = get_project_root()/\"docs/latex/overleaf/tables/\"\n",
    "plots_dir = get_project_root()/\"docs/latex/overleaf/gfx/\"\n",
    "\n",
    "\n",
    "met_dirs = listdir_only(models_dir)\n",
    "skl_results = pd.DataFrame()\n",
    "for metric in met_dirs:\n",
    "    tmp = pd.DataFrame()\n",
    "    print(metric)\n",
    "    exp_dirs = listdir_only(models_dir/metric)\n",
    "    for exp in exp_dirs:\n",
    "        try:\n",
    "            res = pd.read_csv(models_dir/metric/exp/'best_models/1/test_sklearn_metrics.csv')\n",
    "            res.loc[:,'model_str'] = exp\n",
    "            tmp = pd.concat([tmp,res])\n",
    "        except Exception as e:\n",
    "            continue\n",
    "            \n",
    "    if len(tmp)>0:\n",
    "        tmp.loc[:,'target_metric'] = metric\n",
    "\n",
    "    skl_results = pd.concat([skl_results,pd.DataFrame(tmp)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "7154fa6c-be1b-42b7-897e-974c05b48da2",
   "metadata": {},
   "outputs": [],
   "source": [
    "skl_results_long = (process_model_str(cleanup_per_level_entries(skl_results), sep='_')\n",
    "                    .reset_index(drop=True)\n",
    "                    .sort_values('code_level',key= lambda s: s.apply(['All', 'Chapter', 'Subchapter', '3 digits', 'Billing'].index), ignore_index=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb75f30c-526e-49a8-bda2-ac5678ca9734",
   "metadata": {},
   "source": [
    "## Performance plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e1be496f-bd6e-416c-b584-aaa126fed9b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def bar_plot_results(results_df,x_value,x_group,subplot_rows=None,subplot_cols=None,\n",
    "                     reference_row=None, reference_val='relative', values_subset = None, groups_subset = None,\n",
    "                     abs_value_in_percent = True, improvement_direction=\"max\", average=False,axis=None,\n",
    "                     legend_kwargs = {}, show_only_ref_value_abs = True,\n",
    "                     values_legend_dict = {}):\n",
    "    \"\"\" Make a bar plot showing the results.\n",
    "        results_df: a df with expected columns: [\"target_metric\",\"input_data\",\"model_name\",\n",
    "        \"code_subset\",\"code_level\",\"metric_name\",\"metric_value\",'metric_std'] (long table format)\n",
    "        x_cols: column(s) used for the x axis, the columns should be ordered from outer to inner\n",
    "        subplot_rows: a list containing the name of the column to use to put in different rows\n",
    "        subplot_cols: same with columns\n",
    "    \"\"\"\n",
    "    # https://scipy-cookbook.readthedocs.io/items/Matplotlib_LaTeX_Examples.html\n",
    "    # https://matplotlib.org/stable/gallery/lines_bars_and_markers/barchart.html#sphx-glr-gallery-lines-bars-and-markers-barchart-py\n",
    "    # https://matplotlib.org/stable/gallery/lines_bars_and_markers/bar_label_demo.html#sphx-glr-gallery-lines-bars-and-markers-bar-label-demo-py\n",
    "    # TODO check and enforce having n_groups*n_values in the df\n",
    "    if axis is None:\n",
    "        fig, ax = plt.subplots()\n",
    "    else:\n",
    "        ax = axis\n",
    "        fig = ax.get_figure()\n",
    "        \n",
    "    groups = results_df.loc[:,x_group].unique()\n",
    "    if groups_subset is not None:\n",
    "        groups = groups_subset\n",
    "    else:\n",
    "        groups = results_df.loc[:,x_group].unique()\n",
    "    n_groups = len(groups)\n",
    "    x_groups = np.arange(n_groups)\n",
    "    if values_subset is not None:\n",
    "        values = values_subset\n",
    "    else:\n",
    "        values = results_df.loc[:,x_value].unique()\n",
    "\n",
    "    for value in values:\n",
    "        if values_legend_dict.get(value) is None:\n",
    "            values_legend_dict[value] = value\n",
    "        \n",
    "    n_values = len(values)\n",
    "    bar_width = 1.0/(n_values+2)\n",
    "    x_val = np.arange(n_values+2)[1:-1]/(n_values+2.0)+bar_width/2.0\n",
    "    \n",
    "    # Get the corresponding subset of observations in case subset has been specified\n",
    "    results_df = results_df.loc[results_df[x_value].isin(values),:]\n",
    "    results_df = results_df.loc[results_df[x_group].isin(groups),:]\n",
    "    results_df[x_group] = pd.Categorical(results_df[x_group], categories=groups, ordered=True)\n",
    "    results_df = results_df.sort_values(by=x_group)\n",
    "    # Only keep necessary columns\n",
    "    results_df = results_df[[x_group,x_value,'metric_value']]\n",
    "    \n",
    "    if average:\n",
    "        results_df = (results_df.groupby(by=[x_group,x_value])\n",
    "                       .aggregate(func=['mean','sem'])\n",
    "                       .droplevel(0, axis=1)\n",
    "                       .reset_index()\n",
    "                       .rename(columns={'mean':'metric_value','sem':'metric_value_sem'})\n",
    "                      )\n",
    "        # Hacky bugfix: if sem is na then the yerr will not be created but still be used inside computations for bar_labeling \n",
    "        results_df.loc[:,'metric_value_sem'] = results_df.loc[:,'metric_value_sem'].fillna(0)\n",
    "    else:\n",
    "        # Check that if no averaging must be performed values are unique\n",
    "        if not (results_df.groupby(by=[x_group,x_value]).size() == 1).all():\n",
    "            raise RuntimeError('If not averaging all (x_group,x_value) tuples must have only one observation.')\n",
    "    \n",
    "    if reference_row is not None:\n",
    "        def standardize(x):\n",
    "            if reference_val=='relative':\n",
    "                if average:\n",
    "                    x.loc[:,'metric_value_sem']/=x.query(x_value+'==\"'+reference_row+'\"')['metric_value'].squeeze()/100.0\n",
    "                # get relative value\n",
    "                x.loc[:,'metric_value']/=x.query(x_value+'==\"'+reference_row+'\"')['metric_value'].squeeze()\n",
    "                x.loc[:,'metric_value']-=1  # get relative improvement\n",
    "                x.loc[:,'metric_value']*=100  # as percentage\n",
    "\n",
    "            elif reference_val=='absolute':\n",
    "                if average:\n",
    "                    # nothing to do\n",
    "                    pass\n",
    "                x.loc[:,'metric_value']-=x.query(x_value+'==\"'+reference_row+'\"')['metric_value'].squeeze()\n",
    "\n",
    "            else:\n",
    "                raise ValueError(\"Unknown reference_val value, allowed values are : relative, absolute.\")\n",
    "            return x\n",
    "\n",
    "        results_df_trans = results_df.groupby(by=x_group).apply(standardize)\n",
    "        if improvement_direction == \"min\":\n",
    "            results_df_trans.loc[:,\"metric_value\"] *= -1\n",
    "        elif improvement_direction == \"max\":\n",
    "            pass\n",
    "        else:\n",
    "            raise ValueError\n",
    "    else:\n",
    "        results_df_trans = results_df\n",
    "                \n",
    "    # TODO check that values are sorted!!!!\n",
    "    if reference_row is not None:\n",
    "        ax.hlines(0,xmin=0,xmax=n_groups,\n",
    "                  label=values_legend_dict.get(reference_row)+\" (baseline)\")\n",
    "        if reference_val=='relative':\n",
    "            ax.set_ylabel('Relative improvement over baseline (%)')\n",
    "        else:\n",
    "            ax.set_ylabel('Absolute improvement over baseline')\n",
    "        tmp = results_df_trans.loc[results_df_trans[x_value].isin(values),'metric_value'].abs().max()*2.0\n",
    "        ax.set_ylim((-tmp,tmp))\n",
    "    for val,x in zip(values,x_val):\n",
    "        extra_kwargs = {'error_kw':{'elinewidth':.75, 'capsize':1.5, 'capthick':.75}}\n",
    "        if average:\n",
    "            extra_kwargs['yerr']=results_df_trans.loc[results_df_trans[x_value]==val,'metric_value_sem']\n",
    "        if reference_row is not None and val==reference_row:\n",
    "            rects = ax.bar(x+x_groups, 1e-10,**extra_kwargs)\n",
    "        else:\n",
    "            rects = ax.bar(x+x_groups, results_df_trans.loc[results_df_trans[x_value]==val,'metric_value'],bar_width,\n",
    "                           label=values_legend_dict.get(val),\n",
    "                           **extra_kwargs)\n",
    "        unseen_bars = results_df_trans.loc[results_df_trans[x_value]==val,'metric_value'].abs()<tmp/100\n",
    "        ax.scatter(x+x_groups[unseen_bars],np.zeros_like(x_groups[unseen_bars]),color=rects.patches[0]._facecolor,marker='|')\n",
    "        if ((not show_only_ref_value_abs) | \n",
    "        ((show_only_ref_value_abs) & (reference_row == val))):\n",
    "            if abs_value_in_percent:\n",
    "                ax.bar_label(rects,\n",
    "                             results_df.loc[results_df[x_value] == val, 'metric_value'].apply(\n",
    "                                 lambda x: f'{x:.1%}'),\n",
    "                             padding=7, fontsize=7, rotation=90)\n",
    "            else:\n",
    "                ax.bar_label(rects,\n",
    "                             results_df.loc[results_df[x_value] == val, 'metric_value'].apply(\n",
    "                                 lambda x: f'{x:.1f}'),\n",
    "                             padding=7, fontsize=7, rotation=90)\n",
    "                \n",
    "    ax.legend(frameon=False,**legend_kwargs)\n",
    "    ax.set_xticks(x_groups+.5, groups)\n",
    "    ax.set_xlim(-.1,len(x_groups)+.1)\n",
    "    #plt.show()\n",
    "    return fig,ax\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fd6cf70",
   "metadata": {},
   "source": [
    "### Fig2: Best target metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7523c9e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_pretty_names_dict = {\"overall_micro_F1\": r\"$\\mu$F1\",\n",
    "                             \"overall_NDCG\": \"NDCG\",\n",
    "                             \"overall_micro_aucPR\": r\"$\\mu$AUPRC\",\n",
    "                             \"overall_micro_AUROC\": r\"$\\mu$AUROC\",\n",
    "                             \"overall_macro_F1\": \"macro F1\",\n",
    "                             \"overall_macro_AUROC\": \"macro AUROC\",\n",
    "                             \"overall_macro_aucPR\": \"macro AUPRC\"}\n",
    "plots_dir = get_project_root()/\"docs/latex/overleaf/gfx/\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "2c32b56c-ed7b-428a-8186-752ab7f912c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPMAAACTCAYAAABbAi9lAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAi7ElEQVR4nO2daXhURdaA35NggLCEJQQYlmEZZRNEtnFUBEWRTRjWRBYTgUhAXBHjwgjqsI2AoqgIahIFWQZQhGFQ9sFxI4yyiuAHMqIgYJAAGQmQ8/24nSbpbJ3O7SVNvc/TT/e9t27VSeWeW1XnnKoSVcVgMJR+QvwtgMFgsAejzAZDkGCU2WAIEowyGwxBglFmgyFIMMpsMAQJPldmEWkkIm+JyDLH8WARmS8i74hIBV/LYzAECz5XZlU9qKojcpzqq6rxwFKgn6/lMRiChTL+FgDIjlo5DLR0vSgi9wH3AVSoUKFt06ZNfSiawRB4bN++/aSq1nA9HwjKnE194IjrSVWdB8wDaNeunaampvpaLoMhoBCRw/md97kyi0h1YDJwvYg8CXwgIq8D5YH7fS2PwRAs+GPM/IuqJqhqY1WdqqrvqepoVY1T1XO+kCE5OZmmTZvSuXNnkpOTfVGkweB1Aqmb7TPi4uJITk5m8+bN/hbFYLAN42c2GIpJoPbsrghlDtTKN5RO4uLiqFWrFps3byYuLs7f4ji5IrrZplttuBKwVZlFpCrQBMt3vF9VT9mZv8FgKBhbutkiMlBElgOvAX2AvsDrIrJCRAbaUYY3Md3w/PFVvXz77bfEx8czcOBAHn30UY4fP+61sorD2bNnvZa3V+pWVUv8AXoDks95AXrbUYaq0rZtW/WUTp06FevYYOGLehk5cqQeP35cJ02apN99953ec889zmtJSUnapEkT7dSpkyYlJeU59ib169fXuLg43bRpU55rdtSLp3kAqZqPftjSMqvqh45CEJEQEblDRPoCFVT1QzvKCHYCoXfgLxkyMzOpVq0ax44do3HjxlSocHm+jauxyZfGpx49evDCCy+wbds2evTowbRp07xaXkmxq5stOQ6nADWBcMAospsEgoXUXzL07NmTIUOGMGKENf/mrrvu8lnZRREZGcn48eNZs2YNN9xwA0yKsD7ff2J954O/Xop2uabeFpFox28pNKUB8N8/fO/evSQkJDB06FBGjx7N3r17fVZ2QQwaNIjFixfTrl07ALp37+72vUXVY0nq+dlnn8113LlzZ7fu89dL0RZrtqreKyI9RORt4B/AWayW+c925B+M+MtdNnXqVF599VUqV65Meno6Dz74YMAZ/WJjY0lJSXErbVH1WJJ6joqKynU8d+5cEoqdi++wq5tdESiHNSc5FIgBMlQ13Y78DfahOdZJzz06yk3LlJa0TGnJtmPbaJmSZ2aqV3EqshtdWl/SunVrf4tQKHZ1sz8AwoC6QEdgOFDd0VIbAognn3ySxx57jOjoaB577DHGjx/vb5G4+eabmT9/fpGuIF+/YNatW5fr5XfDDTfYku/M6F7MjO7FD3t3MTO6ly15gv0RYJeAEIdl+z0RWWxz/iUm+yE4eOwgLVNasit2l9fKOnv2LBUrVvRa/p7QokUL5s2bl/dCdsv3/Tnrd8P6eZIkJyczbdo0atWq5bQs20GzZs2IiopixIgRVKlShXvvvdc2xSkJiYmJ1K1blzZt2jB8+HDq189bJ4GEXS1zXyATOAo8lH1SVbNsyj+gcNeo0qJFC+69994ix2v+7NLGxsa6ndZbhp0yZcrQp08flixZwoQJE1i7dq0t+bq2gMVtBf/4xz/y4Ycf0rNnTyZPnszAgYEd/2SXMt8KvK+qa1X1YvZJsehjUxkBg7sPdWnwU7praPImQ4YMcf6uV68ekyZNcuu+b5o245umzcj48ku+adrMdrkiIqzeSvv27XnjjTdISkoCIPnrTI6dzaJz8jmSk5MDIkYA7FPmMGC5iLwvIrMdn5XAcq6QyRwFkcdPCX71Vbo7PvUET2W++eabnb/tapXtwPXlmz1kimsdxr6xldgcVyFPIMsfp0336gumMOxyTS0DljkmWvwBy9d8QH040eLpp58mLi6Oq6++2ldFFomnfkrwnuvK0/Fp9oOZ8d/DfNO0Gc32fWObzJ07dyYqKgpVZefOnVx33XUsXboUyN0Knrr9FFU7Vi1W3iXhjTfeIDU1lS5durBgwQJ69uzJaJ+VXnxsnc+sqqdUdZuqfulLRQbYsWMHr7/+Ov369WPBggWcP3/el8XnS35+Sm/hbqtYnPHpqa2nuHj6IgenHuT907/aL7SDYcOG0b59e9577z169OjhVGTI3Qr6UpEBNmzYwLx585gzZw6rV69mx44dPi2/uATN4gT16tVj1qxZLFq0CBEhJibG3yLlwZt+SnfH8YWNT3O1glutVvCaadfQ6MlG9I2o4jXZR4wY4Zwx9dNPPxWa1lcvGIDq1asjIiQmJgJQtmxZ2/LedugHzvx2ntc2fWbbMMpWZRaRIS7Hfd28r7OIbBWRuSLS2ZOy27dvD1gVPmTIEN5///3LY9NCxqfeNKL89ttvLFy4kL/97W98+OGHhXZncz6kU2rXLlKmko5Pd+/ezeeff57rWklawZL4TtesWcPevXvp2rUr586dY82aNQWmLe4LJqfSbDv0Q7HkeughyzGTHSver599ezS0b1iPxO6dGXPrn2zzDNimzCISCox0WLBDRCQMGOnm7YoVAlqOfNbOdofhw4d7cptXGTduHGFhYfz444+cPHnS+XDkR3EfUk/dRI8//jgzZ84kKSmJLVu2MGrUKLfvff/0r5y8eJHY/x621Sj3zDPPsGfPHtLS0rhw4QInT560Le+cStO+Yb1i3Vu5cmU++ugjzp07x8svv0y1atVsk8sb2BXOGQusB1oDGxyfVcDHbmaxVVW7A4lALquRiNwnIqkiknrixAm3ZSqO/9RbXLp0iYEDB6KqDB8+nIsXLxZ9k5c5c+YMBw4cYObMmSQmJhIaGur2vX0jqvCPRo1Jqf97W/3M//rXvzh79izlypWjRYsW3HPPPW7f660XDMADDzzAqVOn6NWrF23btuW5557Lk8afMQKu2DWfOUVVbwW6quptqnqrqt6pqrPdvD87uOQUUNbl2jxVbaeq7WrUyLMjh5Pp06cTFxdHSkoKAwcOpGbNmp7+OXnwtEtbo0YN+vXrR4cOHQC45pprLufpMj71FYcPH+aXX35xHmdkZHitLHfrLTw8nGeffZbIyEjCw8OLVYYnLxh35apatSoxMTGEhIRw0003ERkZWSzZioMdrki7fcDXisgMrLBOAFT1tqJuEpF+wJ1AFWCOJwUfOHCA5ORkbr/9dtavX8+YMWOseVs24KnLpU6dOhw9epQyZcrQq1cvevbseTnP1mHEtQ4DoGVD31lpXcejzZp5zxda3Hrr0qULXbp08Zo82bgrV4MGDYiOjiYmJoY+ffrQokULt/LP2VsYk5zs1kvGDlek3cqcAHRU1czi3KSqK4AVJSn4xIkTzJ8/n6ysLD755BNOnDgBUUXf5002btzIkiVL6NixI5988gmjR7vvpfTkgXCHQYMGOX+rKl9//bXTWltSchqbqtsos7+YMGGC83d8fDzTp0+H/xV9X9+IKk67R7PSNp85BxuApiKyB8fujr6Kz54zZw779+9n2bJlJCUlWS6Xv7s7ZPcOrq6NsLAwt+/11gNRuXJl3nzzTedxcV4wRdG+YT2nkam0KzIU8OIbUsgNfsZuZf6T45ONAkV2s+2gXr161KtnPUjjxo2zTv7dFyUXjDddG57y9NNP5zqePHmynyQJfPJ/8b3nP4GKwO4IsFuBrsAQhxHMJ4pcXHwVeOC6l3SnTp28Vpa7NGzYMNdxoLtb7ObIE1s58sRWzh88zZEnthaatrS9+OwOGokF1gD/FJFQEfFz25g/vops8hXFeUB9hatM7sqVy6r7dbFML7bj7ouvpI2DXf8/u8M5R6rqHUCaql4CqtucvwHvrVQRCOQKhmntvo3BnwRK42C3Ml90rAemIlIeCMrFCQyGQMRuZU4ElgHNHN9P2px/kQRSN81g8CV2G8C+VNVuqlpbVXuq6jY783cHb3fTPB3fBOJLJhBlMniO3QawO0RkpYhsEZF/ici/7MzfW3gzvjebQBwLBqJMvmDprn9y4lwaA99zb83w0vLSs9vP/Degh6oetTlfr+KviB2DfxjUsjuDWlq7ZtSN61hk+lyhlh6s3+26qqm3sHvMvBu4YHOehlJKzhZw6a5/+lscv+HOdNXi9hbyw+6WeQuwT0SyF4hSVb3F5jKKjT/XkbqSydkCGgqnuL2F/LBbme8DflfciRbepiQzlLL9uNk+3eiG9kxKMBjspshutog84fjuJiJfisiDhST/GGuiRahjtZGgWWMs0Mg5Q+lK7sIaLuNOy3wHMA0YAtwEfAq8XEDamxyfbHw20eJKI+cMpUENu+cac41q+lBQzFoyFA93lLmciNwDHFfVCyJS4IxOVb1VRK4Caqhq4cssGmzFjjGXoXTjTjf4Hiylnygi5ShkJZDSMtHCYAhG3FHmC0AHYK7jd+VC0pqJFgaDn3BHmd8GXsKyUl8C7i4krZloYfArOaO1irtOdmnHHWUOVdV9bt7j94kW0W98xt6f0n1dbJEEolzBKFPOAI3irpPtLZl8hTsGsI0iMhf4nYjMBtYVkvZ7Ve1WHAFEpALwGtb+zptVdWFx7jf4j5xhirUyTtmmPAbPKFKZVfV5EbkWa7G+b1V1ZyHJ54hIOPBPYIWbMdr9gGWqukpElgBuK3P0G5/lObf3aDrnMi9a185PyHM943C5XMeP36y5jsNd8jxSqzcAJ8MOsKJWb7ZyDoADXGIs5yibjwz5UZhcnsqUTbZM2fhOpiakZYVz3d1T+WHvLnJ2al1lclcuO+pp70/pRL/xWZ56yk8uX8sExf//LRn1J9yhQGV2uKNcaS0irVX1nfzuUdVBjrFyd+AFEamrqp2LkKEusMvx+5LrRRG5DyuyjPr16xeRlQsNbs5zqp3rCZe18n868Gtu4ZpbuxQc+GcF52+AkHJlKNuw+EH3+cnlqUy24qFM5/93kZ8O/GpkclMm8NL/j8JbZnF83wmcAbYD1wMRQL7K7KANcAuWkn7hhgxHHGm/Jp/xuKrOA+YBtGvXLtdrL783VvQbn/FzWBm332auvJqwMdfx/Y58Oi+qnCtP1+OiKIlcryZs5PNv1/LxV4upHF6NEVMetiUopKQyARy7FELM2bLOejIyFS4TYJtcrhSozKqaAiAi0ao6OPu8iBQYOygim4F/Aymq+rCbMqzA6p73xNqfypAPNzTpxufffszDvWcRF2eC6gx5cccAlikiTwJfYW0MV+AUR1XtLCK1gQYiUtudMbOqngPudVPeIlky6k90XlSYK7xocraCFW6wpxW0Qy67MTK5RyDKlB/uuKYGAt8BrYD/AwYUlFBExmP5pXsDSSJSKqcY3dCkG5XDqzlawbg815OTkzl27FiJNvkyGOzGnZb5Etb4uYzjO4+RKge9VdUZGCwinwDTSyRhABIXF+fTiQz3z7W61X/fV8X525CXQKwnX8rkTsv8LvAHLAPY1cCCQtJmisiNIlJeRDoSJKuOmJbYUBpwp2Wuq6rZ22V9JCJbCkkbhxUF9hesLrn/dzy3AV+3xKWBQGwFr3TcUeYfReRp4D9YbrV8jVoiIsBkVXV/23uDwWAb7nSzhwHfYhnA9gFD80ukqgocFZEOIlLGrDRiMPgWd1rmfqr6d3C2vgMoeLPUDo5PNmalEZvIOW433X5DfrijzAk4lFdVVUScx644tnQ1eAGjwIaicKcbXFZEqgKISDWgXEEJRaSHiHwiIptFZKsjqstgMPgAd1rmx4GVIqJY3ebxhaSdBHRW1QzH7KnNwD9KKqTBYCiaIltmVf1UVW9R1U6q2llVPy0k+VdA9sLUVbAmT5Qq7p97G/fPvY061xiXS2nEGzEBpSXOwG5rcytgi4h8gbW7RUtHd7tUbCDnTQLxgSipTIH4N8XFxbFv375Ct4IJhDy9gqqWmk/btm21KJKSkrRJkybaqVMnTUpKKjJ9QXTq1Mnjew0lIxjr3s6/CUjVfPSjsMUJCuxjqurG/M6LSEtgBFZXWxxpfRpEYqy+hiuVwgxgBa2krkC+yowVxz0aMAvgGww+pjBlft6D/P4PqwsQFBMsDL7HBMd4TmHKvAGrFc6JUHhU11LgvyLyneNYNQC2dDWUHowCe05hywblieZyxFrnXSXvMk8AbdS9VTkNBoONuLOl61Ui0l1EXsdar6uwdbG3EiRzmA0Gu/CVC68wa/Yg4C4gHPgIuEZVuxSRXzvg3yJy3HFsutmGKx5fDR0Ka5knY+0yMUWt5W7PF5WZqt6oqk1UtaPjUyoVORCDIQyGohDLB13ARZFWWDtOtAGuxfIhf6qq513SPamqU0XkXVyMZkX5mUUkDhgE/Bd4VVV3FZS2Xbt2mpqaWugfZCg+rmuFu4sJd/UPIrJdVfOsv1/oRAu1tqLZ6cigEdAfeAZwNY697fjOux9M0WQBGUAo8LPrxRLtaGEoNWRlZRESkn9HcfPmzfzlL3+hRYsWtG/fngEDBvDII49w6NAhNm3a5GNJAxd3Zk0BoKoHgRccH9drPzu+DxeWhyNCbKrL6eGq+o6jF/AE8KhL3vnuaJHfPlPuUNSuBMnJySxdupT69evTrVs3WrVqxeTJkzl9+jTLli3Lk/6hhx6iSZMm1K9fn169enkkU3a5kZGRZGVlUbVqVTp2LChm5zLbt28nNTWVUaNGeVyuHSQnJ7Np0ybKly9P7dq1uXDhArt372bp0qWsW7eOLVu2cPz4cWbNmsWZM2eYMGECUVFR9O3bl/Xr15OWlsb1119PZmYmO3fuJD09nZdeeonq1S9v7x0dHc3YsWOdx2+//TYDBhS46vMVidvKbAeOLnRBT/xxoKIPxQFgz549rF69msTERMaOHUuTJk0YM2ZMLsV866238n1w0tLSEBHCw8NZtGgRmzZtIioqisTERPbs2cPChQv5+eefGTlyJIcOHWLz5s1UqlSJKVOmcOzYMWbOnImq0rhxY6pUqeLMMyQkhOTkZLZs2UKjRo0ICQlh6NChudI//PDDtG3blhdffNHvygxw5513MnjwYLp06cKGDRuYMmUKe/bsITQ0lKysLC5cuMD69etJTU3lmWee4eqrrwZg/fr1REdHc9NNN9GvXz9WrFjBli1bWLRoUS7lXbJkCbt372bQoEHcdpvp3ueH7cosIr8D6qjqNhEJV9WMItLfhzUmrw486245nu4l5UpqairXXXcdAOnp6VSqVInXXnuN1atXM3r0aOe1/Pj6669p3rw5AF27diU2NpYBAwaQlZVF2bJl+e2336hZsybvvvsuDRo0oFWrVvTp04eyZcvy2muvUb58ecqXL8+uXbvybYm7detGdHQ0d999d5702ZQrV46ff/6ZmjVr5rnfl1SubO34UKNGDQDCwsI4f/48r7/+OitXriQlJYWMjAxUNU93OiLC2oDPWpXq8ndOXFtmQ15sVWYReQrLUNZURNoBy7F2hCwQRzfab2zbto3HH3+cc+cub6vp2jIXRFpaGlWqVCEjIyPXgygizJ49m/Hjx6OqTJw4kccff5wdO3Ywfvx4nn/+ebKyshg2bBitWrUCyNdqXqFCBcCa2eaaPpuqVauSnp5eImX2piGrefPmTJ48mW+++Ybbb7+dMWPGMGnSJGrXrk3v3rm3Nu3SpQsPPvggp06d4sUXXyw034SEBL766isee+wxZsyY4TX5SxOFWrOLnZnIJlW9Ncf3Bjd8027jDWt2z549qVu3LpUrV2bXrl3ExMQQGRnpVOZffvmFp59+mnXr1jFy5EiefPJJ5707d+5k7dq1REVF8fHHH1OrVi1q1qxJYmIiK1asYMOGDdSsWZODBw9y4403cuDAAX788UfmzJnDmTNneOqpp6hduzaVKlXi97//PZGRkZw8eTLXd69evYiJiWH69Om50k+cOBGAwYMHk5SURNmyZW2tF0PgUpA129b5xljBJXWwZlXVBNbYmb8785mLQ2Zmpg4bNszj+7OysnTUqFE2SlQ8zpw549fyDf6BAuYz290yN8YKNmmCtcb2BFX9P7vyD0Q/89atW2nbti3h4eE+L3v//v2EhobSuHHjEuUzM9ozK/y4JatLVK7BMzzyM3tAFVWNsTnPgMYdF5K3uOaaa/xWtt0U5mcGOH78OC1btmT79u3UrVuXSZMmMWDAAK699lrWrl3LsWPHAFi2bBnVq1enefPmJCYmcvjwYSZNmkRERARnz55lypQpREVFMWHCBNLS0sjKyuKOO+6gf//+vvpTvYbdyjxYRF7A2mRumap+YXP+l5kU4eF9p+2Vw+ATP3NKSgovvvgiycnJTJhQcGxSQkKC084AMHHiRGbMmEFkZCQHDx7k+eefp1u3btSpU4e//vWvAGRmZnq3gnyErcqsquMARKQNECcii1W1oZ1l2E1ycjKvvPIK27ZtY//+/UybNo0dO3Zwyy23kJ6ezuDBg7njjjvYuHEjKSkpVKhQgXLlyjFr1izmzJnDzp07CQsLo2LFikybNs3ff47f8Laf+YsvvmDZsmX079+fwoaG8+fPZ8qUKYwZMwaAjIwMIiMjAWjUqBHHjh1jz549uXzVYWFh3qgSn2O3a6oS1kbrfYHywN/szN9btGzZkgULFtChg7WzTpcuXZgxYwaqSu/evbn99tuZPXs2H3zwASJCZmYmu3fv5siRI8ybZ3nWLl686M8/we9408/873//mx9++IGEhASOHTvG+vXrqVq1KidOnACsLni1atVIS0sjPj6e2267jVGjRjF06FDCw8NJS0ujWrVqfP/999SsWZMWLVqwfft22rWzhp2ZmZlBodB2d7PfAz4EElT1pM15e40BAwbwzjvv5PHhigiNGjXixIkT1KtXz/mQhYWFsXfvXufD8PLLL7Nu3TqWLl1K+fLlfS5/SfGmIcsOP3NycjLLly+nbt26nDx5kkceeYSZM2fyyCOPsHLlSk6fPs3cuXNZtGgRAOHh4XTo0IFVq1YxadIkxo0bR0REBOnp6UydOpWaNWvy1FNPOVvvLl26BMWY2RZrtojUVtWjDmu266ypgyUuwEEua7ZNY+bsmOiIiAiSkpLIysoiMjLS2TL36dOHlStX0rdvXz744APAepPv37+fBQsWOLvWcXFxzJkzh4oVfR6RarjC8LY1exhWl3oCljJn95MUGG5TGbmx2ZDVsWNHZs2aRUREBBs2bOChhx4iPT2dBx54ABFh7NixxMbGUqlSJcqVK8eMGTOoU6cO8fHxVKhQgbJly5bawI0jT2z16L660/xnyTfkxW4/81BVXZDjuK+qvm9X/oHoZw4GAkGZ7XZNtWjRgg4dOjBx4kT+8Ic/EBERwaxZs/I1ZJY2vO5nFpFQYISILMRqmUOBkYBtymwITALRNdW/f386dOjAwIEDGTt2LDExMahqHkNmMGGLMotILBAHtMZaolewlhxaY0f+hsAnkFxTK1euJDY2FoDly5ezevVqbr755nwNmcGELcqsqilAioi0V9VtduRpKF0Ekmsqe5LM5s2b6d+/P6NHjyY2NpaIiAiOHDnizDdYXFLZ2O2ayhSRBUBtLu815ZX5dS1TWnp0367YApcYc/LVV1/x5z//mX379lG+fHmnxbtXr17s27ePxYsX06BBg1zjs6ioqDyhhJmZmYwbNw5V5cKFCwwePJhOnTp5JLc38aYhyx+uKVdCQ0OJjo5m3rx5+RoygwW7DWCfAUOBN4F44F5Vfdqu/HMawOxSZteVRqZOncqECRNo06YNISEhDBs2rEBlzj7Xv39/7rrrrlxTFhcvXsyrr75Ko0aN6N7dmtIdbC2BwT8UZACze3/m845ZUiGq+h0Fbz4XMLiuNHLVVVeRlpbGkCFDWLVqVaH3zp8/n/j4eOf4bP78+dx4443Obt6ePXto3769M71RZIM3sVuZ14pIOeA9EfkaaxpkQLNt2zaaN2/uXGlk+fLlHD16lLFjx/Ltt9+yf//+fMdnAPHx8cyfP9/ZXYyPj2f9+vV89NFHAM6wwWyCzXpqCCzsnmiRPdPgDccn4Dl06BCTJ0+mcuXKHD9+nOXLl7Nq1SrKly/Pzp07eeutt3j22WcZNWoUO3bs4OTJk7z00kusXp1/CGTOUML4+HgeffRRVq1axaVLl4iJiQnIMbMhOLArnHMrl8M4Jcdv1MZdLewOGrlw4QIjRozgnXfesS1Pg8HbeDVoRFXdGhuLSAdgHPCDqj7mODcOaAhchTVBwz6LXBFcddVVRpENQYOtY2YRaS0i74vIBhEJFZFclmxV/RJIzJE+DGsL2LHALvLZLlZE7hORVBFJzR63GgyGvNhtAJuNNbEiBGgOPCIiq3N8olzSVweyNfQwUNc1Q1Wdp6rtVLVddkCCwWDIi91BI6Kqp0REVXWXiOxR1VyrxYlIgxyHvwCRjt/1cexrZTAYio/dLXOKiKwEGonIUiAp50URuQZr9c7bReQ+Vc0E/iMis4GWwCc2y2MwXDHYOWtKgCNY3exGwCHX1UZUdT8wxOWcR3PQvmnazCM5m+37xqP7DIZAxzZlVlUVkTGq2ger+1xqKW5stmssdmhoKL/++itjxoyhdevWpSI+21D68cbGcWuA/2Dtu4yqPmN3GXaSX2x2cnIyzz33HMuWLWPYsGEF3uu6rOubb75Jjx496N69OxcvXqR///507drVeQ5MFJjBe9g9Zp4FTAfWYc1r3mBz/rZT0tjsnLHYu3fvdsZilylThvDw8FznwMRnG7yHrcqsqltcP3bm7w1KGpudMxa7efPmzljsixcvkpGRwbXXXmvisw0+wdYpkN7GF7tAhoeHs3DhQmds9sKFC52x2VWrVs0Vm509jn7llVdo0KABXbt2dY6ZT58+TUJCAm3atOHRRx8FMPHZBlsoKJzzilZmE5ttKI34aj5zqcLEZhuCiStamQ2GYKJUdbNF5ARWDLc3iAQCbUsdI5N7XGky/V5V80xUKFXK7E1EJDW/cYg/MTK5h5HJwnSzDYYgwSizwRAkGGW+zDx/C5APRib3MDJhxswGQ9BgWmaDIUiwfdZUacKxc2U8UAOoCCxW1a+MTEam0ihL0CqziMQBA7DmVu8FmgEXHZ9NqroEeAF4TVW/cyyuMEFELgIRWLtaVgE+VNV3RCQ5x/0Pqep5H8t0BvgA+BxYp6rLReRWh5xlgPGq+pOPZboEPOjI6g5VbRwA9XQceAXLx7tTVeeWpJ5KKMt/gReBhqp6qyO/XLIAR4HXseryoKrOLG59ZRO0yuxgrqquFpHFwG/Aw6p6FkBErgU+BX4nIvGO9LOBkar6HI4ljERkOfAO8D+s+voVuOBrmRwynAXCsR4SgATgbqzFE0cAz/tSJkc9JYhIayDNcc3f9bQHWKWq74rIMhF5i5LXU0nqZ7iILMuRl6ssm4E9qvqKiLwrImGO5bSKTbCPmeNF5FMge/uJl0RkrojcCTTF+sf3U9VE4BRWN+lc9s0iMp7L65jdr6rxwE9ArkUKfSTTYVW9Geth+Eu2iKqaRQErm/pApmxGAm87fvu7ntYAbURkJlAVawXYktZTSesnJ66y1AV+cFw77pDXI4K9ZZ4PbMTaKucSud+orYFWwIciMg1rZZReWG9VRORR4JiqrgZw/APAqvCKvpYpe3MAVc3IsX9xloiEYK1segTPKUk9hQO/c2wUGAj19D/gEUe6lQ45SlpPHtdPPrjKcgS4znGtBiVYcivYlTn74f8SmARcdIxlvlDVJBEZAsxT1SdEJAJorapnRaQ3Vgu4UUTqq+pkx5u+PNbbfqQfZOoIxDpkeM+R1Tys7XOvIsfmAr6SyXFrNODsRgZAPVXAGjOHAimqmiUiJa4nT+tHROYC14vIDLV2cXGV5SgwWKwVand42sWGK9zPLCJlgPuwujYhwJuq+qORychUGmW5opXZYAgmgt0AZjBcMRhlNhiCBKPMBkOQYJTZYAgSjDIbDEGCUWaDIUj4f4YQhheVIdSKAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 251.059x155.163 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = bar_plot_results((skl_results_long\n",
    "                           .query('model==\"Vanilla\" and input_data==\"Drugs, Sex, Age\"')\n",
    "                            .query('model_id==0')#.query('crossval_id==0')\n",
    "                            .query('metric_name.str.startswith(\"RE@\")')\n",
    "                            .query('code_subset==\"All codes\" and code_level==\"All\"')),\n",
    "                           \"target_metric\",\n",
    "                           \"metric_name\",\n",
    "                           values_subset=[\"overall_micro_F1\", \"overall_NDCG\", \"overall_micro_aucPR\", \"overall_micro_AUROC\",\n",
    "                                          \"overall_macro_F1\", \"overall_macro_AUROC\", \"overall_macro_aucPR\"],\n",
    "                           groups_subset=[\"RE@25\", 'RE@50',\n",
    "                                          \"RE@75\", \"RE@90\", \"RE@100\"],\n",
    "                           reference_row=\"overall_micro_F1\",\n",
    "                           reference_val='relative',\n",
    "                           legend_kwargs={'loc': (0,-.012), 'ncol': 2},\n",
    "                           abs_value_in_percent=False,\n",
    "                           improvement_direction=\"min\",\n",
    "                           average=True,\n",
    "                           values_legend_dict=metrics_pretty_names_dict\n",
    "                           )\n",
    "ax.set_ylabel(\"All codes\\nrelative improvement (%)\",labelpad=-5)\n",
    "ax.set_ylim(-12.5,10)\n",
    "#ax.set_title('RE@ All code levels')\n",
    "fig.tight_layout()\n",
    "if mpl.get_backend()=='pdf':\n",
    "    # The inline backend distorts the scaling and outputs figures \n",
    "    # with wrong dimensions\n",
    "    fig.savefig(plots_dir/\"opt_metric_AllCodes.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9383508b",
   "metadata": {},
   "source": [
    "### Fig S3: best target metric per level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "49177cba-3295-4fdc-b0dd-cc9a1cc17377",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe4AAAEuCAYAAABI//HiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABaqUlEQVR4nO3de7zUc/7A8de7kjrpRhdspbJWSYSyS5LkUkRCF8p2SCmStUqiJYpqtwghFZ3kki4WJfujqLCWsiLRukclSqXLodt5//74fud0zjQzZy7f78x3znk/H4/zODNnvvP9vs/MvOfz/X6uoqoYY4wxJjuUy3QAxhhjjImfFdzGGGNMFrGC2xhjjMkiVnAbY4wxWcQKbmOMMSaLWMFtjDHGZJEKmQ4gWbVq1dKGDRtmOgxjMu6DDz7YpKq1Mx2HVyy3jXFEy+2sLbgbNmzI8uXLMx2GMRknImsyHYOXLLeNcUTL7UAV3CJSDhgJVAOWq+r0DIdkjDHGBErQ2rg7A78D9gBrMxyLMcYYEzhBK7iPBd5V1b8CA8IfFJF+IrJcRJZv3Lgx/dEZY4wxGRa0gnstsMW9vS/8QVWdrKotVbVl7dqlpi+OMcYYE7dAtXEDLwAPi0gbYGmmgzHGGGOCJlAFt6rmA30yHYcxxhgTVEGrKjfGmEDIy8ujSZMmnHXWWeTl5WU6HGMKBeqK2xhjgiI3N5e8vDwWL16c6VCMKcauuI0xxpgsYgW3McYYk0V8LbhFpKaI/ElE/igiNf08Vrys3cqY1CWa2yJSTkTuFZGHRaR3OmI0prTypY1bRLoCPYDdwLeAAA1FpCLwnKrO9uO48bB2K2OSl0Juh2ZF3EwKsyIuXLiQRYsW0bNnT0aPHk3Hjh3p1atXsrszJiv51TltF3C5qmrRP4qIABf5dExjjP+Sze3QrIiPi8gcYFHY8/sB/QAaNGgQdSdTp05lwoQJdOnShaVLl9K3b1969epFXl4eY8aM4fDDDyc3N5fc3Nzk/jtjsoAvBbeqvhy67S4c0h44BHi96GPGmOySQm6vxblKhyizIgKTAVq2bKnhj4fs27eP7du3IyI45woOq0kzZYkvbdxSNKPgPqAukANYoW1MFksht18AzheRh0lhVsRrrrmGGTNmMHHiRK644go6dOiQ7K4OYP1fTLbwq6r8SRH5l6o+j9MGZowpHZLKba9mRezYsSMdO3YEYPZsb7vKJHvV/v777/Pee+/Rvn17xo0bR58+fWjdurWnsRlTlC9X3Kp6NbBdRJ4E3gc2AvnAJX4czxiTHkHL7d69M99BfeTIkbRp04bevXszduxYpkyZUviYXcUbP/hVVX4IUAmYBZTH6YWar6rb/DheuPz8/MKfBx54gPz8/HQc1phSL9O5He64446L+li6Cs169erRokULunfvTu3atalWrVrhY7m5uRx++OEsXry48Io+kZi8+h++//77pJ9rgsevqvIXganAocCJwDXAFSLypKpe49MxCzVq1IgLLrgAcKqxVq5cyZNPPun3YY0pC14kg7ndrVu3wtuqyooVKxg6dGjEbdPVYS3Uzj548GAATjzxxKjbJhpTpO3j7UF//fXXs3v3bipXrkxBQQEHHXQQEyZMiOu4Jtj8njltH1BOHc8C1/p8PMAZ61m9enWGDx/OeeedZ4W2Md7LSG5Xq1aNWbNmMWvWLGbPns0555yTjsPG1Llz52L3+/Txd4HD8Kv4aPLz85k6dSrr16/nkUceYe/evb7GZdLHr4K7C87Qjx+Am0J/VNWCkp4oIlVE5AMR6ZTswZs3b864ceOYPXs2GzZsSHY3xpgDJZ3bXrjjjjuK3b/33nvTcdi4qWpC7e5+VudXqlQJgFGjRgFQrpzNcF1a+FVV3g74Z5RJGi5W1ZdiPHcoTvtZSipUqMBtt92W6m6MMcWlktspa9SoUbH7hx56qJ+Hi8uCBQsAp9CeOnUqffv2jfu5flbnP/bYYwA0bdoUgGHDhnl+DJMZfhXcFYG5bjJ/5/6tIU712jPRniQi5wCf4nR+ifR4XLMrhevduzfTp0+Pe3u/fPrppzz00EPs2LGDqlWrcuONN8bsXGNMACWV26XZnXfeSffu3alTpw75+fls2rQp0yEBUHzIPbz00kv0798/Q9E4pk6dSv369ZkwYQKVK1fmrLPOYtCgQRmNKRv5NXPaHGCOu/jA73HGe36hqltKeGo7oApwHPCriCwoWgUX7+xKzz77LC+//DLly5dHVbnoomDMsjp69GgeeeQRqlWrxrZt2xg0aJANETFZJYXczji/pkVdunQpY8eOpUGDBjRr1ow///nPnuzXay1atAD8ex3i8cEHH/DOO+/wyiuvUK5cOa69Ni1dI2KKdTIR1Kl0/briBsBN5mUJbH8HgIjkApuSbTdbsmQJM2fOLLx/ww03cMUVVySzK08VrV0MPxs2JpskmttB4Fe1dE5ODnfffTeLFi0iJyfH032n4tdff2Xu3LmsW7eOevXqcemllwIHvg7JFE7JFmivvfYa1apVK2xvD2txyYhYJxNBnUrX14I7Waqal8rzd+3axSuvvEL9+vVZu3Ytv/76q0eRpWbYsGEMHjyYX375hRo1ajBkyJBMh2RMRCLSClipqr9lOpZs0b59e9q3b5/pMAoNGDCAbt260aJFC7777jtuuOGGiCNskimcki3Qvvrqq2L3iw7ly9TVbRBPJkri93rcPcPud/HzeCGPPvooW7ZsYcGCBWzZsoWJEyem47AlatasGZMnT+b555/n8ccfp1mzZp7t22ZoMqkSkUXu75E4w7vmxtg24dz2YsRIiH3eS1axYkUuuOACjj/+eC644ILCXuYlSedr+8YbbxTejneYm9e++uorPvzww8L70eYFCBLfCm4RKQ9cK45y7nq9aWnQyMnJoVevXtx222307NkzUNVXRXk5XWOmPvSmVAl9HzRU1euA6pE2SiG3PRkxAvZ5j0e7du3o1q0b3bt3p0ePHpx55plxPS+dr22o3T2STM0yV/RkoqRjZOoE0q8pT3sDC4EWOOvuLgLmAa/5cbxsFepRblcPJiDWiMjrwP+JSAUiLL+ZbG4XGTHyY5TH+4nIchFZvnHjxpT+CeO44oormDVrFs8//zwzZ85kzZo1vh0r3u+w9evX83//93/s3LmThx56iCpVqkTdNtETCK9OOGKdTIQfI55j+vH97tciI9NVtR1wnqqerartVPV8VX3Qj+PFbUR15+fbt53fadatW7fCn65duzJ16lTAnzPcVD8sdjJR9qhqLnAhTmG8D7g4wjbJ5nY74E/AlUBfdy3vovudrKotVbVl7dq1vfh3SlTaP+PRvm/8EO932I033siWLVvo1KkTp5xyCvfcc0/cx/Dr/Xr99deLtWv/6U9/cm54VF748f3ud+e040VkHEXO3FX1bJ+PmRI/O0hUq1atWPIMGDDAs32HS7U3ZFB7Uxr/uFfTvYA6wCk4c5J3jbJ5Qrnt1YgRL/nxGQ/S8KF0ft/Eq2bNmvTo0YMpU6bQunVrnn766bif69d30tChQ6lXrx4nn3wy11xzTUJzhGSK33Pg9QfOdc/Mzw56oQ3+tu8kO12jn1cG69ev55lnnmHnzp0AVlCXbdeq6rnAZlXdCxwWY9ukcltV81R1vhfBBlGQ2t6DOD1sw4YNC9vcO3fuTM2aNTMdEn/84x95+eWXufDCC7n33nvp2jXauWpwpFxwi0iseq1FQBMRKe92YinTk+UmO12jn18GAwYMoEqVKvTr149169Yxa5YnfYd8U9qrNzNsr7tsp4pIZSDWVbHldsAFcXrY4cOH8/zzz9O3b19eeuklqldPf5NluFAMrVq14vHHH2fatGkZjqhkSVWVi8hMVe0hIjcDHUTkJ1W9KsKmp7k/IQqk7ar7gGqrdB04i9StW5dLLrmEc889lwEDBvDLL79kOiTef/993nvvPdq3b8+4cePo06cPrVu3BqwK32dDgTlAU/d3rMmtM5rbpZXvVe2httpvd7ptuOnN94jLsv563/4NMhBX+/bt6du3LzfccAMtWrTg2WefpV+/fgds13x6cwC+3vA1zac3Z2XvlWmLMVyyZ8mhq+yTVfV84OhIG4U6sQA93U4saU3sIFVbecnLq86jj3beuipVqjBx4kQqVPCv20O8cY8cOZI2bdrQu3dvxo4dy5QpU3yLyeynqu+ragdVPUJVL1TVqDOjZTq3i8lwp9OokoirtH5nhQRxWdZHH32Uf/zjHzz99NO88cYbrFixwrN9j+/eifHdO/H9pysZ3z3l6QsKJVtw54vIk8B/3cUGIi706nZ2WQC86lapzU7yeBkTlHF7RXmZ3EUnG6hWrRpz50adcyNl8cZdr149WrRoQffu3alduzbVqlXzLaZ4lea+ACLypoi8EeknxnOyPrdjWXvbW6y97S12ff0La297K9PhlBrxtrs3n96c5tObs2zDssIrXb/Url2bGjVqMG7cOF577TWWLYtvJt/PmjTlsyZNyX//fT5r0tTXGMMlW3BfCoxS1QeAg4BoK8cX7eyyj9idXQIpmXF72ax3796eXcEke5LToUMHAAYPHgzAiSee6DwQR1x+nVhlW1+ARBS5Yn4PuAPoANwOvB/jaVmf2ya9BST41+4e6co23u+CCy+8sPD2mDFjii0Sk7diNxt2FHBW3k62vBWcdXSSLbjbAZNFZCXOcJBo67Il0tmlbIpQGKXzqv7111+nb9++hdVDp59+etRtE619SPYkp3PnzsXu9+kT7bzwQH6dWIX6AkyePJlhw4axbt06z/YdIG2B/6jqbpxCu22MbS230yxUC5DpmgC/qn9jSebqNt7vgvDvmxtvvHH/PlpUZPXAqizOrULNNpnvAR+SbMF9J9ARZzzmPpxlOCNJpLNLmVGswFux+4DH03lVH96+89FHH0XdNlO1D7Gmhk3XSU6svgBBaD7xSB7wrog8D7zj3o/GcttnXhWQJX3fmOyTbMG9z/1Rt4074n4S6ewCICKXiMgUEXlJRM5LMrbAK1bgtaiY0ViSbd/x06RJk+jbty8zZ86kU6dO+2cyiiDRk4dkC9lYfQFKS/OJu959a+BG4AxVfTzGtgnltkl/tXRIIt834Ve28VzdBqEWIF22vLWFvb/s5evRX/PPX7ZmLI5kC+6/44zjPA74P/f+AUTkXLcQXiIiS0VkaaydquqLqtoXyAW6JxmbSUCs9p1MeeONN5g8eTITJ05k/vz5fPzxx4WPFW1zSubq1otCVlX31wJksN3dayLSHHgRmA38U0ROjLFtQrltslvRz/Cyb77PdDgZU7NNTf4w5g80HtaYLtVrJLUPL74Pkiq4VfUVnDGbzYHzVfXVKJv+Heivqm1V9UxVjW95GhgOPBL+Rz8WIsiWL1W/xGrfSZRXr+Vhhx2GiBRe5VasuP8qoWibUzrndV+wYAELFizglVde4dJLL6V79/jPK7PoivxRYKCqtsXpt3JADhaRbG4HUibabbNJ0c9wq0b1U95fWaq+X/bN92z/bRePvvkueXl5nnwfJFRwi8g0EXnSHQr2BDAWeMK9H8knwJ4E9i8iMhZ4VVX/G/64HwsRZNGXalolc2Xr1Wt50003AXDRRRcBcOmllya9r3jEE/edd97JqlWr2LRpE/n5+WzatCnq/rL4ZLACELqcWguUj7FtorldJprBEpHJ4USZFqn63sse3Omqvv/nL1vZtHcvvb9bEzXXWzWqz9COZ3F9u9MytsjICPf3X4H/AB8AJ+G0i0WyBFgtIp+597WEM/MbgXOA6iLye1WdlGB8xiO5LSrubw9L8wlNkyZNit1v2zZW5+b9Ql9++d+t4bMmTWm6+rMSnhG/pUuXMnbsWBo0aECzZs2iDhnJbZgX1+xuQVqMoohHcDqnfQs0JPYVd0K5raovAi+KSE1gHLbEb8YUKyDP2RKY3tJFv3OaN6oZ1p5cLemq6UQUy8sau0vsE9Cleo3CuJqmMYcTKrhVdQ2AiLRQ1ZvcP38uItdHeUo/4Eh3eEk8+38IeCiRmIyJpWgiHp6/hVaN6hdWh97yfPxrXeTk5HD33XezaNEicnJyij2WzElOEKduVdWnReRZnJkRN5awgldCuV1E1GYwd59ZsTpTNgsvIIOqZpuahScVXUZHnOPLc8Xy0odZ+EJX/6GagHpj2iS1n2Q7py0RkQUiMkZEFgDROqa8RilfiCCLq0XTKlM9T71um2vfvj2jRo3yILLgEZEnVLVAVX/EGTESawHnhHI7E81gpnQJr5YOYoe5dJUHSU1Mrap3isjhQANggqpuiLJpa4pXo5e6hQiCeOXk1VldKkIdfEJXt90bDS3hGaVTePU9ELEKP7zqHMhEVXrj0A1VVRGJuAaBK9HctmYwk5JI1dKh797x3TsV6wQmO16lW/OOaY8xvDwY/+ocX46T7Opg9XCmRPw98KWIjFbVA055VLWdiBwE1FbV9amFWnpFaiNNewwJtu0EQfhqPdk8CWmkE8AMnBBuEpFrgX/jrPz1c7QNE81tawYzfmvVqH5hrVq3RukvtNMp2arrPOAZ4CLgWWB6pI2CuBBBquOA/eDHEKeEY0hwUph09Yj1c9hIUKaQDJDewCHAQKAKEHVQfxBzOyQ00UkmJjvJRkGZVMRrs1a+ysadm+n67KDAfNeDN3Elu4ZjJVV9x739togcHGW7a1W1jYi8qar7RCTjCxFksre0SZzXnUX8rk5LphYgvFkhkU5zXlLVfGBCnJsHLrdNcpLpBBaEaumSdGvesTCuernpby6Mxou4ki2454nIq8DHwInAvCjb2UIEaeB3e3LRK5Zsr5b2ojotG5sVfGC57bFH+jurqK77fGvh7aAqS9XSQZTszGljcaYlfQHorapjomxqCxGYtIpnQoRUJdqsUDSmaFWR4bMrpZvb6/v8BJ5iuV1KxfN5DRfUaunSKtnOafeq6h3Aj27Cj1LV4RE2/VZVO6QWojHxy9SECLEUjSmaolcw56w+uli7ezpGBbi9yK/HWXsgHlmV235OzhNLUCc7iSWez2u4oFZLl1bJVpWfFrrhJny0RZwnikgO8Crwgqr+kOTxyqxMfeGYssmdl+G/uFXfqnpnlE0tt+OQLZOdmOySbMH9m4icw/5hIxG7+6pqN7f9qyPwDxGpp6pnJXlMY4y/7o93Q8ttYzIn2eFgVwOdcNq2LsQZRhLNycCZQD3gvSSPV2bYMJbExBrK8kj/N3ik/xsldvax9rlCK3BOxC8A3gGqlrB9YHLbywUqjAm6ZGdO+xH4S0nbichinC+A6apa4vYme0XqEZuOISNezGfsdftcJhZH8MjTOPOI366qe0XkL0DEsWlBy+1Eq6SzqQd3uKInmtc1uSkoC9SYNEq2qjwuqnqWiBwBNBSRI9LdDtb98Xf5dP02T/f5/N238dO3X6e0j1Ti8usLx4/XKnzISDJfOH7ElaqSYirpZCLAhUZlVf2XiNzq3pdoG5bG3PZCOuJK9EQz1Zj8+Lx68Tr9c/x/2fT9Dk/iCQnq5ypcwgW3iAjQXFU/jmPbITjzF68ATnInaxgbY/sqwKM4beaLVfWZROMzwWU9T+OXoauqz0VkKHCoiNwMrIq2oeW2MZmTcMHt9iK/F2e605JcrKqF39Ai8jYQNbmBS4E5qjpPRJ7HmVY1Lt0ff/eAv336wzZ27t67/7FdYSPW3L9/un4b3R9/l/xlywBY+b91XHTtQ2ytcQwAH3+1mTaDX+XIY2qwcVdzftAqxY4Xen68isUVFlP+mkoAFOx6iPw1/bj1DAXgq1ce5tYzBrD1kF0A/FS+gJnu7YKDLwZgU8UveOHwi3mLnQB8wT4GspOD44gtnpjC4/r+8/f4csNmzvhlL4d9O5+6x59bGFcoppBQTCHxxJRIXEDhaxUS/lqVFFO8caUSU9xxNT+T45s7y1u/uqs6rxaJ6/nrTsMPqnq9iHTCybuvVPWBGJsHJ7fj/LxC9DzyIofijSvVmIoKzOfVw5jiiSvn8XfZtHUH6/buofvj77L28Mx/3+SElSdAwnHFm9vJVpXvFJHHgA/YP2zkyQjb7XaHin0ItAT2lLDfesBK9/a+8AdTXrO34RnF7i7/cTkA2/dsZ/mPy2nZqhUA5ZdWJadVK0KrLv+vcgWOPKYGaz91QivYV8DaT1dSu7ITQ8Fve9n1zS8c3CiJKTnDYmrp/n7zoKq0rNsS6hIzJofz+4tXq1DvuP0d2spVquBpTOFxHduqFcf2HFhs2/1x1cBzMeICCl+rkANfq8zHBGmKKwkiMkNVr8Jt1xaRB1X1piibBye34/y8Qqw8qgF4mEMx4kokpm/+/Qq69ze+eHUSe067gEanX5hcLCXEVCgTn9ewmCC+uIoq+p55JsHXav0XWwHY9evewtu+xAWgqgn/4PQiL/YTZbv6wEScsZ4TgQYl7PcqoJN7e2asbU855RQtSbdJ/9bax5wU9fHj847X4/OO15xjc/T4vOML/962bdti2xW9P3PEUG16VH1VVf1+6FL9fuhS/VP9Fvr90KUlxhNvXCXFEOl+vNukElM8+0zkmPGIN65YghiTqndxAcs1iTzW/Xl3EtAH+Ay4xv3pByyN8ZxA53ZIonmUSg7FE9e0adP02GOP1bZt2+q0adM8OWaqMcXLy7i8iOmFcR9os8Ylf1YSkUpcE69bpBOvW6S/P+LEwtupipbbyV5xPw10BWoDj7mJX4zbFn6vqkZdYSiCF3AmdriQ6POfG2O8VQDsBTa7vwX4DWda4wNYbicvjWurmzS7YZKzHP3s1TUKb/sl2XHcT+PUl1yhqnuB0eEbuGcLP4jIqSJSQUTKiUjM46nqTlW9WlUHqAedV56/7jSOO7JaqrsppvtdY6jTsHFK+/AjrlTFE1NeXh4bNmxwlthM03jnbH2tsomqfqSq01W1NfA/nGrvr1Q14vCJ0prbXghiXKU1pi63nEyt+od4FJEjiK9VJMkW3LVVdRLOWXksp+J0WHkdWAQsTPJ4vljZeyUre6+k1eGtWNl7ZclPyCJ+FLK5ubmsXr3aWVzDrhpKHRGZirMG9yHAn0UkUr+VkEDnthfScaKaiZNhk/2SrSr/SUS6A5VFpAsQcQynqrZLOjKTEquSM0k4SlXPDd0RkaiFcVnI7XTkkOWpSUayV9zX4LRv/xent+i1kTYSkQtE5G0RWSwib7ntW8aYYForIsNEpIM7nnutiJwtIgc02FluG5M5yV5xj1TVIaE7IjIcGBVhuxHAWaqa764ktBh4JcljGpOwolWRdnVTom+BisAf3fvfAG0ABcKnzBqB5bYxGZFQwS0iDYGjgXOKnIVXAM4hcsH9IVATyMfpzLYiyTgDKbRO8sH/qZ6WNZNN4qywjp+q3p3A5qU6tzPFTjRNPBK94j4KOAMnUc/AGTayB7g9yvYnAEtE5GfgUGCTiLyF0zH1zKQiNsb4QkR6AdcBxwG/AJtV9YB5J1ylLreDUGhaYW3ikVDBrapLcJL1PqAW0Aj4VlXXR9nen7kZjckgL77gg1BIRDAIZ1nPRTi1aFOjbVgacztA74MxMSXbxn0Hzlq8nwDHi8gKVb0zfCMRaY4zI1NN3JWGEpy0wWSRgBZGnvPifwvo65OvqvtEZA9wJNAi2oaW28ZkTrIFdztVbRu6IyJLo2w3AxgARLwiN6VLQAsjE7/RIlIJuBtnGtMHY2xruW1MhiQ7HOwTEblCRJq447k/EZHGIhI+pdhXOHOtrgn9pBauKWtsgor0UdX/w+lVvga4EafKPBrL7TLE8jBYkr3izgHOc39C/oYzbOSaIn+bBXwnIl+697O240o6hVc5AyVWQadaTR3Uau4gxVLaichknA6o63Cqv8PzuaiEcltErsWZbe0I4G+qusKruI3/LA+DJamCW1WvjnPT24CTVTXizGomskhJUlLSpJpYlpgGaFR05rQSJJTbqjoVmCoiJwEXYcPHTIqCerGRDkkV3CJyDjAEZ9a0AmCrqkYayPwWJa/Ta4zJoCJzMnwrIjfhdDpVAFUNn3glJGpuux3XwhceugZn9bFBwAEdWY1JVFkrrItKtqr8PpzhIi8DnYF7o2zXEnhHRH5y71tVuTHBEzrpXsv+ORog8oxpIVFzW1VXAp2KbiwiBwGPAhNU9fvwnYlIP5w1wGnQoEHS/4gxZUGyBfdOVd0mIoqzQlirSBup6umJ7NTawYxJv9CMaSJyDPClqqq75vbvYzwnodzGWUnsD8AAEVmkqrPD9jcZmAzQsmVLTXDfxngiW/oKJVtwT3eHjTwELAXmF31QRIap6mgRmYFb5RYSa6yntYPFpyy37RhfTVLV9uBcPovIJKB90Q1SyO2/+hGwMV7Klr5CyXZOy3Nv/tP9CRdax3d4tH0k0w5m1WkOK6yNT3JCN9wr7pwI25SY20GRzOgMkx528ZGaZDun/Q24EGeBAQBU9ewit390f0cd25lMO5hVpxnjq+nuGtz/xZkZ8anwDeLJ7aBIZnSGSQ8rrFOT7AQs56nqn1T17NCPR/EUbQfr6tE+jTFxUNVJwBXAbKCHqj6W4ZB8ZZOKmGyV6LKeoZnRPhCRzhQfNvJ1lOccCfxOVZeJSI6q5kfazt1HmWgHs2oiE0QichTO+OzfA1+KyN9V9ZsY28ed20FUVnLPvm9Kn0SrykOzowlwifsDUWZYEpHbgeOBJiLSEpgLdEwy1ozw40NvyWMC6jngJuAjnAVGngEi9h4vDbldVtj3TemTaMF9C1BRVTeE/iAihwO7omx/rqq2E5E3VbVARComG2im2IfelCE/4cw/riKyHNgUY9usz21jslWibdxTgEPC/lYFeCLK9rtF5HeAikhdohfwWcPaxUxpIyIzROQpoBrOgkEzgZUcmOtFlbrcNiZbJHrFfZiqfln0D6r6lYgcGmX764HxOGv2TsBZcShQEq0KtytwUwolM7Qr8LltTGmV8HCw8E4oIhLrrLyGqvZIKjIPxFMoW0FsyrrQ0C4RiTSBygFDwlwZzW1jyrJEq8pHAgtE5BoR6ehOUToPuCfK9leKyBsi8g8R+WNKkSYhNzeX1atXs3jx4rgLZ6sKN2WYuD/lgBMJmzUtTEZz25iyLKErblVdJCIrcCZfOQH4AeiqqhE7sajqLQAicjKQKyIzVbVRaiH7y67ATVmlqtOL3heR+TG2zbrcNqa0SLiqXFV/Jnr1WTEiUhW4GOgCVAb+nujxgs7GSJrSQkRGsn/+8SOJUSOXTG6LyBHAO8DFqvpJygEHkH0fmHRIdpGReD2Ls/Rn/2hX5dnOktOUIgvd3wpsxelZHk0yuT0EZ1a2Usu+D0w6+FJwi8gRqvoD8BecL4FqIlINos+wZozJDLe6+3+qukRE6gA347R1PwSsD9u2xNyOsoDQWziTtJwbJQZbQMiYOPl1xX0VTtXZcPbPtAZRZlgzxmTUw0Ab9/bTOPMybASmAeeHbVtibkdZQOgx4AjgT8BhhA0fS3QBIauSNmWZLwW3qobauxap6tOhv4tIFz+OZ4xJyW539rPDgMNV9XkAETlgfHeyua2qA9ztRgBzUg3YCmtTliW7OliJRKQ80Ecc5dwlO6/163jGmKTtFJGrcaq3ZwOISAUir8edUm6r6ojS2jHNmHTxq427N5CLs1DBIpzqtN3AgjieW+p7nhoTMFcAfwbeBWa4fzsSuC98w1Ry2xjjDb+qyqcD00WklaouS/Dppb7nqTFBoqrbgUfC/vYd8F2EbVPJbWOMB/weDrZbRJ7G6ZQiAKp6NljPU2OyXNTcNsb4y7c2btck4C73OP1wquIAp+epqnYq+gM0BLoCHYDrwnemqpNVtaWqtqxdu7bPoRtjYoia28YYf/l9xb3LXT2snKp+KSJtYm3sdc9TY4xvEsptY4x3/C64/yUilYBn3TnO34/nSao6ws+gjDEpSyq3jTGp87XgVtUx7s3H3R9jTClguW1M5vg1HOwt9i9WIEVuo6pn+nFMY4z/LLeNyTy/hoNZe5cxpZDltjGZ52uvchFpISL/FJFFIlJeRO7w83jGmPSw3DYmc/weDvYgzsID5VR1H2DjPI0pHRLKbRE5RETGi8jDInJRWiI0ppTyu+AWVd3C/nYwv3uxG2PSI9Hc7utuUwB872dgxpR2fhek00XkJaCxiMzCWSbQGJP9ouZ2lFkRfwKeB97AmQ+9R9EHbVZEY+LnW8EtIgKsxalOawx8o6qb/DqeMSY9SsrtKOtxDwe2qOoe9/nFJLoetzFlmW9V5aqqwPWq+rOqLrNC25jSIcncngrcICKPA7P8jdCY0s33NmcRWQD8F6dtC1W90+9jGmP8l0huq+oGoHeaQjOmVPO74L7f5/0bYzLDctuYDPF7ytMlfu7fGJMZltvGZI7fw8GMMcYY46FAjasWkUOAu4GKwGuqOi/DIRljjDGBErQrbpukwRhjjIkhY1fcNkmDMcYYk7iMFdw2SYMxxhiTuKBVldskDcYYY0wMgeqcZpM0GGOMMbEF7YrbGGOMMTGIM+1w9hGRjcAan3ZfCwja3OpBjAmCGVdZi+koVa3t077TznI7EIIYEwQzrrTndtYW3H4SkeWq2jLTcRQVxJggmHFZTCaaIL4PFlP8ghhXJmKyqnJjjDEmi1jBbYwxxmQRK7gjm5zpACIIYkwQzLgsJhNNEN8Hiyl+QYwr7TFZG7cxxhiTRQI1jjuTRKQ8zlzptYFDgJmq+qHFdKAgxhWkmIIUiwnm+xHEmCCYcQUppqDEUiYKbhHJBS4HfgY+BZoCe92fN1X1eeAfwKOq+qU73epwEdkLVAdygRrAy6r6lIjkFXn+Taq6K80xbQdeBP4DvK6qc0WknRtnBWCIqq5PNCYP4toHDHJ3da6qHh2A1+on4GGc4Rofq+qkVF6rFGP5DngAaKSq7dz9FYsF+AF4DOe1/FpVxyf6epUllttpi8tyO0C5XSYKbtckVZ0vIjOB34C/qOoOABE5Hvg3cKSI9HW3fxC4VlXvAd52t5sLPAX8ivPabQX2pDsmN4YdQA7OBwagP3AFcBzQBxiZ7rjc16q/iLQANruPZfq1WgXMU9UZIjJHRJ4g9dcqldfnGhGZU2Rf4bEsBlap6sMiMkNEKqrq7gTjK2sst32Oy3I7WLldljqn9RWRfwPz3fsTRGSSiJwPNMH5EFyqqkOBLThVITtDTxaRIcA09+4NqtoXWE/YQilpimmNqp6B88H4WyhEVS3AmbiiXgoxpRJXyLXAk+7tTL9WC4CTRWQ8UBM4jNRfq1Rfn6LCY6nH/iVtf3LjNbFZbvsfV4jldgByuyxdcU/BWS70cZyqiqJnUi2AE4CXRWQMznrgnXDOphCRvwIbVHU+gPtmgPPiH5LumNTtUaiq+bJ/EbUCESkHNADWphBT0nG5j+cAR6rql26MmX6tfgVudrd7yY0j1dcq6dcngvBY1gInuo/Vxqm2M7FZbvscl/u45XZAcrssFdyhZHgfGAHsddsm3lPVaSLSE5isqreJSHWgharuEJGLcc5+3xCRBqp6r3uGVxnnLO/aDMTUBmcxlsrAs+6uJuOsrnYQMDSVmJKNy31qd6CwuigAr1UVnHaw8sB0VS0QkZRfq2RfHxGZBJwkIuNUdTAHvm8/AFeKyIPAR1ZNHh/LbX/jcp9quR2Q3LbhYC4RqQD0w6m+KAdMVdV1FtOBghhXkGIKUiwmmO9HEGOCYMYVpJiCEosV3MYYY0wWKUud04wxxpisZwW3McYYk0Ws4DbGGGOyiBXcxhhjTBaxgtsYY4zJIlZwG2OMMVnECm5jjDEmi1jBbYwxxmQRK7iNMcaYLGIFtzHGGJNFsnaRkVq1amnDhg0zHYYxGffBBx9sUtXamY7DK5bbxjii5XbWFtwNGzZk+fLlmQ7DmIwTkTWZjsFLltvGOKLltlWVG2OMMVnECm5jjDEmi1jBbYwxxmSRrG3jNsYEl4g0Bu4Aqqvq5SLyKrAG2KGqgzMbnTHZza64jTGeU9WvVbVPkT/l43zf/JihkIwpNazgNsakQ1dV7QccISInhD8oIv1EZLmILN+4cWMGwjMme1jBbYzxnaoWuDd/Ag6J8PhkVW2pqi1r1y41Q9KN8YW1cRtjPCcihwH3AieJyDCgCU51eQXg75mMzZhsl9GCO0IHliuBdsDBwABV3ZnJ+IwxyVHVn4H+mY7DmNIoo1XlETqwdFHVvsAs4NIMhWWMMcYElqdX3CJSEzgWUOBzVd2S4C7U/b0GaB5h//2AfgANGjRIIVJjTLw8yGtjjIc8KbhFpCvQA9gNfAsI0FBEKgLPqersBHfZAFgb/kdVnQxMBmjZsqWGP26M8Y4PeW2M8YBXV9y7gMtVtVhhKiICXBTtSRE6sLwoIo8BlYEbPIrNGJOcpPLaGOMvTwpuVX05dFtEygHtcYZ8vF70sQjPi9SB5VkvYjLGpCbZvDbG+MuTzmnuGXjIfUBdIAew5DYmS1leGxNMXvUqf1JEuru3JeaWxphsYXltTAB5UnCr6tXAdhF5Engf2Igz2cIlXuzfGJN+ltfGBJNXVeWHAJVwxl+Xx+mJmq+q27zYvzEm/SyvjQkmr6rKXwQqAvWANsA1wGHumboxJju9iOV11lq4cCHDhg3jk08+oWfPnjz99NOZDsl4xOuZ0/YB5dTxLHCtx/s3xqSf5XUWmjp1KjfddBN9+/YlLy+PhQsXZjok4xGvCu4uOJM0/ADcFPpjkRWBjDHZx/I6i+3bt4/t27cjIhQfIGCynVcFdzvgn6r6L1XdG/qjODp7dAxjMmr9+vU888wz7NzprH2zePHizAbkP8vrLHbNNdcwY8YMJk6cyBVXXEGHDh0yHZLxiFczp1UE5rrjPr9z/9YQp4rtGY+OEUh5eXmMGTOGww8/nNzcXHJzczMdkvHJgAEDuPrqq+nXrx9///vfmTVrFmeddVamw/JT0nldFlf+++abb9ixYwfNmx+wzEJGdOzYkY4dOwIwe7bNTluaeDUcbI6qXorTeeVpnKTOVdVLVXWuF8dIRKxOGXl5eTRp0oSzzjqLvLy8lI+Vm5vL4YcfzuLFi+MqtENXax9//DG//vprysc36VO3bl0uueQSJk+ezLBhw1i3bl2mQ/JVKnnt58p/69at47333gvM63/nnXfy2GOPMWnSJObPn8+gQYMyHVJEvXv3Brz/DkzEp59+Sv/+/enVqxcDBgzg008/TevxSwtPVwdzVw1a5uU+kzF16lQmTJhAly5dWLp0KX379qVXr16AU9Dm5eVlpJrz6quvpkqVKuzevZujjjqKlStXMnPmzLTHYZJz9NFHA1ClShUmTpzI1VdfneGI0sOjvPZk5b/bb7+dffv2Ub9+fV544QUqVKjAvffem2Joqdm6dSt79+5l7NixANxyyy0ZjSea6dOnAwd+B6az1nD06NE88sgjVKtWjW3btjFo0KC0nzyUBhldj9svQe2UUbFiRSZOnMgPP/zAHXfcwWGHHZbpkEwChg4dWni7WrVqzJ3rXHRm8gomC0Vd+U9VW6pqy9q1a0d98s6dOxk7diwDBw5k7Nix/Pbbb37GGpeTTjqJ//73v/Ts2ZMbb7yRKlWqZDokAObOnUvnzp1p06YNXbp04d///nfE7RKtNUxF0fVqgvTdHE1Qc9vTgltEeobd7+Ll/uOVSKeMdL4xO3bsoE+fPtSpU4dRo0axZUvkZY0z9WH58ssv03as0iBU9ZjOL75MSCavReQwEZnEgSv/dQVeSDaWnJwchgwZwkMPPcStt97KwQcfnOyuPHP11VczY8YMBg0axNChQ7nnnnsKHwvP5XTm9htvvMFLL73Eaaedxty5c3nsscc823ey/8ewYcMYPHgw3bt3Z/DgwQwZMsSzmJK1YcMGfvvtNx5++GHGjBnDjz/+WPhYYHNbVT35wZlZ6U2cOY3L4XRsecWr/Yf/nHLKKZqstm3bxryf6v6i2bdvn+7YsUMLCgp0+fLlmp+fn/I+UzVkyBAdMmSIDh48WFu2bKlDhgxJy3FLsnbtWv3Pf/6ja9euLfb3adOm6bHHHqtt27bVadOmZSa4CNL1fkUCLFef8izdea1x5PZ3332n7777rn733XfJv2g+euyxx4rd9/r7Jl7dunXTt99+W2+44QZVVe3Xr5+nMWXyM++l/v3766BBg/Tll1/W//znP9qtW7dijwcxt72a8rQ3sBBoASxyf+YBr3mx/1SFrowyrVy5clSpUgUR4ZRTTilsc8qk7du306RJEwYOHMgJJ5zAwIEDMx0St99+Ow899BDLli3joYce4o477ih8LJNnwGPHjiU3N5fp06fTtWtXbr311rQeP92Cmtf169fnT3/6E/Xr12fSpElAsKo0W7RokdHjh4wYMYL33nuPESNGAE5exSOdr2UQvpu3bNnC1q1bueiii/jjH//oeROmH6+nV73Kp6tqO+A8VT1bVdup6vmq+qAX+0/U0qVL+frrr+nVqxfdunWjT58+JT8pA0IJ7sUbm+w+HnvsMapXr86UKVOoUKFCYcegTH4RBrEdE+CLL74gLy+PGTNmMHv2bLZv3x7X8+J5LYNU8IQELa8jCeVQJk/ofvzxR/bs2cO0adN4+OGHOeaYY9J6/GgOPfRQBg4cyLx583j44Yc55JBD4npeOl/LohcvmcqBpk2b8oc//KHwfv369T3dvx+vp6e9yoHjRWQczjhPAFT1bI+PUaLnnnuOXbt2cf/991OjRg169+7NmWeeme4wovrkk0/YsWMHf/rTnwBverqn0lP0sssuo02bNixZsiTq/hLdZzLbh4TaMevXr8/atWv3t2OOqO78/nanc3vEL2mLCWDjxo1MmTKFgoIC3n77bTZt2hQxpnDxvL+ZHO0Qh0DkdSRVq1bNdAiMGTOGKlWqcPrpp9OkSRP++te/BqI2LWJcjV50HozxefXTjBkzqFevHo8++igVKlTgzDPPZMCPtwGQC+T9tpPFZ/0AuYsT3neyuX3XXXcVuz9s2DDfj5kqrwvu/kAbVd3t8X4TsmrVKqpWrUqdOnUAqF69eibDKXTrrbdSt25d1q9fT506dZg2bRqPP/64L8dKtCCoU6cOXbt2ZdKkSfTv39+TfSZbGI0ePZrvv/+edevW0aVLl5hnwOmKCWDixIl8/vnnzJkzh2nTpjlVkLMPrDUuhZPyBCKvgWLjflWVUaNG8dxzz2UwIti7dy/79u2jQ4cOlCtXjhkzZmQ0npB44mo+3RmZ9/WGr2k+vTkre6/0NaZly5bx0UcfFU4Ic9NNN0HN+J5bUl55dfLbu3dv58TLo5NyP3g9HGwR0EREyotIORHJyHCzkSNHFhu6c/7558f9XD+ra7Zv384XX3zB+PHjGTp0KOXLl894TOFatGjhfkirw7dv7//wplmkdsxwzac3p/n05izbsKzwCyjWazW+eyfGd+/E95+uZHz3TknF1L59e9avX0/r1q1p1qxZxO0SrRpLNa40CEReA1x11VXMmTOH2bNnM2fOHL766quMf1579OjBl19+SdOmTbnkkks4++zIlRGJfl7jeTyeuJo0aRIzrkSl8nmtVq0aP//8M1OmTGHOnDmFE1IB5K3YzYYdBZyVtzPi/xorr7zMoSDUlpTE6wQ8DXgQp0PLIvd3QkTkLBF5S0QmichZyQTRtm3bYlXjXbrEPyqtpC/dVBJpzZo1/Pzzz4X38/PzPYkpFapabAaqWJNfhEvXCUXRzj5Fk3vLWwcOp/Pztbr11lsZP34806ZNY8mSJVx33XWe7j/AUs5rr/Tq1Ys777yTu+66i7vuuitmZ8p0fT737dvHmDFjaNmyJQcddFBhTV88JxQlfV4jPR7v/9W6dWtmz57N559/zosvvuic5GTYyJEj6dy5M5s3b2bXrl08/PDDhY/ltqjI6oFVWZxbpfBKNh3vX3ifqKVLl3q2b79OykusKheRFqq6QkRq4zRDzFPV1ZG2VdV2InIQUFtV1ycZkwI7gEpEmKgh01KpGlmwYEGx+02bNvUoquRdc801VK9end27d/PQQw8xatQoHq0T33P9qiZav349K1eu5IwzzuCJJ56gXbt2+4/ZoiK5LSoC0LxRTba8tYW9v+zl69Ffk6d5vlZLb9++nW3bthXWAFx//fVA8ZOJL9fWo2abmoVVj7NGO2tz5H+3hs+aOO9309Wf+RajHzzKa08cf/zx9O3bl4EDB3LiiSfG7LiYrmrMUJ+aCRMmUKNGDf785z/72qcm3v+rW7duhbdVlRUrVjC0Z4wnpIGIcMkllxTeX7VqFZHrrdL//gW1T1Qk8bRxjwfaAyOBJcA0nDPwA7jDR3oBdUTkZGCmqnZNMKa3VHWJiNQF7gcy/FHzTsREKlKlnwkiwoQJE/jf//7HgAEDis1sFBKq1oP0tIXdeOONdO3alU6dOjFq1CjuueeeqIsk1GxTk5ptnEayP44ey2djxhYWkpEKyGXffM/233bx6JvvclheYgX9mjVris2KFaoxCT+ZKG08ymtPPProo0ybNo1Ro0bx888/s2LFCgg70Uz35zXUpyY041tQ+tRUq1aNqVOnFt4fMGAA8GzmAoID5iYfNWoUzx174Hbhbe/hJ8Be5nYifaKC0n8lnoI7R0QOBg5W1edEJHLPJce1qtpGRN5U1X0ikvCAON2/1u8WnJWECsU7n3FQRU6kYDj22GMZNGiQ0x+gX1iV9DlbCgvHeIWqhUJVRLc8Pz/u59aoUYMePXowZcoUWrduXWyRmFS1alSfVo2czm6JJl3EGpNf/+lJXKmcUKRBynntldq1a1OjRg3GjRvHbbfdxrJly+DCyNumqzZm5MiRxabvTKRPjZ+Kzn8AOHO6P/RszNwO1Qr5VUN01VVX0blz58ILhK+++goiFNzJSDa3Y71/RV+r3IZ5xWoBPmvStMQLBb/EU3A/DbwE3CUilYBvYmy7V0QOAVREKgMFMbaNSEQuBc4HagATiz6mqpOByQAtW7Y88NIw4CImUhw9F/10+eWXs2DBAlSVqVOnMnnyZFje05eryHjPVhs1akT37t3p0aMHl1xyCccdd5wnx081rnirHosWGP/8pRpdqtcoMYZUTiiSlUAzWMp57ZULL9xfSo8ZM8ZpI/15eMTCqGhtTG7v3ISPFe/nom3btsXuJ9Knxk+NGjUqdv/QQw8F/Kkhive16tWrFzfffDPg5FCjRo3g6xsjbptMHiXjmGOOYeXKlezcuTNm0xwBOpkusXOaqj4CjACOBHYDd8XYfCgwB2jq/o5/QNz+472gqtepandVXZzo82PKcO/TaIlUVKSep4lae9tbrL3tLXZ9/Qtrb3sr5rZ33nknq1atYtOmTeTn57N58+a4jvFZk6Z81qQp+e+/X3hmXpJ4O44ddNBBVKpUiYoVK3LQQQexd+/euPYP8M9ftrJp7156f7cm7g4t8cZVrVo1Zs2axaxZs5g9ezbnnHNOxO1qtqnJH8b8gcbDGvv2ZeOR8e7vkTj9SaZF2S7lvPZK586di92/8UbnS79ox6ZotUOJdnbys6NjkBQvILfG9ZyiNUR5bg1RPK/Vsccey4IFC1iwYAGXXnoptWrVirpteB4lk9vxuPHGG9myZQudOnXilFNOKTbXfFCVWHCLyETgEmCYW409Jdq2qvq+qnZQ1SNU9UJVzegSn0GcjSpoli5dyo4dO6hUqRLNmjXjz3/+c6ZD4osvvmD69OmFM5Tt2LEj7ud2qV6DVxofzfQGR5X4ZRs6wYn3JCdijUl2K9YMhnNifoCg5XWy/BwxEq+iheR9RxwR8+Q3XcMEkznRbNWoPkM7nsX17U5L6KQmdKGwceNG8vPznUmM4pRIbici1DRXrlw5WrduHfNkIhnhJzleiGc4WFNVvQ0IDbiLOvhYRM4VkZdEZImILBUR7/rVJ6GsnDGnIicnh7vvvptatWqRk5MTc9tkzsyTET5D2caNG307ViLiqTGJpOiVgp+vWxJCzWCTYjWDeZHXXgzzTEU8NUR+fF+ED1/MotoYXwTxQiG8aa5mzcg1NsnWhiZ7khNLPG3cO0TkjwAichIQqxH278AFqvqDF8GZ+IR3BOveKPGe6u3bt6d9+/YxtynaZthldMnV10XPNOXC2+nWvGPhlW29MW2iPi/iDGVZrEv1Ggl/Saej96qqPiIiH+A0gy0jejOYF3kd6GGefintIw4SFbpQWLRoUYkXCumSStNcpsRzxd0HZw3dfJwhIf1ibPsJsMeDuEwpUPRMs1vzjnE/LzRD2aGHHsott9wSdYYyr8xa+Sobd26m67ODAtOkko7aogSawbzI67dUtSNOe/ndEWLpJyLLRWS5XzUsfrWRmsS1b9+eUaNGZToMILWmuUx9pkq84lbVTcDgOPe3BFgtIqF+8aqqwR7JnoJQm2g8V5HJCh+eEaTJO4p+aK8P3hCmuHVr3rHwxKJervfvYYA1VdX2IvKmez9aM1jKeR1rmKf7uO8jRorWfDT16bNarKakxu79PZIzKIgxBUkqTXPJfKa8qE2LesUtIm+KyBvuWfBGEXnH/R2rY0o/4EhVbeP+BLrQ9qIHdzKKdYJZkfF1G5KWaGeRIF7ZBlEiHeZSFG8zWMp5LSKXisjjwAzChnkmw+sc8qojWLGaEo8LyEQ7UyYbU6J9MtL4eU1IvJ0NJ06cSOPGjZkzZ06x9csj8aKfjxe1aVGvuN11eBGRZ4C2qrpTRKoAU6M9B3gNZzGCVThtWkXPtI2r2FR+I6onPNlJ+BlbtijDV7ZB1Qe4jZKbwVLOa1V9AXgh+VCLC8+hbBXEWqtk+mQEUbxTptavX79wBcJbbrkl5raJ9vPxSzyd05rgTIayE6gO/CHGtq3dnxAFArFub5Al2oEl/AM5/tU5foaX1Yqe5Byev6VwkhOv9pnNVY8JNINZXvukpKrWYh08d7yaUF+RsiwdzZiZFE/ntP7AY+4QkElA1Hk63av084CeqtpOVS25MyTRamkvqh6L7mPZN98ntQ+vFa2W8qLQDt9nNhbaiTaDWV5nTrIdPE3pFs/MacuAS4EeQBdVfT/atu5iBAuAV921eyOvDJFFsnUSl27NO7K479PMvvKhuKrfvCiM/CgkS0t/gET52R+gSOH7P6ChqrYGGgJfRtq+NOa1yax05HUQ+9R41R8gnmU9ewIDcSZnaCwiE1U12qoPgVmMIBmRenCna2k5E1lpactMVJr6A8TbDJbVeR2vZOcdMIlLR16X5j418bRx3wC0UdW97pq8S3FmXIokMIsRZLN0Ta5vyrxQM1gNYCvRm8HKRF4XXeilWyOrlk5E0avb65rcFIhOdqVZPAV3AXAE8L37O1bSBmYxAj94MUNZPILSc9H4x49Oc4lS1WXuanx1gB9VdV+UTUt1XpvUlZar22zpeBpP57TrgYlu57SHca7Ao/k26IsRhM8dbEwm+NEfIFFuM9hbwDjgHRHpFWXTwOe1MV7Ilo6n8Vxxf6uqhWvpiUi1GNtOFJEc4FXghSDOWW5zBxtTKN5msMDntfFXEGqIskG65tiI54o7fNKEqJMoqGo3nHnNfwD+ISKLkw/NJKJox5pZK1/NdDgmO4SawSBGM5jldfrEM6QyE72lg1BDlA3imRXNi/cvnivuSiXcD3cycCZQD3gvmaBMcfHMrmQda0wSQs1gNXHmEI/VDGZ5nQZFe1tHm3q1tLQnl1VevH/xFNxvisjTwL+B04DF0TZ0z8TfAaar6l+SisgcoOjsSm/8pwGP/OcN1n2+lUf6v5HZwEy2i6sZzPLamGCJZ3Wwv4lIC5wxnuNU9aMY254lIkcADUXkCGsLMybQXgDOiXEfsLw2JmjiueJGVVcAK0raTkSG4MxhvAI4yZ2wYWwK8aWk++Pv8un6bZk6fFRBjCvVmEJX/17XBJTG1ypA4moGC1pegzfvgR8drlKN65/j/8um7+NfDzoeQfy8ltb3L13i6ZyWiItVtaOqDlPVDsBFiTxZRKqIyHQRmeIOVTEmaY/0f4NH+luzQgxvisjTInK9iMwgejNYSnkNwcxt63CV3cry+xfXFXeIiLQCVqrqb1E22S0ipwMfAi2BPQnGcykwR1XnicjzwDPxPrH74+8e8LdPf9jGzt179z+2a3ixx/PXOBcYBbseIn9NP249QwH46pWHufWMAWwd7PTO/virzbQZ/CoFh18MwKaKX/DC4RfzFjsB+IJ9DGQnB0eIIZJicUWJKSQUU8jWQ3YB8FP5Ama6twsOvrjYNqG4QuKJK5GY4onLi5gSjSvVmOKNK5WYAHLcY3y6fhvdH3+XtYcn9lo9f91pJcYYjwSawVLNa/Azt5P4vMbzHny38nW+2LmJFs/eQOMfLqbR6ReWGGsqn42cx99l09YdrNu7J67PBaTn8xrtezBaXKnGFE9cqeZQMnElGhPgW27HM1f5IlVtLyIjcWZYqgdE+wTn4syy9DfgK6B3XFHsVw9Y6d4+YBYnEemHu2ZwgwYNEtw10PCMYndbAt/8+xU2/rqb7U9P48fTLqDR6RdSfmlVclq1Isfd7n+VK3DkMTVwpnWGL16tQr3jmhfup1ylChzcKMn5diPEVEzd4ncPjInCuDxTUkwQR1wexwQevFaZjwlg/RdbAdj1617Wf7G12Gcp3eJsBssltbwGP3M7ic9rPO/BMY0u55iLL08slkTiivDZKMqXz0USMfmeR2ExQclxpSWHEnytwmMCn95DAFWN+QO86f6e4f5+O8p2AjxV0v5KONZVQCf39sxY255yyilakm6T/q21jzmpxO3CtW3bNqX7fsWVyjFL4kVMqsGMK4gxqXoXF7BcU8i7WD9e5LUGKLdDJl63SCdet0h/f8SJOvG6RUnvx+u4Xhj3gTZrXPL/n4igfV5VS+f7Fx6TF3FFy+14qsrXiMjrwHQRqUCEs2X3BEBF5AcRORX4L+5kDqqayIIEL+CMK70QmJfA84wxPvAoryFguX3DJGdJ8dmraxTeNiZbxNM5bQbQUZ2lPAuAO2JseyowFngdWAQsTCQYVd2pqler6gBVjbsNLJrnrzuN446MNUNr6vLy8tiwYUNC63WnI65EBTEmCGZcQYwpGSLyloj8T0TecX8vFZF5IvKnsE1TymvIztxORqpxdbnlZGrVP8TDiIL5WgUxJghuXOHiueK+Q1UXgXOWLSJ/Bd6OtKGqtvMyuGyQm5trS9iZbPUp0F1V14vIkcAI9+dFnMIaKJt5bYItiDUm6Ywp6hW3iFwtIm8BJ7tn4m+5K4R9HeM5F4jI2yKy2N2+5G6YAZPMFbQxWeoUYLN7ewvQQlXXA/lFNyoNeW1MaRL1iltVpwHTROQiVY23TWoEcJaq5rurCS0GXkk5yjSyK2hThtwDLBSRfTid0EaKSHngsbDtRpDleW1SU/SCxr4jMy/WFfcw92Y3EXmq6E+M/X0IhNbKrEEcs62ZxFmtgPGCqr6sqmeoaltVPVNV56nqPlV9PmxTy+s0CWpu5+bmsnr16pirXpn0idXG/aT7+8DR8dGdACwRkZ+BQ4FNbnW7quqZScYYKEE487QzXuMFEbkaZ0x2YQ9xVY3UOFcq8zoIuRwuKHGYYItVVf6je7Nt2EN7RORwVT1gaT9V9WZKpwCzxIouiF+EJqYBwBmqujvWRqU1r+0zarJVPL3KO+J0XFmBc+ZdF9goIv1V9eqiG4pIc6APTrWaAKjqn70M2ASXfRFmnYVAExFZBShEHp9teW3Kimy5+Iin4K6uqleE7ojIq6raVUQiDQmbgXMWv96rADMtW95IE5m9fzGd5v6EKM4qYOFKXV6b7OdHbmfLd0Q8BfceEbkV+BjninuX2/M00tpzX+FM0ZbMIgSBlC1vpInM3r/oEhifXery2mS/spzb8cyc1hX4FjgR+Abo5vY87RBh21nAd+5Yz9C4b5MFgtqbNahxpSqT/5eIPOj+fsudo2FpCflqeW08VVrzOl2iXnGLyEmq+iFwBrDJ/cG9H21x49uAk1X1B0+jNL4L6tlrUONKVSb/L1W9yf3dJs6nWF4bT5XWvE6XWFXlJ+CM3wxPbiV6wf0Wya3Va0zWyPZ2c3eJ3gMXXgZU9c4If7a8NiZAYg0Hm+7+vjuB/bUE3hGRn/bvJnvHeRoTSTYW1mESXSQk5bwWkVygG/Ad8Iiqroz9DGOyT7pO6mNVlb+Jc1ZeDTgK+Bz4A/CtqraK9BxVPd2PIEubbL9iM1mvfCIbe5TXBThzoJcHfixh2xJZDpkgStdnMdYVdzsAEXkGaKuqO0WkCjA1fFsRGaaqo0VkBmFVcDbe80D2RWMyLFrbdrFmsGTz2h33PTrsz9eo6lMicgJOm/lfw57TD+gH0KBBgxL/gaDmUBBPKIIYk0mNqEZs6tq/gcgHwMWqus5d+m+eqp4Stk1dVf1RRI4Kf76qrvE0YlfLli11+fLlJW531llnsXjxYj9CMCYQROQDVW2ZwPaiqioiB4wqKToBix95LSKHA/eoar9o28Sb28aUdtFyO55x3NcBj4lIDWArzkQMxYSmR/WrkDbGeGo8zhXvIvZfSQthE7B4mdfuFfXJwGFAIv1mjDFhSiy4VXU5cHEaYjHGpIGq/tX9He8ELF4cc3K6jmUO9Ej/aAOBYrthUqSJ9JJXUFBAuXKRpw9ZvHgxf/vb32jWrBmtWrXi8ssv5+abb+abb77hzTff9DSObBfPFXdC3Or036nqMhHJUdV8r48RL2vbMeZAInIsTs3ZLziTq4wGqgJ/V9VXozwnI3nd/fF3k3re89fFXhclLy+PWbNm0aBBAzp06MAJJ5zAvffeyy+//MKcOXMO2P6mm27i2GOPpUGDBnTq1CmpmELHrVWrFgUFBdSsWZM2bUoeSv/BBx+wfPlyrrvuuqSP64W8vDzefPNNKleuzBFHHMGePXv45JNPmDVrFq+//jpLlizhp59+4v7772f79u0MHz6cOnXq0KVLFxYuXMjmzZs56aST2L17Nx9//DHbtm1jwoQJHHbYYYXH6N69OwMHDiy8/+STT3L55Zdn4t8NNE8LbhG5HTgeZ+GClsBcnEVK4n1+Lh4OGbHC2piIpgJDcNbWfg1oAezEGSZ2QMGdal4HwapVq5g/fz5Dhw5l4MCBHHvssVx//fXFCuEnnngiYiGxefNmRIScnByee+453nzzTerUqcPQoUNZtWoVzzzzDD/++CPXXnst33zzDYsXL6Zq1arcd999bNiwgfHjx6OqHH300dSoUaNwn+XKlSMvL48lS5bQuHFjypUrR69evYpt/5e//IVTTjmFBx54IOMFN8D555/PlVdeSfv27Vm0aBH33Xcfq1atonz58hQUFLBnzx4WLlzI8uXLufPOOznmmGMAWLhwId27d6d169ZceumlvPDCCyxZsoTnnnuuWEH9/PPP88knn9CtWzfOPtvbq/3SJGbBLSKnAN8DPwOdgF9V9bUYTzlXVduJyJuqWiAiFROMx9MhI8aYiPao6n8ARORLVd3o3v4tyvap5nXSSrpyjtfy5cs58cQTAdi2bRtVq1bl0UcfZf78+QwYMKDwsUhWrFjBcccdB8B5551H7969ufzyyykoKODggw/mt99+o27dusyYMYOGDRtywgkn0LlzZw4++GAeffRRKleuTOXKlVm5cmXEK+wOHTrQvXt3rrjiigO2D6lUqRI//vgjdevW9eT1SFa1atUAqF27NgAVK1Zk165dPPbYY7z00ktMnz6d/Px8VPWAKvHq1asDICLFfhcVfsVtIos1jvsJnA4ru4DaOCsDbRORy2P0CN0tIr8DVETqus+Ntn/fh4wYYyJqLCL34OT3UUVuN4qyfdx5HVTLli3j1ltvZefOnYV/C7/ijmbz5s3UqFGD/Pz8YoWOiPDggw8yZMgQVJW77rqLW2+9lY8++oghQ4YwcuRICgoKuOqqqzjhhBMAIs7LXaVKFQBU9YDtQ2rWrMm2bdsyXnBHc9xxx3Hvvffy2Wefcc4553D99dczYsQIjjjiCC6+uHgXqfbt2zNo0CC2bNnCAw88EHO//fv358MPP2Tw4MGMGzfOz38hq0QdDiYiS1S1rXt7pao2d2+/Ga1Ti4gcDdwLHAusBoar6lcJB2VDRoyJWxLDwdpGe0xVl0TY3pO8jpcfuX3hhRdSr149qlWrxsqVK+nRowe1atUqLLh//vln7rjjDl5//XWuvfZahg0bVvjcjz/+mH/961/UqVOH1157jcMPP5y6desydOhQXnjhBRYtWkTdunX5+uuvOf300/niiy9Yt24dEydOZPv27dx+++0cccQRVK1alaOOOopatWqxadOmYr87depEjx49GDt2bLHt77rrLgCuvPJKpk2bxsEHH+zp62KCLVpuxyq431HV1u7ti1R1nnt7saqeFeU5p6jqBykEWWzIiKp+Em1bK7iNcSRacCex/5TyOlFe5/aePXvo06cPTz31VFLPV1UGDBjApEmTPIspETt27GDw4MEZO77JnGTGcfcTkfLuEp6hQrsicH+M51wpIv8APgDmqOp7iQRpQ0aMCaSU8jrTDjrooKQLbXCqxXv27El+fj45OTkeRhaf9evXM2TIkJT2Mb57cj3hb3l+fkrHNf6INeXpqgh/2w28HOM5twCIyMlArojMVNVo7WbGmCxgeU1cw7b88oc//CFjx/ZarHHcAD/99BPNmzfngw8+oF69eowYMYLLL7+c448/nn/9619s2LABgDlz5nDYYYdx3HHHMXToUNasWcOIESOoXr06O3bs4L777qNOnToMHz6czZs3U1BQwLnnnstll12Wrn/VV14PB6uKM1lLF6Ay8Hcv92+MST/LawPpGcc9ffp0HnjgAfLy8hg+fHjUWPr371/YLwDgrrvuYty4cdSqVYuvv/6akSNH0qFDB373u98xatQoAHbv3u3vC5RGXk/A8izOFXl/Vd3k8b6NMZmRubweUT3J5/1SeDMvL4+HH36YZcuW8fnnnzNmzBg++ugjzjzzTLZt28aVV17JueeeyxtvvMH06dOpUqUKlSpV4v7772fixIl8/PHHVKxYkUMOOYQxY8Z49I9lJ7/Hcb/33nvMmTOHyy67jFjraEyZMoX77ruP66+/HoD8/Hxq1aoFQOPGjdmwYQOrVq0qNha8YsW0jWL0nScFt4gcoao/AH/BXQpURKoBqOrXXhzDGJNepSmvmzdvztNPP82pp54KOEOSxo0bh6py8cUXc8455/Dggw/y4osvIiLs3r2bTz75hLVr1zJ5stP1Zu/evZn8FwLBz3Hc77zzDt9//z39+/dnw4YNLFy4kJo1a7Jx40bAqUY/9NBD2bx5M3379uXss8/muuuuo1evXuTk5LB582YOPfRQvv32W+rWrUuzZs344IMPaNnS6du1e/fuUlN4e3XFfRVO9dlwnAQPvSMKXOPRMYwx6VVq8vryyy/nqaeeOmB8tIjQuHFjNm7cSP369QsLk4oVK/Lpp58Wfuk/9NBDvP7668yaNYvKlSunPf5U+d3JzItx3Hl5ecydO5d69eqxadMmbr75ZsaPH8/NN9/MSy+9xC+//MKkSZN47rnnAMjJyeHUU09l3rx5jBgxgltuuYXq1auzbds2Ro8eTd26dbn99tsLr8rbt29fatq4S1zWM6GdifRS1aeL3O+iqv/07ABF2HAwYxxpGA6WtryGsNz2qKq8Vq1aVK9enWnTplFQUECtWrUKr7g7d+7MSy+9RJcuXXjxxRcB5+rs888/5+mnny6sHs/NzWXixIkccsghqfx7xsQtlWU94z1AeaCPiDyDc2ZeHrgW8C3BjTH+SiavReRU4Bbge1Ud7P7tFpyZ2Q7CaSv37oohTm3atOH++++nevXqLFq0iJtuuolt27Zx4403IiIMHDiQ3r17U7VqVSpVqsS4ceP43e9+R9++falSpQoHH3ywTYBiAsGTK24R6Q3k4ixW8CFOgu8GFqjqgykfIAK74jbG4dcVdyp5LSINgYGqOtid/2GaqvYUkYHAR6r6VrTnWm57b+1tUV/umOqN8XYYnNfDwZo1a8app57KXXfdxe9//3uqV6/O/fffH7GjYTby9YpbVacD00Wklaou82KfxpjMiievo605EHb/MGCje3sNUC/CfmwdgoAL4nCwyy67jFNPPZWuXbsycOBAevTogaoe0NGwtPF6ONhuEXkaOAK3I4uq2tpsxmS3qHntLr17wLRc7hV3yM9ALfd2A+Dj8O3dWRMng3PF7WHsxkNBGg720ksv0bt3bwDmzp3L/PnzOeOMMyJ2NCxtotdZJGcScJe7337Aux7v3xiTfgnltYj8AWdRknNEpJ874+J/ReRBoDnwts/xGp/EGg42btw4zjvvPN+HgwH07duXKVOmFPZYv+yyy3jllVdYvXo11atXZ+3atYX7tSvuku1S1a9EpJyqfikimZsn0BjjlYTyWlU/B3qG/S2pRsbm05sn8zRW9l5Z4jYffvghl1xyCatXr6Zy5cqFvc87derE6tWrmTlzJg0bNizWnlqnTp0DptvcvXs3t9xyC6rKnj17uPLKK2nbNuoCbKVSJoaDhStfvjzdu3dn8uTJETsaliZeDwe7DZgA9AYGAO/HWpozFdaBxRhHGoaDpS2voXhue1Vwr1q1ivnz5zN06FAGDhzI6NGjGT58OCeffDLlypXjqquuilpwh/522WWXcdFFFxVbhnPmzJk88sgjNG7cmI4dOwKla6IPk1nRctvTqnJVHaOqv6nq46raws/kNsakR2nI6+XLl3PiiScCsG3bNg466CA2b95Mz549mTdvXsznTpkyhb59+xa2p06ZMoXTTz+9cC3vVatW0apVq8LtrdA2fvOk4BaRt0RkqftT9PZSL/ZvjEm/0pTXy5Yt47jjjmPnzp2A05nphx9+YODAgfzvf//j888/j7s9tW/fvixcuJD/+7//AyicWjOkNLapmmDxajiYtWUbU8qUprz+5ptvuPfee6lWrRo//fQTc+fOZd68eVSuXJmPP/6YJ554grvvvpvrrruOjz76iE2bNjFhwgTmz488VWjR6Tb79u3LX//6V+bNm8e+ffvo0aNHmWvjNunldRt3C5zep9WA84DbVPVezw5QhLVxG+NIQxt3C9KU1+B9bu/Zs4c+ffrw1FNPebZPY9IhLW3cwIM4ky+UU9V9gI3hNib7ZXVeH3TQQVZom1LF64JbVHULzupB4P1wM2NM+lleGxMgXifgdBF5CWgsIrOAadE2DOpCBMaYA8Sd18YY/3m5OpgAa3Gq1BoD36jqpmjbq+r7IjIUGOg+vyJwcpGFCM4AkpsZ3xjjiUTz2hjjP88KblVVEbleVTvjzE1cyBYiMCY7xcprY0xmeN5WJSILgP8CBQCqeqctRGBMdouU15mNyJiyy+uCO+75iN2FCO4CmrkLEUwWkdBCBAcDj3ocmzEmORlbzPizJk2Tel7T1Z+VuE2ic5WHz01evnx5tm7dyvXXX0+LFi3K/HzlJn08LbhVdUkC23q2EIExxj+J5DVE7Xj6Ps4V+xpVDW82812kucrz8vK45557mDNnDldddVXU54bWfu7RowcAU6dO5YILLqBjx47s3buXyy67jPPOO6/wb2Czpxl/eT0czBhTxqnq+8DQsD/vBCoC69MfUepzlRedm/yTTz4pnJu8QoUK5OTkFPsb2Hzlxl9WcBtjkiYizUVkfthPnQibtlfVa4ALROTQCPvpJyLLRWR5aL5wL6U6V3nRucmPO+64wrnJ9+7dS35+Pscff7zNV27SxiZSMMYkLc6Op6hqgXtzC1Apwn587Xia6lzl4XOT33LLLSxYsIBffvmFYcOGcfLJJ9t85SZtPJ2rPJ1srnJjHH7PVZ6ooh1PcTqZzsaZNvU3YLOq3hbr+TZXuTGOaLltV9zGGE9F6ngK/DkTsYDNVW5KH2vjNsYYY7KIFdzGGGNMFrGC2xhjjMkiWds5TUQ24sxp7odaQNAWUghiTBDMuMpaTEepam2f9p12ltuBEMSYIJhxpT23s7bg9pOILA9SL10IZkwQzLgsJhNNEN8Hiyl+QYwrEzFZVbkxxhiTRazgNsYYY7KIFdyRTc50ABEEMSYIZlwWk4kmiO+DxRS/IMaV9pisjdsYY4zJInbFbYwxxmQRm/LUJSLlgb5AbeAQYKaqfmgxHSiIcQUppiDFYoL5fgQxJghmXEGKKSixlImCW0RygcuBn4FPgabAXvfnTVV9HvgH8KiqfikiAgwXkb1AdSAXqAG8rKpPiUhekeffpKq70hzTduBF4D/A66o6V0TauXFWAIaoalLrHqcY1z5gkLurc1X16AC8Vj8BD+OMs/xYVSel8lqlGMt3wANAI1Vt5+6vWCzAD8BjOK/l16o6PtHXqyyx3E5bXJbbAcrtMlFwuyap6nwRmYmzStFfVHUHgIgcD/wbOFJE+rrbPwhcq6r3AG+7280FngJ+xXnttgJ70h2TG8MOIAfnAwPQH7gCOA7oA4xMd1zua9VfRFoAm93HMv1arQLmqeoMEZkjIk+Q+muVyutzjYjMKbKv8FgWA6tU9WERmSEiFVXVFneOzXLb57gst4OV22WpjbuviPwbmO/enyAik0TkfKAJzofgUlUdirNmcG1gZ+jJIjIEmObevUFV+wLribAWcRpiWqOqZ+B8MP4WCtFd83gNUC+FmFKJK+Ra4En3dqZfqwXAySIyHqgJHEbqr1Wqr09R4bHUA753H/vJjdfEZrntf1whltsByO2ydMU9BXgDeBynqqLomVQL4ATgZREZAxTgfBAfdB//K7BBVecDuG8GOC/+IemOSd2hAKqa79TWAFAgIuWABsDaFGJKOi738RzgSFX90o0x06/Vr8DN7nYvuXGk+lol/fpEEB7LWuBE97HaONV2JjbLbZ/jch+33A5IbpelgjuUDO8DI4C9btvEe6o6TUR6ApNV9TYRqQ60UNUdInIxztnvGyLSQFXvdc/wKuOc5V2bgZjaAL3dGJ51dzUZmAocBAxNJaZk43Kf2h0orC4KwGtVBacdrDwwXVULRCTl1yrZ10dEJgEnicg4VR3Mge/bD8CVIvIg8JFVk8fHctvfuNynWm4HJLdtHLdLRCoA/XCqL8oBU1V1ncV0oCDGFaSYghSLCeb7EcSYIJhxBSmmoMRiBbcxxhiTRcpS5zRjjDEm61nBbYwxxmQRK7iNMcaYLGIFtzHGGJNFrOA2xhhjsogV3MYYY0wW+X84hTDR/d866QAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 502.117x310.325 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, axes = plt.subplots(2, 2)\n",
    "for level, ax in zip([\"Chapter\", \"Subchapter\", \"3 digits\", \"Billing\"], axes.reshape(-1)):\n",
    "    bar_plot_results((skl_results_long\n",
    "                      .query('model==\"Vanilla\" and input_data==\"Drugs, Sex, Age\"')\n",
    "                      .query('model_id==0')\n",
    "                      .query('metric_name.str.startswith(\"RE@\")')\n",
    "                      .query('code_subset==\"All codes\" and code_level==\"'+level+'\"')),\n",
    "                     \"target_metric\",\n",
    "                     \"metric_name\",\n",
    "                     values_subset=[\"overall_micro_F1\", \"overall_NDCG\", \"overall_micro_aucPR\", \"overall_micro_AUROC\",\n",
    "                                    \"overall_macro_F1\", \"overall_macro_AUROC\", \"overall_macro_aucPR\"],\n",
    "                     groups_subset=[\"RE@25\", 'RE@50',\n",
    "                                    \"RE@75\", \"RE@90\", \"RE@100\"],\n",
    "                     reference_row=\"overall_micro_F1\",\n",
    "                     reference_val='relative',\n",
    "                     legend_kwargs={'loc': (0,0),'ncol':2},\n",
    "                     abs_value_in_percent=False,\n",
    "                     improvement_direction=\"min\",\n",
    "                     average=True,\n",
    "                     axis=ax,\n",
    "                    values_legend_dict=metrics_pretty_names_dict\n",
    "                     )\n",
    "    if level!=\"Billing\":\n",
    "        # Keep the legend only on the 4th plot to reduce clutter\n",
    "        ax.get_legend().remove()\n",
    "    #ax.set_title(level+\" codes\")\n",
    "    ax.set_ylabel(level+\" codes\\nrelative improvement (%)\",labelpad=-2)\n",
    "fig.set_size_inches(dual_col_fig_size)\n",
    "fig.tight_layout()\n",
    "if mpl.get_backend()=='pdf':\n",
    "    # The inline backend distorts the scaling and outputs figures \n",
    "    # with wrong dimensions\n",
    "    fig.savefig(plots_dir/\"opt_metric_perlevel.pdf\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "133eb3c7",
   "metadata": {},
   "source": [
    "### Fig 3: Multitask learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e11dcfe3",
   "metadata": {},
   "outputs": [],
   "source": [
    "ndcg_improve_title = \"Relative NDCG improvement (%)\"\n",
    "methods_pretty_names = {\"Vanilla\": \"Naive Multitask\",\n",
    "                        \"multitask\": \"Re-weighted Multitask\",\n",
    "                        \"focal\": \"Focal Loss\",\n",
    "                        \"outL2reg\": \"L2 regularization\",\n",
    "                        \"imbal\": \"IRW\",\n",
    "                        \"imbalNorm\": \"IRNW\",\n",
    "                        \"HierL2\": \"Hierarchical regularization\",\n",
    "                        \"TreeMin\": \"TM loss\",\n",
    "                        \"MCLoss\": \"MC loss\",\n",
    "                        \"flatRollDown\": \"Roll Down\",\n",
    "                        \"flatRollUp\": \"Roll Up\"\n",
    "                        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "727fd724-806b-4df1-853d-87a0859d1c32",
   "metadata": {},
   "outputs": [],
   "source": [
    "plots_dir = get_project_root()/\"docs/latex/overleaf/gfx/\"\n",
    "fig,ax = bar_plot_results((skl_results_long\n",
    "                  .query('target_metric==\"overall_NDCG\"')\n",
    "                  .query('metric_name==\"NDCG\" and input_data==\"Drugs, Sex, Age\" and code_subset==\"All codes\" and code_level!=\"All\"')),\n",
    "                    \"model\",\n",
    "                     \"code_level\",\n",
    "                    values_subset = [\"Vanilla\",\"Per level\",\"multitask\"],\n",
    "                  reference_row=\"Vanilla\",\n",
    "                 reference_val='relative',\n",
    "                    legend_kwargs={'loc':\"lower left\"},\n",
    "                  improvement_direction=\"max\",\n",
    "                  average=True,\n",
    "                  values_legend_dict=methods_pretty_names\n",
    "                )\n",
    "ax.set_ylim(-2,2)\n",
    "ax.set_ylabel(ndcg_improve_title,labelpad=0)\n",
    "#ax.set_title('NDCG')\n",
    "fig.tight_layout()\n",
    "if mpl.get_backend()=='pdf':\n",
    "  # The inline backend distorts the scaling and outputs figures \n",
    "  # with wrong dimensions\n",
    "  fig.savefig(plots_dir/\"multitask_AllCodes.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28d44288",
   "metadata": {},
   "source": [
    "### Fig 5: class imbalance correction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "617f8aef-17b1-461e-b74b-cd4655f8eeb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = bar_plot_results(\n",
    "    (skl_results_long\n",
    "     .query('target_metric==\"overall_NDCG\"')\n",
    "     .query('metric_name==\"NDCG\" and input_data==\"Drugs, Sex, Age\" and code_subset==\"All codes\"')),\n",
    "    \"model\",\n",
    "    \"code_level\",\n",
    "    # values_subset = [\"Vanilla\",\"focal\",\"imbal\",\"imbalNorm\"],\n",
    "    values_subset=[\"Vanilla\", \"focal\", \"outL2reg\", \"imbal\", \"imbalNorm\"],\n",
    "    reference_row=\"Vanilla\",\n",
    "    reference_val='relative',\n",
    "    legend_kwargs={'loc': (0, 0),\n",
    "                   'ncol': 2\n",
    "                   },\n",
    "    improvement_direction=\"max\",\n",
    "    average=True,\n",
    "    values_legend_dict=methods_pretty_names\n",
    ")\n",
    "ax.set_ylim(-2, 2)\n",
    "ax.set_ylabel(ndcg_improve_title, labelpad=0)\n",
    "#ax.set_title('NDCG')\n",
    "fig.tight_layout()\n",
    "if mpl.get_backend() == 'pdf':\n",
    "  # The inline backend distorts the scaling and outputs figures\n",
    "  # with wrong dimensions\n",
    "  fig.savefig(plots_dir/\"imbalance_AllCodes.pdf\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "358e070d",
   "metadata": {},
   "source": [
    "### Fig 4: HMC methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "88d29803-b336-4c9c-bd98-f6f9c52371e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = bar_plot_results(\n",
    "    (skl_results_long\n",
    "     .query('target_metric==\"overall_NDCG\"')\n",
    "     .query('metric_name==\"NDCG\" and input_data==\"Drugs, Sex, Age\" and code_subset==\"All codes\"')),\n",
    "    \"model\",\n",
    "    \"code_level\",\n",
    "    #values_subset = [\"Vanilla\",\"TreeMin\",\"MCLoss\",\"flatRollDown\",\"flatRollUp\"],\n",
    "    #values_subset = [\"Vanilla\",\"HierL2\",\"TreeMin\",\"MCLoss\",\"flatRollDown\",\"flatRollUp\"],\n",
    "    values_subset=[\n",
    "        \"Vanilla\", \"HierL2\", \"TreeMin\", \"MCLoss\", \"flatRollDown\", \"flatRollUp\"],\n",
    "    reference_row=\"Vanilla\",\n",
    "    reference_val='relative',\n",
    "    legend_kwargs={'loc': (0, 0),\n",
    "                   'ncol': 2\n",
    "                   },\n",
    "    improvement_direction=\"max\",\n",
    "    average=True,\n",
    "    values_legend_dict=methods_pretty_names\n",
    "\n",
    ")\n",
    "ax.set_ylim(-2, 2)\n",
    "ax.set_ylabel(ndcg_improve_title, labelpad=0)\n",
    "fig.tight_layout()\n",
    "if mpl.get_backend() == 'pdf':\n",
    "  # The inline backend distorts the scaling and outputs figures\n",
    "  # with wrong dimensions\n",
    "  fig.savefig(plots_dir/\"hmc_AllCodes.pdf\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9ca3968",
   "metadata": {},
   "source": [
    "### All methods, all levels RE (not in manuscript)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "ff93caf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 1)\n",
    "plots_dir = get_project_root()/\"docs/latex/overleaf/gfx/\"\n",
    "target_metric = \"overall_NDCG\"\n",
    "for level, ax in zip([\"All\"], [axes]):\n",
    "    fig, ax = bar_plot_results(\n",
    "        (skl_results_long\n",
    "         .query('target_metric==\"'+target_metric+'\"')\n",
    "         #.query('model_id==0')\n",
    "         .query('input_data==\"Drugs, Sex, Age\" and code_subset==\"All codes\"')\n",
    "         .query('code_level==\"'+level+'\"')),\n",
    "        \"model\",\n",
    "        \"metric_name\",\n",
    "        values_subset=[\"Vanilla\", \"multitask\", \"HierL2\", \"TreeMin\", \"MCLoss\",\n",
    "                       \"focal\", \"outL2reg\", \"imbal\", \"imbalNorm\", \"flatRollDown\", \"flatRollUp\"],\n",
    "        groups_subset=[\"RE@25\", 'RE@50', \"RE@75\", \"RE@90\", \"RE@100\"],\n",
    "        reference_row=\"Vanilla\",\n",
    "        reference_val='relative',\n",
    "        legend_kwargs={'loc': \"lower left\", 'ncol': 2},\n",
    "        abs_value_in_percent=False,\n",
    "        improvement_direction=\"min\",\n",
    "        average=True,\n",
    "        axis=ax,\n",
    "        values_legend_dict=methods_pretty_names\n",
    "    )\n",
    "    #ax.set_title(level)\n",
    "    ax.set_ylabel(\"All codes \\nrelative improvement (%)\")\n",
    "\n",
    "#fig.suptitle(target_metric)\n",
    "fig.set_size_inches(dual_col_fig_size)\n",
    "fig.tight_layout()\n",
    "if mpl.get_backend() == 'pdf':\n",
    "    # The inline backend distorts the scaling and outputs figures\n",
    "    # with wrong dimensions\n",
    "    fig.savefig(plots_dir/\"all_methods_RE_Allcodes.pdf\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "87240ff1",
   "metadata": {},
   "source": [
    "### Fig S5: All methods, per level RE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "0adaf27e-8897-4ecd-9a9d-28ce06f200bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig,axes = plt.subplots(2,2)\n",
    "plots_dir = get_project_root()/\"docs/latex/overleaf/gfx/\"\n",
    "target_metric = \"overall_NDCG\"\n",
    "for level,ax in zip([\"Chapter\",\"Subchapter\",\"3 digits\",\"Billing\"],axes.reshape(-1)):\n",
    "    fig,ax = bar_plot_results((skl_results_long\n",
    "                      .query('target_metric==\"'+target_metric+'\"')\n",
    "                      #.query('model_id==0')\n",
    "                       .query('input_data==\"Drugs, Sex, Age\" and code_subset==\"All codes\"')\n",
    "                       .query('code_level==\"'+level+'\"')),\n",
    "                        \"model\",\n",
    "                         \"metric_name\",\n",
    "                    values_subset = [\"Vanilla\",\"Per level\",\"multitask\",\"HierL2\",\"TreeMin\",\"MCLoss\",\n",
    "                    \"focal\",\"outL2reg\",\"imbal\",\"imbalNorm\",\"flatRollDown\",\"flatRollUp\"],\n",
    "                      groups_subset=[\"RE@25\",'RE@50',\"RE@75\",\"RE@90\",\"RE@100\"],\n",
    "                      reference_row=\"Vanilla\",\n",
    "                     reference_val='relative',\n",
    "                        legend_kwargs={'loc':\"best\",'ncol':2},\n",
    "                      abs_value_in_percent=False,\n",
    "                      improvement_direction=\"min\",\n",
    "                      average=True,\n",
    "                    axis = ax,\n",
    "                    values_legend_dict=methods_pretty_names\n",
    "                    )\n",
    "    ax.text(.5,.9,level,\n",
    "        horizontalalignment='center',\n",
    "        transform=ax.transAxes)\n",
    "    ax.set_ylabel(\"Relative improvement (%)\")\n",
    "    # Remove legend\n",
    "    ax.get_legend().remove()\n",
    "# Get legend labels from the last axis\n",
    "handles, labels = ax.get_legend_handles_labels()\n",
    "# Add a global legend for the figure\n",
    "large_height_fig_size = (dual_col_fig_size[0],dual_col_fig_size[1]*1.4)\n",
    "fig.set_size_inches(large_height_fig_size)\n",
    "fig.legend(handles, labels,\n",
    "           ncol=5, loc='lower center',\n",
    "           borderaxespad=-.6,\n",
    "           frameon=False)\n",
    "#fig.suptitle(target_metric)\n",
    "\n",
    "fig.tight_layout()\n",
    "if mpl.get_backend() == 'pdf':\n",
    "    # The inline backend distorts the scaling and outputs figures\n",
    "    # with wrong dimensions\n",
    "    fig.savefig(plots_dir/\"all_methods_RE_perlevel.pdf\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64cea2c5",
   "metadata": {},
   "source": [
    "#### Paired T tests for RE reduction with multitask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "98bd716d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "metric_name  code_level\n",
       "RE@10        3 digits      0.656410\n",
       "             Billing       0.217445\n",
       "             Chapter       0.234473\n",
       "             Subchapter    0.701420\n",
       "RE@100       3 digits      0.157139\n",
       "             Billing       0.009495\n",
       "             Chapter       0.327324\n",
       "             Subchapter    0.347741\n",
       "RE@25        3 digits      0.208445\n",
       "             Billing       0.158638\n",
       "             Chapter       0.095687\n",
       "             Subchapter    0.398124\n",
       "RE@33        3 digits      0.207102\n",
       "             Billing       0.194130\n",
       "             Chapter       0.591996\n",
       "             Subchapter    0.074378\n",
       "RE@50        3 digits      0.663938\n",
       "             Billing       0.085027\n",
       "             Chapter       0.991810\n",
       "             Subchapter    0.290222\n",
       "RE@66        3 digits      0.128975\n",
       "             Billing       0.043938\n",
       "             Chapter       0.837657\n",
       "             Subchapter    0.101531\n",
       "RE@75        3 digits      0.123275\n",
       "             Billing       0.035212\n",
       "             Chapter       0.983653\n",
       "             Subchapter    0.004126\n",
       "RE@90        3 digits      0.147565\n",
       "             Billing       0.000280\n",
       "             Chapter       0.785006\n",
       "             Subchapter    0.247203\n",
       "dtype: float64"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check whether differences in RE are significant between naive multitask and per level multitask\n",
    "from scipy.stats import ttest_rel # paired t test\n",
    "tmp = (skl_results_long\n",
    " .query('target_metric==\"overall_NDCG\"')\n",
    " .query('code_subset==\"All codes\"')\n",
    " .query('model.isin([\"Vanilla\",\"Per level\"])')\n",
    " .query('metric_name.str.startswith(\"RE@\")')\n",
    " .query('code_level!=\"All\"')\n",
    " ).drop(columns=[\"target_metric\",'input_data','model_id','code_subset'])\n",
    "\n",
    "tmp = tmp.pivot(index=['metric_name','code_level','crossval_id'],\n",
    "          columns='model',\n",
    "          values='metric_value')\n",
    "pvals = tmp.groupby(level=['metric_name','code_level']).apply(lambda x: ttest_rel(a=x['Vanilla'],b=x['Per level'],\n",
    "                                                                                        alternative='two-sided').pvalue)\n",
    "pvals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "10090a30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "metric_name  code_level\n",
       "RE@100       Billing       0.009495\n",
       "RE@66        Billing       0.043938\n",
       "RE@75        Billing       0.035212\n",
       "             Subchapter    0.004126\n",
       "RE@90        Billing       0.000280\n",
       "dtype: float64"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pvals[pvals<.05]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "39d48bb0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "metric_name  code_level\n",
       "RE@10        3 digits      -0.012823\n",
       "             Billing       -0.306387\n",
       "             Chapter       -0.002306\n",
       "             Subchapter     0.004392\n",
       "RE@100       3 digits      -2.492116\n",
       "             Billing      -27.278219\n",
       "             Chapter       -0.011745\n",
       "             Subchapter    -0.184514\n",
       "RE@25        3 digits      -0.043045\n",
       "             Billing       -0.609257\n",
       "             Chapter       -0.002492\n",
       "             Subchapter    -0.009711\n",
       "RE@33        3 digits      -0.048803\n",
       "             Billing       -0.763115\n",
       "             Chapter       -0.001971\n",
       "             Subchapter    -0.018629\n",
       "RE@50        3 digits      -0.037066\n",
       "             Billing       -1.463397\n",
       "             Chapter       -0.000091\n",
       "             Subchapter    -0.029548\n",
       "RE@66        3 digits      -0.281706\n",
       "             Billing       -3.461922\n",
       "             Chapter        0.003230\n",
       "             Subchapter    -0.087680\n",
       "RE@75        3 digits      -0.541335\n",
       "             Billing       -5.080210\n",
       "             Chapter        0.000331\n",
       "             Subchapter    -0.091378\n",
       "RE@90        3 digits      -1.161443\n",
       "             Billing      -11.513733\n",
       "             Chapter       -0.004664\n",
       "             Subchapter    -0.111315\n",
       "dtype: float64"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp.groupby(level=['metric_name','code_level']).apply(lambda x: (x['Vanilla'] - x['Per level']).mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab9c6c02-3cb2-416d-be50-8c2b4bfaad58",
   "metadata": {},
   "source": [
    "## Averaged metrics correlations"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17d95a20",
   "metadata": {},
   "source": [
    "### Fig S4: Spearman rank correlation between different metrics values and rank errors for different recall values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "2be257e6-c412-4019-8393-5b6fe31c5c31",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric_name</th>\n",
       "      <th>metric_value</th>\n",
       "      <th>code_level</th>\n",
       "      <th>model</th>\n",
       "      <th>target_metric</th>\n",
       "      <th>input_data</th>\n",
       "      <th>code_subset</th>\n",
       "      <th>model_id</th>\n",
       "      <th>crossval_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>micro_F1</td>\n",
       "      <td>0.455192</td>\n",
       "      <td>All</td>\n",
       "      <td>Vanilla</td>\n",
       "      <td>overall_macro_AUROC</td>\n",
       "      <td>Drugs, Sex, Age</td>\n",
       "      <td>All codes</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>macro_F1</td>\n",
       "      <td>0.026901</td>\n",
       "      <td>All</td>\n",
       "      <td>multitask</td>\n",
       "      <td>overall_NDCG</td>\n",
       "      <td>Drugs, Sex, Age</td>\n",
       "      <td>All codes</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>samples_F1</td>\n",
       "      <td>0.407974</td>\n",
       "      <td>All</td>\n",
       "      <td>multitask</td>\n",
       "      <td>overall_NDCG</td>\n",
       "      <td>Drugs, Sex, Age</td>\n",
       "      <td>All codes</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>micro_AUROC</td>\n",
       "      <td>0.979523</td>\n",
       "      <td>All</td>\n",
       "      <td>multitask</td>\n",
       "      <td>overall_NDCG</td>\n",
       "      <td>Drugs, Sex, Age</td>\n",
       "      <td>All codes</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>macro_AUROC</td>\n",
       "      <td>0.235360</td>\n",
       "      <td>All</td>\n",
       "      <td>multitask</td>\n",
       "      <td>overall_NDCG</td>\n",
       "      <td>Drugs, Sex, Age</td>\n",
       "      <td>All codes</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   metric_name  metric_value code_level      model        target_metric  \\\n",
       "0     micro_F1      0.455192        All    Vanilla  overall_macro_AUROC   \n",
       "1     macro_F1      0.026901        All  multitask         overall_NDCG   \n",
       "2   samples_F1      0.407974        All  multitask         overall_NDCG   \n",
       "3  micro_AUROC      0.979523        All  multitask         overall_NDCG   \n",
       "4  macro_AUROC      0.235360        All  multitask         overall_NDCG   \n",
       "\n",
       "        input_data code_subset  model_id  crossval_id  \n",
       "0  Drugs, Sex, Age   All codes         0            0  \n",
       "1  Drugs, Sex, Age   All codes         0            2  \n",
       "2  Drugs, Sex, Age   All codes         0            2  \n",
       "3  Drugs, Sex, Age   All codes         0            2  \n",
       "4  Drugs, Sex, Age   All codes         0            2  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skl_results_long.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "97cac96b-a22b-4563-8d47-fc5aeed055a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of models used to compute metrics correlations:50\n"
     ]
    }
   ],
   "source": [
    "wide_skl_results = (skl_results_long\n",
    "                    .query('code_subset==\"All codes\"')\n",
    "                    #.query('model==\"Vanilla\"')\n",
    "                    .query('code_level==\"All\"')\n",
    "                    .pivot(index=('model','code_subset','input_data','target_metric','code_level','crossval_id'), columns='metric_name', values='metric_value'))\n",
    "rank_errors = ['RE@10', 'RE@25', 'RE@33', 'RE@50', 'RE@66','RE@75', 'RE@90', 'RE@100']\n",
    "other_metrics = list(set(wide_skl_results.columns)-set(rank_errors))\n",
    "\n",
    "re_df = wide_skl_results.loc[:,rank_errors]\n",
    "other_metrics_df = wide_skl_results.loc[:,other_metrics]\n",
    "\n",
    "# Compute the matrix of Spearman correlation coefficients between metrics and RE@P\n",
    "def compute_RE_correlations(x):\n",
    "    return re_df.apply(lambda y: scipy.stats.spearmanr(x, y,nan_policy='omit').correlation,axis=0)\n",
    "other_metrics_corr = other_metrics_df.apply(compute_RE_correlations,axis=0).transpose()\n",
    "\n",
    "print(\"Number of models used to compute metrics correlations:\" + str(len(wide_skl_results)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "888b6db2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sort the rows of the metrics correlation according to the average correlation value\n",
    "mean_row_corr = other_metrics_corr.mean(axis=1)\n",
    "other_metrics_corr = other_metrics_corr.loc[mean_row_corr.sort_values(ascending=True).index,:]\n",
    "# compute approximate spearman correlation coefficient: https://stats.stackexchange.com/questions/18887/how-to-calculate-a-confidence-interval-for-spearmans-rank-correlation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "385c7fcc-63af-481a-a049-da65e9c2f822",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtAAAARaCAYAAABlrEUYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAEAAElEQVR4nOzdd3wUdfrA8c+T3U3vnZAGoXeQpmBXUKxYzt5/9nKn53mnZ/es59nOXs9yZ++KiopKObFQlN5CQkjvPdlk9/v7YzYhkARYj+yG8LxfL16Znfnu7PdhdmefeeY7s2KMQSmllFJKKbV7AvzdAaWUUkoppfYmmkArpZRSSinlBU2glVJKKaWU8oIm0EoppZRSSnlBE2illFJKKaW8oAm0UkoppZRSXtAEWimllFJKKS9oAq2UUkrt40TEJiLn+LsfSu0tNIFWSiml1B3ASyLysYhM9XdnlOrtNIFWSiml9mEiEgeca4yxA88BT4vIED93S6leTRPovdC+dKpNY+179pU4QWPtq/pgrC8Df/RMLweygXAAEXH4qU9K9WqaQO+d9qVTbRpr37OvxAkaa1/VZ2IVkVHALKDcM+sYoB4Qz+PrRKRYRE7yR/+U6q00gd7L7Eun2jTWvhfrvhInaKwaa+8nIgK8D1wF/E1E1gAjgFeNMUtEpD9wIvAIcK6IfC4iw/3VX6V6E02g9z770qk2jZU+F+u+EidorBpr73cRsMIY8yRwKtAA3GGMmetZ/kdgizHmXmPMicCHwDEi0havdLFOpfYJmkDvRfalU20aa9+LdV+JEzRWNNa9JdYM4P880/VApWceIjIROBMYJyKXAhhjngIeMsbUeR4bn/dYqV5C9P2/d/Ac6a8HHgbOBmKAr4CPjTFzPafa3gE+AqYAwcC1xpg1furyb6ax9r1Y95U4QWNFY90rYxWRVGAJMNQYUyUibwOLgBewqu6RWENXCkQkGTgF68BhAPCVMWaOn7qulF9oAr2XEJH/A2YZY07y7Kg/AmYaY8o8yx8C+htjTvM8vhwIA542xtSJiOwt1QKNte/Fuq/ECRorGuteEeuO/RGRdOBK4B/AYcBlxphDOiwfDmwCDgKuAJYBeYDT83gDcJ0xptJXMSjlT3Z/d0Dttu5OtZV1ONVWLSKXGmOeMcY8JSIBxhg37HWn2jTWvhfrvhInaKwa614Q6479McZsAf4sIiHA5UDgDsvXiMg44BzgEWPM/A6L/yMitwBjgW97st9K9RZagd4L7Uun2jTWvhfrvhInaKxorHtNrB0r0iISBdwGHAi8YIx52jOE5WqsqvM3WNXqw4DXjTHPi8ixwCnGmPP9EoBSPqYXEe4FPDuujgKAfwGBInI6kGCMecQYU2uMOQlrJ1cmIkcAT2KN1asFfgZuEpGXRCTGdxHsPo2178W6r8QJGisa614ba4fkOcAYU22MuQ6r4j5JRA4GYoGZWENX7sIaunEx1l05LgY+BaJFJNKznski8pSIzPRDOEr1OK1A78U8p9o+BwKNMfvvsGwccC1W9WD+DstuARYYY771UVf/Zxpr+7Jx9JFY95U4QWPtsGwcGuteEWvH4SciEgCcAEwDngZeMsYc6Fm2AHgRKAaONMZcKyI2YAgwCetA43BjzDe+j0KpnqMV6L1QWzXEGNMIHA98LyI/ichlHZYfBHwA/CoifxWRr8W6CAasiz/O93nHf4M9FOsFO66vN9pXtuu+EidorP9LrJ4krFfa0/ul3sgY4+4QpxvrntfpgANYASAiZ2INZfncs2yRiNgBt7HuPlINbG5Lnj2JuFJ9gr6Z90L70qm2PRRrlIhEeFZ5qoh85jm12qvswe0a4VnPUSLylYgc6odwurUH44zxrOdCsX4h7RCfB7MLezDW6LZ1ikiaiIzxbSS7tqffv1if1aWe5/Yqe3C/1LYPPkBELhGRI/0QTrc6xClYFeZawGBdLPkxcCMwF+u2foHAJmNMa4cLFJ8EruuwPreIxIrIQT4MQ6keoXfh2It1PL1mjPkFuKjDqbY1WBesjDPGXO1pFw+0AkdjVQVqPFWeWuB74DMR6ZWn2vZArLVi/UrYGuB14D4R+Qi4B3B12OH73Z6I1bOezz1J5v0i8i1wM9DSW2L9H+PcYoypFOtnlU8HVmH9FHEpcJExpsL3EXXvf4h1FlasVZ5YbwOigeGeWC8wxhT7PKCd2IPv3zdExA3cLNYPefzRGFPo+4i69z9u17Z98OFYwxweB64QkQ+BvwGtveizaoAGEZkD3Apcj5VIXw8cAkQBNxhjqkWsixFF5D5ggzHmw7b1iMgVWPfIHurZtmcbY7J9HI5Se4RWoPuAfelU2/8SqzGmBVhpjHkFq2IyGes6gF7xJbWj/yHWIM8ymzHmdeBkrC9yW2+M9TfGucDz9D9iVb2uNcZMx7oTQphPA/DCb4g1jW2xPoa1Dc81xkwCFgOHez7Hvc7/8llti8kY8xbWPYaPAm5ve2/3Nr9xu37vefpVWL/udz9wHDDIWk2v/Ky+i5VAPwVcCMzHuqXdpZ7kOdSTPMdgbbdL257rqbwfDrxsjJkKLAVG+TwIpfaQXrnjVd7bjVNtmcCfsRLHZjyn2jqs4kmsHV7b+twiEguMMjtcAONvvyHWzR1iFU/bw4BCrKvoIz1ty40xr/kwlF36jdu1WURCgX+LyFygP9AExItILXAZkG+MedXX8XTnN8S50dM2GDhARC40xrxojHmwbZ2eiu3Ivfz968I6M3QEVoX2cRE5yRjznjHmThGJNsa0ehLOaOP5UY/e4jds1+yO+yURGQs8gXXw8Khn3rVAaR/4rK4XkUCsW97927OaTKAAazhEa2/cBxtjNgIniEgKUGGMaeqw+CrPPmYG8IwxZh2AiIRh3ZnkOuA7T9sKrGr0R542CcaYUh+FodT/TO/C0UeJyMlYlceOp9pcdH+qbarZ/len2k+1Ab36VNvuxOppJ8BUrOT5ROBvxpgPPcnJlcAIYDNwozFmma/j2B27ihWo7XBaeQLWPWk/wbqY6UfP9p6K5xQxcJXnC7FX2Y04azokLCOwfl7578aYrzzzrsDa1kPYu9+/f/Kc5l8L/IhV+bsPeN8Y84jn+QOxxqLGAinA740xP/o6jt3h5Xb9P+BgrGT5Os+8mVg/8jGcvf+zWuP5PN4B7I/1WT0TyDPGXOYZtjKNveA93EasX2h8GjgGOK/tIF2su46kG2Mu9jy2A7nAccaYpSJyJ1Y1ehzWz59/2NX6lepNNIHuw0RkENbPsv4IrAR+MsYUeJaFGmMaPKfacoFJHaoFF2OdMn3CGDNPRB4H5hpjPvJLILuhi1h/Nsbke5ZNwdo5zwLisE6hfmWM+dlTRbkKq/L1vIich/XrYo+bXjaOts0uYs0wxuR2aPsY8Lnx/HCDZyhLq2f6bmCFMeYNX8ewO3bx/k00xpR0aPsisNgY86wn8ZjB3v3+7RjrdKyDvUM8j2cBhxpj/iQix2N9VvOMMfeKyAVYY8KvMsZs8EMou7SzWD3Lw4HpwPNYQwDmG+sahoFYwwZy9uLPasftGtvWb8/wjhFYB/eXAoOB84BH96L3cMcfYjkS60DncWAh1rUm/wU+Nsa0iMjfscbyX4N15vNgrIssBwK/x/oZ8Spfx6CUN3QIRx/mxam2Z3dxqq0Ma7xwr915dxer5wDhU6yhG6caY+a1PcdTkU7DqkZ/DzxvjHnZ13331k5iDQeeEZHFxpjbRWQYUIVVmWwzS0QysCrSwVgVrl6ZQO8kTgfwqoh83yHOIqBJrPvyXkkfef96rMa6SLJNBpDhef+ejXXKv9SznpdE5OPeNpSjo13EClbiXAvcZYz5tMP8BOAk+sBn1eN2ESk3xtwB5GAd5H+NdfHhq8Cf2Lvew8bznsQY8yXwpYgEGmOcItIKhHuS5zFYPwc+GatSnYo1jrrY8/2zH9DopzCU2m298kIxtWcZYwq6+JL6N1ZF9gTglw7zr8M61f+VMcYl1l06/g/4GHr3fZShc6zGmEpgIvAacJvntGrbMmOM+QHrdlMhYt3Kb685qOwi1jqs6tUgEfkRq/IVCvzU4WnzsS5Umov1JfUM9O7t2kWcLWwf50NYxYBPgUuwKvJ94v3rUQ+Eisj3njMKpwAPYI2JTsfajvuLyBdijYnutclzRzvGKpYwrIOhe4Bn2+Z72veZz6rHA8BoEVkM3IH1K4V3YB0ALjPGfLm3vYc9+1TToY9t49k3Yf0C47NYd5K53xizBZiAdfavrbBxOPA24OrNcSoFOoRjn+Tlqbb7gVRjzFkdn7e3kO1/TesIrOTjYazxk60dlvUDHgT+YPbSC1l2iHUC1h0bfvJUbIM8CTZi3Wf3TuDPxhin/3r82+wQ53jP7OVYBYEHsN6/H/WF929HYt07tz/wX2NMrljjvMH6VbhGEXkb624O33e/lr1H2/YSkWDA2cc+qx33wQOASmPdqjAE6/qEPvceFutahRnAJ8aYjZ5Y3wVu8+ynRmANW8nGOiu6V8ap9h17zRG82nO8PNV2HtZ4RPbGHZrx3F7KUxj5Cmi7yOwkYJqI/MVT0YzGqn40+6+3/5sdYl3aYdHZwAQRudZY459TsarQt2D9wMNeZYc4O15A5hIRJxDWV96/sC3ZMp3vxBCCdRDYKCKpWBf8lvu+h3tOxwSxw/Y6AesHSm7sQ59V02G7bu4wv7GPvocDjDGrsYYjtRkDRBlj2s6QnYBVsV6wt8ap9i2aQO+jOlQ/2r6wdjzVNh3rgrsHPdWC9qrf3qaLWMG6M8V04AcRWYQ1/vAZY93xYK+PdQf/BkZjxboAGA+8YYyp21tj3ckXbDZ99P3bhc1Yw5IMcCzW+Nle9UMj3uom1vex7gi0L3xWoW++h7vqdw7WQW/bEKwM4AFPoq1Ur6dDOFQnHU61fWyM2eTv/uxpO5w+HQIciVX1+NW/Pdvzuoh1JrDQ9NJbf+0Jff3925GIjATOxars/dtsf2/3vd6+9FntaF95D4t1n+u/YiXTb5le9quaSu2MJtBqO3tzlcNbGmvfs6/ECV3HujePkd2ZfX279kX70vtX9U2aQCul1F5KEw6llPIPTaCVUkoppZTygt4HWimllFJKKS9oAq2UUkoppZQXNIFWSimllFI9QkReFJESEVnZzXIRkcdEZKOI/Or5IbC2ZUeJyDrPsr/4rte7pgm06paIXOLvPviKxtr37CtxgsbaV+0rse4rce7D/gUctZPlRwODPf8uAZ4CEOtn7J/wLB8BnOG5xWOvoAm02pl9aaemsfY9+0qcoLH2VftKrPtKnPskz6+oVuykyQnAK55f5lwMRItIP2AysNEYk22McQJveNr2CppAK6WUUkopf+kP5HV4vNUzr7v5vYL+lPfew+f3G/Tc4nCfuM+hxtr37CtxgsbaV+0rsfoxTvH1Cw6TJFOP09cv26O2UrUKaOow61ljzLNerKKr7WB2Mr9X0ARaKaWUUsoH6nFynRzm727sUdeZ95qMMRP/h1VsBdI6PE4FCoDAbub3CppA72Xu+W6Wv7vQo246eE779B/en+LHnvSsR2b/0D594hNZfuxJz/vgyk3t09NvTfdjT3rewju3tE8fcW+GH3vSs766Mbd9+qgHM/3XER/4/Pqc9uljHhngv474wKd/2Nw+valmvh970vOyIg/ydxfUNh8BV4nIG8AUoNoYUygipcBgERkA5AOnA2f6sZ/b0QRaKaWUUkr1CBF5HTgEiBeRrcBtgAPAGPM0MAeYBWwEGoALPMtaReQq4AvABrxojFnl8wC6oQm0UkoppZQPCEKA9LH7N+xiVLIx5oxdLDfAld0sm4OVYPc6fWwrKqWUUkop1bM0gVZKKaWUUsoLmkArpZRSSinlBR0DrZRSSinlIwHi89tPqx6gFWillFJKKaW8oAm0UkoppZRSXtAEWimllFJKKS/oGGillFJKKR/pc/eB3kfpVlRKKaWUUsoLmkArpZRSSinlBU2glVJKKaWU8oIm0EoppZRSSnlBLyJUSimllPIBEcGmFxH2CboVlVJKKaWU8oIm0EoppZRSSnlBE2illFJKKaW8oGOglVJKKaV8JEDE311Qe4BWoJVSSimllPKCJtBKKaWUUkp5QYdw7INKcut4/4EV5K+vJiY5lGOvGc7gSQldtv37ad9SVdzY/jg5K4KrX5jO0s+28u79Kzq1v/71g4npF9pjffdWxdZGvnpiMyWb6olIDOKQ/8sgY1xUl22b61v59vlcsn+qwrgNCQPDOPVvw3e5rLeoLXKy7N+lVOc1ExLrYMypcSQO77wttiyuZdlrpZ3mH3lHGqFxDnK/r2H9F1U0VbuIywpm/NkJhET3nl1FQ2kLGz+poK7QSVCUnYFHxRCTFdxl24Ifain4oRZnnYvACBspUyJImRyxy2W9RX1JC2veK6cm30lIjJ0hx8UQNziky7ZbFtWQt6iW5loXQZE20qdFknaAFc+iB/JpqnK1tw3v52DK1f18EsPuqituYdXb5VRvbSYk1s7wE2KJH9p1rDkLashdUENzjYvgSBsZB0WSMT0SgMWPFVJX0oJxQ3iSg6HHxRA7sOv3h7/UFTv59Y1yavKsWEecFEfCsK5j3fxdNTnfWbEGRdkZcHAkmQdZsa56r5zCZfU469wkjAhh0sVJvgxjlzauy+HGqx5g5fL1pKYnc+sDv+fAwyd1apezaSt//f3fWbcym5aWFsZNHMGdD/+RjIH929s0NzVz7LSL2Lwxj7Mvns3tD/7Bh5Eo1ZlWoLshIkZE/tHh8fUicrtn+nYRyReR5SKyQUTeE5ERHdo6ROQ+z7KVIvKjiBztWRYuIk+JyCYRWSYiS0TkYl/G9uadyyndUs+sK4YTYBNev305TXUt3bbPHBvDabeM5bRbxnLUpUOteeNi2+edetMYbA4hPCaQyITe9UX12UObqMhv5MAL0rHZhDkPbqS5vrXLtl8+vpl1C8oZeXgCB12QTnRy0G4t6y1+fqmEuuIWRs6OI8AGP71QTEuju1O7uEHB7Hd+Ivudn8iEcxMIsENQhI3gaDuVuc0s/08ZIVF2RpwQS/nGRn59o8wP0XRv/fvlNJa1MuDIaAJssO7dMlqbOsfZWN7C5rlVIDDgyGiMCzZ/UUVzdetOl/UmK98so760lcGzYpAAYcV/uo61oayFDZ9WgcDgWTEYF6z/pJKmqm3xRGcGMfK0OEaeFsegmdG+C2I3/frvUupKWhh2XCwBAcIvr5V2+f6tL21h3UeViAjDjovF7Ya1H1bS6Ik1OjOI4SfEknVEFLUFTla9Xe7rUHZp2Sul1Je0MOzEWMQmLHu5pNtY13xQCSIMOzEW4zKsfr+Cxspt2zVlfJgvu+6Vay+6i+z1W7jpniuxO+xcff5t1FbXdWpXVFCK22245qYLOPmso1n07RJuuuaB7dr88/6XKSrofOC/txEgQAL61L991b4b+a41AyeJSHw3yx82xowzxgwG3gTmiUhbGfcuoB8wyhgzCjgOaCttPQ9UAoONMeOBo4DYngpiRwUbqinaVMuYw/sxdXYG036XSXN9Kyu/K+r2OTHJoQzdP5Exh6cweLIVYmy/UMYcnsKYw1OwBwbgajHsNysVm733vKVKsuspy2lg6PQ4xh6dxPjjk3E2uNjwfWWnttVFTWz6oZIh0+OYdnYqIw5L4MirBu5yWW9RlddMTb6T/vuFM/DgKLIOi6K1yVCwrPOXVVi8g9SJ4aRODMfmENytkL5/BAE2oXxjIxjImB5B1iFRRKUGUbSqAWedq4tX9b26Qif1xS3Ejwql36QIUqZG4Go2lK9p6NTWGOtvUISNqAHBOMIDEBsE2GWny3qL2gIndYUtJI0JJW3/CNKnW7GWrNxJrJF2YgcFExhhI8C+fTwhsXbih4WQPDaMuCFdVzv9pSbfSW1hC/3GhZI+LYKMgyJpbTIUr6jv3Lgt1igbcYODCfLEavPEOvS4GBJGhBA7KJgAm9Dbrteq3tpMbUEL/caHkTk9kgGHWLEW/dI51rbtGhxlI35IyLbt6rCCGnlSHJmHRPqy+7tt1S/rWbNiI8eecjjnXDybC688lbqaej778LtObSdMGcXrcx7j3EtO4tYHfk90TCQb1uS0L1+7chMvPfk21/zlfN8FoNQu9J7zsr1PK/AscC3w1501NMa8KSLHAGeKyHPAxcAAY0yzZ3kx8JaIZAGTgTONMW7PslLg/p4LA5xNLlqarASoONtKqCLjrUpxlKdiXFHY2PWTgeVz81n2RT5h0YHMuHgIE49J2275jx/nIQEw6di0btbgOy3NLlqbrUpOea4VU1hsIADhcdbfmuKmTs8r32q1Ld5Yz5NnLkECYNwxyUw/N22ny/yp1enG5bS+YWsKnACERNs8f62Pdn3ZziuqOQtrQSBzmnV8FxRhPb9iUxPRaUHUl7aAgYaKVgLDbT0Sx664Wty4W6w4G0qsMyWBnn4GRlhxNlV2jjM03kHGYVHkzqtm2VNFVnX2uFgcYTYcYbZul/mTy+nG5Ym1rsjapsFRVp+CPH8bKzrHGpbgIGtmNJvmVrH44UIQGHFy7HbbrHBZPYVL63GEBTBoZjQpE8N7Opyd6hhrbWFbrHbPX0+s5V3Emuhg8KxoNnxWxcK/F4DAqN/Ftcfa2mT45vatANhDAhh5alyPx7Irrg6f1dpC6z3cFmNbzA1dxBqe6GDosTGs+7SS+ffmg8CYM+IJ8tNncVcaG5pobLT2r+tXZwOQ1M+qQSWnJAKQl1PQ6XmBgY726RVL11JVWcPM4w8GwO12c9PVD3D2/81m9IRhPdp/pbyhCfTOPQH8KiIP7LIlLAWGAYOALcaYmi7ajAR+aUuefWXB69nMe3kjAEGhO+x4PRWO7oo0E49NJSEtjFanmy+eXceH/1jFwAlxxHrGOZfn15O9tJzBUxJ6xdjnJe8X8sNb1g46MGSHanhbOaeLklTbF3lrk4uj/5jFL5+VsOSDQtLHRu50WfrYrsdT+8LGL6tY91kVAPbgHWLqPtR29aUtlK5vJGlECKFx1hdYyvhwchbWkLOwlpyFte3rbat4+UP+f2vJm299nGyBO/aj+0Bb6l0U/lRHWLKDtIOiyJtfTfbnlUQNCCLAJt0uC4r0324xd34Nm+d5Yg3qept2xVnnYuv3tUT0czDgsCg2z6tm3ceVxGQFExxlJ2VSOKHxDtythk1fVLH2gwpiBgYTEuu/WDd/W8OmL6uBLmJt08VsZ52LLYtqiUgJZNARUWz8qpo1H1QQNyiY4Gg7tkBhv4sTqS9pYf2nVWz8oopJlyX3YCS7tmleNRu/sGK1exFrc52L3AU1RKYEMnhmNBu+qGL1u+XEDQ7uVdcltHn20df5533/AiAsYvvvA+PZ/8pOdkrZG7Zw2Zl/JTU9mdv+/nsA3nntM7ZuKeKef97AOk9SXltTR3lZFXHx0Xs+CKV2U+/7BPYixpgaEXkFuAbovkRr8TrDEJG/AqcCicaYlC6WXwJcAvDMM89wySWXePsSAIyf2Z+M0TEAOIJsPHv1YqpLrSpBdZn1N6afdUq31WlVqu2BVqJ96DmD2tdTsLGGRW/lUJ5X355A//RxHsbAlOPTf1Pf9rThh8STMtyqptoDA3j7r2uoK7eqW3UVVuUnMtEav9zaYh3H2B0B7fNSRkQwaGosjdWtbF1RQ3VRM4mDwrpdxlifhredtCkRxHounrM5Alj4cAGNngvFGj1jeUPjrI9420GArUMinLOoBgxkTt92CtjmEKb/IYWaAidiE1a+U055dhNhcf7bVSSOCSMyzdo+AQ5hxb9KcNZY8TlrrXiDPZV3d6sVZ4BdqM5pxlnrInm/MOKGhtBQ4mTLtzXUbnWCodtlQSP8F2u/CeFEZ1rbNMAhLHmmmKZqK8bmGutvW9LbMdbK7Caaa1z0nxxOwohQ6opayP6qmuotToJH2xlw6LYDvbpCJ1sW1tJQ1uLXBDplvzBiMrdt1x+fLKbJ875ti7mrWCs2NdFc7SJtajiJo0KpLXKy8QsnVbnNJEfbCbAJ8UNCiB8SQvGKBio2NeOsdxHox7MLqRPDiR2wbbsu/mdRe4xtMYd6YnV5YrXZhYoNTTRVu0g/IIKk0aHUFjpZ/1kVVTnNhIzrfV/fs0+fycSpowEIDgnitJlXtY9bLi60/qZmWBevNjdb++WgIOvM4Ia1OZxz3LUEBjl49eNHSEy2zhwU5pdQUVbFsdMubH+dD9/8ksDAQO59/AbfBLZHyT49brgv6X2fwN7nEazq8ku7aDce+BnYCKSLSIQxpnaHNquBsSISYIxxG2PuBu4Wkc4DVQFjzLNYw0hgp/WnnYtNCSU2ZVs1IDkrghXzCknKDOeHD7cQFGpj5EFWhebhcxbQUO3kts9nUJRdy9zn1jFkSgJul2HZF/k4ggJIGmglqK0tbpZ+nk90UjBDpnZ9Fw9fi0oOJip524WM8RkhrF9UTlx6CCs+LyEwJIDB+1sHE69c9StNta1c8Z+JJA4MJS4jhLxfa1j5ZQmr55UiAdBvWDhx6SHdLvOnsHgHYfHbTn1G9g8kf0kdEckOchbWYA8WUsZZyf/Xd+XhrHNx7EMDACsh2bK4lpAYO0kjt703jNuw8r1yolKDqMxtpnRdI1mHRWEL9N8OPzjGTnDMtl1VaKKDslWNhCbUUrSkDlugEOe528jSJwtpaXCz/19SCfI8p3RFA4HhNkpXWGOHQ2LtuD3ngLpa5k8hsfbt+hCe7KD41wbCEh3k/1CHLUhI8Gyv7x8qoKXBzSG3p7U/p2h5PUGRtvbxtKHxduqKnGyaW0XckBCMGwqX1hPgEMKTHZ074EOhcY72Mx9g3RmkcHkD4UmB5H1fiy1ISBptvX8XPJBPS72bI+5Ob4+1cKkVa+EyT6wJDsrWNVL0Sz3RmUE0VbmoymkmMDwAR6h/E5bQeAehHT6rESkOCpfVE57sYMuiWuxBQvJYa7vOv2crzno3M+/PIMRz4Jq/pI6gSBv5S6xYwxKsdZWsaqC2yCoMNFW1kre4ltis4PblvpY+IIX0AdtqQcNGZfHpu/MYPHwA/3nhQ8IiQjnq+IMAmDHxHCrLq/m14HMKtpZw1jG/p7qylmtvvohffl7NLz+v5thTDmfW7EMZMtzab21Ym8Nj977EQUdM5qyLTvBLjEq10cOgXTDGVABvARd110ZETgZmAK8bYxqAF4DHRCTQs7yfiJxtjNmIlWT/TURsnmXB/Ibq9f/idzePJT4tjDlPrsHV6ub028YTEtF5hxsa5cC44esXNzD32XVEJ4Vw1l0T2sdPr55fRH2Vk4nHpBEQ0Muu1PE46tosYlKCWfDSFlytbo7+4yCCwjonSSLC0ddmEZUcxLfP59JU18qMawYSnxG602W9yX7nJxKe6GDV++W4W2HihUk4dhyy41Gw3Lr1Vca0CGSHbVe2oYlf3igjf0kdAw6KZPhxPrvGdbcMnR1HSJydzV9W4XbB0JPjsAd33pVFpASSeWQ0bpdh02eVuF2GgUdFE5YcuNNlvcnI0+IIjbezYY7Vx9FnxOPYcWgSEJkaxOBZVjzrPqrA3WoYclwMEf0CcYTaMG7I/qqajV9UERxjZ8xZ8X4dqtKVsWfGE5ZgZ+3HFbhdhrFnJ3QZa1RaEEOPjcHdaljzfgXuVhh+YiyRKYE4QgOo3uJkzfuV5C6oIXpAEBMuTNzpsAF/GHd2AmEJDtZ+YMU6/rwEHCGdP6vR6UEMOyEGdyuserccd6thxMmxRPa33qfZ31Sz7hProujaghZWvFlO5ebO13j4y0PP38KAwWncc9MTtDhbeOxftxMZ3flWkVs251NRVoXL5eLBO57lDxfdyR8uuhOAwcMyOfrEQzj6xEOYPM065Zc+oD+jxg/1aSxK7UjaxiWp7YlInTEm3DOdBGwGHjDG3O65nd3FQCkQBqwE/mqMWe1pHwj8DTgJaALqgVuNMV+ISCTwd6yEuwJraMgbxpjHd9ElA3DPd7P2aJy9zU0Hz2mf/sP7U/zYk571yOwf2qdPfCLLjz3peR9cual9evqtvWOoT09ZeOeW9ukj7s3wY0961lc35rZPH/Vgpv864gOfX5/TPn3MIwP81xEf+PQPm9unN9XM92NPel5W5EHg4+IVQGZAvLk16Bhfv2yPuqjplSXGmIn+7oev9a4SRC/Sljx7pouB0A6Pbwdu38lzncANnn87LqsBLt2DXVVKKaXUXsC6D3TvOiOifhsdwqGUUkoppZQXNIFWSimllFLKC5pAK6WUUkop5QUdA62UUkop5QuC3ge6j9CtqJRSSimllBc0gVZKKaWUUsoLmkArpZRSSinlBR0DrZRSSinlA4Jg0zHQfYJuRaWUUkoppbygCbRSSimllFJe0ARaKaWUUkopL+gYaKWUUkopH9H7QPcNuhWVUkoppZTygibQSimllFJKeUETaKWUUkoppbygCbRSSimllFJe0IsIlVJKKaV8QIAAEX93Q+0BWoFWSimllFLKC5pAK6WUUkop5QVNoJVSSimllPKCjoFWSimllPIJ0R9S6SN0KyqllFJKKeUFTaCVUkoppZTygibQSimllFJKeUGMMf7ug9o9uqGUUkqpPcfnN2TOsieaB8JP9fXL9qhTqp9cYoyZ6O9++JpWoJVSSimllPKCJtBKKaWUUkp5QW9jt5d5Y811/u5Cjzp9+EPt0/d8N8uPPelZNx08p336wtfG+rEnPe/Fs39pnz70rgw/9qTnfXNLbvv0Eff23Vi/unFbnMc8MsCPPel5n/5hc/v08f8c6Mee9LyPrs5un15W+rYfe9Lzxif0rWEUyvc0gVZKKaWU8gEBAsTnQ69VD9AhHEoppZRSSnlBE2illFJKKaW8oAm0UkoppZRSXtAx0EoppZRSPiHYRGuXfYFuRaWUUkoppbygCbRSSimllFJe0ARaKaWUUkopL+gYaKWUUkopH7DuA621y75At6JSSimllFJe0ARaKaWUUkopL2gCrZRSSimllBd0DLRSSimllI8EiPi7C2oP0Aq0UkoppZRSXtAEWimllFJKKS9oAq2UUkoppZQXNIFWSimllFLKC3oRoVJKKaWUD4iI/pBKH6FbUSmllFJKKS9oAq2UUkoppZQXNIFWSimllFLKCzoGeh+Uv6mcZ275ks2rSkjoH8n5fz2UMdMyum3vbG7lL7NfozCnkhlnjuWCmw8D4OojXqCsoKa9XcbQBO57/+we7783SnLreP+BFeSvryYmOZRjrxnO4EkJXbb9+2nfUlXc2P44OSuCq1+YDsCitzfz33dyqa1oIjIumGm/y2T/kzJ9EcJuq8pvYuHzWynPaSQ83sHUc/vTf3REp3Yb5lew8Lmtneaf8tAwIhICWfxKPpt/qKapppXUcREc+ccBvuj+bqsvbWHdB+XUFjgJjrYz+JgYYgeFdNk27781bF1ci7PWRVCEjdQDIkmdav2ftDS62TingrJ1jRg3RPQLZPxFSb4MZZfqS1pY8145NflOQmLsDDkuhrjBXce6ZVENeYtqaa51ERRpI31aJGkHWLEueiCfpipXe9vwfg6mXN3PJzHsrrpiJ7++UU5NXjMhsXZGnBRHwrCuY938XTU539XQXOMiKMrOgIMjyTwoEoBV75VTuKweZ52bhBEhTLq4d21TgNoiJ7/8p4zqvGZC4hyMOjmWxOGhndrl/VDL8n+XdZp/+G2phMY52PJ9LRvmVtFU4yIuK5ixZ8YTEt17vtaz12/lzj88zZpfs0lJS+CGey5k/0PHdtn23KNuInvdVtxuNwOGpHLdneey3/4jAFj+w1ruueF5cjcVkDU0jVsevpThYwb6MpQ9SsdA9w26FfdB/7z+MwqyKzjnzwdhswfwyLWf0FDb3G37955cTEVxbZfLhk3sz9UPzuLqB2dxxh+n91SXf7M371xO6ZZ6Zl0xnACb8Prty2mqa+m2febYGE67ZSyn3TKWoy4dCkDZ1nrmPLEWCYBZVwzH1ermk8fWUFXS2O16/OG7J7dQXdjM5DP7EWATvvlnLs4GV6d2ycPCOfiKdA6+Ip2DLksjwC6ERNkJi3G0txkwNcqXXffK6rfLaChrZdBRMYhNWPVmGa1N7k7tGspb2PR5FSIw6KgY3G7YOKeSpupWANZ9UE7xrw30mxDOoKNjCIntPYlHm5VvllFf2srgWTFIgLDiP93EWtbChk+rQGDwrBiMC9Z/UklTVWt7m+jMIEaeFsfI0+IYNDPad0HspmWvlFJf0sKwE2MRm7Ds5RJaGjvHWl/awpoPKkGEYSfGYlyG1e9X0Fi5LdaU8WG+7LrXlr5cSl1JCyNmxxFggyUvdR1r3KBgJpyXwITzEhh/TgIBNgiKsBEcbadqSzO/vFFGcLSN4cfHUL6hkRVvlfshmu7ddOmj5GzM57o7zsXusPPn/3uI2pqGLtuOmTSEP91zAf933cmsX5nDXdc+A0Bzk5M/XfgPGuob+eOd51FeWsUNFz6Ey9X5/0spX9prE2gR+VZEJu7hdf4iIq/v7HVEJFNEVnqmDxGRahFZJiJrROS2LuavFZEHOzw/XESeEZFNIrJKROaLyJQ9GcfObF5dQu66Ug44ZhgzzhzHrPMm0Fjn5Ie5G7psn7uulDmvLOXkK/bvcnliahTjDx7AAbOGMnZ6Zg/23HsFG6op2lTLmMP7MXV2BtN+l0lzfSsrvyvq9jkxyaEM3T+RMYenMHiyVak2bgNAZHwwWfvFER4bhN0RgCOw93x8ynMaqdjSxMCp0Qw/Mp6RRyfQ0ugm58fqTm0jEgMZuH80A/ePxuYQ3K2GwQfFEGC3fl526rn9GXlU11V6f6stdFJf1ELi6FD6T4kg7YAIXM2G0lVdfClbm42gCDsxWcEEhtsQOwTYhcaKVsrWNJI4OpSBR0TTb3wYw2bH+TaYXagtcFJX2ELSmFDS9o8gfboVa8nKzrGatlgj7cQOCiYwwkaAJ9Y2IbF24oeFkDw2jLghXVd2/aV6azO1BS30Gx9G5vRIBhwSSWuToeiX+k5t22INjrIRPyRkW6wOK9aRJ8WReUikL7vvleq8ZmrynfSfEMaAgyIZeEgUrU2GwuWdYw2Nc9B/v3D67xdOgF1wuyBtajgBNqF8YxMYyDggkoEHRxGVFkTxqgac9Z0Pmv1h7YrNrF+Vy8zZ0zjtoqM467JjqKtt5OuPF3fZ/o93nsdBM/Zj8oGjcQQ5CAiwtueir5dTXlrNqefP5HcXzuTEMw8jf0sJPy9a5ctwlOqk95Vc/EREhmMdUBwkImHGmM57s64tMMYcKyJhwHIR+WSH+SHAMhF53xizCHge2AwMNsa4RWQgMHxPx9NRc2MLzU1WdSZvg3U6MDYxHIC4ZOsUb0le50TL7TY8d+uXzDhjHFmjuz4NuuDD1cz/YDWRsSGcfu10Dj15VE+EsNucTS5amqwvkOLsOsBKfAGiEqy/FYXdV46Xz81n2Rf5hEUHMuPiIUw8Jo2E9HBmXjKEuc+t55FzFyABcPKfxxAWHdTD0exca7ObVqdVhanc2gRAaKxVRQ7z/K0tde50HevmVSACQw7tXcljRy6nG1eLlTXVF1vxBEXYrL+R1t+O1cc2ofEOBh4ZTfZXVfz4WCEIDJsdS2CYjbKt1nugNt/Jgr/lQQCkTo0ga0aML0LqVsdY64qsWIOjPLF6/jZWdI41LMFB1sxoNs2tYvHDVqwjTo4lMNzW3qZwWT2FS+txhAUwaGY0KRPDezqcnXI53bicVqy1hdZZobZYg6Osr6aG8s6xhic6GHpsDOs+rWT+vfkgMOaMeII6xNrbtHaItabQs109Qy2Co61+15d3f2YMIPe/NSCQ4RmWExhuHcBXZDcRlR5IXWkLGOv/LDDMP/8XjQ3NNDVaZzM3rt4CQGK/WACS+ln7mK25xV0+t66mgcOH/x8AEVFh3PLQZQAUbCnZbj2JKdZ68nOLgdE9EIVSu2ePJtCeJPItIBWwAXcBQ4HjgBDgv8ClxhgjIt8Cy4D9gATgXOBGrE/Em8aYm0UkE/gc+AEYD6wHzjXGbFeCEZEZwB1AELAJuMAYUyci9wHHA63AXGPM9Tvp/pnAq1jJ7PHA6ztp24kxpl5ElgBZQEmH+Y0ishzoLyJZwBTgLGOM27M8G8j25rW89fELP/Puk9ZRf0hY4I79tiZkx2fBd++vojS/hovvHE7eeivxbqhtpqaigcjYUA47ZRQpA2JwNrt44+GFPH/7V4yckkZiqv9O/y94PZt5L28EICh0hy+R7kMFYOKxqSSkhdHqdPPFs+v48B+rGDghjqAQG9+/l0vyoEgOO28Q817eyMePrmLg+FiiEv1XyVvxaQnL37feao7gHarhZtfPrylupmB1HaljIohICNz1E/xky8Iacr+1xtrbgnbYejuJ01nvYusPtYQnO8g8JIqcb6vZ8Gkl0QOCMa3WE10thhG/iyf/h1ryFtYSkxVCbFZwT4WyS7nza9g87zfEWudi6/e1RPRzMOCwKDbPq2bdx5XEZAUTHGUnZVI4ofEO3K2GTV9UsfaDCmIGBvt12MqmedVs/MI6cLfvGGubLmY317nIXVBDZEogg2dGs+GLKla/W07c4OBeNf63o01fVbP+8yqg+1i72y+BNWylbH0TicNDCI2zDo5TxoeT+99achdZ/9rWa3PsbE096+XHP+TZB98BICx8+31j23eNSNf9CwkL5sm3biZnYz6P3vkaTz3wJs++d1undrtaz94gYC/uu9pmT+9tjgIKjDHHAIhIFPClMeZOz+NXgWOBjz3tncaYg0Tk98CHWMl0BbBJRB72tBkKXGSMWSQiLwJXAB2HRMQDNwNHeJLYPwPXicjjwGxgmCdhj95F308DjvS83lV4mUCLSBwwFeugIaHD/BhgMDAfmAwsN8bs1jk2EbkEuATgmWee4ZJLLvGmS+0OPGE4QyekABAYbOf2s9+i3DOmuaLYqtK2Jb0tTqvi4wi0U15YS01FI3+Z/Vr7uhZ+vBZHoJ1L7jqS2ZdtG3mSs6aEOS8vpTCn0q8J9PiZ/ckYbVURHUE2nr16MdWlVnW2usz6G9PP2rG3Oq3NYA+0Eu1DzxnUvp6CjTUseiuH8rx6mupbqSlrZvLx6YyYnkRxdi1fvbiBLaurGO3HBHrQ9BiShlhjPW2BAcy5axMNFVYVq77S+tuWGLtarEq1zbEt0V43rwIMDDu891afAZLHhROVYSW1Nruw7IVimmusbddca/0NibF2ZW5PYhxgF6o2N+GscZEyMZz44aHUFbeQM6+amjxne+IYnRFEwohQWupdVG1upqmiBfyYQPebEE50pvX6AQ5hyTPFNFV7YvXE3Nb3jrFWZjfRXOOi/+RwEkaEUlfUQvZX1VRvcRI82s6AQ7d9JusKnWxZWEtDWYtfE+jUieHEDtgW6+J/FrXH2jZOPdTTP5cnVptdqNjQRFO1i/QDIkgaHUptoZP1n1VRldNMyLjemUCnTg5vPzCzOYRFjxTS6Bmf3nZxZ1ti3HYGomMinPvfWjCQOX3b0BSbQ5h2TT9qCp0EBAgr3yunIruZ0Dj//R8c+7uDGT9lGABBwYFceNytlBRUAFBSZP3tn54IgLPZ2kcFBllx2+02ph4yhqmHjOGrjxfz88JVVJbXkOJpX1xgje8uLbTW0zZfKX/Z05+0FcCDInI/8IkxZoGInCwiNwChQCywim0J9EcdnrfKGFMIICLZQBpQBeR5hj4AvAZcQ4cEGitpHQEs8hyRBgLfAzVAE/C8iHwKfEI3RGQSUGqMyRWRrcCLIhJjjKmk67pPx3kHisgywA3cZ4xZJSKHeOb/ipWQ32eMKfL2iNkY8yzwbBev6ZWktGiS0qLbH6cPjef7OetIGxTHl2/8SkhYIJOPtJLH62a9TG1lI/9achVTjxpC2mArucrbWM67Tyxm7PRMjjh9DFvWl/HmIwsZe+AA3K1uFny0hsBgO2lD4n9rN/eI2JRQYlO2Xc2enBXBinmFJGWG88OHWwgKtTHyoGQAHj5nAQ3VTm77fAZF2bXMfW4dQ6Yk4HYZln2RjyMogKSBEdSUWackl39ZQERcEL98VQBAfKp/L1SKSAwiInHbMJKYtGCyF1cR3T+ItV+X4wgOIGOSlTi9d8M6mmpdnPO8NcTG1epm44IKwuIcpI7d/k4dectr2oeE1Je3sP7bcpKGhROV7J8hKyGx9u0SvbAkByUrGwhLdJD/Ux22ICF+hLXNf3isgJYGNwfdnEawJ6ku/rWeoAgbJb9ao7JC4+2EJTkIS3JQmd1Ewc91FC6rhwCITPfvsJwdYw1PdlD8qyfWH6xYE0ZasX7/kBXrIbentT+naHk9QZG29rHDofF26oqcbJpbRdyQEIwbCpfWE+AQwpMdnTvgQ6HxDkLjt/UhIsVB4bJ6wpMdbPFUVJPHWrHOv2crzno3M+/PIMSTIOYvqSMo0kb+EivWsARrXSWrGqgtspKzpqpW8hbXEpsV3L7cH8LiHYR1iDUyJZCCpfVEJAeSu6gGe5DQzxPrN3dvxVnnYtaDmYB1oJT3Qy0hMTYSR2w7YDduw6r3K4hKDaQqt5mydU0MPDQSmx+vzUjNTCI1c9twv8EjMvjig0UMHJrKOy9/SVh4CIcfaxVeZh/wB6rKa1iU8yr/nbecLz/6nrGThlKUX8avP60nLiGK6NgIph0+jtj4KN55eS5h4SF88J95pKQnMHHaSH+FqRSwhy8iNMasx6oirwDuFZFbgSeBU4wxo4HngI7lnbZbP7g7TLc9bvsW2TFx3PGxYFW5x3n+jTDGXGSMacWq+L4LnIg1FKQ7ZwDDRCQHawhIJHCyZ1k50HFgZCzQ8b5CC4wx440x+xljnt5h/hisISmXi8g4rIOHsSL+vYfNVQ8cTcqAWF69fz6tLS6ueegYwiI7V91SB8UxZeYQpswcwohJqQAkpUcxcGQSETHBuF2Gt//5X15/eCHxKRFc99hx7WOre4vf3TyW+LQw5jy5Blerm9NvG09IROcv0tAoB8YNX7+4gbnPriM6KYSz7ppAZHwwqcOiOPqKYbha3Hz8yGpaW9wc9/sR9BvUuy5UOviKdKL6BfHjfwpxuwyHXJ1BUDdjIXN/qqGp1sWQQ2KRgO0P7FZ+WsqSN60LLSvzmlj0Qj4l63f3koCeN+LUOELj7Gz8vBLjsoZgOEI6f6Qi+weRNTMa02pY/2kFbpdh8DExhCcHIiKMODWOkBg7G+ZU0NroZvhJcYQn9a6hLCNPiyM03s6GOZW4XYbRZ3QTa2oQg2dF43YZ1n1UgbvVMOS4GCL6BeIItWHckP1VNRu/qCI4xs6Ys+IJiuxd1dpxZycQluBg7QfWthp/XgKOkM7v3+j0IIadEIO7FVa9W4671TDi5Fgi+1vbLvubatZ9UglAbUELK94sp3Jzk09j2ZUJ5yUQnuhg9ftW//e7IBHHjkPOPAp/sW7Jl35ARKfPavnGJn59s5z8pfVkHhTJsGNjfdH93XbP09eQkZXCQ7e9Qouzlfuev5aIqM6Fh6iYcFYu3cj9N77Af56dw7gpw3jktT8jIgQFB3L/C9cSGhbM329+idj4KB54/jpstt5zEbfaN0n7GNg9sTKRFKDCGNMkIicC5wP7A5lYY6IXA+8YY273jIG+3hjzs6die70x5ljPer4FrsdKVDcDBxhjvheR54C1xph/dGiTCywBDjPGbBSRUKwx2AVAqDGmRERigY3GmE57F08ymwtMNcbke+YdCtxsjDlcRK4CJgHne4aCPAqUG2Pu3LHfHda5YzzXApONMWeIyFvAOuBWz/oGAyOMMR/u4r/XALyx5rpdNNu7nT78ofbpe76b5cee9KybDp7TPn3ha13fF7WvePHsX9qnD72r+/uN9wXf3JLbPn3EvX031q9u3BbnMY/0rvuE72mf/mFz+/Tx/9x77z28Oz66etvlOMtK3/ZjT3re+IRTYedDz3vEUEc/81Tceb5+2R51ePH9S4wxe/SuaHuDPV2CGA38XUTcQAtwOVb1dwWQA/z0G9a5BjhPRJ4BNgBPdVxojCkVkfOB10Wk7fzrzUAt8KGIBGN9SK7tZv0HAfltybPHfGCEiPTDGkIxDPhFRAzwM9bFjt54GrheRAYA/wf8A9goIg1YFe4/ebk+pZRSSinlJ3s0gTbGfAF8scPsn7ES2h3bHtJh+lvg2x2Xee7C4TbGXLaL58/DqhLvaPJu9PlbrHHUHee5gI4/03XVTp777a7mG2Magf4dmly8q34ppZRSSqneSQcRKaWUUkop5YXedRXJDowxOcAe+2UOEfkrcOoOs982xty9p15DKaWUUqo7Nv/eR0DtIb06gd7TPImyJstKKaWUUuo308MgpZRSSimlvKAJtFJKKaWUUl7Yp4ZwKKWUUkr5iwgEePmrxKp30gq0UkoppZRSXtAEWimllFJKKS9oAq2UUkoppZQXdAy0UkoppZRPCAF6H+g+QbeiUkoppZRSXtAEWimllFJKKS9oAq2UUkoppZQXdAy0UkoppZSPiE1rl32BbkWllFJKKaW8oAm0UkoppZRSXtAEWimllFJKKS9oAq2UUkoppZQX9CJCpZRSSikfENGLCPsK3YpKKaWUUkp5QRNopZRSSimlvKAJtFJKKaWUUl4QY4y/+6B2j24opZRSas8RX7/g8OD+5qW0y3z9sj1q/423LjHGTPR3P3xNK9BKKaWUUkp5QRNopZRSSimlvKC3sdvLfJZ7t7+70KOOzvhr+/Sji0/zY0961u+nvtk+fdU7k/zYk573+Ck/tU/PfCDTfx3xgS9uyGmfnvXQAP91pIfNuW5z+/SJT2T5sSc974MrN7VPn/TUYD/2pOe9d/mG9umFhc/4sSc9b3q/S/3dBbWX0wRaKaWUUsoXBMSuJ//7At2KSimllFJKeUETaKWUUkoppbygCbRSSimllFJe0DHQSimllFI+IYhNa5d9gW5FpZRSSimlvKAJtFJKKaWUUl7QBFoppZRSSikv6BhopZRSSilf2AfvAy0iRwGPAjbgeWPMfTss/xNwluehHRgOJBhjKkQkB6gFXECrMWaizzq+C5pAK6WUUkqpPU5EbMATwJHAVuAnEfnIGLO6rY0x5u/A3z3tjwOuNcZUdFjNocaYMh92e7fsW4dBSimllFLKVyYDG40x2cYYJ/AGcMJO2p8BvO6Tnv2PNIFWSimllFI9oT+Q1+HxVs+8TkQkFDgKeLfDbAPMFZElInJJj/XyN9AhHEoppZRSPiACYhN/d2NPixeRnzs8ftYY86xnuqtgTTfrOQ5YtMPwjWnGmAIRSQS+FJG1xpj5e6DP/zNNoJVSSiml1G9VtpOL+7YCaR0epwIF3bQ9nR2GbxhjCjx/S0TkfawhIb0igdYhHEoppZRSqif8BAwWkQEiEoiVJH+0YyMRiQIOBj7sMC9MRCLapoEZwEqf9Ho3aAVaKaWUUkrtccaYVhG5CvgC6zZ2LxpjVonIZZ7lT3uazgbmGmPqOzw9CXhfRMDKV/9jjPncd73fOU2glVJKKaV8Qva5+0AbY+YAc3aY9/QOj/8F/GuHednA2B7u3m+2b21FpZRSSiml/keaQCullFJKKeUFTaCVUkoppZTygo6BVkoppZTyBWGfGwPdV+lWVEoppZRSygtagd4HbdlYwqM3vM+GlQUkpcZw+e3HMOGgwd22dza1cOWsJ8jPLuPYc6dwxZ3HAbDq51yeuPkjtmaXkTE4kd/fP5tBo1J8FcZuKcqp4Y27fyJvXSWx/cI4+brxDJuS3GXbO2Z/QmVRQ/vjlMHR3PDKDAAeuugrinNqcLsMyQMiOfGacWSNT/BJDLurYmsj3zyVQ0l2A5EJgRx4UTrpY6O6bNtc38qCF/PY/HMVbpchYWAoJ905DIBfPi3ml0+Lqa9sISzGwbjjkhhzdJIvQ9mpuuIWVr1TTk1+MyExdoadEEv8kJAu2+YuqCF3UQ3NNS6CIm1kHhhJ+rRIABY/Xkh9cQvGQHiigyHHxhA7MNiXoexSXbGTFW+WU721mdBYO8Nnx5EwtOtYN8+vJnd+DU01LoKj7GQeFEnmgZHUl7aw8u1yaguduF2G6PQgRp4SR1i8w8fR7FxtkZNl/y6lOq+ZkFgHY06NI3F4aKd2WxbXsuy10k7zj7wjjdA4B5u+qWbTN9U017QSHGUn69AoBh7S9efAX2oKnSx7rYSqvCZCYx2M+V0CSSM6x5r7fQ1LXy3pNH/GXRmUrW/sdllYXO/YtjkbCrn/uldZt2ILyalx/OFvpzH5kBFdtr3kmPvIWV+I2+Umc0g/rrr9VMZNtb6XTp18E0Vbt/043aARqbz01c0+iUGp7mgF2gsiUtfFvNtFJF9ElovIahE5Y4fls0XEiMiwDvMyRaSxw3OeFhGfbYsHrn6LvE1lXHzz0djtAdxz5RvU1zR12/4/j31DWWH1dvOcTS3cc9nrNNY1c8kts6gqq+Oey1/H5XL3dPe98sqtiynJreXEa8Ziswv/+uv3NNY5u22fNS6Bc++cyrl3TuW4K8a0zx8wOo6Trh3PjAtGkL+hijfu/ckX3ffK3EeyqcxvYvp5aQTYhS/+kU1zfWuXbec9mcP6BeUMPyyeAy9IIyo5CICqwiYW/isPEZh+Xhpul2HBi3nUlnX/f+ZrK14vpb60haHHxiI24dfXSmlp7Py+qy9rYd0nlYgIQ4+Nxbhg7UeVNFVZ/yfRGUEMOz6WgYdHUVvoZPW75b4OZZeWv1ZKfUkLw4+3Yl3+SknXsZa2sPbDShBh+PGxuF2GNR9U0FjZSlN1K8YYBs+MJnVSOOUbmlj5Vu+L9eeXSqgrbmHk7DgCbPDTC8Vdxho3KJj9zk9kv/MTmXBuAgF2CIqwERxtp66khZXvliMCI2fH4XYZVrxTTmNl158Df/n5pSJqi52MPjkesQk/Pl9ES6OrU7v4wSFMujCJSRcmsd95SVaskTZCou07XdZb3HH5C+RuKuKq207B7rBx66XPUlfT2GXbURMH8vu7TuO8a49h46qt3H/9q9stHzt1MLc9eRG3PXkRl9082xfdV2qnNIHeMx42xowDTgCeEZGOh/9nAAuxfn2no02e54wBRgAn9nw3YdPKArLXFHHw8WM47typzP6/aTTUNrPws65/3GfzmiI+eOG/nPWHw7ab//O3G6gsq+OYc6Zw7DlTmHHafhTlVbJi8WZfhLFbtq6rpGBDFROOTOPAUwZzyOlDaKpv4Zd5W7t9TmxKGCMO6MeEI9MZPnVbpfrE349j5PQUhkxMxO4IwHNj916jdHMD5bmNDJ4ey+ijEhl7bDLORhebFld2altd3Ez2j1UMnh7L/mf2Z9gh8Rx+xQAAjCdfCYsNJHV0JKHRDmwOwe7oHfHW5DupLWyh37hQ0g+IIPPASFqbDcUr6js3NtafoEgbcYODCYqwEWCHALsVy9BjY0gYEULcoGDEJtA7QmxXvbWZ2oIW+o0PI2N6JJkHRdLaZCj6tXOsxhNrcJSNuCEh22J1CDGZwUy9sh8Z0yMZMTsOR2gAdcW954AIoCqvmZp8J/33C2fgwVFkHRZFa5OhYFmnmgVh8Q5SJ4aTOjEcm0Nwt0L6/hEE2ATj+Y8IjraTMKzt/0Hat3lvUJXXTPVWJ6kTIxh4cDSDDo+mtclN/tLO29WKNYLUiRHtsWbsH0mATXa6rDdYv2ILG1dv5YgTJnHSBYdw2iWHU1/bxLefLO2y/dW3n8q0I8ew3/ShOALtBOywj+2XHscBR4zmiBMnMeWQkT6IQKmd6z2Hqn2AMWaDiDQAMUCJiIQD04BDsX668vYuntMqIv8FBvVUv5oanTQ3tgCQs64YgPjkCM9f63R20ZbOiZbb7ebRv7zPsedOYciY/tstK9pqtY/zPD8+OcqzngqYltUDUeweZ1MrziarklOwyaqaRyVYp7yjE61TpOUFXSRbHj9/lsNPc3IIjwni2MtGM/X4gQA01rVw89HWL4yGRDg4/aaJPRbD7mppdtHabGW85VusoSdhsYEAhHtO4dYUN3d6XuVWqwJUsrGeZ89ZhgTAmFlJHHB2KjH9g9n/rP58/598/vOHlYjAYVdmEhLlv1PCLqcbV4uVGNUVWYlfUKS16wqKsgHQWNG5whiW4GDw0dFs+LyKRQ8WgMCoU+MIDLee09pk+PZO62DKHhLAyJPjejyWXXE53bicbbFan9lgT4zBnspiY3nnWMMTHQw5Job1cypZcH8+CIw5PZ4gT6xtqvOaaWlwkzSm83ABX2vtEGtNgbVdQ6Jtnr9WrPVlO68c5yysBYHMadb+LCIpkBEnxLL6owrm3bUVBMafnUBQhG2n6+lp28Wab30mO8fastN1bF5YbcU6PdKrZb7U1OCkqdHaltlrCwCI7xcNQEK/GAAKtnQefgNQV9PIcaOvByA8KpQ/P3jOdsu/ePsHPn9rMdFxEVx644kce+a0ngih5wmITWuXfYEm0HuQiEwANhhj2gamnQh8boxZLyIVIjLBGLN0h+eEAocDt/ZUv955egH/efQbAELCg7Zb1la56qr69uXbSyneWsXv7xtPzroiAOprm6gu76oC5lmRnyuzX7+2li9eWA1AUOj2b++dxQqw/wkDSUyPoNXp5uMnf+WtB5YweGIicSnhBIXYufzRgyjOreXjx3/ls+dWceXjh/RcILth2YdF/PR2IQCOkB12yO2xdg62LRltaXYz49qBrPi8hGUfFpE2JpL4jBB+/ayE+MxQJp3aj5/eLmT+C1tIHRVJeFxgT4bTrc3f1ZD9lXUwZAvaIZ6dbFNnnYsti2qJ6BdI1hFRbPqqmjUfVhCbFUxwtB1boLDfRYnUl7awfk4VG7+sYtIlXY+P95Xsb6rZONf7WJvrXOQurCEyJZBBM6LZOLeKVe+VEzsouD1BqytpYcmLJYTE2hkxO7YHo9g9G7+sYt1nVQDYg7uOdWe7k/rSFkrXN5I0IoRQzwFjc62L7O+qieofyNBZMaybU8mKt8tIGBJCSIz/vu42zK1k7Ryr6NApVk+wO4u1rrSF0nWNJI0I7TS+eWfLfO0/T37BSw99CkBo+PbXE5j2OLsONCQsiIdev4bcjcU8dfd7vPDgxzz69rUAHHfWdNKyknA2t/LMPe/z4J//zYTpQ0lJj+/BaJTaOU2g94xrReRiYCBwVIf5ZwCPeKbf8DxuS6CzRGQ51t7zQ2PMZzuuVEQuAS4BeOaZZ7jkkkt+U+cOP3k8IydlABAY7OBPpzxHWWENAOXF1t/kNKs60NJsVXwcQXZKC6qpLq/nyqMfb1/XN+//giPQzuRDhwJ0ux5/mXR0JgPHWBf3OYJsPHbZPKpKrYprdalVpY3rFwZAq9OqVNsDrUrQjPO3XdyydX0l376+npItdcSlhGOzBzB0cjJDJyfzy7ytbFhSQl1VM+HR2x+Q+NLQg+PpN8yqvNkDhfduWUd9uVX9qauwqlmRiVbS62qxKtU2RwARCda8lOERZE2JobGmlfyVtVQXN9NU10p9RQujZiQwcFIMFVsa+eGNAorW1zFof/8kXSkTwojJtP6fAxzCT08V01RtvU+ba6xt2JYcuVutL+kAu1CxqYnmGhdpU8NJHBlKXZGTjXOdVG1pJjnaToBNiBsSQtyQEIpXNFC5qRlnvYvAMP9VK/tPDCdmgJV42BzC4seLaKq2YmyLOSTWitXlidVmFyo2NtFc7SJ9/wiSRoVSW+hkw+dVVOU2ExJtp7bIyY9PFxFgFyZflkRwpP93/WlTIojNaos1gIUPF9BYZcXa6Ik1NM4Tq+egz9ZhKFHOohow21ddyzY00lTlInN6JP3GhFFT4GTtJ5VUbG6if0y4T+LqSvrUSOIGWWfCbA5h/j/y28dlt8XcdhDQZawLqsHAgIM6Xwy5s2W+dtSpUxkz2TqZGhjs4MoTH6S00DpwKCusAqBfmpX0OputfVRgkBW33W5j0sEjmHTwCL79dClLF62jqryO6Lhwzv39rPbX2LAyjzef+Yq87GJNoJVf+X8v2jc8bIx5UEROAl4RkSwgDDgMGCUiBrABRkRu8DynbQx0t4wxzwLPtj38rZ3rlx5Lv/Rtyc+AYcl89/EK0ockMue1HwkJD2LaUdaYsosPe4SaygbeW30rBx07msyh1t0XcteX8O9H5rHfwYM55uzJZAxOJDo+jDmv/UBoWCBz31xCUmo0o6cO+K3d3CPi+4cT33/bF2XKoCiWfZlH8oBIFr2/iaBQO2MOTQXg7tM+o77ayQPzTqJgYxWfPr2C4fv3w+Vy89NnuTiCbKRkRbFmcRHLv85jwOg4Kksa2LyijIjYYMKi/FORbROVFERU0rYEPi4jhA2LKohNC2Hl3BIcIQFkTbUOaP79+5U01rRy6WsTSBgYSlx6CFtX1LDqq1LWflOGBEC/oeHtifa6+RWERjtYv8C68j26n//uThEa52hPLgDC+zko+qWB8KRA8hbXYgsSkkZbB0ULH8ynpd7N4XelE+JJvgqW1RMYYaNwmXXmJCzeQdm6Rop/rScqI4imahdVuc0EhgfgCPXvqdUdY43o56BwWT3hSQ62fG/FmuwZfrHgvq04693MuDejPdEsWFJHUKSNAs942rAEB42Vrfz4VBEtDW4GHxVN1ZZmqrY0kzLefwklWNuh451AIvsHkr+kjohkBzkLa7AHCynjrO369V15OOtcHPuQtX9xtxq2LK4lJMZO0shtw1Ha/u+2/lRHcKSNrT9ZY6jDE/1bme0q1q1L6ohMCSR7vifW8VasX92RS3O9i+MftobCuVsNuYtrCYm1kzxy+6E3O1vmDykZCaRkbLs7UdaI/nz94c8MGJLCB6/MJzQ8mIOPGQ/AWQfeRnVFPXM3PsoP367im4+WMGpiFiUFFaz8eROxCZFExYaxaU0+z977AVMPG4mr1c3nby8mKNhB1rD+3XVDKZ/QgTh7kDHmPeBn4DzgFOAVY0yGMSbTGJMGbAam+7OPADc8diqpA+N57m+f0dLi4sbHTyM8qvOtsdIHJzJ91iimzxrF6CmZAPTLiGXw6P4EBju48YnTCQ4L4pk75xAVF86NT56BrZeN7TrnjqkkZETwwWO/4Gpxc/7f9ic0onPiGxYdhNtt+Oy5lXzy5Apik0O58L5pRCWEEBYZSO7qct79xzK+e3MDA8fGc/Hfp/e6CwmP/P1AolOCWfhyHq5Ww8xrswgK63yMLCIc+YeBRCYFseDFLTTVtXLEVQOISw8hMSuMaeem4mpxM/+FLbha3Bx0UTrxmf7/cm4z5vR4whLsrPukAuMyjD0rofMQFiAqNYghx8TgbjWs/bACtwuGnRBLREogjtAAqvKcrP2wki0La4jODGL8+Ym9bpuOPTuBsEQHaz6qwN1qGH9uAo6QzhXyqLQghh0fg9sFq98rx91qGDE7lsiUQBrKW3DWuTFuWD+nil9eK+OX18r8EM3O7Xd+IuGJDla9X467FSZemIQjtOuzAQXL63HWucmYFoEEbNtmMRlBjJwdi7vV8Otb1v/D6FPjiEr135mirky6IInwJAcr3i3D7TJMviiZwG5jrcNZ5yJzWuR2se5qWW9w2xMXkZ6VxON3vENLSyt3PnMxEVGd9yWR0WGsXpbDwze/wdvPz2PM5EHc9/IViAhRseG43W5e+PvHPH3P+ySnxnL3i5cRnxzt+4D2BBHEHtCn/u2rpH3sqtolEXEDBR1mPQREAnXGmAc9bfYD/gMUA/cYYz7v8PxrgOHA/cAnxphRXry8Afgs9+7/KYbe7uiMv7ZPP7r4ND/2pGf9fuqb7dNXvTPJjz3peY+fsu2WfzMfyPRfR3zgixty2qdnPeTfszE9ac512+62c+IT/rto2Bc+uHJT+/RJT3V/v/y+4L3LN7RPLyx8xo896XnT+10Kfrj3zojIdPOfidf7+mV71Phvfr/EGOP/K+t9TIdweMEYs8tDLWPMEmBoN8se6/DQm+RZKaWUUkr1Evtu7V0ppZRSSqnfQCvQSimllFI+IOh9oPsK3YpKKaWUUkp5QRNopZRSSimlvKAJtFJKKaWUUl7QMdBKKaWUUr7guQ+02vvpVlRKKaWUUsoLmkArpZRSSinlBU2glVJKKaWU8oKOgVZKKaWU8gVBx0D3EboVlVJKKaWU8oIm0EoppZRSSnlBE2illFJKKaW8oGOglVJKKaV8RGxau+wLdCsqpZRSSinlBU2glVJKKaWU8oIm0EoppZRSSnlBx0ArpZRSSvmCiN4Huo/QraiUUkoppZQXNIFWSimllFLKC5pAK6WUUkop5QUdA62UUkop5QMi6BjoPkK3olJKKaWUUl4QY4y/+6B2j24opZRSas8RX7/gqNhM89aMW339sj1q5JsXLTHGTPR3P3xNK9BKKaWUUkp5QRNopZRSSimlvKAXEe5lXv71Cn93oUedN+bJ9ul7vpvlx570rJsOntM+/X//Hue/jvjA82ctb58+/p8D/dcRH/jo6uz26ZOeGuzHnvSs9y7f0D69L71///bNTP91xAduPvSL9un8+qV+7EnP6x82wT8vLILYtHbZF+hWVEoppZRSyguaQCullFJKKeUFTaCVUkoppZTygo6BVkoppZTyBf0hlT5Dt6JSSimllFJe0ARaKaWUUkopL2gCrZRSSimllBd0DLRSSimllI+Izee/IK56gFaglVJKKaWU8oIm0EoppZRSSnlBE2illFJKKaW8oGOglVJKKaV8QUTvA91H6FZUSimllFLKC5pAK6WUUkop5QVNoJVSSimllPKCjoFWSimllPIBEXQMdB+hW1EppZRSSikvaAKtlFJKKaWUFzSBVkoppZRSygs6BloppZRSyicEsWntsi/QraiUUkoppZQXNIFWSimllFLKCzqEYx9UkF3Ji7d/Q87qUuJTIjj7Lwcy6oC0bts7m1u59dS3KMqt5vDTRnHOTQcC8MVrvzD33yuoLq0nOiGMmeeM5cgzR/sqjN1SklvH+w+sIH99NTHJoRx7zXAGT0rosu3fT/uWquLG9sfJWRFc/cJ0ln62lXfvX9Gp/fWvH0xMv9Ae67u3qvKbWPB8HmWbG4mID2Tqef1JHR3Rqd36+RUseDav0/zfPTyciIRAvn8ln+zFVTTVtJI2LoIZ1w/0Rfd3W22Rk1/+U0Z1XjMhcQ5GnRxL4vCut0P2N9Vkf1dDc3UrQVF2sg6NZMDBUQBUZDfx61tl1Be3EN4vkLFnxBOdFuTLUHapptDJstdKqMprIjTWwZjfJZA0onOsud/XsPTVkk7zZ9yVQdn6xm6XhcU5eqTfv8W+8v4Fa7/0wd9XUbChhuikEI69ehiDJsV32fYfZ8ynqrip/XFyVgRXPrc/AP99J5fv382ltqKZiLggpp2SydST0n0Sw+7YsHYz1195JyuWrSE1I4W7HvwTBx++f6d22Ru38Oer/8aaVRtpcbYwYdJo7n30RjIHWt9Lt1z/AB+/+yVlpRUcftR0Xn7nUV+HolQne00FWkSOF5G/9PBrXCsiTSIS1WHe+SLy+A7tvhWRiZ7pHBFZISK/iMhcEUneYf6vIvKdiGR0eP65IrJSRFaJyGoRub4n49rRU3/5ksLNVZx+/TRs9gCe+NNcGmqbu23/0TM/U1Fcv928otwqXv/7fwkQOP36abha3fz7/oWUF9X1dPe98uadyyndUs+sK4YTYBNev305TXUt3bbPHBvDabeM5bRbxnLUpUOteeNi2+edetMYbA4hPCaQyIRgX4WxW755IpeqgmamnJWC2IV5j+XgbHB1atdvWBiHXJnOIVemc/Bl6QTYhZAoO2Ex25KpgVOjfdhz7yx9uZS6khZGzI4jwAZLXiqhpdHdqV1dSQur3q9ABEbMjsO4DCvfraCxshVXi5ufXyihtckw8qQ4nLUulrxYgnEbP0TUvZ9fKqK22Mnok+MRm/Dj80W0NHbepvGDQ5h0YRKTLkxiv/OSCLBDUKSNkGj7Tpf1JvvK+xfg7b+toCyvnqMvH4rNLrxx56873y+NieHUm0dz6s2jmXHJYADKt9bz2ZPrkADh6MuH4m41fPr4WqpLmrpdj69decFNbFyfw633XofDbueyc/5MTXVtp3ZFBSW43YY/3nQpvzv7OBZ88wN/uvKu7docf8oMX3W7Z3nuA92X/u2r9prIjTEfGWPu2522YvktsZ0B/ATM9vJ5hxpjxgI/AzftMH8M8C1ws6dvRwN/AGYYY0YCE4Dq39DX3yR3TSl568qZcvQgjjh9FDPPGUtjnZOfvszusn3e+nK+eO1XTrx84nbzjbESjZjEMEZO7U9UfCj2QBuOQFuPx7C7CjZUU7SpljGH92Pq7Aym/S6T5vpWVn5X1O1zYpJDGbp/ImMOT2HwZKtSHdsvlDGHpzDm8BTsgQG4Wgz7zUrF1ot2HGU5DVRsaSJr/2hGHBnP6KPjaWl0s/nHqk5tIxKDyNo/hqz9Y7AFCu5Ww5CDYgmwCwD7n9ufUUd3XaX3t+q8ZmrynfSfEMaAgyIZeEgUrU2GwuX1nRt73qPBUTYShgYTFGkjwC4E2IWS1Y0017rIPDCCzAMjSZsaQUN5K2Ubek/yUZXXTPVWJ6kTIxh4cDSDDo+mtclN/tLOsYbFO0idGEHqxAhsDsHdChn7RxJgk50u6y32lfcvQMGGGoo21TL6sH5MOTGdA0619kur5hd3+5yYfiEMnZrAmMP6MdhTqXZ7jhkj44MYOCGO8Ngg7I4A7IG9Y7+08pe1rF6xnhNPmcn5l/6Oi68+i9qaej794OtObSdOHcu7XzzHBZedxl0P3kB0bBTr12z7TrrrwRu4+KqzfNl9pXapV5QgRCQT+BxYCEwFfgFeAu4AEoGzgBHARGPMVSKSBDwNtJ2buxwoAD4DvgH2B04UkauAowED/M0Y8+ZO+pAFhAN/wkqC//UbQpkPXNPF/O87zL8RuN4YUwBgjGkCnvsNr7XbmhtbcDa1ArB1YwVgJb4AsUnhAJTm13R6ntttePH2bzn8tFEMGJm43bJ+mTGc+vupvPPYYm484Q0kQPi/Ow8lMjakJ0PZJWeTi5Ymq2pVnG1VwyPjrUpxlKdiXFHY2PWTgeVz81n2RT5h0YHMuHgIE4/ZfmjLjx/nIQEw6djuh7z4Smuzm1an9S1amWclfqGeKlxobCAAtSXOna5j7bxyRGDoYXE92NP/TavTjctpJcM1hVY8wZ7qaXC0dcBWX965eheeFMjw42JY80kl39ydDwLjzoonKMJGQ7n1eQiOstYT4llPQ3kL4L/38Hax5ltnhdr61lYxri/rvlIJsHlhNQhkTo/0apmv7SvvX/Dsl5o9+6XNbfsla7hQlOfvzvdLBSz7ooCwaAdH/t9g9puVSkJ6GEdePJivnt/AY+cvQgJg9g2jCIsO7OFoutfY0Ehjg7Ut167aCEByivXd0a+/9XfL5vxOzwsM3Hb24Jelq6mqqGbWCYf3dHeV+p/0igTaYxBwKnAJVhX4TGA6cDxWQvtBh7aPAd8ZY2aLiA0r8Y0BhgIXGGOuEJGTgXHAWCAe+ElE5htjCrt5/TOA14EFwFARSTTGdB44uHPHAp0Hy8JRHfo/CliyOysTkUuw/j945plnuOSSS7zsjmXOv5bz4dM/AxActv2Yx7ZKsnRRjFrwwVrKCmq48PZDyNtQDkBDnZOaCmtH/9XrK0gfGs8Jl03kw6d/5tV7FzB8cv/2pNwfFryezbyXrR13UOgO1XDP2fnu6m4Tj00lIS2MVqebL55dx4f/WMXACXHEesY5l+fXk720nMFTEnrF2OdfPylh2ftW1coRvEPVyewiWKCmuJmCVXWkjokgIsF/X7q7sumratZ/XgWAPajrgLqa21zrYvP8GiL7BzL0qGjWfV7FyrfLiR/cOUHeNnDDv1XZDXMrWTunEgB78I596f6z2qautIXSdY0kjQjtNL55Z8v8YV95/wIsfGMz37xiVVR33C+1vfe6C3W/Y1KJTwul1enmy+c28NFDaxg4PpbAEDs/vL+F5KwIDj0vi29e3sSnj1nLovw0vOzJh1/h4XufBSA8Imy7Zdu+a7rfqJvW53DhadeRlpHCXf/4U891VKk9oDcl0JuNMSsARGQV8LUxxojICiBzh7aHAecCGGNcQLWIxAC5xpjFnjbTgdc9y4tF5DtgEvBRN69/OjDbGOMWkfewkvkn6Pjdur2O878RERfwK56hGh3mJwElO8zfLcaYZ4Fnu3g9r0w7bihDxvcDIDDIxt3nf0ClZ0xzZYn1Nz7Fqki1OK0qiSPQRkVxHbWVTdxy6lvt6/r+0/U4AgMYtX8alSX1HHrqSCYcOoCtGyp474kf2fhLMZNn+C+BHj+zPxmjYwBwBNl49urFVJdaFZHqMutvTD8rgWr1xGr3DDs59JxB7esp2FjDordyKM+rb0+gf/o4D2NgyvG94yKdQQfGkDTU+pKyBwbwyZ0bqa+wqpMNldbftsTC1WJV+myObYnK2nnlYGD4EV1fvNRbpE4OJzbLSghsDmHRI4U0VlkV5KYqaxuGehJCV4tpb1e+oZGmahcZ0yNIHhNGTaGTdZ9WUZnTTGic3fP8Hdfj311i+tRI4gZZ70+bQ5j/j3waK60+Nu4k1jY5C6rBwICDotjRzpb5w77y/gUYNyOlfb9kDwrg+Wt+osazX6ops840bNsvWbG2DcU45OxtF0EWbqzlv2/nUra1gab6VmrKmpl0XBrDpyVSnF3H1y9tJG9VFVGHJPssto5OOfMYJh8wDoDg4CBmH3kRhfnWQVJRQSkAaZkpADQ3W2cXgoKsbbx+TTanHXMZgUEO3vz0aZKSe+8wnP+JoPeB7iN6UwLd8So2d4fHbna/nx0HB+52KUlExgCDgS89R8eBQDZWAl2OVd3uKBYo6/D4UGNMGZ0d6unTv4A7geuAVcB+wLzd7d//KjE1ksTUbads04bE8cPnG0nJiuWbt1YSHOZg4hHWTvovJ7xOXWUjzyy+mMkzskgdFAtA/qYKPnjqZ0ZPS+Ow341qryb899P1RMWH8v2c9QAkZ/j3yzk2JZTYlG3V4eSsCFbMKyQpM5wfPtxCUKiNkQdZXy4Pn7OAhmont30+g6LsWuY+t44hUxJwuwzLvsjHERRA0kDrLgCtLW6Wfp5PdFIwQ6b2jh17ZGIQkYnb7hoRmx5M9uIqYlKDWfN1OY7gADInRQPwzp/W0lTr4rwXrLukuFrdbFhQSVicg7Sx29/pYMuyGiq3Wl/u9RUtrPumnOTh4UQl++cOFWHxDsLit1VMI1MCKVhaT0RyILmLarAHCf3GWtv8m7u34qxzMevBTEI9z9n6Uz3BkXbyf7Z2D2GJdsITHQRGBJCzsBZ7UAB5i2sJibUTP9i/F4Z2irV/IFuX1BGZEkj2/BrswULKeCvp/OqOXJrrXRz/cBYA7lZDrieO5JHbnyHZ2TJ/2Vfev9B5v5Q0MJwV3xSRmBnOjx/lERRqY8RBSQA8et5CGqpbuGXO4RRl1/LV8xsYPCUet8uwfG6BtV8aEE5tufUV+ctXBUTEBfLL19bJ1bi0sM4d8JGMAalkDEhtfzx81GA+fHcuQ4Zn8erz7xAeEdY+NOPg8SdRUV7F+uKFFGwt4tSjL6GqsoY/3Xo5y35aybKfVnLCqTMB+PrzBaxdvQmAgq3F/Odf7zN1+n4MHNQ7ihlq37S3HgZ9jTXuGRGxiUhXA/rmA6d5licABwE/drO+M4DbjTGZnn8pQH/PnTN+AqZ1uLvGRCAI6HwfpS4YYxqxLho8V0RigXuBBzqsL0hEuho33WMuu/cIkjOjeePBRbS2uLni7zMIi+z85dI/K5ZJR2Yx6cgshu5nVQ0SU6PIHJHAgJGJnP7HA2h1unj13gW0Ol2cfeOBpA/tXdWg3908lvi0MOY8uQZXq5vTbxtPSETn09ehUQ6MG75+cQNzn11HdFIIZ901oX389Or5RdRXOZl4TBoBAb3n4quODrkig6h+Qfzw7wLcrW4OuzqDoLCuL+rM+amapppWhh4ah+wQz4pPS/j5TevLuGJLEwtf2Erx+i4u0vOTCeclEJ7oYPX75bhbDftdkIhjx+E6QHR6ECNOjMXdaljxttV21ClxRPUPwuYIYOIFidiDhJXvlRMYbmPihYmd/i/8bdIFSYQnOVjxbhlul2HyRckEdhErQMHyOpx1LjKnRXaKY2fLeot95f0LcOpfxxCfFsZnT63D1Wo47daxhIR33i+FRQXidsO8lzbx5XMbiE4K4Yw7xxEZH0z/oVEcdfkQWlvcfPLoWlwtbo69Zhj9sjrf+s9fHn/xbrIGZ3DnjQ/hbGnhqVfuIyq6c/9ysrdSXlaJy+Xivtse58oLbuLKC7Zdj//UI69y763/BGDNyg3ccNXf+Hnxcl+FoVSXpK2S6NdOWBcRfmKMGeV5/C/P43falgEPsv1FhM9iXUTowkqmC3dYhwAPsBsXEYrIZuBoY8zaDvMeAoqNMfeLyAnAbVgHHHXANcaYpZ52OZ5+le2wzu3mi8g/gRJjzF0icgHwR6wquQFeNMY8tIv/JgPw8q9X7KLZ3u28MU+2T9/z3Sw/9qRn3XTwnPbp//v3OP91xAeeP2t5+/Tx/+x99+Tdkz66etudA056arAfe9Kz3rt8Q/v0vvT+/ds3M/3XER+4+dAv2qfz65f6sSc9r3/YBPDDRQ+jk7PMh2c/4OuX7VFZ/zhliTFm4q5b9i29YgiHMSYH6+K6tsfnd7PsX555xcAJXayq4zoM1h01dnklgjFmQBfzrusw/SHwYTfPzdyd+caYqztMv4R1lxGllFJKKbWX6RUJtFJKKaVUnyeyT//4SF+yTyXQIjIaeHWH2c3GmCn+6I9SSimllNr77FMJtOc2eeP83Q+llFJKKbX30vMISimllFJKeWGfqkArpZRSSvmLABKgtcu+QLeiUkoppZRSXtAEWimllFJKKS9oAq2UUkoppZQXdAy0UkoppZQvCGDz+Q8gqh6gFWillFJKKaW8oAm0UkoppZRSXtAEWimllFJKKS/oGGillFJKKZ8QvQ90H6FbUSmllFJKKS9oAq2UUkoppZQXNIFWSimllFLKCzoGWimllFLKFwTEprXLvkC3olJKKaWUUl7QBFoppZRSSikvaAKtlFJKKaWUF3QMtFJKKaWUT+h9oPsK3YpKKaWUUkp5QRNopZRSSimlvCDGGH/3Qe0e3VBKKaXUniO+fsEx/YeYTy9/zNcv26PSbzl6iTFmor/74Ws6BloppZRSyhcEsPk8b1c9QIdwKKWUUkop5QWtQO9l8uuX+rsLPap/2IT26R+LX/FjT3rW5KRz26ffWHOdH3vS804f/lD79DXvTfZjT3reYyf92D599ksj/diTnvXaBavap098IsuPPel5H1y5qX166l/S/NiTnrf4vrz26es+3N+PPel5D53wvb+7oPZyWoFWSimllFLKC5pAK6WUUkop5QUdwqGUUkop5SP6Qyp9g25FpZRSSimlvKAJtFJKKaWUUl7QBFoppZRSSikv6BhopZRSSikfEBHEprXLvkC3olJKKaWUUl7QBFoppZRSSikvaAKtlFJKKaWUF3QMtFJKKaWUj+h9oPsG3YpKKaWUUkp5QRNopZRSSimlvKAJtFJKKaWUUl7QMdBKKaWUUr4ggE383Qu1B2gFWimllFJKKS9oAq2UUkoppZQXNIFWSimllFLKCzoGWimllFLKJ0TvA91H6FZUSimllFLKC5pAK6WUUkop5QUdwrEP2rB2M9dfeScrlq0hNSOFux78Ewcfvn+ndtkbt/Dnq//GmlUbaXG2MGHSaO599EYyB6YBcMv1D/Dxu19SVlrB4UdN5+V3HvV1KLu0eX0Bd1/3Amt/zaFfWjx/vPtsph4yusu2Fx59B5vXF+ByuxkwOIU/3HEG4/cfxpbsIu69/iU2rsmj1eli1H5Z/PmB80jNTPJxNDuXv6mcZ275ks2rSkjoH8n5fz2UMdMyum3vbG7lL7NfozCnkhlnjuWCmw8D4OojXqCsoKa9XcbQBO57/+we7//uqtjayNdPbqZkUwORiUEcfFE66eOiumzbXN/Kdy9sYfNPVRi3IWFAKCf/bThr5pXx1RObO7U/76kxRCYG9XQIu626oJnFLxVQkdtEWJyDSWcl029UeKd2mxZWsfjFgk7zT3hgEOHxgQC4Wtx8ems2tcVOhhwWw6Sz+/V4/71RW+Rk2b9Lqc5rJiTWwZhT40gcHtqp3ZbFtSx7rbTT/CPvSCM0zsGmb6rZ9E01zTWtBEfZyTo0ioGHdP3+8JfGshayP6ukvthJUJSdjCOiiR4Q3GXbkl/rKfi+Bmedi8jUIAbOiiUwwgZAzldVlK9poLXBTXRWMENPifdlGLtUntfIl49vomRTPZGJQRx6cSYZ46O7bNtU18q3z+eQ/WMlbrchcWAYv7tnJAAvXLyUmlJne9uEzFDOfmSML0JQqltagd4HXXnBTWxcn8Ot916Hw27nsnP+TE11bad2RQUluN2GP950Kb87+zgWfPMDf7ryru3aHH/KDF91+ze55fKnyN1YyO9vPwO73cZNFz9BXU1Dl21HTxrEdX87iwuvPZ4Nq7Zw9x9fBKC0sBLjNlxy/WyOPf1Afpy/iruve9GXYeyWf17/GQXZFZzz54Ow2QN45NpPaKht7rb9e08upqK483YHGDaxP1c/OIurH5zFGX+c3lNd/k2+eHgTlflNHHh+GgE24bN/bKK5vrXLtl89sZn1C8oZcXg8B16QTlQ/K0lJGRnBzGsHMvPagRx5zQAC7EJotJ2wWIcvQ9mlRc9spaaomQmnJRFgFxY8tRVng6tTu6ShoUy7tD/TLu3PARenEGAXgiNthEZvi2fFR6U0VLb4svte+fmlEuqKWxg5O44AG/z0QjEtje5O7eIGBbPf+Ynsd34iE85NIMAOQRE2gqPt1JW0sPLdckRg5Ow43C7DinfKaazs+v3hLxs/rqCxooX0w6KRANj4QTmtzZ1jrSt0svmzSgLDbaQfHEVNXjOb51Zu1yaui4OM3uKzf2ygIr+Rgy7MIMAufPL3Dd1+Vr98fBNr55cx8ohEDr4ok+h+2x9Q9B8Zwaw/DmLWHwcx/bx0X3S/ZwiILaBP/dtlyCJHicg6EdkoIn/pYvkhIlItIss9/27d3ef6016XQIvItyIycQ+v8xcReX1nryMimSKy0jPdtrGXichaEXlwh+eeKCK/epatEJETd1h+vWfZSs9rn7sn49mZlb+sZfWK9Zx4ykzOv/R3XHz1WdTW1PPpB193ajtx6lje/eI5LrjsNO568AaiY6NYvya7ffldD97AxVed5auue23dilw2rNrCkSdO5dQLj+CMS4+ivraReZ/81GX7P9xxJgfOGM+k6SNwBNoJ8FzoMWbSYJ7+4CZOvehI/nj32UTGhLF5Xb4vQ9mlzatLyF1XygHHDGPGmeOYdd4EGuuc/DB3Q5ftc9eVMueVpZx8ReczDwCJqVGMP3gAB8waytjpmT3Yc++UZtdTltPIkOlxjDk6ifHHJeFscLHx+8pObauLmsj+oYoh02M54KxUhh8azxFXDgAgKimIIdPjGDI9DntgAO5Ww/DDErDZe88usSK3kcq8ZjImRzH08FiGz4ilpdHNlp9rOrUNTwgkc0oUmVOisDmseLIOjCbAbv1gQ2VeE2vnVjDmhARfh7FbqvKaqcl30n+/cAYeHEXWYVG0NhkKltV1ahsW7yB1YjipE8OxOQR3K6TvH0GATTDGABAcbSdhWAhBETYC7NL+/9Ab1Bc7aShpIW54KMkTwkmeFIHLaahY29ipbW2edQCcOC6c5IkRhCUFUrWxiZZG6yAq84ho+k3qfEaiNyjJrqc0p4FhB8YzblYyE47vh7PBxYb/VnRqW1XUxMbFlQw9MJ7p56Qx8rAEZlydtV2bqMQgBkyMYeiB8WR2U8VWvY+I2IAngKOBEcAZIjKii6YLjDHjPP/u9PK5frHPD+EQkeFYBxIHiUiYMaZ+N5+6wBhzrIiEAMtE5H1jzCIRGQs8CBxpjNksIgOAL0Uk2xjzq4hcBhwJTDbG1IhIFHBiD4TWrrGhkcaGJgDWrtoIQHJKIgD9+lt/t2zunBAGBm6rXv2ydDVVFdXMOuHwnuzq/6ypoZmmRutU36Y1eQAk9oux/qZYf/NzO5/+BairaWDmyKsAiIgK5aZ/XAiAI3Dbx2TN8s3UVNZz6DF79BjuN2lubKG5yarm5G0oAyA20foyjUuOAKAkr7rT89xuw3O3fsmMM8aRNbrrYSgLPlzN/A9WExkbwunXTufQk0f1RAi7paXZ1V6dK9tiJRltleLwOGt4Qk1x50p7xVbrPV+8sYGnz1qKBMDYY5KYdk7adu1Wzi1FAmDUkf5PLlub3bQ6rVirtloxhcbYPX+tmOtKd15F3vBtJSIw6GDr/W7chh/+VcCQw2KIGxDSU133WqvTjctpJbw1BdZnNiTa5vlrxVxftvPKcc7CWhDInGa93yOSAhlxQiyrP6pg3l1bQWD82QkEeYY8+IurxY27xYq1wbP92oZhtP1tru4cqz3UOqCr3dpMWLKDJk8l3VntwhHi35i6st1nNdc60xcea31GIzyf1eripk7Pq8izPtfFG+p4/PSfkAAYf2wyB563bQja6m/LWP1NGSFRdqafnc6oIxN7NBa1x0wGNhpjsgFE5A3gBGB1Dz+3x+2RBFpEwoC3gFTABtwFDAWOA0KA/wKXGmOMiHwLLAP2AxKAc4EbgdHAm8aYm0UkE/gc+AEYD6wHzjXGbHfuXURmAHcAQcAm4AJjTJ2I3AccD7QCc40x1++k+2cCrwLDPc95fSdtOzHGNIrIcqC/Z9b1wD3GmM2e5ZtF5F7gT8A5wE3AocaYGs/yauBlb17TW08+/AoP3/ssAOERYTv2HwCR7is0m9bncOFp15GWkcJd//hTz3V0D3j1iTk8/48PAAgN32FMoRUq3YUaEhbMY2/+idwNhTz+tzd59oH3ePLdbWeMcjcW8qfzH6FfWjzX33NOD/TeOx+/8DPvPrkYgJCwwO2WtW1Xuoj1u/dXUZpfw8V3DidvvZV4N9Q2U1PRQGRsKIedMoqUATE4m1288fBCnr/9K0ZOSSMx1T/jSJd+UMSPb1njex0h21eI28LsKk5Xi/VF3tLkYuZ1A1nxeQlLPygibUwk6WOtWKqLmshbUUPG+KheMfZ59WdlrPjI2ib24B1ibZvYSTG1tsRJ0Zp6UkaHt4993rSwirqyFqacH92elDsb3TTVtBIc6b8aysYvq1j3WRUA9uAdgtrFZxWgvrSF0vWNJI0IITTOOrhornWR/V01Uf0DGTorhnVzKlnxdhkJQ0IIifFfrIU/1JK/yBouFRDYdaxdiRsWSsny+vZ/bc+VXvpT0D+/V8DiN61iTOAOCf6292/nvrd/VpvdzLp+ML/MKeLn9wtJHxdNxtgoRs1IIqZ/MC6nm4Wv5vHVU9mkjYkkKqnrceOqV+kP5HV4vBWY0kW7/UXkF6AAuN4Ys8qL5/rFntqjHAUUGGOOAfBUVb/sUIZ/FTgW+NjT3mmMOUhEfg98iJVMVwCbRORhT5uhwEWequ6LwBVYlV0864wHbgaOMMbUi8ifgetE5HFgNjDMk7BH76Lvp2FVhIcCV+FlAi0iMcBgYL5n1siO/fT4GbhSRCKACGPMpt1c9yXAJQDPPPMMl1xyiTdda3fKmccw+YBxAAQHBzH7yIsozC8GoKjAqsamZaYA0NxsVYKCgqwv3/VrsjntmMsIDHLw5qdPk5Ts/yrdzsz63TTGThkCQFCwg0uOv5uSAuuUYUmhdZo/Jd2KwdnsqQQFWV++druNKQePYsrBo5j3yU8sWbSGqvJaouMiyF6Xz5Wn3E9goJ0n3/kL8UnRPo6sswNPGM7QCdZ2Cwy2c/vZb1HuGdNcUWyd+m5LelucVuXKEWinvLCWmopG/jL7tfZ1Lfx4LY5AO5fcdSSzL9u2f8pZU8Kcl5dSmFPptwR62MFx9BtmVdbtQQG8+9e11Jdb79P6CutvW/Lb9kVscwS0z0sZHsGgqbE01bSydUXtdtXqlXNLwcDomb2jmjXggGgSBltjWm2BAXx5bw4Nnqpjo2f8cni89X7tGGubDd9WgoHBh8S0z2uoaKG51sWc27YNv8r5vhqbXZh6QUrPBrQTaVMiiM2yEiCbI4CFDxfQWGUNTWj0VGND46yvKJenemtzbEu+chbVgIHM6ZHt88o2NNJU5SJzeiT9xoRRU+Bk7SeVVGxuon+M/4Y6xI8KIyLVej8G2IXV/y7FWWvF6qyz/gZFWbG6W017uwC7MOKsBBpKWxARcr+uoja/meDo3nnyePihCaSMsLaHPTCAt25cRa3ns1rn+Rvl+Vy2et6/9g6f1f4jIhi8fyyNNS3kraihuqgJxkYx5dT+7a9Rkt3A0o8KqSxo2msTaAnonQdA/4N4Efm5w+NnjTHPeqa7CnbHw8alQIanADoL+AArr9qd5/rNnvoUrgAeFJH7gU+MMQtE5GQRuQEIBWKBVWxLoD/q8LxVxphCABHJBtKAKiDPGLPI0+414Bq2T0ynYo2JWeSpngYC3wM1QBPwvIh8CnzSXadFZBJQaozJFZGtwIsiEmOMqaTrjdRx3oEi8itW4n2fMaaobbVdPLdtXlfLuuV5A7a9CX/zmyZjQCoZA1LbHw8fNZgP353LkOFZvPr8O4RHhLUPzTh4/ElUlFexvnghBVuLOPXoS6iqrOFPt17Osp9WsuynlZxw6kwAvv58AWtXW8cCBVuL+c+/3mfq9P0YOMh/F3j0z0ikf8a2ZGjQiDS+/PAHBgztz3svzyM0PLh9+MXvpv2Fqopavs1+lsXfrOCrj35kzKRBFOdX8OvPG4lNiCIqNpzi/HKuOOleaqrqufQvJ7Ny2SZWLtvEjBOn+itMAJLSoklKi25/nD40nu/nrCNtUBxfvvErIWGBTD5yEADXzXqZ2spG/rXkKqYeNYS0wXEA5G0s590nFjN2eiZHnD6GLevLePORhYw9cADuVjcLPlpDYLCdtCH+u7o/KjmYqORtX5RxGSGsX1RBbFoIK74oxRESwKD9rYTx1atX0FTbymX/3o+EgaHEpYewdUUNK78sZfW8MiSA9mTc1eJmzTdlRMQHkjmhd9ylISIxkIjEbWcTolODyP2xmqiUIDZ8W4k9OID0iVaC8vFNm2iua+W0p4YD4Go1ZC+qIjTOQf8x25LFjElRRKda/39V+c2s+LCUfqPCGHxoDP4UFu8gLH7bMLHI/oHkL6kjItlBzsIa7MFCyjjrjNnXd+XhrHNx7EPWGHZ3q2HL4lpCYuwkjdx2EV1bJXrrT3UER9rY+pN1IBme6N+LQ4Oj7dslvaEJDsrXNBAS76BkWR0BgULsUGt4zS/PFdHa6GbSdf0xbkPuvGrCkhzUFTqpyW0meVI4AZ4DicpNjTR6hoQ4a1yU/FJPZFogwX66GDY6OZjoDp/V+MxQ1i0sIy49hF8/LyYwxMag/WMBePmK5TTWtnLVG5NJzAojPiOULb9Ws2JuMau+toZVpQyLoCyngYWvbWHAhGjcbsOab0uxBwYQn9F7L57cB5UZY7ob17gVK69rk4pVZW7XdkbeMz1HRJ70FEl3+Vx/2iNXzBhj1mNVkVcA93quoHwSOMUYMxp4Duh4qNhWAnJ3mG573LaX2TFh7Cop/bLDoPMRxpiLjDGtWONm3sUaW/z5Trp+BjBMRHKwhoBEAid7lpUDHb9hYoGyDo8XGGPGYA09uVxExnnmrwJ2fCNNAFZ73iT1IjJwJ33qcY+/eDdZgzO488aHcLa08NQr9xEVHdGpXU72VsrLKnG5XNx32+NcecFNXHnBTe3Ln3rkVe699Z8ArFm5gRuu+hs/L17uqzB2y11PXkZ6VjKP3v46LS2t3PPslUREhXVqFxkdxqqlm3jwpld547m5jJ08mH+88gdEhK05JVSW1+JyuXny7re55bKnuOWyp/wQzc5d9cDRpAyI5dX759Pa4uKah44hLLJzhSZ1UBxTZg5hyswhjJhkHVglpUcxcGQSETHBuF2Gt//5X15/eCHxKRFc99hx7WOre4OZf8giJiWYBf/Kw93q5ug/DiIorHMtQESYeW0WUclBzH8hl+a6Vo68ZiBx6dYX76YfKmmsaWXkkQm9tiI07dL+RCYHsfTNYtythgMvTyUwtOuxr3lLamiudTHooOjt4onqH0T6xEjSJ0aSNNSKPSIxkLjM3jMeGmC/8xMJT3Sw6v1y3K0w8cIkHN3EWrC8Hmedm4xpEdvFGpMRxMjZsbhbDb++VY671TD61DiiUv0/PKejrONiCYl1sGVeFcYFg0+I6zRkBwCxLiTc/EUlFWsaSZoQRtpB2w72Cn+oJe87K/doKG1h8+eV1OY7O6/HT46+bhCx/UOY/2IurlbDMX8aTHB415/Vo/84iOjkYL55LoemulaO+v0g4jNCCY60Y9zw39e3svCVPCISgjjuxiHtY6tVr/cTMFhEBohIIHA624qoAIhIsngqoSIyGSs3Ld+d5/qTtI+V/F9WIpICVBhjmjx3nDgf2B/IxBoTvRh4xxhzu2cM9PXGmJ9F5BDP9LGe9XyLNYa4DNgMHGCM+V5EngPWGmP+0aFNLrAEOMwYs1FEQtl2dBJqjCkRkVisAeixXfQ5wLOOqcaYfM+8Q4GbjTGHi8hVwCTgfM9QkEeBcmPMnV30+1qsiwLP8CTSb2NdRJjjGc/9FdbBxHIRuQJrbPhpnosII4HTO5zu6I4ByK9fuotme7f+YRPap38sfsWPPelZk5O23XjljTXX+bEnPe/04Q+1T1/z3mQ/9qTnPXbSj+3TZ7800o896VmvXbCqffrEJ7J20nLv98GV20bcTf1L2k5a7v0W37dtuOl1H3Z9h56+4qETvoedXlHQM8ZmDDNzb3zB1y/bo5Ivn75kJxVoPMMyHsHKB180xtztuaECxpinPfnW5VjXrTUC1xlj/tvdc3syFm/sqSEco4G/i4gbaMH6jzgRqyKdg3UU4a01wHki8gywAdiu5GeMKRWR84HXRaStvHAzUAt8KCLBWB+Oa7tZ/0FAflvy7DEfGCEi/bCGTgwDfhERgzWO+cZu1vU0cL2IDPAkyX8GPhYRB9b/xw3GmOWetk8B4cBPItLiWf6PXf93KKWUUmqvJgK7ce/kvsQYMweYs8O8pztMPw48vrvP7S32SAJtjPkC+GKH2T9jJbQ7tj2kw/S3wLc7LvNUbd3GmMt28fx5WFXiHe2yzOV57ak7zHMBHX+e66qdPLdjvxvZdhcOjDHvAe9181wDPOD5p9T/s3ff8TWdfwDHP0+2DBIkkhBJ7L1q1qxN7dGiZtVqabVoraK0tEqrS1tatbp+raK0VFWLKmrvESRGQoYgiciQPL8/zhWZ5KrcG3zfXnndc8/znHO/T87N9b3P/Z5zhRBCCPGAebTeBgkhhBBCCPEf5ctr4WitQ4D79s0NSqlJQM9Mq3/IT7U0QgghhBDiwZAvE+j7zZQoS7IshBBCCCH+s0cigRZCCCGEsDYFKBupnn0YyFEUQgghhBDCDJJACyGEEEIIYQZJoIUQQgghhDCD1EALIYQQQliCAvWIfZHKw0qOohBCCCGEEGaQBFoIIYQQQggzSAIthBBCCCGEGaQGWgghhBDCIhTKRlk7CHEfyAy0EEIIIYQQZpAEWgghhBBCCDNIAi2EEEIIIYQZpAZaCCGEEMISFGAjc5cPAzmKQgghhBBCmEESaCGEEEIIIcwgCbQQQgghhBBmkBpoIYQQQggLUbYyd/kwkKMohBBCCCGEGSSBFkIIIYQQwgySQAshhBBCCGEGpbW2dgwid+RACSGEEPePsvQD1ihdWW+a+a2lHzZPFelVfY/Wura147A0mYEWQgghhBDCDJJACyGEEEIIYQa5jN0D5kr8eWuHkKc8nP3Sli/FH7ZiJHnL27lK2vK/4UutGEneq1usf9ryBzuetmIkee+l+t+nLT/MY00/zpmb21sxkrw3semvacsjf6xjxUjy3sc9dqUtD1xa1YqR5L3F/Q9ZOwTxgJMEWgghhBDCAhSgbC1eei3ygJRwCCGEEEIIYQZJoIUQQgghhDCDJNBCCCGEEEKYQWqghRBCCCEsQQE2Mnf5MJCjKIQQQgghhBkkgRZCCCGEEMIMkkALIYQQQghhBkmghRBCCCGEMIOcRCiEEEIIYREKZStzlw8DOYpCCCGEEEKYQRJoIYQQQgghzCAJtBBCCCGEEGaQGmghhBBCCEtQoGyUtaMQ94HMQAshhBBCCGEGSaCFEEIIIYQwgyTQQgghhBBCmEFqoIUQQgghLESuA/1wkKMohBBCCCGEGSSBFkIIIYQQwgySQAshhBBCCGEGqYF+BJ04HsSLz7/KgX2HKOlfgrfnTKd5yybZ9h0/dgorf1xLZGQUrds257sVi3PVll+cPH6GMc9P4eC+o/j5F+fNORNo1vLxbPtOHjuL1T+uJyoympZtm7BsxSdpbQs+XsbCT5YTcSmSYj5eDBvVj8EjnrHUMHIl+GQYb73yJccPhuDjV5Qxb/WlfrOq2fZ9tt0bBJ8MIyU1lcCyvox+ozc1G1S4a1t+cCkkhu/e2sX5E1co7ONC91dqUqGed7Z93+i6liuX4tPu+5Z159WlrQF4b/BGwkNiSE3ReAcWpMuLNShd09MiY8it+zVWgOTEFGb330DkuVgadS9Dj7G18jx+c0ScjWPl7EOEnryGh7czHV6sSNk62R+Pd5/+i6vhN9Lue5d2Y9SXjdi77gIr3jmUpf/Yb5vi4eOcZ7GbK/rCDf78NISIM/EU9HSg8eCSlKxeKNu+iddvsnXReYJ3XyU1ReNZyplu042/xwO/hHPgl3CuX0nGxcOeGh2LUa1dMUsO5Y6uhiXwz5ehXA5JwLWoPXX7+lC8qluWfqe2XmHbF6FZ1nefU47UFM32r8K4cj6BlBSNZ+kC1B/gS8FijpYYwv2nFMh1oB8KMgP9CBoycBRBJ0/x5tuvY2dnx6B+I4i5FpNj/649Ot5TW37w/MBXOXUymGlvj8POzo6h/cYQcy02x/6de7TLsu7MqbNMfW02NjaKaW+P42ZyMpPHvk3ohUt5GbrZXh/xKWdPXeSlab2xs7Nl4pBPiIuJz7Zv1TpleOXNZ3j25U4EHTnHW2MW5aotP1g6ZQcRZ2Pp8mJ1bO0Uiydt50ZcUo79S9fwpP/0+vSfXp+Oz1dLWx9YtQjdXq5J60GVCA26ynezdlkifLPcr7EC/LboCNcisn8+5AffT99P5LnrtH++Ija2im+n7SchLjnH/gHVPXj69eo8/Xp12g4rb6yrUThtXc+J1bC1V7h6OFDQ08lSw8iVDfPOcCU0gUYD/LCxU/w29wyJ129m23fT/BBObr1MxeZFaTzIj0LeRuJ49WICfy8+j1LQaIAfqSmarYvOExuV8/PD0rZ8eoFrF5Oo09sbG1vF5k/OkxSfkqVfsQouNBlRgiYjStBoaAls7BROhexw9rAn/koyWmuqd/WiTGN3Lh65zvZFYVYYjRAZ5fsEWinVSSk1Po8f42WlVIJSqlC6dQOVUh9n6veXUqq2aTlEKXVIKXVQKbVZKeWfrl8JpdRqpVSQUuq0UuoDpZRDuva6SqktSqkTSqnjSqkvlFIWmR45uP8whw8dpXvPzjw3bADPjxpCbEwsq1f9mm3/t+dM5/lRz5ndlh8c2n+MI4dO0KVnOwYN682wUf2JjYlj7arfs+3/5pwJDBvVL8v61NRUALx9i9H4ifp4FiuKo6MDjo4OWfpay4lDZwk6co5WXerT89mW9B7WluuxN9i0NvukcPQbfWjcuiZ1GlXC3sEOGxubXLVZ24UTVwgLukqtVn407lGWZr3KkXA9mQObLuS4TWFfFyo97kOtViWpWP/27G2Xl2pQuZEv5Wp7YWdvg1L5a1bofo417NRVNn8XRNvBlS0RutnCgq5x6XQs1Vr4UL+rPw2fCiDx+k0Ob875TaqHtzPlG3hRrYUvZesaM9WFfZyp1sKXai18sXOwISVZ81j7Etja5Z/ncGRwPJfP3qBso8JUbetF9Q7eJN1I4fSOK1n6XgtP5My/VynbqDAN+hSnQrOitHg+EABtvCzhUtiBElUL4uxuj629ws4+fzyPL5+9wZVzCQTWL0SFlkWo1LYoyTdSObvrWpa+bp4OBNZ3J7C+O7YOitSbmrKN3bGxU3iWdabtxFJUbFWEen19cXSx5WpoghVGJERG+b6EQ2v9M/Bzbvoq439ApfWtl5Zc6w3sAroCi83Y7gmtdZRS6g1gMjDEFMNPwKda685KKVtgAfAWME4pVQz4Aeiltd5u6t8dcAPyZHooPv4GN+KNjzuPHjkOgI+v8Z+rb3EfAM4Gn8uLh7Y4Y6zGi+uxI0GAkfgC+BQ3bs8G55yAZKdMuUAmTR/NzKkf0LhmJ2xsbHj/sxkU9Sx8HyM3X0J8Igk3jNmm08fOA+Dl42Hc+hq3oWcjs902LiaeNpVHAuBWyJmJc5/NVZs1JCXcJCnBmLUKO23851vIswAA7l7G+87LYddz3H73uhB2/RqCq4cjHYZXpX6nUgDciEtmcrvVABRws6fXxNp5NobcyouxpqZqvpu5m0bdS+NXybrP2fSSElJINo01/EwcAAWLGjPFhUwzxtEXb2S/MbB/Qyj7fgvFxd2B1kPKUftJvwzt/645j7KBOh38ctiD5SQnpnAz0fhv6fI542XepbDxBty1iD0AMeGJWba7csEYf8Sp6yzotw9lA9XaF+PxviXwKO5Eg2eKs/2bUL4ZfRiloPkLARQoZG+JIWXrZmIqN5OMcV49b7wOO3vYZbiNjcz5UwWAk39GoxSUa2Y8V9O/+YkKvkHi9RT8axe877ELYS6rJtBKqQBgPfA3UB84AHwFvAF4Ac8AlYDaWuuRpuTzM6CUaRcjgDBgHfAn0ADoopQaCbQDNPCm1vr7O8RQGnAFxgETMS+BvmU78KJpuTmQoLX+CkBrnaKUehkIVkpNBV4Almitt5vaNfDjPTxmrn34/qfMnjkPAFc31wxtxsOT72bf7tX8979i7sxPAXB1c8nQdnus5u0zKjKaRZ99Q+VqFRgzcThzZ37GpDEzadSsLr7Fs69HtYRln/zKF3NXAeDsmukjamOoOY61gIsTH34/jrNBF/n4ze9ZMPsn5q8Yf9c2a/hj+XF++/IoAI7OGV+yTIcUchhng86l8Crpxs2kVNbMP8j/Zu+hbG0vivi64ljAjhEfNCH8bCxrPj7IuoVHeOHjZnk3kFzIi7EG7Y4g+tJ1erWvnZaUJ1xPJu5KAq4e1itt2PrtGTYtOQWAo7NtxsZbz98ctq3doQSefi7cTErltwUnWD33CKVqFaGwqc75cuh1zuy9TNl6nvmi9nnf6kvs+uEiAPYFMs2Gpx3XrKNNSTYakxNTaf1yKQ6tj2Df6kv4VStIUf8CHFwXQdEAZ+r09GHXDxfZ8uU5SlQpiGsR63w6dvjXSA6sMt602zuZP+sfE57IxaPXKV7NFVfPjGO4djGRTfPOGrXU/XzuS7zWoHh4/r991OWHGegyQE9gKMYscB+gEdAJI6Fdla7vh8BmrXVX08yuK+ABlAcGaa2fV0p1B2oA1YGiwC6l1Bat9cUcHr838C2wFSivlPLSWkeYOYa26eKsDOxJ36i1jlFKnTONtQqwJDc7VUoNxfi98PnnnzN06FAzwzL06tOd+g3qAOBUwIn2LbsTFmr8Oi6GGR+RlgwwZmkSE41ZEEfHB/MEjZ59OlK3QU3AGGvnlv25GBoOwKUw47CWDCgBQGKiMXt7t1KMf7b8y8WwCPo99xRtOzTn2JEgZk//mD07D+DbzXoJdPunGlK9XjkAHJ3sGdrpLSLCogGIuGh8HOxb0vhoOynRmPVxcDRmp+zsbKnXtAr1mlZh09pd7Nl2jKuXY3Ev4nbHNmuo0y6AUtWMcdg72vLh8E1cjTRm5q5FGrN5RXyMN0s3k4wZTTsHIyFrPbBS2n4unLzCX9+eJOJcHEV8XbG1s6F8XW/K1/XmwKYLBO2JIO5qIq7u1nvu58VYr0TEE3clkdn9NqS1715/Fjt7G3pNrJP3g8pBzTbF8a9qfFJi72jLglE7uBZpzFpeizJuPXyM2ffMY32iX5m0/YSdimHb/0K4fP56WgK9a815tIZ6nUpaZjB3Ub5pUXwqGH8/dg6Kn14/wfXLxutPXLTxt1nQy3gdSkk2ZnBt7W1wMyWRvhXdKF3PgxsxNwk9HMu18EQS4m5yPTqZKq09KVXHg+hzN9j5XRiXTsZRpoF1Pmko3dADr3LG89POXrHurWDio43a7vgrxq2bp/EalH6ct5z86wpoKN88Y/xXQxP47Z0QbO0UrccH4uxuvVl2IW7JDwl0sNb6EIBS6gjwh9ZaK6UOAQGZ+jYH+oMxswtcU0p5AGe11jtMfRoB35raw5VSm4E65FwG0gvoqrVOVUr9hJHMf8LteYHM0q//0zQrHoFRwgHGG8zsts1pfY601gswyj8yP65ZAgL9CQhMK9GmcpWK/PTjz1SoWI5FC5fh6uZKp87GyXN1azxB9OVozkcYpR4b1v/BsSMnAAi9cJGli7+lYaP6lC4TeMc2a/EP9MM/8PZHtpWqlGP1j+soX7E0Sxb+D1c3F57s3BKARjU6En35Cqcj/gVg4/otHDeVfYRduMTXi1fQoFFtSpr2t+K7tRTz9uSn738BoFSZAAuOLKvi/l4U9/dKu1+mkh+/r95JYPni/LRkE86uTjzxpFGW8FTD8VyNjuWvMwvY8echNv78L9XqlCE8NJqDu09R2LMQhQq73rHNWooWd6Vo8duP71umEPt+P493YEG2rTyNo7Md1Z4w3hS99fQ6rl9LYvamboSdusovnx2iYgMfUlJS2bXuLPaOtviWLsSxHZfY/8d5AqsW4UpEPMGHonAr7IRLIevWtefFWD2KFcCnlHF6x6XgGNZ/cYQK9b1p2K20VcZ4S2FfZwr73p4d9i7txqFNFykW4MrO1edwdLalchPjDer7/bYSfy2Jqetbc+lMLBsWnqBcPU9SUzT7fgvF3tGGYqWMBPVmcip714fiXsyJcvXzx1VVChVzpFC6q0YU8S9A0LZoCvsV4PCGCOwL2FC6vvFm4uuXDnMj5ibDltfCs5QzRUoW4MKhGI5sjOT4n1EoG/Ap75qWgJ7YEo2zuz0ntxpvnt19rPepgpuXA25et/+GPPycCN55DffijpzYFI29kw3+tY3n4srxQSTGpvDMAuONX8rNVE5vvYJLEXtKVLv9Zv365SR+mxVM4vUUanYvRtTpeKJOxxNY392iYxMis/xwZkX6wq/UdPdTyX2Cn74oMNefjSilqgFlgd+VUiEYyXRvU/NljNnt9AoDUenuPwH4A0eA6aZ1R4AMxZRKqYKAH3Da1P5YbmPMCwu++pAyZUszefwMkpOTWbR0PoXcs7+E0kfzPueNKW8DcOTwMUa/8Bo7t+++a1t+Mf+rdyhdNoBp498lOTmZz5fOoZB79vVz8+d9xVtT5gFw9PBJxr4wjV3b91GjVmWmzhpLUmIyE19+i6TEJGa+N5HK1cpbcCR3N2P+cEqW9uaDad+SnHyTmQtewK2QS5Z+Bd1dOLL3NHMmLuO7hRuoXrcsc5eORil1x7b8ot8b9fH0d2PVhwdISU5l4JsNcHbLmvi6uDuSmqpZt/Awa+cforC3M8++3ZBCngVwKejA2aOXWTF3H5u/D6JU9aIMebdRvhon3J+xegcWokZzP2o090u7TF/R4q74Vcg/9dAAT02uTlE/F36df4yUm6n0mlqTAm5ZZxqdC9mjU+GPRUFsWHAC92IFeGZGrbT66aNbLnH9ahK1n/TDJp9eLqzVS6Vw93Xi7yXnSbmpafNyaRxdsv53p5Si1ehSFCzmyNZF50iIu0nLkYEUKVkAr9IuNOxfgpTkVLZ8eY6U5FSaDC5J0QDrl6zc0nh4CQp5O7Dr20uk3tQ0fcEPBxfbbPue2x1DQmwKZZt6oNIdt9iIJBJiU9CpsPeHcLZ8eoEtn5p3HosQeUFpfc8Tm//9wY0a6LVa6yqm+4tN93+81QbM4XYN9HfADq31PFMJhwtGUpt+H92AYUB7U9tuoJ7WOsvp3EqpWUCM1npWunXBQDMgAdgJ1NdaXzJdfeNroKJptjrEFFeUUsoHOASUA65glKJ8qLVeaorzM9PjjDHNWP8LPKW13ml6zL7AxuxiTEcDXIk/n9tf7wPJw/n27PGl+MNWjCRveTtXSVv+N3ypFSPJe3WL9U9b/mDH01aMJO+9VP/26RYP81jTj3Pm5vZWjCTvTWx6+wpFI3+0XsmLJXzc4/ZVewYuzf4a8g+Lxf2ND74t/bi1ylXVmz9ZZemHzVMFW5fZo7W2/lnYFpYfZqDN8RLwhKm8Yw9GvXFmK4GDGCckbgJevUNi2svUP/P2vbTW4abH+1UptR+YB/TO7gofpvrqb4EXTCcFdgV6KqWCgJMYyfhEU99w0+POMV3G7hjQGMj5QsxCCCGEECLfsGoNtNY6BOOkulv3B+bQtti0LhzonM2u0u9DY1xRY1wuHj9Lsa7W+pV0y6uB1TlsG5Dp/qh0y+eBHL9hxHQFjsZ3i08IIYQQQuQ/D9oMtBBCCCGEEFaVH67CkeeUUlWBZZlWJ2qt61kjHiGEEEI8ghQZTpIUD65HIoE2XSavhrXjEEIIIYQQDz4p4RBCCCGEEMIMkkALIYQQQghhhkeihEMIIYQQIl/IZ1/YJO6NzEALIYQQQghhBkmghRBCCCGEMIMk0EIIIYQQQphBEmghhBBCCCHMICcRCiGEEEJYhJIvUnlIyAy0EEIIIYQQZpAEWgghhBBCCDNIAi2EEEIIIYQZpAZaCCGEEMISFDJ1+ZCQwyiEEEIIIYQZJIEWQgghhBDCDJJACyGEEEIIYQapgRZCCCGEsBCl5DrQDwOZgRZCCCGEEMIMkkALIYQQQghhBkmghRBCCCGEMIPSWls7BpE7cqCEEEKI+8fixci1KlTTf3+51tIPm6dcGvnv0VrXtnYcliYz0EIIIYQQQphBEmghhBBCCCHMIJexe8B8c2S0tUPIU30qz0tb/mz3AOsFkseG116Stjx9U0srRpL3pjTfmLa87PAoK0aS9/pV+Shtec2ZadYLJI91LDUtbfmP8+9ZLQ5LaOH3Stry6ZgtVowk75Uu2CRtOTE23oqR5D1HN2drhyAecJJACyGEEEJYggJs5DrQDwMp4RBCCCGEEMIMkkALIYQQQghhBkmghRBCCCGEMIPUQAshhBBCWIhSUgP9MJAZaCGEEEIIIcwgCbQQQgghhBBmkARaCCGEEEIIM0gNtBBCCCGEpcjU5UNBDqMQQgghhBBmkARaCCGEEEIIM0gCLYQQQgghhBmkBloIIYQQwgKUUigbuQ70w0BmoIUQQgghhDCDJNBCCCGEEEKYQRJoIYQQQgghzCAJtBBCCCGEEGaQkwiFEEIIISxFyUmEDwOZgRZCCCGEEMIMkkALIYQQQghhBkmghRBCCCGEMIPUQAshhBBCWIh8kcrDQRLoR1Do6WgWTtlI8NEIPIu70X9CM6o19M+xf1LiTSZ2/5qLIVdp1bsaAyc9AcD1mESWzvqLvX+dIeWmJqCSJ1OW9LTUMHLlYvBVls/Yzrnjlyns48rT4+pSqb5vtn0ndVpB9MXrafdLlPVg0jcdAfh+zr/s+T2E2OgEqjQqzgvvt7BI/OaIPHud1XOOERYUg3uxArQfVY4ytYtk2/f9Ptu4Gp6Qdt+7tCsjFtQDYO+6MLZ8HULs5UT8q7nTZWxFCno6WWQMuRF6Jpovp2wi5GgERYsXpN+EJlR9vGSO/ZMSbzK5x3dcCrlKy15V6T+pKVtXHWPh639k6Tt3fX88ixfMy/DNcv5UJB+N/5lTRy5SrLg7Q6e2o2bj0jn2T0q8yUtPfkZo8GXa96vD8GntCQu+zCeT1hJyIpybySmUq1GC52c8iY9/YQuO5O7OBoUz99XvCTp0gWIlCjNyeldqNymfbd+RnT7gbFA4qSmp+JctxvApnahWz/i9rPhiCz8t2kJ0RAxFihWi+3NN6DqosSWHclenToQwYeRsDu8/SYmS3kyZ/RKNW9TJ0i/k9AUmvfQuJw6fITk5mRq1KzH9/TH4lyqe1icxIZEODQcTfOo8fYd0Zdqc0RYcSc7+2bGdF19+iZNBQVSqWJFPP/qEmjVqZun389o1jJ80kdCwUOrWqcuC+Z8RGBBAZFQkT3buyKnTp7GxsaFG9Rp8MPc9KleqbIXRCJGVlHA8gj5+dR1hwdE882pjbO1s+XDMr8THJubYf+WnO4kOj8uyfsHrv7PtlxM07VaZfq81wdvPPQ+jvjeLJm3l0tlrdB9dG1s7GxaO38yNuKQc+5etWYxn32rMs281puuoWhnaarcKyONo/5sf3zpM1PnrtBleFls7xf+mHyIh7maO/f2rudNjUmV6TKpMqyFlAAg9EcPPc49RsKgjrYaUIeTAFdZ+cMJSQ8iVT1/dwMXgK/QZ1whbOxs+HrP+js/f1Z/tyvL8rVC7OM/Pbs3zs1szbGYr7OxtKFTEGQ8vl7wO3yxzRq/gwpkoBk9sja2dDe+M/IHrsQk59v/uo81EXYrJsO5yeCypWtP7pWa06F6DA9vO8PGENXkdutlmjlrO+VMRDH+9E3Z2tkwfsZS4mBvZ9q30WAAvvNGFZ15sxamjYcx99X8AXAiO5NPpq1FKMfz1TtxMTuGTqauICLtiyaHc1cuDZ3Dm5DkmznwBO3s7Rg2cSuy1rK+xl8IiSU3VvDhxEN2face2v/Yw8cXZGfp89M4SLoVFWir0XElISKB33z7ExsXx7qx3iIiIoHf/vqSkpGTodyn8Ev0GDaBgQTdmzXiLffv38dzwoWntrVu15oO57zN08HP8ve1vXp0w3tJDESJH+SqBVkr9pZSqbe047kQp9YVSqtId2qcrpVpaMiZzhByL4NyJKBq0L0/r3tVp178mN+KS+HdDULb9z52IZN2yfXQbUT/D+ojz19j9x2keb1eOXqMb0rRrJYa+2coSQ8i18ycucyHoCrVbB9LsqQq0fKYiCdeT2fvH2Ry3KeLrStWGJajTOpBKDW7P8jw9ti4t+uR42K3uYlAsl07HUeWJYtTr4keDHiVJvJ7CkS0ROW7j4V2AcvWLUrW5N2XqGDPVIQeuoDXU7lCc+t388CnjxskdUcRfS7bUUO4o5Fgk505EUb9dOVr2rkbb/jW4EZfErt9PZ9v/3Iko1i/bT9cRdTOs9yxRkPrtylG/XTnsHW25mZxKk64VsbO3tcQwcuX0kYsEHwunSccqPNmvLp0HNyA+LpFt645m2z/4eDg/L9pB7xebZlhfoZYfs74dSIf+dRk6tR1u7gU4F5Tz88Iagg5f4PTRMJ7oXJPOAxrRfUgT4mMT2PrrwWz7j5jSiQYtK1GrUVkcHOywMX0krlM1AEW9C1GrUTkKe7ph72iHg6O9xcZyN0cOnOTYoVN06NGCfkO68uwLPYmLuc661Zuz9K1Vrwrf/voh/Yd2Y8rsl3D3KEjQsZC09uOHT/PV/B94cfxAyw0gF377fQPhEREMe24Iw4YMZUC/AYSEhLB565YM/f73ww8kJiYy7pWxPD98BJ06dGTbP9s4feYMnkU9eeP1qbRt04ZmTYzntI1NvkpZxCPukS7hUErZaa1znqLLhtb6ubu0T/lvUd1/iTeSSUowhnn+5GUACptm2goXcwUg4kJMlu1SUzULp/5Bq17VKVWlWIa2C6eN/Zw5Es6zdeZjY6No07cGvV9plGfjyI2khJtpYw09dRUAdy9n060x5qjQrDM9t+z89TQ7fjmNq4cTXV6oScPOZfM24P8gKSGF5ERjRic82BhTwaJGqUVBT0cArlzMfgYP4MDvF9m/4SIu7va0GFyGx9r74urhAMDZw1fxKedGdOgNtIar4TdwLmSdJCT98/dCkPG88yiW+fl7Lct2qamaRdM20bJXVUpV8cpx/3/+cARlo2jWw/ofDSfeSCbxhvFm5ewJI8ktXMwoKSnqbdyGn8s6m5qaqvl4ws+071uHstWKZ2izd7j9piDoYBixV2/weNuKeRK/ORJuJJF4w/g0KPj4JcBIfAE8fdwBuHjucrbbXo9JoHuNqQC4FizAK+88BYBfaS+eG/8kX77zK4OeeAcbG8XYOb1wL+Kal0O5qxvxCdy4YXxycPLoGQCK+RQFwNvXeG6eDwnLsp2Dw+2/uUN7j3P1SgxtOhnJZGpqKhNHzabvc12pWqtCnsZvrpCzIQD4+hjlcsWLG8/J4JCQTP2MyQxf38z9gildqhSHjxymXqPHjTZfX+a8k3H2/YGkAKmBfijcNYFWSrkA/wNKALbADKA80BEoAPwDDNNaa6XUX8A+4DHAE+gPTACqAt9rrScrpQKA9cBOoCZwEuivtY7P9LitgTcAR+A0MEhrHaeUehvoBNwENmitx+YQ92IgAagMFANe0VqvVUoNBJ4EnAAXpVRH4CNTjHbANK31aqWULfAO0AbQwEKt9UemMY41jfNLoLapfZHW+n3T467VWv+olGoBzDHtdxcwQmudqJQKAZaYfof2QE+t9fG7HYt7tXbRHn76dCcATi4O2XfK5u95y8ojRIXF0PiNFpw3JS7xsUnERMdzM9lI3BJu3GTUu+34/bsDrF20h6oNSlKlQc71qHltw9LD/LLQmLVycsmU8Gljdiq7sQI06lKWYv6FSE5MYdUne/lm1g7K1/amaHG3PIz43m37/ix/LQ0GwNE508zpraHmMNZaT/pStIQzN5NS2fjFada+f5xSNT2o3LQYu9eGsnuN8XNrv3YO1pv5+eWrvaz6dBeQ9Zhq0zFV2RzUrauOERkWy7PTKqQl3vFxScRE36Bg4QIAhJ+/xtGd56nWyD9f1D6vWLCN7z40ZiILuGb8W00bazYHdeOP+4i4cJWRszpx9kQ4APGxiVy7fJ1CRYw3GxfORPHW8O/wKmHUUlvb95/+ybJ5GwBwdnXM0Kb1nZ/ABVwceGf5UM6djmDhzLUsee835nw3gquX41i1+G9KV/al3+jWLJu3gY+n/ETNhmXSknJrWPDBt3z09mIAXNycM7Td6bjeciboHMP7TKJESW+mvvsSAD8uX8eFc5eY+dGrnDAl5bExcVyOukqRou73fxD/QW7GmF2/0qVKs2blanbv2cP0t2Ywd977LJj/Wd4GK0Qu5WYGui0QprV+EkApVQj4XWs93XR/GdABuFVUl6S1bqKUeglYjZFMRwOnlVLvm/qUBwZrrbcppRYBz2Mkmpj2WRSYDLTUWl9XSr0GvKKU+hjoClQwJezud4k9AGgKlAb+VEqVMa1vAFTTWkcrpWYCm7TWz5r2969SaiNG8h8I1NRa31RKZT7jpgZQXGtdxRRzhliUUk7AYqCF1vqkUmopMAKYZ+oSpbWupZR6HiMhzzKzrZQaCgwF+Pzzzxk6dGjmLrnSuFNFytcy3uHbO9oxvf8PaTWht269TMlDcpIx02fvYMflS3HERN9gQvdv0va1be1x7B1safl0VQAq1PKlTqsyxFy5wZGdFwg/f40qDe4pzPuiXvvSlK5uzOg4ONkx57n1XA03Tgy8Gmm8Ryvqa8xGJScZbwJuzdC1e7Za2n7On4jmj2+OEnEuJt8m0NVb+VCyijsA9o42fPnSHmKijFmumCijJtjd20gUbyalArcT4abPBKbt5+KpOLb/eI7LF+Lx8CnAoPcfIyI4DhtbxbpPgjh3+CoePgUsNawsGnWsQLmaxvPXwcmON/uvSHveXokwjq1niVvP39vH9PKlWGKjbzC5x3dp+/pn7QnsHWwZ/EZzAP784TBaQ/Onq1hsPHfSvGt1KtU23oA6ONox/umvuGyqab4cHguAl+lcg+RE09+qox1RF2O4Fh3PS0/eTi7+WnUQewdbRs3qxLmgSCb3XYK9gx1vLu9PYS/rP6dbda9NlTrG89DRyZ7R3T8m8qLxSULUJePWx8942U0yjdXB0fgvy9bOlsealOexJuXZ+utB9v9zimvRcez/5xRRl67RoW8DGrauQvDxiyyes56je87StIO7hUd4W9debahd33jNdCrgyNNtRqbVLYdfNG5L+PsAkJhozMo7OhpvoIKOh9Cv48s4ONqzbM08vLyNcquLoRFER12lQ8Nn0x5n9fe/4+DgwKyPX7XMwHIQ4B8AQGhYKABhYWGm9f4kJCRgY2ODg4MDAf7GyeuhoZn7Gdu7urrSqkVLWrVoyaIlX7Fi5U+SQIt8IzcJ9CFgjlLqHYyZ1a1Kqe5KqVcBZ6AwcITbCfTP6bY7orW+CKCUOgP4AVeB81rrbaZ+y4EXSZdAA/WBSsA20ztRB2A7EIMxq/yFUuoXYO1dYv+f1joVCDI9/q3PuX7XWkebllsDnZRSt2aynYCSQEvgs1slHun633IGKKWU+gj4BdiQqb08EKy1Pmm6vwR4gdsJ9E+m2z1At+yC11ovABbcunuXsebIy68QXn6F0u6XLFeU7etOUrxMEf74/hBOLg7UaWWUKoztsJTYKwks2vU89dqWpURZ48X6wqnL/DR/J9Ua+dPy6aoEVPLCr2wRjuw8z6YfD7N51VFsbBXlavrca5j3hWcJNzxL3E4Oipf1YPfvIfiUcmfLipM4udhTs7nxoj2txyquX01k3pY+hJ66wupP9lH5cV9SUjQ7fz2NvaMtvmU8ADj09wXCTl8F4Ep4PH+vCqJcrWJ4lbTerGVh3wIU9r2d2BYr5crhP8Px9HdJmz2u1MQTgI8Gbif+WjKTfmlG+Jk4Nn55mrJ1i5Caojnw+0XsHW3wCnQlNUWz/tOT+JRxI/REDGf2RtOgR0nsHa1XG5z5+etXrgg71wVRonRh/vjfYZxc7Knd0rgCw2sdlxN7JYGF/w6jXpuylChjPH9DT0ezcv6/VGtYkuZPGcnyzeQUtq4+ThEfN6o3DrD4uLLjXdID75IeafcDKhRj69rDlCzrybqvd1PA1YHH2xq1+CNafUzMlXj+d2gijdpXxr+c8cbxXFAE336wmVpNytCuT20iw64xqc9iYq/doO8rzTm5P5ST+0Np0tG6bxp8/Yvg63/7KjGlKvrw18/7CChXjDXLtuPs6kjjdsab2kFPvM216OusPT6LXZuPs3ntASo/FkBE2FWO7AnBw9ONgh4u+JQ09vfHyj0U8SrIplV7AShRytPyA0ynZKAvJQNvX/2nQpXS/LJiE2UrBvLNl6txcXOmbacmALSu3Y8rl69xMGw9YRcieObJl7h2JZaXJw/mwO6jHNh9lA49WtC+6xOUq2i8AQk6HsKHs76iScu6PDO4s1XGmF6bVq3x8vRkwZdf4ObqxpJlS/D396dp4ya4eBSkUsWK7N25m549ejB52hTmznuPiIgIfl67hscbPE7pUqVYsnwpBw8epFq1ahw+fJjz589Tu9Zj1h6aEGnumkCbZk8fA9oDs5RSGzASwdpa6/NKqWkYSectt06HT023fOv+rcfLnAxmvq8wktzemeNRStUFWgC9gJFA8zuFn8P96+nWKaC71jrDpQaUkbnnmLRqra8opapjlHi8ADwFPJuuy92KnG79blKwcC36C7PbsnDqRr6evZWivm68OLcdLgUds/QrUboIJUob/yG5uRcAdlLMrxCBlY166JHvtmPh1I0snfkXRXzcGD6zNX5li1pyKHf17IzGLH/rH1bM201hHxeem9kEZ7esZSyu7o6kpmrWfH6ApISb+JRyp/OIGrh7Gh+3/r7sCEF7jY/GQ4Ou8PVb2+k/5XGrJtCZ9ZhUmdVzjvHbZ0G4eznR8/WqFHDNWrfsXMgenar5c/EZkhNT8PR3ocWzpSlY1PgdnD14lT1rw7B3sqFulxK0HJzzZdOsYcQ7rfly6ia+efdvivi6MXJO22yfv8VLF6Z4aWMG89iuC4CRjAdWNhLN3RtPExt9g+4j66WdhJbfjH2/Gx9NXMOXMzfg5VuIVz/siWvBrJcULFnWk5JljSSxoIczsBlvfw/KVPXl0I4QrkUbn74sfff2pfusnUBnNvHDvsx97X98NuNnvIp7MHl+f1wLZf3kw83dmeP7zrFp1V7sHe2oUieQIRM7oJSifHU/hr/eiVWL/+bD13+iiFdBRs3oSulK2V+60lre++J1Jo6azcyJn1DcrxgfLp5GQfesnwqcCw4lOuoqAHPeWJC2vkOPFpStEEDZCgEAeGzdB0DJwOJUqZn9pf8sycnJia+XLmf0mJcZ89o4KlWsyPwPP8HWNuMbcR9vH5YuWsyEyZMYP3kidWrXSZth9iziyfoNG1i46EtcXVxp37Yds2e+bY3h3Hd3K2URDwaVVmuWUwelfIForXWCUqoLMBCjBCIAoyZ6B/Cj1nrarfpgrfVupVQz03IH037+wihViAKCgce11tuVUguB41rruen6nMWYmW2utT6llHLGqMEOA5y11hGmkopTWutsL2ZqqkX2wigvCQQ2A2UwEu/aWuuRpn4zgYLAKFNZSE2t9T6l1HCMWehet0o4TCUft2IMwShXiVFK1QAWa61r3KqBNv2cTDeGxcA+rfUHphro2lrrKNNVR+ZorZvd8UCYkvlvjoy+S7cHW5/K89KWP9s9wHqB5LHhtZekLU/flG8v2nJfTGm+MW152eFRVowk7/Wr8lHa8poz06wXSB7rWGpa2vIf59+zWhyW0MLvlbTl0zFb7tDzwVe6YJO05cTY+Dv0fPA5GrXoFs9kH6tcXW//NvMH1g82x+ree7TW+foKankhNzOfVYF3lVKpQDJGHW8XjBKNEIyT48x1DBiglPocCAI+Td+otY40nez3rVLq1tTSZCAWWG2qL1bAy3d5nBMYiXMxYLjpTUDmPjMwyioOmmadQzCS7i+Acqb1ycBC4ON02xUHvlJK3Tq7akKmMSQopQYBPyilbp1EKMVbQgghhBAPuNyUcPwG/JZp9W6MhDZz32bplv8C/srcZroKR6rWevhdtt8EZP1qJqibzbqcbNNaZ0iytdaLMU7uu3X/BjAsm1huAq+YfrKNEcj4TRtG+8B0y39gXGkkc5+AdMu7gWaZ+wghhBBCiPzpkb4OtBBCCCGERcn3wTwULJ5Aa61DgPt29opSahLQM9PqH9LPBAshhBBCCHG/PPAz0Frrt4C3rB2HEEIIIYR4NMgHCUIIIYQQQpjhgZ+BFkIIIYR4MKhH7jrQSqm2wAcYlz7+Qmv9dqb2Z4DXTHfjgBFa6wOmthCMK7ClADfz0+XyJIEWQgghhBD3nVLKFvgEaAVcAHYppX7WWh9N1y0YaGr6grp2GN/AXC9d+xNa6yiLBZ1LUsIhhBBCCCHyQl2ML707o7VOAr4DMnzfvNb6H631FdPdHRhfnJfvSQIthBBCCCHyQnHgfLr7F0zrcjIYWJfuvgY2KKX2KKWG5kF890xKOIQQQgghLEEBNg9dDXRRpdTudPcXaK0XmJazG6zObidKqScwEuhG6VY31FqHKaW8gN+VUse11lvuS9T/kSTQQgghhBDiXkXd4eS+C4BfuvslgLDMnZRS1YAvgHZa68u31mutw0y3EUqplRglIfkigZYSDiGEEEIIkRd2AWWVUoFKKQegF/Bz+g5KqZLAT0A/rfXJdOtdlFJut5aB1sBhi0V+FzIDLYQQQggh7jut9U2l1EjgN4zL2C3SWh9RSg03tX8GTAGKAPNNl/i7dbm6YsBK0zo74But9XorDCNbkkALIYQQQliIevhqoO9Ia/0r8GumdZ+lW34OeC6b7c4A1fM8wHskJRxCCCGEEEKYQRJoIYQQQgghzCAJtBBCCCGEEGaQBFoIIYQQQggzyEmEQgghhBCWoh6tkwgfVjIDLYQQQgghhBkkgRZCCCGEEMIMkkALIYQQQghhBqmBFkIIIYSwAKUevS9SeVgprbW1YxC5IwdKCCGEuH8snsnWrlpD71y5ydIPm6fsyhbZY/rq7UeKlHAIIYQQQghhBkmghRBCCCGEMIPUQD9gfgqaaO0Q8lS3sjPTlr85Mtp6geSxPpXnpS2/v72H9QKxgJcb/Ji2vPrU61aMJO91LjMjbXlz6MdWjCRvNS0+Mm353/ClVowk79Ut1j9t+VL8YStGkve8naukLSfGxlsxkrzn6OZsvQeXqcuHghxGIYQQQgghzCAJtBBCCCGEEGaQBFoIIYQQQggzSA20EEIIIYRFKJSS60A/DGQGWgghhBBCCDNIAi2EEEIIIYQZJIEWQgghhBDCDFIDLYQQQghhCQqwkRroh4HMQAshhBBCCGEGSaCFEEIIIYQwgyTQQgghhBBCmEFqoIUQQgghLEVKoB8KMgMthBBCCCGEGSSBFkIIIYQQwgySQAshhBBCCGEGqYEWQgghhLAUJUXQDwOZgRZCCCGEEMIMkkALIYQQQghhBkmghRBCCCGEMIMk0EIIIYQQQphBTiIUQgghhLAQLecQPhQkgX4EXTgVxfxJv3L68CW8ShRi8OutqNGoVI79kxJvMqbTl4QFR9P2mVoMmdoGgPE9FnP+9GVSU1LxK1OUARNaULlOSUsNI1dCT0ezcMpGgo9G4Fncjf4TmlGtoX+O/ZMSbzKx+9dcDLlKq97VGDjpCTavOsqCyb9n6Tvvt0F4Fi+Yl+GbJTwkhu9n7uH8iasU9nGm28s1KF+vWLZ9Z3Rbx5VL8Wn3fcsWYuySlmn3kxNTmDNgI5Hn4mjYvRTdx9TM8/hz6/ypKD6euJbThy/iVaIQQ15vQ83GpXPsn5R4k9EdFxAWHE37vrUZOrUtYSHRzJ/8C2dPRJCcnEL56sUZPr0dPv6FLTiSuzsbdInZY7/h5KHzeJcozIszelCnacVs+47oMIeQoEukpqTiX86bF6Z0o3r9MgDEXovnoyk/su33w6TcTKFcVT8+XDHagiO5u+CTYbz1ypccPxiCj19RxrzVl/rNqmbb99l2bxB8MoyU1FQCy/oy+o3e1GxQ4a5t+cXJ42cY8/wUDu47ip9/cd6cM4FmLR/Ptu/ksbNY/eN6oiKjadm2CctWfJLWtuDjZSz8ZDkRlyIp5uPFsFH9GDziGUsN447+2bGdF19+iZNBQVSqWJFPP/qEmjWyvo78vHYN4ydNJDQslLp16rJg/mcEBgQQGRXJk507cur0aWxsbKhRvQYfzH2PypUqW2E0QmQlJRyPoPdfWU3omcsMnNACWzsb5r64iuuxCTn2/+Hjv7l8KTbL+vK1SjB4cit6Pt+Q4GPhfDrp17wM+558/Oo6woKjeebVxtja2fLhmF+Jj03Msf/KT3cSHR6XYV3F2sUZObstI2e3ZcSsNtjZ21KoiDMeXi55Hb5Zlk39l/CzsXR+sRq2tjYsmbyDG3HJOfYvVaMo/d6oS7836tJhRJUMbRu+Osa1iBt5HfI9mfvySkLPRPHsxFbY2dny7os/3fH5+/3HW7I8fy+Hx6BTNb1ebEKLbtU58E8wn0z6Ja9DN9v0FxZz7lQ4z0/piq29LdOGLyIuJvvjUrl2IC9O70H/0W05dSSUd8d9k9Y2e8zXbFy5myd71WfktO4UD/C01BBy7fURn3L21EVemtYbOztbJg75hLiY+Gz7Vq1ThlfefIZnX+5E0JFzvDVmUa7a8ovnB77KqZPBTHt7HHZ2dgztN4aYa1lfY2/p3KNdlnVnTp1l6muzsbFRTHt7HDeTk5k89m1CL1zKy9BzJSEhgd59+xAbF8e7s94hIiKC3v37kpKSkqHfpfBL9Bs0gIIF3Zg14y327d/Hc8OHprW3btWaD+a+z9DBz/H3tr95dcJ4Sw9FiBxZNYFWSnVSSj1QfxF3i1kpVVsp9aElYzLHmaOXCDkeQaMnK9Gu72N0HFSX+LhEtq8/kW3/kOMRrF28i6dGNcrSNnBCC2o/UYaqDQKwd7BD5bNrW4Yci+DciSgatC9P697Vade/Jjfikvh3Q1C2/c+diGTdsn10G1E/w3qvEoVo0L48DdqXx97RlpvJKTTtWgk7e1tLDCNXLpy4SljQNWq18qNR99I07V2WhOs3OfDnhRy3KeLrQsXHvanZyo8K9b3T1oedusbm74JoPTj7mU5rOnPkEiHHw2ncoTLt+9am06B6xMcl8s/6Y9n2Dzkezpqv/qXXqCYZ1leo6cdb3/TnyX51GDKlDW7uBTgfFGmJIeRa0OHznD4aSosuj9F1YBOeGvIE12MT2PzL/mz7vzC1Gw1aVaFWw3I4ONihbIy/x7CzUWxdf5DmnR9jyPhOtHu6Pq/NzR+zlLecOHSWoCPnaNWlPj2fbUnvYW25HnuDTWt3Zdt/9Bt9aNy6JnUaVcLewQ4bG5tcteUHh/Yf48ihE3Tp2Y5Bw3ozbFR/YmPiWLsq66dcAG/OmcCwUf2yrE9NTQXA27cYjZ+oj2exojg6OuDo6JCn8efGb79vIDwigmHPDWHYkKEM6DeAkJAQNm/dkqHf/374gcTERMa9Mpbnh4+gU4eObPtnG6fPnMGzqCdvvD6Vtm3a0KxJU4B8dyzFo82qJRxa65+Bn3PTVxnZmdJap97PGJRStlrrlLv3NNwtZq31bmD3/Yjtfkm8kUziDWMm8twJI0koXMwNgCLexm34+atZtktN1Xw66VfaPlOLMlV9srTHxyYyqP4HALgUdOL5t9rnRfhmSbyRTFLCTQDOn7wMQGHTTHHhYq4ARFyIybJdaqpm4dQ/aNWrOqWqZF/2ALDpf4dQNormPbP/aNmSkhJukpRgPHUvnrkGQCHPAgC4m26jQ6/nuP3udWfZ9etZXN0daT+iMvU7BpKaqvl+1h4adS9NyYr5o5wh/fP37MkIIPfP308m/UK7vrUpUy3j89fe4fabn1OHwoi9eoMGbaz/MX/CjSQSbiQBcOZ4GABFvQsB4OnjDsDFc1HZbhsXc4Mu1SYA4FqoAOPe7QNASJAxI3niwDnalhuDja0NPZ5txrBJnfNsHLmREJ+YNtbTx84D4OXjYdz6GrehZ7N/UxMXE0+byiMBcCvkzMS5z+aqzVri429wI974lOTYEeMNvLev8TrjU9y4PRuc85vd7JQpF8ik6aOZOfUDGtfshI2NDe9/NoOintb/uw05GwKAr48vAMWLFwcgOCQkU7+zRj/fzP2CKV2qFIePHKZeI6O0pbivL3PemZ3Xoec9ZfoRD7w8ezunlApQSh1XSn2hlDqslPpaKdVSKbVNKRWklKqrlBqolPrY1L+YUmqlUuqA6edx0z6OKaXmA3sBP6XUu6b9HVJKPX2Hx2+mlNpi2udRpdRnSikbU1ucUmq6Umon0EAp1Vcp9a9Sar9S6nOllK2pX1ul1F5TPH+Y1qWPuacplgNKqS3pHnetabmwUmqVUuqgUmqHUqqaaf00pdQipdRfSqkzSqkX8+o4AKxauINB9T9gUP0P+GL6hgxtWt/6fWXdbtOKg0SEXqNpl6pEhxsfL8bHJnIt2vhY1cnZgSlf9WLw5FYkJ97kuw+35uUwcmXtoj0Mb7yA4Y0XsHjmX9l3ymasW1YeISoshsadKnAlwijhiI9NIib69kfI4eeucmTneao39M8Xtc+blp9kSvu1TGm/lp/m7s/Qpu90YIH6nQLpP6MefabUwdbehh9n7+Ny2HX+/SWEKxfjqd3On2uRRqlAQtxN4q7kXPaS135a+A/9671H/3rvseCN9dn2yW6Uf6zYT8SFqzzRpSrRphKO67EJXLt8+01F6JnLzBzxg1FLPaVNXoRvlm/nb6RLtQl0qTaBDyb/mKHt1iHN6ZgWcHFkzjcv8OL0HiQl3OSrOUZJSnKi8YbyRnwSU+cPomrtUnz76UZ2bz2eZ+PIjWWf/EqbyiNpU3kk705clrHxzk9fCrg48eH34xjzZl+SEpNZMPunXLVZy/z3v6KKfxOq+Ddh0piZGdpu/a2a+wFeVGQ0iz77hsrVKvDV9x9QqWp5Jo2ZSVio9Us4Mrs9xjsPMnO/0qVKs2blaqZOnkLYxYvMnfd+3gYqhBnyega6DNATGArsAvoAjYBOwERgVbq+HwKbtdZdTQmsK+ABlAcGaa2fV0p1B2oA1YGiwC6l1Bat9cUcHr8uUAk4C6wHugE/Ai7AYa31FKVUReA1oKHWOtmUrD+jlFoHLASaaK2DlVLZva2fArTRWocqpdyzaX8D2Ke17qKUag4sNcUPUAF4AnADTiilPtVaZyhYVUoNNf3u+Pzzzxk6dCj3ommXKlR4rAQAjk72TOq9jMvhxizsrcTYq4QRfnKS8Z+tvYMdly/GEBMdz5hOX6bta8vPR7B3sOP5me2xtbOhesNAqjcMZPtvxzm84ywx0fEULOx8T3HeD407VaR8LWM2w97Rjun9f0irab5162VKfjOM9VIcMdE3mND9dt3otrXHsXewZch04+S6P344jNbQ4mnrzz4D1GlXklLViwBg72jLR8M3c9VUt3wtypjtKuxrHIubScZMtZ1p5rXVwNuzraEnr7L5uyAiz8VyNfwGcVcTmdN/Y1r7nt/OYedgw9MTHsv7QWXjiS7VqPSYHwAOTvZM6LUkrab51q2XnztwO1m0d7Qj6mIM16LjGd1xYdq+Nq8+jL2DHSNnduB8UCSv91+OvYMtM5b2pbCXmwVHlb02PepSta5xQq+jkwOjur5P5MWrAERdMm59/IxjnpRovFw4ONoDYGdnS+0mFajdpAKbf9nPvn+CuBodh7ef8dJVrV4pmrSvwdXoOPZuO0nY2ShobMHBZdL+qYZUr1cOMF6XhnZ6i4iwaAAiLl4BwLekUaud3VjrNa1CvaZV2LR2F3u2HePq5Vjci7jdsc1aevbpSN0Gxgl0TgWc6NyyPxdDwwG4FGZ8qlIywHiNTkw0ZuXvVorxz5Z/uRgWQb/nnqJth+YcOxLE7Okfs2fnAXy7ed9x27wW4B8AQGhYKABhYWGm9f4kJCRgY2ODg4MDAf7GCd2hoZn7Gdu7urrSqkVLWrVoyaIlX7Fi5U8smP+ZBUciRM7yOoEO1lofAlBKHQH+0FprpdQhICBT3+ZAfwBTScU1pZQHcFZrvcPUpxHwrak9XCm1GahDziUV/2qtz5ge/1vT9j8CKcAKU58WwGMYyThAASACqA9s0VoHm2KKzmb/24DFSqn/AdlNczQCupu236SUKqKUKmRq+0VrnQgkKqUigGJAhs/wtNYLgAW37uYwxrvyLumBd0mPtPv+5b3Y9ssx/Mp48tu3eyng4kCDNuUBGNVmAbFX4vl6/1geb1+RkuWM/8DOB0Xy/Ud/U7NxKdr0qcm+rWf4Z90xytcsweWLMZzYF4p7URfcPArca5j3hZdfIbz8CqXdL1muKNvXnaR4mSL88f0hnFwcqNOqLABjOywl9koCi3Y9T722ZSlR1khMLpy6zE/zd1KtkT8tTcnyzeQUtq4+ShEfN2o0CbT8wLJRpLgrRYq7pt33KVOIfRvP412qIP/8dAZHZzuqNzM+Ep3VawPXryXy9h9dCDt9jV8/O0zF+t6kpGh2rz+LvaMtPqUL4V7MGZ9SxhuMS8Ex/PblMSrUL8bjXXO+Sktey/z8Dajgxd+/HKFkWU/Wf7PH9Pw16rWfb/MpsVfi+e7AazRqVwn/sl4AnDsVyXcfbqFWk9K07fMYkRevMbnvMmKv3eCZl5tx8kAYJw+E0biDdc/w9/Uviq9/0bT7pSv6smn1XgLK+bB62d84uzrRpH11APo1fZNr0XGsPzmXf/86xl9r9lK5dikiwq5weM8ZPDzdKOThQiEPF0pV8GXv3ydZ+/U21n2/ExtbG6rUtt4xBSju70Vxf6+0+2Uq+fH76p0Eli/OT0s24ezqxBNP1gbgqYbjuRody19nFrDjz0Ns/PlfqtUpQ3hoNAd3n6KwZyEKFXa9Y5s1+Qf64R/ol3a/UpVyrP5xHeUrlmbJwv/h6ubCk52NN+qNanQk+vIVTkf8C8DG9Vs4bir7CLtwia8Xr6BBo9qUNO1vxXdrKebtyU/fG584lCoTYMGRZa9Nq9Z4eXqy4MsvcHN1Y8myJfj7+9O0cRNcPApSqWJF9u7cTc8ePZg8bQpz571HREQEP69dw+MNHqd0qVIsWb6UgwcPUq1aNQ4fPsz58+epXcs6b+KFyE5eJ9DpP/dNTXc/1YzHTl/EaW7lUOak89b9hHR1zwpYorWekL6jUqpTNttn3JnWw5VS9YAngf1KqRqZumQX7619pv/dpGDBevTR73Xi00m/snjWH3gWL8iYD7rgUtApSz+/MkXxK2P8Z34rMS5W0p3SVXw4dTCMoANh/L3mKHYOtlR4rAT9xz2R704kfGF2WxZO3cjXs7dS1NeNF+e2w6WgY5Z+JUoXoURpI4F2cy8A7KSYXyECKxv1ibt+P0VM9A16jGqAjU3+GuMt/abV5ftZe1j94UEKezvT/816FHDLOovlWsiB1FTN+i+OkpSQQrFAN9oPrUwhzwIU8iyAd6CRQLvsjQSOUaS4C34VPLLsx1peea8rn0xcy6KZv+NVvBDjPuyOa3bP37Ke+JU13gC67TSev94lPShTxYdDO0PSSpGWzfkzbRtrJ9CZTf54IO+O+4b501dSrLgHUz8dhFuhrJ/wFHR35uj+s2xctQd7Rzuq1inN8Emd0/4eX//E2M8HU36kmK8HE+f1o1QFX0sP545mzB/OW2MW8cG0b/EuUYSZC17ArVDWK90UdHfhyN7TbFi5HXsHe6rXLcuo159GKXXHtvxk/lfvMOaFqUwb/y4lSvry+dI5FHLPvixs/ryv2L7VOK3m6OGTjH1hGvM+m8HT/bowddZYFn32LRNffotiPp7MfG8ilauVt+RQsuXk5MTXS5czeszLjHltHJUqVmT+h59ga5vxxGsfbx+WLlrMhMmTGD95InVq10mbYfYs4sn6DRtYuOhLXF1cad+2HbNnvm2N4dx/+ez5KO6NSquVvN87VioAWKu1rmK6v9h0/8dbbcAcoLbWeqRS6jtgh9Z6nqmEwwUonGkf3YBhQHtT226gntY6S9GXUqoZsI7bJRzrgAVa6xVKqTittaupXyVgNUYJR4SpVMMNiMeou04r4dBaRyulBqaLubTW+rRpP/uAQYA7MFZr3cF0NY5IrfUMUzzva61rKqWmAXFa6zmmbQ8DHbTWIXf4lWqAn4Im5uK3/+DqVvZ2feA3R0ZbL5A81qfyvLTl97f3sF4gFvByg9u1vKtPvW7FSPJe5zIz0pY3h35sxUjyVtPiI9OW/w1fasVI8l7dYv3Tli/FH7ZiJHnP2/n25SwTY7O/hODDwtHNGaxwOl/t6jX1znWbLf2wecqueKE9Wuva1o7D0vLTNWFeAp4wlXfsAbKbCloJHAQOAJuAV7NLntPZDrwNHAaCTdtnoLU+CkwGNiilDgK/Az5a60iM+uOflFIHgO+z2f+7ppMZDwNbTHGlNw2obdrv28CAO8QqhBBCCCEeAHlWNmCaTa2S7v7AHNoWm9aFA9ldVyn9PjQwzvSTG/Fa6yxX6rg1+5zu/vdkkyBrrddhzFynX7c4XczdsnnMv0w/t+qms4xJaz0t0/0qmfsIIYQQQoj8KT/NQAshhBBCCJHvWfWLVO4HpVRVINNFREnUWtfDNBMshBBCCCHE/fLAJ9Cmy+TVsHYcQgghhBDi0SAlHEIIIYQQQpjhgZ+BFkIIIYR4EGhAy2WgHwoyAy2EEEIIIYQZJIEWQgghhBDCDJJACyGEEEIIYQapgRZCCCGEsAgFSoqgHwYyAy2EEEIIIYQZJIEWQgghhBDCDJJACyGEEEIIYQapgRZCCCGEsBQpgX4oyAy0EEIIIYQQZpAEWgghhBBCCDNIAi2EEEIIIYQZpAZaCCGEEMISFFID/ZCQGWghhBBCCCHMIAm0EEIIIYQQZpAEWgghhBBCCDNIAi2EEEIIIYQZ5CRCIYQQQggL0XIS4UNBZqCFEEIIIYQwg9JaWzsGkTtyoIQQQoj7x+JzwY/VqKV3/L7Z0g+bpxy8Cu7RWte2dhyWJjPQQgghhBBCmEFqoIUQQgghLEVJEfTDQBLoB8zVNcetHUKecu9YIW059vfTVowkb7m1Kp22fGPnBStGkvcK1CuRtpwcEWvFSPKevZdb2nJibLwVI8lbjm7OacvxN+KsGEnecy7gmrZ8Pm6XFSPJe36uddKWn5wXaMVI8t4vo4OtHYJ4wEkJhxBCCCGEEGaQBFoIIYQQQggzSAmHEEIIIYSlSAn0Q0FmoIUQQgghhDCDJNBCCCGEEEKYQRJoIYQQQgghzCA10EIIIYQQliI10A8FmYEWQgghhBDCDJJACyGEEEIIYQZJoIUQQgghhDCD1EALIYQQQliCAq2kCPphIDPQQgghhBBCmEESaCGEEEIIIcwgCbQQQgghhBBmkARaCCGEEEIIM0gCLYQQQgghhBkkgRZCCCGEEMIMkkALIYQQQghhBrkOtBBCCCGEpchloB8KMgMthBBCCCGEGWQGWgghhBBCPBKUUv2Bgdz5swANKK31Ezl1kBnoR8yOo3tpOKozxbpVo+nobhw4fSTHvlHXoin9TAM8OlXko5WL0tYPePslAnrXw6NTRcZ9NsMSYd+THUf20GDYkxR9siKNn+/E/qDDWfpEXb1Mw+Ed8e5YFd/O1Wk3pjdHg08CoLVm2pfvUq7X43g+WYnHnm3Nir9+sfQwcmX7gV3UfaYV7o1L06B/O/YdP5SlT+SVy9Tr14aizcrh1bwCrUf04Mjp42ntfSYMw7dVFZzr+/HynMmWDD/X/vl3B7WaNcDVryh1WzZm38H92fZ7enA/vMqVxKFYQV6aMCZt/Y5dO2nyZEs8y/rhWdaPp57tS2RUlIWiN88/O7ZTu0FdChb1oH7jx9m3f1+2/X5eu4ZK1atSyLMwrdq3JTgkJEN7ZFQkxQNK4lTQhfc/nJf3gd+j48eP0/yJFni4F6FG9Zps3PhHjn23b99O3br18XAvwuMNGrFv334ATp06Rbu27fErUZJiXj506NCJM2fOWGgEuRd0PJiuLQZTpkhDmtXswZY/dmTbL/jUOZ5uP4JqJVtS0acZfTqNJOTMhbT2KePmUKtUW0q61WVgj5ctFX6uxYUn8c8HF1k/NoTNMy8QefxGjn2DN1/jz+nnWT82hD9nXCBkS0xa25GfLrPx9XP8+nIIuxaGWyJ08XBbhZFAD7jDz632HEkC/QhJSEpkwNsvEXfjOm89N57Iq5cZ8PZoUlJSsu0/fuFMEpISs6x3sHegQ/2WeR3uf5KQlEjf6S8QF3+dWcMmEXHlMv1njMp2rK3qNGHuqDcY3KEP2w7tYsLnbwHw595tvPf953gX9mTGkPFcvBzOiDmvknwz2dLDuaOExAR6TxhGbPx13nlpChFXInlm4vBsx9qmwRPMG/cWQ7r24+99Oxn/wfS0Nkd7Bzo2bWPJ0M2SkJDA08/2JS4ujjnTZxERGUGvwf2zHaejgwOd23XIsv7kmVMUKVyEma+/QdsWrVj1y89MmPG6JcI3S0JCAr379iE2Lo53Z71DREQEvfv3zTLWS+GX6DdoAAULujFrxlvs27+P54YPzdBnzKvjuJGQc+KSXwwc8CwnTwbx9tuzsLO3p1/f/ly7di1Lv4SEBPr07ktcbBzvvPM2ERER9H2mHykpKYSFXSQ1NZVJkybRr19f/tz0Jy88P9IKo7mzUc9O5lTQWV6fNRo7eztG9J9AzLW4LP0uXYwkNTWVVyYNpWffDvz957+8NvKtDH069mhlqbDNtm9pJNcjkqnQpTDKVrFvSQTJN1Kz9LsemcyxVVdAKSp0KYxO0RxdGc2NKzfT+vjWdLFk6HlLqYfr5wGjtY7RWp/TWp8DigJO6e5fBh4DfE33cyQJNKCUqqmU0kqpNunWBSilDmfqN00pNda0vFgpFayU2q+UOqCUapGun4NSap5S6rRSKkgptVopVSJdu7dS6jtT+1Gl1K9KqXJ5Pc6Ne7YQcTWKwe1781z7PvRt1Z2z4Rf4+/C/Wfr+vmcLv+36k5e6D87StnDMu/Rq3jmvw/1Pfv93MxFXoniu4zMM6dSXfm17EnLpPFsP7szQr6h7EV4f+Apt6jalSY36ANjYGH8Wqdp4oQ/0LUnzWg0p6OKKawEXbFT++rP5bfufRERHMrR7f4b1GMCAjr0ICTvHlr3bM/Tz9CjC1GHjaPN4c5rWfhwAZXN7LF9N/4hn2vWwaOzmWL/pd8IjIxg26DmGDxrCwD79CD4XwuZtW7P0Xfrpl/R9qneW9b269mTlsu8Z0v9Z5s/5AICjx4/leezm+u33DYRHRDDsuSEMGzKUAf0GEBISwuatWzL0+98PP5CYmMi4V8by/PARdOrQkW3/bOO0adb1tw2/8ev6dYwZ/Yo1hpFr+/cf4NChQ/Ts2YNhw4cyatRIYmJiWLVyVZa+G37bQEREBEOGPsfQYUMYMKA/ISEhbNmylfr16/HbhvUMHzGMOXPfpXBhD44ey1/H9/CBExw9FETnHq0ZMLQnQ0b2ITbmOr+uyjrj/li9avyw/nMGDnuK6e+Oxd2jICeP3Z5Rn/7uWJ57IevzPD+4diGR2LBkfGq6ENCoIIHNCnIzQXPpwPUsfbU2bp0K2VK0XAEc3GyxsQMbeyM5q9ytCAHNCloyfPHo+AhIBVBKKeBvjNnnuUqpSXfaMH9lAveBUsr2HjbrjfFLM/eVaJzWugYwGvgs3fqZgBtQTmtdFuPjgp+UCbAS+EtrXVprXQmYCBS7h7jNcjY8FACfIsZD+RbxBiDk0vkM/eJuXOeV+dN4vd8rlPD0zeuw8sRZ05h8ihpjLV7UNNaLWd9QHgk5Qamn6tF90mB8ixbjneFG+UKLxxozpFNfVm5ZR+3n2hAdc5UvJ7yPre29PMXyztkwY6y+nqaxevkAEByadayHTx/Hv10NurzcH19Pb94dPc1icf5XIefOAuDrbYyvhE9xAILPhuR6Hw4ODmnLG/40EpbGDRrepwjvnxDTmHx9jL+/4sVNY81UnhFy1vQ78c3cL5i4uDhGvvwSM6a+gZ9fCfKb+Ph4oqKiiIqK4sgRo5TM19c4tmnjCA7Jsl3WMRu3IcHBGY7v3j17iY6+QqOG1j++N+ITiI66SnTUVY4fOQWAt48nAD7FvQA4FxKaZTsHB/u05QN7j3L1Sgx1G9a0QMT3JiUplaS4FJLiUoi9aHxS51TI1nRrnHIVf/lmlu1cvewp38GDKyGJbJkVSkxoElWeKoqja/56rRUPpUJa61Om5cZAAa11Z6Apd8kJLZ5Am2Z2jyulvlBKHVZKfa2UaqmU2maara1r+vlHKbXPdFvetK2tUmqOUuqQUuqgUmqUaX2IUmqKUupvoKdSqrepz2Gl1Dt3iUcBPTDecbRWSjndw7C2A8VN+3MGBgEva61TALTWXwGJQHPgCSBZa52WcGut92uts06j5TGN8bZfZfoI5oMVX+DsWIDmNR8n8uplAKJjrnI1LuvHqQ8KrbMfK0ApX39WzvyKyQNe5uLlCOb9sBCAoPNn+N8fq2n+WCO+njIfL4+ijJjzKtdvxFs0dnPdaaylSwTw8wfLmTJ0LBejwnlv+aeWDu++yen5mxv//LuDoaNfoFb1mrw+bsL9Du2+u9MxzanfnPffw7mAMy2btyAyMhKAy9HRXLlyJW+DzaX335uHf8lA/EsGMuaVsRnacjteoy9Z+p48eZKnnuqFv78/c+bOuX9B36NP5y2lRmBragS2ZsrYjPHkZqynT57luV7j8PP3YfqcsTn2s7bTm66x8fXzbHz9PEdXXM6+UzbDTIxL4ezWGAr6OvDYs14U9HXg6IrL3LiaNdkW4j5L/yRrDvwBoLW+CWRf32pirRnoMsAHQDWgAtAHaASMxZiNPQ400VrXBKZgzOgCDAUCgZpa62rA1+n2maC1bgRsAd7B+EXUAOoopbrcIZaGQLDW+jTwF9D+HsbTFmOW+dbYzmmtYzL12Q1UBqoAe3KzU6XUUKXUbqXU7gULFtxDWBn5FzNmdcKiLgFw8XK4aX0JEpISSUpOAiA06hInL5yhzoj2TFsyF4B5KxbyxS/f/OcYLMXf2w+4PdawW2P19sswVgDXAi60qN2YV595gRKePqzc/CsAv+74g2vXY+nVogsdG7WmWc3HCYsK5/i5U+Qn/r7GWEMjTGONNG4DfP1ISEzIOFZnF1rWa8r4Z1+iRDFffvpjreUDvkcBJf0BCL0YluE2oKQ/CQkJJCUl5bhtelu3b6NDr26UCgjkl+9+wtXFNW8C/g8C/AMACA0zZiXDwkxj9c841gB/0+8kNHO/AC6EXuDEyRNUfawGk6YYdd5z3pvLZwv/+2vJ/dDnmd6sWfsza9b+zMpVKwAIDTXiTxtHgDG+xMREEhON8zFyGrN/QAAAx44dp03rdtjZ2fHrurX4+HhbZkB30L33k3z988d8/fPHLFkxD4CLYREAXAoz3tz4+Rsz6YmJSSQm3n4unzx+hqfaDcPOzpZv186nmHdRywZvhhK1Xak7vBh1hxej9lDjE7GEaymmWyNPcS5szESn3NSk3DTePEQHJZBwLQXvas4Uq+qMdzVnbiZqroZkPQdHiPvslFJqhlKqO0aO+T2AUqowd0mgrXUZu2Ct9SEApdQR4A+ttVZKHQICgELAEqVUWYxLidz6HKsl8JnpnQFa6+h0+/zedFsHozwi0rT/r4Em3E5wM+sNfGda/g7oB/xketzspF//rlJqNuAF1DetUzlse2t9rqfLtNYLgFv/2+UUT661fKwJnoWKsGjdd7gWcGH57yso6VWcRlXqUrRrFSqULMP2j9cw5MlnaFOnGQB/H/qXL379hl5PdKZTQ6NE/Ketv7LvlPGR64nzp1m64Qda126Kd2Gv/xrifdOqblM83Yvw5ZpvcC3gwrL1P+BfrASNq9XDo115KvqXZefCdSz/7UcOnj5GtdIVOXzmOOcjwqhVvhoAgT4lAfhy7TckJCWwfuefONjbE+Cdvz4Ob9PgCbw8irLwp2W4OruwZM13+Pv40aRWA9waBlCpVDl2f/MHS9d+z8GTR6lWrhKHTx3j/KVQHqtUPW0/P/7+M3uPHwTgeHAQX63+lrYNm6eVwVhb2+at8CrqyYLFX+Lm6srib5YR4OdP04aNKeDrQaXyFdm/xahx/9+qFew9YFy14tiJEyxavoR2rdpwKfwSHXt3R6MZ3HcAGzf/iYuzCx3atLPm0LJo06o1Xp6eLPjyC9xc3ViybAn+/v40bdwEF4+CVKpYkb07d9OzRw8mT5vC3HnvERERwc9r1/B4g8cpXaoUI4YOp31bY1xbtm7ls4Wf80zvPnTr0sW6gzMJDAwkMDAw7X6VqlX48ccVVKxYkYVffIGbmxuduxjnWtSoXovLly8TEXmJ1m1a4+nlyRcLvzR+N0uW4u/vT5Mmjblw4QJt27bjSvQVpkx9nV27drNr12569rRubb9/YHH8A4un3a9YpQw/r/idchVLseyLFbi6udCuc3MAnqjVk+jLVzl+aTNhF8J5ut0Irl6JYezrw9m/+wj7dx+hU4/WAPyx/m9OHD0NwMXQcL5dvIr6jWoRWKak5QcJOBe1x7no7bITN197Lu67jqu3Pee2xWLnqPCu7gzAlpkXSLqeSpt3/ClQxEhFQvfE4VjQltA9Rp20i6exr4gj8cReMkpCEq7e5PyOWAqXdkprf+A8eOfdPcyGAtOA/sB4rfWtE00cgGF32tBaM9Dp31amprufipHUzwD+1FpXAToCt8oqckpOAa6n65Mrpnrp7sAUpVQIRjF5O6WUG8aZmB6ZNikMpL/m1TiMGefJwBLTulOAv2kf6dUCjgJHMM7wtDgnB0e+eu19XAo4M+GLWRQtVITFr83LUtNbs2wVOjdsQ+eGbahZtgoAlQLKUa5EKQCmLZnLx6bL2m09tJOXPp7CqdAQi47lbpwcHFk6+SNcCjjz2qdv4ulemCWvf5RlrEUKFWbDv38x+sPX+XbjKtrWe4Ivxr8HQKdGbRj91FDOhYcy7pPpeLgVYuFrcylSqLA1hpQjJ0cnls/8FFdnZ8a9Pw1Pj6Isn/lplrEWdS/Cb/9s4sV3JvLNryto17AlX037KK198iezmPf15wBs3vMPL8x6laCz+ecSYE5OTnz7xVJcXFx4ZfJreBb15NsvlmRbkz7pzam8N/9DAP7atoXhY0Zx8lQQh44eIf5GPDdu3ODF8WPoN/xZXp40ztJDuSsnJye+XrocVxcXxrw2Dk9PT75ZsjzLWH28fVi6aDFXr15j/OSJVK9WnYWfGsfwsVq16NalK926dKVWLaNutkrlypQvV97i48mNr75aRNmyZRg/fgLJSUksXbYEd3f3LP2cnJxYvnwpLq4ujBv3Kp6enixfvhRbW1vOnAkmKjKKlJQUpk6ZxsABgxg4YJDlB3MXHy6aQekyJZkxYR7JScnMXzKTQu6Z/8uAs8EXuBx1hZSUFN6Z9gkjB01m5KDbl5j8/IPlvD31EwCOHT7Fa6NmsnvHQYuN425q9PXExdOe46uiSU3R1BzgiX2BrH+v7iUdqdDZg9SbcGTFZVJvaip1L0zB4kZN+5k/r3FirVF6FBuWzKHvL3MlOMGiYxEPF6WUv1LKH+N8tbnAi8DmW+u11pe01rvuuI9b9VeWopQKANaakmOUUotN93+81QYEAcu11iuUUtOAgVrrAKXUcIxZ6F5a65tKqcJa62hT8ltbax2llPIBdmAkqVeA34CPtNars4mlDfCK1jr91TeWABu11suUUruB17TWf5im83cA7bTWpzPFrYC9GO9eflNKvYdxUIZrrVNMF+1+EWN2HNN+vtBaLzQ9Zh3AWWu9+Q6/Og1wdc3xO3R58Ll3rJC2HPv7aStGkrfcWpVOW76x88Idej74CtS7PWOfHBFrxUjynr3X7SQoMTZ/18r/F45uzmnL8TeyXn7tYeJc4HaJz/m4O/5/+sDzc62TtvzkvMA79Hzw/TI6GKwwF/xYzVp6++a/Lf2wecqxkMserXVta8dhDqXUQYzjbwNUBG6ddV8SOKG1rni3feTXq3DMBmYppbYB6d+ufoExyINKqQMYtdMZaK0vAhOAP4EDwN7skmeT3hhXxEhvRbr99gcmK6X2A5uAN0y10pkfUwNvAq+aVk0AEoCTSqkgoCfQVZsAXYFWpsvYHcH4+CAshxiFEEIIIcR9orWuprWuCmwGGmitA7TWAcDjwM47bmxi8RporXUIxol0t+4PzKEt/XWRXze13wReMf2k32dApvvfAHc94y39Y6db9zPws2n5KMZVM+66rdZ6BUbyjdY6ERhl+slu2zDgqbvFJ4QQQoiHiAItNdD5SUOt9fO37mitdyilPs/NhtY6iVAIIYQQQghrOqmUWgB8a7rfFziRmw0fmQRaKbUTcMy0ut+tq4EIIYQQQohHSj9gBDASoyb6b2B+bjZ8ZBJorXU9a8cghBBCCCHyB611AvC+6ccsj0wCLYQQQghhdffw7aki7yilKgItMK529ofWOleXO8uvV+EQQgghhBAizyilemJ80Z4PxhXUZiulnsnNtjIDLYQQQgghHkUTgUZa60ilVDuMywz/A3x9tw1lBloIIYQQQjyKbLTWkaZlpbVOAXL1HfEyAy2EEEIIYSlSAp2fJCmlPLTWVwAnpdQn5NcvUhFCCCGEECIfeAFwA65gXAv6DLko3wBJoIUQQgghxKNpP9BWKRWotZ5uzoaSQAshhBBCiEfRKiAZ8FBKbQLmAV9prbvebUNJoIUQQgghLEVqoPMTX611DaWUI7BTaz1NKVUiNxvKVTiEEEIIIcSj6IRSqoLWOhFAKeUEOOVmQ5mBFkIIIYQQjyJPYJ9SagfgD+wCPsjNhpJACyGEEEKIR9G0dMsJQJDpknZ3JQm0EEIIIYRFKLSSIuh85Gym+wWVUgW11pnXZyEJtBBCCCGEeBStwTitUwMuGGUcJ4FKd9tQEmghhBBCCPHI0VpXS39fKVUHGJibbeUqHEIIIYQQ4pGntd4F1M9NX0mghRBCCCEsQT2EP3cbslJtlVInlFKnlFLjs2lXSqkPTe0HlVK1crvtf6WU8s/8A7ys1N0L1aWEQwghhBBC3HdKKVvgE6AVcAHYpZT6WWt9NF23dkBZ00894FOgXi63/a/S10A7AoHAaa11xbttKAm0EEIIIYTIC3WBU1rrMwBKqe+AzkD6JLgzsFRrrYEdSil3pZQPEJCLbf+TbGqgKwEv52ZbSaAfMO4dK1g7BItxa1Xa2iFYRIF6ufrW0IeCvZebtUOwGEc3Z2uHYBHOBVytHYLF+LnWsXYIFvPL6GBrhyAeDsWB8+nuX8CYZb5bn+K53Pa+0lofVUrlqgZaEmghhBBCCIvQpp+HSlGl1O509xdorReYlrOrJc78C8ipT262/U+UUovSPY4NUBk4kJttJYEWQgghhBD3KkprXTuHtguAX7r7JYCwXPZxyMW2/9XadMuOQDLwbW42lAT6AXMp/rC1Q8hT3s5V0pYvx4dYL5A8VsQ5IG05/kac9QKxgPQf8SfExVsxkrzn5Hq7bCM5ItaKkeSt9KU4ibEP9zFNX4qTcjraipHkPdvShdOW43ecv0PPB59zfb+7dxL3wy6grFIqEAgFegF9MvX5GRhpqnGuB1zTWl9USkXmYtv/RGv9U6ZV3yql/gb+uNu2kkALIYQQQoj7Tmt9Uyk1EvgNsAUWaa2PKKWGm9o/A34F2gOngHhg0J22vZ/xmS5bd4sNUBXwzM22kkALIYQQQog8obX+FSNJTr/us3TLGnght9veZ2vSLd8EzgLP5GZDSaCFEEIIISzAOIXwoTuJ8IGV+TJ25pBvIhRCCCGEEI8cpZSDUqqTUqqpudvKDLQQQgghhHgUrcK48oaHUmoTMA/4Smvd9W4bygy0EEIIIYR4FPlqrTtjfF14F631VYzL5d2VzEALIYQQQliIJtXaIYjbTiilKmitjyulUEo5AU652VASaCGEEEII8SjyBPYppXYA/hjXrf4gNxtKAi2EEEIIIR5F09ItJwBBWusrudlQEmghhBBCCPEoOpvpfkGlVEGtdeb1WUgCLYQQQghhMXId6HxkDaAwDoojEIjxjYiV7rahJNBCCCGEEOKRk/mLVJRSVYFRudlWLmMnhBBCCCEeeVrrQ8DjuekrM9BCCCGEEOKRo5Qak+6uLfAYcCE320oCLYQQQghhEVquA52/uKRbvgmsBlbkZkNJoIUQQgghxCNHaz39XreVBFoIIYQQQghAKbUT4wtWlNY6MKd+kkALIYQQQggBaK3r5aafJNBCCCGEEBai5TrQ+YZSqjVwSGt9USlVFqgKrNdax99tW7mMnRBCCCGEeBS9C0QrpdyB9UAb4H+52VASaCGEEEII8ShK1VonAu2B/2mthwHFc7OhJNBCCCGEEOJRlKSU6gyMAH4xrbPNzYaSQD+CTh4/Q8fmffH3qEWjGh35a+M/OfadPHYWVQOa4uNSlX7dX8jQtuDjZdSp2AZ/j1rUrdSWLz/9Oq9DN9vJ46do17w7vh7lqVejOX9u3JJj3wljp1ExoDZFXQLp3X1w2vrTp4Lp3K43Zf1q4l+sCt079CX4zFlLhG+W48eP0/yJFni4F6FG9Zps3PhHjn23b99O3br18XAvwuMNGrFv334ATp06Rbu27fErUZJiXj506NCJM2fOWGgEd/fP9u3UqV+XQkU8aNDocfbt35dtv5/XrKFy9aq4Fy1M63ZtCQkJASAyMpJ6j9enSDFPPH2K0aptG44cPWLBEeTeP//uoFazBrj6FaVuy8bsO7g/235PD+6HV7mSOBQryEsTxuS6LT/5Z8d2ajeoS8GiHtRvfIfjunYNlapXpZBnYVq1b0uw6bjeEhkVSfGAkjgVdOH9D+flfeD3YNuendR8sjHOFX2o0+kJ9h4+kKXP9r27aNSzLUVqBlKkZiA9XxhA5OWou7blJ9sP7KJu39Z4NCnD4wPas+/EoSx9Iq9cpn7/tng+UZ5iLSrSZkRPjpw+AcDOQ3toPqQrvq2q4NuqCn0mDCPyymVLD+P+0xp06sP182AbAQwAftNa/62UcgNm5GZDSaAfQc8PfJVTJ4OZ9vY47OzsGNpvDDHXYnPs37lHuyzrzpw6y9TXZmNjo5j29jhuJiczeezbhF64lJehm23owBcJOnmaGW9Pwt7Onmf7jSTmWkyO/bv26Jhl3cWwcFJTU3lt0mj69OvB5j+3MfqF8XkZ9j0ZOOBZTp4M4u23Z2Fnb0+/vv25du1aln4JCQn06d2XuNg43nnnbSIiIuj7TD9SUlIIC7tIamoqkyZNol+/vvy56U9eeH6kFUaTVUJCAn369iE2Lo7Zb79DeEQEffr1JSUlJUO/S+GX6D9oAAXd3Jj55lvs27+P54YNTWtv3ao1H7z3PkMGP8ff2/7mtQn571gmJCTw9LN9iYuLY870WURERtBrcP8sYwVwdHCgc7sO2e7nTm35RUJCAr1Nx/XdWe8QERFB7/7ZH9d+gwZQsKAbs2aYjuvwoRn6jHl1HDcSblgyfLMkJCbw1AsDib0ex9xJbxJ+OYKnRw3KMtagkFMU9SjCrFen0q5ZK1b+tpbxs6fdtS2/SEhMoM/E4cTFX+edl6YQER1J34nDs33+tq7fjPfHvslzXfvy9/6djP/QuCxv0LkzFHH3YMbz42nd4AlW/7WOyZ/MtPRQxENOa71Xa91Na/2m6X6s1vqH3GwrCXQ6SqmaSimtlGqTbl2AUupwpn7TlFJjTcuLlVLBSqn9Sqm9SqkG2aw/oJRqkW77ukqpLUqpE0qp40qpL5RSzpYY46H9xzhy6ARderZj0LDeDBvVn9iYONau+j3b/m/OmcCwUf2yrE9NNd51evsWo/ET9fEsVhRHRwccHR3yNH5zHNx/hMOHjtGtZ0cGD+vPiFGDiY2J5edV67LtP2vONIaPGpxlfd36tVjz2/c8N3wAs+ZMw6OwOyeOBuV1+GbZv/8Ahw4domfPHgwbPpRRo0YSExPDqpWrsvTd8NsGIiIiGDL0OYYOG8KAAf0JCQlhy5at1K9fj982rGf4iGHMmfsuhQt7cPTYMcsPKBu/bdhAeEQEQ58bwrAhQxnYf4AR99aMnyr874cfSExMZOyYsTw/fASdOnZk2z/bOHPmDJ6enkybMpU2rdvQrGlTAGxs8t/L4PpNvxMeGcGwQc8xfNAQBvbpR/C5EDZv25ql79JPv6TvU72z3c+d2vKL3343jusw03Ed0M84rptzOK7jXjEd1w7GcT1t+oTktw2/8ev6dYwZ/Yo1hpEr6zdvJDwqguHPPMuIvoMZ1LMvwefP8tfOvzP069WhO6sWfM3Q3gP5dMZcAI4EHb9rW36xYftfRERHMqRbP4Z278+Ajr0ICTvPlr3bM/Tz9CjC1GHjaPN4c5o+1hAAG2X8PT7VujM/vLuIwV378tFrswA4duakZQcixB3kv/857hOlVK5qWDLpDfxtujXHOK11DWA88Hk260cDn5niKgb8ALymtS4PVMQ489PtHuLNlfj4G1yOusLlqCscO2Ikft6+xQDwKW7cng3O1Ve/pylTLpBJ00eza/s+GtfsxOEDx5n90VSKeha+v8GbyRhrNJejojl2xPhPxcfXGwDf4sbt2eBzZu3TweH2m4J9ew9yJfoqDRrVuU8R37v4+HiioqKIioriyBGjDMHX1weA4sWNcyCCg0OybBdy9qypr6+pr3EbEhycYax79+wlOvoKjRo2zLMxmCPkbAiQLm7f7MeYNj6fTP1CggE4fOQwJUsF0LlbV3x9fXn37dl5HbrZQs6ZxuBtHM8SPqYxmH4HD5O043rreN167mYqz8j6vL19XOPi4hj58kvMmPoGfn4lLBD1vQk+b7z2FC9mOq7exliCz2UsCUv/d7hh658ANK7z+F3b8ouQi+cB8PU0XnOLexm3IWFZX3sPnz5OQPuadH2lP76e3swePRUAB/vb49y4czMADWvk6vK8QliE1RJo08zurdnXw0qpr5VSLZVS25RSQaZZ2rpKqX+UUvtMt+VN29oqpeYopQ4ppQ4qpUaZ1ocopaYopf4Geiqlepv6HFZKvXOXeBTQAxgItFZKOd3DsLYAZbJZv53bZ3W+ACzRWm8H0IYftdbh9/B4uTL//a+o4t+EKv5NmDQm40dgWhvXo1TKvH1GRUaz6LNvqFytAl99/wGVqpZn0piZhIVat4Tjo/c/p7z/Y5T3f4zXxkzL0HZ7rGYO1iTo5Gn6PTWUkv4lmDXnjf8a6n/2/nvz8C8ZiH/JQMa8MjZDmzljNXXN0PfkyZM89VQv/P39mTN3zv0L+j66dS3Vu40x8++idKnSrFm1mqmvT+HixYu8N+/9vA30PsjtWB8GuX3upu835/33cC7gTMvmLYiMjATgcnQ0V65cydtg/6O7jXXbnp08N34Uj1WpztQXX811W35zp3GWLhHA6nnLmTJ0LBejwnl/+WcZ2rcf2MWIt8ZRs0JVJj33skXizWv6Ifv3IFJKbVJKva+UGqCUqqaUMvt7Uaw9A10G+ACoBlQA+gCNgLHAROA40ERrXROYAtzK/oYCgUBNrXU1IP3Zawla60YYyew7QHOgBlBHKdXlDrE0BIK11qeBvzAuaWKujkDWMyWgLbDKtFwF2JObnSmlhiqldiuldi9YsOAewjH07NOR79cs4Ps1C/h65acAXAw18vVLYREAlAwwZm0SE5NITEy66z7/2fIvF8MiaN+5BW07NKd95xbExV5nz86sJ8RY0tN9urFizTJWrFnG9yu/AkhL6i+GGWMuGeAHQGJiIomJibna74ljQXRu0ws7O1tW/voN3j5eeRC9efo805s1a39mzdqfWblqBQChoWEAhIUZtwEB/kDGsQb4+5v6hmbo6x8QAMCxY8dp07oddnZ2/LpuLT4+3pYZ0F0E+AcAWeMOCPAnISGBpKQkUz/T+MJM/S6GZdje1dWVli1aMv7V1yhRogQrVv5kqSHkWkBJ0xhMsd+6DSiZcawPg7TjGpbpuPrncFwzH3//AC6EXuDEyRNUfawGk6a8DsCc9+by2cJ7f93MC4F+JQG4cMl0XMMvAhDgV5KExIzHdcu///Dksz0pXTKQX7/6EVcX11y15QcBPsZrbGikMb6wSOO119/Hzxhn8u1xujq70LJeE14b9CIlivny06a1aW1/79tJl1f6E1jcn9XzluPq7GLBUYiH3DDgH4wccS8QoZT6SynVLbc7sPY3EQZrrQ8BKKWOAH9orbVS6hAQABQClpi+HUYD9qbtWgKfaa1vAmito9Pt83vTbR3gL611pGn/XwNNuJ3IZtYb+M60/B3QD/jJ9LjZSb/+XaXUZCASGJxp/WzAC6ifw35ypLVeANz6H+Ce3+b5B/rhH+iXdr9SlXKs/nEd5SuWZsnC/+Hq5sKTnVsC0KhGR6IvX+F0xL8AbFy/heOmso+wC5f4evEKGjSqTUnT/lZ8t5Zi3p789L1x9ZdSZQLuNcz7IiCwJAGBJdPuV65SgZU/rqFCxbJ8tfBrXN1c6djZOCmyfo0WRF++wtkIo/xhw/pNHD9i1NiFXQhj2eLveLxRPZycHOnctjdXrlxl4pQx7Nm1nz279tOtZ9YTDi0pMDCQwMDAtPtVqlbhxx9XULFiRRZ+8QVubm507tIZgBrVa3H58mUiIi/Ruk1rPL08+WLhl7i5urFkyVL8/f1p0qQxFy5coG3bdlyJvsKUqa+za9dudu3aTc+ePaw1zDRtWrfGy9OThV8aY1u8dIkRd+MmuLoXpFLFiuz5dzc9e/Tg9alTeO/994iIiODnNWt4vMHjlCpViqXLlnLg0EGqV63GoSOHOX/+PI899pi1h5ZF2+at8CrqyYLFX+Lm6srib5YR4OdP04aNKeDrQaXyFdm/ZScA/1u1gr0HjKtWHDtxgkXLl9CuVRt8innfsS2/aNPKOK4LvvzCeD4uM45r08ZNcPEwjuvencZxnTxtCnPnmY7rWuO4li5VihFDh9O+rfF3vWXrVj5b+DnP9O5Dty5drDu4TNo2bYlXEU8+/+Yr3Fxc+eqH5QSUKEmzeo1wLO9F5bIVOLBuG3sPH6DD4KfRWjP46X5s3PYXzgWc6dii7R3b8ovWDZrh6VGUL35ajpuzK0vWfIe/jx9NajWgYKNAKpYqx+6vN7J07f84GHSEamUrc/jUMc5fCuWxitUB2HfiEF1f6Y/WmkGde7Pp3624OBWgfeNWVh6deBhorYOAIKXUGKCM1jpEKVUGeEcp5a21nn+3fVh7Bjr99F9quvupGMn9DOBPrXUVjNndW2UVipwTyuvp+uSKqV66OzBFKRUCfAS0M13O5DLgkWmTwkD66waN01rX0Fq30lqnP+FwHMYs+2RgiWndEcCq/2PP/+odSpcNYNr4d0lOTubzpXMo5F4w+77zvuKtKfMAOHr4JGNfmMau7fuoUasyU2eNJSkxmYkvv0VSYhIz35tI5WrlLTiSu/v8qw8oU7YUr49/i6TkJL5c+nGOY/143gKmTzEqfY4cPs7LL0zg3+17CD5zjqioy6SkpDBj6myGDnyRoQNftOQwcuWrrxZRtmwZxo+fQHJSEkuXLcHd3T1LPycnJ5YvX4qLqwvjxr2Kp6cny5cvxdbWljNngomKjCIlJYWpU6YxcMAgBg4YZPnBZMPJyYmvly7H1cWFsa+Ow8vTk6+XLsfWNuPpDj7ePixZtJir164xYdJEqlerzsLPjFMTihb15LcNGxg1+iW++eZb2rdtx+IvFlljOHfk5OTEt18sxcXFhVcmv4ZnUU++/WJJlrECTHpzKu/N/xCAv7ZtYfiYUZw8FXTXtvwi/XEd89o4PD09+WZJ9sd16aLFXL16jfGTTcf1U+O4PlarFt26dKVbl67UqlUTgCqVK1O+XP56PXJydOK7jxbh6uzCy29OxKuwJ999tCjLWA+dOEL8jXhuJNxg1LRXeWb0EEZPH3/XtvzCydGJ5W99iksBZ8a9Pw1Pj6Isf2t+lnF6uhdmwz9/8tLsiXy7bgXtGrZg0RvG8/XwqePEJ9zgRmICL8+ZzMApIxnz3lRrDEc83JyBcwBa61NAT4xL292VulWbZGlKqQBgrSk5Rim12HT/x1ttQBCwXGu9Qik1DRiotQ5QSg3HmIXupbW+qZQqrLWONiW/tbXWUUopH2AHRrJ6BfgN+EhrvTqbWNoAr2it0199YwmwUWu9TCm1G+Okvz+UUoVN+22ntT6dPu5M+0w/HoXxEcF4YD/wL/CU1nqnqW9f02PdqYBYA1yKP3yHLg8+b+cqacuX40OsF0geK+IckLYcfyPOeoFYgHOB2x8vJ8TFWzGSvOfkevtiOskROV8a8kFn73X7nOfE2If7mDq63T6mKaej79DzwWdb+vZJ4PE7zlsxkrznXN8PzJhou19q1qqhN2/L/qpXD6pCzl57tNa1rR3HvTDllv7ARK31RaWUN7Bdax145y2tPwN9N7OBWUqpbWT8ZpgvMN4xHFRKHcConc5Aa30RmAD8CRwA9maXPJv0BlZmWrci3X77A5OVUvuBTcAbplrpXNHGu5Q3gVdNJwv2AuaYLmN3DGgM5HxxYiGEEEIIcV9pracB24D1SqlQjPPY5uZmW6vNQAuzyQz0Q0RmoB9OMgP98JEZ6IeTzEDfPw/yDHR6pitxpGqdu69XtPZJhEIIIYQQQljVrQtT5NYjl0ArpXYCjplW97t1NRAhhBBCCPFwUkr1x/jOjzt9AqExqjSeyKnDI5dAa63lq4yEEEIIYQUa40JjwopWYXzfx3/yyCXQQgghhBDi0aS1juE+XLghv1+FQwghhBBCiHxFEmghhBBCCCHMICUcQgghhBAWonP8ImXxIJEZaCGEEEIIIcwgCbQQQgghhBBmkARaCCGEEEIIM0gNtBBCCCGEhWi5DvRDQWaghRBCCCGEMIMk0EIIIYQQQphBEmghhBBCCCHMIDXQQgghhBAWI9eBfhjIDLQQQgghhBBmkARaCCGEEEIIM0gCLYQQQgghhBmkBloIIYQQwiK0XAf6ISEz0EIIIYQQQphBEmghhBBCCCHMIAm0EEIIIYQQZpAa6AeMt3MVa4dgMUWcA6wdgkU4F3C1dggW4+TqbO0QLMbey83aIViEo9ujc0xtSxe2dggW41zfz9ohPJQ0RhW0ePDJDLQQQgghhBBmkARaCCGEEEIIM0gJxwPm74ufWzuEPNXIZ1ja8smrG60YSd4q594ybTk8/pgVI8l7xZwrpi3fuHHDipHkvQIFCqQtJ0fEWjGSvJW+POXmpRgrRpL37LwLpi1f33rWipHkPZfG/mnLK9yft2Ikea/71fnWDkE84CSBFkIIIYSwGLkO9MNASjiEEEIIIYQwgyTQQgghhBBCmEESaCGEEEIIIcwgNdBCCCGEEBah5TrQDwmZgRZCCCGEEMIMkkALIYQQQghhBkmghRBCCCGEMIMk0EIIIYQQQphBTiIUQgghhLAQLV+k8lCQGWghhBBCCCHMIAm0EEIIIYQQZpAEWgghhBBCCDNIDbQQQgghhMXIF6k8DGQGWgghhBBCCDNIAi2EEEIIIYQZJIEWQgghhBDCDFIDLYQQQghhERqt5TrQDwOZgRZCCCGEEMIMkkALIYQQQghhBkmghRBCCCGEMIPUQAshhBBCWIiW60A/FGQG+hEUEnSRER1n0zxgJH0aTeXfv47m2Hfok2/TuuxLtCw1iufazmT/jqC0tp51J9LYd3jaz6CWb1oifLOcPnGWXq1fokqx9rSpM4i/N+3Otl/I6Qv06ziWeqW6UdOvE4O6vsa54LC09phrcbw6/B1q+3ehZomOPNP+FUsNIddOHj9Nh+Z9KOlRnYY12vPXxm059p009i2qBDTG26USfbuPSFv/3bKVeLtUyvJz7myoJYaQK8ePH+eJJ57A3d2d6tWrs3Hjxhz7bt++nbp16+Lu7k6DBg3Yt29fWtuYMWMICAjA2dmZbt26WSJ0s/zz7w5qNWuAq19R6rZszL6D+7Pt9/TgfniVK4lDsYK8NGFMrtvyk23/7qBm88dx8fekTqvG7M1prM/1w7NCSex9CvHixLFZ2iOjovCuFIi9TyHe+/TDPI763mw/tJt6A9tSuEU5Gg5+kv0nDmfpE3n1Mg2ebYdX64p4t6lMm1FPceTMiSx9SnasiWuTAD74doGlws+1IvVK0XLbJLqEf0DzzeNxr+6Xbb/KUzrT/thMulz6gNb/TqFE18dy1SaEtUkC/Qh6Y8SXnD19iZFTe2Bnb8uUYQuIi7mRbd8qtUvx0oynGfDyk5w6coF3xi7L0F69flmmzh/M1PmDGT65qyXCN8srz83kTNB5Jrw5DHs7O14aOIPYa9ez9Au/eBmdmsqo8f3p3qcN//y1l0kvvpfWPnHkHNb8sIke/doyceYISpbyteQwcmXEwLGcOnmGaW+/hp2dHUP6vUzMtdgc+3fp0S7LugaN6/DZ4jl8tngOH3/xNg4O9nh6FcHH1ysvQzfLgAEDOHnyJG+//Tb29vb07duXa9euZemXkJBA7969iY2N5Z133iEiIoJnnnmGlJSUtD49evSwZOi5lpCQwNPP9iUuLo4502cRERlBr8H9M8R+i6ODA53bdch2P3dqyy8SEhJ4+rl+xMXFMueNmURERdJrSPZjdXBwpHO7jjnu65Upr3EjISEvw/1PEhITeOb14cTFx/H2yNeJuBJF3ykjsh1r63rNeP/lGTzXpS/bDvzLhI8zTlC8+uEbJCTmz7HaONpRf+kQ7FwdOThxBU6eBam/ZAjYqAz9vJpVoMIrbUgIj+HQlJU4+bhTe34/lJ3NHduEyA/kmfgfKKUeuBKYk4fOceroBVp2rkO3Qc14emgLrscm8Nfavdn2HzWtJw1bVeOxRuWxd7DDRmV8AfQpWYTHW1alZZc61GtW2QIjyL2jB09x/PAZnuzejGeGdGbgC92Ji41n/c9bsvStWbcSy395j75DuzD5nRdw93Dj1PGzAJwPucjva7fxZPcneGXKYLo904ZZH2ed/bKmQ/uPcuTQCbr0fJJnh/Vh2KgBxMbEsXbVhmz7vzVnEkNHDciy3j+gBF16tqdLz/Y4OjmSlJRM7/7dsLe3z+sh5Mr+/fs5dOgQPXv2ZPjw4YwaNYqYmBhWrlyZpe9vv/1GREQEQ4cOZdiwYQwYMICQkBC2bDGO/9y5cxk1apSlh5Ar6zf9TnhkBMMGPcfwQUMY2KcfwedC2Lxta5a+Sz/9kr5P9c52P3dqyy9ujXX4wOcYMXAIg3r3I/jcWTb/k3Wsy+Z/Qb+evbLfzx+/s3bDesaNHJ3HEd+7DTv/IiI6iiFd+zG0az/6P/k0IRfPs2X/jgz9PN2LMOW5sbRu8ARNazUAwCZd8rlhx5+s2/YHL/cZbtH4c8u7VWWcihXizBdbOPPlFkKW/4NLQFE8G5XL0E+ZxnQ9OJKIP4+RHHOD5LhEdKq+Y5sQ+cEDlwDmllKqPzAW40vnDwKTgUWAJxAJDAKuAQeA/7N33/E1X38cx1+HiiytkQRBEnvWpvbeq7T23nSookaN0lZbXbp0qU2rG60OLa1NUXurSpQgiR0k/OT8/rhXiMS4rdxEvJ8e93G/93vO/d7Px/eOzz33fL/JZ62NM8Z4A3uAfEAQ8L6z/3mgj7V2tzFmBnACKANsNMZ8AbwNeAEXgB7W2j3Obc0AigC7gBDgCWvtBmNMA+B5ICOw33mf6OT6v4g5f5GYCxcB+Hu3Y1qCX87MAPjnzAJA+MHIJO8bfeYCzR90FIu+D3gz/I0uCdoXffUHP3+5lszZMtHv2ZY061g1OVK4bRfOx3DhQiwAe3ceACB7Tj8AcgQ6rg+FHkl0Pw+PqwXitk17OHXyLA1bVAeIL6S3bdpDmVzNSZc+HV37tWLo832SL5HbcP78BS6cd4xA7drhmFpzZaQ4MFcOAMIO/POvtz976pekS5eOzj3b/sdI/5vz589z/vx5AHbs2AFAYKDjF4BcuXIBcODAgUT3CwsLu2Hf2rVrJ2/Q/1HoQWfsOXICkDunM/aw0JQKKdkciM/VuZ9yOq7/DgulTvXb20b0uWieGD6I8SPH4uvjkyxx3glhRw4BEOjneH3m8ndch4YfhHIJ3zt3/L2bKr2aOvr75+DVAc8BEH3+HAPfHM24vsPw9fZ2V+gu8QnOBsCFI6cc14dPOtaH+BG5/OpUlGO/7WL/5KXk71uL3K3KcfnCRVa3/xDi7E3b7n46D3RakCZHoI0xxYFRQB1rbSlgIDAJmGWtLQl8Crxrrb1SQNd03rU5sMhaewmYDAyw1pbDUYh/cM1DFALqWWuHALuBGtbaMsBzwMvOPo8DJ52P9yJQzhmbH45ivp61tiywAUjWCbWffbCI5g8+Q/MHn+GtUZ8naLtyMIO5bmT5Ci+fjEyc+xQDX2zHxdhLTH3j+/i25p2q8fzHvRn9Xg8yeKTnjeGfEn4wKvkSuQ1T3v2SygVaU7lAa14YNilBm7XON94b5Arw975/eLzTWHIF5WD0q08AcPHiJcBRnL81bRRlHyrOlHe/ZPXSpEft3eX9t6ZSPLgqxYOrMnJIwp93r+R6o/16K6F/H2TF0rXUrl+NoOBc/znW/2LixIkEBQURFBTE4MEJXyqu5Plf/09S0q1ep2nJv9lPr096G28vL+rXrENklGMw4PiJE5w8dTJZYrxTbpZrvlwhLHhjFmN6DeFI1DHenvsxABM/+wivjJ7UrVidyJPHAThx5iQnzyaexpRqXMnPJix+MxXMTp52FTm2ZCdrOn9MTORZyn3QlfTeHjdtE0kN0uoIdB3ga2ttFIC19oQxNXfv8QAAnDtJREFUpjJw5Uih2cBrzuUvgHbA70B74ANjjC9QBfjqmje2jNds/ytr7ZVJaw8AM40xBXGMdl8ZyqwGvON8/O3GmK3O9ZWAYsAq57Y9gDVJJWGM6Qv0Bfj444/p27evi/8NDo3aVKJkxQIAeHhm4ImWbxB5xPHBEuUcIciZxzE6ezHWUSx6ZHSkcd996alQsxgVahZj6Q8b2bhqD6eOR5M5my9dBzaJf4x92//hi48X88/fxwgM8vtXcd4JLdvXp1ylEgBk9PSgY+NBHA13FPXHjjg+bHIHO0Z9LsY6RuU9MjrekP/aHUa3h4fi4ZGBmd+9RkAOxyhKrqDsAJSvVIIGzatzIuo0a5dv5uCBcKrUKuu+5K7TtuPDPFTZ8fieXp60qNeZ8MPHADgS7rgOCskNQKwz14wZb+/DZ9bUL7HW0q1P0j+Xu1OnTp2oUqUKAF5eXtStW5fDhx0HNYaHO35RCQkJASA21vHrQ8aMGQkODga4Yd/ULCTIGfuR8ATXIUHBxMTEkC5dOjw80kYhkTc+V+d+Onokfv3t5vpP+GF2/7WX4tWuHmT22qS38PH2YeSgockUueuCczpej4cjHTmGRx1zrs9DTKwz1wyOXH29fahbsQZ1K9ZgxsLP+fb3H/hwxOscjghn78H9lOlUJ367b376Id6e3gzvljqmJJ0Lc7zXegVmcV5njl+fLuN92DiLvXSZnI1L4vGAN2FfrCN84RZyNChB3q5Vub9wTvyrF7ph28lNYSmVmki8tFpAG7jleWKutH8HvGKMyYpjlPg3wAc4Za0tfYP7XnsU2ovA79baVsaYEGDpNTHcKLZfrbW3nJhorZ2MYyT82nhdFhjsT2Cwf/zt/MVysWTBBvIWCmT+rOV4+3pSs2kZADpVH8vpE+f45a93+GPpDn7/7k9KlM9PRPgJtm/YT1b/+3kgqw/7dx1m8ivzqVSnOJf/F8fPX60lo2cG8hdJ2dHKPCE5yROSM/524eL5+PHbpRQsEszcaQvxyeQdPzWjUYWenDxxmk2HvufIoQi6NH+G0yfP8PSoHmzdsJutG3bT9NHaFC9VkELF8rJm+Wa+nPkj3362iPTp01H2oZSd8x2cNw/Bea8e2V6sRGEWfP0jhYsWYOYnn+ObyYemD9cHoGrpJpw4fpK/I/4E4Nefl7HbOe0j/NBRPp3xNZWrlSdfgRAuXrzIF3PmkytPTuo1rOH+xK6TN29e8ubNG3/7wQcf5Ouvv6Zo0aJMmTKFTJky0bJlSwBKlSrF8ePHiYyMpGHDhgQEBPDJJ5/g6+vLzJkzCQ4OpkYNR04//fQTO3c6zkBz+PBhpk+fTvXq1SlQoIDbc7xeozr1CfDzZ/KMqWTy9WXGZ7MJyRNMzarV8QrMQrHCRdm8/A8Avpz/DRu3OM4usmvPHqbNmUnj+g3JmT3HTdtSiyu5fjxzKr6+vkyfO5uQPEHUrFIdz9xZKV64KJuXOuYIf7ngG/68ks/e3Uz9dCZN6jXk8R59aFq/IQDLVq/kw+mf0LlNBx5p9nCK5ZWUBg/Vwj+LH1Pmz8HX24dZP3xBcI7c1ChdiQdq56do3kKsn/kLs3/8kq37dlKyYDG279/NP8cOU65IKQD6PdKNRlXqArBi01omz5tFx4aP0LJW4oOCU8rRX3cQE3GGfL2q87/oGEI6V+FcWBSRK/fy6In3Ob0znMVVxhMd6vi1IF/P6qT3zEDOhiW4HHuJc2FReOXJcsM2kdQgTU7hAJYAbY0x2QCcxfFqHCPMAJ2AlQDOucfrcIwWL7TWXrbWngEOGGPaOO9vjDGlbvBYDwBXzvHV/Zr1K4G2zvsXAx50rl8LVDXGFHC2eRtjEh5ZkczGvt+LoPzZmfT811y69D9e+LgPmR5IPJfu/sw+7NwUylujP+erKb9RsmIBJsx8HGMMD2T1JS4ujqmvf89HL88jR+6svDStP345MrszlVt685NnyVswN6+M/phLly7x9rTR3P+Ab6J+B0OPcCLqFJcvx/HmC1MZ3PtlBvd2zMYxxjBxykiC8ubkxeGTOH3yLK9+OJxCxfIm2k5K+mD6a+QvmJdxI17l0qVLTJ41kQcy359037en8dJzjrOM7Ny+hyFPPMe6NY7C5McFizkedYLOPdqQLl3qe4uYPn06BQsWZMSIEVy8eJHZs2eTOXPmRP08PT2ZM2cOvr6+DB06FH9/f+bMmUP69OkBePvttxkzZgwA27Zt44knnmDNmiR/DHI7T09P5k6ZhY+PD4NHD8ffz5+5U2bGx36tUePHMvEDxynblq5aTv8hA9j7175btqUWnp6efD55Jj4+vgweMwJ/Pz/mTk4615HjxzHxw/cAWLpqBf2feYq9+/dRvnRZHm3WkkebtaRcKcdgwINFi1GkoFvfWm/JM6Mns59/Hx8vH4a9+wL+WbIx+4UPEuXqlzkbv6xdysA3RzN30bc0rlKXqWPeBqBskZK0qtWEVrWaULaI42OleL4iFA5O+S9+V8TF/o8/uk/hf9GxlJrQhtios6ztNiXR/OXw7zaz5+1f8AnKRunX2nLx5HnW953BxRPnbtp2N7M4pmSlpX/3KmNt2kzeGNMNGApcBjYB43AcROiH8yBCa+1BZ9/WwFdALWvtMue6vMCHQE4c0zI+t9a+4DyIcKG19mtnv8rATOc2fwO6WGtDjDE+zvWFnI9fAmhvrd1njKkDvMrVaSGjrbXf3SIlC7DyyMf/5b8l1auWs1/88t5TNz6/792uUOZ68cvHzu9KwUiSX3bvovHLFy4kfbrEtMLLyyt++VLEjU8heLfLEJApfvl/R8+kYCTJ774cV7+EnluRtqcO+FQPjl/+JvPjKRhJ8nv01Adw41+Kk02psiXsLyu/dPfDJqscPsX/tNaWT+k43C2tTuHAWjsTRwF7rTo36Ps1172QrLUHgEZJ9O1+3e01OIrkK8Y4r2OAztbaGGNMfhyj4mHO+/wGVLjdXEREREQk9UizBXQq4A38bozJgKM4f8xaezGFYxIRERGR/0gFdDKx1p4F7rmfNERERORGLFbngU4TUt8RQiIiIiIiqZgKaBERERERF6iAFhERERFxgQpoEREREREX6CBCEREREbdJm39/416jEWgREREREReogBYRERERcYEKaBERERERF2gOtIiIiIib6A+ppA0agRYRERERcYEKaBERERERF6iAFhERERFxgeZAi4iIiLiFJc5eTukg5A7QCLSIiIiIiAtUQIuIiIiIuEAFtIiIiIiICzQHWkRERMQNrOZApxkagRYRERERcYEKaBERERERF6iAFhERERFxgeZAi4iIiLiJRXOg0wJjrU3pGOT2aEeJiIjcOcbdD1iybFG7cPl0dz9ssgrOVPlPa235lI7D3TSFQ0RERETEBSqgRURERERcoDnQd5lLuyJTOoRklaGof/xyXNiplAskmaULzhy/HLMxPOUCcQPPsoHxy5cPnEzBSJJf+rxZ4pf/d/h0CkaSvO7L9UD88r20Ty9uO5aCkSQ/jwezxy9f2p3GP2uK+N+6U7LQeaDTCo1Ai4iIiIi4QAW0iIiIiIgLVECLiIiIiLhAc6BFRERE3MBaNAc6jdAItIiIiIiIC1RAi4iIiIi4QAW0iIiIiIgLNAdaRERExC0scTYupYOQO0Aj0CIiIiLidsaYrMaYX40x+5zXWZLok8cY87sxZpcxZocxZuA1beOMMYeNMZudlybuil0FtIiIiIikhBHAEmttQWCJ8/b1/gcMsdYWBSoBTxhjil3T/pa1trTz8mPyh+ygAlpEREREUsLDwEzn8kyg5fUdrLVHrLUbnctngV1ALncFeCMqoEVEREQkJWS31h4BR6EMBNysszEmBCgD/HHN6ieNMVuNMdOSmgKSXHQQoYiIiIgbWCCONPeHVPyMMRuuuT3ZWjv5yg1jzGIgRxL3G+XKgxhjfIFvgKettWecqz8EXsTxX/si8CbQ05Xt/lsqoEVERETk34qy1pa/UaO1tt6N2owxx4wxOa21R4wxOYGIG/TLgKN4/tRa++012z52TZ9PgIX/JoF/Q1M4RERERCQlfAd0cy53AxZc38EYY4CpwC5r7cTr2nJec7MVsD2Z4kxEBbSIiIiIpIQJQH1jzD6gvvM2xphAY8yVM2pUBboAdZI4Xd1rxphtxpitQG1gkLsC1xQOEREREbewWJvm5kD/a9ba40DdJNaHA02cyysBc4P7d0nWAG9CI9AiIiIiIi5QAS0iIiIi4gIV0CIiIiIiLtAcaBERERG3sMRpDnSaoBHoe8zqjX9QtmVNfEvlpuKjddm0c2uiPms3r6dGxyb4P1QA/4cK0HZgDyJPRCXoE3kiipxViuBRLICJ0953V/guWbVhLaUbVcWrUHbKN63Jxu1bEvVZ8+c6qj3SgKwPBpP1wWBa9+9K5PGruc5f9AOFapbFu1AO6rRrxoF/wtyZwm1bvXk9FdrV44FKeancsSGbdm1L1Gft1g3U6tGCHDWLkqNmUdoP7UPkyeMAWGsZ894r5G1YlsyV81HqkRp89UuiswmluFUb1lKmcTW8i+SgQvNaSe/Tjeuo1roh2UqFkK1UCG0eS7hP2z7eDb/SebkvX1aeGjvMneG7ZNX6tZSpVwWffAFUaFSDjds2J9mvXd+u+BcPJkPuzDw1amiCtn/CD/FIzw48UDCQbEWD6PJkHzdE7rp7ab+u3rSOco/WJlO5IB5qWz/p9+AtG6jZpRkBVQsRULUQ7Qb1SvAe/M/Rwzz6VDeyVMyLf5WCdBvxuDtTuC2rN/5B2Ydr4lsyNxUfqcumHUnkuWk9NTo0wb9iAfwrFqDtUzf4rKlcBI+iqfezRu5NKqDvITGxMbQb2JPoc+d4Y8QLRByPpP3TPbl8OeG34b2h+8mWJRsvDx5Doxr1mP/rDzz7xgsJ+gx+ZTQXYmPcGb5LYmJiaNO/G2ejo5k45iWORUXS9rFuiXM9sJ9sWbPxyohxNK5dn3k/f8/wV8YCcDTiGB0H9OJ+30y8NvIF/ty+hR5DHkuJdG4qJjaGjsP6cPZ8NK8NHsuxE1F0HN43Ua5/hf2NX+asjH9qJA2r1mHBbz8y6p3xAPz2xwremDGJnH4BvDxwNOGRR+k7bjCXLl1KiZSSFBMbQ9vHu3P2XDRvjnLs03ZPdE+U574D+/HLko1Xho+jca36zFu0kBETxsa3Z/TISMsGTd0dvktiYmJo17cr0eeieWPsy0RERtK+X+LnL4CHR0YebtQs0XprLa17d2bx8qUM6T+AV0Y9j3+2bO4I3yX31H6NjaH94F6cPRfN60OfJ+JEJO2H9E6ca+h+smXOyktPj6ZRtbrMX/IDI996EXDs17ZP92DJ2uUM7v44Lw8ag1+W1LVfb/uzJsz5WTPkJp81L6fuzxq5d6mAdoHzvIRf36LPanfF46qfVyzh2PFI+rXvTv8OPen+aEcOHDrIsnWrEvRr3+QR5r0/mz7tuvHBuDcA2Ll/z9XtLF/CD78v4pleT7o1flf8tHQxx6IieKxLLx7r0pue7Tpz4J8wlq5dmaBfhxaPsmDKXPp16sFHL78FwM59uwH4/LtviL0Yy/DHB/Fk9760bNiUFevWsD/sgNvzuZlFq37n2PFI+rbpRr+23en+cHtCDx9k+Z8Jn4ptG7Xk67dm0PvRLkwa9SoAO//eC0BcXBwAeXMHU/ehGtzvm4lM3j6kS5d63iJ+du7T/p178liXXvRo2ynJfdq++aPM/+Qz+nbszocvOc65v2Pf1efv7Lcn0/mR9m6N3VU///4rxyIj6N+1N491602P9p05cDCMZWtWJuo7e9IndHk0cT5LVy9n49bNPN33CYY9MYg+nboz8fkJ7gjfJffSfl208jfHe3C77vRv34PurToQevggy9YnfK22a9KKb9+bRZ82XXn/udeBq+/BS9etYuPOrQzs0o+hvQbQu3UX3hz+ottzuZmfly/hWFQk/Tp0p3/HW3zWfOD8rHne+Vnz1931WSP3rtTz6XgLxpj0ybBNl+aAW2vDrbWtb9Gnyn+LKvmEHjoIQGB2xx/uyZ09EIADhxJOS/Dw8Ihf/mXl7wBUL1cJgOhz0Tz5/FDGDxpNUM7cyR7zvxXqzCkwhyPXXDkcuf59MDRBv2tzXbT8NwCqV3Tswiv/L7mc28h9g22ktNBw5371zwFALuf+PeDc31d4ZLia669rlgJQrcxDANSrXJN+bbvz7eKFlG5dkxOnTzHjpUmkT3/HX3b/2pXpM7mcz9sr++PAP6EJ+iV4/sbv08puiPDOuZJr/PM3Zy4A/g4Lve1t7NzrKETm/fgd9xfMSdYieXhv6kd3NtA74J7ar4cdr8lcAc79eqP34Gtfq6sc78HVyjreg3f97dyvi38gc8W8+FUuwKRPpyRv4C4KPXzdZ038Pr2Nz5ry13zWjBvK+MGjCQpMvZ81rrIW4uzlNHW5V92xAtoYE2KM2W2MmWKM2W6M+dQYU88Ys8oYs88YU9F5WW2M2eS8Luy8b3pjzBtX/pqMMWaAc32oMeY5Y8xKoI0xpoOzz3ZjzKu3iCfaGPOmMWajMWaJMcbfuX6pMeZlY8wyYKAxppwxZpkx5k9jzKIrfxbSGFPAGLPYGLPFuY38zhy3O9uLG2PWOf8izlZjTMErj+u8NsaY152xbjPGtHOur+WM4Wvn/9enzj9T6XbWWpwxJdm+euMf9B09kLLFSzHmCcfcytenTsLLy4t6VWsRcSISgOOnTnLy9Cm3xPxv3SrXVRvW0nvYAMo9WJqxTw//V9tILW65Xzevp/8LQyhbtCSj+w0BHNN2Pv/xW+pVqsnnr08hIKsffcYN4tyF826L21XONG++T0cMoFyJ0owdmPQ+vVtYXH/uXbwYC8B9993H11PmEJIniCHjnmXv338lS4x3yj21X2/1Wt20jr5jB1G2WEnGPO54D469eBGADPfdx5dvTyckVxDPvDaGvaH73RP0v3BbnzWjrvusmeL8rKlSi4jjd89njdw77vQIdAHgHaAkUAToCFQDngFGAruBGtbaMsBzwMvO+/UF8gJlrLUlgU+v2WaMtbYasBx4FagDlAYqGGNa3iQWH2CjtbYssAwYe01bZmttTeBd4D2gtbW2HDANeMnZ51PgfWttKaAKcOS67fcH3rHWlgbKA4eua3/EGWcpoB7w+jV/s70M8DRQDMiH489UJmKM6WuM2WCM2TB58uSbpHp7QnIHAXD4WLjjOsKRUkiuIGJiY7jofGMGWLFhDc36tidfnhB++OQLfH18ATh09DB7/t5HiSaVGfmm42fD16e8y4dzp/3n+O6kkNzBABw+4sg1/Jgj17x5gomJSZjr8j9W0aRbG/IHhfDTrG/ic83r3MYh5zYOX7ON1CQk0LlfnfszPOKoY32uPI79eulqris3ruXhAZ3IlzuY7yZ9iq+3DwALl//C6egzdGz6KA/XaUyditUJjzjKLucUj9Tgyv/7oaNX9ofjOiR3cKLn7/J1q2naoy35g/Ly48yv4/fp3eJKrvHPX+d13qDEz98bCXa+3pvUbUiLhk1pUrch1lpCD6auA2Hvqf2ay7FPDjlzDL/+PfhSwvfg5o91IF+eEBZ+9Hn8azUkMA8AjWvUo0XtRjSuUc+xXw8n/MUpJYU48zx8NOF7Z0juJD5r1q+hWZ/25Au6jc+aT97lw89S12eN3Lvu9GnsDlhrtwEYY3YAS6y11hizDQgBHgBmOkdrLZDBeb96wEfW2v8BWGtPXLPNL5zXFYCl1tpI5/Y/BWoA828QS9w1950DfJvENgsDJYBfnd+M0wNHjDGZgFzW2nnOeGKcj3nt9tcAo4wxuYFvrbX7rnv8asBc6/ibncecI94VgDPAOmvtIec2Nzv/bxJNbrTWTgauVM72BnnetkbV6xKQzY/Jn88kk48vM775jJBcQdSsWBWvB3NSrEARNn+3nE07t9K8XwestfRq05nFq5fh4+VNs9oNebxjL5rUbADA8vWr+PCzaXR+uC2PNGj+X8O7oxrXqkeAnz8fzZmGr48v076YQ0juIGpVqoZHfj+KFyrC1l/WsHH7Fpp2b4u1lt4duvLrit/x8fameb3GtGvxCM++9jyvf/QOEVGRzF/0A9UqVCJ/cN6UTi+BhlVrE5DVj0++nkUmbx9mLPic4MA81ChXBd+KQRTLX5g/v/yNTbu28fCAzlhr6dGyI0v+WIGPlxdNazQgn/PLwuSvZ3EhJoafVi7GI4NHfHGeGjSqVY+AbP58/Ok0Mvn4Mv3LT+P3acaC/hQvVIQtP69m4/YtNOvh2Ke92nVl8crf8fb2oXndRgB8ufBbNjjPaLHzrz1M/WIWTWo3IGdAjhTMLqFGtesT4OfPx7On4uvry/TPHSPINStXwzM4G8ULF2XzkjUAfPndt/y5ZRMAu/btZupns2hStwGN6zQgwM+feT9+R/6QvHz7wwJ8fXwpXaJkSqaWyL20XxtWq0NAVj8mf+l8D543l+DAPNSsUAXvMrkolr8wm+YtY9POrbR4vCMW6PlIJ5asWY63lzfNajVwvI9n9WPe4h/Inycv3/66EF9vH0oXKZHS6cVrVOMmnzUlnJ813y9n045rPmtaOz9rvJ2fNZ160aSW87Nm3TWfNQ1T12eN3Lvu9Ah07DXLcdfcjsNRrL8I/G6tLQE0Bzyd7YYbF4jnrunzX1y7/Wu3ucNaW9p5edBa2+B2Hsta+xnQArgALDLG1Lmuy822ce3/02XcdD5uz4yezH1rKj7ePgx+eTT+Wf2Y+9aURPNct+3ZyfkL57kQc4GnXhxBl2f6MeilkQCUK1GaRxs259GGzSlXvDQAJQoWpUi+gu5I4bZ5enryxfvT8fXxYdALzxKQzY8vPpiRKNetu3bE5/rkmKF0eqo3A8c6fhbOGZCDT9+dwqkzpxn68hjKFH+QaW98kBLp3JRnRk8+ffVjfL18eOaNsQRkzcanr36cKNft+3ZyPuYCF2JjePrVUXQb+TiDXxsDQMs6TRjc7XHCwg8x+PUxZLk/M9NefBe/LFlTIqUkeWb05PP3p+Pr7cOgFx379PNJ0xM/f3df3acDxg6l08A+PD3u6k/9z776PBM/mQTA0jUr6Pfs0+xJZdMaPD09+fyjGfh4+zJ47Aj8/fyY+9HMJOekj3x5LBM/fg+ApatX0H/YU+zdvw8vLy+++HgmHh4ZeWr0ULy8vPjqk9kE+Pm7O52buqf2a0ZPPnvjE3y9fRjy6hjHe/CbnyTO9cprNeYCA19+li7D+zN4guM92MvTi7lvTiFjBg8Gvvws3p5efPHWNAKypZ79muCz5pXR+GfzY+7bSXzW7E3is2Z8Ep81JUoDUKJQ6vuscZ1N8TnLmgN9Z5grc5P+84aMCQEWOotjjDEznLe/vtIG7APmWGu/McaMA7pba0OMMf1xjEK3t9b+zxiT1Vp7whgTCpS31kY5pz+sBcoBJ4FFwHvW2iRPVmuMsUAHa+3nxpjRQHZr7QBjzFLgGWvtBmOMB7AT6GKtXWOMyQAUstbuMMasBSZYa+cbYzLiGJ0OuJKjMSYfjhF3a4x5Gwi11r5tjIm21voaYx4B+gFNgKzABuAhHFNbnrHWNnPGOQnYYK2dcYv/YgtwaVfkbeyNu1eGolc/BOLCTqVcIMksXXDm+OWYjeEpF4gbeJYNjF++fOBkCkaS/NLnzRK//L/Dp1MwkuR1X64H4pfvpX16cduxFIwk+Xk8mD1++dLuNP5ZU8RxWJS7H7d46QL28yVvuvthk1VJv5Z/WmvLp3Qc7ubus3C8BrxijFmFoyC9YgpwENhqjNmCY+50AtbaI8CzwO/AFhzzm2/2lx7OAcWNMX/imDf9wvUdrLUXgdbAq87H3YxjvjNAF+ApY8xWYDVw/e+A7YDtzikYRYBZ17XPA7Y6Y/0NGGatPXqTeEVERETkLnDHRqBTmysjwSkdxx2kEeg0RCPQaZNGoNMejUCnTRqBvnPu1RFot8y9FRERERGI496dN5yW3PUFtDHmDyDjdau7pLHRZxERERFJJe76Atpa+1BKxyAiIiIi94675k95i4iIiIikBnf9CLSIiIjI3cBisffwuZPTEo1Ai4iIiIi4QAW0iIiIiIgLVECLiIiIiLhAc6BFRERE3MISpznQaYJGoEVEREREXKACWkRERETEBSqgRURERERcoAJaRERERMQFOohQRERExE10EGHaoBFoEREREREXqIAWEREREXGBCmgRERERERdoDrSIiIiIG1hribNxKR2G3AEagRYRERERcYEKaBERERERF6iAFhERERFxgeZAi4iIiLhJHDoPdFqgEWgRERERERdoBPouk6Gof0qH4DbpgjOndAhu4Vk2MKVDcJv0ebOkdAhuc1+uB1I6BLe4l/apx4PZUzoEt8lQ5N75rBH5NzQCLSIiIiLiAo1Ai4iIiLiFxVrNgU4LVEDfZaJ/+zulQ0hWvnXyxS+fXbw/BSNJXpnq5Y9fPjV/ZwpGkvwytywWv3xxR0QKRpL8PIoHxC9f3n8iBSNJXunzZ41fvhx2KuUCcYP010wlu3jqXMoF4gYemX3il89eSLvPX4BMXllv3UnkJjSFQ0RERETEBSqgRURERERcoCkcIiIiIm5gscRpDnSaoBFoEREREREXqIAWEREREXGBCmgRERERERdoDrSIiIiIm2gOdNqgEWgREREREReogBYRERERcYEKaBERERERF2gOtIiIiIgbWCxxaA50WqARaBERERERF6iAFhERERFxgQpoEREREREXaA60iIiIiJvoPNBpg0agRURERERcoAJaRERERMQFKqBFRERERFygAlpERERExAU6iFBERETEHazF6iDCNEEj0CIiIiIiLlABfY9Zs30DlXo3IVujolTr14LNe7cn6hN56jhV+jYje9MS5GxWkkaDOrDzwB4AXp75Dpnq5k90SY3Wbv+Tyn2b4tekKNUfa8HmfYlzjTp1nKr9mpOj+YMEtihF48Ed2HlgLwAvz3qH++sXSHRJjdbu2kjVp1uRvU1pag5pzZb9O2/YN+r0CfJ3rUqWVsV5b/50ANbt3kyDEZ0I7lSJ4E6V6PrqQKJOn3BX+Ldt9cZ1lGtVi0xl8vBQm3ps2rk1UZ+1m9dTs1NTAioXJKByQdo93ZPIE1Hx7f8cOcyjA7qSpUII/pUK0G34Y27M4Pat+vMPyjStjnfRnFRoUZuN27ck6rNm43qqtWlEtjJ5yVYmL22e6Ebk8au5tn2iO35l83FfgWw8NW6YO8N3yaoNaynTqCrehbJToWnNJHMFaPtYN/xKhnBfSBaeem5ogrZX3n+T4ErFub9oLjo80ZMzZ8+4I3SXrV67hnJVHyJT9qw8VLMqm7ZsTrLfgh++p2jZktyfIxv1mzXiQFhofNs/hw7xaMd2ZMkVgH9QIN369HRP8C7as3sP9Wo3wi9zTsqWqsiSxb/fsO/aNX9QuWJ1/DLnpHrl2mzedPU5UKJIae73zhZ/qfpQTXeEL3JTKqDvITEXY+k87gmiL5xjwmOjiDgZRZcXnuTy5cQ/J9WvWJOJTz1Pr+YdWbV1Hc9++DIALWs0Yvqot5k+6m3eHDAOgFIFirszjdsSczGWzi84cn2l/ygiTh6n6wsDbpBrDd4c4Mx123qe/eglAFpWb8y0kW8zbeTbvPHkWABKFSjm1jxuR8zFWLq99jTRF87xUs/hRJ46TrfXByWZK8CIqa8QczE2wbr94aFkuz8z47oOpn656ny/djFjZ73pjvBvW0xsDO0H9eTs+WheH/4CEccjaT+4V6I894X9TbYsWXlp0BgaVa/L/MU/MHLiiwBYa2k7sAdL1ixncPcneHnwc/hlyZYS6dxUTGwMbZ/oztlz0bw5ajzHjkfQbkCPxLmG/oVflmy8MmwsjWvVZ96ihYx4bVx8e8aMGWnZoKmbo3dNTEwMbft342x0NG+OeYljUZG0e6xbks/fjB4etGzYLNH6b3/6jjGvj6d8yTKMeGIQX/0wj+fefMkd4bskJiaG9t06cTY6mtdfmkBEZATtu3VOlOvRY8fo0qs792fKxCsvjGfjls30fqwv4HwOd+nAkqW/MXjAQF5+/kX8/PxSIp1b6tmtL3v37uPlCS+SIUMGunXuwenTib/YxMTE0LlDd6LPRvPKq+OJiIiga6eEz/eq1aowbeYnTJv5Cc+PH+vONESSpAL6OsaYpcaY8ndoW+OMMYeNMZudlwnO9U8aY/4yxlhjjNve+X5Zt5SIk1H0btGJPg93pmvjNoQe+YcVW9Ym6OefORvP9RhMg4dqUbNMZQDSpTMAFMtbmNZ1mtO6TnMuXIwBoGfzDu5K4bb9um6ZI9fmnejTojNdGrUh9Og/rNjyR4J+fpmzMab7YBpWrEmN0pUASJfO8bIolrcQrWs3o3XtZvEFZ89mHd2byG1YvHEFEaeO06tRe3o37kDnuo8QduwQK3esT9T31z9XsGj9Uga2Sjhi9Wj1Jswd+T49GrblrcfGAbD74F/uCP+2LVrxG8eOR9KvXQ/6t+9B90c6EnroIMvWr0rQr12TVnw7aTZ92nbl/bFvALDzr90ALF23ko07tzCwaz+G9h5A7zZdeHPEeLfncis/L1vMsagI+nfqyWOde9GjTWcO/BPG0j9WJujXvtmjzJ/8KX07dOfDFx1feHbs2x3fPnvix3Ru2c6tsbvq56XOXLv04rEuvenRzpnr2pWJ+s5+5xM6P9I+0fplzr6D+z7JyCefIYd/dmZ+PTfZY3fVosW/cCwign69+tC/d1+6d+5KaFgoy1YuT9Dvi2++IjY2lmGDhvBE38d4uGlzVq5Zzf4Df7N0xTI2bt7EwMefZOjTQ+jdvSdvvvJaCmV0Y1s2b2Xbtu20bvMIffv35skBj3PmzFkWzPsuUd9fFy0mIiKC3n170qdfL7p060xoaBgrll99DgSHBNGwUX1at3mEevXruDOVO8piibOX09TlXqUCOvm9Za0t7byMcK5bBdQDwtwZSNiRQwAE+uVwXPs7rg8c+SdR3x0H9pDv0Yo88mxPAv1yMOHxMQnarbVMX/g59/v40rZui2SO3HVhRx055fTLDkAuZ66hRw8m6rvjwB7ytXmIR0f2ItAvO68+NjpBu7WW6T98zv3evrSp0zyZI3dd2DHHfs2ZLQCAQGfOoUcT7tfoC+cY/PHzjOnyNLn9cyZo88jgEb/82ybHh1aV4nfke+Qdc+CwY9/lyp7Dee3I4cChhC+ja3P5dZXjJ+Nq5RxfBHftd0zPmffrD2QuH4LfQ/mZNOeT5A38Xzjwz5VcHTnmzhHoWH/wulw9rub6ywpHrtUrVHFHiHfMlf2XK8f1uYbe9jb8s/kDjkJ6/ZaNRJ08ztnosxw/mbqmIR0Ic+aa05FjrsBcjvWhoQn6hTqnawQm0W/XbscXpHnfLyBzoD9+QTmZ9PEHyR36bTl//jzHo45zPOo4O3Y4ppEFBjr2a2AuRy6hB0IT3S8szPF8z+nsmyu+79Xn+9xPvyBX9hDyBRdm1ow5yZaDyO26KwpoY4yPMeYHY8wWY8x2Y0w7Y8xzxpj1ztuTjTHG2XepMeYtY8xyY8wuY0wFY8y3xph9xpjxzj4hxpjdxpiZxpitxpivjTHeSTxuA2PMGmPMRmPMV8YYX+f6CcaYnc77vuFqPtbaTdba0P/43/LfWQuAwSRqypcrmPmvzmBMj0EcOX6Mt7+YnKB9+eY17D8cSru6LfH18nFLuP+FvUWu816ZzujugzhyPIK3v0xYUC3fvJb9h0NpW/fhuyRXx7XzJRHvnXlT8fbwok6pqkQ65zefOHuKU9Gn4/us3bWRAZPGUDp/cYa3e9xtMf8b8XkmsU/BMV+675inKVusFGOecMyXjb14EYAM993Hl+9MJyRXEM+8Ooa9ofvdEvO/Ff/8NUnnuurPP+g9YgDlSpRi7FOpd67z7bhVrknp37knRfIX4rk3XqLyw3XxzOgJgGfGjMkS451yu7le2+/qczgDX86ZS0hQMM88O5y9f+1L3mBvw9sT3yNvUCHyBhVi6ODhCdrs1TemW27n+v+Xbj26MHPONCZP/RCPDBkYOGAwoaFuHX8SSeSuKKCBRkC4tbaUtbYE8DMwyVpbwXnbC7h2YtxFa20N4CNgAfAEUALoboy5MuGxMDDZWlsSOAMkqBacUytGA/WstWWBDcBgY0xWoBVQ3HnfW/3+O+iaKRwNXUnaGNPXGLPBGLNh8uTJt77DLQTnzA3A4cgjAIRHHQMgJGduYi7GcvHSxfi+vl4+1C1fnWGdnyS3f07mLf0xwbamfu/4ebRX89Q3pQEgOEceAMIjjzqunbkG58hz41w7PeHIdVnCXKct/AxIxblmd+zX8OOOHI84r4OzJ9yvh6OOsvfw31R4sinjZk0E4O1vpzDlJ8e+XLVjA21e6EdIjjx889zkVPdlIW+uIAAOHXM+fyMc1yG5g4iJjUmwT1dsWEPz/u3JlyeEhZM/x9fbkUtILsfzonGNerSo05jGNephrSX0cOJfJlJS3jzOXI+GA3DYmXNIHmeuF6/munzdapr2bEP+oLz8OP1rfH183R/wf5A3dzAAh45cn2swMTEJc70Rv6zZ2PjTCtZ+9xu7ft9AYPYcBOXKjY93KnsOBztzDT8MQLgz55DghLmGBIcAcDiJfiFBjudG4waNaNGkGY0bNHI8h8NSvqDs0KkdCxZ+w4KF3/DN/C8BOHzYEfuRcOd+DXH8H8TGxhIb65gaFxzsyCnc2Tfc2Tc4xLF+6PAhtGzVgvYd2vJI61ZcvnyZv/al7i+9kvbdLeeB3ga8YYx5FVhorV1hjHnUGDMM8AayAjuA7539v7vmfjustUcAjDF/A3mAU8A/1torkyfnAE8B144mVwKKAauc34I9gDU4iu0YYIox5gdg4S1if8ta6/IoNYC1djJwpXK2/2Yb12pQsRb+mbMx9fvPyOTty6yfviI4R26ql6pE5gaFKBpSkHVTf2b2z1+z7a+dPFigGDv+3s0/EeGUK1wyfjuRJ6NYuOpXKpUoR/F8hf9rWMmifsWajlwXfoavtw+zf76S60NkaVSYoiEF+eOTn5jz89ds3b+LkvmLsv2AI9eyCXI9zsLVi6lUvBzF86bOXOuVrY7/A9mY9vMX+Hr5MGfJtwQF5KJa8Qr4tS5JkTwFWPPuAvo06UjD8o6j11duX8+Un+bSvlYLWlRuwJb9O2n7Yn+stXSt35rft6zG29OLxhVqp3B2VzWsXoeArH5M/mIGmbx9mfHtZwTnykPNClXxLhVIsQKF2TR/OZt2bqXFYx2wFnq27syS1cvw9vamWa2GNKpel4Csfsxb/AP5g/Ly7a8L8fX2oXSREimdXgKNatYjIJs/H382nUw+vkz/ag4huYOo9VA1MhYOoHjBImz5aRUbt2+hWa92WGvp1a4Li1ctxdvLm+Z1GwHw5Q/z2LBtMwA7/9rD1C9m06R2fXIG5EjB7BJqVKseAX7+fDxnmiPXL5y5VqpGxvx+FC9UhC2/rAHgy++/ZcO2TQDs3LeHqZ/PokmdBlhreX/mJxTKW4BFyxaz9++/eHvcqymZVpIa1mtAgL8/k6dNIZOvLzPmzCI4KJia1Wrg7fcAxYoUZdOa9bR9pDWjn3+ON955i2ORESz44XuqVqpM/rz5CMyRkwB/f+Z9v4D8+fLx7Xfz8fX1pXTJkrcOIJnlzRtC3rwh8bdLPFicb76eR9GiRZg6ZTqZMvnSoqVjGly5Ug9x/PgJjkQepH7DevgH+DP1k+n4+voye+YcgoODqF6jGju27+T5seOp36Au/7v8P+Z+9gVeXl4UL140hbL87+7lecNpyV0xAm2t3QuUw1EQv2KMeQ74AGhtrX0Q+ATwvOYuV04xEHfN8pXbV740XF+QXn/bAL9eM3+5mLW2l7X2f0BF4BugJY7R8LuCp0dGZo2dhI+XN8PefxH/LNmY9dx7pE+fPkE/vwey8su6pTz99hg++2UejSrVYcrIifHts37+mkv/u0SvVHhA3RWeHhmZNeY9fDy9Gf7BePwzZ2Xm6MS5ZsvszPXdMcz9dT6NKtVmyoiruc5e9BWX/neJns1S34GSV3h6ZGT60In4eHrz7NRX8HsgKzOGTkyUa5kCJXi4SkMertKQMs4zpxQLLkSh3PnYEbaH87EXuHAxhqGTx9N74lCGf/JySqRzQ54ZPfls4hR8vX0YMmE0/ln9mPvmlER5btu7k/MXLnAh5gIDx4+gy7D+DH55FABenl7MfWsqGT08GPjSs3h7evHF29MJcM6hTS08M3ry+XvT8PX2YdD4kQRk9efz96YlznXPDs5fOM+FmAsMGDeMTk/34ekXRsS3P/va80ycMgmApWtX0m/U0+z5O3UdHOrp6cnn70/H18eHQS88S0A2Pz7/YEaiXAGenTCOiZOd+axZQb8RA9mz/y/SmXQsWPQDj40axIp1axgzcDhPdOvj7lRuydPTk8+mz8bXx4chzw7D38+fuTNmJ8o1Z44czJ4ynVOnTzNizChKP1iSTz74GAAvLy/mzpjjeA4PHYy3lxdfzPqMAP+AlEjppqZOn0zBggUYOWIMFy9eYsbsaWTO/ECifp6ensyaMw0fXx+GDx2Jv78/M+c4nu/ZsmXl8uXLvDR+AuPGvEhQntx8+vms+PnSIinFxM9LSsWMMYHACWttjDGmJdAdqAyEAOmBtcDX1tpxxpilwDPW2g3GmFrO5WbO7SwFngGigANAFWvtGmPMJ8Bua+2b1/QJA/4E6lhr/3LOkc4NhAPe1toI53SOv6y1WW8Q9zgg+kYj0MaYUKC8tTYqqfbrWIDo3/6+ja53L986+eKXzy5Ouz/RZap39dzZp+bf+JzNaUHmlldP/XdxR0QKRpL8PIpfLWIu709dB7DdSenzX33Luxx2KuUCcYP0wZnjly+eOpdygbiBR+arU17OXki7z1+ATF5ZgRscQJGMCpXMbSctfNLdD5usGgY/+6e1NnUdde4Gd8sUjgeB140xccAl4DEco7/bgFAg8fm6bm0X0M0Y8zGwD/jw2kZrbaQxpjsw1xhz5UiU0cBZYIExxhPHi2+Qqw9sjHkKGAbkALYaY3601vb+FzmIiIiIiJvdFQW0tXYRsOi61RtwFLTX9611zfJSYOn1bcaYECDOWtv/Fvf/DaiQREgVbzPucTdY/y7w7u1sQ0RERNIKS5yNS+kg5A64K+ZAi4iIiIikFnfFCPSd5jwH8x077N4YMwpoc93qr6y1qe9vyYqIiIjIf3JPFtB3mrNQVrEsIiIicg9QAS0iIiLiBhadBzqt0BxoEREREREXqIAWEREREXGBCmgRERERERdoDrSIiIiIO1jL5TjNgU4LNAItIiIiIuICFdAiIiIiIi5QAS0iIiIi4gLNgRYRERFxA8d5oONSOgy5AzQCLSIiIiLiAhXQIiIiIiIuUAEtIiIiIuICzYEWERERcQtLnM4DnSZoBFpERERExAUqoEVEREREXKACWkRERETEBSqgRURERERcoIMIRURERNzAWoizOogwLdAItIiIiIiICzQCfZfxrZMvpUNwm0z18qd0CG6RuWWxlA7BbTyKB6R0CG6TPn/WlA7BLdIHZ07pENzGI7NPSofgNpm87o3nr8i/pRFoEREREREXaARaRERExC0slzUHOk1QAX2XOTV/Z0qHkKyunc6QlnO9Ns/IWRtSMJLk59+1fPzyhfWHUzCS5OdVIVf88v/2RqVgJMnrvkJ+8cuX959IwUiS37VTcS6eiE7BSJKfR1bf+OWTFw6lYCTJL4tX7pQOQe5ymsIhIiIiIuICFdAiIiIiIi7QFA4RERERN7BAXFxcSochd4BGoEVEREREXKACWkRERETEBSqgRURERERcoDnQIiIiIm5hidN5oNMEjUCLiIiIiLhABbSIiIiIiAtUQIuIiIiIuEBzoEVERETcwVri4jQHOi3QCLSIiIiIiAtUQIuIiIiIuEAFtIiIiIiICzQHWkRERMQNLBBn41I6DLkDNAItIiIiIuICFdAiIiIiIi5QAS0iIiIi4gLNgRYRERFxA4vlss4DnSZoBFpERERExAUqoEVEREREXKACWkRERETEBSqg7zFrd22k6tOtyN6mNDWHtGbL/p037Bt1+gT5u1YlS6vivDd/OgDrdm+mwYhOBHeqRHCnSnR9dSBRp0+4K3yX/NdcAbq99jQhnSuTpVVxhk4e746w/5U/9m6h5qgO5O5VhbrPdWZr6O4b9o06c5IiT9QjoFsF3v9xdvz6w8eP0vXtIYT0qU7+/rXo/9Fod4TukjVb1lOxYz0yV81H5S6N2LR7W6I+kSeP81CnBvjVKEhArcI06PcoO/Zf/f/4bunPlHikKlmq5adh/9aEHj7ozhRu26o//6BM85r4lMhFhVZ12LhjS6I+azatp3r7JviVz49f+fy0HdCdyBNRAESeiKLcw7V4oHQwWcqEUKdzC7bv3eXuNG7Lqj//oEzT6ngXzUmFFrXZuD2JXDeup1qbRmQrk5dsZfLS5oluRB6PumVbarP6jzWUq16JTDmz8VDtamzasjnJfgt++J6i5Utxf6Af9Vs05kBYKADWWka/MJaQYgW5P9CPBx8qy1fzvnFfAi7Ys3sfDWu3JEfm/FQsVZPfFi+7Yd8RQ56jcEgZsnrnof0j3RO0rV2znmoV65Mjc35qVW7Mlk2JX/d3DQtx9nKautyrVEDfQ2IuxtLttaeJvnCOl3oOJ/LUcbq9PojLl5N+AYyY+goxF2MTrNsfHkq2+zMzrutg6perzvdrFzN21pvuCN8ldyJXAI8MHjR7qG5yh/ufxFyMpeekYZyLOc8LHQcRefoEPScNv+GBKqM/fTNRrtZaur87jGU7/uDxJl14rt1T+GXK4o7wb1tMbAwdhvfl7LlzvDpoLBEnIuk0ol+S+7Rhldq8Pexl+jzShZWb/mDE2y8AcDQqgm6jnyCTTyZeHjCKTbu30ef5Qe5O5ZZiYmNo91QPos9F88azLxIRFUn7p3olynVf6H78Mmfl5Weeo3HNesz75QdGvP58fHujGnV5b+yr9OvQgxXr1zB0wnPuTuWWYmJjaPtEd86ei+bNUeM5djyCdgN6JJHrX/hlycYrw8bSuFZ95i1ayIjXxt2yLTWJiYmhfffOnI2O5vXxE4iIiKB9jy6Jcj167Bhd+vTg/kyZeGXceDZu3kzvJ/oBsGTp77z+zkRy5MjBK+PGE37kCL2f7M+lS5dSIqWb6tPtSfbt/YvxE8ZwX4b76NH5Mc6cPnPD/q1at0i0LiYmhu4d+hF99hwvvfockRGRdO/U/4bv5XJ3McZkNcb8aozZ57xO8oPHGBNqjNlmjNlsjNng6v2Twz1fQBtjWhhjRiTTtkOMMRecO/zKxcMYU8QYs8YYE2uMeSY5HjspizeuIOLUcXo1ak/vxh3oXPcRwo4dYuWO9Yn6/vrnChatX8rAVj0TrH+0ehPmjnyfHg3b8tZj4wDYffAvd4TvkjuRK8Ang16jfe3Eb+qpyZKtq4k8fYLudVvTs24bOtZswcHIcFbt+jNx3y2rWLR5BU827Zpg/cpdG9gSuov+DTvxVNNudK3VivGdhrgrhduyaPXvRJyIpG/rrvRr3Y1uLdoTGn6Q5X+uSdDPP0s2xvYfRsOqdahZoSoAxhgAvvplAbEXYxna7Ukea9eTFjUbsWrzH/x9KNTd6dzUz8uXcCwqkv4de/BYp570aN2JA4fCWLZuVYJ+7Zs+wryP5tC3fTc+eN7xRXbnvj0A+Gf144WnR9K4Zj1qV6oGQDrn/0Nq8vOyxRyLiqB/p5481rkXPdp05sA/YSz9Y2WCfu2bPcr8yZ/St0N3PnzRkeuOfbtv2ZaaLFr8C8ciIujXszf9e/Whe+euhIaFsmzligT9vvj2K2JjYxn29BCe6Nufh5s2Z+Wa1ew/8Hf8X7HLF5KXurVq88D995PJ15d06VLXx/nWzdvZvm0nj7Z5mN79u/P4gD6cPXOWBfN+TLL/hDdf4PEBvROtX7zodyIiIunZtyu9+nWjU7f2hIUeZOXyNUlsRe5CI4Al1tqCwBLn7Rupba0tba0t/y/vf0elrldcCrDWfmetnXA7fY2Dq/9n+507/MrlInACeAp4w9V4/4uwY4cAyJktAIBAv+wAhB79J0G/6AvnGPzx84zp8jS5/XMmaPPI4BG//NsmxwdcleLlSW3uRK53i4NR4QDkzOIPQGAWR85hkYcT9IuOOc/QmRMY3foJcmfLkaBt7+EDACzc8BvBfauTr38tPvnl8+QO3SVh4Y59F+jviD1XgGN/HQgPS9R3+/7dBDcsRcuBXQgMyMHrgx2jsqHhjukagQGObQRmd24jlU3jOHDIGaczvlw5AgH4+5+EuXp4XH09/rLyNwCqV6gcv27b3p0EVi5Ksz7tyZU9J2+OfClZ4/43DvzjyDWXM9fczlwPHLxJrit+B6B6hSq3bEtNruSUK6cjx1yBzlzDDiToFxrm6BeYM2fCfqGh1K9dl/69+vLNgnmUqlye4ydPMPPjaaRPn94tOdzM+fMXOB51guNRJ9i5w/EFJmeg87WWy5FL2IHEr9ebCQtzvu4DsyfYTuiB1PWalX/tYWCmc3km0NLN9//X0nQB7RwB3m2MmWKM2W6M+dQYU88Ys8o53F/RGNPdGDPJ2T+7MWaeMWaL81LFuY1dxpgPgI1AHmPM687tbTPGtHM1LmtthLV2PZCiv7lZ67g2141KvTNvKt4eXtQpVZVI5/zmE2dPcSr6dHyftbs2MmDSGErnL87wdo+7LeZ/67/kerdxpoohYa6TfpiJl4cntR58iMgzJwE4GX2aU+fOEPu/iwDcl/4+Zjz1OkF+gYz+bCL7j7r2YedO8fuUxKOq+XOH8N27n/Jcv6EciTzGxNkf3nQj1z8vUhsbH2fS7av+/IM+IwdStngpnntyaPz6AkF5+XHqlzw/cAThEUd5Y8p77gj3P7G32Cer/vyD3iMGUK5EKcY+Ney221KjG70vJe539f9kz769zP3qC+rVrsuXMz8lu38AfZ7sz7lz55I73Ft6d+KHFAwqRcGgUgwbPCZB26326227S16zctuyW2uPADivA27QzwK/GGP+NMb0/Rf3v+PuhT+kUgBoA/QF1gMdgWpAC2AkMP+avu8Cy6y1rYwx6QFfIAtQGOhhrX3cGPMoUBooBfgB640xy6/swCTkN8Zsdi6vstY+cbuBO58kfQE+/vhj+vbte4t73Fxw9twAhB8/BsAR53Vw9tzEXIwlnTF4ZPDgcNRR9h7+mwpPNo2/79vfTsHH04tn2vRn1Y4NtB//GHlzBvHNc5Px9fL5T3ElhzuV690gyM8xOhV+IgKAIycd10H+gY5c06XD474MHD5xjH1HQqk8vHX8fd/9YSbeGb0oGJgXgPqlqtK4bE3+/Gsbuw79RVhkOPlzBLs5o6QFB+YB4HCE46UWHum4DgkMIiY2xpGn8xcSX28f6lWqSb1KNZm+4DO+Xfw9H495k5DAIMc2jjm3EXHUuY08bs3lVvLmdsZ51PHrQrgz3ry5gx25mnTxo64r1q+mRd+O5A92FMu+Pr7x2/H18aV+tdrUr1abqV/N4eufFzDllXfdnM3N5c3jyPWQM9cr+yYkT1CiXJevW02LPu0pEJSPH6d/nSDXm7WlFnmDHK+lQ+GOX4fCjzhyDgkKISbG+Rz28CAk2NHvcLhz/x9x/p8EBzPv+wWcPnOaTm3b83CzFvz06yKmz5nFrj27KV+2nLtTSqB9p0epVKUCAJ5enjSp+wjhhx2xHwl3vNaCQhz7OzbWcRxGxowZb7rN4GDHazP8sOP+4c7tBIekrtfs7bPEpb0/pOJ37bxkYLK1dvKVG8aYxUCOxHdjlAuPUdVaG26MCQB+NcbsttYu/5fx3hH3QgF9wFq7DcAYswPHXBlrjNkGhFzXtw7QFcBaexk47ZyQHmatXevsUw2Y62w/ZoxZBlQAvrvB4++31pb+N4E7n4BXnoT2Zn1vR72y1fF/IBvTfv4CXy8f5iz5lqCAXFQrXgG/1iUpkqcAa95dQJ8mHWlYviYAK7evZ8pPc2lfqwUtKjdgy/6dtH2xP9ZautZvze9bVuPt6UXjCrX/a3h31J3IFeDblT+x6a8dAOz5Zz+zfv2aBuVqkiOrf4rldr26Javgd39WZv72Db6e3ny27DuC/AKpWrQcOXtUokiufCx/+Qt61WtLg9LVAVi160+mLfmKtlWb0rxCXXL75cDv/qws3PA7ebPn4fsNv+Hj6c2DwYVTOLurGlapTUBWPz75Zja+3r7M/O5zgnPmoUa5ymSqHEyxfIXZ8PkSZn3/BVv37qBkoeJs37eLf44eplyxUgC0btCCMe+/wpuzPyDiRCTfLfuZKqUqki93SMomd51GNeoSkM2fj+fOwNfHl+lff0pIriBqVqyKZ7EcFC9YhM0LV7Bxxxaa9emAtZZebbqwePUyfLy8aVanITO++Ywtu7ZTqmgJtu3ZycHwQ5R/sExKp5ZIo5r1HLl+Np1MPr5M/2oOIbmDqPVQNTIWDqB4wSJs+WkVG7dvoVmvdo5c23Vh8aqleHt507xuo5u2pSYN6zUgwN+fydOnksk3EzPmzCI4KJia1arjHZCZYkWKsmnVOtq2as3oF8byxrtvcSwiggU/fE/VSpXJnzcf+UPyATB5+hQuxFzgx19+TlB0p6SQvMGE5L0aR/EHi/Lt199RpGghpk2ZjW8mX1q0bAxAxVK1OHH8BP9EOubs//LTEnbtdCwfPhzOrOlzqVq9EvUa1sY/wI9pn8zG19eHT2d+TlBwHqrVqJw4AEkpUdfNS07AWlvvRm3GmGPGmJzW2iPGmJxAxA22Ee68jjDGzAMqAstx1GG3vH9ySNNTOJyuPd1A3DW347j9LxDX/jZ21/5u5OmRkelDJ+Lj6c2zU1/B74GszBg6MdHcuTIFSvBwlYY8XKUhZQoUB6BYcCEK5c7HjrA9nI+9wIWLMQydPJ7eE4cy/JOXUyKdm7oTuQKMmzWRSQscp7VbsX0dAz8Yy1/hCecrpjRPj4xMfeIVfDy9GP3pm/jdn4UpT75C+nQJcy2dtxjNK9SleYW6lM5bFICiufNTMDAELw9Ppj05gYwZMjBi9mt4eXgyfcBr+N+fNSVSSpJnRk/mvPwRvt7eDJ04Fv8sfsx55aNE+9QvczYWrf6NpyY8y2c/fk3javWY/oJj6kJOv+zMGD+J02fP8Oy74ylVqASTx05MiXRuyjOjJ5+/MwUfHx8GvzwK/2x+zH1naqJct+3ZyfkL57kQc4GnXhhO58F9eXr8s4DjIMKfly/mibFDmbPgS5rWbsCsN24wlSUFeWb05PP3puHr7cOg8SMJyOrP5+8lntO7bc+O+FwHjBtGp6f78PQLI27Zlpp4enry2bRZ+Pr4MGTkMPz9/Zk7bVaiXHPmyMHsydM4dfo0I8aOonTJknwy6SMAWjZvwZABTxN28CCDRgwlS+YszPhoCn7Z/FIipZuaPP09ChTMz+gRL3Lp4iWmzf6ABzI/kGTf997+iOfHvALAjm27ePqJYfyxZj2enp5Mn/MhPr7ePDt0HH7+fkyf82GqmPMtd8R3QDfncjdgwfUdjDE+xphMV5aBBsD2271/cjFX5iWlRcaYEGChtbaE8/YM5+2vr7ThOJCvvLX2SWPM58Baa+3bzikcPkDW67bxCNAPaOJs2wA8ZK09eqvHT6J9HBBtrb2dgwktwKn5Nz6XcVqQuWWx+OW0nOu1eUbO2nCTnnc//65XByYurD98k553P68KueKX/7c3dZ6H+E64r9DVYu3y/tR5Hvg7JX3+q18iL56ITsFIkp9H1qvTXk5eOJSCkSS/LF65IQUGxEKK+dlRc1L3mZ1c1bfc9D9vNgJ9M8aYbMCXQBBwEGhjrT1hjAkEplhrmxhj8gHznHe5D/jMWvvSze7/3zK6PffCFA5XDAQmG2N6AZeBx4Dr5zbPAyoDW3AUtcOSKp5vxhiTA0fhfT8QZ4x5Gihmrb3xCTJFRETkrmYh/lSEAtba40CiP7bgnLLRxLn8N47jzm77/u6Qpgtoa20oUOKa291v0DbDue4YjlOiXO/abVhgqPPi0uNfs/4okPtW9xcRERGR1OdemAMtIiIiInLHpOkRaHcxxjwIzL5uday19qGUiEdEREREko8K6DvAeZq80ikdh4iIiKRmlss2zZ0H+p6kKRwiIiIiIi5QAS0iIiIi4gIV0CIiIiIiLtAcaBERERE3sBbi4jQHOi3QCLSIiIiIiAtUQIuIiIiIuEAFtIiIiIiICzQHWkRERMQtLHE2LqWDkDtAI9AiIiIiIi5QAS0iIiIi4gIV0CIiIiIiLtAcaBERERE3sOg80GmFRqBFRERERFygAlpERERExAUqoEVEREREXKA50CIiIiLuYC2XreZApwUagRYRERERcYEKaBERERERF6iAFhERERFxgbHWpnQMcnu0o0RERO4c4+4HzFMki316Wm13P2yyeqbqvD+tteVTOg5300GEIiIiIm5gscTFxaV0GHIHaAqHiIiIiIgLNAJ9lzn57faUDiFZZXmkRPzyia+2pWAkyStrmwfjl49+tCoFI0l+OfpXjV8+tyIsBSNJfj7Vg+OXY7ceTcFIklfGkjnily/uiEjBSJKfR/GA+OVLkWdTMJLkl8E/U/zy6QvHUjCS5PeAV/aUDkHuchqBFhERERFxgUagRURERNwkTn9IJU3QCLSIiIiIiAtUQIuIiIiIuEAFtIiIiIiICzQHWkRERMQdrCUuTnOg0wKNQIuIiIiIuEAFtIiIiIiIC1RAi4iIiIi4QHOgRURERNzAApd1Hug0QSPQIiIiIiIuUAEtIiIiIuICFdAiIiIiIi7QHGgRERERt7DExcWldBByB2gEWkRERETEBSqgRURERERcoAJaRERERMQFmgMtIiIi4gYWiNN5oNMEjUCLiIiIiLhABbSIiIiIiAtUQIuIiIiIuEAF9D1m7e5NVBv0CDnalaXWM23Z8vfOG/aNOn2CAt2rk/XRB3lvwYz4dTWGtCZ3x4oEdXqIZmO6s/PgPjdF75o/dm+i+jOPkrNjOWoPv0WuZ05QsFcNsrUtyaTvZsSvqzm0DXm6VCSoayWaj+3BrlSa67r9W6kzvhvBA2pT/+WebD2454Z9o86epNgzTcn5WDU+/PWz+PXv/DSLss+2It/AevSb8hxnL5xzR+guWbNtAw91b0TWuoWo2qspm/dsT9Qn8tRxKvdsTECDouRoWJyGA9qy4+89ifoENS+Db40Q3pk72V3hu2T1pnWUb12H+8sHU6ldfTbt2pqoz9otG6jVtTnZqxUme7XCtB/ci8gTUQC8+OEbeJbKmeiSGq3euI5yrWqRqUweHmpTj007k8h183pqdmpKQOWCBFQuSLune8bnCpCxRPYEl9ZPdXNjBrdv9R9rKVuzMr65/ahYtzqbtm5Osl+7nl0IKBiER8D9DBwx5LbbUpM9u/fSoHYzAjIHUb5UVX5bvPSGfdeuWUeVirUJyBxEjcr12bzp6nPgg0mTebBIeQIyB1GyaAU+/nCKG6JPJtYSF5e2LvcqFdD3kJiLsXR/fRDRMed5qccwIk8fp/vrg7l8OekDGp6dNoGYi7GJ1tcrU43X+4yiR8N2rN75J6NnvJ7cobss5mIs3d8cTPSFc4zvNpSIUyfoMXEIl+OSznXk9FeJuRiTaH3dMtV4rdcoejZoy+pdfzJ61hvJHbrLYi7F0nvyaKJjzvN86wFEnTlBn8mjb5jrc1++Q8ylhPt14calTPhuMqWCi/BUoy589+dvvPrdJ+4I/7bFxMbQaUx/os9HM+HJMUScjKLzc48l+fxt8FAt3hr0Ir1bdmbVlnU8O2l8gvZh7z5PTGzi/Z1axMTG0GFIb86ei+b1oc8TcTyKDkP6JMp1X9jfZMuclfEDR9GwWl3mL/mRkW85cn2kXjNmTfiQWRM+5O1nXwagdJESbs/lVmJiY2g/qCdnz0fz+vAXiDgeSfvBvZLONUtWXho0hkbV6zJ/8Q+MnPhigj6t6jdj9msfMfu1jxjU/XF3pnFbYmJiaNezM9HnonnjhVeIiIygfc+uST6HM3p48HCTZklu52ZtqUmvbo+xd+9fvDRhHBky3Ef3zn04ffpMon4xMTF07dCL6LPRvPzq80RGRNKtU28uX77M/r/+ZuSw50iXLh0vTRjH/y79j+FDRnPo0OEUyEjkqru+gDbGtDDGjEjpOFxhjBlnjHnGuTzDGNPaHY+7eNNKIk4dp2fDdvRq1J5OdVoRFnGYlTvWJ+r768YVLNqwjKda9kiw3u+BrIzqMID6ZatT/cGKAKQzqe9ptHjzSiJOO3Nt2J7ON8l18aYV/PznMp56uGeC9X73Z2VU+ycduZa4kqtxS/yu+G3HWiLPnKB7jVZ0r/kIHao24+DxI6zeuylR3yXb1/DLtlU82aBTgvVr9jn6Pla/A0837kbA/dn4Yu2Pbon/dv3yx1IiTkTRp1UX+rbqQtem7Qg98g/LN69N0M8/czae6/0MDSrXpmbZygCkS3d1v/2y9nd+WrWEQR37uzV+Vyxa+RvHjkfSr113+rXrTrdWHQg9fJBlG1Yn6NeucUu+eXcmfdp05f0xrwGwa79jtL14wSK0bdySto1bcsH5ZaF3m67uTeQ2LFpxJdce9G/fg+6PdCT00EGWrV+VoF+7Jq34dtJs+rTtyvtjHV9kd/61O0GfovkL0bRWA9o2aUXVsg+5LYfb9fOSXzkWGUG/7r3p37MP3Tt24cDBUJatWpGo76yPptK5bYckt3OzttRiy+ZtbN+2g9ZtWtKnf08eH9CPM2fOsmDewkR9f130GxERkfTu253e/XrQuVsHwkIPsnL56vi/2hcYmJNadWoQkN2fjBkz4pkxo7tTEkngrj+NnbX2O+C72+lrjDGAsda6/Hc0jTHprb27zz0TduwQAIHZApzX2QEIPXaImtf0i75wniEfv8iYTgPx8fROtJ2dB/dR85k2AOTMGsDLPYYlb+D/wsEIx+hEzqwJcw07dggevNovOuY8Qz4Zz5iON8611rC28dt6qfvwZI7cdQejjgCQI7MfADkz+wMQFhVO9Wv6nYs5z/C5bzCyZX98Mnol2Ea2TFkAWL13Ex7pM3Ai+hT/i7vMiejTZPV9IPmTuA1hR5zPX78cAOTyd1yHhh+EclUT9N3x926q9Grq6O+fg1cHPAdA9PlzDHxzNOP6DsPXO/H+Ti1CDx8EIDDAmWt2x9SLA4fC4KGre9Ujg0f88q+rlwJQrVylBNuy1jL16znc75uJ9k0eSc6w/5UDzlxzZU8i12skyHXV7wBUK1c5QZ9XPn6Llz+aSFDO3Lw96hWa1mqQbHH/G6EHHTkF5nTkmDswFwAHwkJTKqQ76vz581w4fwGAnTt2AZAz0JFrrlyBAIQdCEt0v7Cwg0n2DT0QRs3a1Rn34iief+5lKpSuRrp06Xj/47fx8/dL3mREbiH1DR1ewxgTYozZbYyZYozZboz51BhTzxizyhizzxhT0RjT3Rgzydk/uzFmnjFmi/NSxbmNXcaYD4CNQB5jzOvO7W0zxrS7yePXMsb8boz5DNhmjEnvvO96Y8xWY0y/a/oOc25vizFmgnNdH2ffLcaYb4wxqewT2zF3yVw3qvrO/Kl4ZfSkdukqRJ05AcDJs6c4FX0agLw5gvh6zMeM7PAkR09G8u6C6e4N+1+wNulc350/zZFrycpEnT4OwIno05yKdvzMmDdHEF+P+ohn2zlyfe9uyPXKfiVhrpN++RQvD09qFq1A1NmTAJw4d4ZT587QrUZLCuQI5tXvPqHxq33I6CxWPK8pWlKbG+1TgHy5QljwxizG9BrCkahjvD33YwAmfvYRXhk9qVuxOpEnnfv7zElOnj3tvsD/hZvlCo750v3GDqJssZKMfuyZBG1L163ir4N/06Hpo/h6+yR7rP+VM9VEz98rVm9cR98xT1O2WCnGPDE0fv0zvZ7ki7en8cG4Nzh55hRdh/Xn/IXz7gj5X7vVfr3bvDPxffIHFSd/UHGGDR6ZoM2VXK/tGxUZxeQPp/FgyRJ8+sUMSjzo2PbhQ+F3PgE3cJwHOi5NXe5VqbqAdioAvAOUBIoAHYFqwDPAyOv6vgsss9aWAsoCO5zrCwOzrLVlgPJAaaAUUA943RhzsyNrKgKjrLXFgF7AaWttBaAC0McYk9cY0xhoCTzkfOzXnPf91lpbwblul/P+t80Y09cYs8EYs2Hy5P9+oFNw9twAhB8/5ryOcKwPyEXMxVguXrrkWB91lH2HD1BxQHPGzX4LgLfnTWXKz58D4OvlTZ3SVXimdT9yZcvBgtWL/nNsd1pQgGNk50quR05ExK+PuRjLxf85cj183JHrQ0+34PlP3wYcXyCmLHLm6ulN7VJVeObRvo5c16TCXP0cT98jpyIBOHoqKn59zKWruYafjOCvo2FUG9eR8fM+BGDSojlMX/Yt2Xwzs2TUDH4aMYVVz88lxwN+5MqaHe/rRqpTUnBOx/P3cKRjxD086phzfR5iYmO4eOlifF9fbx/qVqzB8G4DyB0QyLe//+C4b0Q4ew/up0ynOoz5aAIAb376IZO/neXOVG4pJFcQAIePOXONOBq//vpcV/y5hhaPdyRf7hC+/3BuoiJ5yteO3PqkwukbAHmduR6Kz9VxHZI7iVw3rKF5//bkyxPCwsmfJ8j1pUFjeLhuE3q17kLdyjWJPn+Of46mriIrJCgYgMPhjrgOHwmPXx8TE8PFixdveN+7QYdObZm/8EvmL/ySr+Y7DlAOP+zIMTzcsV+DQhz7OzY2lthYx7EYwcFBSfYNDglixfJVhIcfofnDTWjavBHNH27C2bPRrF+3wX2JiSThbpjCccBauw3AGLMDWGKttcaYbUDIdX3rAF0BnNMtThtjsgBh1torEyWrAXOd7ceMMctwFMM3mgayzlp7wLncACh5zZzlB4CCOArx6dba887HPuFsL2GMGQ9kBnwBl6ova+1k4Erl/J8Pda1Xphr+D2Rl2qIv8fX04dPf5hEUkItqxSvg37Y0RfIUYPXb8+jduCMNyjsmdazavp4pP39O+1otaFGpPp/+No9tB/bwYN7C7Ajby6GoI5QtkPoOTKpX2pHr9F++xNfLhzm/zSPIP5BqxSsQ0L4MRfLkZ9Wb8+jdqAMNy9UAYOWODUxd9DntajR35Pr7fLaH7qZESBF2OnMtkz/15VqneCX8MmVh5vL5+Gb0Zu6qheTJlpMqhcqQ+4maFM6Zl6XPzaZHrUep/2AVwDFVY/qyb2nzUCOala3N0VNRTFv6Dfmz5+H3HX+wP+Ifxrd9OmUTu06Dh2rhn8WPKfPn4Ovtw6wfviA4R25qlK7EA7XzUzRvIdbP/IXZP37J1n07KVmwGNv37+afY4cpV6QUAP0e6UajKnUBWLFpLZPnzaJjw0doWatxSqaWSMNqdQjI6sfkL2eSyceXmfPmEhyYh5rlq+BTNjfF8hdm47dL2bRrKw8/3gmLpeejnViyZjk+Xt7xUxcijkfx3W8/U7l0BUoULJrCWSWtYXVnrl/MIJO3LzO+/YzgXHmoWaEq3qUCKVagMJvmL2fTzq20eKwD1kLP1p1ZsnoZ3t7eNKvVkJ+WL2buwq+pUaEKp86cZtHK3/DPmo28uYNSOr0EGtWtT4CfP5NnTiWTry8zPptNSFAwNatWxytnFooVKcrm5X8A8OX8b9i42XFswq69e5g2ZyaN6zckZ/YcN21LSSF5gwnJGxx/u/iDxfjm6wUUKVqYaVNmkimTLy1aOqZWlS9VlRPHT3A48m/qN6yDf4AfUz+Zia+vL3NmziUoOA/ValRh6+ZtAHz5+Tdkz5Gdr774FoD8BfK7P0GRa9wNBfS1pwuIu+Z2HLcf/7Xn43L1t7Lr7zvAWpugEDbGNCLpAncG0NJau8UY0x2o5eJj31GeHhmZPuRNhk55iWenT6BIngK83X8c6dOnT9CvTIHilClQHIBzzp9AiwYVpFDufBw4+g+LN65gxi9f4uPpTcNyNRnffWiix0ppnh4ZmTboDYZNfZmR01+lSJ4CvNVvLOnTXZdr/uKUye/INTrGMXevWFBBCuXKy4GjB1m8aSUzfv3KkWvZGrzYLRXmmiEjk/u8yMjPJzLmq3conDMvr3cenijX0sFFKB1cBIBzsY5ci+bKR8EcwUScPs5PW5ZzMCqcLD73M6RpD3rWetTtudyMZ0ZPZj//PoPfeo5h775A0bwFmTR0QqLnr1/mbPyydilTv/sMXy9vGlepyytPjAagbJGSlC1SEoBzztP0Fc9XhMLBBdybzC14ZvTk0zcm8/TLIxny6hiK5S/MB2NfT5Trtr27OO983g58+VkAggJzxxfQM+fP5dL/LqXa0Wdw5PrZxCkMfGkEQyaMpliBwnw47s0kct3J+QvOXMc7jhsPDsxDs1oNCQ7MzdHIY4yc+CKXL1+mXPFSvPrMuATzplMDT09P5k6dxVMjhjB49HCKFS7KRxPfTZQrwKgXxxL2j2Nu8NKVy1m6cjm/zvuBnNlz3LQtNZky/QOeenwIo0aMI09QbqbPnkzmzImPqfD09GTGnE8YOuhZRgwdQ5GihXn3/TdInz49ZcqVZvyEcUz+cCpDBz1LjpzZeX3iyzxYsngKZCRylbky1yg1MsaEAAuttSWct2c4b399pQ14AyhvrX3SGPM5sNZa+7YxJj3gA2S9bhuPAP2AJs62DTimXhxN4vFrAc9Ya5s5b/d13q+NtfaSMaYQcBioDjwH1LPWnjfGZLXWnjDGRAHFgJPAj8Bha213Y8w4INpa+8a1Od3iv8MCnPw28Xlv05Isj1wd4T3x1bYUjCR5ZW1z9UjGox+tuknPu1+O/lcP8Du3IvEBRGmJT/Wro2+xWxO9paQZGUteLdQu7ohIwUiSn0fxgPjlS5FnUzCS5JfBP1P88ukLx1IwkuT3gFd2cH1A7T/LWSiT7TWprLsfNlm91HD5n9ba8ikdh7vdDSPQrhgITDbG9AIuA48BR67rMw+oDGzBUZQOS6p4voEpOKaNbHSe0SMSxwjzz8aY0sAGY8xFHMXySGAM8AcQBmwDMiW1UREREbk33Mt/fCQtSdUFtLU2FChxze3uN2ib4Vx3DHg4iU1duw0LDHVebvX4S4Gl19yOw1EYX3/wItbaCcCE69Z9CHyYRN9x1yx3v1UcIiIiIpJ63A1n4RARERERSTVS9Qi0uxhjHgRmX7c61lqb+v6UlYiIiIikKBXQgPM0eaVTOg4RERFJw6zmQKcVmsIhIiIiIuICFdAiIiIiIi5QAS0iIiIi4gLNgRYRERFxA4slLhX/ATu5fRqBFhERERFxgQpoEREREREXqIAWEREREXGB5kCLiIiIuElcXFxKhyB3gEagRURERERcoAJaRERERMQFKqBFRERERFygOdAiIiIibmAtxMXpPNBpgUagRURERERcoAJaRERERMQFKqBFRERERFygOdAiIiIibhJnNQc6LdAItIiIiIiIC1RAi4iIiIi4QAW0iIiIiIgLNAdaRERExC2szgOdRhiryex3C+0oERGRO8e4+wGzF/Cx7V4r4e6HTVbvPbruT2tt+ZSOw900hUNERERExAUqoEVEREREXKA50HeZ419sSekQklW2dqXil9NyrtfmeWzymhSMJPll71s5fvncirAUjCT5+VQPjl+O3XwkBSNJXhlL54xfvrgjIgUjSX4exQPily9FnE3BSJJfhoBM8ctnLkSmYCTJ734v/xR5XGshzsalyGPLnaURaBERERERF6iAFhERERFxgQpoEREREREXqIAWEREREXGBDiIUERERcRP9IZW0QSPQIiIiIiIuUAEtIiIiIuICFdAiIiIiIi7QHGgRERERN7BoDnRaoRFoEREREREXqIAWEREREXGBCmgRERERERdoDrSIiIiIO1iLtZoDnRZoBFpERERExAUqoEVEREREXKACWkRERETEBZoDLSIiIuImcXFxKR2C3AEagRYRERERcYEKaBERERERF6iAFhERERFxgeZAi4iIiLiBBeLidB7otEAj0PeYP/ZsosawNgR2rkDtEe3ZcmDXDftGnTlBoT618Gtfmknfz7ztttTiXsp13V9bqf1CF4KeqEn98d3ZenDPDftGnT1JscGNydGvCh/88ln8+nd+nEmZ4Q+Td0Ad+k4ew9kL59wRukvWbNvAQ90bkbVuIar2asrmPdsT9Yk8dZzKPRsT0KAoORoWp+GAtuz4e0+iPkHNy+BbI4R35k52V/guWb15HeXb1uX+iiFU6tCATbu2JuqzdssGanVrTvbqRchevQjth/Qm8sRxAKy1jH73ZfLWL8MDD+WlZKvqfLVogbvTuC2rN66jXKtaZCqTh4fa1GPTziRy3byemp2aElC5IAGVC9Lu6Z5EnoiKb//nyGEeHdCVLBVC8K9UgG7DH3NjBrdv9bq1lK1VGd88flSsV51NWzcn2a9dry4EFArCI/v9DHx2yG23pSZ7du+lfu0m+GfOTblSlVmy+Pcb9l275g+qVKyJf+bcVK9cl82bHM+BT2d/zgPeAYkuYWEH3ZWGSJJUQN9DYi7G0mPiM0RfOMf4rs8Qefo4Pd96hstxl5PsP3LGa8RcjHG5LTW4p3K9FEuvj0cSHXOe59s8ReTZE/T+eNQNcx3zxdtcuBSbYN3Cjb/zyoKPKR1SlIGNu/Ldn0uYsOBjd4R/22JiY+g0pj/R56OZ8OQYIk5G0fm5x7h8OXGeDR6qxVuDXqR3y86s2rKOZyeNT9A+7N3niYlNxfs0NoYOz/Th7LloXn9mHBHHo+gwtG+iXPeF/U22zFkZP3AUDavVYf5vPzLynRcBWPLHct6YPokc/gG88vRowiOO0mfsIC5dupQSKd1QTGwM7Qf15Oz5aF4f/gIRxyNpP7hX0rlmycpLg8bQqHpd5i/+gZETHblaa2k7sAdL1ixncPcneHnwc/hlyZYS6dxUTEwM7Xp2Jjo6mjdeeIWIyAja9+qa5HM4o4cHDzduluR2btaWmvTq1o+9e//i5QnPkyFDBrp37s3p02cS9YuJiaFLh56cPXuOl199gciISLp26snly5epWr0yU2d+zNSZHzN56vt4eHgQEOBPYGDOFMhI5Ko0WUAbY5YaY8qndBw3Y4wJNcb4OZej3fGYSzavIuL0cXo0aEvPBu3oVLsVYRGHWbVjQ6K+izetZNHG5Qxo0cOlttTiXsr1t+1riTxzgu41H6FHrUfpULU5B6PCWb1nU6K+S7av4Zetq3iyYecE66/0fax+R55u0p2A+7PxxZof3RL/7frlj6VEnIiiT6su9G3Vha5N2xF65B+Wb16boJ9/5mw81/sZGlSuTc2ylQFIl85c3c7a3/lp1RIGdezv1vhdsWjV7xw7Hkm/tt3o17Y73Vq2J/TwQZZtWJ2gX7vGLfnmnZn0ad2F90e/BsCu/XsBsM6fifPlDqFupRo84Hs/mbx9SJcudb3tL1rxmyPXdj3o374H3R/pSOihgyxbvypBv3ZNWvHtpNn0aduV98e+AcDOv3YDsHTdSjbu3MLArv0Y2nsAvdt04c0R4xM9Vkr7+bdfORYZQb8evenfow/dO3bhwMFQlq1akajvrA+n0rlthyS3c7O21GLL5m1s27aD1m1a0ad/L54Y0J8zZ86yYN73ifr+umgJERGR9O7bnT79etKlW0fCQg+yYvkqQkKCad2mFa3btCKjpycXL16kc9eOZMiQIQWyErkqdb2TpnLGmLt6znhY5GEAcmYJACAwq+M6NOJQgn7RMed5ZupLjGn/FLmz5bjtttTkXsr1YFQ4ADmz+AMQmNlxHRZ1OEG/czHnGfbpa4xs1Z9cWbMnaPO7PwsAq/duZFPoTk5EnyI65jwnok8nd/i3LeyIY98F+jn2RS5/x3VoeOKfcnf8vZu8LcrRamh3Av1z8OqA5wCIPn+OgW+OZlzfYeTJHuimyF0XetiRU2CAM1dnrAcOJ8zVI4NH/PKvq5cCUK3sQwDUq1yT/u26882v31PqkZocP32SGS+/T/r06ZM7fJdcySlX9iu5OkYWDxwKS9AvQa6rHFMBqpVzfEG68qVh3q8/kLl8CH4P5WfSnE+SN/B/IfSgI6fAHI4cc+fMBcCBsNCUCumOOn/+PMejjnM86jg7d+wEIKdzpDgwl+M69EBYovtdmY4RGN83MMm+06fOJF26dHTv1SV5EnAHC3HWpqnLvcptBbQxxscY84MxZosxZrsxpp0x5jljzHrn7cnGGOPsu9QY85YxZrkxZpcxpoIx5ltjzD5jzHhnnxBjzG5jzExjzFZjzNfGGO8kHreBMWaNMWajMeYrY4yvc/0EY8xO533fuEncM4wxE40xvwOvGmPyG2N+Nsb8aYxZYYwp4uyX3Rgzz5nfFmNMFef6+c6+O4wxfZPhv/Zfu/K0d/63x3tvwXS8PDypVbIykWdOAHAy+jSnos/ctC01U64wadEcvD08qVWsIlFnTwJw8txpTp07Q7earSiYI5gJCybT+JXeZMyQEYCM1xQtqY11vnFfnydAvlwhLHhjFmN6DeFI1DHenuuYjjLxs4/wyuhJ3YrViTzpmCt84sxJTp5NPV8UkhKf6w3aV29eR7/nB1O2aElG938GgL2hfzH3h2+pV7kmX7w5lYCsfvQdO4hzF867Kep/58rnsblBtqs3rqPvmKcpW6wUY54YCkDsxYsAZLjvPr58ZzohuYJ45tUx7A3d75aY/y3LjZ/Dd6N3Jk4iX1BR8gUVZejgZxO03ez1er2k+v799wGW/b6CevXrEBwcdAejFvl33Dmi2ggIt9Y2BTDGPAD8aq19wXl7NtAMuPL7zkVrbQ1jzEBgAVAOOAHsN8a85exTGOhlrV1ljJkGPA7EF8POKRKjgXrW2nPGmOHAYGPMJKAVUMRaa40xmW8ReyHnNi4bY5YA/a21+4wxDwEfAHWAd4Fl1tpWxpj0gK/zvj2ttSeMMV7AemPMN9ba47fzH+YsuPsCfPzxx/Tt+9/q72B/x2hH+IljABw5ERG/PuZiLOnSpcPjvgwcPn6UfeEHqDT44fj7vrNgGt4ZvW7aNuSRPv8pvjvpXso1yM8xWhN+0pHjkVORjvXZAom5FEs648z1ZAT7joZR9bn28fd97+fZeHt4MqhpD5aMmcXOQ39xv5cvXd4fSsylWHwyerk/oRsIzpkbgMORRwAIjzrmXJ+HmNgYxz51Fvy+3j7UrViDuhVrMGPh53z7+w98OOJ1DkeEs/fgfsp0qhO/3Tc//RBvT2+Gdxvg5oxuLCSXo0A4fMyZa8SR+PXX57riz7W0eqoL+fOE8P0Hn+Hr7QPAwmW/cDr6DB2btubhOo35acViZsyfy66/91K+eGn3J3UDeZ25Hro+19xJ5LphDS0f70T+oLwsnPx5fK4hufIA0LhGPVrUacy6LX+yfd8uQg8fpFBIfnendEMhQcEAHD4SnuA6JCiYmBhnrh6p90vrrbTv1JZKVRy/gHh5edKwbnPCDztyPBJ+FIDgEMf+jo11HIeRMWPG+IL4cHzfIwn6AkyfOgtrLb36dE/+RERugzsL6G3AG8aYV4GF1toVxphHjTHDAG8gK7CDqwX0d9fcb4e19giAMeZvIA9wCvjHWntlotwc4CmuKaCBSkAxYJXzm6wHsAY4A8QAU4wxPwALbxH7V87i2ReoAnx1zTfjjM7rOkBXAGvtZeDKkNZTxphWzuU8QEHgtgpoa+1k4MopAv7z7yR1S1fF/4GszPj1K3w9ffj093kE+QdStXh5sncsR5Hc+Vn5xjf0atieBmVrALBq5wam/vIF7Wo0o8VD9YiOOX/DttTkXsq1TolK+GXKwsxl8/D19Gbuqu/Jky0nVQqXIddj1SkcmJdlYz+lZ61Hqf9gFQBW793E9KXf0KZSY5qVq83RU5FM+/0b8mXPw+871rL/2EHGtxuUwpkl1OChWvhn8WPK/Dn4evsw64cvCM6RmxqlK/FA7fwUzVuI9TN/YfaPX7J1305KFizG9v27+efYYcoVKQVAv0e60ahKXQBWbFrL5Hmz6NjwEVrWapySqSXSsGptArL6MfmrWWTy9mXm/M8JDsxDzfJV8Cmfh2L5C7Px69/ZtGsrDz/ZCYul5yOdWLJ2OT5e3jSt2YB8uUMAmPzVTC7ExvDTisV4ZPAgJDBPyiZ3nYbV6zhy/WIGmbx9mfHtZwTnykPNClXxLhVIsQKF2TR/OZt2bqXFYx2wFnq27syS1cvw9vamWa2GNKpel4Csfsxb/AP5g/Ly7a8L8fX2oXSREimdXgKN6tQnwM+fyTOmksnXlxmfzSYkTzA1q1bHKzALxQoXZfPyPwD4cv43bNziODZh1549TJszk8b1G5Ize46btqWkvHlDyJs3JP52iQeL8c3X8ylatDBTp8wkUyZfWrR0HPxYvlQVjh8/TnhkKPUb1sU/wI+pn8zE19eX2TM/Iyg4iOo1qgJw8eJFPpvzBXny5KZBo9T1/iv3LrcV0NbavcaYckAT4BVjzC/AE0B5a+0/xphxgOc1d7lymoC4a5av3L4S9/VF5fW3DY5R7kRHWxhjKgJ1gfbAkzgK4Bu5cj6vdMApa23pm/S99jFqAfWAytba88aYpSTM0a08PTIy9enXGT7tFUbNfI0iefIzsc9zpE+XcE5kmfzFKZO/OOCYNwtQNE9BCubKG99+o7bU4p7KNUNGPuk7nmfnvsmYL96mcGBe3ug8IlGupUOKUjqkKADnYi8AUDRXfgrmCCHi9HF+2ryMsKhwsvg8wJBmPelVu7Xbc7kZz4yezH7+fQa/9RzD3n2BonkLMmnohERzev0yZ+OXtUuZ+t1n+Hp507hKXV55YjQAZYuUpGyRkgCcc56mr3i+IhQOLuDeZG7BM6Mnn742macnjGTI689RLH8hPhjzRqJct+3bxfkYx74c+MpIAIJy5qZpzQa0rNuEId0fZ+4P3zL41dHkzRXExOHjU93ZKTwzevLZxCkMfGkEQyaMpliBwnw47s3Eue7dyfkLzlzHjwAgODAPzWo1xMvTi7lvTeWp8cMZ+NKzFArJzxdvTycgm7/b87kZT09P5k6ZxVMjhjB49HCKFS7KR2++m+S89FHjxxL2j2Nu8NJVy1m6ajm/fvsDObPnuGlbajJl+kcMeHwQI0eMJU9QbmbMnkLmzA8k6ufp6cnMOVN5ZtAIRgwdTZGihXn3/Ynx/y/fLfiBqMgoRj83ItUdBOs6q/NApxHGumkCuDEmEDhhrY0xxrQEugOVgRAgPbAW+NpaO85ZaD5jrd3gLEKfsdY2c25nKfAMEAUcAKpYa9cYYz4Bdltr37ymTxjwJ1DHWvuXc450biAc8LbWRhhjsgJ/WWuz3iDuGThGzL923l4NvGWt/co5Z7uktXaLMeZzYK219m3nFA4foDbQ21rb3DlXejPQyFq71BgTiuPLQ5QxJtpa65vEw1/LAhz/Ysut/7PvYtnalYpfTsu5XpvnsclrUjCS5Je9b+X45XMrEh9AlJb4VA+OX47dfCQFI0leGUtfPYXYxR0RKRhJ8vMoHhC/fCnibApGkvwyBGSKXz5zITIFI0l+93v5w40PK0g2fnm9bNNxqesL+381q/v2P621qfrMZ8nBnV/lHgTWGWM2A6OA8cAnOKZozAfW/4tt7gK6GWO24pgC8uG1jdbaSByF+lxnn7VAESATsNC5bhngym/VnYBexpgtOKacXJkgOxCobYzZhqNoLw78DNznfJwXnY8vIiIiIncxd07hWAQsum71BhwH+V3ft9Y1y0uBpde3GWNCgDhrbaKTuV53/9+ACkmEVPE24+5+3e0DOA6IvL7fMa4W09dKcnKltTbkmuVbjT6LiIiISCpxV5/XWERERORuYS3ExcWldBhyB9y1BbS1NhS4Y4dYG2NGAW2uW/2VtfalO/UYIiIiInL3u2sL6DvNWSirWBYRERGRm7rbzwcjIiIiIuJWKqBFRERERFygAlpERETETeKsTVOX/8IYk9UY86sxZp/zOksSfQobYzZfczljjHna2TbOGHP4mrYm/ykgF6iAFhEREZGUMAJYYq0tCCxx3k7AWrvHWlva+VegywHngXnXdHnrSru19kd3BA0qoEVEREQkZTwMzHQuzwRa3qJ/XWC/tTbF/6ytCmgRERERSQnZrbVHAJzXAbfo3x6Ye926J40xW40x05KaApJcdBo7ERERETewQFzcf5s3nAr5GWM2XHN7srV28pUbxpjFQI4k7jfKlQcxxngALYBnr1n9IfAijv/aF4E3gZ6ubPffUgEtIiIiIv9WlLW2/I0arbX1btRmjDlmjMlprT1ijMkJRNzkcRoDG621x67ZdvyyMeYTYKFrof97msIhIiIiIinhO6Cbc7kbsOAmfTtw3fQNZ9F9RStg+x2N7iZUQIuIiIhISpgA1DfG7APqO29jjAk0xsSfUcMY4+1s//a6+79mjNlmjNkK1AYGuSdsTeEQERERcQ9r0+Ic6H/NWnscx5k1rl8fDjS55vZ5IFsS/boka4A3oRFoEREREREXqIAWEREREXGBCmgRERERERdoDrSIiIiIm8TZuJQOQe4AjUCLiIiIiLhABbSIiIiIiAtUQIuIiIiIuEBzoEVERETcwILOA51GaARaRERERMQFxlp9E7pLaEeJiIjcOcbdD5gl2NPWGZHH3Q+brL59/K8/rbXlUzoOd9MItIiIiIiICzQHWkRERMQdrOZApxUqoO8yx7/YktIhJKts7UrFL0fO2pCCkSQv/65Xf+069OqvKRhJ8ss9vH788qn5O1MwkuSXuWWx+OWzv+5PwUiSV6b6+eOXz686mIKRJD/vqkHxy3Fhp1IuEDdIF5w5fvn0hSMpF4gbPOCVM6VDkLucpnCIiIiIiLhABbSIiIiIiAs0hUNERETEDSyWOJ39LE3QCLSIiIiIiAtUQIuIiIiIuEAFtIiIiIiICzQHWkRERMRNdB7otEEj0CIiIiIiLlABLSIiIiLiAhXQIiIiIiIuUAEtIiIiIuICHUQoIiIi4g4WbFxKByF3gkagRURERERcoAJaRERERMQFKqBFRERERFygOdAiIiIibmDRH1JJKzQCLSIiIiLiAhXQIiIiIiIuUAEtIiIiIuICzYEWERERcROrKdBpgkagRURERERcoAL6HvPHnk3UGNaGwM4VqD2iPVsO7Lph36gzJyjUpxZ+7Usz6fuZt92WWvyxdws1R3Ugd68q1H2uM1tDd9+wb9SZkxR5oh4B3Srw/o+z49cfPn6Urm8PIaRPdfL3r0X/j0a7I3SXrQ/dQf23+5F/VFMav/s42w7vS7JfnhENElx6zRqboP149ClKvtCaPCMa8NHyr9wRukvW7tpI1adbkb1NaWoOac2W/Ttv2Dfq9Anyd61KllbFeW/+dADW7d5MgxGdCO5UieBOlej66kCiTp9wV/guWbvjTyr3a4pf06JUf7wFm/dtT9Qn6tRxqvZvTo7mDxL4cCkaD+nAzgN7AbDWMm7q6xRqXwX/psUo17MB3yz9wd1p3JY1W9dTsVtDstQuSJWeTdi0Z1uiPpEnj1Ope2P86xUhe/1iNHyyLTv+3nPLttRm1Ya1lG5UFa9C2SnftCYbt29J1GfNn+uo9kgDsj4YTNYHg2ndvyuRx6Pi2+cv+oFCNcviXSgHddo148A/Ye5M4bbt2b2PBrWbE5A5hPKlqvHb4qU37Lt2zTqqVKxLQOYQalRuwOZNW+PbPpj0CQ8WqUhA5hBKFn2Ijz+c6oboRW5OBfQ9JOZiLD0mPkP0hXOM7/oMkaeP0/OtZ7gcdznJ/iNnvEbMxRiX21KDmIux9Jw0jHMx53mh4yAiT5+g56ThN8x19KdvEnMxNsE6ay3d3x3Gsh1/8HiTLjzX7in8MmVxR/guibl0kX5zXuBc7Hmea9aPyOiT9P/0xRvm2qRENSZ1eJZJHZ6lX402CdrGfv8hMZdik7xfSou5GEu3154m+sI5Xuo5nMhTx+n2+iAuX046zxFTX0m0T/eHh5Lt/syM6zqY+uWq8/3axYyd9aY7wndJzMVYOr/wBNHnz/FKv1FEnDxO1xcHJJlr/Qo1eHPA8/Rq1pFV29bz7McvAfD7xlVM/OJjcmT158U+Izhy/BiPvTGMS/+75O50biomNoaOox8j+nw0rw4YQ8SJKDqPfizJXBtUqsVbQ8bTu1VnVm7+gxHvvXhbbalFTEwMbfp342x0NBPHvMSxqEjaPtYtUa57D+wnW9ZsvDJiHI1r12fez98z/BXHl92jEcfoOKAX9/tm4rWRL/Dn9i30GPJYSqRzS726Pcbevft5acJYMmTIQPfO/Th9+kyifjExMXTt0Ifos9G8/Oo4IiOi6NapL5cvX2b/X38zcthY0qVLx0sTxvK/S/9j+JAxHDp0OAUyErlKBXQKMMbUMsYsdC53N8ZMcsfjLtm8iojTx+nRoC09G7SjU+1WhEUcZtWODYn6Lt60kkUblzOgRQ+X2lKLJVtXE3n6BN3rtqZn3TZ0rNmCg5HhrNr1Z+K+W1axaPMKnmzaNcH6lbs2sCV0F/0bduKppt3oWqsV4zsNcVcKt+33PeuIjD5J10rN6Va5Be3LN+LgiaOs+Xtrkv0LBgRTv2glHi5Vm4ohJRJsZ/GutTxes527QnfJ4o0riDh1nF6N2tO7cQc6132EsGOHWLljfaK+v/65gkXrlzKwVc8E6x+t3oS5I9+nR8O2vPXYOAB2H/zLHeG75Nd1y4g4GUXv5p3o06IzXRq1IfToP6zY+keCfn6ZszGm+2AaVqxJjdKVAEiXzvG2HmfjAMgbGESdslW538cXXy8f0pnU9bb/y9qlRJyIpE+rLvR9pCvdmrUj9Mg/LN+0JkE//yzZGNvnGRpWqk3NslUA4nO5WVtq8tPSxRyLiuCxLr14rEtverbrzIF/wli6dmWCfh1aPMqCKXPp16kHH738FgA79zl+Qfv8u2+IvRjL8McH8WT3vrRs2JQV69awP+yA2/O5mS2bt7F9205at3mYPv178PiAvpw5c5YF8xYm6vvrot+JiIikd99u9O7Xnc7d2hMWepCVy1cTF+d4HgcG5qBWneoEZPcnY8aMeGb0dHdKd4aFuLi0dblXpb53mGRmjEmfEvdNDcIiHd/Yc2YJACAwq+M6NOJQgn7RMed5ZupLjGn/FLmz5bjtttTkYFQ4ADmz+AMQ6Mz5yv/BFdEx5xk6cwKjWz+RKJ+9hx0fSAs3/EZw3+rk61+LT375PLlDd9k/J48CkP0BPwByOq8PnjiSZP93f/+Mws89TKUJnVm8ay0A52Iv8Oy8dxnRqCeBmQPcELXrwo45nqc5szmfv37ZAQg9+k+CftEXzjH44+cZ0+VpcvvnTNDmkcEjfvm3TY6ipUrx8skW878V5swppzPHXH6O52bokYOJ+u4I3UO+tg/x6KheBPpl59X+jmlGdctVp0+Lzsxb/hPlezfkxJlTTH32LdKnT11vY6FHHLkGOnPM5dxnoeH/JOq7/e/dhDQvS6tnuhHon4PXBj53W22pReghx1SLwByOHHPlCATg74OhCfp5eFx9ni5a/hsA1Ss6vhgccG4jl3MbuW+wjZRw/vx5jkcd53jUcXbucBT8OQOdueZyXIcdSPwcDgs76OzrfA7kcuQUeuAgBQsVYNyLI1m7Zj0VStdg65btvD3pNfz8syV7PiI3k+oKaGNMiDFmtzFmijFmuzHmU2NMPWPMKmPMPmNMRedltTFmk/O6sPO+6Y0xbxhjthljtpr/t3ff4VFUbxvHvwcknZoCCan0gDTpLYB0EcRCERBQEMtPVFQEpNgQxcIrdkrooIgK2ACRooKgoFIVRSlCghACoSdRct4/dgkJSYBV2I3x/lzXXjsz58zs87CT8OzZMxNjBjm37zbGjDbGrAa6GmNudfbZaowZd5F4ThhjnjLGfAM0Msb0NsZ8a4zZaIyZeLaoNsa0N8Z8b4zZZIxZ7tyWa5z5xdkLgY0x2ba/umgavl4+tKjRiKRjjvmhR04cJeXEsQu25WeZuZI919c+meHIp3oDko4dAZz5nDxG2l/pAFxV+Cqm3/8CkUFhjJw7nt/+yJ/zDc+yzmzPzxXg3ubdmdR7NONuepCjp09w39vPcjo9lTe+mIdvEW/iKtYh+WQKACmnjpFy6rg7Q3fJ2SvZzz9/JyyIx8/Ll2trNiHJOb/58PEUUk4czeyz7qfvGfTaKGqVr8bQ7ve6Lea/yzqTPT9XgHJhUSwYO42RfQezP/kgL8+fDMCOvTt5d/kirq3TlDmj3yCkZBD3vPgoJ0+fcmvsrjqXa8628mWjWTR+FqMHPMz+Qwf4vzlvXVJbfnWh9xUc86UHPDqIOtVr8fiDQ//WMdxpwvg3KB9ZnfKR1Xn0oRHZ2vL6ec1N1nPgUFIyk96cRvUa1ZgzbypXV6/Kow+NIGFf4mWPX8QV+fU2dhWArsBAYD3QE2gKdAYeA/oAcdbav4wxrYGxwM3O/jFAbWdbqSzHTLXWNjXGhAHrgDrAEeAzY0wXa+3CPGLxB7Zaa0cbY2KBoUATa+2fxpg3gF7GmMXAZGdMu7K87vY84rwkxpiBzpyYOHEiAwcOvNRdcxUVXBaAxMMHANh/+GDm9tT0NAoVKoTXVUVISP6DHYm7aPjQDZn7Tlg0FT9v3wu2PXzTnf8ovsspMsgxgpHozHH/EcdzZHBY9lwPH2DH/t00GnpL5r6vfDIDP29fKobFANCmZhM6XNOc737dwk/7fmVPUiLly0S5OaO8RZR0jNr8cfSQ8znZsb1UGVL/TKeQMXhdVQSA4R36Z+636ucNLN62msSjSSSmJPFr0l6av3RuysPrq+bhW8SHB1r1clcqFxRVOhyAxGTn+et8jiod7nhPjcGriBcJh/7gl4Sd1LuvY+a+L38wBX8fXx7pejdrtm2gx5h7iAmN5P3Rkwjw9Xd/MhcRVSYCgMRDjm8XzuYcVSYiW64AAb7+tKrbjFZ1mzFj8TwWfPEpbzz8HJ+uW87Rk8fp0aoLnZq2Zem3K5m5ZD7bf/+VOpVreCaxXESHOnJNSHLm6sw5KjSC1LRUx8/q2Vz9/GldP47W9eOY9tE7fLDyE9567MWLtuUX0eGO3xsJ+x3FX+IBx7dEMRFRpKY6c3WOPn/5zRo63dGDClExLJ75PgH+AY6+zmPscx4jIcsxPO3WXl1p1Lg+AD6+PrRv1YXEBEd8iYmO58hox/udlua4PsHb25uoqEhHn/P6RkVH8tWXa0hM3M/tA26jY6f2/LhtO8889Tzrv/2OsuFh7ktO5Dz5tYDeZa3dAmCM2QYst9ZaY8wWIBooDswwxlTEMbhYxLlfa+Ata+1fANbarJfXz3M+1wNWWWuTnMefA8QBC/OI5QzwvnO5FY7Ce73zU7QvcBBoCHxprd113uvmFeclsdZOAiadXXVl39y0qtWE4OKlmL5sPgE+/sxZuYDI4DCaVKtL6Z51qBJentUvvk//dj1oe00cAGt+3ED8Z/PoHnc9nRu05kTqqTzb8pNWNRoTVKwUM1a8T4CPH3O/+JDIoDCaxNYh9PaGVClbji/HzqN/6260rdUMgDU/fcfU5fPp1qQjneq1IjyoDEHFSvHxhpXElI7gow0r8Pfxo3pUvvoigZaV6xMUUIJZ6z7C39uXdzYsIaJkaRqVq0H0Yx2oVDqK5YMns2L7t3zww3IalavB0dMnWPnLegL9ixNRsgz9Gt9A69gGAKzduZkZaz/klmta07F6Mw9nd07ra5oRXDyQqUvmEeDrz+zlHxAZUpam1eoRdEsNqkRUYO0ri7jzup60q9scgNVb1zNl8dv0aNGZzo3asum3H+n29N1Ya+nT5hZWbvoaPx9fOtRr6eHssmtTvznBJQKJ/2guAb7+zFoyn6jS4TSr0YCSHSoTG1WRbyYvZvbS99j820/UKB/L1p3b2XswkWucxXFMqKMoif94LqnpqSz5ZiVeRYoQXSbck6nl0LZhC4JLBjFl4SyK+vkz4+N5RIWGE1e7EcWalyM2phIbZi1j5ifvsnnHj9SoWJWtv21n74EE6sTWBLhgW37SoUVrQoKCeWv2VAL8A5g6bzbR4ZG0aNgUr/JBVKtUhc2freX7rZvo2K8b1loG3NqHZV+txN/Pj06tO9C9800Mf/5JXnhrAgcPJbFw6Sc0rdeQ8lExnk6P6JgoomPOFfLVqlfl/fcWUSW2ElOnzKRo0QA6d3F8sK1bsxmHkw+TkPQrbdq1JDgkiPjJMwkICGD2jHeIjIqgaVxjNm903H3m3Xfep3SZEObP+wCA8hXKuT/By8ACNkM3gi4I8t0UDqesl85nZFnPwFH0Pw2stNZeDXQCzl5NYMi70DyZpY8rUq21Zy+RNsAMa20t56OytfaJC7xuXnF6hI+XN/EPvoC/jx8jZjxPcPFSxD/4AoULZZ8TWbt8NTo3bEPnhm2oVa4qALERFalYNuaCbfmJj5c38f97Fn8fX0bOeYmgYiWZct+zOXKtFVOVTvVa0aleK2rFxAIQG16eimHR+Hr5MPW+5/AuUoRhs57H18uHaYOeJ7hYqdxe0mN8injxZs+R+Hn58sRHbxLkX4I3e43KkWvZkiEcPH6YsYun8MqKudQoW5Hp/cbgdVURaoZXomP1ODpWj6NG2UoAVC4TQ4WQSE+klCsfL2+mDRmPv48fw+OfJah4KaYPGZ9jTm/tCldzQ+N23NC4HbUrVAOgalQlKoWXY9uenzmVdprT6akMmTSGAeOHMHTyWE+kc0E+Xt7MHPkq/r5+DH1zDMElSjFj1Ks5cg0sXorPvl3Fg6+M4u3PF9K+QUumDBsPQOem7Xiw20B+P5DAkNefomTR4kwe+hKBxfPZ+evtw+yn38Df158hE54kuGQgs59+M0euwSUC+WzdSh54cQRvL3mfDo1bMXX0hIu25Sc+Pj7Me30aAf7+DH5qOCGBQcx7Y3qOXDf/tI1Tp09xOvU0940aQq/7B/DA444pHKEhZZjzyhRSjh1lyNhR1K5WnakvvuGJdC5qyrTXqVixPCOGPUl6+p9MmzWREiWK5+jn4+PD9NmTCAjwZ9iQ0QQFBzJj9iQKFy5M7To1GfPc46SlpTNk8AjS0tJ5YfwzVK9RzQMZiZxjbD77kzjGmGjgY2fRiTFmunP9vbNtwA5gtrX2fWPME0A/a220MeZuHKPQPc5O4bDWHjbG7AbqWmsPGWNCyT6FYynwqrV2UR7xnLDWBjiXqwKLcEzhOOicqlEUOAV8T5YpHM7XXZBHnC2AR6y11xtj+jlju+8i/zQWIHleznuGFiSB3c+NGiXNzHl3kIIiuM+5C9f2jVvmwUiuvPChbTKXUxbmfd/mgqBEl6qZy8eX/ebBSK6som3KZy6fWpPzorCCxK/JuQ+RGXtSPBeIGxSKKpG5fPR07hchFxTFfUPB9QG1f6xYuLdtOCj04h3/RZYN2/OdtTb/XY19heXXEeiLeR541hizBsj60X0K8Duw2RizCcfc6WystfuB4cBKYBPwfV7Fcy77/giMxDFvejOwDAh1TgcZCHzgfN2z00XyilNERERE/qXy3Rxoa+1u4Oos6/3yaKuUZbdRzva/gIecj6zHjD5vfS4w9xLjCThvfR7nCuSs2xcDi8/btjaPOFcBq5zL04HplxKLiIiI/Lv9l++dXJD8W0egRUREREQ8It+NQHuK8z7P3udtvu3s3UBEREREREAFdCZrbQNPxyAiIiIi+Z8KaBERERF3sGA1B7pA0BxoEREREREXqIAWEREREXGBCmgREREREReogBYRERERcYEuIhQRERFxAwtkWOvpMOQy0Ai0iIiIiIgLVECLiIiIiLhABbSIiIiIiAs0B1pERETETfSHVAoGjUCLiIiIiLhABbSIiIiIiAtUQIuIiIiIuEBzoEVERETcwUKG5kAXCBqBFhERERFxgQpoEREREREXqIAWEREREXGB5kCLiIiIuIXFZlhPByGXgUagRURERERcYKzVJ6F/Cb1RIiIil49x9wsWDfOytQaUcffLXlGrn977nbW2rqfjcDeNQIuIiIiIuEBzoEVERETcwOo+0AWGCuh/mcPzt3g6hCuqVNfqmcvJc3/wYCRXVmDP2pnL+8Yt82AkV1740DaZyykLf/RgJFdeiS5VM5ePL/vNg5FcWUXblM9cPvX17x6M5MrzaxyZuXxm1xEPRnLlFY4pmbl87HSSByO58or5Bns6BPmX0xQOEREREREXqIAWEREREXGBpnCIiIiIuIluflYwaARaRERERMQFKqBFRERERFygAlpERERExAWaAy0iIiLiJhkZmgRdEGgEWkRERETEBSqgRURERERcoAJaRERERMQFmgMtIiIi4gbWgs3wdBRyOWgEWkRERETEBSqgRURERERcoAJaRERERMQFKqBFRERERFygiwhFRERE3CRDFxEWCBqBFhERERFxgQpoEREREREXqIAWEREREXGB5kCLiIiIuIn+kErBoBFoEREREREXqIAWEREREXGBCuj/mG+2/0CzR24mtGcdWg7txqadP+bZ99Cxw1TsH0dgtxq89uH0S27LL775eSNxw7oR1rcBLUf0ZNOun/Lse+jYESrddS1Bva7htU9mZm4P6nVNtsdt4x9yR+guW797G21evovyIzrS4ZV72ZKwI9d+EcPaZnv0n/l4tvbkEynUeOoWIoa15a0v57sjdJes++l7mjx4I6W71qL5w7ew6bcLnL9HD1O+TxNK3liNVxdOA+Db7RtpO6wXUb0aEtWrIX3GPcCho4fdFb5L1m37jkZ3dSSoYyzN7u3Mxh1bc/Q5lJJMk7s7UaZTdcJuqEmHh2/lx12/AGCt5Yn4F6jUozHBHatS5462vL/qE3encUnWbl5P/T7tKNmiIo1vv44fft6So0/SkWQa9u1AcKsqlG5dlXb/68a2nT9ftC2/WbNhHbU7NMWvShnqdWrB91s35eiz9vtvaXpLOwJrRhNYM5qu9/QhKflQZnu3e/sSVCuGq8qV4v7HH3Vn+C75efsvtGl5HcElwqlTsxHLP1+ZZ991a7+hcf3mBJcIp1mjVmz8YTMAc2a9Q3G/kByPPXt+d1caIrlSAf0fkpqeRr+XHuLE6ZOM6TuEgymHuX38w5zJOJNr/8emjSM1PdXltvwgNT2N2ycM4UTqKcb0fpiko8ncMeHRvHOd+QKpf+aeT6d6rZh037NMuu9Z/tfxtisZ9t+S+mc6d81+ipNppxh9/V0knTjC3XOezjPX665uymu3Due1W4dzV1zXbG2Pf/QmqX+muSNsl6Wmp9H3+Qc5cfokz9wxlKSUZPq+MJgzZ3LPc1j8s6SmZ8/lt8TdBBYrwRN9HqJNnWZ8tO5zHp/5kjvCd0lqehq9n/ofJ06d5Nm7RnDwSDJ9nh6Ua65t6sXx0qAn6X99T9ZsWc/wic8AsPL7NYyfN5EypYJ5+s5h7E8+wD0vPsqff/3p7nQuKDUtlZ4j7uHEqROMu38UBw8fovfIe3LNtW3DFvzfw2MYcGNvVm/8hmGvPH1JbflFaloq3e7tx/GTJ3hpxDMcOJRE9//1y5Hrjl2/EVQykGeHPkGHFm1YsPRjhj137sOut5c3Xdp2dHf4Luvf9y5++eVXxj73JEWKFKFf7wEcPXosR7/U1FRuu/UOjh8/ydhxT5F0MIk+ve7gzJkzNGnWiPgZE4mfMZFJ8a/j5eVFSEgwYWGhHsjon7MWMqwtUI//KhXQV5AxZrcxZosxZqPz0di5fYkxJsUY87E74/l842oOHk3mjnbd6d+uB72vvZE9BxNYvW19zr4/fMWS777g/hvucKktv1i+aQ0HjyZze+uu3NGmG71adGFPUgJrftyQo+/nG9ew9IcvGXR9v1yPVSm8HO1qx3FTo3Y0rFz7CkfuupU/f0vSiSP0adiJvo0606Nue34//Adrd27OtX/FkCjaxDbkhpotqR99dbbjfP7TOu5t3t1dobvk8++/4mBKMv3b92BAh1vp3eom9hzYl+v5u+y7r1i6fhUP3Jj9HL252XW8/djr3N6uG/93zxMAbP/9V3eE75Jl337BwSOHGNCpF3d27s1t7buy+4+9fLX5m2z9gkoEMqrfQ7Sr35y4Wg0BKFTI8Ws9w3mlUkxYJNde04Ri/gEE+PpTyOSvX/ufrVvFwcNJ3HnjbQy8qQ99r+/O7sS9fPnD2mz9gksG8vjAR2jXqCXN6zQGzuV6obb8ZMmqzzlw6CB3976De27rz+3derFr7x5WrVudrV+PTjezcPJcBvbsx5vPjAdg245zI+qzXp5E75t6uDV2V23auIUtW7ZxS9cbufPu/vxv0N0cO3acRQs+ytF32dLlHDyYxICB/bjzrju4rW9P9uz+na++XEN0dBS3dL2RW7reiLePD+np6fTu05MiRYp4ICuRc3QXDhcYYwpba3Mf7spbS2vtofO2vQD4AXddnsguze8HEwAILRUCQFhgaQD2HNgH1c/1O5F6iocnj2FUzwfw9/HLdowLteUne5ISAQgt6czVmfPugwnEZel3IvUUj0x9hlHd78PfO/d8xi+cwksLJhMeWIZx/YbR7pq4XPt5yt4jfwBQungQAKHO598P7wdyFvyvrJzLhBVzKFsihDE33Efr2IacTDvN8AWvMKz9Hfh5+botdlfsObAPgNBA53sa5Dh/d/+xl+Y1Gmb2O3H6JA9NfJJRtz1IwHnnqFcRr8zlFT84ipbG1epe0bj/jj1/7AUg1Jlj2aAyAOze/zvUbpyt77bdP9P0ns6A499k3N0jAWhVpxl3du7N5A9ns+DLxfh4eTPvqUkULlzYXWlckt37HbmGBTtyLBviGFncnbg3R9+tv22n8e3XZfZ//oHRl9SWX+zauweAsqXDAAgvE+bcvhtontnPy+vcefrZlysAaFa/kXuC/AdOnTrF6VOnAfhxm2N6VahzpDisrPN93bUnx35np2OEZfYNO9e35bl+0+JnUKhQIfr1z3/fBMp/T/77iP4PGGOijTHbjTFTjDFbjTFzjDGtjTFrjDE7jDH1nY+vjTE/OJ8rO/ctbIx50TlivNkYM8i5fbcxZrQxZjXQ1Rhzq7PPVmPMuL8Tp7V2OXD88mX+91jnVy/GmGzbX1k4FV9vH1rWaMSho8kAHD5xlJQTxy7Ylp/lleurH03H19uHFtUbkXTMMRf2yPGjpJx05HN/p37MePBFxvcfScrJ4wx8/TFOpZ12b/AusjhzxeRou7d5dyb1Hs24mx7k6OkT3Pf2s5xOT+WNL+bhW8SbuIp1SD6ZAkDKqWOknPL4aZqns98cnv+eTlgQj5+XL9fWbEKSc37z4eMppJw4mtln3U/fM+i1UdQqX42h3e91W8x/V17nL0C5sCgWjJ3GyL6D2Z98kJfnTwZgx96dvLt8EdfWacqc0W8QUjKIe158lJOnT7k1dldl5ppLW/nwaBaNn8XoOx9m/6ED/N+cty6pLb/K6xw+a82GdQwYNog6V9fi8QeGujGyv2fC+NcoFxlLuchYhjw0PFvbhc7h8+XWd+fOXXyx8itat7mWqKjIyxi1yN9TEEegKwBdgYHAeqAn0BToDDwG9AHirLV/GWNaA2OBm539Y4DazrZSWY6Zaq1taowJA9YBdYAjwGfGmC7W2oUXiGelMeYMkGatbeBKIsaYgc64mDhxIgMHDnRl9xwiQ8oCkJh8AID9hw9mbk9NT6NQoUJ4XVWEhOQ/2JGwiwYPds7cd8LCePy8fS/Y9sjN/yy+yykq2DGCkXjYmeuRpMzt2XM9wI7E3TR85MbMfSd8NA0/b18evnEAo3vcn7l9xeav+Xj9ChKSD1AxLNp9yVxEREnHyN0fRw85nx0fbCJKlSH1z3QKGYPXVY6vO4d36J+536qfN7B422oSjyaRmJLEr0l7af7SuSkPr6+ah28RHx5o1ctdqVxQVOlwIMv563yOKh3ueE+NwauIFwmH/uCXhJ3Uu+/cHNGXP5iCv48vj3S9mzXbNtBjzD3EhEby/uhJBPj6uz+Zi4gqEwFA4iHHtwtnc44qE5EtV4AAX39a1W1Gq7rNmLF4Hgu++JQ3Hn6OT9ct5+jJ4/Ro1YVOTduy9NuVzFwyn+2//0qdyjU8k1guokMduSYkOXN1PkeFRZCalur4WT2bq58/rRvE0bpBHNM+fIcPVnzCW4+9eNG2/CImIgqAfX84viFLOOB4jg6PcuRqCmWOPn/57dd07t+DClExfDrjPQL8AzwTtAt69OpGw8aO/+Z8fX1o16oTiQmOHPcnOt/XaEfxm5bmuD7B29s7syBOyOy7P1tfgGnxM7HW0v/Oflc+kStM94EuGApiAb3LWrsFwBizDVhurbXGmC1ANFAcmGGMqQhY4OxEqtbAW9bavwCstVkvzZ/nfK4HrLLWJjmPPweIAxZeIJ7cpnBcEmvtJGDS2dW/c4ysWtdqSnDxUkz77F0CfP2ZvWIBkcFhNK1Wj5AetakSUZ41Ly1gQPtbaVfHMU1h9bYNxC99h+5xnejcsA0nU0/l2ZaftKrZhOBipZi+/D0CfP2Zs2ohkcFhNKlal9K31aNKeHlWj5tP/7bdaVu7GQBrftpA/LJ36d7sejo3aMWyjauZv/pTmlStw9GTx1m+6WuCipUkyvlBJL9oWbk+QQElmLXuI/y9fXlnwxIiSpamUbkaRD/WgUqlo1g+eDIrtn/LBz8sp1G5Ghw9fYKVv6wn0L84ESXL0K/xDbSOdfzHt3bnZmas/ZBbrmlNx+rNPJzdOa2vaUZw8UCmLpnnOH+Xf0BkSFmaVqtH0C01qBJRgbWvLOLO63rSrq7j6/DVW9czZfHb9GjRmc6N2rLptx/p9vTdWGvp0+YWVm76Gj8fXzrUa3mRV3evNvWbE1wikPiP5hLg68+sJfOJKh1OsxoNKNmhMrFRFflm8mJmL32Pzb/9RI3ysWzduZ29BxO5xlkcx4Q6io/4j+eSmp7Kkm9W4lWkCNFlwj2ZWg5tG7YguGQQUxbMoqifPzM+nkdUaDhxtRtRLK4csTGV2DB7GTM/eZfNO36kRsWqbP11O3sPJFAntibABdvyk/YtWhMSGMzEOVMp6h/AtHfnEB0eSYuGTfGuGEy1SlXYtORrvt+6ietv7+YoGLv34fPVK/Hz86dTq/YAvPvxB2zYshGAH3/9mfh5M7muZVtCQ8p4MDuIiYkmJiY6c/3q6lV5/72FxMZWJn7KDIoWDaBzl+sBqFuzMcnJySQm7aZNu1YEhwQRP3kGAQEBzJoxl8ioSJrFNQEgPT2dubPnERERTtv2rT2RmkgOBbGAznrZfUaW9Qwc+T4NrLTW3miMiQZWOdsNeRepJ7P0+dfy8fJm6uAXeTR+LI9NG0eViAr8312PU7hQ9jmRtctXo3b5agCcSHVMV6gaWZFKZWMy2/Nqyy98vLyJv38cQ6ePY8TMF6gSXp7xA0blzLVcVWqXqwrAyVTHV9uxERWoGBbDmYwMDqQc4sm3J3AmI4OaMbE81euhzNHc/MKniBdv9hzJyEWv8cRHb1IpJIpxNw/OkWvZkiEcPH6YsYuncCYjgxplKzKq4114XVWEmuGVqBleCYCTaY67kVQuE0OFkPzzVamPlzfThoxnyKQxDI9/lioRFZhw75M55vTWrnA1tSs4Lo48+55WjapEpfByzF2xIHMKzpBJYwCICA7LdwW0j5c3M0e+ysOvPcHQN8cQG1WBVwaPzZFrYPFSfPbtKqZ+Mhd/H3/aN2jJ2LtHANC5aTse7DaQecsXMeT1p4gOjeCF/40msHip3F7SY3y8fZg95g0GvzSKIS8/SWxMRV4fOi5HrsElAvls7UriF84hwNePDk1a8dygURdty098vH145/Vp3D96CIOfHk61ilV4a+zLOXLdsn0bp5xTbQY9PgSAqLIRmQX08HFPsifBMUd81dqvWLX2Kz6f+6HHC+jzTZn2FoPuHcxjwx4nIjKc6bOmUKJE8Rz9fHx8mDE7nkcGD2PYkJFUia3MK6+Pz/x3+XDRJxxKOsTI0cPy5cWh8t9kbAG6BYmzIP7YWnu1c326c/29s23ADmC2tfZ9Y8wTQD9rbbQx5m4co9A9zk7hsNYeNsbsBupaaw8ZY0LJPoVjKfCqtXZRHvFk7ptLWwvgEWvt9ZeYngU4PD/n/VELklJdz13NmDz3Bw9GcmUF9jx3cd++ccs8GMmVFz703LcTKQvzvm9zQVCiS9XM5ePLfvNgJFdW0TblM5dPfV2w78fr1/jch8gzu454MJIrr3BMyczlY6eTPBjJlVfMNxg8MCjmV9rLVr41xN0ve0VtnJDwnbU2/12NfYUVxBHoi3kexxSOh4AVWbZPASoBm40xfwKTgdey7mit3W+MGQ6sxPGD92lexfOFGGO+AqoAAcaYfUB/a+3Sv5WNiIiI/GtkaA50gVCgCmhr7W7g6izr/fJoq5Rlt1HO9r+Ah5yPrMeMPm99LjD3EuOJzmN7/plYKiIiIiIu0WQiEREREREXFKgRaE8xxnwDeJ+3+bazdwMRERERkYJDBfRl4Or9nUVEROQ/yILNKDg3b/gv0xQOEREREREXqIAWEREREXGBCmgRERERERdoDrSIiIiIG1h0H+iCQiPQIiIiIiIuUAEtIiIiIuICFdAiIiIiIi7QHGgRERERN7G6DXSBoBFoEREREREXqIAWEREREXGBCmgREREREReogBYRERERcYEuIhQRERFxBwsZGbqKsCDQCLSIiIiIiAtUQIuIiIiIuEAFtIiIiIiIC1RAi4iIiLiBBWxGwXr8E8aYrsaYbcaYDGNM3Qv0a2+M+dkY86sxZliW7aWMMcuMMTuczyX/WUSXTgW0iIiIiHjCVuAm4Mu8OhhjCgOvAx2AqsCtxpiqzuZhwHJrbUVguXPdLVRAi4iIiIjbWWt/stb+fJFu9YFfrbU7rbXpwDvADc62G4AZzuUZQJcrEmgudBu7f5lSXat7OgS3CexZ29MhuEX40DaeDsFtSnSpevFOBUTRNuU9HYJb+DWO9HQIblM4xm3fDntcMd9gT4cgclZZYG+W9X1AA+dyaWvtfgBr7X5jTIi7glIB/e9hPB2AiIiI/H1ph/5c+vOU/UGejuMy8zHGbMiyPslaO+nsijHmc6BMLvuNsNYuuoTj51b/ePxm2iqgRURERNzAWtve0zG4m7W29T88xD4gIst6OJDoXD5gjAl1jj6HAgf/4WtdMs2BFhEREZH8aj1Q0RgTY4zxAnoAHzrbPgT6Opf7Apcyon1ZqIAWEREREbczxtxojNkHNAI+McYsdW4PM8Z8CmCt/Qu4D1gK/AS8a63d5jzEc0AbY8wOoI1z3T2xW+vxaSQiIiIiIv8aGoEWEREREXGBCmgREREREReogBYRERERcYEKaBERERERF6iAFhERERFxgQpoEREREREXqIAWEREREXGBCmgRERERERf8P8Kl8GEMB/YnAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 720x1440 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# from https://matplotlib.org/stable/gallery/images_contours_and_fields/image_annotated_heatmap.html#sphx-glr-gallery-images-contours-and-fields-image-annotated-heatmap-py\n",
    "def heatmap(data, row_labels, col_labels, ax=None,\n",
    "            cbar_kw={}, cbarlabel=\"\", **kwargs):\n",
    "    \"\"\"\n",
    "    Create a heatmap from a numpy array and two lists of labels.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    data\n",
    "        A 2D numpy array of shape (M, N).\n",
    "    row_labels\n",
    "        A list or array of length M with the labels for the rows.\n",
    "    col_labels\n",
    "        A list or array of length N with the labels for the columns.\n",
    "    ax\n",
    "        A `matplotlib.axes.Axes` instance to which the heatmap is plotted.  If\n",
    "        not provided, use current axes or create a new one.  Optional.\n",
    "    cbar_kw\n",
    "        A dictionary with arguments to `matplotlib.Figure.colorbar`.  Optional.\n",
    "    cbarlabel\n",
    "        The label for the colorbar.  Optional.\n",
    "    **kwargs\n",
    "        All other arguments are forwarded to `imshow`.\n",
    "    \"\"\"\n",
    "\n",
    "    if not ax:\n",
    "        ax = plt.gca()\n",
    "\n",
    "    # Plot the heatmap\n",
    "    im = ax.imshow(data, **kwargs)\n",
    "\n",
    "    # Create colorbar\n",
    "    cbar = ax.figure.colorbar(im, ax=ax, **cbar_kw)\n",
    "    cbar.ax.set_ylabel(cbarlabel, rotation=-90, va=\"bottom\")\n",
    "\n",
    "    # Show all ticks and label them with the respective list entries.\n",
    "    ax.set_xticks(np.arange(data.shape[1]), labels=col_labels)\n",
    "    ax.set_yticks(np.arange(data.shape[0]), labels=row_labels)\n",
    "\n",
    "    # Let the horizontal axes labeling appear on top.\n",
    "    ax.tick_params(top=True, bottom=False,\n",
    "                   labeltop=True, labelbottom=False)\n",
    "\n",
    "    # Rotate the tick labels and set their alignment.\n",
    "    plt.setp(ax.get_xticklabels(), rotation=-30, ha=\"right\",\n",
    "             rotation_mode=\"anchor\")\n",
    "\n",
    "    # Turn spines off and create white grid.\n",
    "    ax.spines[:].set_visible(False)\n",
    "\n",
    "    ax.set_xticks(np.arange(data.shape[1]+1)-.5, minor=True)\n",
    "    ax.set_yticks(np.arange(data.shape[0]+1)-.5, minor=True)\n",
    "    ax.grid(which=\"minor\", color=\"w\", linestyle='-', linewidth=3)\n",
    "    ax.tick_params(which=\"minor\", bottom=False, left=False)\n",
    "\n",
    "    return im, cbar\n",
    "\n",
    "\n",
    "def annotate_heatmap(im, data=None, valfmt=\"{x:.2f}\",\n",
    "                     textcolors=(\"black\", \"white\"),\n",
    "                     threshold=None, **textkw):\n",
    "    import matplotlib\n",
    "    \"\"\"\n",
    "    A function to annotate a heatmap.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    im\n",
    "        The AxesImage to be labeled.\n",
    "    data\n",
    "        Data used to annotate.  If None, the image's data is used.  Optional.\n",
    "    valfmt\n",
    "        The format of the annotations inside the heatmap.  This should either\n",
    "        use the string format method, e.g. \"$ {x:.2f}\", or be a\n",
    "        `matplotlib.ticker.Formatter`.  Optional.\n",
    "    textcolors\n",
    "        A pair of colors.  The first is used for values below a threshold,\n",
    "        the second for those above.  Optional.\n",
    "    threshold\n",
    "        Value in data units according to which the colors from textcolors are\n",
    "        applied.  If None (the default) uses the middle of the colormap as\n",
    "        separation.  Optional.\n",
    "    **kwargs\n",
    "        All other arguments are forwarded to each call to `text` used to create\n",
    "        the text labels.\n",
    "    \"\"\"\n",
    "\n",
    "    if not isinstance(data, (list, np.ndarray)):\n",
    "        data = im.get_array()\n",
    "\n",
    "    # Normalize the threshold to the images color range.\n",
    "    if threshold is not None:\n",
    "        threshold = im.norm(threshold)\n",
    "    else:\n",
    "        threshold = im.norm(data.max())/2.\n",
    "\n",
    "    # Set default alignment to center, but allow it to be\n",
    "    # overwritten by textkw.\n",
    "    kw = dict(horizontalalignment=\"center\",\n",
    "              verticalalignment=\"center\")\n",
    "    kw.update(textkw)\n",
    "\n",
    "    # Get the formatter in case a string is supplied\n",
    "    if isinstance(valfmt, str):\n",
    "        valfmt = matplotlib.ticker.StrMethodFormatter(valfmt)\n",
    "\n",
    "    # Loop over the data and create a `Text` for each \"pixel\".\n",
    "    # Change the text's color depending on the data.\n",
    "    texts = []\n",
    "    for i in range(data.shape[0]):\n",
    "        for j in range(data.shape[1]):\n",
    "            kw.update(color=textcolors[int(im.norm(data[i, j]) > threshold)])\n",
    "            text = im.axes.text(j, i, valfmt(data[i, j], None), **kw)\n",
    "            texts.append(text)\n",
    "\n",
    "    return texts\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10,20))\n",
    "\n",
    "#im, cbar = heatmap(np.abs(other_metrics_corr.to_numpy()), other_metrics_corr.index, other_metrics_corr.columns, ax=ax,\n",
    "#                   cmap=\"YlGn\", cbarlabel=r\"|Spearman's $\\rho$|\",vmin=0, vmax=1)\n",
    "im, cbar = heatmap(other_metrics_corr.to_numpy(), other_metrics_corr.index, other_metrics_corr.columns, ax=ax,\n",
    "                   cmap=\"PiYG_r\", cbarlabel=r\"|Spearman's $\\rho$|\",vmin=-1, vmax=1,cbar_kw={'shrink':.745})\n",
    "texts = annotate_heatmap(im, valfmt=\"{x:.2f}\", threshold=.65, fontweight=560)\n",
    "\n",
    "fig.tight_layout()\n",
    "plt.show()\n",
    "fig.savefig(plots_dir/\"metrics_rankerrors_correlation.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e50f488-8e75-4872-8163-a50228377ab8",
   "metadata": {},
   "source": [
    "# Models global ranking performance (RE@R)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "917f09d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_NDCG/drugSexAge_AllCodes/best_models/1/\"\n",
    "                              +\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "icd9_concepts = pd.read_csv(datadir / \"icd9_concepts.csv.gz\")\n",
    "icd9_allcodes = icd9_concepts[icd9_concepts.concept_code.isin(predictions.columns)].concept_code\n",
    "icd9_chapters = (icd9_concepts[icd9_concepts.concept_code.isin(predictions.columns)].query('concept_class_id==\"Chapter\"')\n",
    "                    .concept_code)\n",
    "icd9_subchapters = (icd9_concepts[icd9_concepts.concept_code.isin(predictions.columns)].query('concept_class_id==\"SubChapter\"')\n",
    "                    .concept_code)\n",
    "icd9_3dig = (icd9_concepts[icd9_concepts.concept_code.isin(predictions.columns)].query('concept_class_id.str.startswith(\"3-dig\",'\n",
    "                                 'na=False)')\n",
    "             .concept_code)\n",
    "icd9_billing = (\n",
    "    icd9_concepts[icd9_concepts.concept_code.isin(predictions.columns)].query('concept_class_id.str.endswith(\"-dig billing code\",'\n",
    "                        'na=False)')\n",
    "    .concept_code)\n",
    "del predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eb2dae17-dbaa-49e3-a6ad-a190e4a1f905",
   "metadata": {},
   "outputs": [],
   "source": [
    "def dummy_ranker(y_true_learn, y_true_test, pos_label=1):\n",
    "    y_true_learn = np.asarray(y_true_learn)\n",
    "    freq = (y_true_learn==pos_label).mean(axis=0)\n",
    "    freq = freq.reshape((1,-1))\n",
    "    return np.concatenate([freq]*y_true_test.shape[0],axis=0)\n",
    "\n",
    "from sklearn.dummy import DummyClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "b131febb-48db-4b8d-86d8-3a36fff0ee3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def rank_errors_at_K(y_true, y_score, k, pos_label=1):\n",
    "    y_score = np.asarray(y_score)\n",
    "    y_false = np.asarray(y_true)!=pos_label\n",
    "    arg_sorted_scores = (-y_score).argsort(axis=1)\n",
    "    sorted_errors = np.take_along_axis(y_false, arg_sorted_scores,axis=1)\n",
    "    rank_errors = np.cumsum(sorted_errors, axis=1, dtype=np.float64)\n",
    "    return rank_errors\n",
    "\n",
    "def precision_at_recall(recall,precision_at_k, recall_at_k):\n",
    "    if not isinstance(recall,float) and recall>1.0 or recall<0.0:\n",
    "        return ValueError('recall value must be a numeric value between 0 and 1.')\n",
    "    # Get the index of first rank achieving the recall value\n",
    "    indices = np.argmax(recall_at_k>=recall, axis=1)\n",
    "    # Get the precision a the given recall\n",
    "    p_at_r = np.squeeze(np.take_along_axis(arr=precision_at_k,\n",
    "                       indices=indices.reshape([-1,1]),\n",
    "                       axis=1))\n",
    "    p_at_r[np.any(np.isnan(recall_at_k),axis=1)] = np.nan\n",
    "    return p_at_r\n",
    "    \n",
    "def get_positive_ranks(y_true,y_score,pos_label=1):\n",
    "    y_score_arr = np.asarray(y_score)\n",
    "    y_true_arr = np.asarray(y_true)==pos_label\n",
    "    ranks = (-y_score_arr).argsort(axis=1).argsort(axis=1)+1 # guarantees all ranks>0\n",
    "    pos_ind = np.nonzero(y_true_arr*ranks)\n",
    "    ranks = ranks[pos_ind]\n",
    "    \n",
    "    if isinstance(y_score,pd.DataFrame):\n",
    "        return pd.DataFrame({'index':y_score.index[pos_ind[0]],\n",
    "                             \"column\":y_score.columns[pos_ind[1]],\n",
    "                             \"prediction_rank\":ranks\n",
    "                            })\n",
    "    elif isinstance(y_true,pd.DataFrame):\n",
    "        return pd.DataFrame({'index':y_true.index[pos_ind[0]],\n",
    "                     \"column\":y_true.columns[pos_ind[1]],\n",
    "                     \"prediction_rank\":ranks\n",
    "                    })\n",
    "    else:\n",
    "        return pd.DataFrame({'row_id':y_score.index[pos_ind[0]],\n",
    "                     \"col_id\":y_score.columns[pos_ind[1]],\n",
    "                     \"prediction_rank\":ranks\n",
    "                    })\n",
    "def compute_predictions_ranks(y_score):\n",
    "    y_score_arr = np.asarray(y_score)\n",
    "    ranks = (-y_score_arr).argsort(axis=1).argsort(axis=1)+1 # guarantees all ranks>0\n",
    "    if isinstance(y_score, pd.DataFrame):\n",
    "        ranks = pd.DataFrame(ranks,columns=y_score.columns,index=y_score.index)\n",
    "    return ranks\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "07a817a3-e908-420d-82bb-89a5305b61f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_dir = get_modelsdir() / \"mimic-omop/sequentialNN_3/overall_micro_aucPR/drugSexAge_AllCodes\"\n",
    "tables_dir = get_project_root()/\"docs/latex/overleaf/tables/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9673a651-7480-44b7-b5e3-860222d48756",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "259"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_visits = pd.read_csv(datadir/'test_visits_id.csv.gz').squeeze()\n",
    "train_visits = pd.read_csv(datadir/'train_visits_id.csv.gz').squeeze()\n",
    "oh_diagnoses_df = pd.read_parquet(datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "test_oh_diagnoses_df = oh_diagnoses_df.loc[test_visits,:]\n",
    "train_oh_diagnoses_df = oh_diagnoses_df.loc[train_visits,:]\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "861bb1df-08a7-4baf-a7a0-1d346518fdbc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train a dummy classifier\n",
    "dummy_classifier = DummyClassifier(strategy='prior')\n",
    "dummy_classifier.fit(X=np.empty_like(train_oh_diagnoses_df),y=train_oh_diagnoses_df.to_numpy())\n",
    "dummy_pred = dummy_classifier.predict(X=np.empty_like(test_oh_diagnoses_df))\n",
    "dummy_probas = dummy_ranker(train_oh_diagnoses_df,test_oh_diagnoses_df,pos_label=1)\n",
    "dummy_predictions = pd.DataFrame(dummy_probas,columns=test_oh_diagnoses_df.columns,index=test_oh_diagnoses_df.index)\n",
    "del train_oh_diagnoses_df, oh_diagnoses_df\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed422fc6",
   "metadata": {},
   "source": [
    "### Fig. 1d: Recall at K all codes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "id": "12828471-6c30-437f-8413-be960ed63063",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compare actual recall for models learned on different code sets\n",
    "code_subset = icd9_billing\n",
    "max_K = 50\n",
    "res = {}\n",
    "oh_diagnoses_df = pd.read_parquet(datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "for crossval_suff in ['','_1','_2']:\n",
    "    test_visits = pd.read_csv(datadir/(\"test_visits_id\"+crossval_suff+\".csv.gz\")).squeeze()\n",
    "    train_visits = pd.read_csv(datadir/(\"train_visits_id\"+crossval_suff+\".csv.gz\")).squeeze()\n",
    "    test_oh_diagnoses_df = oh_diagnoses_df.loc[test_visits,:]\n",
    "    train_oh_diagnoses_df = oh_diagnoses_df.loc[train_visits,:]\n",
    "    predictions = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_micro_F1/drugSexAge_AllCodes\"\n",
    "                                +crossval_suff+\"/best_models/1/\"\n",
    "                                +\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    predictions_subset = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_micro_F1/drugSexAge_Sagi\"\n",
    "                                +crossval_suff+\"/best_models/1/\"\n",
    "                                +\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    missing_columns = predictions.columns[~predictions.columns.isin(predictions_subset.columns)]\n",
    "\n",
    "\n",
    "    dummy_classifier = DummyClassifier(strategy='prior')\n",
    "    dummy_classifier.fit(X=np.empty_like(train_oh_diagnoses_df),y=train_oh_diagnoses_df.to_numpy())\n",
    "    dummy_pred = dummy_classifier.predict(X=np.empty_like(test_oh_diagnoses_df))\n",
    "    dummy_probas = dummy_ranker(train_oh_diagnoses_df,test_oh_diagnoses_df,pos_label=1)\n",
    "    dummy_predictions = pd.DataFrame(dummy_probas,columns=test_oh_diagnoses_df.columns,index=test_oh_diagnoses_df.index)\n",
    "\n",
    "    # Add back the missing columns and assign them 0 probability\n",
    "    predictions_subset.loc[:,missing_columns] = 0.0\n",
    "    predictions_subset = predictions_subset[predictions.columns] # reorder columns\n",
    "    for pred,label in zip([predictions_subset, predictions, dummy_predictions],\n",
    "                        ['Cherry Picked Training', 'All Codes Training', \"Dummy\"]):\n",
    "        r_precision,recall = precision_recall_at_K(test_oh_diagnoses_df.loc[:,code_subset], pred.loc[:,code_subset], k=None, pos_label=1, rectified_precision=True)\n",
    "        mean_recall_K = np.nanmean(recall,axis=0)\n",
    "        q1_recall_K = np.nanpercentile(recall,25,axis=0)\n",
    "        q3_recall_K = np.nanpercentile(recall,75,axis=0)\n",
    "        df = pd.DataFrame({'mean_R@K'+crossval_suff:mean_recall_K,\n",
    "                                    'q1_R@K'+crossval_suff:q1_recall_K,\n",
    "                                    'q3_R@K'+crossval_suff:q3_recall_K})\n",
    "        if res.get(label) is None:\n",
    "            res[label] = df\n",
    "        else:\n",
    "            res[label] = pd.concat([res[label],df],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "45c6bbbf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max Recall@K difference:5.51%\n",
      "All Recall@K: \n",
      "4     0.255480\n",
      "9     0.368433\n",
      "19    0.488827\n",
      "29    0.559845\n",
      "49    0.644533\n",
      "Name: mean, dtype: float64\n",
      "Sagi Recall@K: \n",
      "4     0.249294\n",
      "9     0.350027\n",
      "19    0.455884\n",
      "29    0.518238\n",
      "49    0.589346\n",
      "Name: mean, dtype: float64\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQAAAACjCAYAAABljcCSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAq1klEQVR4nO2deXyU1dX4v3cyayZDFghh3zcRARUQkSUIVQrqq9YqVbHqa6211v6qb22tti5vaWttq9VaFLXFV3GrVqrSqqiAIooL4oIKggRIQghZZpJMZn/O749nMoaQZRISZkju9/PJJ8/Mc5/nOZPMOffcc889V4kIGo2mZ2JJtQAajSZ1aAOg0fRgtAHQaHow2gBoND0YbQA0mh6MNgAaTQ8mZQZAKTVAKbVZKRVUSlmbOfe6UmqjUmp+qmTUaLo7KlV5AEopJ+ACngPmi0i00bl7gCeBj4EXRaQwJUJqNN2clHkAIhIUkeoWTk8E3haROqBWKeU5gqJpND2GdI0BZMjXrokPyE2lMBpNd8XadpOUEGt03AvwNm2glLoSuBLA7XafOG7cuCMjmUaTxnzwwQcVIpKfbPt0NQAfK6VOxowB9BKRmqYNRGQ5sBxgypQp8v777x9hETWa9EMptbs97VM5C2BTSr0KTAJeVkqdpJS6N37698BS4FXgN6mSUaPp7qTMAxCRCNB0im9T/FwxcOoRF0qj6WGkaxBQo9EcAbQB0Gh6MNoAaDQ9GG0ANJ3Ka6+9xpw5c5g9ezbnnHMOlZWVXHrppezYsaPLnllYWMicOXOYNm0azz33HGVlZSxdurTZtitWrOChhx5q9/0b8Pl8FBYWUlhYSHZ2NoWFhVx22WWtXr9lyxYefvjhFs//6Ec/apc8nUm6TgNqjkIqKiq4/fbbefHFF/F4PGzfvp1wONzu+xiGgcViafF1c7z22muEQiHmzZvHO++8w0033dTu5yZDdnY269atA2DmzJmJ49bknDx5MpMnT27xnvfee2+L57oa7QFoOo3Vq1ezZMkSPB4zc3vMmDH0798fML/kM2fO5LbbbgNgx44dnHbaacyZM4df//rXAFx66aVcc801LFiwgBUrVnDBBRewaNEi7rzzTu677z7A7E1b6jHdbjcul4uioiIuvvhiAFatWsX06dOZO3cu69evT7QtKSlh4cKFlJaW8uKLLzJ79mxmzJjBSy+9BMDy5cuZPn06P//5z9v83I3lLi0tZe7cucycOZOrr74agHXr1nHzzTcDMGnSJC655BImTZrEli1bANOQgOlp3HjjjUydOjXhMTz//POceOKJXHnllYl2nYk2AJpOY9++fQmFb0phYSEbNmzg3//+NwA33XQTDz/8MOvXr2fr1q0UFxcDcMopp/DKK68AkJOTw+rVq7n22mtZvXo1AE899RTf+c53mn3GgQMHiMW+TiI1DIOlS5eydu1a1q5dy6xZswAoLS3le9/7Hg8++CD9+vXjD3/4A6+//jrr1q3jzjvvJBqN8vDDD7NhwwbOOuuspD57g9x9+vRhzZo1bNiwgZqaGr788suD2pWVlbF8+XL++te/8sgjjxxyn/PPP58NGzYkzt1xxx288cYb3HLLLezfvz8pWdqDHgL0dLasBO+ejl+fMwQmXwRA//79KS0tbbbZhAkTAHC5XABs27aNJUuWAOD1eikpKQHgxBNPTFzTcOxyuejbty979uxh06ZN/OY3h+aGzZs3D6vVym9/+9vEewcOHGDo0KGJZza45/fffz9Lly5l4MCBlJeX8/nnnzN/vpmSUl5enrjOarUeJE9rNLSrrKzkBz/4AV6vl6KiokP+HqNGjcLpdDJw4EC8Xm+zfyebzZaQNSMjA7fbjdvtpk+fPknJ0h60AejpxJW3M1i4cCHnnXceixcvxuPxsGPHDtxuNwBKqYPajh07lrvvvpv+/fsTi8VQSrFs2bKDxtCNjy+88EKuv/56pk2bdsi9wIwBWK3m17moqAiA/Px89uzZQzAYxOl0YhgGADfffDOrVq3imGOOYdq0aRx33HG8/PLLZGRkEIlEUEqxe/duYrEYH374YVKfvUHWxx9/nLPPPptLL72Uiy66iKbL7RvL3txS/KafzTAM6uvr8Xq9VFRUJCVLe9AGQNNp5Ofn88tf/pIzzjgDESEvL6/F6PfSpUu5/PLLCYVC2Gw2nn322VbvPW/ePL773e8mxtLJYLFYuPHGG5kzZw5ut5tbbrkFALvdzsqVKznvvPO46667uO6665g3bx5KKcaPH899993HZZddxowZM5gzZ07yfwDg1FNP5ZJLLmHVqlXtuq4lbrjhBmbPns3kyZMpKCjolHs2JmUFQToTvRio+xOLxViwYAFr1qxJtShHlGg0itVqpaSkhCuvvDIRC2kJpdQHIjIl2ftrD0CT9lRVVfGtb32L73//+6kW5YjzzDPPsGzZMvx+P/fcc0+n3197ABpNN6K9HoCeBtRoejDaAGg6nVGjRvHkk08mXhcWFhKNRrn11lt59dVXD2pbX1/PlVdeSWFhIaecckqzc+NNWbduHbfeemu7ZNq2bRuFhYVMnz6dgoKCRNJNa7z00kstjrlbSzc+mtAxAE2n8tFHHzFr1ixeeOEFFi9e3Gb72267jTlz5rB8+XJEhDfffLNL5Bo7dizr1q2jqKiIm2++mcceeyxxrqUU3gULFrR4v379+nVZuvGRJKUegFLqLqXUm0qpPzd5f45SapNS6h2l1FWpkk/Tfv75z39y9dVXU19fTygUarP9xo0buegiMxdBKcXs2bMBuPbaa5k9ezZnnHEGPp8PgMsvv5z58+fz6KOPJq5/6KGHmDVrFrNmzWLz5s1UVVVRWFjI3Llzufbaa1t9dmFhITfccAOXXHIJW7ZsYc6cOUyfPj2RaNSwcKioqIhTTz2V8847jxNPPJHi4uKD0o1PPvlkrrnmGiZPnpxIJb7//vuZPn06P/vZzw5aTJRupLIk2AmAW0RmAXal1NRGp68Hvg3MAFpfaqVJKzZv3szUqVNZsGDBIe5+srz33nv4/X7eeOMNFi9ezP3338+7775LRkYGr776KiNHjgTMxUfPP/88b7zxBv/617+4/fbb2bx5M4WFhaxdu5Y///nPbTwJzjnnHB577LGEh/DOO++wZs0aAoHAQe2qq6t5+umnue666w7JWaisrORXv/oVq1ev5oEHHiAajbJixQreeustzj333A79DY4UqRwCnIxZ84/47+nAe/HXW4Fs4ADgP/Ki9RxWvLOTosqO/4mH9XZz6XRTIXfu3Mmnn37KggULCIVCjBkzhkWLFrX7njt37uSEE04AYMqUKaxfv56hQ4dy/PHHA2ba7dtvv81XX33FRx99xNy5cxPXzpkzhzfeeIMLL7yQb37zm4l045ZoSOHdtWsX119/PfX19Wzbto3y8vKD2o0fPx6LxcLAgQMPWdqcn59P3759ARIZe0OGDCEjI6PVVYDpQCoNQA6wM37sA45tdG4V8GL8+PbmLm5cFnzIkCFdImBPoEF5O4Nnn32Whx56iHnz5gFw1llnHbQ4pzlmzJjBypUrE2mzb731FiNGjEgsCHr//fcZOXIkw4cPZ+3atQCJ9Nzhw4czdepUnnnmGQAikQixWIzbbze/MpMnT27TADSM/ZctW5Zw12fOnNmuFN6m5/r06cPevXsxDIOPP/641eenmlTGALyYNf/h0Nr/dwIzgdHAJUqpzKYXi8hyEZkiIlPy85Mug67pQlavXs2MGTMSr8ePH8+GDRtaveaWW25h/fr1iVmAnTt3Mm3aNFwuF7NmzeLxxx/nqquu4qSTTkqs99++fTtg9ryLFi1i9uzZzJ07l9/97ne8++67zJw5k5NOOimxwCcZFi1axDXXXMP555+P3W7v2B8gjtVq5bvf/S4zZszg8ccfx2azHdb9upI2E4GUUl8CGYAACggBXwI3icgnHX6wGQP4voh8Xyn1V2CFiLwbP7cWOEtEauPHZ4uIr6V76UQgTbrRkMK7adMm/va3v/HAAw8cked2RSrwM8ALmCW7pwHnYm7o+QBmkK5DiEjDzsBvAh+JyLtKqXtF5EfAHcCrSikD+E9ryq/RpCP33nsvq1atIhwOJ5XbkCqS8QDWisjcpq+VUuvSZdde7QFoNCZdkQpcqpT6q1LqsrirXqaUsgKdvzhZc9Szbt06hg4dyrx58ygsLOSJJ55ItUiaVkjGACwB1gD5mNN1F4lIVETO61LJNEctS5Ys4bXXXuM///kPK1euZPPmzakWSdMCbRoAETGAbcDbmL1+51cm1HRLXC4X119/PS+88EKikMeKFStYsWJFIrvu/PPP5/jjj+ef//wnp512GjNnzsTv97d5/o477kjk6a9atYo//vGPqfyoRy1tGgCl1BPAnzA9gSXAxV0tlKb7MGDAgIMWBjWmurqaJ598kp/+9Kc88sgjvPLKKyxcuJCXX365zfMXXnghTz31FAD/+Mc/uOCCC47YZ+pOJDML0D9dgn2azqeoqAi/v+OZgG63m2HDhrV4vqSkhMWLFxOJRAAzUaYhcaYhu27AgAGJoqEDBgygurq6zfODBw+mqqqKyspKvF4vgwYN6vBn6MkkYwDWKqWOO5w5f0360pryHi7BYJC7776b2267jb/85S8AfPLJJ0ycOBE4OIOuuUy7ts6fddZZXHXVVZx55pld9hm6O8kYgIuBS5VSEcxEIBGRMV0rluZo5tFHH+Xtt98mFotx5ZVXMmnSJEpLS1m4cCG9e/futOd8+9vf5rrrrmPZsmWdds+ehi4Jpjlqqa6u5oorrmizonBPotMyAZVS80TkNaXUhU3PicjjHRVQo+kMvvjiC6644opmNwnRJE9rQ4CGqMrojtxYKXUssE1Eoh25XqNpjXHjxrW50EjTNi0aABF5JP77tmRvppRaLSKLlFI3AN8AKoG260JpNJqU0NoQYA3mCsBDEJHTWrjMGf99nIh8Qym1voV2Go0mDWhtCHBFB+7nUkpdAZTFXx+6iZtGo+kwMcMgGDEIRmIEozFC0Rh1oSi1wQg1wUi779eaAWj/3UyjcTrwa6WUC1jRgXtoND2OSCyu1HHFDkZi1MYVuy4UwVsfobSmngO1IWqCEarqQ/gCEaKGJPIiLJb297etGYCVfF0EpDECnNrcBSLymVJqOHAB8DDwbrsl0mi6GZGYQSAcIxCJEojEqA9H8QYi1AYjVPlD7Kmup6IuiDcYodIfoi4YJSaCIYJCYbGA05pBjstGtstOb7eDY/vn0D/bSbbTjsdppZfThstmZUTrhZAPobUg4NyWzrVEfLlwEJgpIsuVUndhBgNban8XMAXYLCI/bvS+E7gPGA5sjRcJ0WjSjsbKHYzG8Iei1AQj+AKmcu/11rO/Nkh1fZgDdUGCEYOYYSBAhlI4bBZyMx3kuez08zg5flAu/Xu5yMm0k+204XHacFgzcFgtB/3uSG/fHK0FAS8SkZVKqV80PSciLU2+jhORU+NlvMAsJdbS/RNlwZVSy5RSU0WkoSrwtcDjIvJakp9Do+kSGhS8PhKlPhzFF4hQXR+mqj7E7io/ZTUBqvxhDtSFqI9EiRmmO56hFHarhbxMB73ddobluZk5Mp8Cj5Mcl51sl40shw2nLQNnFyh2srQ2BGgo+FHSjvtZlFIeQOKFPFtLM2ytLHghMFAp9UvgTyLyfDtk0GiSJhozqG+i4N5AhCp/kBJvgBJfPRX+EPtrgwQjMQzDXMxktSjy3A76uB0MyXVzSiPlznHZybRbcdoycNkycNpMBW+8niFdaG0I8HL89yNKqT6Y7vguEWmtEtAtwHpgWPz39a20zaHlsuAjgbuBXwDrlFL/1glFmo5iGEJ9JIo/FMUfjlLlD1NRZyr4rqo6KupC7K8LUheMHhRQy3bZyHc7GZRj9t4DemWSm2knN9OOy2Yl0/61gqejcidDm4uBlFI/Ai4HPgWOVUqtEJFmNyoXkfXACUqpPm0YCmi9LLgPWC8iIaXUDqCAJp6I3hdA0xjDEPxhU8H9oSi+QJgKf4iymgBfHqhjf02AfTVB6sOmkluUwmW30r+Xk/wsBxMH5jAwx1TwvEw7HqeNTJuVTLsVl+3Iu+YJRMCIQDQEsQjEQhAJQH0FeEugphj85RD0mT/tJJnVgEuAqSISjdcC3AgcZACUUrfQxN1vsIgi0uzGHpgVhr4PPA3M5+Apw43ARKXUZkxv4kDTi0VkObAczMVASXwOzVGOiBCImPPe/lCU6voQFf4Q+3xBvqqoZX9tkH01AfwhM4quUGTaM+if7aKfx8WM4fkMyHHR2+1IuOluu9mTWzOO0BYZIhALm4ocDZvHwRqo3Qe+YqgphWC1qczhWlPpm0unsTrA7gFXNmT2gX4TIWcI8H/tEicZA7ANsAPR+O8vm2nTsFfSaZj7BmwCpgKulm6aRFnwRzA9gwdFJJzsB9Ic/URiBnWhSCLB5UBdiAO1QXZW1LGvJkCpL0CVP5TocZw2C/16uSjIcnLi4DwG5WbSx+0gz+3A7bCS5bCSabN2XS+e6KWDplJHA+CvBO8e8O2B2v2mUgd8YDQzklUK7G5w9gJHL8gdDr0GQvYgyMwDqxOsdshwQIYdLC3G1ttNi8uBG6UC5wBDgS+AYzDjACe1cM0LInJmo9erRaT9m8O1E70c+OjEMIS6+LRZVX2IfTUB9lb52VVZR1ltkDJfkFA0Zk6ZWRS9Mx30z3YyKCeT4b2z6Otx0tvtwOO0keUwe/NO78kbeuxoEMJ1pjJXfgmVO003vL7KdM+/vsBUaIsVHNlmD53VF3KGmj/uPnGFjitzhh1U58ncmRuDdCQVuK9SahpmNH8q0KcD99B0QwxDzPnxYJjymiDFvnq27vOxu9JPsbeecNQABW67lQHZLgblZDJrRF/6Z7vIz3KQm+kgK96bd1rQTcR0xcP1EIj32JU7TVe8vhJCNSRy4UTMY4sdMnNMt3vgFOgzCjz9weoyldrqAIvNNAJHAa3NAuzuwP0uBX6LuYT4S8zgoaaHEYkZ1AQi1IYiVNSF2FPt55MSLzsq69jnrSdmCNYMxYDsTAbnZjJ3dAH9czIp8DjIdtnxOGydMy4XAyL1EKg2lfvAl+AtMnvuYE1Do6/b27PAlWe63sNmQe4wcOWCPdN0vzvR9U4XOnV3YBH5HDhbKZUvIocE7jTdj0A4ii8YodofZl9NgJ0HatleXsNeb4ADdUEMEawWC4NzMzmmoBcXHD+YATlu+rgd9HLa8DitZFg6qOgipoLXV0J1EZR/Ab7d4D9guuwJ3RawZZrud69BMGwm5I2IK7fbdMN7KO02AEqpTBGpb+HcQuAuYL9Sqi9wnYj8+zBl1KQJhiF4A2Gq68Psra5nx4FatpZ52VXppyYQAQSP08aQXDdTh+YxOt9D/14u+vYyE2Q8Dlv7A3FGzHTF6/bD/q1w4AvzuGHKS+JjblsmZBVAn9Ew7kzIGQzOHLC5OnWM3d1oLRX4TyJyXZP3coAXaXlzkF8C00TEF2/7EqANwFFKJGbgrQ9TURdiV2Ut7+2pZOu+Gir8IUSELIeVkX08nH5Mf8b09TCgVyZ5bru5MMXejr7FiEHQa469938KlTugtszs3RuC1JYMyOxtRsfHnQG9R5s9ut2tFfwwaO2/lKGUuktEfgKglOoHrAbubOUaadjJV0S8Sik9P3+UICLUBqNUB0KU1QT5sryWD4ur2HGgDl8gjMWiGJKbyUlDe3PcwBwG5WSSm+kgx5WksotAxG/23mWfQtknUFtqBuDMBmbwzJ1vjr1HzoO84eaY3Oo8aoJqRxutBQF/rJS6Wyl1N2Za7vPAz0TkP63c72Wl1CvAO5i5/a90oqyaTsQwhOr6MPtrA+yp8vNhcTWflfko8dYTEzMaP7KPmzMmDGB8v2wG5mSSl+nA47S2HYGPBMzElpLNsG+LqeixyNfuemYfc0ps4vnQe4zZk/fgcXgqadV0i8j/iy/ZfQs4X0TeaqP9bUqpScBY4FkR+ajzRNUcDg09fHltgF2VdbxTVMHHJV4O1IVQCgZmZzK+Xy8unjqMwbnuRLacw9ZK5DsWhrpys0cv3Qy+vRBu2GVIzN48fywc+y3IHWomulg6Ne6sOUxaiwF8ydcFQYLAirjlb3FjEKXUEuAxEflImSwRkUe7QG5NEgQjMQ7UBdlT7ef93ZW8u7uS4up6lFIMzcvklBH5TB6Ux5A8s3fv5WwhSCdiJsF4d8PujVD+mRl5B0CZyS45w2DSd0yFz+ytFf0oobUhQEfKgV/aoPAiIkqp7wLaABwhDEOo8Ico9QXYWurlrV3lbN9fS8QQ8rMcTBmcx/dmjGJoXlY8g64Zd14MM+peuQuK1kPF9q8j7hYrZA+G4XNgwPHg6WeOzzVHLcmsBnRgJvQcA2wSkZWtNHcrpVRc+S1AVifJqWmBcDRGeW2IrypqWfflft7bU4kvEMVtz2DCgGx+OHsM4wqyKYhPxR3Sw0fqTTf+q3VQ8j4EqgBljslzhsL4c6D/RDMYl2FLxUfUdCGtGgClVBbwD8zsvhXABUqp/wEeaSHR5wngdaXUBmAGoHcQ6gLqQhH2+QJ8ts/LK1+UsbXMh2EIw3tncfbEQUwelMeA7Ez6ZDmwNc6mM2LmgpSqXbDtP2bvHgub02jZA2HUN2DIdO3C9yDa+i/fAPwFOBMzzVcBo4DdSqlsEXmocWMR+XN8EdGxwBMi8lnni9wzCYSj7Kny8/auCl7+Yh/F3nocVguTB+by8/njGVPQi74eJ1mORr10NATecijaAMWbzLl1EXNlWZ+xcMq1kDcSnNl6Lr2H0pYBOEVEfqWUmg48C8QwS3k9g5kQ9FDTC+JKrxW/EwhHY5TVBNlSXMmqj0vYXl5LtsvGnFF9+X9zxzEk101+luPrnPloCLx7YefrsGejmVwD5mKVQVNh6Exw9wabW8+ra4C2DUBDdzJTRH4ZH9ffEB/j53StaD2TSMygrCbAF2U+Xvi0hI9KvGQoxYwRffjejFGMyvfQJ8thBu+MqDnfvuNV2Pu2uehFMN35cWfCkJPMfHftzmtaoK1vxhdKqQnA0/FtvgzgGaXUWL6u59dhWioLHj+ngC3AvU2HGt2R6voQ2/fX8OyWvby3pxIRmDQol5tPP5Zj+pk14DMsFjNot2cjfP48VO8GxFy9Nu5Ms5fPzNMKr0matr4pt2KW3VqMWWtIYZb6fgqznBcAzW0h3kBLW4m3URYc4CygPJkPcbQSjRnsrfaz7styVn28l0p/mGP7Z/Oz+eM5dkAO/Xq5sFmUmT77weOw5x0zndbuhkEnwcnXgqdAK7ymw7SVCViqlLoR+BtQCjgwC3Te2KReQEdyBlorCw7wHUxD0+3wBcJs3+/jiQ/28P6eSnJdds48bhBzRvdlcK4bp1WZC2LeeAYOfG7OzecMgSmXwYATzBVuGg1mhqdhGMRiMQzDaPf1bXYdIvIJcL5SKhezhFhVM22S3kK8ETm0UBZcKXU6ZlnxaDIyHg0YhrCvJsDbXx3gyc27KasJMGlgLnecfTzjCnqR67KZ69nXPQlVO83Vb32Phfm3mYtjdJS+xyEixGIxotEowWCQuro66urqCAaDRCIRIpFIoox5R2ktFbhZt14pdYhb38GtxL20XBb8CsxqxItbke+oKAseisTYVVnHcx/vZc0X+8hQFhZNGMjC8QMYlufC7v0KNt5nzsln2MwyU9OvNte260h9t8QwDAzDIBqNEg6H8fv9+P1+AoEA4XCYSCRySG+ulMJms2G323G73Ykfu91ORkYGFosFSwcKq7TWu7bHrW+pfmBr5qm1suCjgVXAQMx44AYR+eKgG6d5WfBQJMYX+308+NYOtpRUMyTPzfVzx3PikFwKlBe15QFzpZyymG79oj+a03Wao5aGHjsWixEOhwkEAgnlDoVCLSq21WrFbrfjdDrJycnB7XbjdDqxWq0J5e6qjUdaWwuQtFvftH6gUioPc4fgC4FZLVzTYllwEZkcv8+lgLWp8qczoUiMrWU+7n9zO5/t8zF1aG/+8u2pjM614i5aA2teNMtV5R8D8283S1Ppnj7taRhnR6NRAoEANTU11NbWJhS7OVe8QbEdDgf5+flkZWXhcrkOUuxUk0xZ8ENozq2Prxk4G1PxpwK/Al4RkfbsLdgh0qEseDga4/MyH/es38b2/bXMHJHPpSeNYISlFNuHK8yade58mHyROV2nI/dphYgQjUaJRCKEQiGqq6upra0lGAwSi8UOaquUwuFwkJmZicfjwe1243A4sFqtCVc8VVuFpaQsuFLq75hu+z+BHwArROTvyV5/NBOJGWzf7+Pe9dvZus/H7FF9uWneGIZWvUPGW/eY2XnDZkHhjWaNOk3KaHDRG3rzUChEXV0dlZWV1NXVJdxzi8WCy+XC4/EwcOBAMjMzj4g7ngqSKguulBoEDObrPYqalgzvD9QA+zAj+mk3Ju9sRIQSb4C7133OO7sqOGV4PisumMCwr54m4/WNZn79iZfD0JN1b3+EaOqmN0TNA4EAoVDokPZKKex2Ox6PhyFDhiTG3VZrElWPugnJLAe+BXPLr+FAEVALnN64jYgsUEoVYM7drwbGxCsErxWRQGcLnWpqAhGe+KCIx9/fxbH9snn4v8Ywatf/kbH2I+g9ChbeCdlD9Ni+k2lQ8FgsRjAYxOfzUVNTk5gWa0yDcjeMv7OzsxMK3t168cMhma5pIWaSzlpgLvBYc41EZD9m7cC7lVJjgIuB3wMTOkXSNMAwhE27DvCbNZ9hy7Dwu/lDmLJnBba3t0P/SXD2A5CVn2oxj1oa9+B+v5+amppEBD0cDiMiiAhKKZRSOJ1OMjMzEwE2h8ORNsG1o4VkDEBdfPGPwtwcdGxbF4jIdswg4K8OU760wVsf5s5Xt/LWVxV8f0pfzq55GteHn8Hgk+DUv+nxfTswDINwOIzX66WiooJAIHBID26xWHA4HLhcLnJzc/F4PIk574ZeXHP4JGMAXlRKuYCHMXcKfrZrRUovDEPYVFTB/770CUOyHTwz9j367HrXjOR/62Fz8Y3mEBoSXUKhEDU1Nfh8vkPG4jabDbfbzaBBgxIKrpX7yJJMKvBd8cP/o72bjx/l1Iej/G7NVjbuPMD/jK7iG1X/IEONgnMfNtfVaw4aj1dVVVFfX080evAW2Ha7HZfLRd++fcnJyUmMxfU4PPUkEwRcCVzcaBjwmIhc1PWipZbdlXX8z3Obyc0I8lzBM3jqozD/FijoNiGNdtHYba+qqsLv9x+k6A29+bBhw/B4PNhsNjIyOmkXX02XkcwQoL/Es4XiRqBb56vGDIPntuzlvje2cXneF1zIWjKOWQwTzuuWu8M2xTAMIpEIfr+fqqqqRDJMQ/DNarXidrsZPHgw2dnZOBwO7bIfxSRjAJRSaqyIbIsXAum2/+1wNMbt//mUj3bt5bG85xiY64ZTHzR3rulGiEgiV93r9VJbW0sgECAajTYs9kokw+Tm5pKXl5dIhtF0L5L5j/4Y+LtSqg9mgY6ru1ak1OAPRbnu2fexH/iEZ3P+jX3yRXDcuUf1MtyG9NaGbLeampqDgnANKa1ut5uCggKysrISkXbtuvcMkgkCfoxZ4rvbUl4T4JonNzKzfh1XD96P9fQHIKtvqsVKigYlD4VC1NbW4vV6qa+vJxwOJ9oopRLz5Xl5eTidTjIyuv9wRtM2yQQBp2Am+HiAE4GlIvKzLpbriLF9fw0/Wfka11he4PTjx2GZdUtap+42JMmUlZXh8/kSbjuYgTin06mj7ZqkSeab/ifgXOApEYkqpaZ2sUxHjK0l1dzwyL/4dfbLHL/whzB8ZqpFOgjDMAgEAlRWVlJZWUkgYGZVK6Vwu90MHTqU3NxcbDabVnJNh0jGAEREpFwp1a0W+GzZW8mvVvyD3xZsYuJ37jaLa6aYaDSKz+ejrKyMurq6xDJUp9NJXl4eo0ePxuVyafdd02kkYwC+UkrdAGQrpX6MmQ14VPPeV2UsffQfLB26jeMW/9nctjoFxGIxampqKC0tpba2FsMwyMjIwOPxMGrUKHr16qVdeE2XkowB+AHw35glvELANZ318Jb2BYivQFwQf3mziLzWWc/cUlTG7x5Zyf+OLee48/8IVkdn3bpNDMOgrq6OkpISfD4fhmFgsVjweDyMHTuWXr166d5dc0Rpa3NQF5AhIg/EX38TeA0oPNwHt7EvwP+JyG3x3Yeejz/zsNlXXcdvH1nJjcfUcdz5vzkiiT2RSIT9+/ezf/9+QqEQSimysrIYPXo0OTk5WuE1KaW1qsA/xuz9bUqpezBrAuyj8zyAFvcFEJFd8fdDdFJxEX8wwi/uf5iL+lcw5fxfd6nyiwjV1dUUFRURCARwOBwUFBRQUFCA3W7vsudqNO2lNQ9gMTAecAMlwGwR2dKJz86hhX0BGnEr8EBzF7enLHgkZnDzsgeZ7ijmzMv+t8uUPxgMUlRURFVVFUop+vXrx8SJE3UGnSZtae2bWS8iBlCrlPqwk5UfWt8XAKXUOUDvlrYWS7YsuGEIv39kJbmBIi7/8S9QNmcniP41kUiEkpISysrKiMVieDweJkyYgMfj0cE7TdrTmgE4Xin1CmYdwOMaHUsrm320hxb3BVBKTQR+CCw63Ic88+rrlOzexu9/+ENs7tzDvR0A4XCYkpISysvLiUajOJ1ORo4cSe/evfXCGM1RRasGoCsf3Nq+AMCdmHsQvqyU8onIf3XkGXv27WPVmxv59bdPJ6vv0MOSNxwOU1xcTHl5ObFYDJfLxYgRI8jLy9OBPM1RS1JVgbuKpluCx5UfETm9+SuSJxKNcsffV3Du2CxGTGx2b5I2MQyD0tJSiouLMQwDl8vFqFGjyMvL0z29plvQbaNT9/ztfvrZgpz7nRvafW00GmXXrl2Ul5djt9u10mu6Ld3SALz19no+K67g7p/8EEs73HO/389XX32Fz+fD7XYzadIksrKyulBSjSa1dDsDUOOt5MGX1vPjMwvx5La9pNcwDPbt20dxcTHRaJS8vDymTJmC09m5swUaTTrS7QzAskcfYnK/TI6fWthqO8Mw2Lt3L8XFxdhsNoYNG0Z+fr528zU9im5lAHZ8sYVtFUH+csNPWmwjIuzevZuSkhIcDgcTJkwgOzv7CEqp0aQP3cYAiGGw/Ll/sXDSKDLdza/uC4VCfPzxxwAcd9xx9OqVmlWAGk260G0MwOsvP0MwZuHc/1rc7PnKyko+//xzCgoKGDVqlM7S02joJgZAxODpTVu56pwzD4n6iwjbt2/nwIEDjB8/nrw8vZOPRtNAtzAAlZXlFPTK5PhJUw56PxgM8umnn6KUYtq0aXolnkbThG5hAOrCwo+WXJZ4LSLs3buX3bt3M2DAAEaMGKFdfo2mGbqFARjUN5/8fHPOPxQKsXXrViKRCCeccAJutzvF0mk06Uu3MABWqxURoaioiOLiYvLz8xkzZoye09do2qBbGADDMHj33XexWCy619do2kG3MACBQID+/fszePBgPdbXaNpBtzAAbre7zbJgGo3mUFI6SFZK3aWUelMp9ecm7w9QSr2ulNqolJqfKvk0mu5OygxA47LggL3JlmM/B27GrER8cyrk02h6Aqn0AJorC97AROBtEanDLErqOdLCaTQ9gVQagBygJn7sAxpX7MwQEWnhnEaj6SRSGQT00nJZ8Fij40NKhsPB+wIAdUqpdNqzsA9QkWohWkHL13HSWTaAse1pnEoD0GJZcOBjpdTJwMdALxGpaXpx430B0g2l1PsiMqXtlqlBy9dx0lk2MOVrT/uUDQFEZDPQUBbcaCgLHj/9e2ApZmzgN6mSUaPp7qQ0D6CVsuDFwKkpEUqj6UHoZPmuIS2HJo3Q8nWcdJYN2imf+jrYrtFoehraA9BoejDaAHQC8dTlhr0OrfH3fqqU2qCUWqmUsqVQtpPiKdVvKqXuSifZ4rJMaCTf35VJ2sgXl/E6pdSG+HHayKaUGqaU2q+UWhffvLfd8mkD0DlUAfOAdwCUUvnAXBGZiTmVeXbqRGM3cGo85bqvUmpWGskGsE1EZsTlA5hCGsmnlHIAk+LH6fR/bWCNiBSKyGkdkU8bgE5ARIIiUt3orWnAuvhx0zTnI4qIlIlIMP4yiplmvS7+OqWyAYhIpNHLEDCGNJIPuAJ4JH6cNv/XRsyNe08/oQPyaQPQNeTQcppzSlBKTcTMYvOSfrKdpZT6FOiLOTWdFvLFXeg5IvJ6/K0c0kS2OPswDeZczGS6KbRTPm0AugYvLac5H3GUUnnAX4D/Js1kAxCR50VkAlCC6aWki3xLgMcbvfaSPrIhIiER8YtIFHgR2EE75dMGoGt4D5gTP55PPDaQCuJByceAn4pIWTrJBokxdgM1QAbpI99Y4AdKqZeAYzF72HSRjSarZE/BNADtkk8bgE5AKWVTSr2KGSx6GRgOvBGPHE8GVqVOOr4NTAXuUEqtA0aSPrIBLFBKrVdKrQcKMI1VWsgnIj8TkdNFZAGwVURuSxfZ4sxSSn2glNoIlIrIJtopn04E0mh6MNoD0Gh6MNoAaDQ9GG0ANJoejDYAGk0PRhsAzWETz0l/NX7siUf1z0u1XJq20QZA02kopbKAF4D7ROSZVMujaRttADSdRRZmNtpfReTpVAujSQ5tADSdxTggG3g+1YJokkcbAE1n8T6wEniqoSaCJv3RBkDTaYjIH4CtwN+V3qb5qEAbAE2nIiK/AOqAe1Iti6Zt9FoAjaYHoz0AjaYHow2ARtOD0QZAo+nBaAOg0fRgtAHQaHow2gBoND0YbQA0mh6MNgAaTQ/m/wO6NMh4uGumtQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 251.059x155.163 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for label,col in zip(['Cherry Picked Training', 'All Codes Training', \"Dummy\"],\n",
    "                    ['#FF800E','#006BA4','#ABABAB']):\n",
    "    df = res[label]\n",
    "    for stat in ['mean_R@K','q1_R@K','q3_R@K']:\n",
    "        cols = df.columns.str.startswith(stat)\n",
    "        mean_stat = df.loc[:,cols].aggregate(['mean','sem'],axis=1)\n",
    "        if stat=='mean_R@K':\n",
    "            plt.plot(range(1,len(mean_stat['mean'])+1), mean_stat['mean'],label=label,color=col,linewidth=.5)\n",
    "            plt.fill_between(range(1,len(mean_stat['mean'])+1),mean_stat['mean']-mean_stat['sem'],mean_stat['mean']+mean_stat['sem'],alpha=.3,color=col)\n",
    "        else:\n",
    "            pass\n",
    "            #plt.plot(range(1,len(mean_stat['mean'])+1), mean_stat['mean'],color=col,linestyle='--')\n",
    "            #plt.fill_between(range(1,len(mean_stat['mean'])+1),mean_stat['mean']-mean_stat['sem'],mean_stat['mean']+mean_stat['sem'],alpha=.3,color=col)\n",
    "    # keep computation for further analysis\n",
    "        if stat == 'mean_R@K':\n",
    "            if label=='Cherry Picked Training':\n",
    "                sagi_recall_k = mean_stat['mean'][0:max_K]\n",
    "            elif label=='All Codes Training':\n",
    "                all_recall_k = mean_stat['mean'][0:max_K]\n",
    "plt.gcf().set_size_inches(single_col_fig_size)\n",
    "plt.xlim(1,max_K)\n",
    "plt.legend(frameon=False)\n",
    "#plt.title('Recall@K for billing codes on the complete code set')\n",
    "plt.ylabel('Recall@K billing \\n All codes')\n",
    "plt.ylim(0,1)\n",
    "plt.xlabel('K')\n",
    "plt.savefig(plots_dir/\"recallatK_uF1_allcodes.pdf\")\n",
    "print(\"Max Recall@K difference:\"+str(np.abs((sagi_recall_k-all_recall_k)).max()*100)[0:4]+'%')\n",
    "print(\"All Recall@K: \")\n",
    "print(all_recall_k[[4,9,19,29,49]])\n",
    "print(\"Sagi Recall@K: \")\n",
    "print(sagi_recall_k[[4,9,19,29,49]])\n",
    "del sagi_recall_k, all_recall_k"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f82d1e4",
   "metadata": {},
   "source": [
    "### Fig. 1c: Recall at K cherry picked F1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "3bbb5226",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compare actual recall on the cherry picked for models learned\n",
    "# on different code sets\n",
    "code_subset = icd9_billing\n",
    "max_K = 50\n",
    "res = {}\n",
    "oh_diagnoses_df = pd.read_parquet(datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "for crossval_suff in ['','_1','_2']:\n",
    "    test_visits = pd.read_csv(datadir/(\"test_visits_id\"+crossval_suff+\".csv.gz\")).squeeze()\n",
    "    train_visits = pd.read_csv(datadir/(\"train_visits_id\"+crossval_suff+\".csv.gz\")).squeeze()\n",
    "    test_oh_diagnoses_df = oh_diagnoses_df.loc[test_visits,:]\n",
    "    train_oh_diagnoses_df = oh_diagnoses_df.loc[train_visits,:]\n",
    "    predictions = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_micro_F1/drugSexAge_AllCodes\"\n",
    "                                +crossval_suff+\"/best_models/1/\"\n",
    "                                +\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    predictions_subset = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_micro_F1/drugSexAge_Sagi\"\n",
    "                                +crossval_suff+\"/best_models/1/\"\n",
    "                                +\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    missing_columns = predictions.columns[~predictions.columns.isin(predictions_subset.columns)]\n",
    "\n",
    "\n",
    "    dummy_classifier = DummyClassifier(strategy='prior')\n",
    "    dummy_classifier.fit(X=np.empty_like(train_oh_diagnoses_df),y=train_oh_diagnoses_df.to_numpy())\n",
    "    dummy_pred = dummy_classifier.predict(X=np.empty_like(test_oh_diagnoses_df))\n",
    "    dummy_probas = dummy_ranker(train_oh_diagnoses_df,test_oh_diagnoses_df,pos_label=1)\n",
    "    dummy_predictions = pd.DataFrame(dummy_probas,columns=test_oh_diagnoses_df.columns,index=test_oh_diagnoses_df.index)\n",
    "\n",
    "    code_subset = code_subset[code_subset.isin(predictions_subset.columns)]\n",
    "    for pred,label in zip([predictions_subset, predictions, dummy_predictions],\n",
    "                        ['Cherry Picked Training', 'All Codes Training', \"Dummy\"]):\n",
    "        r_precision,recall = precision_recall_at_K(test_oh_diagnoses_df.loc[:,code_subset], pred.loc[:,code_subset], k=None, pos_label=1, rectified_precision=True)\n",
    "        mean_recall_K = np.nanmean(recall,axis=0)\n",
    "        q1_recall_K = np.nanpercentile(recall,25,axis=0)\n",
    "        q3_recall_K = np.nanpercentile(recall,75,axis=0)\n",
    "        df = pd.DataFrame({'mean_R@K'+crossval_suff:mean_recall_K,\n",
    "                                    'q1_R@K'+crossval_suff:q1_recall_K,\n",
    "                                    'q3_R@K'+crossval_suff:q3_recall_K})\n",
    "        if res.get(label) is None:\n",
    "            res[label] = df\n",
    "        else:\n",
    "            res[label] = pd.concat([res[label],df],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "76582584",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max Recall@K difference:0.70%\n",
      "Min Recall@K difference:0.17%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQAAAACjCAYAAABljcCSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqY0lEQVR4nO2deXzU5bX/3yf7npANQiCsgiCbEjCsCctVBNdbq1SKICpWf7hft2or9Lq09Va9LsWiraCCWkW5Lq1aUERkUUQWKSIgazAkZN9nMnN+f3wnMYTMZBInmZA879eL18x3vs/3+Z6Z8Jzvs5znc0RVMRgMnZMAfxtgMBj8h3EABkMnxjgAg6ETYxyAwdCJMQ7AYOjEGAdgMHRi/OYARKS7iGwVkSoRCWrk3MciskFEpvrLRoOhoyP+igMQkTAgHHgbmKqqNfXOPQW8BuwA3lPVLL8YaTB0cPzWA1DVKlUtdHN6GLBRVcuAUhGJbkPTDIZOQ3udAwjUH7smxUAXfxpjMHRUgpou4hcc9d7HAEUNC4jIfGA+QGRk5MgzzzyzbSwzGNoxX3311QlVTfK2fHt1ADtEZAzWHECMqpY0LKCqS4AlAOnp6bply5Y2NtFgaH+IyKHmlPfnKkCwiKwGhgMfisi5IvK06/QfgYeB1cAj/rLRYOjo+K0HoKp2oOES32bXuaPA5DY3ymDoZLTXSUCDwdAGGAdgMHRijAMwGDoxxgEYDJ0Y4wAMhk6McQAGQyfGpw5ARP63wfEiX9ZvMBh8i0/iAEQkCogF0kUkBRBX3WN9Ub/BYGgdfBUI9DNgLjAYWOH6rBp400f1GwyGVsAnDkBVlwHLRGSSqn7iizoNBkPr4+tJwEMi8oaIfCgiQSJyu4/rNxgMPsTXDmAJ8N9AsEvh50If128wGHyIrx2AquoOH9dpMBhaCV87gAIRuQIIFZFLgOM+rt9gMPgQXzuA64GzgUJgDPArH9dvMBh8iK/1AJyqep+IBAPnYcUDGAyGdoqvewDvuF4XApdiSXu7RUSeEJHPGokgzBSRzSKySURML8JgaCV8Pgnoeu2pqtcDYe4Kisg5QKSqTgBCRGRUvdN3Aj/HiiS8xsc2GgwGF752AEEi8hCwv/bYQ9kxWJp/uF4z6p3bhRVaHAqU+9hGg8HgwtcO4ArgK+AREQnHs6BnHFCr9ttQ+38V8B7wLbC8sYtFZL6IbBGRLXl5eT/RbIOhc+JrB1AKpAFPYmn2r/NQtghL8x9O1f5/DBgPnAFcLSIRDS9W1SWqmq6q6UlJXsugGwyGejTpAERkr4h8LyL7Xa+7ReQdERnaSPHXgAjgLay8f3/3UPVGYIrr/VRgU71zDqBIVW2AEwj24rsYDIZm4k0P4E3gl8AAYBZW1/z3wF8aKRuuqo+q6hpV/T3WGL5RVHUrUCUin2EtH35RLy/AH4DVIrIR+ERVi73/SgaDwVu8iQPIUNX7XO83isgjqnqXiNhqC4hId9fbHSJyJfAFkI6V2cctqnprg+ObXa8fAB94+R0MBkML8cYBHBORPwNfAqOAHBEJAk7UK7McawlQgHP4MQLQP7nHDYaOSo2NmsIj2PL2Y8vbS3VhNjWleZRU2ThW7bbD7RZvHMBs4BKsCbnVwCpVdQKX1xZQ1UnNvrPBYPgRhw0tPort+H7s+XuxFRzBXppHVXU1ObZQsm2h/FAdyg/2cHKc0VQHRmIPisYRmo4zNJqIsAiSE8KA55t12yYdgKo6RWQP1hNfsGbnG53dF5HfNnL975plkcHQ0bCVoyf2YTu+G3vefmyF2diqSimvEY5Vh3GkOoxsWziHHXGUSDS24Giqg9JxhsYQFBxKYmwo3WLCSYkNY0xsBD26RBAfGUp0aDBhwYGEBAYQEGBF3T/eTNOadAAi8irWGv1hLAeguF/eqw0AEmAkEN1MewyG0wtHDZRmww87sefswl5wiKqKUirtkFsTwsGqcI7YIjjoTCKPOKqC+2MPPgcJiSA4UEhKCCMlJpzUuHBGdIkkKTKUuIgQYsODCQ8OIjQoAJHW21LjzRAgRVWzvKlMVesH7bwiIqtaYpTB0K4oPwHHv8GZswt73l5sZQVU1CjFduFwVQR7HYnsdXbjB0mjImgoGhROUEAAXSJCSO0eTmpcBFO7RJISHUaXyFBiwqwnd1hQYN2T21944wA+EZGhqrqzqYIiUl8FuCfQq8WWGQxthaMG8vdC9lfUHP83NUXHqKpxUmGHbFsI++1d+E67870zmdLA/8AWGEFgoBAWFEj3lAh6xUeSGR9Jr/gIEqLCiA0LITwkkODA9p92wxsH8EtgrojYcQ0BVHWAm7LXu14VyAd+8dNNNBh8gDqh6DAc+RLbsR3UFB6h0u6gwBbAgaoIvtMUvtNUcmQUlYExSEAgQYFCUnQoPbtE0ichkqy4SJKiw4iPCCE6PJjw4MBW7Z63Bd5MAp7hbWWqWrdzT0TOAva10C6DoWVUFsKxbdiPbMGWu5eKykrKa5RDVeHsrkni35pGjgynMiiTgKAgIkOC6Nkjgl5dIrkgMYrU2HCSosOIDgsmIjjI71301satAxCRKaq6RkSuanhOVVe4ueZ9VZ0hIndhCYLkAzN9Zq3BUEtFPs5Dm7Af2kx1/kHKbU5yqoP4troLu5w9+V56UBY0lIDgMIIDA0jtGkG/pCguTIqhT2IU8REhRIUGExoc6O9v4lc89QB6uF697gHw4/7/Yar6HyLyacvMMhhc2MqoObSZ6u83UJm7j6IqJ/uqIvjOZjX0wpBhVAZPJDg4mC5RIfTtHcXQpBguT44mJSacmHBrwu1076q3Fm4dgCvZB6ranPx+4SJyHfCD69j86gbvqKmGnJ1U71tHVc4uysqr2F8Vzs6qLuxw9KAw+AyqQscQHhpKWmoE/ZKiuTI5mp6ucXlUaBBBp8GkW3vD0xDgX7gJ5VXV89xcdh1wPvCwSw9g6U810NABsVVg37eWqr0fU1V4jOPVwXxTGc22mp4cklQqQgcTGBpOalIEg7rFcFW3OHp2iSAxKozI0EACA0xD9xWehgDXNbcyVf038O96H/2t2RYZOhY1NqoPbKD829WU5x5kT2UkOyrj2KU9KQodR01oHDHhIQxMi+HclFh+6eq6x4aHmCd6G+DJAdjbzApDx0AVLT5M5c53KT2whcPl8GVZF7529KIwbDjOiKn07BrFWSmxXNM1hrSEKBKjQokI8bU4tcFbPP3y9Xf41UeBya1mkeH0oaaa6n2fUb77IwpOZPN1WSybK7txJLgn1ZFXkRgbzYhBXZiXlkDvhEi6RIQS1sln3dsbniYBvd7hJyITPdTjVhZMRJ7A0g3YWl8bQETCgGeBPsCuWp0Ag39xlORSsuP/KNu/ie9KhC/L4/lWUykNP5fg6GTOOjOW8/skMiQljviI0E6/xHY64GkScJaqLheRXzc8p6oNxT5nu157YGkCbgVGAEdxv3OwThZcRBaLyChV/dJ1+hZghaquada3MfgOVWz5hyjaupKSw9vZUhLJpoqu5ASnYo+5gp4pCZzdswsXpHahd0IUMWHBZqntNMTTEKBW8CO7qUpcOQAQkTeA4apa4xINed3DZY3Jgtc6gCwgVUR+Azyuqu+cernB19iLcyj54mUKDu7gi5IYNlV2IzcsDWfMHAYNSOS8PokM7hZLt5hw05XvIHgaAnzoel0mIolY3fEDqnrC3TVAfyAEqHG99vVQNo4ftw8XA2fVO9cPS1n418BaEfmHK924wYdUV9vJ+fdairevYmu+8GVFMnlhvSBuHkMHJ3Jx7wQGdo2lW0wYIUGmwXdEvNEDuBmYB3wDnCUiS1X1KTfFHwG+FpHjQDLwoIeqi3AvC14MfKqq1SKyD+hKg56IiMzHkh4nLS2tqa9hAFSVgrwc8ja/ynff7+HT4kRyAxJxJlzA4LN68fM+SQzoGkNydJhZguskeCsJNqpet34D0KgDUNU3RGQlkAiccEmHuWMjcAOWdPhUTg4a2gAME5GtQG/glMwfqroEWAKQnp5utAfd4KixcXzPRnK2vM3nObCjKp6yyJ50T7uGzEmpDEmJIzUuwkzYdVK8cQB7OLlbv9ddQRHpiyXpHQPMEJFbVfWJxsqq6lYRqZUF314rC+6a8f8DsMxVz/Ou/AAGL6mqKCNny1sc2bWeD/LjOUoizoRJnDNqANf2SWRAcgxdIkLMpJ3Bq1DgOOCAiHwLDAIOeKhvCXAH8KSrx3Ah0KgDAI+y4D9g7SY0eImtOJec9S9y6PvdvF+UwuGgNKK6/4Kpo/tybVo8PbpEmHG84RR8GgqMJRaywzxZ2oaaomwK1z7DnqM5/F9xDw6G9KdLygSmT+zJbb0TSI4O7/D72Q0/DU+rAIdaUF+BiFwBhIrIJcDxFltmaJSayjJyP1vC/r07eacojf3hI0jomsb08anc1S+JpKgw07U3eI2vg7CvB+4DCrHW+W/wcf2dE6eTgq0r+f7LD3iroCt7QwYSlzKfC8alcluvBFJiI8yT3tAimu0ARCRCVSvcnL65No2YWI+hv2ItIRpaQOXx7zj+0ZP844dgPqsZSGDXX3LhtN7c2DeZVNPoDT7A0yTg46p6R4PP4rCSg453c1mUiFyjqi9iJQ/9wleGdhYcNXZyNrzMjq83sqqkN3mx08g4ux/3n9WdAckxZiLP4FM89QACReQJVb0dQES6Ae8Dj7m7QFXvE5G/icjlwAeq+pxvze24lJ84yoF/Ps5bhwPYFdCfrn3nceW03ozsmUBsRIi/zTN0UDxNAt4qIk+KyJNYYbnvAPeo6j8blhWR5/lRPSgAGApki8gSVZ3vc6s7EHnb/snX61aysrAnRV3Gcf6k4cwblEJ308U3uMHpdFJTU0NFRQWlpaWUlJRQXl5OTU3zo+U9zgGo6m2uLbufA1eo6uduir7S4PjFZlvSiXDW1JC97m98/NXX/KNyAAl9r+YX085gdO8kI47RiVHVusZdVlZGaWkpZWVlVFZWYrfbUVVUlYAAK11YaGgoERERxMXFkZaWRnh4eLPv6WkOYC8/CoJUAUtdy0unJAZR1U9d1wwFvlFVdU0CDm22RR0Yh72KA//4E2/vymOznsngodezaHRfBiTHmqd9J8LpdGK32yktLaWwsJDi4mJsNhtOpxMRISAggLCwMCIjI0lISCAqKoqwsDCCgoJ8vsTraQjQHDnwWp5U1Smu61VEHseK8+/UOKsrOfTPx1ixq5itQUPIHPOfPDkijdS4CLNm3wGpbeAlJSUUFxdTWlpKdXU1DocD4KRGHhMTQ//+/YmKiiIwsO3ly73ZDRiKtZQ3CNjcIAFoQ4IbHIf+BNtOe7TGzuEPHuOl7fl8HTSUaRPHsuTsXiREdeqf5bSntpteXl5OcXExJSUlVFZW1o3Baxt4eHg4MTEx9O7dm8jISIKD259oikcHICJRwBvAo1i79a4Ukf8ClqnqKTv0gL0i8hDwCTAJDxuHOjSq5G1azstrv2A9w7ho8hXcOiKNmHAzm386oKo4HA4qKyspKiqitLSU8vJybDbbSWPwkJAQoqKi6NatGzExMYSEhBBwmkmWN9UDuBt4BrgImIs1H9AfOCQisar6QoPyv8LaQ/CfWPoBzUkq0iEo3bueN1at4P3KQUyacAPLxpxBdJhp+O2N2gm3qqoqioqKKCwspKKiArvdEsMWEYKCgoiKiiI2NpaePXsSERFBYGDHisNoygGMU9XfikgGsBJwYIX4vokVEHSSA1BVO7C4NQxt7zjKTrBhxUKeze7B0PS5PD9pGPFRYU1faGgTVJWKigoKCwspLCykpKQEp9NJaGgo0dHRJCQk0K9fP0JDQ0+7p/hPoSkHUDumH6+qvxGRAOBu1wRfXG0hEXlZVWfXWzmAplOJdwycDo5++Cee2FxAUdcpPHrLVPokRPvbKkM9ysrKOHDgAMXFxQQFBREfH8+QIUOIiYlpd2PytqYpB/CtiAwB/u5K9OkE3hSRgfyo54er8XcDZgH7VLXAm5u7kwV3nRNgG/B0I0ONdoEtewevr/gzb9vO5vrLZ3H+4O5mOc9PqCo1NTVUVVVRUlJCWVkZFRUVVFRY21aSk5Pp0aMHUVFRBAc3nKvuvDTlABZiiXzMBF7CeqoHYqn91u30E5E7sOYIdgHDReRBVX3DU8VNyIIDXAzkNu/rtBFOB/veWshDO4PpOfxqll84mnATwNMmOJ1OysrKOHHiBIWFhVRXV6NqdTpFhODgYCIiIoiKiiIxMZGoqChCQoz6kTuaigQ8JiL3YeX4O4a1rNcVuK+BXsAVwDkuFaAYYBXW6oEnPMmCA/wCz7LifsF+bDsvvrKEjxjFA9dezIi0eH+b1OGoXWYrLS2luLiY4uJiKisr6wJlQkNDiYuLo3///kRGRvpl/byj0ORjS1V3AleISBdA3HTvK2plu1W1xMs/RhxuZMFF5HzgUywdwnbzaM37+Cnu+bSQAaOv4dVp5xBslHNbTO1SW3l5eV00XEVFBU6npSNb29Cjo6NJTU0lLi6uXa6jn+54CgW+ys3nqOqKBh9niMh3tUWwknrsxfMkYBHuZcGvw1IjnunBvraTBbdXsPnFO3k4Zwi/nnMTGX2TWvd+HQxVxWazkZubS15eHlVVVTidTgICAggODiYmJoaUlBTTyP2Ap6er16HAqhrRgnt7kgU/A2sYkYo1H7heVb9tcM82kQV3/rCLJS/9mc8ipvLSf11IXISJ4vNEbWPPz88nPz+fsrIyHA4HAQEBxMbG0rt3b6Kjo1slrt3QfDztBWjVIB5PsuCqOgJAROYCQQ0bf1tRseNd7nt7E0nnzOXlGelmhr8BdrudwsJCTpw4QWlpaV0obEBAANHR0SQmJjJgwAAzCdeO8UYW/BRU1SeS3e5kwesdL/XFfVrCkQ8f584Ndmb/5w1cNNxkHrLb7RQUFJCbm0tpaSmqiojU7Vjr3bs3YWFhnSqIpiPga1nw0x+nkw0v3cMfD/fiDzdey8Busf62qM1RVUpLS8nOzqawsLCusUdHR5OcnMyZZ55puvAdBK9kwUWkB9ATa4IPoFHJcBH5CCs8+G1XWPDphbOG/3v2dl6vOoeX/usqYjrBeF9VqaysJC8vj/z8fCorK1FVQkND6datG/369SMkxOxl6Kh4sx34QawsPX2Ag0ApcL6b4tcB1wB3uyIHn/fX+L3Z2Ct5/Zk7+VfQBJbdfgXBHUx8U1XruvEFBQWUlJTgcDhQVYKCgoiLi6N3797ExMQQFNRuVl4NrYw3f+npWEE6tVt8G8p/1aGqh4FFIvJH4M/Adtck30Oquvanm9tKVBXz0jP3sTF6Kn+5/lICT/NxbG1jz83N5fjx41RXV+N0OgkMDCQyMpL4+Hh69epFaGioCaLp5HjjAMrqSXyFAAPdFRSR4Vi9gFFYuwd7YImErsKK/Gt/VBay5Kn72ZU8g2fnTD8tZ/odDgdFRUXk5uZSXFxMTU1N3bJb3759iYmJ6XDbWA2+wZtH3XsiEo6V5GMPVoSeO+7FGv9nqOpjqpqnqseB3/jAVt/jrOHlPz/AnpSLeXLu6dP4HQ4Hx48fZ+vWraxfv55NmzZx6NAhIiIiGDZsGOPGjWPs2LGcddZZdOnSpU0b/5o1a8jMzGTixIlcdtll5OfnM3fuXPbt29dq98zKyiIzM5PRo0fz9ttvk5OTw8MPP9xo2aVLl/LCC83bW5aVlVX3vri4mKysLLKysoiNjSUrK4trrrnG4/Xbtm3jr3/9q9vzN998s9tzrY03ocC12X1fcv1rFNdW4ShV/biROlY3confWfv87XwcMoG/zT6/XXeDa2fljx49SkGBFYkdFRVFWloa8fHx7Wbp7cSJE/zud7/jvffeIzo6mu+++w6brfmZ3WujBN0dN8aaNWuorq5mypQpbNq0ifvvv7/Z9/WG2NhY1q5dC8D48ePr3nuyc8SIEYwYMcJtnU8//bSPrfSeJv/niMhyV/cfsWhUE1BVnVhKQak+trFV2LNyIc8WDGXJjT9vl43f6XSSk5PDF198wfr169m9ezeRkZGce+65jB8/nhEjRpCYmNhuGj/A+++/z+zZs4mOtvQQBgwYQEpKCmD9Jx8/fjyLFlnxZfv27eO8884jMzOThx56CIC5c+eyYMECpk2bxtKlS7nyyiuZMWMGjz32GM8++yxgPU3dPTEjIyMJDw/n4MGD/PKXvwRg1apVZGRkMGnSJD799MfOa3Z2NtOnT+fYsWO89957TJw4kbFjx/LBBx8AsGTJEjIyMrj33nub/N717T527BiTJk1i/Pjx3HTTTQCsXbuWBx54AIDhw4dz9dVXM3z4cLZt2wZYjgSsnsZ9993HqFGj6noM77zzDiNHjmT+/Pl15XxKrda4u3/Ax56OG5w7iCUhvh9LD/C7pur3xb+RI0dqc8hf/1f92cLHtbCsqlnXtTY1NTV69OhR3bhxo3722We6bds2LS4uVqfT6W/TvOLRRx/V995775TP58yZo2+99Zaqqo4ePVpVVa+44go9fPiwqqrOnDlTjxw5onPmzNEVK1aoquqLL76o8+fPV1XViooKveCCC1RV9d5779XPP//8pPozMzPVbrdrbm6uTpgwQQ8cOKCzZs1Sh8Oh6enpWlFRoaqqDodDX3zxRV20aJFecMEFevToUXU4HHXXV1dX6+TJk9Vut+vo0aPVbrfr559/rpmZmY1+33HjxtV9v1q7q6ur1W63q6rqrFmz9LvvvtNPPvlE77//flVVTU5O1srKSl2/fr3edtttJ9WTmZmpW7du1aqqKp0wYYKqqo4dO1bLysr06NGj2r9//yb/BsAWbUbb8WYSUERkoKrucQmBNPrIcQ0B5qgrR0B7pebIVm79KJvf33QLcZH+X+evqanh6NGj/PDDDzgcDuLi4hgyZAiRkZHtsmfiiZSUFI4dO9bouSFDhgDUJa/Ys2cPs2fPBqCoqIjs7GwARo4cWXdN7fvw8HCSk5M5fPgwmzdv5pFHHjml/ilTphAUFMSjjz5a91leXh69evWqu2dtb+m5557j4YcfJjU1ldzcXHbv3s3UqZZ6fe2GpV69ehEUFHSSPZ6oLZefn8+NN95IUVERBw8ePOX36N+/P2FhYaSmplJUVNTo7xQcHFxna+3KTWRkJImJiV7Z0hy8cQC3Ai+KSCKWQMdNjRVSVaeI3InnSUL/4qzhoWUr+MWl8+nf1X8Rfna7ncOHD5OTk4OqkpCQwPDhw4mIaMmeqp/ItuVQdLjl18elwYhZAEyfPp3LL7+cmTNnEh0dzb59+4iMjAQ4xZkNHDiQJ598kpSUFBwOByLC4sWLTxrS1H9/1VVXceeddzJ69OhGHeOaNWvq4hcOHjwIQFJSEocPH6aqqoqwsLC6rcYPPPAAq1atYtCgQYwePZqhQ4fy4YcfEhgYiN1uR0Q4dOgQDoeDr7/+2qufodbWFStWcOmllzJ37lxmzZpVJ1ZSS33bG55r7HdyOp1UVFRQVFTEiRMnvLKlOXgzCbgDGOtlfdkicg+wAUtAFFXd0HLzfMvHy35LcdJIpp/d9jKFDoeDI0eO1D3pkpKSSE9PJzTUz70QV+P1BUlJSfzmN7/hwgsvRFWJj493O/v98MMPM2/ePKqrqwkODmblypUe654yZQpz5sypG0t7Q0BAAPfddx+ZmZlERkby4IMPAhASEsLy5cu5/PLLeeKJJ7jjjjuYMmUKIsLgwYN59tlnueaaaxg7diyZmZne/wDA5MmTufrqq1m1alWzrnPH3XffzcSJExkxYgRdu3b1SZ31kca80EkFRNKxkoNGAyOBh1X1HjdlG+YEVFWd5wM7PZKenq5btmzxWKbo32v41d+38soDdxIU1HYTZ4WFhXWz4YmJifTt29f/jf40xOFwMG3aNP71r3/525Q2paamhqCgILKzs5k/fz7vv/++x/Ii8pWqpntbvzdDgMexdP5fV0vya5S7gqrqeUHUX9gruf/1j3ngulvbpPGrKrm5uezdu5eQkBAGDx5cNzNuaD4FBQX87Gc/44Ybbmi6cAfjzTffZPHixZSXl/PUU0/5vH5vHIBdVXNFpEnRjeb0FtqSZc8tou9Z4xnSM7lV7+N0Ojly5AhHjhwhMjKSUaNGmae9D4iPj+eTTz7xtxl+YebMmcyc6VYY6yfjjQP4XkTuBmJF5FasaEB3eN1baCuObvuIT4oTWXrzha12D5vNxp49eygqKiIxMZGMjAyzocZwWuBNf/hGLNHOjUA1sMBDWbuq5uJGSKQhIvKEiHwmIv/b4PMHRWSj698Ub+pqFKeT369ax6Ibrm1xFZ4oLCxk8+bNfPnll8THxzN+/HgGDRrU6Rt///79ee211+qOs7KyqKmpYeHChaxefXJQaEVFBfPnzycrK4tx48axbNmyJutfu3YtCxcubJZNe/bsISsri4yMDLp27VoXdOOJDz74wO2Y21O48elEU8lBw4FAVf2L6/gCYA2Q5eYSr3sLTeQFeElVF7myD73jumezWfP6/xCX0p9eSV1acnmjOJ1ODh06xNGjRwkPDzfj+wZs376dCRMm8O6773rVdV20aBGZmZksWbIEVeWzzz5rFbsGDhzI2rVrOXjwIA888ACvvPLjplZ3IbzTpk1zW1+3bt1aLdy4LXHbA3A14K+xtvTeKiLvAz/Dcw+gYW/B0y6HxvICAKCqB1xvq/GyN9EQR8lxXtxt48FrZ7fk8lOoqqpi27ZtbNiwAZvNxpgxY0hPTzeNvwFvvfUWN910ExUVFVRXVzdZfsOGDcyaZS1FiggTJ04E4JZbbmHixIlceOGFFBcXAzBv3jymTp3Kyy+/XHf9Cy+8wIQJE5gwYQJbt26loKCArKwsJk2axC233OLx3llZWdx9991cffXVbNu2jczMTDIyMuoCjWo3Dh08eJDJkydz+eWXM3LkSI4ePXpSuPGYMWNYsGABI0aMqAslfu6558jIyOCee+45aTNRe8NTD2AmMBiIBLKBiaq6zVNlrnH/cqALlnpQd8BdlEkcbvIC1GMh8JfGLm5KFvypJU9yyeSLfrKwR1FREbt370ZV6du3L127dj3tIvTakq1bt7Jo0SKmTZvG6tWrmTFjRrPr+PLLLykvL2fdunW88sorPPfcc0yaNInAwEBWr17NI488gs1m48SJE7zzzjusW7eOwsJC5s2bx4IFC8jKymLhwoWNBto05LLLLmPMmDFUVlaydu1aRIRJkyZx++23n1SusLCQ1atX8+qrr7Jy5UouueSSunP5+fn89re/xW63s2DBAqZOncrSpUv5/PPP2bJlC5s3b272b9BWeHIAFWpt8CkVka+bavwAIvIYcCFwBFdyUCw1ocYown1eAETkMiBBT81BAHiWBT/+zcfsrIrn9ixv45dOpaKigh07dgAwbNiwuoi2jsbSTfs5mF/e4ut7J0QyN6MfAPv37+ebb75h2rRpVFdXM2DAgBY5gP3793POOecAkJ6ezqeffkqvXr04++yzASvsduPGjXz//fds376dSZMm1V2bmZnJunXruOqqq7jgggvqwo3dURvCe+DAAe68804qKirYs2cPubknZ6UbPHgwAQEBpKamnrK1OSkpieRka4WpNmIvLS2NwMBAj7sA2wOeHMDZLo0/AYbWe6/qXhV4vKoO8vLebvMCiMgw4P8Bzf/fo8ojK9fw6+tua/alYIXpfvPNN1RUVDB48GC6dPHd/EF7pLbx+oKVK1fywgsvMGWKNW978cUX43A4PF4zduxYli9fXhc2+/nnn9O3b18++ugjALZs2UK/fv3o06dP3VJgbXhunz59GDVqFG+++SZg/e0cDge/+93vAGsbblMOoHbsv3jx4rru+vjx45sVwtvwXGJiIkeOHMHpdNY9RNorHh1AC+rbJCJJqprXVEH1kBcAeAwrB+GHIlKsqpd4ru1Hdq19laCYFPqnNi97j81mq2v4/fr1q9vGavCe999//6StuoMHD2b9+vUer3nwwQe57bbbeP7557HZbNxwww3MmTOHZcuWMWHCBKKjo1mxYgVxcXEsXryYKVOm0KtXL9LS0khKSmLGjBlMnDiRwMBAJk+eTGZmJr/+9a+x2+11G3y8YcaMGSxYsIDBgwf/ZBHUoKAg5syZw9ixYxkzZky7zkbcZCiwV5W40oBhSYZ1BY66Tqm6Tw3mM+qHAt+86Dc8cMvddO3i3eSczWZj586dVFZWcsYZZ7RKvLWh81Ebwrt582b+9re/8Ze/NDqV5XNaIxS4SVTV6zRircmR7WtwhsZ61fhVlb1793L8+HEGDhxYN4YzGHzB008/zapVq7DZbF7FNvgLX/UAfgHkqOon9T6bBHRT1Vd/8g2aoLYHcPdDD3DtnBsZ2NOzKNGJEyfYvXs3ycnJDBgwwMzqGzoMfukBYGkENNw3+anrX6s7AICiQ99QpBFNNv7du3dTXFzMueeeaxJeGDo9vtoaZ3ctGdbhOq7xUf1N8vRrrzHvkovdnldVvv76a2pqasjIyDCNv5VYu3YtvXr1YsqUKWRlZfHqq23i/w0txFcOQEUkof4HIpIEON2U9ynqsHOwMpSMYUMaPe90Otm8eTPR0dEMHTq0LUzq1MyePZs1a9bwz3/+k+XLl7N161Z/m2Rwg68cwH9jLdldKyLnicj1wD+AVk0xXkveiTwuy5rQ6Dm73c7GjRtJS0ujf//+bWGOwUV4eDh33nkn7777bp2Sz9KlS1m6dGldeO0VV1zB2WefzVtvvcV5553H+PHjKS8vb/L8H/7wh7qNOqtWreJPf/qTP7/qaYtPHIBaab9+BiQDlwKJwOWqus4X9TdFpTOACxuJty4tLWXjxo0MGjSI7t27t4UphgZ07979pJ2B9SksLOS1117jrrvuYtmyZXz00UdMnz6dDz/8sMnzV111Fa+//joAb7zxBldeeWWbfaeOhM/2raqVTfjRJgu2Ar1Sup3yWU5ODnv37mX06NGEhYX5warTg4MHD1Je3vJQ4MjISHr37u32fHZ2NjNnzsRut5JFqyvVOPwYXtu9e/c61eDu3btTWFjY5PmePXtSUFBAfn4+RUVF9OjRo8XfoTPTITeu79u3j7y8PMaNG9euEme0Rzw13p9KVVUVTz75JIsWLeKZZ54BYOfOnQwbNgw4OYS2sVDbps5ffPHF/OpXv+Kiiy5qte/Q0elQDsDhcLB161ZCQ0PJyMgw6/t+4uWXX2bjxo04HA7mz5/P8OHDOXbsGNOnTychIaHpCrzk5z//OXfccQeLFy/2WZ2dDZ8EAvmb9PR0XbNmDdu3b+eMM84wcfydhMLCQq677romJcU7E80NBOoQ/ePq6mp27tzJueeeaxp/J+Hbb7/loosu4tZbb/W3Kac1HaIHMGzYMN2+fbvp8hs6PZ2yBxASEmIav8HQAjqEAzAYDC3Drw7Agyx4dxH5WEQ2iIj3qg4Gg6FZ+M0B1JcFB0IaJBG5F3gAS0/Q+2yQBoOhWfizB+BWFhwYBmxU1TIsUVKjvW0wtAL+dABxQInrfTGWlHgtgfrj8kTDcwaDwUf4MxKwCPey4PWlZE+RDIeT8wIAZSLiKWdhW5MInPC3ER4w9rWc9mwbwMDmFPanA3ArCw7sEJExwA4gRlVLGl5cPy9Ae0NEtjRnLbatMfa1nPZsG1j2Nae834YAqroVqJUFd9bKgrtO/xF4GGtu4BF/2WgwdHT8uhlIVW9tcHyz6/UoMNkvRhkMnQgTCNQ6tMuhST2MfS2nPdsGzbSvQ+wFMBgMLcP0AAyGToxxAD7AFbpcm+swyPXZXSKyXkSWi4jfksOJyLmukOrPROSJ9mSby5Yh9ex7USzajX0uG+8QkfWu9+3GNhHpLSLHRWStK3lvs+0zDsA3FABTgE1QJ4k+SVXHYy1lXuo/0zgETHaFXCeLyIR2ZBvAHlUd67IPIJ12ZJ+IhALDXe/b09+1ln+papaqntcS+4wD8AGqWqWqhfU+Gg2sdb1vGObcpqhqjqpWuQ5rsMKs17qO/WobgKra6x1WAwNoR/YB1wG1yf3azd+1HpNcvafbaYF9xgG0DnG4D3P2CyIyDCuKrYj2Z9vFIvINlqx8EO3EPlcXOlNVP3Z9FEc7sc3FD1gOcxJWMF06zbTPOIDWoQj3Yc5tjojEA88A19LObANQ1XdUdQiQjdVLaS/2zQZW1Dsuov3YhqpWq2q5qtYA7wH7aKZ9xgG0Dl/yY7LUqbjmBvyBa1LyFeAuVc1pT7ZB3Ri7lhIgkPZj30DgRhH5ADgL6wnbXmyjwS7ZcVgOoFn2GQfgA0QkWERWY00WfQj0Ada5Zo5HAKv8Zx0/B0YBfxCRtUA/2o9tANNE5FMR+RToiuWs2oV9qnqPqp6vqtOAXaq6qL3Y5mKCiHwlIhuAY6q6mWbaZwKBDIZOjOkBGAydGOMADIZOjHEABkMnxjgAg6ETYxyA4Sfjiklf7Xof7ZrVv9zfdhmaxjgAg88QkSjgXeBZVX3T3/YYmsY4AIOviMKKRvuzqv7d38YYvMM4AIOvOBOIBd7xtyEG7zEOwOArtgDLgddrNREM7R/jAAw+Q1X/B9gFvCgmXfNpgXEABp+iqr8GyoCn/G2LoWnMXgCDoRNjegAGQyfGOACDoRNjHIDB0IkxDsBg6MQYB2AwdGKMAzAYOjHGARgMnRjjAAyGTsz/BxQM9TKy5l40AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 251.059x155.163 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for label,col in zip(['Cherry Picked Training', 'All Codes Training', \"Dummy\"],\n",
    "                    ['#FF800E','#006BA4','#ABABAB']):\n",
    "    df = res[label]\n",
    "    for stat in ['mean_R@K','q1_R@K','q3_R@K']:\n",
    "        cols = df.columns.str.startswith(stat)\n",
    "        mean_stat = df.loc[:,cols].aggregate(['mean','sem'],axis=1)\n",
    "        if stat=='mean_R@K':\n",
    "            plt.plot(range(1,len(mean_stat['mean'])+1), mean_stat['mean'],label=label,color=col,linewidth=.5)\n",
    "            plt.fill_between(range(1,len(mean_stat['mean'])+1),mean_stat['mean']-mean_stat['sem'],mean_stat['mean']+mean_stat['sem'],\n",
    "            alpha=.3,color=col,linewidth=.5)\n",
    "        else:\n",
    "            pass\n",
    "            #plt.plot(range(1,len(mean_stat['mean'])+1), mean_stat['mean'],color=col,linestyle='--')\n",
    "            #plt.fill_between(range(1,len(mean_stat['mean'])+1),mean_stat['mean']-mean_stat['sem'],mean_stat['mean']+mean_stat['sem'],alpha=.3,color=col)\n",
    "    # keep computation for further analysis\n",
    "        if stat == 'mean_R@K':\n",
    "            if label=='Cherry Picked Training':\n",
    "                sagi_recall_k = mean_stat['mean'][0:max_K]\n",
    "            elif label==\"All Codes Training\":\n",
    "                all_recall_k = mean_stat['mean'][0:max_K]\n",
    "plt.xlim(1,max_K)\n",
    "plt.legend(frameon=False,loc='lower right')\n",
    "#plt.title('Recall@K for billing codes on Sagi code set')\n",
    "plt.ylabel('Recall@K billing \\n Cherry Picked subset')\n",
    "plt.ylim(0,1)\n",
    "plt.xlabel('K')\n",
    "plt.savefig(plots_dir/\"recallatK_uF1_Sagicodes.pdf\")\n",
    "print(\"Max Recall@K difference:\"+str((sagi_recall_k-all_recall_k).max()*100)[0:4]+\"%\")\n",
    "print(\"Min Recall@K difference:\"+str((sagi_recall_k-all_recall_k).min()*100)[0:4]+\"%\")\n",
    "del sagi_recall_k, all_recall_k"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84955d84",
   "metadata": {},
   "source": [
    "## Fig. 6: Recall@K summary at billing level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "17132e4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compare actual recall for models learned on different code sets\n",
    "code_subset = icd9_billing\n",
    "max_K = 50\n",
    "res = {}\n",
    "oh_diagnoses_df = pd.read_parquet(datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "for crossval_suff in ['','_1','_2']:\n",
    "    test_visits = pd.read_csv(datadir/(\"test_visits_id\"+crossval_suff+\".csv.gz\")).squeeze()\n",
    "    train_visits = pd.read_csv(datadir/(\"train_visits_id\"+crossval_suff+\".csv.gz\")).squeeze()\n",
    "    test_oh_diagnoses_df = oh_diagnoses_df.loc[test_visits,:]\n",
    "    train_oh_diagnoses_df = oh_diagnoses_df.loc[train_visits,:]\n",
    "    predictions_F1 = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_micro_F1/drugSexAge_AllCodes\"\n",
    "                                +crossval_suff+\"/best_models/1/\"\n",
    "                                +\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    predictions_NDCG = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_NDCG/drugSexAge_AllCodes\"\n",
    "                            +crossval_suff+\"/best_models/1/\"\n",
    "                            +\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    predictions_subset = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_micro_F1/drugSexAge_Sagi\"\n",
    "                                +crossval_suff+\"/best_models/1/\"\n",
    "                                +\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    missing_columns = predictions_F1.columns[~predictions_F1.columns.isin(predictions_subset.columns)]\n",
    "\n",
    "    dummy_probas = dummy_ranker(train_oh_diagnoses_df,test_oh_diagnoses_df,pos_label=1)\n",
    "    dummy_predictions = pd.DataFrame(dummy_probas,columns=test_oh_diagnoses_df.columns,index=test_oh_diagnoses_df.index)\n",
    "\n",
    "    # Add back the missing columns and assign them 0 probability\n",
    "    predictions_subset.loc[:,missing_columns] = 0.0\n",
    "    predictions_subset = predictions_subset[predictions_F1.columns] # reorder columns\n",
    "    for pred,label in zip([predictions_subset, predictions_F1, predictions_NDCG, dummy_predictions],\n",
    "                        [r'Cherry Picked Training $\\mu$F1', r'All Codes Training $\\mu$F1', 'All Codes Training NDCG', \"Dummy\"]):\n",
    "        r_precision,recall = precision_recall_at_K(test_oh_diagnoses_df.loc[:,code_subset], pred.loc[:,code_subset], k=None, pos_label=1, rectified_precision=True)\n",
    "        mean_recall_K = np.nanmean(recall,axis=0)\n",
    "        q1_recall_K = np.nanpercentile(recall,25,axis=0)\n",
    "        q3_recall_K = np.nanpercentile(recall,75,axis=0)\n",
    "        df = pd.DataFrame({'mean_R@K'+crossval_suff:mean_recall_K,\n",
    "                                    'q1_R@K'+crossval_suff:q1_recall_K,\n",
    "                                    'q3_R@K'+crossval_suff:q3_recall_K})\n",
    "        if res.get(label) is None:\n",
    "            res[label] = df\n",
    "        else:\n",
    "            res[label] = pd.concat([res[label],df],axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "293bed46",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best model recall@K:\n",
      "[[4, 9, 15, 19, 24, 29, 49], 4     0.261025\n",
      "9     0.377330\n",
      "15    0.460006\n",
      "19    0.500021\n",
      "24    0.538463\n",
      "29    0.570452\n",
      "49    0.657491\n",
      "Name: mean, dtype: float64]\n",
      "Best model recall@K:\n",
      "[[4, 9, 15, 19, 24, 29, 49], 4     0.249294\n",
      "9     0.350027\n",
      "15    0.421880\n",
      "19    0.455884\n",
      "24    0.490826\n",
      "29    0.518238\n",
      "49    0.589346\n",
      "Name: mean, dtype: float64]\n"
     ]
    }
   ],
   "source": [
    "# Plot main text figure\n",
    "plt.figure()\n",
    "for label,col in zip(['All Codes Training NDCG', r'All Codes Training $\\mu$F1', r'Cherry Picked Training $\\mu$F1', \"Dummy\"],\n",
    "                    ['#b600bc', '#006BA4', '#FF800E','#ABABAB']):\n",
    "    df = res[label]\n",
    "    for stat in ['mean_R@K']:\n",
    "        cols = df.columns.str.startswith(stat)\n",
    "        mean_stat = df.loc[:,cols].aggregate(['mean','sem'],axis=1)\n",
    "        if stat=='mean_R@K':\n",
    "            plt.plot(range(1,len(mean_stat['mean'])+1), mean_stat['mean'],\n",
    "                     label=label,color=col,linewidth=.5)\n",
    "            plt.fill_between(range(1,len(mean_stat['mean'])+1),\n",
    "                             mean_stat['mean']-mean_stat['sem'],\n",
    "                             mean_stat['mean']+mean_stat['sem'],\n",
    "                             alpha=.3,color=col)\n",
    "\n",
    "    # keep computation for further analysis\n",
    "        if stat == 'mean_R@K':\n",
    "            if label=='All Codes Training NDCG':\n",
    "                best_recall_k = mean_stat['mean'][0:max_K]\n",
    "            elif label==r'Cherry Picked Training $\\mu$F1':\n",
    "                cherry_f1_recall_k = mean_stat['mean'][0:max_K]\n",
    "                \n",
    "plt.gcf().set_size_inches(single_col_fig_size)\n",
    "plt.xlim(1,max_K)\n",
    "plt.legend(frameon=False)\n",
    "#plt.title('Recall@K for billing codes on the complete code set')\n",
    "plt.ylabel('Average Recall@K billing')\n",
    "plt.ylim(0,1)\n",
    "plt.xlabel('K')\n",
    "if mpl.get_backend()=='pdf':\n",
    "    # The inline backend distorts the scaling and outputs figures \n",
    "    # with wrong dimensions\n",
    "    plt.savefig(plots_dir/\"recallatK_bestmodel.pdf\")\n",
    "ks = [4,9,14,19,24,29,49]\n",
    "print(\"Best model recall@K:\")\n",
    "print([ks,best_recall_k[ks]])\n",
    "print(\"Best model recall@K:\")\n",
    "print([ks,cherry_f1_recall_k[ks]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2f2cd4a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "for label,col in zip(['All Codes Training NDCG', r'All Codes Training $\\mu$F1', r'Cherry Picked Training $\\mu$F1', \"Dummy\"],\n",
    "                    ['#b600bc', '#006BA4', '#FF800E','#ABABAB']):\n",
    "    df = res[label]\n",
    "    for stat,linestyle in zip(['mean_R@K','q1_R@K','q3_R@K'],\n",
    "                              ['-',':','-.']):\n",
    "        cols = df.columns.str.startswith(stat)\n",
    "        mean_stat = df.loc[:,cols].aggregate(['mean','sem'],axis=1)\n",
    "        \n",
    "        plt.plot(range(1,len(mean_stat['mean'])+1), mean_stat['mean'],\n",
    "                    label=(label if stat=='mean_R@K' else None),\n",
    "                    color=col,linewidth=.5,\n",
    "                    linestyle=linestyle)\n",
    "        plt.fill_between(range(1,len(mean_stat['mean'])+1),\n",
    "                            mean_stat['mean']-mean_stat['sem'],\n",
    "                            mean_stat['mean']+mean_stat['sem'],\n",
    "                            alpha=.3,color=col)\n",
    "\n",
    "        # keep computation for further analysis\n",
    "        if stat == 'mean_R@K':\n",
    "            if label=='Cherry Picked Training':\n",
    "                sagi_recall_k = mean_stat['mean'][0:max_K]\n",
    "            elif label=='All Codes Training':\n",
    "                all_recall_k = mean_stat['mean'][0:max_K]\n",
    "plt.gcf().set_size_inches(dual_col_fig_size)\n",
    "plt.xlim(1,max_K)\n",
    "# Add a first legend for colors\n",
    "color_legend = plt.legend(frameon=False)\n",
    "plt.gca().add_artist(color_legend)\n",
    "\n",
    "# Add a second legend for linestyle\n",
    "lines = [plt.plot([],[],linestyle=linestyle,\n",
    "                  label=label,color='k',\n",
    "                  linewidth=.5)[0] for linestyle,label in zip(['-',':','-.'],\n",
    "                                                              ['Average','Q1','Q3'])]\n",
    "plt.legend(handles=lines,frameon=False)\n",
    "\n",
    "#plt.title('Recall@K for billing codes on the complete code set')\n",
    "plt.ylabel('Recall@K billing')\n",
    "plt.ylim(0,1)\n",
    "plt.xlabel('K')\n",
    "if mpl.get_backend()=='pdf':\n",
    "    # The inline backend distorts the scaling and outputs figures \n",
    "    # with wrong dimensions\n",
    "    plt.savefig(plots_dir/\"recallatK_bestmodel_Q1Q3.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "faccbc2c",
   "metadata": {},
   "source": [
    "## Fig. S7: Recall@K summary at 3 digits level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "330a071a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compare actual recall for models learned on different code sets\n",
    "code_subset = icd9_3dig\n",
    "max_K = 50\n",
    "res = {}\n",
    "oh_diagnoses_df = pd.read_parquet(datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "for crossval_suff in ['','_1','_2']:\n",
    "    test_visits = pd.read_csv(datadir/(\"test_visits_id\"+crossval_suff+\".csv.gz\")).squeeze()\n",
    "    train_visits = pd.read_csv(datadir/(\"train_visits_id\"+crossval_suff+\".csv.gz\")).squeeze()\n",
    "    test_oh_diagnoses_df = oh_diagnoses_df.loc[test_visits,:]\n",
    "    train_oh_diagnoses_df = oh_diagnoses_df.loc[train_visits,:]\n",
    "    predictions_F1 = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_micro_F1/drugSexAge_AllCodes\"\n",
    "                                +crossval_suff+\"/best_models/1/\"\n",
    "                                +\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    predictions_NDCG = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_NDCG/drugSexAge_AllCodes\"\n",
    "                            +crossval_suff+\"/best_models/1/\"\n",
    "                            +\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    predictions_subset = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_micro_F1/drugSexAge_Sagi\"\n",
    "                                +crossval_suff+\"/best_models/1/\"\n",
    "                                +\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    missing_columns = predictions_F1.columns[~predictions_F1.columns.isin(predictions_subset.columns)]\n",
    "\n",
    "    dummy_probas = dummy_ranker(train_oh_diagnoses_df,test_oh_diagnoses_df,pos_label=1)\n",
    "    dummy_predictions = pd.DataFrame(dummy_probas,columns=test_oh_diagnoses_df.columns,index=test_oh_diagnoses_df.index)\n",
    "\n",
    "    # Add back the missing columns and assign them 0 probability\n",
    "    predictions_subset.loc[:,missing_columns] = 0.0\n",
    "    predictions_subset = predictions_subset[predictions_F1.columns] # reorder columns\n",
    "    for pred,label in zip([predictions_subset, predictions_F1, predictions_NDCG, dummy_predictions],\n",
    "                        [r'Cherry Picked Training $\\mu$F1', r'All Codes Training $\\mu$F1', 'All Codes Training NDCG', \"Dummy\"]):\n",
    "        r_precision,recall = precision_recall_at_K(test_oh_diagnoses_df.loc[:,code_subset], pred.loc[:,code_subset], k=None, pos_label=1, rectified_precision=True)\n",
    "        mean_recall_K = np.nanmean(recall,axis=0)\n",
    "        q1_recall_K = np.nanpercentile(recall,25,axis=0)\n",
    "        q3_recall_K = np.nanpercentile(recall,75,axis=0)\n",
    "        df = pd.DataFrame({'mean_R@K'+crossval_suff:mean_recall_K,\n",
    "                                    'q1_R@K'+crossval_suff:q1_recall_K,\n",
    "                                    'q3_R@K'+crossval_suff:q3_recall_K})\n",
    "        if res.get(label) is None:\n",
    "            res[label] = df\n",
    "        else:\n",
    "            res[label] = pd.concat([res[label],df],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c738075c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best model recall@K:\n",
      "4     0.342303\n",
      "9     0.481591\n",
      "15    0.579209\n",
      "19    0.624796\n",
      "24    0.670722\n",
      "29    0.706593\n",
      "49    0.799066\n",
      "Name: mean, dtype: float64\n",
      "Best model recall@K:\n",
      "4     0.270630\n",
      "9     0.383631\n",
      "15    0.461141\n",
      "19    0.498365\n",
      "24    0.532485\n",
      "29    0.558910\n",
      "49    0.624691\n",
      "Name: mean, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# Plot main text figure\n",
    "plt.figure()\n",
    "for label,col in zip(['All Codes Training NDCG', r'All Codes Training $\\mu$F1', r'Cherry Picked Training $\\mu$F1', \"Dummy\"],\n",
    "                    ['#b600bc', '#006BA4', '#FF800E','#ABABAB']):\n",
    "    df = res[label]\n",
    "    for stat in ['mean_R@K']:\n",
    "        cols = df.columns.str.startswith(stat)\n",
    "        mean_stat = df.loc[:,cols].aggregate(['mean','sem'],axis=1)\n",
    "        if stat=='mean_R@K':\n",
    "            plt.plot(range(1,len(mean_stat['mean'])+1), mean_stat['mean'],\n",
    "                     label=label,color=col,linewidth=.5)\n",
    "            plt.fill_between(range(1,len(mean_stat['mean'])+1),\n",
    "                             mean_stat['mean']-mean_stat['sem'],\n",
    "                             mean_stat['mean']+mean_stat['sem'],\n",
    "                             alpha=.3,color=col)\n",
    "\n",
    "    # keep computation for further analysis\n",
    "        if stat == 'mean_R@K':\n",
    "            if label=='All Codes Training NDCG':\n",
    "                best_recall_k = mean_stat['mean'][0:max_K]\n",
    "            elif label==r'Cherry Picked Training $\\mu$F1':\n",
    "                cherry_f1_recall_k = mean_stat['mean'][0:max_K]\n",
    "                \n",
    "plt.gcf().set_size_inches(single_col_fig_size)\n",
    "plt.xlim(1,max_K)\n",
    "plt.legend(frameon=False)\n",
    "#plt.title('Recall@K for billing codes on the complete code set')\n",
    "plt.ylabel('Average Recall@K 3-digits')\n",
    "plt.ylim(0,1)\n",
    "plt.xlabel('K')\n",
    "if mpl.get_backend()=='pdf':\n",
    "    # The inline backend distorts the scaling and outputs figures \n",
    "    # with wrong dimensions\n",
    "    plt.savefig(plots_dir/\"recallatK_bestmodel_3dig.pdf\")\n",
    "ks = [4,9,14,19,24,29,49]\n",
    "print(\"Best model recall@K:\")\n",
    "print(best_recall_k[ks])\n",
    "print(\"Best model recall@K:\")\n",
    "print(cherry_f1_recall_k[ks])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bad527f7",
   "metadata": {},
   "source": [
    "# Performance metrics comparison with other papers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee0c70e2",
   "metadata": {},
   "source": [
    "Reconstruct missing dev and training sets model predictions to compute the adaptive threshold performances. Requires a bit of fiddling to the missing keras serialization functions in the current version of `keras_utils`. As the missing objects are losses and metrics their missingness does not influence predictions.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f80f9a43",
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy.testing import assert_almost_equal\n",
    "# Ignore missing metrics serializers warning\n",
    "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\n",
    "# Generate predictions for the development set using the learned models\n",
    "oh_ingredients_df = (pd.read_parquet(datadir / \"oh_ingredients.parquet\")\n",
    "                     .set_index('visit_occurrence_id'))\n",
    "oh_diagnoses_df_sample = pd.read_csv(\n",
    "    datadir/\"oh_diagnoses_rollup.csv.gz\",nrows=1).set_index('visit_occurrence_id')\n",
    "\n",
    "age_sex_df = pd.read_csv(datadir / \"visits_agesex.csv.gz\",\n",
    "                         usecols=['visit_occurrence_id', 'age', 'sex_F'],\n",
    "                         dtype={'age': 'float64',\n",
    "                                'sex': 'uint8',\n",
    "                                'visit_occurrence_id': 'uint64'},\n",
    "                         index_col='visit_occurrence_id')\n",
    "\n",
    "for crossval_suffix,crossval_id in zip(['', '_1', '_2'],[0,1,2]):\n",
    "#for crossval_suffix, crossval_id in zip(['_1', '_2'], [1, 2]):\n",
    "    test_visits_ids = pd.read_csv(\n",
    "        datadir/('test_visits_id'+crossval_suffix+'.csv.gz')).squeeze()\n",
    "    dev_visits_ids = pd.read_csv(\n",
    "        datadir/('dev_visits_id'+crossval_suffix+'.csv.gz')).squeeze()\n",
    "    train_visits_ids = pd.read_csv(\n",
    "        datadir/('train_visits_id'+crossval_suffix+'.csv.gz')).squeeze()\n",
    "    dev_oh_ingredients_df = oh_ingredients_df.loc[dev_visits_ids, :]\n",
    "    dev_oh_ingredients_df = pd.merge(age_sex_df, dev_oh_ingredients_df,\n",
    "                                     left_index=True,\n",
    "                                     right_index=True,\n",
    "                                     how='right')\n",
    "    test_oh_ingredients_df = oh_ingredients_df.loc[test_visits_ids, :]\n",
    "    test_oh_ingredients_df = pd.merge(age_sex_df, test_oh_ingredients_df,\n",
    "                                     left_index=True,\n",
    "                                     right_index=True,\n",
    "                                     how='right')\n",
    "    train_oh_ingredients_df = oh_ingredients_df.loc[train_visits_ids, :]\n",
    "    train_oh_ingredients_df = pd.merge(age_sex_df, train_oh_ingredients_df,\n",
    "                                     left_index=True,\n",
    "                                     right_index=True,\n",
    "                                     how='right')\n",
    "    test_predictions = pd.read_csv(\n",
    "        (models_dir /\n",
    "         ('overall_micro_aucPR/drugSexAge_AllCodes'+crossval_suffix) /\n",
    "         'best_models/1/predictions_test.csv.gz')).set_index('visit_occurrence_id')\n",
    "    model = tf.keras.models.load_model(\n",
    "        (models_dir /\n",
    "         ('overall_micro_aucPR/drugSexAge_AllCodes'+crossval_suffix) /\n",
    "         'best_models/1/model'),\n",
    "        compile=False  # Needed since metrics serializers are missing\n",
    "    )\n",
    "    model.compile()\n",
    "    model.summary()\n",
    "    # Make a sanity check\n",
    "    y_pred_test = model.predict(\n",
    "        x=test_oh_ingredients_df.loc[test_visits_ids, :]#.to_numpy(),)\n",
    "    )\n",
    "    #assert_almost_equal(y_pred_test,test_predictions.loc[test_visits_ids, :].to_numpy(),decimal=2)\n",
    "    \n",
    "    y_pred_dev = model.predict(\n",
    "        x=dev_oh_ingredients_df.loc[dev_visits_ids, :].to_numpy(),)\n",
    "    y_pred_dev = pd.DataFrame(y_pred_dev, columns=oh_diagnoses_df_sample.columns,\n",
    "                        index=dev_visits_ids)\n",
    "    y_pred_dev.to_csv((models_dir /\n",
    "                    ('overall_micro_aucPR/drugSexAge_AllCodes'+crossval_suffix) /\n",
    "                    'best_models/1/predictions_dev.csv.gz'),\n",
    "                          header=True,\n",
    "                          index=True,\n",
    "                          index_label=\"visit_occurrence_id\")\n",
    "\n",
    "    y_pred_train = model.predict(\n",
    "        x=train_oh_ingredients_df.loc[train_visits_ids, :].to_numpy(),)\n",
    "    y_pred_train = pd.DataFrame(y_pred_train, columns=oh_diagnoses_df_sample.columns,\n",
    "                        index=train_visits_ids)\n",
    "    y_pred_train.to_csv((models_dir /\n",
    "                    ('overall_micro_aucPR/drugSexAge_AllCodes'+crossval_suffix) /\n",
    "                    'best_models/1/predictions_train.csv.gz'),\n",
    "                          header=True,\n",
    "                          index=True,\n",
    "                          index_label=\"visit_occurrence_id\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6547f34",
   "metadata": {},
   "source": [
    "### Table S4 hard classification results vs Hansen et al"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3e017ae7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build a table of F1,R,P scores to compare with and Hansen et al 2022\n",
    "from project_miscellaneous import filter_Sagi_diagnoses\n",
    "import sklearn.metrics as skmet\n",
    "\n",
    "\n",
    "oh_diagnoses_df = pd.read_parquet(\n",
    "    datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "base_dir = \"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_micro_F1/\"\n",
    "results_df = pd.DataFrame(\n",
    "    columns=('Experiment', 'Code level', 'Metric', \"Value\", \"crossval_id\"))\n",
    "\n",
    "for exp_dir, exp_name in zip(['drugSexAge_Sagi', 'drugSexAge_AllCodes'],\n",
    "                             ['Filtered codes learning', 'All codes learning']):\n",
    "    for crossval_suffix,crossval_id in zip(['', '_1', '_2'],[0,1,2]):\n",
    "        sagi_codes = filter_Sagi_diagnoses(oh_diagnoses_df=oh_diagnoses_df)\n",
    "        oh_diagnoses_sagi_df = oh_diagnoses_df.loc[:, ~oh_diagnoses_df.columns.isin(sagi_codes)]\n",
    "        visit_ids = pd.read_csv(\n",
    "            datadir/('test_visits_id'+crossval_suffix+'.csv.gz')).squeeze()\n",
    "        test_oh_diagnoses_df = oh_diagnoses_sagi_df.loc[visit_ids,:].sort_index(\n",
    "        )\n",
    "        test_predictions = pd.read_csv(\n",
    "            base_dir+exp_dir+crossval_suffix+\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "        test_predictions = test_predictions.loc[:, ~test_predictions.columns.isin(sagi_codes)].sort_index() # make sure index is sorted\n",
    "\n",
    "        for code_set, code_set_name in zip([icd9_chapters, icd9_subchapters, icd9_3dig, icd9_billing],\n",
    "                                           ['Chapter', 'Subchapter',\n",
    "                                               '3 digits', 'Billing (raw)']\n",
    "                                           ):\n",
    "            y_true = test_oh_diagnoses_df.loc[:,test_oh_diagnoses_df.columns.isin(code_set)].to_numpy()\n",
    "            y_pred = test_predictions.loc[:,test_predictions.columns.isin(code_set)].to_numpy() > .5\n",
    "                                           \n",
    "            f1 = (skmet.f1_score(y_true=y_true,\n",
    "                                 y_pred=y_pred,\n",
    "                                 average='micro',\n",
    "                                 pos_label=1))\n",
    "            prec = (skmet.precision_score(y_true=y_true,\n",
    "                                          y_pred=y_pred,\n",
    "                                          average='micro',\n",
    "                                          pos_label=1))\n",
    "            rec = (skmet.recall_score(y_true=y_true,\n",
    "                                      y_pred=y_pred,\n",
    "                                      average='micro',\n",
    "                                      pos_label=1))\n",
    "            results_df = pd.concat([results_df, pd.DataFrame({'Experiment': exp_name,\n",
    "                                                             'Code level': code_set_name,\n",
    "                                                             'Metric': ['µF1', 'Precision', 'Recall'],\n",
    "                                                             'Value': [f1, prec, rec],\n",
    "                                                             'crossval_id': crossval_id\n",
    "                                                             })])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ab5385e",
   "metadata": {},
   "source": [
    "Compute performance using the adaptive threshold based on dev set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d3d7000",
   "metadata": {},
   "outputs": [],
   "source": [
    "from project_miscellaneous import filter_Sagi_diagnoses\n",
    "import sklearn.metrics as skmet\n",
    "\n",
    "oh_diagnoses_df = pd.read_parquet(\n",
    "    datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "icd9_concepts = pd.read_csv(datadir / \"icd9_concepts.csv.gz\")\n",
    "icd9_allcodes = icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].concept_code\n",
    "icd9_chapters = icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id==\"Chapter\"').concept_code\n",
    "icd9_subchapters = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id==\"SubChapter\"')\n",
    "                    .concept_code)\n",
    "icd9_3dig = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.startswith(\"3-dig\",'\n",
    "                                'na=False)').concept_code)\n",
    "icd9_4dig = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.startswith(\"4-dig\",'\n",
    "                                'na=False)').concept_code)\n",
    "icd9_5dig = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.startswith(\"5-dig\",'\n",
    "                                'na=False)').concept_code)\n",
    "icd9_billing = (\n",
    "    icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.endswith(\"-dig billing code\",'\n",
    "                        'na=False)').concept_code)\n",
    "\n",
    "adaptive_thresh_results_df = pd.DataFrame(\n",
    "        columns=('Experiment','crossval_id', 'Code level', 'Metric', \"Value\"))\n",
    "\n",
    "for crossval_suffix,crossval_id in zip(['', '_1', '_2'],[0,1,2]):\n",
    "# for crossval_suffix, crossval_id in zip(['_1', '_2'], [1, 2]):\n",
    "    base_dir = \"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_micro_aucPR/drugSexAge_AllCodes\"+crossval_suffix\n",
    "\n",
    "    test_visits_ids = pd.read_csv(\n",
    "        datadir/('test_visits_id'+crossval_suffix+'.csv.gz')).squeeze()\n",
    "    sagi_codes = filter_Sagi_diagnoses(oh_diagnoses_df=oh_diagnoses_df)\n",
    "    #sagi_codes = pd.Series(dtype='str')\n",
    "    test_oh_diagnoses_df = oh_diagnoses_df.loc[test_visits_ids,~oh_diagnoses_df.columns.isin(sagi_codes)].sort_index()\n",
    "    test_predictions = pd.read_csv(\n",
    "            base_dir+\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    test_predictions = test_predictions.loc[:, ~test_predictions.columns.isin(sagi_codes)].sort_index() # make sure index is sorted\n",
    "\n",
    "    dev_visits_ids = pd.read_csv(\n",
    "        datadir/('dev_visits_id'+crossval_suffix+'.csv.gz')).squeeze()\n",
    "    dev_oh_diagnoses_df = oh_diagnoses_df.loc[dev_visits_ids,~oh_diagnoses_df.columns.isin(sagi_codes)].sort_index()\n",
    "    dev_predictions = pd.read_csv(\n",
    "            base_dir+\"/best_models/1/predictions_dev.csv.gz\").set_index('visit_occurrence_id')\n",
    "    dev_predictions = dev_predictions.loc[:, ~dev_predictions.columns.isin(sagi_codes)].sort_index() # make sure index is sorted\n",
    "\n",
    "    for code_set, code_set_name in zip([icd9_chapters, icd9_subchapters, icd9_3dig, icd9_billing],\n",
    "                                            ['Chapter', 'Subchapter',\n",
    "                                                '3 digits', 'Billing (raw)']\n",
    "                                            ):\n",
    "        y_true_dev = dev_oh_diagnoses_df.loc[:,dev_oh_diagnoses_df.columns.isin(code_set)].to_numpy()\n",
    "        y_pred_dev = dev_predictions.loc[:,dev_predictions.columns.isin(code_set)].to_numpy()\n",
    "\n",
    "        precision, recall, thresholds = skmet.precision_recall_curve(y_true_dev.ravel(), y_pred_dev.ravel(),\n",
    "                                                                     pos_label=1, )\n",
    "        f1_scores = 2*recall*precision/(recall+precision)\n",
    "        print()\n",
    "        best_threshold = thresholds[np.argmax(f1_scores)]\n",
    "\n",
    "        y_true_test = test_oh_diagnoses_df.loc[:,test_oh_diagnoses_df.columns.isin(code_set)].to_numpy()\n",
    "        y_pred_test = test_predictions.loc[:,test_predictions.columns.isin(code_set)].to_numpy() > best_threshold\n",
    "\n",
    "        precision = skmet.precision_score(y_true_test.ravel(), y_pred_test.ravel(),pos_label=1)\n",
    "        recall = skmet.recall_score(y_true_test.ravel(), y_pred_test.ravel(),pos_label=1)\n",
    "        f1_score = skmet.f1_score(y_true_test.ravel(), y_pred_test.ravel(),pos_label=1)\n",
    "\n",
    "        adaptive_thresh_results_df = pd.concat([adaptive_thresh_results_df, pd.DataFrame({'Experiment': \"Adaptive Threshold\",\n",
    "                                                          'crossval_id': crossval_id,\n",
    "                                                          'Code level': code_set_name,\n",
    "                                                          'Metric': ['µF1', 'Precision', 'Recall'],\n",
    "                                                          'Value': [f1_score, precision, recall],\n",
    "                                                          'Threshold': best_threshold\n",
    "                                                          })])\n",
    "#print(adaptive_thresh_results_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "79379f21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute mean and standard error over our experiments\n",
    "tmp = pd.concat([results_df,adaptive_thresh_results_df.drop(columns='Threshold')],axis=0)\n",
    "tmp = tmp.drop(columns='crossval_id').groupby(by=['Experiment','Code level','Metric']).aggregate([np.mean,np.std])*100\n",
    "tmp = tmp.droplevel(level=0,axis=1)\n",
    "tmp['stderr'] = tmp['std']/np.sqrt(3)\n",
    "tmp.drop(columns='std',inplace=True)\n",
    "tmp['Value'] = tmp.apply(lambda x: str(x['mean'])[0:5] + ' $\\pm$ ' + str(x['stderr'])[0:4],axis=1)\n",
    "tmp.reset_index(inplace=True)\n",
    "tmp.drop(columns=['mean','stderr'],inplace=True)\n",
    "\n",
    "# Concatenate Hansen et al values\n",
    "# Make a DF containing results reported in Hansen et al 2022 table 3\n",
    "hansen_results_df = pd.DataFrame({'Code level': ['Chapter']*3+['Subchapter']*3+['3 digits']*3+['Billing (raw)']*3,\n",
    "                                  'Metric': ['Precision', 'Recall', 'µF1']*4,\n",
    "                                  'Value': [68.46, 69.27, 68.86,\n",
    "                                                   58.05, 57.21, 57.63,\n",
    "                                                   48.45, 47.19, 47.81,\n",
    "                                                   36.98, 36.26, 36.62],\n",
    "                                  'Experiment':'Hansen et al.'\n",
    "                                  })\n",
    "hansen_results_df = hansen_results_df.astype('str')\n",
    "\n",
    "# # Make a DF containing results reported in Sagi et al 2020 table 1\n",
    "# sagi_results_df = pd.DataFrame({'Code level': ['Chapter']*3+['Subchapter']*3+['3 digits']*3+['Billing (raw)']*3,\n",
    "#                                   'Metric': ['Precision', 'Recall', 'µF1']*4,\n",
    "#                                   'Value': [78.3,58.2,66.8,\n",
    "#                                             60.9,43.0,50.4,\n",
    "#                                             56.6,31.5,40.5,\n",
    "#                                             49.9,18.8,27.3],\n",
    "#                                   'Experiment':'Sagi et al.'\n",
    "#                                   })\n",
    "# sagi_results_df = sagi_results_df.astype('str')\n",
    "\n",
    "# hansen_compare_df = pd.concat([tmp,sagi_results_df,hansen_results_df])\n",
    "tmp = pd.concat([tmp,hansen_results_df])\n",
    "\n",
    "# # Pivot the table to a wider one\n",
    "# hansen_compare_df_wide = hansen_compare_df.pivot(index=['Code level','Metric'],columns='Experiment').droplevel(level=0,axis=1)\n",
    "# # reorder rows and cols\n",
    "# hansen_compare_df_wide = hansen_compare_df_wide.loc[['Chapter','Subchapter','3 digits','Billing (raw)']]\n",
    "# hansen_compare_df_wide = hansen_compare_df_wide.loc[:,['All codes learning','Filtered codes learning','Sagi et al.','Hansen et al.']]\n",
    "# hansen_compare_df_wide.style.to_latex(buf=tables_dir/\"hardclassif_scores.tex\",\n",
    "#                 environment='void',hrules=True,clines=\"skip-last;index\",column_format='llrrrr')  \n",
    "\n",
    "# Pivot the table to a wider one\n",
    "hardclassif_cherry_compare_df = tmp.pivot(index=['Code level','Metric'],columns='Experiment').droplevel(level=0,axis=1)\n",
    "# reorder rows and cols\n",
    "hardclassif_cherry_compare_df = hardclassif_cherry_compare_df.loc[['Chapter','Subchapter','3 digits','Billing (raw)']]\n",
    "hardclassif_cherry_compare_df = hardclassif_cherry_compare_df.loc[:,['All codes learning','Filtered codes learning','Adaptive Threshold','Hansen et al.']]\n",
    "hardclassif_cherry_compare_df.style.to_latex(buf=tables_dir/\"hardclassif_scores.tex\",\n",
    "                environment='void',hrules=True,clines=\"skip-last;index\",column_format='llrrrr')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "37b7839e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Experiment</th>\n",
       "      <th>All codes learning</th>\n",
       "      <th>Filtered codes learning</th>\n",
       "      <th>Adaptive Threshold</th>\n",
       "      <th>Hansen et al.</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Code level</th>\n",
       "      <th>Metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">Chapter</th>\n",
       "      <th>Precision</th>\n",
       "      <td>73.65 $\\pm$ 0.32</td>\n",
       "      <td>73.58 $\\pm$ 0.37</td>\n",
       "      <td>68.15 $\\pm$ 0.61</td>\n",
       "      <td>68.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>68.03 $\\pm$ 0.19</td>\n",
       "      <td>67.98 $\\pm$ 0.60</td>\n",
       "      <td>75.61 $\\pm$ 0.54</td>\n",
       "      <td>69.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>µF1</th>\n",
       "      <td>70.73 $\\pm$ 0.04</td>\n",
       "      <td>70.66 $\\pm$ 0.16</td>\n",
       "      <td>71.67 $\\pm$ 0.11</td>\n",
       "      <td>68.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">Subchapter</th>\n",
       "      <th>Precision</th>\n",
       "      <td>66.51 $\\pm$ 0.49</td>\n",
       "      <td>67.24 $\\pm$ 0.02</td>\n",
       "      <td>57.39 $\\pm$ 0.47</td>\n",
       "      <td>58.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>49.36 $\\pm$ 0.11</td>\n",
       "      <td>49.10 $\\pm$ 0.49</td>\n",
       "      <td>60.80 $\\pm$ 0.07</td>\n",
       "      <td>57.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>µF1</th>\n",
       "      <td>56.66 $\\pm$ 0.12</td>\n",
       "      <td>56.75 $\\pm$ 0.32</td>\n",
       "      <td>59.04 $\\pm$ 0.22</td>\n",
       "      <td>57.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">3 digits</th>\n",
       "      <th>Precision</th>\n",
       "      <td>60.55 $\\pm$ 0.44</td>\n",
       "      <td>61.73 $\\pm$ 0.48</td>\n",
       "      <td>46.82 $\\pm$ 1.00</td>\n",
       "      <td>48.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>33.13 $\\pm$ 0.05</td>\n",
       "      <td>32.39 $\\pm$ 0.47</td>\n",
       "      <td>47.65 $\\pm$ 1.08</td>\n",
       "      <td>47.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>µF1</th>\n",
       "      <td>42.83 $\\pm$ 0.06</td>\n",
       "      <td>42.48 $\\pm$ 0.30</td>\n",
       "      <td>47.18 $\\pm$ 0.07</td>\n",
       "      <td>47.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">Billing (raw)</th>\n",
       "      <th>Precision</th>\n",
       "      <td>54.39 $\\pm$ 0.58</td>\n",
       "      <td>55.67 $\\pm$ 0.59</td>\n",
       "      <td>37.90 $\\pm$ 0.54</td>\n",
       "      <td>36.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>22.06 $\\pm$ 0.11</td>\n",
       "      <td>21.50 $\\pm$ 0.34</td>\n",
       "      <td>38.22 $\\pm$ 0.59</td>\n",
       "      <td>36.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>µF1</th>\n",
       "      <td>31.39 $\\pm$ 0.05</td>\n",
       "      <td>31.02 $\\pm$ 0.28</td>\n",
       "      <td>38.04 $\\pm$ 0.01</td>\n",
       "      <td>36.62</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Experiment              All codes learning Filtered codes learning  \\\n",
       "Code level    Metric                                                 \n",
       "Chapter       Precision   73.65 $\\pm$ 0.32        73.58 $\\pm$ 0.37   \n",
       "              Recall      68.03 $\\pm$ 0.19        67.98 $\\pm$ 0.60   \n",
       "              µF1         70.73 $\\pm$ 0.04        70.66 $\\pm$ 0.16   \n",
       "Subchapter    Precision   66.51 $\\pm$ 0.49        67.24 $\\pm$ 0.02   \n",
       "              Recall      49.36 $\\pm$ 0.11        49.10 $\\pm$ 0.49   \n",
       "              µF1         56.66 $\\pm$ 0.12        56.75 $\\pm$ 0.32   \n",
       "3 digits      Precision   60.55 $\\pm$ 0.44        61.73 $\\pm$ 0.48   \n",
       "              Recall      33.13 $\\pm$ 0.05        32.39 $\\pm$ 0.47   \n",
       "              µF1         42.83 $\\pm$ 0.06        42.48 $\\pm$ 0.30   \n",
       "Billing (raw) Precision   54.39 $\\pm$ 0.58        55.67 $\\pm$ 0.59   \n",
       "              Recall      22.06 $\\pm$ 0.11        21.50 $\\pm$ 0.34   \n",
       "              µF1         31.39 $\\pm$ 0.05        31.02 $\\pm$ 0.28   \n",
       "\n",
       "Experiment              Adaptive Threshold Hansen et al.  \n",
       "Code level    Metric                                      \n",
       "Chapter       Precision   68.15 $\\pm$ 0.61         68.46  \n",
       "              Recall      75.61 $\\pm$ 0.54         69.27  \n",
       "              µF1         71.67 $\\pm$ 0.11         68.86  \n",
       "Subchapter    Precision   57.39 $\\pm$ 0.47         58.05  \n",
       "              Recall      60.80 $\\pm$ 0.07         57.21  \n",
       "              µF1         59.04 $\\pm$ 0.22         57.63  \n",
       "3 digits      Precision   46.82 $\\pm$ 1.00         48.45  \n",
       "              Recall      47.65 $\\pm$ 1.08         47.19  \n",
       "              µF1         47.18 $\\pm$ 0.07         47.81  \n",
       "Billing (raw) Precision   37.90 $\\pm$ 0.54         36.98  \n",
       "              Recall      38.22 $\\pm$ 0.59         36.26  \n",
       "              µF1         38.04 $\\pm$ 0.01         36.62  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hardclassif_cherry_compare_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f9b6183",
   "metadata": {},
   "source": [
    "### Table S6 AUROC comparison with Malakouti and Hauskretch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "31de1de5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build a table of F1,R,P scores to compare with Malakouti and Hauskretech\n",
    "from project_miscellaneous import filter_Sagi_diagnoses\n",
    "import sklearn.metrics as skmet\n",
    "\n",
    "\n",
    "oh_diagnoses_df = pd.read_parquet(\n",
    "    datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "base_dir = \"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_macro_AUROC/\"\n",
    "results_df = pd.DataFrame(\n",
    "    columns=('Experiment', 'Metric', \"Value\", \"crossval_id\"))\n",
    "\n",
    "exp_dir, exp_name =('drugSexAge_AllCodes','All codes learning')\n",
    "frequent_codes = oh_diagnoses_df.columns[oh_diagnoses_df.sum(axis=0)/float(len(oh_diagnoses_df)) >= 0.01]\n",
    "for crossval_suffix,crossval_id in zip(['', '_1', '_2'],[0,1,2]):\n",
    "    visit_ids = pd.read_csv(\n",
    "        datadir/('test_visits_id'+crossval_suffix+'.csv.gz')).squeeze()\n",
    "    test_oh_diagnoses_df = oh_diagnoses_df.loc[visit_ids,frequent_codes].sort_index()\n",
    "    test_predictions = pd.read_csv(\n",
    "        base_dir+exp_dir+crossval_suffix+\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    test_predictions = test_predictions.loc[:, frequent_codes].sort_index() # make sure index is sorted\n",
    "\n",
    "    y_true = test_oh_diagnoses_df.to_numpy()\n",
    "    y_pred = test_predictions.to_numpy()\n",
    "                                    \n",
    "    MAUROC = macro_averaged_AUROC(y_true=y_true,\n",
    "                                    probas_pred=y_pred,\n",
    "                                    pos_label=1)\n",
    "    MAUPRC = macro_averaged_auc_PR(y_true=y_true,\n",
    "                                    probas_pred=y_pred,\n",
    "                                    pos_label=1)\n",
    "    results_df = pd.concat([results_df, pd.DataFrame({'Experiment': exp_name,\n",
    "                                                        'Metric': ['macro AUROC', 'macro AUPRC'],\n",
    "                                                        'Value': [MAUROC,MAUPRC],\n",
    "                                                        'crossval_id': crossval_id\n",
    "                                                        })])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "ff6a3483",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 mean       sem\n",
      "Metric                         \n",
      "macro AUPRC  0.302209  0.001554\n",
      "macro AUROC  0.867473  0.000594\n",
      "592\n"
     ]
    }
   ],
   "source": [
    "malakouti_compare_df = results_df.groupby('Metric').aggregate({'Value' : ('mean','sem')})\n",
    "malakouti_compare_df = malakouti_compare_df.droplevel(level=0,axis=1)\n",
    "print(malakouti_compare_df)\n",
    "print(len(frequent_codes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "da76650d",
   "metadata": {},
   "outputs": [],
   "source": [
    "malakouti_compare_df['Naive multitask ('+str(len(frequent_codes))+' codes)'] = malakouti_compare_df.apply(lambda x: str(x['mean']*100)[0:5] + ' $\\pm$ ' + str(x['sem']*100)[0:4],axis=1)\n",
    "malakouti_compare_df.drop(columns=['mean','sem'],inplace=True)\n",
    "malakouti_compare_df['Malakouti and Hauskrecht (696 codes)'] = ['18.5','75.1']\n",
    "malakouti_compare_df.style.to_latex(buf=tables_dir/\"softclassif_scores_malakouti.tex\",\n",
    "                environment='void',hrules=True,clines=\"skip-last;index\",column_format='lrr') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "6e39bb6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Naive multitask (592 codes)</th>\n",
       "      <th>Malakouti and Hauskrecht (696 codes)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>macro AUPRC</th>\n",
       "      <td>30.22 $\\pm$ 0.15</td>\n",
       "      <td>18.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro AUROC</th>\n",
       "      <td>86.74 $\\pm$ 0.05</td>\n",
       "      <td>75.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Naive multitask (592 codes) Malakouti and Hauskrecht (696 codes)\n",
       "Metric                                                                      \n",
       "macro AUPRC            30.22 $\\pm$ 0.15                                 18.5\n",
       "macro AUROC            86.74 $\\pm$ 0.05                                 75.1"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "malakouti_compare_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd189618",
   "metadata": {},
   "source": [
    "### Performance metrics computed on different code subsets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7027d43d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from project_miscellaneous import filter_Sagi_diagnoses\n",
    "import sklearn.metrics as skmet\n",
    "oh_diagnoses_df = pd.read_parquet(\n",
    "    datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "\n",
    "icd9_concepts = pd.read_csv(datadir / \"icd9_concepts.csv.gz\")\n",
    "icd9_allcodes = icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].concept_code\n",
    "icd9_chapters = icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id==\"Chapter\"').concept_code\n",
    "icd9_subchapters = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id==\"SubChapter\"')\n",
    "                    .concept_code)\n",
    "icd9_3dig = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.startswith(\"3-dig\",'\n",
    "                                'na=False)').concept_code)\n",
    "icd9_4dig = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.startswith(\"4-dig\",'\n",
    "                                'na=False)').concept_code)\n",
    "icd9_5dig = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.startswith(\"5-dig\",'\n",
    "                                'na=False)').concept_code)\n",
    "icd9_billing = (\n",
    "    icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.endswith(\"-dig billing code\",'\n",
    "                        'na=False)').concept_code)\n",
    "\n",
    "\n",
    "base_dir = \"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/\"\n",
    "\n",
    "metrics_func_dict = {\n",
    "    'micro_AUROC':lambda y_true,y_score: skmet.roc_auc_score(y_true=y_true, y_score=y_score, average='micro'),\n",
    "    'macro_AUROC':lambda y_true,y_score: macro_averaged_AUROC(*filter_no_positive_label(y_true, y_score),pos_label=1),\n",
    "    \n",
    "    'micro_AUPRC': lambda y_true,y_score: micro_averaged_auc_PR(y_true=y_true, probas_pred=y_score, pos_label=1),\n",
    "    'macro_AUPRC': lambda y_true,y_score: macro_averaged_auc_PR(*filter_no_positive_label(y_true, y_score), pos_label=1),\n",
    "    \n",
    "    'NDCG': lambda y_true,y_score: skmet.ndcg_score(y_true=y_true,y_score=y_score, k=None),\n",
    "\n",
    "    'NDCG@15': lambda y_true,y_score: skmet.ndcg_score(y_true=y_true,y_score=y_score, k=15),\n",
    "}\n",
    "for p in [25,50,75,100]:\n",
    "    metrics_func_dict['RE@'+str(p)] = lambda y_true,y_score,q=p: np.nanmean(kumet.rank_errors_at_percentile(y_true=y_true, y_pred=y_score, q=q, no_true_label_value=None).numpy())\n",
    "\n",
    "n_occurrences = oh_diagnoses_df.sum(axis=0)\n",
    "code_filters_dict = {\n",
    "    'Top50': lambda x: x.sort_values()[-50:].index,\n",
    "    'None': lambda x: x.index,\n",
    "}\n",
    "results_df = pd.DataFrame()\n",
    "\n",
    "def compute_metrics(truth_df,predictions_df, metrics_list):\n",
    "\n",
    "    predictions_df.sort_index(ascending=True,inplace=True)\n",
    "    truth_df.sort_index(ascending=True,inplace=True)\n",
    "\n",
    "    metrics_df = pd.DataFrame()\n",
    "\n",
    "    for code_subset, code_subset_name in zip([icd9_allcodes, icd9_chapters, icd9_subchapters, icd9_3dig, icd9_billing],\n",
    "                                            ['All', 'Chapter', 'Subchapter', '3 digits', 'Billing']):\n",
    "        for filter_name,filter_func in code_filters_dict.items():\n",
    "            if filter_name==\"Top50\" and (code_subset_name in ['Chapter','Subchapter']):\n",
    "                continue\n",
    "            code_subset_dict = {\"Hierarchy level\":[code_subset_name], \"Filter\":[filter_name]}\n",
    "            filtered_codes = filter_func(n_occurrences.loc[code_subset])\n",
    "            for metric in metrics_list:\n",
    "                code_subset_dict[metric] = [metrics_func_dict[metric](truth_df.loc[:,filtered_codes].to_numpy(),\n",
    "                                                                    predictions_df.loc[:,filtered_codes].to_numpy())]\n",
    "            metrics_df = pd.concat([metrics_df, pd.DataFrame(code_subset_dict)])\n",
    "    return metrics_df\n",
    "\n",
    "for crossval_suffix,crossval_id in zip(['', '_1', '_2'],[0,1,2]):\n",
    "    visit_ids = pd.read_csv(datadir/('test_visits_id'+crossval_suffix+'.csv.gz')).squeeze()\n",
    "    test_oh_diagnoses_df = oh_diagnoses_df.loc[visit_ids,:]\n",
    "    metrics_df = pd.DataFrame()\n",
    "    # Compute RE,NDCG and NDCG@15 on NDCG experiment\n",
    "    test_predictions = pd.read_csv(\n",
    "        base_dir+\"overall_NDCG/drugSexAge_AllCodes\"+crossval_suffix\n",
    "        +\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    tmp = compute_metrics(test_oh_diagnoses_df,test_predictions,\n",
    "                ['NDCG','NDCG@15','RE@25','RE@50','RE@75','RE@100'])\n",
    "    metrics_df = pd.concat([metrics_df, tmp])\n",
    "    # Compute micro AUROC\n",
    "    test_predictions = pd.read_csv(\n",
    "        base_dir+\"overall_micro_AUROC/drugSexAge_AllCodes\"+crossval_suffix\n",
    "        +\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    tmp = compute_metrics(test_oh_diagnoses_df,test_predictions,\n",
    "            ['micro_AUROC'])\n",
    "    metrics_df = pd.merge(metrics_df, tmp,on=[\"Hierarchy level\",\"Filter\"])\n",
    "    # Compute macro AUROC\n",
    "    test_predictions = pd.read_csv(\n",
    "        base_dir+\"overall_macro_AUROC/drugSexAge_AllCodes\"+crossval_suffix\n",
    "        +\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    tmp = compute_metrics(test_oh_diagnoses_df,test_predictions,\n",
    "                ['macro_AUROC'])\n",
    "    metrics_df = pd.merge(metrics_df, tmp,on=[\"Hierarchy level\",\"Filter\"])\n",
    "    # Compute micro AUPRC and LRAP\n",
    "    test_predictions = pd.read_csv(\n",
    "        base_dir+\"overall_micro_aucPR/drugSexAge_AllCodes\"+crossval_suffix\n",
    "        +\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    tmp = compute_metrics(test_oh_diagnoses_df,test_predictions,\n",
    "                ['micro_AUPRC'])\n",
    "    metrics_df = pd.merge(metrics_df, tmp,on=[\"Hierarchy level\",\"Filter\"])\n",
    "    # Compute macro AUPRC\n",
    "    test_predictions = pd.read_csv(\n",
    "        base_dir+\"overall_macro_aucPR/drugSexAge_AllCodes\"+crossval_suffix\n",
    "        +\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    tmp = compute_metrics(test_oh_diagnoses_df,test_predictions,\n",
    "                ['macro_AUPRC'])\n",
    "    metrics_df = pd.merge(metrics_df, tmp,on=[\"Hierarchy level\",\"Filter\"])\n",
    "    \n",
    "    metrics_df['crossval_id'] = crossval_id\n",
    "    results_df = pd.concat([results_df,metrics_df])\n",
    "results_df.to_csv(tables_dir/'performance_metrics_persubset.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "8134ae17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>NDCG</th>\n",
       "      <th>NDCG@15</th>\n",
       "      <th>RE@25</th>\n",
       "      <th>RE@50</th>\n",
       "      <th>RE@75</th>\n",
       "      <th>RE@100</th>\n",
       "      <th>micro_AUROC</th>\n",
       "      <th>macro_AUROC</th>\n",
       "      <th>micro_AUPRC</th>\n",
       "      <th>macro_AUPRC</th>\n",
       "      <th>crossval_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Filter</th>\n",
       "      <th>Hierarchy level</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Top50</th>\n",
       "      <th>All</th>\n",
       "      <td>0.905674</td>\n",
       "      <td>0.802586</td>\n",
       "      <td>1.352283</td>\n",
       "      <td>3.626995</td>\n",
       "      <td>7.883311</td>\n",
       "      <td>17.318040</td>\n",
       "      <td>0.871219</td>\n",
       "      <td>0.849181</td>\n",
       "      <td>0.752291</td>\n",
       "      <td>0.685437</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">None</th>\n",
       "      <th>All</th>\n",
       "      <td>0.830641</td>\n",
       "      <td>0.745933</td>\n",
       "      <td>8.648289</td>\n",
       "      <td>37.653492</td>\n",
       "      <td>148.101910</td>\n",
       "      <td>1660.959700</td>\n",
       "      <td>0.983218</td>\n",
       "      <td>0.849517</td>\n",
       "      <td>0.500958</td>\n",
       "      <td>0.092933</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chapter</th>\n",
       "      <td>0.924145</td>\n",
       "      <td>0.918710</td>\n",
       "      <td>0.393832</td>\n",
       "      <td>0.975772</td>\n",
       "      <td>2.115584</td>\n",
       "      <td>4.072876</td>\n",
       "      <td>0.893717</td>\n",
       "      <td>0.848209</td>\n",
       "      <td>0.806640</td>\n",
       "      <td>0.687824</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Subchapter</th>\n",
       "      <td>0.839523</td>\n",
       "      <td>0.738461</td>\n",
       "      <td>1.678969</td>\n",
       "      <td>4.104698</td>\n",
       "      <td>9.825030</td>\n",
       "      <td>26.213955</td>\n",
       "      <td>0.948834</td>\n",
       "      <td>0.852442</td>\n",
       "      <td>0.621387</td>\n",
       "      <td>0.285507</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Top50</th>\n",
       "      <th>3 digits</th>\n",
       "      <td>0.786003</td>\n",
       "      <td>0.710546</td>\n",
       "      <td>1.718675</td>\n",
       "      <td>3.576439</td>\n",
       "      <td>7.147665</td>\n",
       "      <td>13.874050</td>\n",
       "      <td>0.893614</td>\n",
       "      <td>0.857560</td>\n",
       "      <td>0.602602</td>\n",
       "      <td>0.495256</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>None</th>\n",
       "      <th>3 digits</th>\n",
       "      <td>0.754107</td>\n",
       "      <td>0.596264</td>\n",
       "      <td>5.472575</td>\n",
       "      <td>14.877092</td>\n",
       "      <td>39.930580</td>\n",
       "      <td>147.303590</td>\n",
       "      <td>0.969501</td>\n",
       "      <td>0.856241</td>\n",
       "      <td>0.454996</td>\n",
       "      <td>0.133810</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Top50</th>\n",
       "      <th>Billing</th>\n",
       "      <td>0.631120</td>\n",
       "      <td>0.573224</td>\n",
       "      <td>2.410466</td>\n",
       "      <td>4.590089</td>\n",
       "      <td>8.538312</td>\n",
       "      <td>15.415603</td>\n",
       "      <td>0.890304</td>\n",
       "      <td>0.853319</td>\n",
       "      <td>0.494618</td>\n",
       "      <td>0.374549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>None</th>\n",
       "      <th>Billing</th>\n",
       "      <td>0.608781</td>\n",
       "      <td>0.437473</td>\n",
       "      <td>19.504261</td>\n",
       "      <td>52.090378</td>\n",
       "      <td>155.010570</td>\n",
       "      <td>792.124500</td>\n",
       "      <td>0.976287</td>\n",
       "      <td>0.847834</td>\n",
       "      <td>0.298843</td>\n",
       "      <td>0.072075</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Top50</th>\n",
       "      <th>All</th>\n",
       "      <td>0.905154</td>\n",
       "      <td>0.803104</td>\n",
       "      <td>1.315453</td>\n",
       "      <td>3.483554</td>\n",
       "      <td>7.796192</td>\n",
       "      <td>17.123400</td>\n",
       "      <td>0.870141</td>\n",
       "      <td>0.848532</td>\n",
       "      <td>0.753655</td>\n",
       "      <td>0.690634</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">None</th>\n",
       "      <th>All</th>\n",
       "      <td>0.829885</td>\n",
       "      <td>0.745552</td>\n",
       "      <td>8.251551</td>\n",
       "      <td>37.995000</td>\n",
       "      <td>148.247850</td>\n",
       "      <td>1685.589500</td>\n",
       "      <td>0.982994</td>\n",
       "      <td>0.846022</td>\n",
       "      <td>0.496879</td>\n",
       "      <td>0.097259</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chapter</th>\n",
       "      <td>0.921804</td>\n",
       "      <td>0.914997</td>\n",
       "      <td>0.420400</td>\n",
       "      <td>1.008202</td>\n",
       "      <td>2.164458</td>\n",
       "      <td>4.102955</td>\n",
       "      <td>0.891672</td>\n",
       "      <td>0.845961</td>\n",
       "      <td>0.803031</td>\n",
       "      <td>0.692852</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Subchapter</th>\n",
       "      <td>0.839107</td>\n",
       "      <td>0.736686</td>\n",
       "      <td>1.813536</td>\n",
       "      <td>4.388395</td>\n",
       "      <td>10.234887</td>\n",
       "      <td>26.910828</td>\n",
       "      <td>0.948050</td>\n",
       "      <td>0.856852</td>\n",
       "      <td>0.617963</td>\n",
       "      <td>0.297322</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Top50</th>\n",
       "      <th>3 digits</th>\n",
       "      <td>0.790386</td>\n",
       "      <td>0.713955</td>\n",
       "      <td>1.704117</td>\n",
       "      <td>3.603575</td>\n",
       "      <td>7.206663</td>\n",
       "      <td>14.054821</td>\n",
       "      <td>0.891835</td>\n",
       "      <td>0.857295</td>\n",
       "      <td>0.596412</td>\n",
       "      <td>0.493468</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>None</th>\n",
       "      <th>3 digits</th>\n",
       "      <td>0.755212</td>\n",
       "      <td>0.595242</td>\n",
       "      <td>5.818787</td>\n",
       "      <td>15.578311</td>\n",
       "      <td>40.556843</td>\n",
       "      <td>151.469600</td>\n",
       "      <td>0.969405</td>\n",
       "      <td>0.852328</td>\n",
       "      <td>0.449465</td>\n",
       "      <td>0.135516</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Top50</th>\n",
       "      <th>Billing</th>\n",
       "      <td>0.643243</td>\n",
       "      <td>0.586910</td>\n",
       "      <td>2.207820</td>\n",
       "      <td>4.275404</td>\n",
       "      <td>8.243769</td>\n",
       "      <td>15.155574</td>\n",
       "      <td>0.891302</td>\n",
       "      <td>0.853952</td>\n",
       "      <td>0.496044</td>\n",
       "      <td>0.375878</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>None</th>\n",
       "      <th>Billing</th>\n",
       "      <td>0.614398</td>\n",
       "      <td>0.441022</td>\n",
       "      <td>19.716946</td>\n",
       "      <td>54.278923</td>\n",
       "      <td>160.595520</td>\n",
       "      <td>811.021200</td>\n",
       "      <td>0.975898</td>\n",
       "      <td>0.845674</td>\n",
       "      <td>0.298157</td>\n",
       "      <td>0.074440</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Top50</th>\n",
       "      <th>All</th>\n",
       "      <td>0.907414</td>\n",
       "      <td>0.804094</td>\n",
       "      <td>1.265869</td>\n",
       "      <td>3.497906</td>\n",
       "      <td>7.778929</td>\n",
       "      <td>17.225260</td>\n",
       "      <td>0.869723</td>\n",
       "      <td>0.847168</td>\n",
       "      <td>0.752601</td>\n",
       "      <td>0.686761</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">None</th>\n",
       "      <th>All</th>\n",
       "      <td>0.832280</td>\n",
       "      <td>0.749098</td>\n",
       "      <td>7.657939</td>\n",
       "      <td>36.260914</td>\n",
       "      <td>143.128650</td>\n",
       "      <td>1742.779900</td>\n",
       "      <td>0.983087</td>\n",
       "      <td>0.842807</td>\n",
       "      <td>0.499792</td>\n",
       "      <td>0.094625</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chapter</th>\n",
       "      <td>0.924012</td>\n",
       "      <td>0.917680</td>\n",
       "      <td>0.398622</td>\n",
       "      <td>0.979377</td>\n",
       "      <td>2.124000</td>\n",
       "      <td>4.107685</td>\n",
       "      <td>0.893504</td>\n",
       "      <td>0.846220</td>\n",
       "      <td>0.804858</td>\n",
       "      <td>0.670695</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Subchapter</th>\n",
       "      <td>0.840912</td>\n",
       "      <td>0.739286</td>\n",
       "      <td>1.665875</td>\n",
       "      <td>4.096562</td>\n",
       "      <td>9.871790</td>\n",
       "      <td>26.292868</td>\n",
       "      <td>0.949449</td>\n",
       "      <td>0.857856</td>\n",
       "      <td>0.622994</td>\n",
       "      <td>0.284217</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Top50</th>\n",
       "      <th>3 digits</th>\n",
       "      <td>0.790038</td>\n",
       "      <td>0.713875</td>\n",
       "      <td>1.725744</td>\n",
       "      <td>3.627865</td>\n",
       "      <td>7.299743</td>\n",
       "      <td>14.136432</td>\n",
       "      <td>0.890604</td>\n",
       "      <td>0.854967</td>\n",
       "      <td>0.594260</td>\n",
       "      <td>0.489237</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>None</th>\n",
       "      <th>3 digits</th>\n",
       "      <td>0.756531</td>\n",
       "      <td>0.597066</td>\n",
       "      <td>5.256750</td>\n",
       "      <td>14.762240</td>\n",
       "      <td>39.508210</td>\n",
       "      <td>146.466600</td>\n",
       "      <td>0.969747</td>\n",
       "      <td>0.847802</td>\n",
       "      <td>0.452350</td>\n",
       "      <td>0.134570</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Top50</th>\n",
       "      <th>Billing</th>\n",
       "      <td>0.644399</td>\n",
       "      <td>0.585921</td>\n",
       "      <td>2.219843</td>\n",
       "      <td>4.403768</td>\n",
       "      <td>8.581594</td>\n",
       "      <td>15.585254</td>\n",
       "      <td>0.888106</td>\n",
       "      <td>0.849949</td>\n",
       "      <td>0.487074</td>\n",
       "      <td>0.368339</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>None</th>\n",
       "      <th>Billing</th>\n",
       "      <td>0.614019</td>\n",
       "      <td>0.440548</td>\n",
       "      <td>18.830270</td>\n",
       "      <td>50.504300</td>\n",
       "      <td>150.752200</td>\n",
       "      <td>837.016600</td>\n",
       "      <td>0.975759</td>\n",
       "      <td>0.841029</td>\n",
       "      <td>0.298479</td>\n",
       "      <td>0.072597</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            NDCG   NDCG@15      RE@25      RE@50       RE@75  \\\n",
       "Filter Hierarchy level                                                         \n",
       "Top50  All              0.905674  0.802586   1.352283   3.626995    7.883311   \n",
       "None   All              0.830641  0.745933   8.648289  37.653492  148.101910   \n",
       "       Chapter          0.924145  0.918710   0.393832   0.975772    2.115584   \n",
       "       Subchapter       0.839523  0.738461   1.678969   4.104698    9.825030   \n",
       "Top50  3 digits         0.786003  0.710546   1.718675   3.576439    7.147665   \n",
       "None   3 digits         0.754107  0.596264   5.472575  14.877092   39.930580   \n",
       "Top50  Billing          0.631120  0.573224   2.410466   4.590089    8.538312   \n",
       "None   Billing          0.608781  0.437473  19.504261  52.090378  155.010570   \n",
       "Top50  All              0.905154  0.803104   1.315453   3.483554    7.796192   \n",
       "None   All              0.829885  0.745552   8.251551  37.995000  148.247850   \n",
       "       Chapter          0.921804  0.914997   0.420400   1.008202    2.164458   \n",
       "       Subchapter       0.839107  0.736686   1.813536   4.388395   10.234887   \n",
       "Top50  3 digits         0.790386  0.713955   1.704117   3.603575    7.206663   \n",
       "None   3 digits         0.755212  0.595242   5.818787  15.578311   40.556843   \n",
       "Top50  Billing          0.643243  0.586910   2.207820   4.275404    8.243769   \n",
       "None   Billing          0.614398  0.441022  19.716946  54.278923  160.595520   \n",
       "Top50  All              0.907414  0.804094   1.265869   3.497906    7.778929   \n",
       "None   All              0.832280  0.749098   7.657939  36.260914  143.128650   \n",
       "       Chapter          0.924012  0.917680   0.398622   0.979377    2.124000   \n",
       "       Subchapter       0.840912  0.739286   1.665875   4.096562    9.871790   \n",
       "Top50  3 digits         0.790038  0.713875   1.725744   3.627865    7.299743   \n",
       "None   3 digits         0.756531  0.597066   5.256750  14.762240   39.508210   \n",
       "Top50  Billing          0.644399  0.585921   2.219843   4.403768    8.581594   \n",
       "None   Billing          0.614019  0.440548  18.830270  50.504300  150.752200   \n",
       "\n",
       "                             RE@100  micro_AUROC  macro_AUROC  micro_AUPRC  \\\n",
       "Filter Hierarchy level                                                       \n",
       "Top50  All                17.318040     0.871219     0.849181     0.752291   \n",
       "None   All              1660.959700     0.983218     0.849517     0.500958   \n",
       "       Chapter             4.072876     0.893717     0.848209     0.806640   \n",
       "       Subchapter         26.213955     0.948834     0.852442     0.621387   \n",
       "Top50  3 digits           13.874050     0.893614     0.857560     0.602602   \n",
       "None   3 digits          147.303590     0.969501     0.856241     0.454996   \n",
       "Top50  Billing            15.415603     0.890304     0.853319     0.494618   \n",
       "None   Billing           792.124500     0.976287     0.847834     0.298843   \n",
       "Top50  All                17.123400     0.870141     0.848532     0.753655   \n",
       "None   All              1685.589500     0.982994     0.846022     0.496879   \n",
       "       Chapter             4.102955     0.891672     0.845961     0.803031   \n",
       "       Subchapter         26.910828     0.948050     0.856852     0.617963   \n",
       "Top50  3 digits           14.054821     0.891835     0.857295     0.596412   \n",
       "None   3 digits          151.469600     0.969405     0.852328     0.449465   \n",
       "Top50  Billing            15.155574     0.891302     0.853952     0.496044   \n",
       "None   Billing           811.021200     0.975898     0.845674     0.298157   \n",
       "Top50  All                17.225260     0.869723     0.847168     0.752601   \n",
       "None   All              1742.779900     0.983087     0.842807     0.499792   \n",
       "       Chapter             4.107685     0.893504     0.846220     0.804858   \n",
       "       Subchapter         26.292868     0.949449     0.857856     0.622994   \n",
       "Top50  3 digits           14.136432     0.890604     0.854967     0.594260   \n",
       "None   3 digits          146.466600     0.969747     0.847802     0.452350   \n",
       "Top50  Billing            15.585254     0.888106     0.849949     0.487074   \n",
       "None   Billing           837.016600     0.975759     0.841029     0.298479   \n",
       "\n",
       "                        macro_AUPRC  crossval_id  \n",
       "Filter Hierarchy level                            \n",
       "Top50  All                 0.685437            0  \n",
       "None   All                 0.092933            0  \n",
       "       Chapter             0.687824            0  \n",
       "       Subchapter          0.285507            0  \n",
       "Top50  3 digits            0.495256            0  \n",
       "None   3 digits            0.133810            0  \n",
       "Top50  Billing             0.374549            0  \n",
       "None   Billing             0.072075            0  \n",
       "Top50  All                 0.690634            1  \n",
       "None   All                 0.097259            1  \n",
       "       Chapter             0.692852            1  \n",
       "       Subchapter          0.297322            1  \n",
       "Top50  3 digits            0.493468            1  \n",
       "None   3 digits            0.135516            1  \n",
       "Top50  Billing             0.375878            1  \n",
       "None   Billing             0.074440            1  \n",
       "Top50  All                 0.686761            2  \n",
       "None   All                 0.094625            2  \n",
       "       Chapter             0.670695            2  \n",
       "       Subchapter          0.284217            2  \n",
       "Top50  3 digits            0.489237            2  \n",
       "None   3 digits            0.134570            2  \n",
       "Top50  Billing             0.368339            2  \n",
       "None   Billing             0.072597            2  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df = pd.read_csv(tables_dir/'performance_metrics_persubset.csv',index_col=['Filter','Hierarchy level'])\n",
    "results_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "786b0697",
   "metadata": {},
   "source": [
    "#### Table S7 and S8: ranking and soft classification performance metrics on different subsets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0fc5f86",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = pd.read_csv(tables_dir/'performance_metrics_persubset.csv',index_col=['Filter','Hierarchy level'])\n",
    "ranking_metrics = ['NDCG','NDCG@15','RE@25','RE@50','RE@75','RE@100']\n",
    "soft_classif_metrics = [\"micro_AUROC\", \"macro_AUROC\", \"micro_AUPRC\",\"macro_AUPRC\"]\n",
    "non_RE_cols = results_df.columns[~results_df.columns.str.startswith('RE@')]\n",
    "results_df.loc[:,non_RE_cols]*=100\n",
    "results_df = results_df.reset_index(drop=False).melt(id_vars=['Filter','Hierarchy level','crossval_id'],var_name='metric',value_name='value')\n",
    "results_df = results_df.drop(columns='crossval_id').groupby(by=[\"Filter\",'Hierarchy level','metric']).aggregate(['mean','sem'])\n",
    "results_df = results_df.droplevel(level=0,axis=1)\n",
    "results_df['value'] = results_df.apply(lambda x: str(x['mean'])[0:5] + ' $\\pm$ ' + str(x['sem'])[0:4],axis=1)\n",
    "results_df = results_df.drop(columns=['mean','sem']).reset_index()\n",
    "results_df = results_df.rename(columns={\"metric\":\"Metric\"})\n",
    "results_df = results_df.pivot(index=[\"Filter\",\"Hierarchy level\"],columns=\"Metric\", values='value')\n",
    "results_df = results_df.reindex([\"Chapter\",\"Subchapter\",\"3 digits\",\"Billing\",\"All\"], level=\"Hierarchy level\")\n",
    "\n",
    "# All performance metrics\n",
    "results_df.style.to_latex(buf=tables_dir/\"perf_persubset.tex\",\n",
    "                environment='void',hrules=True,clines=\"skip-last;index\",column_format='llrrrrrrrrrr')\n",
    "# Table S7\n",
    "results_df.loc[:,ranking_metrics].style.to_latex(buf=tables_dir/\"ranking_perf_persubset.tex\",\n",
    "                environment='void',hrules=True,clines=\"skip-last;index\",column_format='llrrrrrr')\n",
    "\n",
    "# Table S8\n",
    "results_classif_df = results_df.loc[:,soft_classif_metrics]\n",
    "results_classif_df.columns = results_classif_df.columns.str.replace(pat='micro_',repl=\"$\\mu$\",regex=False)\n",
    "results_classif_df.columns = results_classif_df.columns.str.replace(pat='macro_',repl=\"macro \",regex=False)\n",
    "results_classif_df.style.to_latex(buf=tables_dir/\"softclassif_perf_persubset.tex\",\n",
    "                environment='void',hrules=True,clines=\"skip-last;index\",column_format='llrrrr')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19762015",
   "metadata": {},
   "source": [
    "### LIG-doctor, Rodrigues Jr et al 2021: number of ICD codes in the reported subset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "0e7767a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of patients with >= 2 stays: 7537\n",
      "vs 7483 reported in Rodrigues et al, after filtering for negative duration of stay and stays with 0 ICD codes attached.\n",
      "Number of corresponding visits: 19993\n",
      "vs 19911 reported in Rodrigues et al, after filtering for negative duration of stay and stays with 0 ICD codes attached.\n"
     ]
    }
   ],
   "source": [
    "visit_patient_table = pd.read_csv(get_datadir()/\"mimic-omop/visit_occurrence.csv.gz\",\n",
    "                                  usecols=('visit_occurrence_id','person_id'))\n",
    "# Extract the visit_occurrence_id for patients with more than 1 stay\n",
    "mul_stay_person_id = visit_patient_table.person_id[visit_patient_table.person_id.duplicated()].unique()\n",
    "print(\"Number of patients with >= 2 stays: \" + str(len(mul_stay_person_id)))\n",
    "print(\"vs 7483 reported in Rodrigues et al, after filtering for negative \"\\\n",
    "      \"duration of stay and stays with 0 ICD codes attached.\")\n",
    "mul_stay_visit_patient  = visit_patient_table.loc[\n",
    "      visit_patient_table.person_id.isin(mul_stay_person_id),\n",
    "      :]\n",
    "print(\"Number of corresponding visits: \"+str(len(mul_stay_visit_patient)))\n",
    "print(\"vs 19911 reported in Rodrigues et al, after filtering for negative \"\\\n",
    "      \"duration of stay and stays with 0 ICD codes attached.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "dad8a85c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ICD9-CM</th>\n",
       "      <th>MIMIC III Rodrigues</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Chapters</th>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Subchapters</th>\n",
       "      <td>154</td>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3 digits</th>\n",
       "      <td>1234</td>\n",
       "      <td>941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4 digits</th>\n",
       "      <td>7473</td>\n",
       "      <td>3224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5 digits</th>\n",
       "      <td>8846</td>\n",
       "      <td>2519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Billing</th>\n",
       "      <td>12167</td>\n",
       "      <td>4093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All levels</th>\n",
       "      <td>17726</td>\n",
       "      <td>6851</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             ICD9-CM  MIMIC III Rodrigues\n",
       "Chapters          19                   19\n",
       "Subchapters      154                  148\n",
       "3 digits        1234                  941\n",
       "4 digits        7473                 3224\n",
       "5 digits        8846                 2519\n",
       "Billing        12167                 4093\n",
       "All levels     17726                 6851"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the number of resulting ICD diagnoses\n",
    "oh_diagnoses_df = pd.read_parquet(datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "oh_diagnoses_df = oh_diagnoses_df.loc[mul_stay_visit_patient.visit_occurrence_id,:]\n",
    "icd9_concepts = pd.read_csv(datadir / \"icd9_concepts.csv.gz\")\n",
    "icd9_allcodes = icd9_concepts.concept_code\n",
    "icd9_chapters = icd9_concepts.query('concept_class_id==\"Chapter\"').concept_code\n",
    "icd9_subchapters = (icd9_concepts.query('concept_class_id==\"SubChapter\"').concept_code)\n",
    "icd9_3dig = (icd9_concepts.query('concept_class_id.str.startswith(\"3-dig\",na=False)').concept_code)\n",
    "icd9_4dig = (icd9_concepts.query('concept_class_id.str.startswith(\"4-dig\",na=False)').concept_code)\n",
    "icd9_5dig = (icd9_concepts.query('concept_class_id.str.startswith(\"5-dig\",na=False)').concept_code)\n",
    "icd9_billing = (icd9_concepts.query('concept_class_id.str.endswith(\"-dig billing code\",na=False)').concept_code)\n",
    "# Build the table containing the number of unique codes\n",
    "n_codes_df = pd.DataFrame()\n",
    "for code_set,name in zip([icd9_chapters,icd9_subchapters,icd9_3dig,icd9_4dig,icd9_5dig,icd9_billing,icd9_allcodes],\n",
    "                            ['Chapters','Subchapters','3 digits','4 digits','5 digits','Billing','All levels']):\n",
    "    tmp_df = pd.DataFrame({'ICD9-CM':len(code_set),\n",
    "                            'MIMIC III Rodrigues':sum(oh_diagnoses_df.loc[:,oh_diagnoses_df.columns.isin(code_set)].sum(axis=0)>=1),\n",
    "                            },index=[name])\n",
    "    n_codes_df = pd.concat([n_codes_df,tmp_df],axis=0)\n",
    "#n_codes_df.style.format(precision=2).to_latex(buf=tables_dir/\"unique_codes_perlevel.tex\",\n",
    "#                environment='void',hrules=True,clines=\"skip-last;index\",column_format='lrrr')\n",
    "n_codes_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d4fbcb0",
   "metadata": {},
   "source": [
    "This doesn't agree at any level with the 855 ICD codes reported in the Rodrigues et al paper."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "3932f9a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-Total number (complete set) of ICD9 codes (diag + proc): 19020\n"
     ]
    }
   ],
   "source": [
    "# Check the number of ICD9 CSS mapping see if it equals 855 from the Rodrigues paper\n",
    "# retro engineered from: https://github.com/jfrjunio/LIG-Doctor/blob/master/preprocess_mimiciii.py\n",
    "# And the corresponding file: https://github.com/jfrjunio/LIG-Doctor/blob/master/icd9_to_css_dictionary\n",
    "import pickle\n",
    "icd9TOCCS_Map = pickle.load(open(get_datadir()/'LIG_doctor_ICD-CCS_map/icd9_to_css_dictionary','rb'))\n",
    "print('-Total number (complete set) of ICD9 codes (diag + proc): ' + str(len(set(icd9TOCCS_Map.keys()))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "9e409417",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multi stay unique ICD9 Diagnoses in CCS map: 5137\n",
      "Multi stay unique ICD9 Chapters Diagnoses in CCS map: 0\n",
      "Multi stay unique ICD9 Subchapters Diagnoses in CCS map: 0\n",
      "Multi stay unique ICD9 3 digits Diagnoses in CCS map: 98\n",
      "Multi stay unique ICD9 4 digits Diagnoses in CCS map: 2520\n",
      "Multi stay unique ICD9 5 digits Diagnoses in CCS map: 2519\n",
      "Multi stay unique ICD9 Billing Diagnoses in CCS map: 4093\n",
      "Multi stay unique ICD9 All levels Diagnoses in CCS map: 5137\n"
     ]
    }
   ],
   "source": [
    "# Check how many diagnoses in the filtered stay are in the CCS map\n",
    "ccs_icd9_keys = pd.Series(icd9TOCCS_Map.keys())\n",
    "ccs_icd9_keys = ccs_icd9_keys.str.rstrip(' ')\n",
    "ccs_icd9_keys = set(ccs_icd9_keys)\n",
    "diagnoses_in_map = (\"D\"+(oh_diagnoses_df.columns[\n",
    "    oh_diagnoses_df.sum(axis=0)>=1\n",
    "    ].to_series()\n",
    " .str.replace('.','',regex=False)\n",
    " .str.replace('-','',regex=False)\n",
    " )).isin(ccs_icd9_keys)\n",
    "\n",
    "print(\"Multi stay unique ICD9 Diagnoses in CCS map: \" + str(diagnoses_in_map.sum()))\n",
    "for code_set,name in zip([icd9_chapters,icd9_subchapters,icd9_3dig,icd9_4dig,icd9_5dig,icd9_billing,icd9_allcodes],\n",
    "                            ['Chapters','Subchapters','3 digits','4 digits','5 digits','Billing','All levels']):\n",
    "    print(\"Multi stay unique ICD9 \"+name+\" Diagnoses in CCS map: \" \n",
    "          + str(diagnoses_in_map[diagnoses_in_map.index.isin(code_set)].sum()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "469aa80b-3056-49de-8e66-a8792af844b2",
   "metadata": {},
   "source": [
    "# Information extracted beyond code frequency"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77cfbb22",
   "metadata": {},
   "source": [
    "### Fig. 1b: Positive vs negative examples rank ratios"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b730d3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute the relative difference of ranks between positive and negative examples\n",
    "\"\"\" oh_diagnoses_df = pd.read_parquet(datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "test_visits = pd.read_csv(datadir/'test_visits_id.csv.gz').squeeze()\n",
    "test_oh_diagnoses_df = oh_diagnoses_df.loc[test_visits,:]\n",
    "del oh_diagnoses_df\n",
    "predictions = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_micro_F1/drugSexAge_AllCodes/best_models/1/\"\n",
    "                              +\"predictions_test.csv.gz\").set_index('visit_occurrence_id') \"\"\"\n",
    "code_subset = icd9_allcodes\n",
    "total_ranks = False\n",
    "nbins = 50\n",
    "aggregating_func=np.nanmedian\n",
    "from project_miscellaneous import filter_Sagi_diagnoses, infrequent_codes_mask, sagi_filtered_out_roots_mask\n",
    "oh_diagnoses_df = pd.read_parquet(datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "code_subset = icd9_allcodes\n",
    "rel_rank_ratios_df = pd.DataFrame(index=code_subset)\n",
    "for crossval_suff in ['','_1','_2']:\n",
    "    test_visits = pd.read_csv(datadir/(\"test_visits_id\"+crossval_suff+\".csv.gz\")).squeeze()\n",
    "    test_oh_diagnoses_df = oh_diagnoses_df.loc[test_visits,:]\n",
    "    predictions = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_micro_F1/drugSexAge_AllCodes\"+crossval_suff+\"/best_models/1/\"\n",
    "                                +\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "\n",
    "    for pred,label in zip([predictions],['Model']):\n",
    "        if total_ranks:\n",
    "            pred_ranks = compute_predictions_ranks(pred)\n",
    "        else:\n",
    "            pred_ranks = compute_predictions_ranks(pred.loc[:,code_subset])\n",
    "        pos_examples_median = aggregating_func(np.where(test_oh_diagnoses_df.loc[:,code_subset].to_numpy()==1,pred_ranks.loc[:,code_subset].to_numpy(),np.nan), axis=0)\n",
    "        neg_examples_median = aggregating_func(np.where(test_oh_diagnoses_df.loc[:,code_subset].to_numpy()!=1,pred_ranks.loc[:,code_subset].to_numpy(),np.nan), axis=0)\n",
    "        rel_rank_ratios_df['crossval'+ crossval_suff] = (pos_examples_median-neg_examples_median)/neg_examples_median\n",
    "\n",
    "rel_rank_ratios = rel_rank_ratios_df.aggregate(np.nanmean,axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "6ec1e3a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPAAAACpCAYAAAAV1LCtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAtJUlEQVR4nO2deZzN1f/Hn+8ZQ7bIXrL9aCGJaJeQLzGoDGXNWCNLokL5IpWURF9KWTJkLUNkSaOsEdkSSqQsKftS1jHz/v1xPnfcuXPvnTvMcmec5+Mxj/l8zud8zuf9+dz7vud8znmd9xFVxWKxZE5CMtoAi8Vy5VgHtlgyMdaBLZZMjHVgiyUTYx3YYsnEZMtoA/xRqFAhLV26dEabYbFkOBs3bjyqqoU904PagUuXLs2GDRsy2gyLJcMRkb3e0m0T2mLJxAR1DWyx7Ny5k1dffZWiRYtSpkwZypQpw+zZs8mfPz/h4eE0bNgwIe/58+cZOHAg586d49Zbb6VHjx707NmT2NhYIiMjuffee+nTpw/Dhg0je/bsGXhXqYd1YEtQs2jRItq1a0d4eDht27bl66+/Zs6cOeTOnZt69eolcuDx48dz+vRpsmXLxo033gjAsWPHePHFF1m6dCmbNm2iQ4cOWcZ5wTahLUFOmzZtWLx4MX369OHYsWO8/vrr9O7dm379+nHixIlEeXfs2EGNGjUYM2YMkydP5ty5c9SuXZvZs2dTpUoVAObOncugQYM4e/ZsRtxOqmNrYEtQU6hQIcaMGQNAkyZNKFu2LOPGjePcuXNs2bIlUd4SJUpQsGBBAPLkycPFixfp0KEDsbGxDBgwgIiICFavXk2pUqVYtGgRTZs2Te/bSXWsA1uCmj///JNBgwZx6dIlGjZsyObNm4mOjubUqVO8/vrrAERGRjJy5Eg6depEnz59WLRoERUqVCBfvnwAjB49mp49e5InTx5Gjx7N9u3b+e9//5uRt5VqSGrPRhKRm4AFQAUgj6pe8jg2FbgOGKiqS/2VVa1aNbXDSBYLiMhGVa3mmZ4W78DHgUeB770c6wcMAOo6/y0Wy1WQ6k1oVT0PnBcRb4crAc+rqorIPyKSV1X/SW0bLMFNzZo1M9qELEOyDiwiG4FZQJSqHr7K64Xq5Tb7KeAGIJEDi0hnoDNAyZIlr/JylqzGxn3HfR4rl/0kALsv5k8fY4KAQGrgh4FmwAwROQFMAJbolb08x7ltXw+c9MygquOAcWDega/gGhYv7N+/n+7du1OkSJGEDiFfgoiZM2fyzTffcOzYMRo1akS7du1SVRCxfPnyRPvV3l4c8Lm3+Tk2qvAyAHodqZVim4Kdjf0aeE1P1oFV9SwwWURigDeBycAfIvK+qk5PoR1bReQBYCtwvaqeTuH5livkp59+4rHHHqNr1660bt2aKVOmMG3aNK+CiObNm9O8eXNUlXr16tGuXburEkSkxEEtKSOQJvQTQHsgDPgE07wNAZYCSRxYRMKAxcBdwBIReQVorao9gHeAKUBOYFDq3IIlEO6//35GjBjBwoULKVGiBC+88AK9e/fmhhtuSCKIcDFkyBC6du0KkCCIqFWrFrt27WLu3Ll89tln9O3bl1y5ciU6zzps+hFIE/p+4AVV/c09UUS6e8usqrFAHY/kdc6xA0DtK7DTcpVMmjSJ3r17Ex4eTt++fcmWLZtPQYSq0rdvX+677z6efPJJAL+CiGG/5c6AO7JAYA78DbAHQEzXch1VjVHVH9PUMkuq0qBBAwYMGMCiRYs4evQof/zxB2PHjvUqiPjggw+IiYnh9OnT/PjjjwwZMgS4LIhoOGEt++YtQELDuLF2C3IUsA6cUSQr5BCRb1T1Ubf9parqWcOmCVbIEZwEaxM5i3dieRVyBFID5/TYz+U1lyXLEqwOawnMgVeIyFRgGVATWJWmFlksloAJZBipv4jUByoC01XV/hxfA9haN3MQqBZ6L0bbfEZEaqShPRaLJQUEMg48AyN53O8kKbAyLY2ypD+uGtfVEQRZryMoKxLIO/CNqlozrQ2xWCwpJ5Am9DIRuTPNLbFYLCkmkBq4NRApIq6J+aqqt6ahTZZ0wHZSZQ0C6YW+JT0MsVgsKSeQTqyCmEgaBYBngaeuYBaSJYOxNW7WJJB34CiMeOMWJ75V+zS1yGKxBExAUkpVnS8ivZx9r7FyLMGFrXGvDQKpgc+LyL0AInIX8G9yJ4jISBFZJSLve6Q/IiLrROR7EelyZSZbLBYXgdTAXYDhQBGgP9DNX2YRuRvIraoPi8hYEblHVX9wDvfBhOc5AKwFPrpiyy2JsDXutUkgvdAHgBYpKPMBTLQOnP/3Ay4H3g7kA44AZ7ydbIPaWSyBE0gv9DKMfDIBVfUXVSM/4IrecQq4w+3YF5ig7wBDvJ1sg9oFhq1xLRC4kANM51U14L5k8p/ERJyEpJEnhwPVgUNAjIjMcILmWSyWKyCQJvSfbrsHAuh8WosZL/4MExsryu1YHHBSVS+KSDwmUJ4lAGyNa/FGIE3oV7nchC4B+I0lqqqbROS8iKwCflTV9SIy2olK+Taw1HHexap66irtt1iuaQJpQh9w/ismnnOv5E5Q1ec99ns4/78CvkqZidcutta1JEcgTejJ6WGIxWJJOYE0of8AbsR0PBUFDgKXsLOSLJYMJ5Am9GJgmKruFZGSwABV7ZzGdl2T2CazJaUE4sCVVHUvgKruE5E7kjshK7N8+XL69etH5cqVqV69Ojlz5vS5SNjSpUuJjo7m9OnT3HffffTs2TNVFwmzWAJx4DUisgijpqqKGSa6ZhER8ubNy5kzZyhdujTvvvsuU6dO9bpIWJ06dahTpw6XLl2iadOm9OzZ86oWCbNYPAmkE+slEakM3Ap8oaqb09yqIObhhx8mJiaGc+fO0bBhQ9555x2/i4SNGzeOqVOn0rFjRyDxImEdPviCN+auRePjKPZIU0KyX5fet2PJ5CQ7G0lEcgAPYeJC/yQijyZzSpYmJMQ8spw5cxISEkLVqlUZN24cgwcPpkCBAknyd+7cmRUrVjBx4kTALBI2ePBgYmJiyFW8HCHZc5CzWGlO7bRLyFhSTiBN6MnAGsyiZgNFpD9mwbNrks8//5yYmBjOnj1Ly5YtiYmJITo62usiYYsWLWLdunWcP38+YZU/uLxI2NcT1nJ4zXzOHdrHjbVTMl/EYjEEsrjZUlWtIyLLVLWW52JnaUlWW9wsM/QyZ+YFwjKz7cnha3GzQCb0q4jc7PwvCsSmunUWi+WKCKQJ/QIwDTMtcCbgdWFvi8WS/vh1YBEJAXqp6iPpZI/FYkkBfh1YVeNFJExEcquq1wgaFt8Eyztv5cLZaV4+PyGSfDzC3KFPAPBOXOYb0srMtnsSr8rMn0+y5chFv/kCaUKXB/aIyDYgHqOBruvvBBEZiZn8v8l9ZpKIXAd8AJQBtrtmKVnSlvBy+ShWvAQhYTmSzVsi7B8A9sfmTWuzUp3MbLsn8bEXCI9Vthw54jefTwcWkZudeFjNUnLhZILa9cSsMXzNDkNlBPmyhyDZrNorMyHZsnN99uT7mP3lcE0jHK2qe93/kinTW1A7FzWBxiKyXEQaJ2udJdWQAJrPluBBRAIKwO7XxUVkElBVRMa5/yVTZn7gtLN9CrO2sIuywEIgHPiviCRpAYhIZxHZICIbjiTTfAg2qr29OOFv9Scvu621m3X5ftUK2jZpwDNP1qdn+1acPH6cV3p1Ze/ve9LVjvGjR3Lwr7+8Hmv9+GMpKuuDd99i/ZpVV2XP3FnTmDtr2lWVEQj+3oHDgSpAZcwwUqCcxHdQu1PAClW9ICK7MfOL3WNu2aiUaUyFcYGF6q3gI31H530J2yeOHWPsyLf5cMoscufJyx+/7SY21n+nizfi4+MTJKre9gOhU48XuMl5B76W8OnAqnoeWCsiVVU1PgVl+gtqtwaoJCKbgNKY+NCWTMqKb5bQKKI5ufOYTqPSZcslHJv+ycds3/ojD9aoyXN9+rH39z280b8PFy9e4IEatejS6yVe6dWVXLnysHfPbho82ZTV3y7l7NkzVL3vQXLlzk3Ldp34edtW5sz4lFffHJ5QdmREOHdUqsKmH77nyeateap1JK/06srgPl0pVrQovbp148Sxo9xcsjRvjvow4byJH7xPSEgILSI7MqB3d44dPUyBAoUYNmYcZ/79l16d2nDddTkJCwvjngerJ7rXTeu/Z8QbgwjLHsbTbdrzn/DG9OvxLIf/PkiRYjcxbPTHxMfH07tzWy5evMj1+fLzUE0TffnD995m/XcrkZAQ3nhvDJcuxfFKzy6E5cjOgzVq0blnnyv+DJL9mUuh86KqmzDLsawC4l1B7ZzDbwNvAt8BE1Q15T/XlqDhyKFDFC5a1Ouxex6sztR5X7Hy2xgA/jfsdYaMGM3kOYvYvfMX/j5oGl5V7rmP8TPnApA3Xz7GfvoZrTs8y8pvvgbgq/lzafBE0yTlP1q/IVPnLWHerOlcvHj5azTp02k8+EhtJs9ZxOvvjUlI/+TD/wHQrmsPoqdPoVbd+kz6/EvuebA6Xy+YR/T0KTRt+QwfTf2c8+fPJ7neyKGvMWbSdKJmL6Beoyf4ZvECyt56G1PmLqbcbbcTs3A+3361kDurVGXc9Ghy58kDwK8/b+fw338RFb2QAUPfZfzokfywZjVNW7clavYCOvXoneLn7k4gw0gpxk9Qu78Av0NQmYlgGefNKAoXLcrhQ397PXbLbaYRft11Zkz299920b/nswCcPnWKw3+b99U7KlVOOKeCs31dzpwUKFSIgwf2s3XzBnr1H5ik/PIVKxEaGsqNN5fg+NHLDbnffttDo2fMO6+rGX7m39Ms+mI20xeYvtXfdv/Kjq1b+HzqJC5cuECDJyI4sO8PHvlPPWPHnXd5vacbChZMKHffH78n5Lvjrirs2LqFkJBQyleslOi+9uz6lR/WrCYyIhyAQkWK8VjjJ/hgxDBe7taJRhFP8XDt/3i9XiAE9KIhIneISAMxFLniq1myFDUercuC6Fmc+de8e+79fQ9HXA7t0etdpuwtDP9wIlHRC/l8yQoqVr7bZHN71w2Ry9vhTzRj+JAB3Fm5qtce9J07thEXF8dfB/ZToFDhhPRyZcvy40YzASY+3jQec+e5nl6vDGLAC92Ij4+nTNlytH+uJ1HRC5mxYCnN23akeIlS7Ny+DYCft21Ncj0ROHn8eEK5JUqVZvvWLQBs/3EzJUqV4eaSpdi5I3EZpcuW48FHahEVvZCo6IW89b+PyJYtjL6Dh/LGyA8YPXxoII/aJ4EEtRsA3I15Z10MTAFS1q1nyZIUKFiILr1e5rlnnkZVyZf/Bl4fMcZr3p79/suA3t25ePEC2bKFMWrCFL9l3//wI7zSqyvPPv+i1+NLFnzBsEH9efLpVokimkS2aUmb517ky+iZlChVhjdGfgBA9ZqPcvL4cYYO6MuL/x3CoJeeZ+bkiagqL/QfRNOWz9CrUxvmz57pNUJKr/6D6Na2OWE5svN0m/bUadCImO6deebJ+hQuUowO3Xqh8fH06vQMnVo0IV++/ADcfsedFCpS1NTAIjR4oil5817P9EnjOX/uLA0jngrkUfskkOmEK1W1htt0wm+TWRsp1Qj26YT+mtDBNLXtnUeKUqxU2YDyBouaKS4ujmdbRjBh1hdJjkVGhDNh1jyyZUtc/wSL7anF33t/4+UVhwDf0wkDegcWkVDMdMIQIDRVrcxkXOvvvenByRMn6NWpDU+1bpfRpgQ9gTjwaGA1UApYBrzvP7vFcnXkv+EGomYv8Hk8KnphOloT3AQS1O5zEYkBygF7VPV42ptlsVgCIZCgdkuBJ4CfrfNa3Fm/ZhV17qlIZEQ4kRHhfPvVQlYsXcKf+/fSt7uJ/T9nxqepdr1XenXlz/3JSfG9k5p2pGZZV0sgTehIoBWwTER2ApNVdan/UyzByo7vYvwfT+b8Cg8lHrNs1LQ5z/cdkCjN3cnmzJxGkxZtkrXrSuSTKSE5O1Jy/UDvKT0IpAl9AHhbRIYD/YFFJLPEaFbCdloFztxZ04i7FMcDNUwAl2+XLGLXLzuIjAinc88XyZM3L++9OYjY2Es0bdmGJ5u3JjIinIqVq3Lk0N/0fW0o/+3dnTNn/uX/brmNgW+N4MC+P3jpuY4UKlyEs2eSxpT4MnoW0yeNJyQklA+GD+HOihVo/fhjTJ33FX/u38sHI4ZRp36jRHY8+IgZGVi/ZhWTP/4AVaVFZEd27tjOsq8Xkz1Hdt4c+SE33VyCCWNGJUpbvfybhLL6DRnG/95+g7Nnz1CgYCHe+zgqPR83ENg48J1AW4yueTmJpwcGJZs3b+bjjz/mwoULFCtWjLvuuosVK1Zw9OhRnnrqKZo1uzzF+ciRI7zwwgvkyZOHSpUq8dxzzyVa/kRVObBwPMUfa0dINrseuSdfzp7J5vVrKV6iFNUeeCjRsdr1GnDL7RUSOp06tWjCmKgZ5M6Tl45PP0F4EzMGWqd+QypXu5d3XnuVTj16U7navYx4YxBbNqxnfvQsXhr4BpXurkaTOon1yXFxcUyd+DFT5y3h8N8HGTKgN59Pj0pio6cd7sTGxjJuejRHDh9iyvixTJu/hI3r1jJhzEi69u7Luu9WJkobOOw95n8+k6johez9fQ/5CxTgwymzSG44Nq0IpM0wCFgJVFPVXo7WOaipUqUKH330EZMmTWLbtm00b96csWPHMnbsWBYuTPwhjh8/nq5du/LRRx+xdOlSYmNjOXbsGJ07d2blypUcXbeIQvfUs87rg0ZNmxMVvTDRpAFf7Nyxje6RLWjXtCFHjxzmxLGjwGUJ5Z5dvzJy6GAiI8JZt3oFhw/9xYG9f1C+YiWyZcvGreUTL8t1/NhRbrq5BGFhYRQvUYrTp08nOh6IU7nkkAf37+M2p/yKd1Vh3+97vKa5U6rM/3Hr7RV4uVsnJo/7INlrpQWBNKGTKskzAdHR0YwfP56HH34YgLfeeot58+YlBF93sXfvXkqWNFPsChcuzLFjxxItf8KavZzctoYTW1fZ5U+uAHcZZPmKlRg5fjK5cuUmNjaWsDDzo+h69yxdthyNIp5O0BFfunSJ71etYOeObdxZpSq7fkn8hl6gYCH+3L+P2NhYDv99kOuvN7NYL14wkxF+/flyfl8BDVxSzptKlEyQQW77cTMlSpfxmuZe1sULF3imczdCQkLo1PxJGjZ5ikKF01dp7C+kzqeq2kZEdgGunzIhk6wLHBERQUREBPXr1+fMmTP079+fPn360KBBA/7zn8sdMSVLlmT//v2UKFGCI0eOULBgQTp06EBsbCwDBgwgV/Fy/PvHdrLnL8KpnRu44c7qfq4a/Hh2QnmS2mqmO6vcTY92LYns0p1uL/ane2QL4uPjyZf/Bt6fkLg3t3PPPgx+6Xn++ec0IRLCa+++T2SXHrzcrSMFCxWmYOHCifKHhobSsl0nnnmyPiESwuh3XgOMRrv1449R6e6qXu2oet+DSewsXKQo9z70MK0a1SUsexhD3x/rNQ2g2E3Feb5jG57r3Zc3XnmJuLhL3FyyNAULFU5SbloTiJQyxH1KoYiIJnOSr6B2rvOBLZhQPRP8lXOlUsrFixezePFiYmNjKVCgAMWKFWPXrl38888/1K1blxYtWiQsfxIbG0ufPn3Ily8ft99+O1FnjOTw0Ko53FCpBqE5crJv3odIaBg31m5BjgLep895YqWU6U9mtt0bqSWlXAq4a59nAT4V2MkEtQNoDBwO5AaulPr161O/fn2/eaKiohK2P/30ck0Q5fQ6F324SUJamadfSl0DLZZUwl8TuiJwF1BURFo6yWHATcmU6S2onbsDt8D8CFgslqvEXy90QYx8Mhdwi/N3E9ArmTLz4yOonYjUA1YAl3ydnJmD2lks6Y2/mFgrgBUi8j9VTbpytW9O4juoXUegDdDcz3VtULs05KmJ/qMtZpc4AC6q90lnn3V4OGF7/ZpVrF21IokS60oYM3woa1Yu49U3hyeK0pEVeKVXV7r16UfxEqVSvexA3oFvFZGhwM2YXmiS6YX2F9TuFuALoDimP2u1qv6ScrNTD6u0Cg7WfbeK6V8mlnmmtbwyKxDI03kf6AYcBB4HpvvL7C+onapWVtXHgBHAqIx2XkvqM2HMKFo1rke7Zo04eGA/K5YuYdon4zh39iyVSxfh5IkTzJnxKV/Nn5twzrRPxvHrz9uJjAg3MsWmDenV6Rm+mDWN5TFf8cyT9WnVqC6rlpmulc+mRtGiYR3ee3NwQqwp99jPrrStmzYQGRFOq8b1mDtzasKxkUNf46n6tYiebqKCbFr/Pa0a1yOyaUMWz5vDkH69E8acp4z/kK8Xzkt0jzMnT6RFwzpENm3I77t3cfDAfto1a0SrxvWYMGYUAAf2/UGLhnXo0a4lhw4eBOD8uXO82LUD7Zo1os+z7YiNjWXp4i95ukFt2jVrlBDILyUEUgOfVdVfnOGkn0Uk6SCaB76C2rntR6XMTEtm4MjhQ0mkh8/3G8jCubO55fby3PNAdX7cuJ4tG3/gud59E85r1b4zi+fNISp6IX/u38vxo0eZOGseIkL7Zo345PMviY+Pp0vrpjzwcE3mzPiUqfOW8NPmjWzd9INPe0YPH+pVuvlYoyfo1qcfHZs/QUTLZxIiTt5QsCDx8fEUKXYji+bN4fnbK7Dq26WMmXS5zjp29AhLFnzB1HlLCA0NJT4+njdffZnuL/an6n0P0qlFExo3fZpPxo5OIgF1RcMMf7IpMydP5OsF81j5zde89/EkipcodUVyzEAc+KKI5AR+FZGxQL4UX8VyTeApPRz73tvky5+fkyeOs2XDejo89zzrvlvJ3wf/pNhNxX2Wc1uFioSGhnLs6BH27PqVjk8/DsCxo0c54cgns2XL5vVd2d0JXNJNgBPHjydIN8vdXoGwsLBEzXP3iJN333s/H44Yxp/791K4aFFyXHdZfffnvr1UqHgXoaGhCfn37/2d8o4ks3zFShzYt9erBNRbNMxnn3+Rj0e9y6W4Szzb80VK/V9g4/UuApFSPgYgIj0wwewGp+gKlmsGX9LDYjcVZ93qlXTq0ZupEz8i/w0F/JbjcqwbChTkltsrMG7GHEJDQ4mNjUVEOHhgP3FxcYmiR6oqFy5cYNfuy3plX9JNT1mlK+Jk/gIFEt6776xSlXdfH5hk2uDNpUrz8/atCfni4+O5uVRpdmzdQrX7H+LnbVtp1b5zQoRKdwlombLluL96DeqGmx+k2NhY4i5dYsiI0Wz+YR2Tx33AwGHvpeiZ+xsH9tZUPoxZ3+hQiq4SRLh3Wo0qvIxRhYNDLZVeuPcieyOlaqaFcz5j60bTjH2214tepYdVqt3L8aNHEBFy5c7DXXcnERR5JSQkhLbPdqPDU41BhLK33M5/33qXJ59uRavGdal2/+XZT0883ZK6DZtQp3bNhLTkpJsuPCNO1n+8CQ2bNCMyoiHDP5yYKG+BgoX4T4PGtGpclxzXXcegYSPp2O15Xnm+K7EXY6lZ9zGK3niTVwlos1aRSaJhxiz6kq2bfuDsmTO8NOiNgJ6LOz6llM7CZt5QVW2f4itdAWkRldLTgSFtHNhKKdOHyIjwhGmCqWn77l9/YWbURAYMHZ585jTiqqSUqmpDAlquSTauW8OINwYxdNTYjDYlWQKZ0D+Jy7ORAEivGtiSeqhqllwjOC0iVFa978EkY9LpjaoSSJ90IL3QrhlDAlTFvANbMhGnLsZT5NJFJCxHRptiCRC9dJHTF5NfVzCQXujv3HZXi8iXV2OYJf1ZuPsUzcP2ExJADXwh1EyGPxGX+QIXZGbbPYlXZeHuU8nmC6QJ3dJttwSQbrOWL168SPv27fnpp5/44YcfmDlzps/YVlFRUUybNo2yZcvy+OOPU79+fXr27MmM9b9RqGodcpW47ZqNbbXlyEW2HAlsBqer8+3lIOh8SymZ2fYrJRAppWsmUjngDGY+b7qQPXt2PvnkEwo6g+z+YluJCLlz5+bcuXOUKmVE48eOHaPwvfX55/dtNraVJUsSSBP6NUiIpJHhvSC+Ylu1adOGtm3bcvToUdq3b8/8+fOpXbs2306LIe//VeLCsYM2tpUlyxFIE7ov0Ak4ixMTC6iUxnb5xFdsK5d6J1++fMTGxgLQoUMHPjxUhD+/nsINFR/K1LGt+pVNGhM5LSh5MS5dr5eaZGbbk6OZj/RAeqGbAbepalwq2hMQcXFxdOnShR07dtClSxfKly/Pb7/9xj///EOHDh0AEmJbzZgxg61bt3L8+HF69Lg8d+LwmvkUebAxoTlycnjNfM4d2seNtVuk960ERFb84lnSlkAceBtQAEj38BihoaF89NFHfvO4Yls999xzgFFaDfwJBv4UfLGtrINaUptAHPhrYJ+IHHD2kw0r6ysqpYgMwkyIABigqt9cgc2ZhpLXxVmntaQpgThwP6C0qgY0gSGZqJRTVPU1EckPzAeylAO7O6vrfcxiSUsCceANQEqWFfUZlVJVf3fSL4B3pZiIdAY6AwkrJgQrtna1ZDSBjAPfAewRkRjnL7m4H/nxEZXSjcHAx95OVtVxqlpNVasVLpxumhGLJVMSSA3sM4i7D07iOyolIvIkUFBV/cbWClaCtdZtur1DqpRzrriZL9x0l99FM4KSzGz7lRKIA6c0UI/PqJQiUgkTIC88hWX6JS0jS6anw6aWE1quHQJx4IkYJw4BygP78bNGsKpuEhFXVMofXVEpncB2w4GiwBIROaWqj1/9LaQuaemw1kEtqU0gUsoEuZOI5ALGBHCO16iUqlrvCmzMtBQ+u/Oaas5Z0p9AamB3YjHrJWUZrmas1l+N6nofs1jSkkC00K71gV2TGTJmKXI/pGf8KdsMtgQTgTShb0kPQ9KLlIotrMNaghl/YWV7AdtVNcYtrQ5QUVVHpb1pvvHsdR6VisPF1mEtmQl/NXBTT0dV1aUiMhgY5e2EYCRYx20tltTAnwNfTGF6psT2FFsyM/4c+LyIlFbVP1wJIlIGo2MOWpKrcd2byLan2JLZ8efA/YB5IjIfI94ohVFQtfFzjsViSUf8rcywVUSqAw0x0Si3AcNU9Z/0Mi5QSl7nO5SK7ZSyZGX8DiM5zjojnWzxSUq0ztZhLdcSKVViBQVJatos1a1msQROpnRgTwqf3QlcW9PILBbIJA7sWeN6NpNtb7LlWiWQiBwpRkRGisgqEXnfI/0mEflWRNY4qi6LxXIVpHoNnExQu37AAGArsIDLsbP8YjumLBbviGpKA24kU6BIN+CIqn4mIhHATao62jm2HKilquqsctjSc1jKPagdcBuwEygEHE1VQ9MHa3f6k1ltT87uUqqaRPWfFu/A+YHfnO1TmKB4LkL18i+GK+BdIgdW1XHAOPc0EdmgqtXSwNY0xdqd/mRW26/U7rR4Bz6J76B27vP3kgS8s1gsKSMtHHgt8KizXQf43u3YVhF5QERyA9er6ukkZ1ssloBJdQdW1U2YiRCrgHhXUDvn8DvAm5jOq6EpKHZc8lmCEmt3+pNZbb8iu1O9E8tisaQfaTIObLFY0gfrwBZLJiYoHVhE6ovILyKy2i0tm4h8KiKrRaSfW7pX1VdGIyKRIrJTRJaLyDtOmtd7CCaC9Xl6Q0RKi8gh5xl/7aS95DzfaSISltE2uuMoEV0LH2Rz0pLYKyKtHLXiAhG53l+ZQenAmJ5rz/jTjYGfVbU6UF1EirmrvoDsInJPehuaDMNVtaaqvuzsJ7mHDLQtCZngeXojxnnGdUWkMEYoVB2j9nsiY01LwnHMCM33AN7sdZy4C1AD+BSzTJFPgtKBVfWEqnqG7nFftnQZcA/elzINJnqJyEoRcQ2rebuHYCLYn6c3ajkthheAe4HlTnrQ2a+q51X1hFuSN3tvBX5S1UsEcA9B6cA+yE/SZUu9pQULXwCVgAjgXREJJbjtheC3z5O/MF/4WhjNQTUyl/35ucrvdIZOJ3SakDM9kv9W1eZesp8kscJrN5AXP0uZpgcB3MMREfkVs6jbSZLeQzBxkgx+ninBaaVdABCRBZgvfnHncNDbj7HP096TpOAzyNAaWFX/dt5f3P+8OS8kVnjVAn7Av+orXfB1D67OBxHJCdwCHMH7PQQTGf48U4KI5HXbfQjzg/iIsx/09mM+f097fwUqOi22ZO8hKJvQIlJNRJZibmSpiFwHfOnsrwbWqupf3lRfGWm3By+IyFrMO84wVY3Fyz1kpIGeBPnz9MbDIrJRRNYAB1V1HbDSeb6VMa8xQYOIhDnf67uAJUAZPOx1vifjgVVAW+Bjv2VaJZbFknkJyhrYYrEEhnVgiyUTYx3YYsnEWAe2WDIxWdqBHa3sCUcru0FEWqbw/GnO/8oiUsMtfZQjg7sSm5Y5dqmIPO6WnmpjwiJSU0Qque1PS62y0xsRGSwirTPYhuoiEuVs9xOROzPSHneytAM7bFTVmkBtYKhLRB4IqtrK2ayM0aa60nup6pGUGuJ88Nuc3V+A/iIiKS0nAGpiVGBAovuwXCWqOkxVf8poO1xcCw4MgBO+52+goIh87MwAWSMi9wKIyLsistapIZ920ly1Ym+gg1OTF3f+3ywiI0SksZM3p4j8KIZHRGSFk+8jNydtDMx3tv8ENgEJtbBTTj4R+UxEvhETQ7uck/60U360iCwRkZpO+hLnOuvFhCsqAEQCrzrpoa77cM69y9kuISLfONvNHD3xahEZ6PnsvNkkIgWdaxYRkQqO5ju7iLR0nuFaEZnguncR2Ssio0Vks4g877Ri1onIB87xms69RIvIFhFp5sWOJHY65613rjnJyznLReQdx+6ZItLZuY9VIpLLydPD2V8rIh2dtBudz/Ar3FbkFJEoMYv+JXn2TtpgEZkoIvOd+7jd06ZURVWz7B9QGljqbBfHRMtsAnzipP0fsN7Z3g5kc7ZDnP+7nf+RwAC3cpcDN2NquWgnrTkwBBBgM5DPSR8JNHS2vwbCXHY5ZXzvnOO61jCgubN9FzAbCMUodPI6528Hajp5cjv/ywPfOtuDgdZu9rrKfhwY4Wz3A9pjtLbfA2FO+lzgTo/nmMQmZ7sGsBj4Dijnbo+zPQuo4WyfA4oAOTDywLud9M1AAUyrYZtzf9c79xviuhdfdgL/A+q6f24eti8HGjvbXwEvONujgCed57bQ+QxCgTVAQeB9oIWT91UgytmOAqon8+xHOdstgXfT8jueKZZWuUqqisgyQDHxpqthPiRUdY+IuMTi/YBPRCQeGI5xEr+oWYL1ZqfWaw30wsT3LY1ZWxkgD7BTjGb6mKrGuipkVT0gIhtJPO3tTuAREeni7F9yyjykTgxtEdns/M8JvC8it2EifhbHP4uA18XI9CIwrxW3Y9Z+jnHsyu/suzcTvdmEqq4UkbeArarqaq3UEJGXMM5QisstjoOqetix+yjGccG0RFyfwWY1SqRYETkMuPczlPNh53Cgr4i0Bb4FJnq5b9e1DgBb3LYLABWBCpjZYWB+PEpgJkn8z0lbh5HDJpDMs9/o/N8H/MeLPanGteDAG1U1YRkXMfrZxsAEEfk/4KTTzFuqql86zaMhmC+4i4v4flazgB5AHlXd7ZS1B1Pr/utcMwxTiy/wcv5bmFrWxXaMzHKuc252zBekqIjkAc5j3skBHgPi1KyCUYHLzuLVXufHYznQH/hVVf8RkT0YDXEdVb0kIiGY2sgdbzYhIh2A9Rh5aDVV3YCprR9T1b9EZJZbWYkkf6qJJICuPJXF9FHkxEz+cA907svOHKra3Xnuv4rI55o02qn62BbgZ4yDR6iqikiY85x2YX7sf8P7tE9fz97bNdKMa8GBPZkPhIvRn4ZinC8bsNj5Zb8O48DufAd0F5GKQHePY9OAvcDzYL6YItIbmO98qeKBF4D6QJI1YpxaeAPmCwEmaudHItID8+EvUNURIjIYWA38DhzGOOlaTEfYUsdGFzHAKBFpCDzlccnJmBol3Ln+MREZBXwrInFALPAMpr/ARRKbRGQh5kfpUUzTOFrMeldTMLXkL573GgAHgc8xGuEBqhrn1lrxZWdHEamLaW7HeHFev6jqNuf5rXDKPSemX+NtYIaItMd8vp74evbpitVCpxMiEqmqUVdxvqtmCMM00eqq6t/JnZdZENMp11pVO2awKZmKa7EGzhCuxnkdIkWkFeYdbUpWcl7LlWNrYIslE3PNjANbLFkR68DphCM4mJAK5fiVXIpIT7ftys6QTqogRphR09l2yUxzOiKJZSJSUkTGiBF1NE6t614tIpJXRGY723VF5AcRmev0ZCMib4nIHW75J4nIzRllb0qwDnyFOGOpwVh2ggOr6hZVHZ4KJiVBE8tM96tqLVXdh+lcq6Gq832fbUjLZ+hBZ2C6s90NI2g5CtzljOOiqu7j/uOAPulk21VhO7FSgFP79McET/tNRLYCnTBDT9uBTs4w0l5MOJf7gNWq+qJbGYIZ+z2jqq+7pZfGDKH8ghEyjMSouEIwX7a2qnrOLX8tYCDmMzwOPI1RmRV3xnpjMMMbrYG+wEJVvd8591WMgGIaMBYoi1FA9VaPMDpiJI2vYsZD87il78YswD4WM0a9ADPEVcK5fkegCuYHRYCvVXWIl2e4GDNsp869d8UIND7DKLMqYzrtRokR3UzACFvigRYYhdd4jHpKgM5uohIXEVwWVJxxPq+cznZ/3H70HL4nmVA2QUNayryy2h9G7vcTl+V8vmSD54FimC/UL5ie45qYMdhJQAcvZZfGBL673tlfCZR0tp8Hujvbu71c+23gGffjbvZOcLbn4EgkMcNQeTABxPs5aUWB7zxs8pRw7uCyhHO35zU80n1JHxOeIT5kp86zOAjkwjjb787xdzAO6rpWCD5knh73sdNt+3bMD9crGAVcK8w4/fvAPW751gE3ZPR3Lrk/WwOnnA1q5H7gWzb4pzrDPCJygMtSwdqYWuUTH2Vv08tChDuAKW7ikqUeee8QkTcw2uKiXI4l7IspwDMi8hmwQ1X/FTM76kERcYlI8nmc4ynh3JTMNdzxJX38F+cZipmSWRoP2SnmGf2sqmed67oWhq+IqW0BUNV45x6SyDx9oaq/AK3EBEr8CLPM7QuYH8kvgAZOVsFDPRaMWAdOOXFu2wHJBt3Sl2Bqn3Ei8qyqxvspextGTP8XXJYvuvEqMEhV14pZe8l1jUsiEuKl7IWYpmpujDODafbvVtWRPq5xFO8SzkDwJX182O0+j+Jddloc786zDVOD73LyhuBD5unBMRHJrapn3NJ6AyMwzyOHY1sBpwwBcqrqyRTcb4ZgO7GuDpds0DVjKFlU9X3MNMJPkunE6QZEOT2833I5frCLmcBEEZmLkTK6mA0sdO+Ndq4bC6zA1DDfOMnjgducHuRlGMmk+zlxmPfs1Zj38z8DuUfn3GOYGT/fOmV/ReLJCahpq7pkp8vETG8s76fYt4AGYqb5fevc95vAU66ecIw01pM5QD3XjoiUxGjXf8JMbsjv3OOHTpYHuPyMghor5LBkecQE2Z+kqhHJZjb5JwEDVXV/2lp29VgHtlgyMbYJbbFkYqwDWyyZGOvAFksmxjqwxZKJsQ5ssWRirANbLJmY/wddRBwhYRpCNgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 251.059x155.163 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "infrequent_codes = oh_diagnoses_df.columns[infrequent_codes_mask(oh_diagnoses_df)]\n",
    "infrequent_codes = infrequent_codes.intersection(code_subset)\n",
    "filtered_roots = oh_diagnoses_df.columns[sagi_filtered_out_roots_mask(oh_diagnoses_df)]\n",
    "filtered_roots = filtered_roots.intersection(code_subset).difference(infrequent_codes)\n",
    "sagi_codes = filter_Sagi_diagnoses(oh_diagnoses_df=oh_diagnoses_df)\n",
    "plt.hist([rel_rank_ratios[rel_rank_ratios.index.difference(sagi_codes)]*100.0,\n",
    "        rel_rank_ratios[filtered_roots]*100.0,\n",
    "        rel_rank_ratios[infrequent_codes]*100.0],\n",
    "        label=[\"Cherry picked codes\",\"Filtered out roots\",\"Low frequency codes\"],\n",
    "        bins=nbins,density=True,\n",
    "        cumulative=True,stacked=True,\n",
    "        #alpha=.7,\n",
    "        color=[\"#FF800E\",\"#ABABAB\",'#006ba4cc']);\n",
    "for thresh,vspace in zip([-.9,-.75,-.5,.0,.5],[.09,.09,.08,.055,.03]):\n",
    "    perc = np.sum(rel_rank_ratios<thresh)/(~np.isnan(rel_rank_ratios)).sum()\n",
    "    plt.vlines(thresh*100,0,perc,colors=\"sandybrown\")\n",
    "    #plt.hlines()\n",
    "    plt.text(thresh*100, perc+vspace, f'{perc:.1%}',\n",
    "             ha='center',\n",
    "             va='bottom',\n",
    "             multialignment='center',\n",
    "             fontsize=7.5)\n",
    "plt.hlines(1.0,25,110,colors='black')\n",
    "plt.xlabel('Positive/Negative examples median\\n rank relative difference (%)')\n",
    "plt.ylabel('Cumulative frequency')\n",
    "plt.ylim(0,1.1)\n",
    "plt.xlim(-102,108)\n",
    "plt.legend(frameon=True,\n",
    "           #loc='center right',\n",
    "           loc=(.42,.26),\n",
    "           markerscale=.5)\n",
    "plt.savefig(plots_dir/\"posneg_examples_ranks_ratio_uF1.pdf\",\n",
    "            bbox_inches = \"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "54311a7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max filtered out rank ratios: -0.09422852323494135\n",
      "Max cherry picked rank ratios: -0.048442562963633105\n"
     ]
    }
   ],
   "source": [
    "print(\"Max filtered out rank ratios: \" + str((rel_rank_ratios[filtered_roots]).max()))\n",
    "print(\"Max cherry picked rank ratios: \" + str((rel_rank_ratios[rel_rank_ratios.index.difference(sagi_codes)]).max()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "5b52c783",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8365440747227088"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compute fraction of codes >=100 occurrences with psitive ranks more than halved\n",
    "((rel_rank_ratios[filtered_roots]<=-.5).sum() + (rel_rank_ratios[rel_rank_ratios.index.difference(sagi_codes)]<=-.5).sum())/float(len(filtered_roots)+len(rel_rank_ratios.index.difference(sagi_codes)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "640e0a52",
   "metadata": {},
   "source": [
    "### Fig. S6  Ranks vs code frequency distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "a5581476",
   "metadata": {},
   "outputs": [],
   "source": [
    "# lowess with local linear regression and CI\n",
    "# code from: https://james-brennan.github.io/posts/lowess_conf/\n",
    "def lowess(x, y, f=1./3.):\n",
    "    \"\"\"\n",
    "    Basic LOWESS smoother with uncertainty. \n",
    "    Note:\n",
    "        - Not robust (so no iteration) and\n",
    "             only normally distributed errors. \n",
    "        - No higher order polynomials d=1 \n",
    "            so linear smoother.\n",
    "    \"\"\"\n",
    "    # get some paras\n",
    "    xwidth = f*(x.max()-x.min()) # effective width after reduction factor\n",
    "    N = len(x) # number of obs\n",
    "    # Don't assume the data is sorted\n",
    "    order = np.argsort(x)\n",
    "    # storage\n",
    "    y_sm = np.zeros_like(y)\n",
    "    y_stderr = np.zeros_like(y)\n",
    "    # define the weigthing function -- clipping too!\n",
    "    tricube = lambda d : np.clip((1- np.abs(d)**3)**3, 0, 1)\n",
    "    # run the regression for each observation i\n",
    "    for i in range(N):\n",
    "        dist = np.abs((x[order][i]-x[order]))/xwidth\n",
    "        w = tricube(dist)\n",
    "        # form linear system with the weights\n",
    "        A = np.stack([w, x[order]*w]).T\n",
    "        b = w * y[order]\n",
    "        ATA = A.T.dot(A)\n",
    "        ATb = A.T.dot(b)\n",
    "        # solve the syste\n",
    "        sol = np.linalg.solve(ATA, ATb)\n",
    "        # predict for the observation only\n",
    "        yest = A[i].dot(sol)# equiv of A.dot(yest) just for k\n",
    "        place = order[i]\n",
    "        y_sm[place]=yest \n",
    "        sigma2 = (np.sum((A.dot(sol) -y [order])**2)/N )\n",
    "        # Calculate the standard error\n",
    "        y_stderr[place] = np.sqrt(sigma2 * \n",
    "                                A[i].dot(np.linalg.inv(ATA)\n",
    "                                                    ).dot(A[i]))\n",
    "    return y_sm, y_stderr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8457b626",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Correlation between positive examples ranks and code frequency\n",
    "import scipy\n",
    "code_subset = icd9_allcodes\n",
    "total_ranks = False\n",
    "predictions = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_NDCG/drugSexAge_AllCodes/best_models/1/\"\n",
    "                              +\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "for pred,label in zip([predictions],['Model']):\n",
    "    if total_ranks:\n",
    "        pred_ranks = compute_predictions_ranks(pred)\n",
    "    else:\n",
    "        pred_ranks = compute_predictions_ranks(pred.loc[:,code_subset])\n",
    "    pos_examples_median = np.nanmedian(np.where(test_oh_diagnoses_df.loc[:,code_subset].to_numpy()==1,pred_ranks.loc[:,code_subset].to_numpy(),np.nan), axis=0)\n",
    "    neg_examples_median = np.nanmedian(np.where(test_oh_diagnoses_df.loc[:,code_subset].to_numpy()!=1,pred_ranks.loc[:,code_subset].to_numpy(),np.nan), axis=0)\n",
    "    for subset,subset_name,col in zip([pos_examples_median,neg_examples_median],\n",
    "                                      ['Positive examples','Negative examples'],\n",
    "                                      [\"tab:orange\",\"tab:blue\"]):\n",
    "        x = test_oh_diagnoses_df.loc[:,code_subset].sum(axis=0)/float(test_oh_diagnoses_df.shape[0])\n",
    "        y = subset\n",
    "        mask = ~np.isnan(y) & test_oh_diagnoses_df.loc[:,code_subset].any(axis=0)\n",
    "        x = x[mask]\n",
    "        y = y[mask]\n",
    "        y_sm, y_std = lowess(np.log10(x), y, f=1./3.)\n",
    "        # plt.scatter(x,\n",
    "        #             y,\n",
    "        #             label='rho='+str(scipy.stats.spearmanr(x,y,nan_policy='omit').correlation)[0:6]\n",
    "        #            )\n",
    "        order = np.argsort(x)\n",
    "        #x = np.argsort(np.argsort(-x)) #rank transform\n",
    "        plt.plot(x[order], y_sm[order], label=subset_name,color=col)\n",
    "        plt.fill_between(x[order], y_sm[order] - y_std[order],\n",
    "                    y_sm[order] + y_std[order], alpha=0.3,color=col)\n",
    "        plt.scatter(x, y, s=.6,marker=\".\",\n",
    "                    color=col,\n",
    "                    #alpha=.4,\n",
    "                    )\n",
    "#plt.plot([1,pred_ranks.shape[1]],[1,pred_ranks.shape[1]],label=\"Dummy\",color=\"green\")\n",
    "plt.plot(x[order],np.argsort(np.argsort(-x[order])),label=\"Dummy\",color=\"tab:green\")\n",
    "plt.xscale('log')\n",
    "plt.yscale('log')\n",
    "plt.xlabel('Code Frequency')\n",
    "plt.ylabel('Code median rank')\n",
    "plt.legend(frameon=False)\n",
    "plt.ylim(1,pred_ranks.shape[1])\n",
    "#plt.ylim(1,600)\n",
    "#plt.xlim(1,pred_ranks.shape[1])\n",
    "plt.savefig(plots_dir/\"posneg_examples_ranks_at_frequency.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "413a7a4d",
   "metadata": {},
   "source": [
    "### Fig. 2a Entropy reduction distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "b63182fd-da26-4329-9b84-ed1dbd40ea90",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute mutual information between predicted score and output, compute entropy reduction\n",
    "from sklearn.feature_selection import mutual_info_classif\n",
    "from hot_encoding import _entropy as entropy\n",
    "from project_miscellaneous import filter_Sagi_diagnoses, infrequent_codes_mask, sagi_filtered_out_roots_mask\n",
    "oh_diagnoses_df = pd.read_parquet(datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "code_subset = icd9_allcodes\n",
    "entropy_df = pd.DataFrame(index=code_subset)\n",
    "for crossval_suff in ['','_1','_2']:\n",
    "    test_visits = pd.read_csv(datadir/(\"test_visits_id\"+crossval_suff+\".csv.gz\")).squeeze()\n",
    "    test_oh_diagnoses_df = oh_diagnoses_df.loc[test_visits,:]\n",
    "    predictions = pd.read_csv(\"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/overall_micro_F1/drugSexAge_AllCodes\"+crossval_suff+\"/best_models/1/\"\n",
    "                                +\"predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "\n",
    "    mi = []\n",
    "    for col in code_subset:\n",
    "        mi.append(mutual_info_classif(X=predictions.loc[:,col].to_numpy().reshape((-1,1)),\n",
    "                                      y=test_oh_diagnoses_df.loc[:,col],\n",
    "                                      discrete_features=False,\n",
    "                                      copy=True)[0])\n",
    "        # Set to NaN for each column with no or all positive example in the test set\n",
    "    s0_col = (test_oh_diagnoses_df.all() | ~test_oh_diagnoses_df.any()).to_numpy()\n",
    "    entropy_df['mi'+crossval_suff] = mi\n",
    "    entropy_df.loc[s0_col,'mi'+crossval_suff] = np.nan\n",
    "entropy_df.rename(columns={'mi':'mi_0'})\n",
    "entropy_df['mean_mi'] = entropy_df.aggregate(np.nanmean,axis=1)\n",
    "s = []\n",
    "for col in code_subset:\n",
    "    s.append(entropy(oh_diagnoses_df.loc[:,col].sum()/float(oh_diagnoses_df.shape[0]),logbase=np.exp(1)))\n",
    "entropy_df['entropy'] = s\n",
    "infrequent_codes = oh_diagnoses_df.columns[infrequent_codes_mask(oh_diagnoses_df)]\n",
    "infrequent_codes = infrequent_codes.intersection(code_subset)\n",
    "filtered_roots = oh_diagnoses_df.columns[sagi_filtered_out_roots_mask(oh_diagnoses_df)]\n",
    "filtered_roots = filtered_roots.intersection(code_subset).difference(infrequent_codes)\n",
    "sagi_codes = filter_Sagi_diagnoses(oh_diagnoses_df=oh_diagnoses_df)\n",
    "del oh_diagnoses_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "6f97e655",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of codes with estimate >1\n",
    "(entropy_df.mean_mi/entropy_df.entropy > 1.0).sum()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "b840ece6-fe8b-4ca0-9b40-f3ab0c5fa3bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAACjCAYAAACnmNM/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAggklEQVR4nO2de3hU1dX/PyshQYrRCJEoagAFFRGhXLSIBYSK3JygEPpCgkQLKAQERC5VeUVafxXQRy6t8IJSkTsoSqBEQFAkFRVQuVYpDRKxiCColAoJZP3+OGeGyWUmJ8lMMjPsz/PMM+ey9z7fmWTNvq29tqgqBoMhMomqagEGgyF4GAM3GCIYY+AGQwRjDNxgiGCMgRsMEYwxcIMhggm4gYtIXRH5VETOiEi1Eu5tEpEPReQ3gX62wWAojAR6HlxELgFqAG8Bv1HVc173ZgBLgV3AGlXtENCHGwyGQgS8BlfVM6p60sft24Ctqvof4JSIxAX6+QaD4QKV3QeP1gtNhh+BKyr5+QbDRUW10pMElPNex5cBPxRNICKDgcEANWvWbHnzzTdXjjJD6PPvz8qd9WSN+oHTUQXk5OQcV9Ury5qvsg18l4i0weqDX6aqPxVNoKpzgDkArVq10u3bt1eyREOl8+zlDhNeWmqKN5q8WjEtIUpKSsqh8uQLuIGLSAyQBTQD1onIk0Caqg4HpgCvYw3CPRPoZxsqhgxbFJBy9MqhFcofqUZaFQTcwFU1Hyg6Bfaxfe8w0DHQz7yYCZRRVgSnBm0Mt/Kp7Ca6wSHhZLi+MAZd9RgDr2JCwZDdGIOOPIyBVxKhZMhuymrQxoDDD2PgQSKcDdoYcuRgDDxAhKJBuynNsI1BRy6lGriI7ACWAa+p6nfBlxRehKphO6mtjWFHPk5q8F8DKcASETkJvAKs04s0WmOoGrSbig6UGSKLUg1cVf8LzBeRDcBzwHzgKxGZrqqLgy0wVIgEwzY19sWHkyZ6T+BhIAaYh+UnHgW8C0S8gYerYRtjNoCzJvqvgFGq+i/viyIyLDiSQgNj2IZIwImBbwRyAEREsII4bFDVnUFVVkWEq2EbDCXhxMDHq+oGAFVVERkHbAiurMonXA3b1NgGfzgx8BpFzn8RDCFVRagbtsFQEZwY+GYRWQi8B3QAtgRVUSUSDsZtam5DRXAyTfZ7EekK3AosVtWs4MsKLuFg2L4whm0oC05dVQ8B/8EaZ2unqh8EUZOB4jW3MWxDeXAyD74EKzji1/YlBfwauIi8BLQCPlXVEV7X22NFdVEs19fZ5dRdLsK55jYYyoOTGvzqssQvF5EWQE1V/bWIzBKR1qq6zb49Gsvt9TCwFagUAw8nwzbTYIZA4sTA3xORpqq622GZbbC83LDffwW4DXwvcDlwDDhdFqGRjhlMMwQDJwaeBqSLiHuHElXVG/2kjwfcXm8/Ak287r0NrLGPJ5WU2TtsclJSkgN5vgmnmttgCAZORtEblbHMH7BinkPx2OdTgbuAo8AGEVliL2bxfl6hsMllfHbYYWpuQzApdWcTEaktIlNF5FURqSYi/UrJshXoZB//BvjI69554AdVzQMKsBawBBwZtsjU3gYDzprorwFzgSdU9ZyIPIyfVWSq6t5ZdAuwU1U/EZGZdlz0ycC7IlIAZKnqjwH4DGGJqbkNlYEjV1VVzRSRkfa5lJbBe2rMPh9uv78DvFNWkU4xtbbBUBgnBn5GRG4HEJFmWA4vhnJiam5DZeJkd9FHgVFAHeD3QEZQFRkMhoDhZBT9MNC3ErSUm3Bumpua2xBMnLiqvoflWupBVc3+YmXEeKgZqgKnji5gDa61Au4InpyyYWpug8E/Tpro33idHhaRR4OoJ+Iwq8IMVYmTJvpTXGiiXwfEBlWRwWAIGE6a6IftdwV2ASODpsYh4dw0NxgqEydN9PmVISTSME1zQyjgpIn+FXA11gKRRODfwDlKX1VmMBiqGCeOLlnAjaqaBDQCNqhqI2PcBkPo46QPfpuqHgJQ1VwRaVJahmBh+t4GQ9lwYuAfisharKgsLbGWgxocYvrehqrEySDbGBFpDtwIvK2qnwVdVRhjPNYMoYSTQbbqQFusAbaVItJJVTcGXZm3BtM0NxjKhZNBtvlANNamg+ewVpT5RUReEpEtIjK9yPVL7Mgwm0RkZvkkhwdvNHnVNM8NVY4TA09Q1RnAWfvcb8AH77DJQKyItPa6/RjW7igd3UEgDAZD8HBi4Coi19rviUB+KelLCpvspgPgEpH3RcRVVrGhjF451PS/DSGHk1H0UcAirPDHS4FhpaSPx3fY5BuAacCTwPsistZu9pfIjtwTYdf/Ns1yQyjh18BFJAoYqarty1DmD/gOm/wjsFlVz4rIAayBO+/VaoXionNp7TI81mAwFMVvE11VC4AYEalZhjL9hU3+ELhNRKKB+lg7nBR95hxVbaWqrahxWdHbIYdpmhtCGSdN9MZAjojswYplrqra2VdiB2GT52PV7HPt+OgGgyFI+DRwEbnWjseWUtZC/YRNPgL4/HEIZ0zf2xCK+Guiu5eJzlTVQ96vyhBmMBgqTmmDbH8FWorIHO/rqjo4qKrCANPvNoQD/gy8O/BLoDnWNJmhBEzT3BDK+DRwVT0DbBWRlvZousFgCDOcrCYzxu2Fu2luam5DOODEVdVgMIQpjgxcRJqISDexqBNsUQaDITA4WQ/+NNACy/MsC3gd6BJcWQaDIRA4qcE7q+oDwI+qqpiNDwyGsMGJqyq277jai0+igyspNDGDa4ZwxImBzwSygXrAe8B0/8kNBkOo4GSabIWIbAAaAjmqeiL4skIPU3MbwpFS++Ai8i7QE/jHxWjcZjmoIZxx0kRPB1KB90TkS2C+qr7rP0vkYGpuQzhTag2uqodVdTJWbLUvgLVBV2UwGAKCkyZ6UxF5AfgUuJLCQRR95SkxbLJ9T0Rkp4gMLJdig8HgGCdN9GewnFvG+wuQ6MY7bLKIzBKR1qq6zSuJC/iufHIrjxUdqle1BIOhwjgZRe9dxjJLCpvsbeB9gWVlLNNgMJQDn010EVlgv/9TRPbbr3+KyP5SyowHfrKPfwSu8CrzXmAz1v7iBoMhyPhbD97fPrzJe8moiPjd2QT/YZMHAv2B//GV2YRNNhgChxNf9KJTYqU1r/2FTW4EvA2MBkaKyM1FM1d12OQVHaqb/rchYvAXVfVWoBmQKCL97MsxQF1/BfoLm6yqze2y04FqqvpFID6EwWAoGX+DbLWx3FN/gVXzAuQBI0sr1FfYZK/z18oi0mAwlA9/ffDNwGYRmaGqJytRU5VgmuWGSMRJH/xGEdkoIl+6R9ODrspgMAQEJ44u07H80WcBQ4HfBlOQIXBERUVx1VVXERsbS+mTH4ZQQFXJy8vj22+/paCg4vFOnRj4f1X1CxGJUtV/iMidFX6qoVK46qqrqFu3LnFxccbAwwRV5dSpUwD8+9//rnB5TproeSJSA9gvIrOAyyv8VEOlEBsba4w7zBAR4uLiiI0NTGQ0J6vJuqjqz8BwYB2WL3nEEMnz3iISFOPOzs7mueeeC0hZzz//PPfeey+ff/55QMoLJTIyMsjNzS1zvkD+3fzNg5fUFP8OuAE4GpCnGy56tmzZwrp16wpdKygoICrKhOwPBP764IN8XFfgwyBoMQSR2v+bVaH830/q6vf+9OnTeeedd4iNjeUvf/kL+/bt49ChQ6SmpnLDDTewb98+1q5dS82aNenZsycAc+fOZe/evbhcLiZPnsz48eO54ooruOeee0hISGDmzJmcO3eOMWPG0KlTJ+bPn8+iRYto27YtO3bsIDMzk27durF2rRWiwOVykZmZyfbt25k0aRL5+fmkpaWRmpqKy+WiVatWbN68mYceeoi0tDQ+/vhjJk6cSGxsLOnp6WRnZzNw4EAaN27M7NmzqVu3Li7XhQbrvHnzWLp0KTVq1OCFF16gRo0aZGRkkJeXR5cuXRgxYgSHDh1i0KBB1KlTh9OnTwPw888/89hjj/Hdd9+RkJDA7NmzWbduHdOmTaNmzZoMGzaMe+65p0J/H1/4mwd/KChPNEQcR48eZcuWLWRlZfHRRx/x0ksvMWHCBN58800aN25M27Zt2bZtG9u2bWPMmDGefIMGDeKtt94iMzOT3Nxcjh07xsqVKxERevbsydtvv01BQQF9+vShffv2LFy4kKysLD799FN27NjhU8/zzz/PwoULiYuL44EHHiAlxdrivmfPnowbN45evXqRlpbGpEmTWLhwIbVr16agoICrr76alStX8tRTT/Huu++ycOFCT5nHjh1j1apVZGVlER0dTUFBAePGjWP8+PG0adOG3r1706dPH2bOnMkf/vAHWrZsSbt27QBYsGABXbp0oVevXsybN4/MzEzWr1/PvHnzSEpKwopGHhycbHzwV6xa24OqPhw0RZVIpPa9K5vc3FyaNGkCQPPmzZkyZQrx8fGcOHGCTz75hOHDh7Nlyxa++eYbrrnmGp/l3HrrrURHR3Ps2DH279/PAw88AFjGdfz4ca677jqqVatGs2bNiuX1NpK9e/eSlpYGwPfff8/x48cBaNy4MTExMYWa/7VrWwuaoqKiuOOOO5g8eTK5ubkkJiZyySWXeNIdOnSIZs2aER0d7Un/1VdfcdtttwHQtGlTcnNzOXToEE2bNqVatWqe72T//v3s3LmT+fPnc+bMGXr16sXo0aN58cUXOXfuHI8//jg33HBDGb91ZziZJnvFfhegJVYf3GDwkJSUxJ49ewD4/PPPadCgAQDXXHMNW7ZsYdSoUcyZM4datWr5Lcc9sFS7dm0aN27MG2+8QXR0NPn5+YgIX3/9NefPn2fXrl2ePKrK2bNnOXDggOda06ZN+etf/0rNmjXJz88nJiamUPnezztx4gS1atXy9PtbtGjBM8884/mBcFO/fn127drlSVdQUEC9evXYuXMnd955J7t372bQoEEkJSWxd+9eWrRowb59+wBo1KgR7dq18zT38/PzOXfuHNOnT+eTTz7h5Zdf5sUXXyzz9+4EJwEf/u51mi0iq4OixBBWvPHGG2zfvh2A0aNHc9ddd9GlSxdPHxygdevWHDt2DBHh0ksvpWXLlo7KjoqKYujQodx///2ICDfeeCNTp06lX79+dOnShbZt23rS9u3bl27dutGpUyfPtXHjxpGamoqqEh8fz/z580t8zoQJE+jXrx/Vq1cnPT2d+++/n5SUFFwuF3Pnzi2UNiEhgfvuu48uXbp4+uAjRozw9MHvvfde6taty7Bhwxg8eDB16tThyiuvBODBBx9k5MiRzJs3D1VlwoQJrFmzhh07dnD69GkmTZrk/IsvI1Ja+99rJRnAdcD9qlpqXLZAIHWuV/r8IWjlR3oTvUGDBp7aNNJwD6gFmi+++IJ58+YxZcqUgJddFg4ePMjBgwc95ykpKTtUtVVZy3HSRHevJFPgBBEwDx7phm0oH1u3bmXixImeFkgk4KSJ/ix4IrkYlyhDSBCM2rtNmzbF5uTDHSej6OOw5sT/i2XgCtwWZF0GgyEAOHEXSsGKy3abqjZV1VKN21dcdBF5RkS22q9OvvIHi0h2S61sNm/ezH333UePHj3o378/J06cICMjg5ycnErVMW3aNJ+LMrp161amsiZPnkx2dnaF9CxevJjFixdXqIxA4qQPvgeoBRxzUmApcdFfV9VnRSQeyAQ2lke0oezUmlmxwbYTwy8M+Hz//fe88MILLF68mLi4OA4cOEBeXl6ZyyzqkloeF9WRI0eW+bkXE04MfD2QKyKH7XNV1Rv9pPcZF11V3f8lZyniPGMIH9avX0+fPn2Ii4sDoGHDhp57c+fOZefOnXTo0IGxY8eSk5PDmDFjyMvLo3379jzxxBNkZGRQs2ZN/vWvf9GrVy82btzI6dOnadOmDTVr1mTgwIHs3r2bhQsXMnnyZE/ZLpeL5s2b89FHH5GamsqAAQPIyMhg9OjRJCYmkpGRwffff0+9evX485//7Mk3Y8YMoqKi+N3vflfMZfQ///kP6enpXHLJJcTExBSaggOKubPed999DBkyhCNHjnD11Vcza9YsCgoKeOihh8jLyyM+Pp67774bgKlTp7JlyxZExON2O3ToUKpXr0779u15/PHHg/lnApw10ccD9VW1kf3yZ9zgJy66FxOB/ysps4gMFpHtIrKdn38qKYmhijl69CiJiYkl3mvbti1r165lw4YNADz33HPMmDGD1atX8+WXX/LNN98AcMcdd/Dmm28CcPnll7N06VIGDx7syffWW2/Rq1evYuV3796drKwslixZUqjV8Prrr3P33XezevVqZsyY4bk+c+ZMAIYNG+ZxGV21ahVt27YlMzOTBQsWkJaWxrJlyzhz5kyx57ndWVetWkVycjJ/+9vfuOmmm1izZg033XQTq1evZu3atbRo0YIVK1Zw6aWXArBv3z6OHDlCZmYmU6ZMYdq0afz973/nwQcfZNWqVYwaNarM33t5cGLg27Gmx5zyA77joiMi9wO1VbXEjkowwiabLYADS2JiIt9++22J9xo3bgxAjRo1ADhw4ABDhgzB5XKxf/9+jhw5AlDI3dR9XKNGDRISEjh8+DA7duygdevWxcpv2rQp0dHRXHfddRw7dqHXeODAAW6//XYATzP/1KlTrFy5kkcffRSwXEZnz56Ny+Vi6dKlHD9+3ONaCnjcTovi7c568OBBT7pf/vKX5OTkFHJZdX+W/fv3k52djcvlYsyYMZw6dYrk5GT27dvHI488wsaNldM7dWLgTYAcEdlgv9aXkt5nXHQRuQ3IsF+VxhtNXjXbAAeQe+65hxUrVngij+Tk5HgMvqg7aMOGDZkzZw6ZmZls2rSJFi1aABTqa3sf9+7dmwkTJtCiRYsS10Tv3buX8+fP8/XXX3s8xcByB3V71rlDHcXFxTFhwgSGDx9OQUEBjRo1Yvjw4Z7FHg8//LDHtRRg9+7dxZ7ndmd1l1u/fn127twJwGeffUaDBg2oV6+ex1XXXUbDhg25++67yczMJDMzk5dffpmYmBj++Mc/MnPmTP70pz85+7IriJM+eJ+yFOgvLjowFUgE1onIj6qaXA7NhiomISGB0aNH069fP487qHez2JunnnqKxx57jLNnzxITE8Nrr73mt+x27dp5+tUlsWrVKp588kn69etXKOpJ//79GTp0KMuXL6d+/fqepnnHjh05ceIE48eP59lnny3mMtq/f3/S09NZtmxZiVFUirqz9ujRg0cffZQePXqQmJjIiBEjPH3w3r17Ex8fD1gLZ+rUqYPL5UJEeOCBB7jssst45ZVX+Pnnnz0r3IKNE1fVpKLXVLXsYSrKQaBcVS/WqbFwdFU9f/48KSkprFy5stg9l8vFypUrqVbNSb0U3lSmq+qrWCPeUUBj4Gsc7BFuMJSVkydPkp6ezoABA6paSsTgxFXVE2pCRH4B/NlP8pDiYq25w5UrrriCVatW+bwfDPfUSKesga/ysfYrMxgMYUCpBu61P/g/gX8AC0vLU9WYabHgkp2dTbNmzXC5XLhcLtauXcv69evJzc3lkUceAWDRokUBe155o5MGWkcgy6osnDTRG5WWJtQwU2LF2bRpU4Xyd+zYsdB5nz59eOqppwpd8zbCRYsWkZqaWmq5wY6gWpqOsjzf6WcKJXx+MhEZKSL3FLn2GxEZGXRVhrBi8eLFLFiwwHOelZXFvn37cLlcvPfee2zfvh2Xy0XXrl09taDL5WLixIkMGTKE48ePk5qaSnJyMk888QRgxUDr3LkzaWlpJS4mWb58OZ07d6Zr166eOWj34pLc3FwyMjKK6XCTnZ1Namoq/fr1Y9OmTUyfPp2uXbuSnJzM4cOWR3bRa/Pnz/eUtWfPHvr27UtycjIPPxza4Qn91eC9VXWa9wVVfVdEJgLTSspQ1ZhBtcpj+fLlfPzxxyQlJXHnnYVD6Hft2pVbbrnFMyjWu3fvEqOcdu/endatWzNhwgRGjhxJ69atmThxItu2bWPZsmXFopO6OX/+PHPmzCErK4sjR44wbtw4lixZUkxjUR3e5OXlsWLFCo4ePcrs2bMLRYQdO3ZssSixL774IsuWLSMzM5OcnBxq1arFkiVLghoRNRD4a5v4Wh5U9mVDhoijT58+ZGZmFlrU4Qt3lNPk5GS+++47T5RTb7fOSZMm4XK5+OCDDzhy5EiJ0UnduCOsxsTEkJSUxE8/FV6z4MTo3K6lRSPCHjx4sMRr3lx//fXccsstPPLII8yaNavUZ1Ul/mrwMyJSX1W/cl8QkQZYK8EMBr94u5n6inLq7vs2bNiQlJQUmjdvDsC5c+fYvHlzseikbhISEsjNzSU/P58jR45w2WXWmgX3YhHv9L62AHI/u6SIsL6ixLrLOnv2LEOGDCEqKopevXrRu3dv6tSpU85vKrj4M/DxwCoRycRybqkHdAf6V4awsuAeMX8DM7gWKrRo0YK0tDQyMjJKjXI6atQoRo0axU8//URUVBTTpk0rMTqpm+joaAYOHEj37t2JioryBEjs3Lkz3bp1KxS91VtHmzZtiulMTEwsFhG2pGtghYEeMGAAY8eOZezYsZw7d4769esX0xdK+HVVFZE4oAdWNNWvgTWqeqqStJXqquoxbDNqXiLh6KpqsKgUV1XbmIuPXhgMhrAgrL32Tc1tMPgnLA3cTIc5Q1VR1aDsEW4IHu6/WyAwmzBHMHl5eZw6dSrk52oNF1BVTp06Va4gliURlBpcRF4CWgGfquoIr+t1sXzZLwH+V1Xf9VGEIQC4o6zExsaaWjxMUFXy8vJ8hsQqKwE38FLCJo8HngZ2AWu4EH3VEaZpXjYKCgp8xgw3XBwEowb3GTYZa0eUEaqqInJKROL8TbtdHydMNkZtMJSbYPTB4/EdNjlaL3QIfYVUNhgMASIYNfgP+A6bfN7ruFhIZbDiogOD7dOzKSkpewKuMDAkAMerWoQPjLbyEcrabipPpmAY+FbgEWA5Vtjk17zu7RKRNlh98MtUtdjOBqo6B5gDICLby+O9UxkYbeXDaCsfIrK9PPkC3kRX1U+xFqpsAQrcYZPt21OA57D65v8v0M82GAyFCco0mffUmH0+3H4/DHQsMZPBYAg4oe7oMqeqBfjBaCsfRlv5KJe2Ujc+MBgM4Uuo1+AGg6EChKyBi8hLIrJFRKaHgJY7RORDW89L9rUxIpItIotEJCYEND4uItmhpk1EHhSRjSLyvohcEyraROQXIvI3W9cqEale1dpEpK6IuPf2q2ZfK6ZJRFLt/8c1IuJ3C96QNHBvd1cgVkSK7yNbuRwCOtp66ojIr4G7VfUurCm/nlUpTkSqY29IISJXEiLaROQaoL2qdlLVDljx/EJCG9AF+NjW9QnwP1S9thNYO/N+BCX/LW0jfxRoByzAmpL2SUgaOCW7u1YZqvqtqrp3hz+H5XL7vn1e5fqAgYA7DtLthI62e4FouwafSWhp+xfg9oOOB+pTxdpU9YyqnvS6VNL3dSOwW1XP4UBnqBp4PL7dXasMe3/zBCwPvJDQZ/+it1dV984G8YSINqytomNVtRPwX0JL2z+BO0RkL9bKxwOEjjY38RTXVNI1n4Sqgf+Ab3fXKkFEamFtvPg7Qktff2Cx1/kPhI62H4HN9vEmrFoyVLQNANapahPgb1g+IaGizc0PFNdU0jWfhKqBb8Xqi4Dl7vpRFWrBHvBYCIxR1W+xVse1t29Xtb6bgCEi8g7QBKs2ChVtH2J1ZwCaYwXuDBVtgtXnBcv/vD6ho81NSf9n+4FbRSQaBzpDdh7cHj1vAexU1WFVrKUvMAPYa1/6PdYgx31ALpCuqlW+IYSIZKvqXSIyjhDRJiIvYP3oHAf6AaNCQZuIxAPLsPrh+cBvgUFVqc3ubmUBLYFPgSeBDkU1iUh/YAhwEuinqj/6LDNUDdxgMFScUG2iGwyGAGAM3GCIYIyBGwwRjDFwgyGCMQYeJESkvoictH2d3xeRjaWkTy/Nr7iqEJED5cjTwXYMcp8vCoCOX4nIs/bx0yLyiYg8bZ+LiCwVkUvs81gRyRSRi/p//KL+8JXADlXtYL86lZI2nQsODB7s+c6gEMyysaZ3PAauqqkBKHMc8Bf7uKeq3g7cb58/BCxyuxTbU1xbsaaYLlqMgVcyIjJRRF61a5fPReRmEemI5QiyQkRm2rX/NhFZAMy1a64P7VVFs+zayp1mkYhsF5ERdvnZIlLHPm4nIq8WeX7RspuKyLsisklElotIDTvdVBHZKiKzAfcqpnSvGvNaEXnfPk4SkXdEZLPtd14L6wfrKbv1Eu1uBdiruFbYad8TkYb29fdF5HkRWW+XUb2I7jggQVW/sy/l2Q5I+SJyOXCXqq4u8nWvBVIq9hcLc9z7IJlXYF9YnlEnsRYLvI9VuwBMBKbZx/2AF+zj94FrvfIewwpMCbAduN4+nge4vNLEYRng50AdLFfaJ+y0rwG/KkGXd9kfAEn28QhgGPBL4B2v9Pn2cTrwtH18LfC+fbwc6GwfR3l9zjSv5x6w30di7WoDlrPQSq/P39M+ngP0KKK7NbDQ6zwZy1ElGXgeaApMBl4CrrTT1MBylKry/4eqepkaPLh4N9G9m6g77PdcoLaPvHv0QtTZy1U1xz7+ELjZPv5CVU+paj6wB2gALAVS7P58Y1UtyZXRu+wmwOt2bdwXuAprxdI2AFX9Cjhqp/X2ivLeC6kJ8J6dvsDH53Fzk/0Zin4WcPa9YD9nlar+FmuRyH+xVlV9imX03jEBL2pPLmPgVUNJhpJH4SCY3jHkfxSR6+3jO4Ev7eObReRSu6l6K3BQVU9j/aPPoPAiFG+8y94D9LV/hH4FTMJaadUSrOY31qowsHy3r7WPW3qVsRerz43XoFbRz+PmS/szFP0s4PsHBOALrNZEUR4HpgKXYrmdxmK1asD6odpbQp6LhrDcPjiMaOnup9r08JN2JfCqiHwIFN34/DFgkYicx/qHzQTqAV8Bc4FGwHy90D/9P6xFCI870JgBvCYXIpj8SVU3iMg/RGQr1g+Ae4OzDcDjIrIe+MyrjCew+vNPY/WJO9tpp4lID6CPV9q5WC2GD7AMepADjajqKRE5LiKJqnoUQERSgNWq+rOILMfqKsRgrY8H6A6scFJ+pGJ80cMUEakPvKKqvynhXnOslW+BGLkOGcTaNKOrqv6vg7SxwJtAsoNuQ8RiavAIQ0RSsQayBlSxlICjqluxpr6cpM3jIp8iA1ODGwwRjRlkMxgiGGPgBkMEYwzcYIhgjIEbDBGMMXCDIYIxBm4wRDD/HyYidS8Y8I3GAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 251.059x155.163 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "histbins = np.linspace(0,100,100)\n",
    "entropy_df['entropy_reduction'] = entropy_df.mean_mi/entropy_df.entropy\n",
    "entropy_df.loc[entropy_df.entropy_reduction >\n",
    "               1.0, 'entropy_reduction'] = 1.0\n",
    "lf_hist = plt.hist(entropy_df.entropy_reduction[infrequent_codes]*100, bins=histbins,\n",
    "         density=True, cumulative=True,\n",
    "         label='Low frequency codes',\n",
    "         color=\"#006BA4\",\n",
    "         #alpha=.5\n",
    "         )\n",
    "# plot the remaining codes (neither filtered by frequence nor prior knowledge)\n",
    "plt.hist(entropy_df.loc[entropy_df.index.difference(sagi_codes),:].entropy_reduction*100, bins=histbins, density=True,\n",
    "         cumulative=True, label='Cherry picked codes',\n",
    "         color=\"#FF800E\",\n",
    "         #alpha=.5,\n",
    "         )\n",
    "plt.hist(entropy_df.entropy_reduction[filtered_roots]*100, bins=histbins,\n",
    "         density=True, cumulative=True,\n",
    "         label='Filtered out roots',\n",
    "         color=\"#ABABAB\",\n",
    "         #alpha=.5\n",
    "         )\n",
    "\n",
    "plt.legend(frameon=True, loc='lower right')\n",
    "plt.xlim(0, 100)\n",
    "plt.ylim(0, 1)\n",
    "plt.xlabel('Entropy reduction (%)')\n",
    "plt.ylabel('Cumulative frequency')\n",
    "#plt.title(\"All codes entropy reduction uF1\")\n",
    "plt.savefig(plots_dir/\"entropy_reduction_uF1.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c9f755a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3178068898592916"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fraction of low frequency codes with more than 10% entropy reduction\n",
    "float((entropy_df.entropy_reduction[infrequent_codes]>.10).sum())/len(infrequent_codes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "5c60e3c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mi</th>\n",
       "      <th>mi_1</th>\n",
       "      <th>mi_2</th>\n",
       "      <th>mean_mi</th>\n",
       "      <th>entropy</th>\n",
       "      <th>entropy_reduction</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>concept_code</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>091</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.401017e-03</td>\n",
       "      <td>0.002401</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200.3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2.114348e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.002114</td>\n",
       "      <td>0.001933</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249.11</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.101512e-04</td>\n",
       "      <td>0.000510</td>\n",
       "      <td>0.000554</td>\n",
       "      <td>0.921244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250.33</th>\n",
       "      <td>2.102530e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.002103</td>\n",
       "      <td>0.001494</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272.6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.464833e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001465</td>\n",
       "      <td>0.001192</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275.5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.818862e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001819</td>\n",
       "      <td>0.001037</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279.51</th>\n",
       "      <td>1.221245e-15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.237684e-03</td>\n",
       "      <td>0.001619</td>\n",
       "      <td>0.001642</td>\n",
       "      <td>0.986064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343.1</th>\n",
       "      <td>1.051751e-04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.454350e-03</td>\n",
       "      <td>0.001780</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365.63</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.399159e-03</td>\n",
       "      <td>0.002399</td>\n",
       "      <td>0.001192</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377.16</th>\n",
       "      <td>6.532936e-04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000653</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.908818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>422.93</th>\n",
       "      <td>2.581017e-03</td>\n",
       "      <td>1.221245e-15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001291</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>531.6</th>\n",
       "      <td>8.671627e-04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000867</td>\n",
       "      <td>0.000383</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>531.60</th>\n",
       "      <td>1.509148e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001509</td>\n",
       "      <td>0.000383</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>536.1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6.801691e-04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000680</td>\n",
       "      <td>0.000554</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>642.9</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.697684e-03</td>\n",
       "      <td>0.002698</td>\n",
       "      <td>0.000554</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>645</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.018902e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001019</td>\n",
       "      <td>0.000554</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>645.1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.002275e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001002</td>\n",
       "      <td>0.000554</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>645.11</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.008664e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001009</td>\n",
       "      <td>0.000554</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>648.81</th>\n",
       "      <td>1.221245e-15</td>\n",
       "      <td>1.125980e-03</td>\n",
       "      <td>1.752195e-03</td>\n",
       "      <td>0.000959</td>\n",
       "      <td>0.000880</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661</th>\n",
       "      <td>1.221245e-15</td>\n",
       "      <td>1.933424e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>663</th>\n",
       "      <td>9.197348e-04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000920</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>663.3</th>\n",
       "      <td>1.053068e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001053</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>663.31</th>\n",
       "      <td>1.064285e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001064</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>674.03</th>\n",
       "      <td>6.542335e-04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000654</td>\n",
       "      <td>0.000554</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>716.8</th>\n",
       "      <td>3.121017e-03</td>\n",
       "      <td>1.221245e-15</td>\n",
       "      <td>3.270977e-03</td>\n",
       "      <td>0.002131</td>\n",
       "      <td>0.001494</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>716.80</th>\n",
       "      <td>3.021017e-03</td>\n",
       "      <td>1.221245e-15</td>\n",
       "      <td>3.470977e-03</td>\n",
       "      <td>0.002164</td>\n",
       "      <td>0.001344</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>728.6</th>\n",
       "      <td>1.170227e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001170</td>\n",
       "      <td>0.000554</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>819.0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>8.725468e-04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000873</td>\n",
       "      <td>0.000383</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>823.81</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.476926e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001477</td>\n",
       "      <td>0.001037</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>880.00</th>\n",
       "      <td>5.307373e-04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.104350e-03</td>\n",
       "      <td>0.001818</td>\n",
       "      <td>0.001344</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>951.3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>7.876933e-04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000788</td>\n",
       "      <td>0.000554</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>987</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2.127041e-03</td>\n",
       "      <td>1.221245e-15</td>\n",
       "      <td>0.001064</td>\n",
       "      <td>0.000880</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995.89</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.483193e-04</td>\n",
       "      <td>0.000548</td>\n",
       "      <td>0.000383</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997.32</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.747619e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001748</td>\n",
       "      <td>0.001344</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E002</th>\n",
       "      <td>5.527609e-04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000553</td>\n",
       "      <td>0.000554</td>\n",
       "      <td>0.998190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E955.0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.528472e-03</td>\n",
       "      <td>0.001528</td>\n",
       "      <td>0.001344</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V27.2</th>\n",
       "      <td>9.309513e-04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000931</td>\n",
       "      <td>0.000880</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V30-V39</th>\n",
       "      <td>3.911194e-01</td>\n",
       "      <td>3.765633e-01</td>\n",
       "      <td>3.785119e-01</td>\n",
       "      <td>0.382065</td>\n",
       "      <td>0.389569</td>\n",
       "      <td>0.980737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V58.12</th>\n",
       "      <td>9.372570e-03</td>\n",
       "      <td>6.218540e-03</td>\n",
       "      <td>5.208607e-04</td>\n",
       "      <td>0.005371</td>\n",
       "      <td>0.003716</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V59</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.202235e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001202</td>\n",
       "      <td>0.001192</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V59.6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.218902e-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.001219</td>\n",
       "      <td>0.001192</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        mi          mi_1          mi_2   mean_mi   entropy  \\\n",
       "concept_code                                                                 \n",
       "091                    NaN           NaN  2.401017e-03  0.002401  0.000719   \n",
       "200.3                  NaN  2.114348e-03           NaN  0.002114  0.001933   \n",
       "249.11                 NaN           NaN  5.101512e-04  0.000510  0.000554   \n",
       "250.33        2.102530e-03           NaN           NaN  0.002103  0.001494   \n",
       "272.6                  NaN  1.464833e-03           NaN  0.001465  0.001192   \n",
       "275.5                  NaN  1.818862e-03           NaN  0.001819  0.001037   \n",
       "279.51        1.221245e-15           NaN  3.237684e-03  0.001619  0.001642   \n",
       "343.1         1.051751e-04           NaN  3.454350e-03  0.001780  0.000719   \n",
       "365.63                 NaN           NaN  2.399159e-03  0.002399  0.001192   \n",
       "377.16        6.532936e-04           NaN           NaN  0.000653  0.000719   \n",
       "422.93        2.581017e-03  1.221245e-15           NaN  0.001291  0.000719   \n",
       "531.6         8.671627e-04           NaN           NaN  0.000867  0.000383   \n",
       "531.60        1.509148e-03           NaN           NaN  0.001509  0.000383   \n",
       "536.1                  NaN  6.801691e-04           NaN  0.000680  0.000554   \n",
       "642.9                  NaN           NaN  2.697684e-03  0.002698  0.000554   \n",
       "645                    NaN  1.018902e-03           NaN  0.001019  0.000554   \n",
       "645.1                  NaN  1.002275e-03           NaN  0.001002  0.000554   \n",
       "645.11                 NaN  1.008664e-03           NaN  0.001009  0.000554   \n",
       "648.81        1.221245e-15  1.125980e-03  1.752195e-03  0.000959  0.000880   \n",
       "661           1.221245e-15  1.933424e-03           NaN  0.000967  0.000719   \n",
       "663           9.197348e-04           NaN           NaN  0.000920  0.000719   \n",
       "663.3         1.053068e-03           NaN           NaN  0.001053  0.000719   \n",
       "663.31        1.064285e-03           NaN           NaN  0.001064  0.000719   \n",
       "674.03        6.542335e-04           NaN           NaN  0.000654  0.000554   \n",
       "716.8         3.121017e-03  1.221245e-15  3.270977e-03  0.002131  0.001494   \n",
       "716.80        3.021017e-03  1.221245e-15  3.470977e-03  0.002164  0.001344   \n",
       "728.6         1.170227e-03           NaN           NaN  0.001170  0.000554   \n",
       "819.0                  NaN  8.725468e-04           NaN  0.000873  0.000383   \n",
       "823.81                 NaN  1.476926e-03           NaN  0.001477  0.001037   \n",
       "880.00        5.307373e-04           NaN  3.104350e-03  0.001818  0.001344   \n",
       "951.3                  NaN  7.876933e-04           NaN  0.000788  0.000554   \n",
       "987                    NaN  2.127041e-03  1.221245e-15  0.001064  0.000880   \n",
       "995.89                 NaN           NaN  5.483193e-04  0.000548  0.000383   \n",
       "997.32                 NaN  1.747619e-03           NaN  0.001748  0.001344   \n",
       "E002          5.527609e-04           NaN           NaN  0.000553  0.000554   \n",
       "E955.0                 NaN           NaN  1.528472e-03  0.001528  0.001344   \n",
       "V27.2         9.309513e-04           NaN           NaN  0.000931  0.000880   \n",
       "V30-V39       3.911194e-01  3.765633e-01  3.785119e-01  0.382065  0.389569   \n",
       "V58.12        9.372570e-03  6.218540e-03  5.208607e-04  0.005371  0.003716   \n",
       "V59                    NaN  1.202235e-03           NaN  0.001202  0.001192   \n",
       "V59.6                  NaN  1.218902e-03           NaN  0.001219  0.001192   \n",
       "\n",
       "              entropy_reduction  \n",
       "concept_code                     \n",
       "091                    1.000000  \n",
       "200.3                  1.000000  \n",
       "249.11                 0.921244  \n",
       "250.33                 1.000000  \n",
       "272.6                  1.000000  \n",
       "275.5                  1.000000  \n",
       "279.51                 0.986064  \n",
       "343.1                  1.000000  \n",
       "365.63                 1.000000  \n",
       "377.16                 0.908818  \n",
       "422.93                 1.000000  \n",
       "531.6                  1.000000  \n",
       "531.60                 1.000000  \n",
       "536.1                  1.000000  \n",
       "642.9                  1.000000  \n",
       "645                    1.000000  \n",
       "645.1                  1.000000  \n",
       "645.11                 1.000000  \n",
       "648.81                 1.000000  \n",
       "661                    1.000000  \n",
       "663                    1.000000  \n",
       "663.3                  1.000000  \n",
       "663.31                 1.000000  \n",
       "674.03                 1.000000  \n",
       "716.8                  1.000000  \n",
       "716.80                 1.000000  \n",
       "728.6                  1.000000  \n",
       "819.0                  1.000000  \n",
       "823.81                 1.000000  \n",
       "880.00                 1.000000  \n",
       "951.3                  1.000000  \n",
       "987                    1.000000  \n",
       "995.89                 1.000000  \n",
       "997.32                 1.000000  \n",
       "E002                   0.998190  \n",
       "E955.0                 1.000000  \n",
       "V27.2                  1.000000  \n",
       "V30-V39                0.980737  \n",
       "V58.12                 1.000000  \n",
       "V59                    1.000000  \n",
       "V59.6                  1.000000  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy_df.query('entropy_reduction>.90')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b68b41e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MannwhitneyuResult(statistic=308625.0, pvalue=0.15453434178221834)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy import stats\n",
    "stats.mannwhitneyu(x=entropy_df.entropy_reduction[filtered_roots].dropna(),\n",
    "                    y=entropy_df.loc[entropy_df.index.difference(sagi_codes),:].entropy_reduction.dropna())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8b0dbc8",
   "metadata": {},
   "source": [
    "### Fig. S2 Entropy reduction by code frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "17a8f8cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAboAAAEYCAYAAAAqIzNgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAn6klEQVR4nO3de3xV5ZX/8c8yXIJouCjea/HXKijXAomAQpBrHfmpONpWHRVUKH1VVNpSsdppp79pvbUqMv2hOFi13qDW26i1iIKgQOWiKLFVWwcci6JWAghyC2v+ODvhJJyT7CRnn7PPyff9evFi3/fKVrN8nv08a5u7IyIiUqgOyHUAIiIiUVKiExGRgqZEJyIiBU2JTkRECpoSnYiIFDQlOhERKWgZT3RmdpSZrTazHWbWKsW+F81sqZmNzPS9RURE6rJMz6Mzs2KgHfA4MNLd9yTtuwN4BHgDeNrdh2X05iIiInVkvEXn7jvcfVOa3b2BZe7+ObDVzA7O9P1FRESSZfsdXZHva0JuBjpl+f4iItLCtGr4kIyqSlouASrrHmBmk4BJAO3bt+/fvXv37EQmIlm1Y21Fk8/dcvCxGYwkXrxqY65DiK0PNm3+1N27NPa8bCe6N8xsEIl3dCXuvqXuAe4+G5gNMGDAAF+5cmWWQxQpDL3u65XrEGqZd8Oe2hu6dk177IvDfh1tMM20Y9OtEV79+Aivnd9+MO+Z9U05L+OJzsxaA38A+gB/NLMfAf/i7lOAm4H7SQxW+Umm7y2Sb+KWjOqzX6JqQN1k9eKwDAaTZdEmNolaxhOdu+8G6k4d+FOw7wNgeKbvKZIr+ZSoGtJQIot7K0sknWx3XYrklUJKZHVNXjaj1no+t7gyTS24wqJEJwWtkBNVQ+q20Fpyi0yJq2VTopO805KTV7KGEplaaCIJSnQSey05sdX33qwltdDUIpPmUKKT2CnkxNbYkYstlRKbZJISnWRdISeyujKZ2FpSC04kk5ToJHJdpz9Ta/3gE3MUSBY0N7G1pGSmVptkixKd7KduYmqug0+cntHrxUlLTmxKVJIvlOhagEwnroYUUmLTO7V9lNgkXynR5YlsJ6vGyOfElu1Elk8tOCU2KRRKdFkS50TVWPmU2HLdIlNiE8k9JbqQCilRiYi0JC0q0SlZNY1acIVJLThpKfIq0SlRZUc+JTaIV3KLU1elEplIQqwT3Zt/36zkJrEWdWJTshJpvlgnOskOteDiQ4lNJPOU6EQaIdMtOCU2kegp0bVAasGJSEuiRNcCrCu+oNZ6L47NUST5Ry04kfynRNcC9DouvxJbLltwSmwihUeJrkAlt+LUghORlkyJTlq0OM17E5FoKNEViP3ew+VRd2UhDTZRV6VI/CjR5Ym6iUziQYlNJP6U6ApEPrXgckldlSItjxJdTBVyCy6fuyrVghPJP0p0MVHIiS2fKbGJ5D8luhxpbmLLp67KQpoXJyL5R4kuS9Riyw9qwYkUHiW6iCixiYjEgxJdnlBXZTiN7apUC06k8CnRZUhLbsHl8yhKESl8SnSS9zTgRETqc0CuAxAREYmSWnRN1JK7KnOtOS04vZMTaXmU6GIqnwafiIjEmRJdSGrBiYjkJyU6abR8GmWprkoRUaJLQy04EZHCEEmiM7PbgAHAane/Kml7OXAz4MC97n5nFPfPR3onJyISjYwnOjPrB7R39yFmNsvMSt19RbD7+8B5wAfAMiA2iU4tuPhqzChLdVWKSF1RzKMbBCwIlhcAA5P2VQAdgLbAtgjuLSIiUksUXZcdgb8Fy5uBHkn7ngCeDpZ/lupkM5sETAIoKukSQXj7qBUXTj4NPhERqSuKFl0lUBIslwTr1W4BTgWOBy42swPrnuzus919gLsPKDqwQwThiYhISxJFi24Z8G1gHjASuDdpXxVQ6e67zGwv0DqC++cFDT7JHL2XE5H6ZLxF5+6rgR1mtgTY6+6vmtnMYPdNwAIzWwYsdPfNmb6/iIhIskimFyRPKQjWpwR/Pwc8F8U9w9A7ORGRlkcTxmU/GnwiIoWkoBNdnFpweicXnr4vJyKZVNCJTgqTBp+ISGPow6siIlLQCqpFp65KERGpq6ASneQnvZMTkSgp0Uns6Z2ciDRHXie6OHVV5jNNJxCRQqbBKCIiUtDyukUn+Unv5EQkm5ToMkSjLEVE4imvEp3eybUMGnwiIpmkd3QiIlLQ8qpFJ5mhUZYi0pLEOtH1svdYqe5KERFphlgnujjT4JPM0Ts5EYmSEp1ETtMJRCSXGhyMYmarzOyHZnZYNgISERHJpDAtuiHAecDDZrYJ+E/gj+7ukUYmGZPtwSdqwYlInDSY6Nx9O3CfmT0P/By4D1hnZjPc/aGoA5TCo3dyIpJNYbouzzazp4A5wLPAMcBQ4DsRxyYiItJsYbouBwJT3f1vyRvN7IpoQhIREcmcMJVRXgDeA7CEUQDuvibKwERERDIhTItuurs/D+DubmbXAM9HG1b8aN5cehp8IiJxFibRtauzfmAUgUjh0uATEcmlMInuJTN7AFgIDAOWRBqRiIhIBoWZXnCtmZ0O9AQecvc/RB+WNJcKN4uIJIT9TM96YDmwzcyGRhiPiIhIRjXYojOzh4FOwP8EmxxYHGVQEm8NDT7ROzkRiZMw7+iOdPdhUQciIiIShTBdlwvNrFfkkYiIiEQgTIvuX4DxZlY9usHd/YQIY4qFfJs3p8EnIiKphRl1eXw2AhEREYlCmKLOh5jZLWY2x8xamdkF2QhMREQkE8J0Xd4L3A38wN33mNmlgD7P04KoxJeI5LMwg1HauftTQPVLIIswHhERkYwK06LbYWZlAGbWB/g82pAk32jenIjEWZgW3WRgKnAYcC3w3UgjEhERyaAwoy4/AM7PQiwSE3onJyKFJEwJsIUkyn7VcPfhDZxzGzAAWO3uVyVtLwZ+DRwHVLj7lKYELZo3JyISVtgJ45AYhDIAOLm+g82sH9De3YeY2SwzK3X3FcHuK0l8AeGFJkcckXybIC4iIuGE6br8e9LqB2Y2uYFTBgELguUFwECgOtENA442sx8DtwajOSXPaPCJiOSTMF2X17Gv6/JLQJsGTukI/C1Y3gz0SNr3FeB24EfAIjN71t1r9cGZ2SRgEsCxHTSTQUREmifMqMsPgL8Hfz8DnN7A8ZVASbBcEqxX2wy85O7bgL8Ch9c92d1nu/sAdx/Q5UAlOhERaZ4wXZf3NfKay4BvA/OAkSQqq1RbCvQ2s9VAV+CTRl67xdLgExGRpgnTdbkOOBLYSKIFtoFElZSUXzFw99VmtsPMlgBr3P1VM5sZjLC8CbiPREvvbnfflbkfRZpK0wlEpJCFGXX5B+BGd19vZscC17v7pPpOSJ5SEKxPCf7+EBjd1GBFREQaK8w7ut7uvh7A3d+n9uASERGRWAvToltqZs+SmCLQn8Q7uIKguXMiIoUvzGCUaWbWFzgBeMLdX4s8KhERkQwJ8+HVtsApQE/gTTMbEXlUIiIiGRKm6/I+EtMCRrr7v5rZtUDsSnhJdFQJRUTyWZjBKIe6+x3AzmBds7hFRCRvhEl0bmbHBH8fDuyOOCYREZGMCdN1ORV4kMS0gkeAKyKNSIBoK6FogriItCT1JjozOwC42t3LsxSPiIhIRtXbdenue4HWZtY+S/GIiIhkVJiuyxOB98xsLbCXRI1LlfESEZG8kDbRmdkx7v4BcF4W44mUKqGIiLQ89bXo7gNGADPd/cwsxSMR0OATEWnJGhqM8hugv5nNTt7e0NcLRERE4qK+RHcG8DWgL4npBSIiInknbaJz9x3AMjPrH4y+lAjF6QviKvklIoWkwcooSnIiIpLPwpQAExERyVth5tFhZj2ALwN/ALq4+8eRRiXNppGWIiIJDSY6M7se6Ad0JZHo7ge+Hm1YIiIimRGm63K0u58DbHZ3B9pEHJOIiEjGhHpHZ2ZFJD7TcwBQFG1IIiIimRPmHd1M4GUS7+gWAjMijUhERCSDGkx07v47M3se+Crwnrt/Fn1YmRHn2pZxmjcnIlLIGuy6NLMFwNnAn/MpyYmIiEC4rsvxwIXAQjN7G7jP3RdEGpVklSqhiEghC1MZ5QN3vwkYCPwFeDbyqERERDIkzDy6XsAlwEhgEYmEJzGjCeIiIqmF6br8CYlJ4tPdXSMoREQkr4QZdXluNgIRERGJQtpEZ2a/dfeLzOxdwKs3A+7uJ2QlOhERkWaq73t0FwWL3ZI/1WNmFnlUBUjz5kREciNMCbC6UwnmRhGIiIhIFOrruuwJ9AEON7MLgs2tgaOyEZjUT6MsRUTCqW8wyiEkyn4dCBwfbNsFXB1xTCIiIhlT3zu6l4CXzOwOd9+UxZiaLM61LeNElVBEpCUJ847uBDN7wczeNrN3zOydyKMSERHJkDCJbgbwXWADcBbwUKQRiYiIZFCYRLfd3f8CHODufwYGRxyTiIhIxoRJdLvMrB3wjpnNAjo0dIKZ3WZmS8xsv4+0WsIaM7u8CfGKiIg0SpgSYF8HMLMpwNeBn9Z3vJn1A9q7+xAzm2Vmpe6+IumQM4GPmx5yy6TpBCK58WzFX1n41ju4e8MHS8Z06dSR4cd/mdKuxzT7WvXNo0vVRfkx8BVgYz3XHMS+SeYLSHztIDnRnU8LmHSuSigihWHhW+/wySefcMghh+Q6lBajqqqKVatWMXjw4GgTHTAxzXYHltZzXkfgb8HyZqBH9Q4zGwO8BOxJd28zmwRMAji2g6qNiUhuubuSXJYVFRVRVlZGVVVVRq5X3zy6CU28ZiVQEiyXBOvVLgcuAr5Vz31nA7MBBhxVpL4CEYmVjz76iKuvvpoVK1bQtm1bunbtyu23384JJ4SrdT9+/HjGjh3LuefqwzDZEubDq79h39cLAHD3S+s5ZRnwbWAeiY+13pu073jgCeDoxKXt5WBEp4hI7Lk748aN45JLLuGRRx4B4PXXX2fjxo2hE11cVVVVUVRUlHY9n4UZdfmfwBzgHmAN8Hl9B7v7amCHmS0B9rr7q2Y2M9jXNxjc8ivgdiW57Nix6dZaf0SkaRYuXEjr1q2ZPHlyzba+ffsyZMgQ3J1p06bRs2dPevXqxdy5iaEI7s4VV1zBSSedxBlnnMHHH+8bi7dq1SrKy8vp378/Y8aM4cMPP9zvnuvXr2fEiBH07t2bESNG8P777wOwceNGxo0bR58+fejTpw9LlybeKN1///307t2bPn36cNFFiY/QjB8/nkcffbTmmgcddBAAixYt4rTTTuOCCy6gV69e+61XVVUxbdo0SktL6d27N3fddVfNecOGDePcc8+le/fuXHjhhTWDdVasWMHgwYPp06cPZWVlbN26Ne11PvzwQ4YOHUrfvn3p2bMnS5Ysycw/qDrCjLp8JWn1ZTP7rxDnXFVnfUqd9XvDBigiEhdr166lf//+Kfc99thjvP7666xZs4ZPP/2U0tJShg4dyrJly3j77bd588032bhxIyeddBKXXnopu3fvZsqUKTz55JN06dKFuXPnct1113HPPffUuu4VV1zBxRdfzCWXXMI999zDlVdeyRNPPMGVV15JeXk5jz/+OFVVVXz++edUVFTw85//nFdeeYVDDz2Uzz77rMGf6dVXX2Xt2rUcd9xxLFq0qNb67Nmz6dChAytWrGDnzp2ccsopjB49GoDXXnuNiooKjjrqKE455RReeeUVysrK+OY3v8ncuXMpLS1ly5YttGvXjjlz5qS8zmOPPcaYMWO47rrrqKqqYvv27c3/h5RCmK7LC5JWvwR0iSQSqUXTCUTyy8svv8z5559PUVERhx9+OOXl5axYsYLFixfXbD/qqKMYPnw4AG+//TZr165l1KhRQKKr8Mgjj9zvusuWLeOxxx4D4KKLLuKHP/whAC+++CL3338/kBi80aFDB+6//37OPfdcDj30UAA6d+7cYNxlZWUcd9xxKdfnz5/PG2+8UdMa3Lx5M++++y5t2rShrKyMY45JjIjs27cv69ato0OHDhx55JGUlpYCUFJSUu91SktLa5L+2WefTd++fcM+7kZpMNGx78sFDnxGYh6c1KHpBCKFr0ePHrW6AJPVN88u1feq3Z0ePXqwbNmyRsVQ37ev3T3l/latWrF3796aY3bt2lWzr3379rWOTV53d2bOnMmYMWNqHbNo0SLatm1bs15UVMSePXvS3j/ddQAWL17MM888w0UXXcS0adO4+OKL0/58TdXgOzp3/zd3/zfg/wH/H/g041GIiOSB4cOHs3PnTu6+++6abStWrOCll15i6NChzJ07l6qqKj755BMWL15MWVkZQ4cO5ZFHHqGqqooPP/yQhQsXAtCtWzc++eSTmkS3e/duKioq9rvn4MGDawa+PPjgg5x66qkAjBgxglmzZgGJ1uCWLVsYMWIE8+bN4x//+AdATddl165dWbVqFQBPPvkku3fvDvXzjhkzhlmzZtUc/84777Bt27a0x3fv3p0NGzawYkVi6vTWrVvZs2dP2uusX7+eww47jIkTJ3LZZZexevXqUHE1Vpiuy2tIzKnbDhiJll3vSKIREYkxM+Pxxx/n6quv5sYbb6S4uLhmekH1+7g+ffpgZtx8880cccQRjBs3jhdffJFevXpxwgknUF5eDkCbNm149NFHufLKK9m8eTN79uzh6quvpkePHrXueccdd3DppZdyyy230KVLF37zm98AMGPGDCZNmsScOXMoKipi1qxZDBo0iOuuu47y8nKKior42te+xr333svEiRM566yzKCsrY8SIEfu14tK5/PLLWbduHf369cPd6dKlC0888UTa49u0acPcuXOZMmUKX3zxBe3atWPBggVpr7No0SJuueUWWrduzUEHHVTTFZtp1lBZGzNbCZzs7pmZudcIA44q8pWTDkq7P07fn8t012Um39FppKVI0/1g3jMq/5UjZsYvv3FGzfoP5j2zyt0HNPY6YaYXrAUafqMpIiISQ2EGo8wH3jezD4J1d/f8nhkpIiItRphENx3o6u71FXIWERGJpTCJbiWJaQWSR/ReTkQkIUyi6wG8Z2bV5brc3UdHGFNe0Lw5EZH8ECbRfSPyKEREhA0bNjB27FjeeustPv/8c1q1CvMrunnWrVvHySefzIknnkibNm2YP39+5PfMtjBPUeNqRURS6Dr9mWadv+7GM2qtd+7cmRdeeIFx48Y1eO62bdtCz4dryKhRo3jggQcycq04CjO9YA6JLxjcQ+ITPPMijUhEpIUqLi6mU6dO9R6zfPlyJkyYwMSJ+76N7e4MHDiQyZMn071795rqK3VVVlYyb948rr322lrbFy5cyJAhQ7jtttua/0PEUJivF4yqXjazA4H/iDSiFkpFnEUknd27dzNnzhx+//vf06dPH6ZPn063bt1q9q9bt45du3YxY8YMli5dWvO5HYDt27czY8YMVq5cSUlJCaNGjWLatGk15x555JG88847tG3blrPOOqvmk0CFJEyLLtluoE8UgYiISGpbt27lrrvuomfPnkyePLlWkgOoqKjg/PPPp23btnz66ae1vkawd+9eNm3aRFVVFSUlJXTu3JkDDzywZn/btm1p3749rVq1YuzYsaxduzZrP1e2NJjozOxdM3vHzN4F/gwUbkeuiEgMde7cmddee43zzjuPf//3f+eMM86o9RWFioqKmk/crFmzpla9zIMOOoibb76ZJ554gu9///usX7+ea665pmb/1q1ba5ZfeeUVvvKVr0T/A2VZmK7L4xs6RnJP8+ZE8t/u3bs5/fTTWbNmDWPGjOEXv/gFJ598cs3+wYMHM3jwYD777DOeeWbfQJiKigrGjx9fs5z8Du6jjz7iW9/6Vq37HHHEETXLS5Ys4cc//jFt27bl1FNPrXW/QpG2qLOZXQ1UuPvzSdtGAj3d/fZsBBfnos5xK+KsRCcSDRV1zp1MFXWur0V3bt2E5u4LzOynwO2pTohaISU2ERHJjvoS3a5GbpdG0ChLEZHsqG8wyg4z65q8wcyOA3ZGGpGIiEgG1deimw48aWZPAf8DfBk4A7goG4GJiIhkQtoWnbu/AZwKvAV0JPEB1iHu/mZ2QhMRaVnWrVvH4YcfzrBhwxg9Oju18zds2EC/fv0oLi5mz549OYsjSvVOL3D3rcDDWYpFGkGjLEUKU9i6k5mqdZmuvmYh1b+MvjS2iEih+mmHZp6/eb9N1XUnzznnHKZOnbrf/uXLl3PXXXexc+dOHnroISBR63LQoEH07duXRYsWMWvWrJoSYMkqKyuZP38+r732GjfccAOQqK9ZXFzc6DjyiRKdiEhMpKs7GWWty8bEka8aW+tSREQikq7uZJS1LhsTR75SohMRiYl0dSejrHXZmDjylbous0QTxEWkIQ3VnYyi1mWq+pr/+Mc/Cqr+Zdpal3FQt9ZlNkuAxa2WZV0adSmSHap1mTvZqHUpMaLEJiLSNHpHJyIiBU0tuojonZyISDyoRSciIgVNiU5ERAqaEl1g3g17av0REcm2uBRYThUHwNSpUxkyZAhXXXVVrePTbY8LvaOLKY2yFIm/Xvf1atb5b15S+2MwzS2wHGWh59WrV7Nt2zaWLFnCd77zHVasWEFpaWna7XES6xZdRds29Dru2Jo/IiKFrLi4mE6dOu23vbrA8m233ZbyvOXLlzNhwgQmTpxYa7u7M3DgQCZPnkz37t1ZuHDhfudWVlYyb968WpPMU8WxbNkyRo4cCcDIkSNZvnx5vdvjRC06EZEYa2qhZ0hf7LkphZ4rKytrSoF16NCBioqKerfHSSQtOjO7zcyWmNmMOtt/YmbLgj8jori3iEghaWqhZ0hf7LkphZ47duzIli1bANiyZQsdO3asd3ucZDzRmVk/oL27DwHamFlyZ+397j4IOB34SabvLSJSaJpa6BnSF3tuSqHnQYMG8cILLwCwYMECBg4cWO/2OImi63IQsCBYXgAMBFYAuPt/B9t3AgVVPE4TxEWkuZpSYDldoWdIX+y5KYWeTz75ZIqLixkyZAh9+vShrKwMoGZ0Zt3tcZLxos5mdh2wyt2fM7ORwGB3/1mdY24C1rj7QynOnwRMAmh9SOv+3X61f3M8UzI5jUBFm0UKk4o6506cizpXAiXBckmwXsPMxgGHpEpyAO4+G5gN0O64di3m3y4lNhGRaEQxGGUZUD3QZCRQM9bUzHoD3w3+iIiIRC7jic7dVwM7zGwJsNfdXzWzmcHuW4DDgT+a2ZOZvreIiEhdkcyjc/er6qxPCf4eE8X9RERE0ol1ZRQREZHmUqITEYmJxhRTznYh5bVr1zJ48GCGDBnChAkTao1EjUN89VEJsBzRKEuR/Pfn7ic26/wT//LnWuthiykXFRWFLqScqULP3bp1Y+nSpQBMmDCBlStXpi3q3Jj4skGJrok0QVxEMq24uJji4uJa21IVTT7ggAP221Y3kSxfvpy77rqLnTt38tBD+2ZzuTuDBg2ib9++LFq0iFmzZnHaaafVOreyspL58+fz2muvccMNNwDQunXrmv1t27blS1/6UrPiyyZ1XYqIxFhlZSUlJYmpyR06dGDTpk0pt0Giosmdd97JqFGjePTRR5k+fXqtJAe1Cz3PmjWLRYsWAbB9+3ZuuOEG/vmf/5mpU6eyZ8+e/Qo9P/XUU/Ts2ZOPP/6YQw45pNHx5UqLatHpg6oikm9SFU0uKipKWUi5utDzsGHDmDx5Ml/96lf3u15zCj2feeaZnHnmmUyZMoWnn36acePGNSq+XFGLTkQkxlIVTU5XSDnKQs87d+6sWS4pKaFdu3aNji9XWlSLTkQkk+oOJmmuxhRTrq+QchSFnp977jluvTUxiO74449n9OjRQPqiznEq9Jzxos6Z1O64dv7Vn+7f9G6qOBVx1qhLkfygos65k6mizuq6FBGRgqZEJyIiBa2g39HFqatSRERyo6ATXZzonZyISG6o61JEJCbS1ZOMQy3JdHU4GxNfrupfqkUnItJEv578YrPO/+6dw2utp6onmapuZC5qXaaqwwnha3Hmsv6lWnQiIjGRqp5kqlqSqbbVtXz5ciZMmMDEiRNrbXd3Bg4cyOTJk+nevTsLFy7c79zKykrmzZtXM8cOEvPiOnXqtN+xYeMLE3NUlOgismPTrbX+iIiEUbeeZJxqXaYSNr5c1r9UohMRiZEzzzyTtWvXcvTRR/P000+nrCWZahvsq3XZs2dPJk+eTLdu3fa7fnNqXaYSNr50MWeDEp2ISEykqicZl1qX6YSNL5f1L5XoRERi4rnnnqO8vJzy8nI2btzI6NGja9WSPOCAAygrK0u5LdngwYO59957+e1vf8sXX3xRa19FRQW9e/euWT7ppJOARK3LYcOGMWzYMC6++GIefvhhNm7cWHPe7t27GTlyZE0dzj/96U8AoeNrKOYoFXSty+ZMGFctSxEB1brMpUzVuiyo6QX63pyIiNRVUIkul9SCExGJJ72jExGRgqZEJyIiBU2JTkRECpre0QUaO8pS7+REJC42bNjA2LFjeeutt/j8889p1Srxq33q1KmsXLmSfv36MWPGjJrjU21Pd2whUKITEWmiX31zbLPO//7cp0Mdt3v3bqB2LcxkqQoupyq2XFpaGroIc7YKLmeDui5FRGIiXcHlzZs3M3z4cH70ox+xbt26/c5LVXA5XRHluBVczgYlOhGRmEhXcPnQQw9l8eLFjBo1iuuvv55zzjmHZ599tt5rpSuiHLeCy9mQ112XzZ0g3tzqJyIimZSu4DIkqoScdtppFBUVcfPNN/PAAw/wT//0T2mvla6IcqrtRUVFOSu4nA1q0YWkz+6ISNTSFVzeuXMnv/rVrygvL+fZZ59l5syZ+31+p650RZTjVnA5G5ToRERiIl3B5W3btnH00Ufz/PPPc+ONN9Zq6UHqgsvpiijHreByNuR1Uedsdl2qFSfSMqmoc+60yKLOeicnIiKNpa5LEREpaHnVossmdVWKSLWqqiqKiopyHUaLsmvXLg4wy8i1Yp3ovvKhvjEnIrnVpVNHVq1aVXADNOJs165d3HzTTXQ94rCMXC+SRGdmtwEDgNXuflXS9qOAB4Bi4F/dfUEU96+mwSYi0lzDj/8ygwcPpqqqKtehtBgHmNH1iMO4oH+PjFwv44nOzPoB7d19iJnNMrNSd18R7J4OXA+8ATwNRJroGqLkJiINKe16DKVdj8l1GNIMUbToBrEvgS0ABgLVia43cJW7u5ltNbOD3X1rpm6sUZUiIlJXFImuI/C3YHkzkNz2LPJ9E1I2A52A0IlOn9IREZHGiiLRVQIlwXJJsF4tuZO77j4AzGwSMClY3XnS239ZW7Pz7RGZi7KwHQp8musg8pCeW+PpmTWNnlvTdGvKSVEkumXAt4F5wEjg3qR9b5jZIBLv6ErcfUvdk919NjAbwMxWNmUWfEun59Y0em6Np2fWNHpuTWNmK5tyXsYnjLv7amCHmS0B9rr7q2Y2M9h9M/BzEu/ufpHpe4uIiNQVyfSC5CkFwfqU4O8PgOFR3FNERCSVuJcAm53rAPKUnlvT6Lk1np5Z0+i5NU2Tnlusv14gIiLSXHFv0YmIiDRLbBOdmd1mZkvMbEauY4kzMzvZzJYGz+q2YNs0M3vZzB40s9a5jjHOzOx7ZvZysKznFoKZXWxmL5jZIjM7Ws+tYWZ2oJk9EzyzJ82srZ5bamZ2lJmtNrMdZtYq2LbfszKzC4PffU+bWUl914xloksuIwa0MbPSXMcUY+uB4cGzOszMhgCnufupJKZxnJ3L4OLMzNoCfYLlLui5NcjMjgbK3X2Euw8DdqHnFsbXgT8Fz+xV4FvouaXzGTACWA6p/9sMkt1kYCjwWxJT2tKKZaIjdRkxScHdP3L3HcHqHhJl1hYF63p29bscuC9YLkPPLYwxQFHQopuJnltYfwPaBssdga7ouaXk7jvcfVPSplT/jp0AvOnuewjx/OKa6DoC1ZPJq0uFST3MrDeJaguV6Nk1KPg/wnJ3fzHY1BE9tzAOB9q4+whgO3puYb0LnGxmFSS+7PJX9NzC6sj+zyrVtrTimugqSV9GTOows87AfwCXoWcX1kXAQ0nrlei5hbEZeClYfpFEy0TPrWGXAH909x7AMyTmMOu5hVPJ/s8q1ba04prolpHoo4VEGbHlOYwl1oKXtQ8A09z9IxJfiigPduvZpdcN+I6ZPUei8PgA9NzCWEqiexygL/A/6LmFYSTePUGixmVX9NzCSvU77R2gp5kVEeL5xXYeXTDash+wxt2vyHU8cWVm5wN3ABXBpmtJvKD9v8D7wHh335Wj8PKCmb3s7qea2TXouTXIzH5J4n8MPgUuAKai51YvM+sIzCXxnm438E1gInpu+wleK/wB6A+sBn4EDKPOszKzi4DvAJuAC9x9c9prxjXRiYiIZEJcuy5FREQyQolOREQKmhKdiIgUNCU6EREpaEp0UtDMrKuZbQpqDC4ysxcaOH58Q3XzcsXM/tqEc4YFxQSq1x/MQBwDzezfguXrzexVM7s+WDcze8TMioP1Nmb2lJnpd43kjP7lk5ZglbsPC/6MaODY8eybiFojmK8TiSivTWJYdk2ic/cLM3DNa4BfB8tnu3sZMC5YnwA8WF2WLhgyv4zE0HCRnFCikxbJzH5qZnOC1sbrZtbdzIaTmAT9OzObGbQGV5jZb4G7g5bM0qCK+qyg9VJ9zINmttLMrgqu/7KZHRYsDzWzOXXuX/favcxsgZm9aGbzzKxdcNwtZrbMzO4Eqqu2j09qQR1jZouC5WPN7DkzeymoRdmZROK+LmjNFlW3CoNq+r8Ljl1oZl8Nti8ysxvNbH5wjbZ14j4YONTdPw427QqKFuw2sw7Aqe7+X3Ue97PAec37JybSdK1yHYBIFvSvTgbA35NaNVvd/TIzuwC43N1/YGavA//i7h+YWVcSFSxGuPsWM1sJfMPd3zOze0i0Ut4IjhkO7ABWmNnDwG+Ai4FfApcCd6aIK/nai4P7vh8ky8vM7BWgl7sPCmK5rIGf85fAre4+38wOcPe9ZnYv8Fd3fwDAzKqPnUSiKO7PzGwocDNwTrBvubtPN7PZwCjg6aR7dCfxxYxqNwEPAjeQKFZwm5ndBLQBfuHun5CoYtGrgdhFIqMWnbQEyV2XyV13q4K/3wcOSXPuWnevLh7bwd3fC5aXkvilD/AXd9/q7ruBtcBxwCPAecH7vhPdPVWJouRr9wDuDxLy+cARJCq0rwBw93XAxuDY5CoPlrTcA1gYHL83zc9TrVvwM9T9WSDccyG4z5Pu/k0SRYq3k6giv5pEFZCrkg9tIB6RyCjRSUuWKmHsonZPR1XS8mYz+z/B8mDg7WC5u5kdFHTh9QT+2923kfiFfwe1i0cnS772WuD8IBkPBH5GouJ9f0h0S5L4cgAkaiYeEyz3T7pGBYl3ciQN/qj781R7O/gZ6v4skD6RAvyFREu0ru8BtwAHkShz1QY4ONh3AvtK1IlknboupSVI7roEGFvPsY8Bc8xsKTCnzr4rgQfNrIrEL+6ngC8D64C7geOB+5LeX91Fotjs90LE+F3gXtv3pekb3P15M/uzmS0jkQg3BPueB75nZvOB15Ku8QMS7/uuJ/HObHRw7O1mNhb4RtKxd5NoQS4mkdgmhogRd99qZp+a2eHuvhHAzM4D/svdvzCzecA8Eu8TLw9OOwP4XZjri0RBtS5FmiF4d/af7j4yxb6+JL4qkYmRjrFhZoOA0939X0Mc2wb4PXBWiO5UkUioRScSATO7ELiaxHfICoq7LyMxZSDMsbvQ1ALJMbXoRESkoGkwioiIFDQlOhERKWhKdCIiUtCU6EREpKAp0YmISEFTohMRkYL2v9DfMj8wvsCaAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 502.117x310.325 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "oh_diagnoses_df = pd.read_parquet(datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "code_occurrences = oh_diagnoses_df.sum(axis=0)\n",
    "few_shot_codes = oh_diagnoses_df.columns[(code_occurrences>2) & (code_occurrences<=5)]\n",
    "rare_codes = oh_diagnoses_df.columns[(code_occurrences>5) & (code_occurrences<=15)]\n",
    "infrequent_codes = oh_diagnoses_df.columns[(code_occurrences>15) & (code_occurrences<=100)]\n",
    "medium_freq_codes = oh_diagnoses_df.columns[(code_occurrences>100) & (code_occurrences<=300)]\n",
    "frequent_codes = oh_diagnoses_df.columns[(code_occurrences>300) & (code_occurrences<=1000)]\n",
    "highly_frequent_codes = oh_diagnoses_df.columns[(code_occurrences>1000)]\n",
    "\n",
    "histbins = np.linspace(0,100,100)\n",
    "entropy_df['entropy_reduction'] = entropy_df.mean_mi/entropy_df.entropy\n",
    "entropy_df.loc[entropy_df.entropy_reduction >\n",
    "               1.0, 'entropy_reduction'] = 1.0\n",
    "plt.hist(entropy_df.entropy_reduction[few_shot_codes]*100, bins=histbins,\n",
    "         density=True, cumulative=True, label=r'$1 < n \\leq 5$', alpha=1.)\n",
    "plt.hist(entropy_df.entropy_reduction[rare_codes]*100, bins=histbins,\n",
    "         density=True, cumulative=True, label=r'$5 < n \\leq 15$', alpha=1.)\n",
    "plt.hist(entropy_df.entropy_reduction[infrequent_codes]*100, bins=histbins,\n",
    "         density=True, cumulative=True, label=r'$15 < n \\leq 100$', alpha=1.)\n",
    "plt.hist(entropy_df.entropy_reduction[medium_freq_codes]*100, bins=histbins,\n",
    "         density=True, cumulative=True, label=r'$100 < n \\leq 300$', alpha=1.)\n",
    "plt.hist(entropy_df.entropy_reduction[frequent_codes]*100, bins=histbins,\n",
    "         density=True, cumulative=True, label=r'$300 < n \\leq 1000$', alpha=1.)\n",
    "plt.hist(entropy_df.entropy_reduction[highly_frequent_codes]*100, bins=histbins,\n",
    "         density=True, cumulative=True, label=r'$n > 1000$', alpha=1, linewidth=1.5)\n",
    "# plot the remaining codes (neither filtered by frequence nor prior knowledge)\n",
    "#plt.hist(entropy_df.loc[entropy_df.index.difference(sagi_codes),:].entropy_reduction*100, bins=histbins, density=True,\n",
    "#         cumulative=True, label='Remaining codes', alpha=.5)\n",
    "plt.legend(edgecolor=\"k\",framealpha=1.0, loc='lower right',title=\"Code occurrences\")\n",
    "plt.xlim(0, 100)\n",
    "plt.ylim(0, 1)\n",
    "plt.xlabel('Entropy reduction (%)')\n",
    "plt.ylabel('Cumulative frequency')\n",
    "#plt.title(\"All codes entropy reduction uF1\")\n",
    "plt.gcf().set_size_inches(dual_col_fig_size)\n",
    "plt.savefig(plots_dir/\"entropy_reduction_infrequent_codes_uF1.pdf\")\n",
    "del oh_diagnoses_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "2cc706fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1350\n",
      "1598\n",
      "2061\n",
      "783\n",
      "510\n",
      "410\n"
     ]
    }
   ],
   "source": [
    "print(len(few_shot_codes))\n",
    "print(len(rare_codes))\n",
    "print(len(infrequent_codes))\n",
    "print(len(medium_freq_codes))\n",
    "print(len(frequent_codes))\n",
    "print(len(highly_frequent_codes))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a756d492",
   "metadata": {},
   "source": [
    "# Table S9 Performance per stay type (medical, surgical, newborn)\n",
    "Check whether the models performs about equally on 3 patients subsets: newborns, surgical patients and medical patients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e2560f36",
   "metadata": {},
   "outputs": [],
   "source": [
    "visit_details = pd.read_csv(\n",
    "    datadir/\"../visit_detail.csv.gz\",\n",
    "    usecols=[\"visit_occurrence_id\",\"visit_detail_concept_id\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "4e8ada63",
   "metadata": {},
   "outputs": [],
   "source": [
    "def categorize_visit(x: pd.DataFrame):\n",
    "    \"\"\"\n",
    "    Translate visit detail concept codes to string.\n",
    "\n",
    "    Patients with at least one newborn tag are considered newborns, among the\n",
    "    remaining patients the ones with at least one surgical tag are considered\n",
    "    surgical, the remaining ones are considered medical patients.\n",
    "    \"\"\"\n",
    "    x=set(x)\n",
    "    if 4237225 in x:\n",
    "        return \"newborn\"\n",
    "    elif 4149152 in x:\n",
    "        return \"surgical\"\n",
    "    elif 45763735 in x:\n",
    "        return \"medical\"\n",
    "    else:\n",
    "        return None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "4954c2f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "visit_detail_concept_id\n",
       "medical                    26883\n",
       "surgical                   23947\n",
       "newborn                     8096\n",
       "dtype: int64"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "visit_categories = visit_details.groupby(\"visit_occurrence_id\").agg(categorize_visit)\n",
    "visit_categories.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "7a10e530",
   "metadata": {},
   "outputs": [],
   "source": [
    "from project_miscellaneous import filter_Sagi_diagnoses\n",
    "import sklearn.metrics as skmet\n",
    "oh_diagnoses_df = pd.read_parquet(\n",
    "    datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "\n",
    "icd9_concepts = pd.read_csv(datadir / \"icd9_concepts.csv.gz\")\n",
    "icd9_allcodes = icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].concept_code\n",
    "icd9_chapters = icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id==\"Chapter\"').concept_code\n",
    "icd9_subchapters = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id==\"SubChapter\"')\n",
    "                    .concept_code)\n",
    "icd9_3dig = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.startswith(\"3-dig\",'\n",
    "                                'na=False)').concept_code)\n",
    "icd9_4dig = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.startswith(\"4-dig\",'\n",
    "                                'na=False)').concept_code)\n",
    "icd9_5dig = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.startswith(\"5-dig\",'\n",
    "                                'na=False)').concept_code)\n",
    "icd9_billing = (\n",
    "    icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.endswith(\"-dig billing code\",'\n",
    "                        'na=False)').concept_code)\n",
    "\n",
    "\n",
    "base_dir = \"/home/quentin/Projects/completion_pmsi/models/mimic-omop/sequentialNN_3/\"\n",
    "\n",
    "metrics_func_dict = {\n",
    "    'micro_AUROC':lambda y_true,y_score: skmet.roc_auc_score(y_true=y_true, y_score=y_score, average='micro'),\n",
    "    'macro_AUROC':lambda y_true,y_score: macro_averaged_AUROC(*filter_no_positive_label(y_true, y_score),pos_label=1),\n",
    "    \n",
    "    'micro_AUPRC': lambda y_true,y_score: micro_averaged_auc_PR(y_true=y_true, probas_pred=y_score, pos_label=1),\n",
    "    'macro_AUPRC': lambda y_true,y_score: macro_averaged_auc_PR(*filter_no_positive_label(y_true, y_score), pos_label=1),\n",
    "    \n",
    "    'NDCG': lambda y_true,y_score: skmet.ndcg_score(y_true=y_true,y_score=y_score, k=None),\n",
    "\n",
    "    'NDCG@15': lambda y_true,y_score: skmet.ndcg_score(y_true=y_true,y_score=y_score, k=15),\n",
    "}\n",
    "for p in [25,50,75,100]:\n",
    "    metrics_func_dict['RE@'+str(p)] = lambda y_true,y_score,q=p: np.nanmean(kumet.rank_errors_at_percentile(y_true=y_true, y_pred=y_score, q=q, no_true_label_value=None).numpy())\n",
    "\n",
    "stays_filters_dict = {\n",
    "    'Newborns': lambda x: x.index.isin(visit_categories.query(\"visit_detail_concept_id=='newborn'\").index),\n",
    "    'Surgical': lambda x: x.index.isin(visit_categories.query(\"visit_detail_concept_id=='surgical'\").index),\n",
    "    'Medical': lambda x: x.index.isin(visit_categories.query(\"visit_detail_concept_id=='medical'\").index)\n",
    "}\n",
    "results_df = pd.DataFrame()\n",
    "\n",
    "def compute_metrics(truth_df,predictions_df, metrics_list):\n",
    "\n",
    "    predictions_df.sort_index(ascending=True,inplace=True)\n",
    "    truth_df.sort_index(ascending=True,inplace=True)\n",
    "\n",
    "    metrics_df = pd.DataFrame()\n",
    "\n",
    "    for code_subset, code_subset_name in zip([icd9_allcodes, icd9_chapters, icd9_subchapters, icd9_3dig, icd9_billing],\n",
    "                                            ['All', 'Chapter', 'Subchapter', '3 digits', 'Billing']):\n",
    "        for filter_name,filter_func in stays_filters_dict.items():\n",
    "            code_subset_dict = {\"Hierarchy level\":[code_subset_name], \"Filter\":[filter_name]}\n",
    "            filtered_visits = filter_func(truth_df)\n",
    "            for metric in metrics_list:\n",
    "                code_subset_dict[metric] = [metrics_func_dict[metric](truth_df.loc[filtered_visits,code_subset].to_numpy(),\n",
    "                                                                    predictions_df.loc[filtered_visits,code_subset].to_numpy())]\n",
    "            metrics_df = pd.concat([metrics_df, pd.DataFrame(code_subset_dict)])\n",
    "    return metrics_df\n",
    "\n",
    "for crossval_suffix,crossval_id in zip(['', '_1', '_2'],[0,1,2]):\n",
    "    visit_ids = pd.read_csv(datadir/('test_visits_id'+crossval_suffix+'.csv.gz')).squeeze()\n",
    "    test_oh_diagnoses_df = oh_diagnoses_df.loc[visit_ids,:]\n",
    "    metrics_df = pd.DataFrame()\n",
    "    # Compute RE,NDCG and NDCG@15 on NDCG experiment\n",
    "    test_predictions = pd.read_csv(\n",
    "        base_dir+\"overall_NDCG/drugSexAge_AllCodes\"+crossval_suffix\n",
    "        +\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    tmp = compute_metrics(test_oh_diagnoses_df,test_predictions,\n",
    "                ['NDCG','NDCG@15','RE@25','RE@50','RE@75','RE@100'])\n",
    "    metrics_df = pd.concat([metrics_df, tmp])\n",
    "    # Compute micro AUROC\n",
    "    test_predictions = pd.read_csv(\n",
    "        base_dir+\"overall_micro_AUROC/drugSexAge_AllCodes\"+crossval_suffix\n",
    "        +\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    tmp = compute_metrics(test_oh_diagnoses_df,test_predictions,\n",
    "            ['micro_AUROC'])\n",
    "    metrics_df = pd.merge(metrics_df, tmp,on=[\"Hierarchy level\",\"Filter\"])\n",
    "    # Compute macro AUROC\n",
    "    test_predictions = pd.read_csv(\n",
    "        base_dir+\"overall_macro_AUROC/drugSexAge_AllCodes\"+crossval_suffix\n",
    "        +\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    tmp = compute_metrics(test_oh_diagnoses_df,test_predictions,\n",
    "                ['macro_AUROC'])\n",
    "    metrics_df = pd.merge(metrics_df, tmp,on=[\"Hierarchy level\",\"Filter\"])\n",
    "    # Compute micro AUPRC and LRAP\n",
    "    test_predictions = pd.read_csv(\n",
    "        base_dir+\"overall_micro_aucPR/drugSexAge_AllCodes\"+crossval_suffix\n",
    "        +\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    tmp = compute_metrics(test_oh_diagnoses_df,test_predictions,\n",
    "                ['micro_AUPRC'])\n",
    "    metrics_df = pd.merge(metrics_df, tmp,on=[\"Hierarchy level\",\"Filter\"])\n",
    "    # Compute macro AUPRC\n",
    "    test_predictions = pd.read_csv(\n",
    "        base_dir+\"overall_macro_aucPR/drugSexAge_AllCodes\"+crossval_suffix\n",
    "        +\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    tmp = compute_metrics(test_oh_diagnoses_df,test_predictions,\n",
    "                ['macro_AUPRC'])\n",
    "    metrics_df = pd.merge(metrics_df, tmp,on=[\"Hierarchy level\",\"Filter\"])\n",
    "    \n",
    "    metrics_df['crossval_id'] = crossval_id\n",
    "    results_df = pd.concat([results_df,metrics_df])\n",
    "results_df.to_csv(tables_dir/'performance_metrics_per_stay_type.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "172e322f",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = pd.read_csv(tables_dir/'performance_metrics_per_stay_type.csv',index_col=['Filter','Hierarchy level'])\n",
    "ranking_metrics = ['NDCG','NDCG@15','RE@25','RE@50','RE@75','RE@100']\n",
    "soft_classif_metrics = [\"micro_AUROC\", \"macro_AUROC\", \"micro_AUPRC\",\"macro_AUPRC\"]\n",
    "non_RE_cols = results_df.columns[~results_df.columns.str.startswith('RE@')]\n",
    "results_df.loc[:,non_RE_cols]*=100\n",
    "results_df = results_df.reset_index(drop=False).melt(id_vars=['Filter','Hierarchy level','crossval_id'],var_name='metric',value_name='value')\n",
    "results_df = results_df.drop(columns='crossval_id').groupby(by=[\"Filter\",'Hierarchy level','metric']).aggregate(['mean','sem'])\n",
    "results_df = results_df.droplevel(level=0,axis=1)\n",
    "results_df['value'] = results_df.apply(lambda x: str(x['mean'])[0:5] + ' $\\pm$ ' + str(x['sem'])[0:4],axis=1)\n",
    "results_df = results_df.drop(columns=['mean','sem']).reset_index()\n",
    "results_df = results_df.rename(columns={\"metric\":\"Metric\"})\n",
    "results_df = results_df.pivot(index=[\"Filter\",\"Hierarchy level\"],columns=\"Metric\", values='value')\n",
    "results_df = results_df.reindex([\"Chapter\",\"Subchapter\",\"3 digits\",\"Billing\",\"All\"], level=\"Hierarchy level\")\n",
    "\n",
    "results_df.style.to_latex(buf=tables_dir/\"perf_per_stay_type.tex\",\n",
    "                environment='void',hrules=True,clines=\"skip-last;index\",column_format='llrrrrrrrrrr')\n",
    "results_df.loc[:,ranking_metrics].style.to_latex(buf=tables_dir/\"ranking_perf_per_stay_type.tex\",\n",
    "                environment='void',hrules=True,clines=\"skip-last;index\",column_format='llrrrrrr')\n",
    "\n",
    "results_classif_df = results_df.loc[:,soft_classif_metrics]\n",
    "results_classif_df.columns = results_classif_df.columns.str.replace(pat='micro_',repl=\"$\\mu$\",regex=False)\n",
    "results_classif_df.columns = results_classif_df.columns.str.replace(pat='macro_',repl=\"macro \",regex=False)\n",
    "results_classif_df.style.to_latex(buf=tables_dir/\"softclassif_perf_per_stay_type.tex\",\n",
    "                environment='void',hrules=True,clines=\"skip-last;index\",column_format='llrrrr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cfd1c105",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Metric</th>\n",
       "      <th>NDCG</th>\n",
       "      <th>NDCG@15</th>\n",
       "      <th>RE@100</th>\n",
       "      <th>RE@25</th>\n",
       "      <th>RE@50</th>\n",
       "      <th>RE@75</th>\n",
       "      <th>macro_AUPRC</th>\n",
       "      <th>macro_AUROC</th>\n",
       "      <th>micro_AUPRC</th>\n",
       "      <th>micro_AUROC</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Filter</th>\n",
       "      <th>Hierarchy level</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Medical</th>\n",
       "      <th>Chapter</th>\n",
       "      <td>92.78 $\\pm$ 0.11</td>\n",
       "      <td>92.13 $\\pm$ 0.13</td>\n",
       "      <td>4.281 $\\pm$ 0.01</td>\n",
       "      <td>0.367 $\\pm$ 0.00</td>\n",
       "      <td>0.973 $\\pm$ 0.00</td>\n",
       "      <td>2.137 $\\pm$ 0.02</td>\n",
       "      <td>64.86 $\\pm$ 0.23</td>\n",
       "      <td>78.65 $\\pm$ 0.27</td>\n",
       "      <td>81.81 $\\pm$ 0.07</td>\n",
       "      <td>87.63 $\\pm$ 0.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Subchapter</th>\n",
       "      <td>84.41 $\\pm$ 0.17</td>\n",
       "      <td>72.63 $\\pm$ 0.16</td>\n",
       "      <td>29.68 $\\pm$ 0.30</td>\n",
       "      <td>1.474 $\\pm$ 0.03</td>\n",
       "      <td>4.045 $\\pm$ 0.04</td>\n",
       "      <td>10.43 $\\pm$ 0.09</td>\n",
       "      <td>27.62 $\\pm$ 0.32</td>\n",
       "      <td>79.72 $\\pm$ 0.57</td>\n",
       "      <td>62.29 $\\pm$ 0.02</td>\n",
       "      <td>94.29 $\\pm$ 0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3 digits</th>\n",
       "      <td>75.80 $\\pm$ 0.17</td>\n",
       "      <td>57.94 $\\pm$ 0.14</td>\n",
       "      <td>171.8 $\\pm$ 1.67</td>\n",
       "      <td>4.631 $\\pm$ 0.09</td>\n",
       "      <td>14.24 $\\pm$ 0.08</td>\n",
       "      <td>40.78 $\\pm$ 0.24</td>\n",
       "      <td>14.39 $\\pm$ 0.20</td>\n",
       "      <td>79.97 $\\pm$ 0.15</td>\n",
       "      <td>45.24 $\\pm$ 0.05</td>\n",
       "      <td>96.67 $\\pm$ 0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Billing</th>\n",
       "      <td>66.40 $\\pm$ 0.13</td>\n",
       "      <td>46.54 $\\pm$ 0.10</td>\n",
       "      <td>848.1 $\\pm$ 9.76</td>\n",
       "      <td>12.19 $\\pm$ 0.24</td>\n",
       "      <td>39.96 $\\pm$ 0.25</td>\n",
       "      <td>131.6 $\\pm$ 1.88</td>\n",
       "      <td>8.992 $\\pm$ 0.13</td>\n",
       "      <td>80.75 $\\pm$ 0.16</td>\n",
       "      <td>30.70 $\\pm$ 0.09</td>\n",
       "      <td>97.59 $\\pm$ 0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>83.36 $\\pm$ 0.14</td>\n",
       "      <td>76.24 $\\pm$ 0.29</td>\n",
       "      <td>1827. $\\pm$ 20.0</td>\n",
       "      <td>6.689 $\\pm$ 0.21</td>\n",
       "      <td>33.94 $\\pm$ 0.44</td>\n",
       "      <td>143.1 $\\pm$ 0.59</td>\n",
       "      <td>11.29 $\\pm$ 0.21</td>\n",
       "      <td>80.50 $\\pm$ 0.13</td>\n",
       "      <td>50.12 $\\pm$ 0.03</td>\n",
       "      <td>98.21 $\\pm$ 0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Newborns</th>\n",
       "      <th>Chapter</th>\n",
       "      <td>97.10 $\\pm$ 0.35</td>\n",
       "      <td>96.59 $\\pm$ 0.45</td>\n",
       "      <td>1.157 $\\pm$ 0.06</td>\n",
       "      <td>0.105 $\\pm$ 0.02</td>\n",
       "      <td>0.217 $\\pm$ 0.04</td>\n",
       "      <td>0.635 $\\pm$ 0.04</td>\n",
       "      <td>22.32 $\\pm$ 0.75</td>\n",
       "      <td>72.17 $\\pm$ 1.59</td>\n",
       "      <td>88.87 $\\pm$ 0.41</td>\n",
       "      <td>96.76 $\\pm$ 0.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Subchapter</th>\n",
       "      <td>94.70 $\\pm$ 0.38</td>\n",
       "      <td>93.55 $\\pm$ 0.44</td>\n",
       "      <td>3.350 $\\pm$ 0.18</td>\n",
       "      <td>0.354 $\\pm$ 0.06</td>\n",
       "      <td>0.645 $\\pm$ 0.09</td>\n",
       "      <td>1.450 $\\pm$ 0.09</td>\n",
       "      <td>18.94 $\\pm$ 2.40</td>\n",
       "      <td>74.60 $\\pm$ 1.17</td>\n",
       "      <td>87.46 $\\pm$ 0.30</td>\n",
       "      <td>99.24 $\\pm$ 0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3 digits</th>\n",
       "      <td>88.48 $\\pm$ 0.59</td>\n",
       "      <td>84.74 $\\pm$ 0.67</td>\n",
       "      <td>23.68 $\\pm$ 0.33</td>\n",
       "      <td>2.019 $\\pm$ 0.58</td>\n",
       "      <td>3.805 $\\pm$ 0.88</td>\n",
       "      <td>7.805 $\\pm$ 1.14</td>\n",
       "      <td>14.52 $\\pm$ 0.36</td>\n",
       "      <td>72.98 $\\pm$ 0.32</td>\n",
       "      <td>70.85 $\\pm$ 0.86</td>\n",
       "      <td>99.37 $\\pm$ 0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Billing</th>\n",
       "      <td>41.05 $\\pm$ 0.31</td>\n",
       "      <td>34.62 $\\pm$ 0.38</td>\n",
       "      <td>280.6 $\\pm$ 13.5</td>\n",
       "      <td>25.20 $\\pm$ 6.54</td>\n",
       "      <td>46.08 $\\pm$ 11.9</td>\n",
       "      <td>89.73 $\\pm$ 17.8</td>\n",
       "      <td>9.891 $\\pm$ 0.14</td>\n",
       "      <td>72.63 $\\pm$ 0.71</td>\n",
       "      <td>35.56 $\\pm$ 1.13</td>\n",
       "      <td>98.74 $\\pm$ 0.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>91.91 $\\pm$ 0.43</td>\n",
       "      <td>84.36 $\\pm$ 0.70</td>\n",
       "      <td>383.0 $\\pm$ 23.7</td>\n",
       "      <td>4.354 $\\pm$ 0.96</td>\n",
       "      <td>12.46 $\\pm$ 2.50</td>\n",
       "      <td>37.46 $\\pm$ 6.99</td>\n",
       "      <td>12.62 $\\pm$ 0.19</td>\n",
       "      <td>72.58 $\\pm$ 0.56</td>\n",
       "      <td>73.03 $\\pm$ 0.68</td>\n",
       "      <td>99.55 $\\pm$ 0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Surgical</th>\n",
       "      <th>Chapter</th>\n",
       "      <td>90.28 $\\pm$ 0.01</td>\n",
       "      <td>89.65 $\\pm$ 0.07</td>\n",
       "      <td>4.677 $\\pm$ 0.03</td>\n",
       "      <td>0.516 $\\pm$ 0.00</td>\n",
       "      <td>1.189 $\\pm$ 0.01</td>\n",
       "      <td>2.496 $\\pm$ 0.02</td>\n",
       "      <td>62.18 $\\pm$ 0.72</td>\n",
       "      <td>79.68 $\\pm$ 0.11</td>\n",
       "      <td>76.87 $\\pm$ 0.10</td>\n",
       "      <td>86.86 $\\pm$ 0.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Subchapter</th>\n",
       "      <td>79.96 $\\pm$ 0.06</td>\n",
       "      <td>68.50 $\\pm$ 0.02</td>\n",
       "      <td>30.95 $\\pm$ 0.01</td>\n",
       "      <td>2.438 $\\pm$ 0.04</td>\n",
       "      <td>5.533 $\\pm$ 0.12</td>\n",
       "      <td>12.29 $\\pm$ 0.11</td>\n",
       "      <td>26.36 $\\pm$ 0.52</td>\n",
       "      <td>82.64 $\\pm$ 0.05</td>\n",
       "      <td>56.42 $\\pm$ 0.24</td>\n",
       "      <td>93.65 $\\pm$ 0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3 digits</th>\n",
       "      <td>70.91 $\\pm$ 0.20</td>\n",
       "      <td>53.03 $\\pm$ 0.14</td>\n",
       "      <td>169.3 $\\pm$ 1.39</td>\n",
       "      <td>7.640 $\\pm$ 0.20</td>\n",
       "      <td>19.71 $\\pm$ 0.23</td>\n",
       "      <td>49.83 $\\pm$ 0.26</td>\n",
       "      <td>13.07 $\\pm$ 0.28</td>\n",
       "      <td>82.72 $\\pm$ 0.11</td>\n",
       "      <td>39.68 $\\pm$ 0.02</td>\n",
       "      <td>96.30 $\\pm$ 0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Billing</th>\n",
       "      <td>62.44 $\\pm$ 0.30</td>\n",
       "      <td>44.31 $\\pm$ 0.25</td>\n",
       "      <td>893.6 $\\pm$ 18.2</td>\n",
       "      <td>26.08 $\\pm$ 1.07</td>\n",
       "      <td>66.98 $\\pm$ 2.00</td>\n",
       "      <td>194.0 $\\pm$ 3.44</td>\n",
       "      <td>7.423 $\\pm$ 0.27</td>\n",
       "      <td>82.56 $\\pm$ 0.22</td>\n",
       "      <td>28.08 $\\pm$ 0.14</td>\n",
       "      <td>97.04 $\\pm$ 0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>79.87 $\\pm$ 0.07</td>\n",
       "      <td>69.78 $\\pm$ 0.14</td>\n",
       "      <td>1965. $\\pm$ 44.1</td>\n",
       "      <td>11.08 $\\pm$ 0.80</td>\n",
       "      <td>49.25 $\\pm$ 1.13</td>\n",
       "      <td>186.6 $\\pm$ 3.95</td>\n",
       "      <td>9.550 $\\pm$ 0.24</td>\n",
       "      <td>82.56 $\\pm$ 0.20</td>\n",
       "      <td>45.24 $\\pm$ 0.06</td>\n",
       "      <td>97.91 $\\pm$ 0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Metric                                NDCG           NDCG@15  \\\n",
       "Filter   Hierarchy level                                       \n",
       "Medical  Chapter          92.78 $\\pm$ 0.11  92.13 $\\pm$ 0.13   \n",
       "         Subchapter       84.41 $\\pm$ 0.17  72.63 $\\pm$ 0.16   \n",
       "         3 digits         75.80 $\\pm$ 0.17  57.94 $\\pm$ 0.14   \n",
       "         Billing          66.40 $\\pm$ 0.13  46.54 $\\pm$ 0.10   \n",
       "         All              83.36 $\\pm$ 0.14  76.24 $\\pm$ 0.29   \n",
       "Newborns Chapter          97.10 $\\pm$ 0.35  96.59 $\\pm$ 0.45   \n",
       "         Subchapter       94.70 $\\pm$ 0.38  93.55 $\\pm$ 0.44   \n",
       "         3 digits         88.48 $\\pm$ 0.59  84.74 $\\pm$ 0.67   \n",
       "         Billing          41.05 $\\pm$ 0.31  34.62 $\\pm$ 0.38   \n",
       "         All              91.91 $\\pm$ 0.43  84.36 $\\pm$ 0.70   \n",
       "Surgical Chapter          90.28 $\\pm$ 0.01  89.65 $\\pm$ 0.07   \n",
       "         Subchapter       79.96 $\\pm$ 0.06  68.50 $\\pm$ 0.02   \n",
       "         3 digits         70.91 $\\pm$ 0.20  53.03 $\\pm$ 0.14   \n",
       "         Billing          62.44 $\\pm$ 0.30  44.31 $\\pm$ 0.25   \n",
       "         All              79.87 $\\pm$ 0.07  69.78 $\\pm$ 0.14   \n",
       "\n",
       "Metric                              RE@100             RE@25  \\\n",
       "Filter   Hierarchy level                                       \n",
       "Medical  Chapter          4.281 $\\pm$ 0.01  0.367 $\\pm$ 0.00   \n",
       "         Subchapter       29.68 $\\pm$ 0.30  1.474 $\\pm$ 0.03   \n",
       "         3 digits         171.8 $\\pm$ 1.67  4.631 $\\pm$ 0.09   \n",
       "         Billing          848.1 $\\pm$ 9.76  12.19 $\\pm$ 0.24   \n",
       "         All              1827. $\\pm$ 20.0  6.689 $\\pm$ 0.21   \n",
       "Newborns Chapter          1.157 $\\pm$ 0.06  0.105 $\\pm$ 0.02   \n",
       "         Subchapter       3.350 $\\pm$ 0.18  0.354 $\\pm$ 0.06   \n",
       "         3 digits         23.68 $\\pm$ 0.33  2.019 $\\pm$ 0.58   \n",
       "         Billing          280.6 $\\pm$ 13.5  25.20 $\\pm$ 6.54   \n",
       "         All              383.0 $\\pm$ 23.7  4.354 $\\pm$ 0.96   \n",
       "Surgical Chapter          4.677 $\\pm$ 0.03  0.516 $\\pm$ 0.00   \n",
       "         Subchapter       30.95 $\\pm$ 0.01  2.438 $\\pm$ 0.04   \n",
       "         3 digits         169.3 $\\pm$ 1.39  7.640 $\\pm$ 0.20   \n",
       "         Billing          893.6 $\\pm$ 18.2  26.08 $\\pm$ 1.07   \n",
       "         All              1965. $\\pm$ 44.1  11.08 $\\pm$ 0.80   \n",
       "\n",
       "Metric                               RE@50             RE@75  \\\n",
       "Filter   Hierarchy level                                       \n",
       "Medical  Chapter          0.973 $\\pm$ 0.00  2.137 $\\pm$ 0.02   \n",
       "         Subchapter       4.045 $\\pm$ 0.04  10.43 $\\pm$ 0.09   \n",
       "         3 digits         14.24 $\\pm$ 0.08  40.78 $\\pm$ 0.24   \n",
       "         Billing          39.96 $\\pm$ 0.25  131.6 $\\pm$ 1.88   \n",
       "         All              33.94 $\\pm$ 0.44  143.1 $\\pm$ 0.59   \n",
       "Newborns Chapter          0.217 $\\pm$ 0.04  0.635 $\\pm$ 0.04   \n",
       "         Subchapter       0.645 $\\pm$ 0.09  1.450 $\\pm$ 0.09   \n",
       "         3 digits         3.805 $\\pm$ 0.88  7.805 $\\pm$ 1.14   \n",
       "         Billing          46.08 $\\pm$ 11.9  89.73 $\\pm$ 17.8   \n",
       "         All              12.46 $\\pm$ 2.50  37.46 $\\pm$ 6.99   \n",
       "Surgical Chapter          1.189 $\\pm$ 0.01  2.496 $\\pm$ 0.02   \n",
       "         Subchapter       5.533 $\\pm$ 0.12  12.29 $\\pm$ 0.11   \n",
       "         3 digits         19.71 $\\pm$ 0.23  49.83 $\\pm$ 0.26   \n",
       "         Billing          66.98 $\\pm$ 2.00  194.0 $\\pm$ 3.44   \n",
       "         All              49.25 $\\pm$ 1.13  186.6 $\\pm$ 3.95   \n",
       "\n",
       "Metric                         macro_AUPRC       macro_AUROC  \\\n",
       "Filter   Hierarchy level                                       \n",
       "Medical  Chapter          64.86 $\\pm$ 0.23  78.65 $\\pm$ 0.27   \n",
       "         Subchapter       27.62 $\\pm$ 0.32  79.72 $\\pm$ 0.57   \n",
       "         3 digits         14.39 $\\pm$ 0.20  79.97 $\\pm$ 0.15   \n",
       "         Billing          8.992 $\\pm$ 0.13  80.75 $\\pm$ 0.16   \n",
       "         All              11.29 $\\pm$ 0.21  80.50 $\\pm$ 0.13   \n",
       "Newborns Chapter          22.32 $\\pm$ 0.75  72.17 $\\pm$ 1.59   \n",
       "         Subchapter       18.94 $\\pm$ 2.40  74.60 $\\pm$ 1.17   \n",
       "         3 digits         14.52 $\\pm$ 0.36  72.98 $\\pm$ 0.32   \n",
       "         Billing          9.891 $\\pm$ 0.14  72.63 $\\pm$ 0.71   \n",
       "         All              12.62 $\\pm$ 0.19  72.58 $\\pm$ 0.56   \n",
       "Surgical Chapter          62.18 $\\pm$ 0.72  79.68 $\\pm$ 0.11   \n",
       "         Subchapter       26.36 $\\pm$ 0.52  82.64 $\\pm$ 0.05   \n",
       "         3 digits         13.07 $\\pm$ 0.28  82.72 $\\pm$ 0.11   \n",
       "         Billing          7.423 $\\pm$ 0.27  82.56 $\\pm$ 0.22   \n",
       "         All              9.550 $\\pm$ 0.24  82.56 $\\pm$ 0.20   \n",
       "\n",
       "Metric                         micro_AUPRC       micro_AUROC  \n",
       "Filter   Hierarchy level                                      \n",
       "Medical  Chapter          81.81 $\\pm$ 0.07  87.63 $\\pm$ 0.08  \n",
       "         Subchapter       62.29 $\\pm$ 0.02  94.29 $\\pm$ 0.01  \n",
       "         3 digits         45.24 $\\pm$ 0.05  96.67 $\\pm$ 0.01  \n",
       "         Billing          30.70 $\\pm$ 0.09  97.59 $\\pm$ 0.02  \n",
       "         All              50.12 $\\pm$ 0.03  98.21 $\\pm$ 0.00  \n",
       "Newborns Chapter          88.87 $\\pm$ 0.41  96.76 $\\pm$ 0.19  \n",
       "         Subchapter       87.46 $\\pm$ 0.30  99.24 $\\pm$ 0.04  \n",
       "         3 digits         70.85 $\\pm$ 0.86  99.37 $\\pm$ 0.03  \n",
       "         Billing          35.56 $\\pm$ 1.13  98.74 $\\pm$ 0.09  \n",
       "         All              73.03 $\\pm$ 0.68  99.55 $\\pm$ 0.02  \n",
       "Surgical Chapter          76.87 $\\pm$ 0.10  86.86 $\\pm$ 0.06  \n",
       "         Subchapter       56.42 $\\pm$ 0.24  93.65 $\\pm$ 0.05  \n",
       "         3 digits         39.68 $\\pm$ 0.02  96.30 $\\pm$ 0.01  \n",
       "         Billing          28.08 $\\pm$ 0.14  97.04 $\\pm$ 0.01  \n",
       "         All              45.24 $\\pm$ 0.06  97.91 $\\pm$ 0.00  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e6facce",
   "metadata": {},
   "source": [
    "# Table S5 Model Calibration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "29c1b116",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'calibration' from '/home/quentin/Projects/completion_pmsi/code/libs/calibration.py'>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#from estimating_eces.explicalib.calibration.evaluation.metrics import confidence as confidence_ece\n",
    "import calibration\n",
    "from importlib import reload\n",
    "reload(calibration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "372e3da0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow_probability as tfp\n",
    "import sklearn.metrics as skmet\n",
    "def logit(p):\n",
    "    return np.log(p) - np.log(1 - p)\n",
    "\n",
    "\n",
    "oh_diagnoses_df = pd.read_parquet(\n",
    "    datadir/\"oh_diagnoses_rollup.parquet\").set_index('visit_occurrence_id')\n",
    "\n",
    "icd9_concepts = pd.read_csv(datadir / \"icd9_concepts.csv.gz\")\n",
    "icd9_allcodes = icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].concept_code\n",
    "icd9_chapters = icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id==\"Chapter\"').concept_code\n",
    "icd9_subchapters = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id==\"SubChapter\"')\n",
    "                    .concept_code)\n",
    "icd9_3dig = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.startswith(\"3-dig\",'\n",
    "                                'na=False)').concept_code)\n",
    "icd9_4dig = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.startswith(\"4-dig\",'\n",
    "                                'na=False)').concept_code)\n",
    "icd9_5dig = (icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.startswith(\"5-dig\",'\n",
    "                                'na=False)').concept_code)\n",
    "icd9_billing = (\n",
    "    icd9_concepts[icd9_concepts.concept_code.isin(oh_diagnoses_df.columns)].query('concept_class_id.str.endswith(\"-dig billing code\",'\n",
    "                        'na=False)').concept_code)\n",
    "\n",
    "base_dir = str(get_modelsdir())+\"/mimic-omop/sequentialNN_3/\"\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9a7ef04c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  metric  crossval_id       ECE       MCE  Brier score\n",
      "0   NDCG            0  0.000686  0.029386     0.002816\n",
      "  metric  crossval_id       ECE       MCE  Brier score\n",
      "0   NDCG            1  0.000486  0.018163     0.002869\n",
      "  metric  crossval_id       ECE       MCE  Brier score\n",
      "0   NDCG            2  0.000498  0.021384     0.002863\n"
     ]
    }
   ],
   "source": [
    "results_df = pd.DataFrame()\n",
    "for crossval_suffix,crossval_id in zip(['', '_1', '_2'],[0,1,2]):\n",
    "    visit_ids = pd.read_csv(datadir/('test_visits_id'+crossval_suffix+'.csv.gz')).squeeze()\n",
    "    test_oh_diagnoses_df = oh_diagnoses_df.loc[visit_ids,:]\n",
    "    metrics_df = pd.DataFrame()\n",
    "    # Compute RE,NDCG and NDCG@15 on NDCG experiment\n",
    "    test_predictions = pd.read_csv(\n",
    "        base_dir+\"overall_NDCG/drugSexAge_AllCodes\"+crossval_suffix\n",
    "        +\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    \n",
    "    # Convert the binary multilabel predictions to a fake 2 mod multiclass\n",
    "    flatten_pred = test_predictions.to_numpy().flatten()\n",
    "    pred_probas = np.zeros((flatten_pred.shape[0],2))\n",
    "    pred_probas[:,0] = 1.0 - flatten_pred\n",
    "    pred_probas[:,1] = flatten_pred\n",
    "\n",
    "    ece, mce = calibration.confidence_ece_a(\n",
    "        probas_pred=pred_probas,\n",
    "        Y=test_oh_diagnoses_df.to_numpy().flatten(),\n",
    "        n_bins=500,  # \"sqrt\",\n",
    "        backend=\"accuracies_confidences\",\n",
    "        summarizing_function=(\"average\",\"max\"))\n",
    "    brier = skmet.brier_score_loss(\n",
    "        y_true=test_oh_diagnoses_df.to_numpy().flatten(),\n",
    "        y_prob=pred_probas[:,1],\n",
    "        pos_label=1)\n",
    "    tmp = pd.DataFrame({\"metric\":[\"NDCG\"],\n",
    "                    \"crossval_id\":[crossval_id],\n",
    "                    \"ECE\":[ece],\n",
    "                    \"MCE\":[mce],\n",
    "                    \"Brier score\":[brier]})\n",
    "    print(tmp)\n",
    "    results_df = pd.concat([results_df,tmp])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b1fd29db",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute ECE on AUCPR models\n",
    "for crossval_suffix,crossval_id in zip(['', '_1', '_2'],[0,1,2]):\n",
    "    visit_ids = pd.read_csv(datadir/('test_visits_id'+crossval_suffix+'.csv.gz')).squeeze()\n",
    "    test_oh_diagnoses_df = oh_diagnoses_df.loc[visit_ids,:]\n",
    "    metrics_df = pd.DataFrame()\n",
    "    # Compute RE,NDCG and NDCG@15 on NDCG experiment\n",
    "    test_predictions = pd.read_csv(\n",
    "        base_dir+\"overall_micro_aucPR/drugSexAge_AllCodes\"+crossval_suffix\n",
    "        +\"/best_models/1/predictions_test.csv.gz\").set_index('visit_occurrence_id')\n",
    "    \n",
    "    # Convert the binary multilabel predictions to a fake 2 mod multiclass\n",
    "    flatten_pred = test_predictions.to_numpy().flatten()\n",
    "    pred_probas = np.zeros((flatten_pred.shape[0],2))\n",
    "    pred_probas[:,0] = 1.0 - flatten_pred\n",
    "    pred_probas[:,1] = flatten_pred    \n",
    "\n",
    "    ece, mce = calibration.confidence_ece_a(\n",
    "        probas_pred=pred_probas,\n",
    "        Y=test_oh_diagnoses_df.to_numpy().flatten(),\n",
    "        n_bins=500,  # \"sqrt\",\n",
    "        backend=\"accuracies_confidences\",\n",
    "        summarizing_function=(\"average\",\"max\"))\n",
    "    brier = skmet.brier_score_loss(\n",
    "        y_true=test_oh_diagnoses_df.to_numpy().flatten(),\n",
    "        y_prob=pred_probas[:,1],\n",
    "        pos_label=1)\n",
    "    tmp = pd.DataFrame({\"metric\":[\"AUPRC\"],\n",
    "                    \"crossval_id\":[crossval_id],\n",
    "                    \"ECE\":[ece],\n",
    "                    \"MCE\":[mce],\n",
    "                    \"Brier score\":[brier]})\n",
    "    results_df = pd.concat([results_df,tmp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a8062642",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df.to_csv(tables_dir/\"calibration.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d6728dd3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calibration Metric         Brier score                 ECE                 MCE\n",
      "Optimized Metric                                                              \n",
      "AUPRC               0.282 $\\pm$ 0.00\\%  0.039 $\\pm$ 0.00\\%  1.171 $\\pm$ 0.23\\%\n",
      "NDCG                0.284 $\\pm$ 0.00\\%  0.055 $\\pm$ 0.00\\%  2.297 $\\pm$ 0.33\\%\n"
     ]
    }
   ],
   "source": [
    "results_df = pd.read_csv(tables_dir/\"calibration.csv\",index_col=['metric','crossval_id'])\n",
    "results_df.loc[:,:]*=100\n",
    "results_df = results_df.reset_index(drop=False).melt(id_vars=['metric','crossval_id'],var_name='cal_metric',value_name='value')\n",
    "results_df = results_df.drop(columns='crossval_id').groupby(by=[\"metric\",'cal_metric']).aggregate(['mean','sem'])\n",
    "results_df = results_df.droplevel(level=0,axis=1)\n",
    "results_df['value'] = results_df.apply(lambda x: str(x['mean'])[0:5] + ' $\\pm$ ' + str(x['sem'])[0:4] + \"\\%\",axis=1)\n",
    "results_df = results_df.drop(columns=['mean','sem']).reset_index()\n",
    "results_df = results_df.rename(columns={\"metric\":\"Optimized Metric\",\"cal_metric\":\"Calibration Metric\"})\n",
    "results_df = results_df.pivot(index=[\"Optimized Metric\"],columns=\"Calibration Metric\", values='value')\n",
    "print(results_df)\n",
    "results_df.style.to_latex(buf=tables_dir/\"calibration_perf.tex\",\n",
    "                environment='void',hrules=True,clines=\"skip-last;index\",column_format='llccc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9e3234f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Calibration Metric</th>\n",
       "      <th>Brier score</th>\n",
       "      <th>ECE</th>\n",
       "      <th>MCE</th>\n",
       "      <th>Unnamed: 0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Optimized Metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AUPRC</th>\n",
       "      <td>0.282 $\\pm$ 0.00\\%</td>\n",
       "      <td>0.039 $\\pm$ 0.00\\%</td>\n",
       "      <td>1.171 $\\pm$ 0.23\\%</td>\n",
       "      <td>0.0 $\\pm$ 0.0\\%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NDCG</th>\n",
       "      <td>0.284 $\\pm$ 0.00\\%</td>\n",
       "      <td>0.055 $\\pm$ 0.00\\%</td>\n",
       "      <td>2.297 $\\pm$ 0.33\\%</td>\n",
       "      <td>0.0 $\\pm$ 0.0\\%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Calibration Metric         Brier score                 ECE  \\\n",
       "Optimized Metric                                             \n",
       "AUPRC               0.282 $\\pm$ 0.00\\%  0.039 $\\pm$ 0.00\\%   \n",
       "NDCG                0.284 $\\pm$ 0.00\\%  0.055 $\\pm$ 0.00\\%   \n",
       "\n",
       "Calibration Metric                 MCE       Unnamed: 0  \n",
       "Optimized Metric                                         \n",
       "AUPRC               1.171 $\\pm$ 0.23\\%  0.0 $\\pm$ 0.0\\%  \n",
       "NDCG                2.297 $\\pm$ 0.33\\%  0.0 $\\pm$ 0.0\\%  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0109eca",
   "metadata": {},
   "source": [
    "# Table S10 Best models hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "2218a13e",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_dir = get_modelsdir() / \"mimic-omop/sequentialNN_3/\"\n",
    "met_dirs = listdir_only(models_dir)\n",
    "hyper_parms_df = pd.DataFrame()\n",
    "for metric in met_dirs:\n",
    "    exp_runs = listdir_only(models_dir/metric)\n",
    "    #exp_dirs = [models_dir/metric/exp/'best_models' for exp in exp_dirs]\n",
    "    for exp in exp_runs:\n",
    "        # process model_str\n",
    "        tmp = process_model_str(pd.DataFrame({'model_str':[exp],\n",
    "                                'metric_value':[0.0]\n",
    "            }))\n",
    "        exp_dir = models_dir/metric/exp/'best_models'\n",
    "        # print(exp_dir)\n",
    "        # Read model json file \n",
    "        with open(exp_dir/\"1/model_hyperparameters.json\") as json_file:\n",
    "            model_json = json_file.read()\n",
    "            # convert to to dataframe\n",
    "            model_json = pd.DataFrame({\"metric\":metric,\n",
    "                                       \"exp_name\":exp.split(\"_\")[1],\n",
    "                                       \"crossval_id\":exp.split(\"_\")[-1],\n",
    "                                       **json.loads(model_json)},index=[0])\n",
    "        hyper_parms_df = pd.concat([hyper_parms_df,\n",
    "                                    model_json],\n",
    "                                   axis=0)\n",
    "hyper_parms_df = hyper_parms_df.reset_index(drop=True)\n",
    "hyper_parms_df.loc[~hyper_parms_df.crossval_id.str.isnumeric(),\"crossval_id\"] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "4b5cdf07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>exp_name</th>\n",
       "      <th>crossval_id</th>\n",
       "      <th>dropout</th>\n",
       "      <th>batchnorm</th>\n",
       "      <th>num_layers</th>\n",
       "      <th>units</th>\n",
       "      <th>activation</th>\n",
       "      <th>tuner/epochs</th>\n",
       "      <th>tuner/initial_epoch</th>\n",
       "      <th>tuner/bracket</th>\n",
       "      <th>tuner/round</th>\n",
       "      <th>tuner/trial_id</th>\n",
       "      <th>outhier_lambda</th>\n",
       "      <th>out_L2</th>\n",
       "      <th>IR_cap</th>\n",
       "      <th>focal_g</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>overall_macro_AUROC</td>\n",
       "      <td>AllCodes</td>\n",
       "      <td>0</td>\n",
       "      <td>0.719302</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>3840</td>\n",
       "      <td>relu</td>\n",
       "      <td>50</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0386</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>overall_macro_AUROC</td>\n",
       "      <td>AllCodes</td>\n",
       "      <td>1</td>\n",
       "      <td>0.784277</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>3840</td>\n",
       "      <td>relu</td>\n",
       "      <td>50</td>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0465</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>overall_macro_AUROC</td>\n",
       "      <td>AllCodes</td>\n",
       "      <td>2</td>\n",
       "      <td>0.784277</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>3840</td>\n",
       "      <td>relu</td>\n",
       "      <td>50</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0445</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>overall_macro_aucPR</td>\n",
       "      <td>AllCodes</td>\n",
       "      <td>0</td>\n",
       "      <td>0.649855</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>3840</td>\n",
       "      <td>relu</td>\n",
       "      <td>50</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0192</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>overall_macro_aucPR</td>\n",
       "      <td>AllCodes</td>\n",
       "      <td>1</td>\n",
       "      <td>0.640512</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>4864</td>\n",
       "      <td>relu</td>\n",
       "      <td>50</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0193</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>overall_macro_F1</td>\n",
       "      <td>AllCodes</td>\n",
       "      <td>1</td>\n",
       "      <td>0.642097</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>5376</td>\n",
       "      <td>relu</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>overall_macro_F1</td>\n",
       "      <td>AllCodes</td>\n",
       "      <td>2</td>\n",
       "      <td>0.517681</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>5376</td>\n",
       "      <td>relu</td>\n",
       "      <td>50</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0190</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>overall_micro_aucPR</td>\n",
       "      <td>AllCodes</td>\n",
       "      <td>0</td>\n",
       "      <td>0.736253</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>4608</td>\n",
       "      <td>relu</td>\n",
       "      <td>150</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0229</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>overall_micro_aucPR</td>\n",
       "      <td>AllCodes</td>\n",
       "      <td>1</td>\n",
       "      <td>0.669924</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>1536</td>\n",
       "      <td>relu</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>overall_micro_aucPR</td>\n",
       "      <td>AllCodes</td>\n",
       "      <td>2</td>\n",
       "      <td>0.745444</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>1792</td>\n",
       "      <td>relu</td>\n",
       "      <td>150</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0242</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>69 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 metric  exp_name crossval_id   dropout  batchnorm  \\\n",
       "0   overall_macro_AUROC  AllCodes           0  0.719302       True   \n",
       "1   overall_macro_AUROC  AllCodes           1  0.784277       True   \n",
       "2   overall_macro_AUROC  AllCodes           2  0.784277       True   \n",
       "3   overall_macro_aucPR  AllCodes           0  0.649855       True   \n",
       "4   overall_macro_aucPR  AllCodes           1  0.640512       True   \n",
       "..                  ...       ...         ...       ...        ...   \n",
       "64     overall_macro_F1  AllCodes           1  0.642097       True   \n",
       "65     overall_macro_F1  AllCodes           2  0.517681       True   \n",
       "66  overall_micro_aucPR  AllCodes           0  0.736253      False   \n",
       "67  overall_micro_aucPR  AllCodes           1  0.669924      False   \n",
       "68  overall_micro_aucPR  AllCodes           2  0.745444      False   \n",
       "\n",
       "    num_layers  units activation  tuner/epochs  tuner/initial_epoch  \\\n",
       "0            3   3840       relu            50                   17   \n",
       "1            3   3840       relu            50                   17   \n",
       "2            3   3840       relu            50                   17   \n",
       "3            3   3840       relu            50                   17   \n",
       "4            3   4864       relu            50                   17   \n",
       "..         ...    ...        ...           ...                  ...   \n",
       "64           3   5376       relu           150                    0   \n",
       "65           3   5376       relu            50                   17   \n",
       "66           2   4608       relu           150                   50   \n",
       "67           2   1536       relu           150                    0   \n",
       "68           2   1792       relu           150                   50   \n",
       "\n",
       "    tuner/bracket  tuner/round tuner/trial_id  outhier_lambda  out_L2  IR_cap  \\\n",
       "0               4            3           0386             NaN     NaN     NaN   \n",
       "1               2            1           0465             NaN     NaN     NaN   \n",
       "2               3            2           0445             NaN     NaN     NaN   \n",
       "3               3            2           0192             NaN     NaN     NaN   \n",
       "4               3            2           0193             NaN     NaN     NaN   \n",
       "..            ...          ...            ...             ...     ...     ...   \n",
       "64              0            0            NaN             NaN     NaN     NaN   \n",
       "65              3            2           0190             NaN     NaN     NaN   \n",
       "66              2            2           0229             NaN     NaN     NaN   \n",
       "67              0            0            NaN             NaN     NaN     NaN   \n",
       "68              1            1           0242             NaN     NaN     NaN   \n",
       "\n",
       "    focal_g  \n",
       "0       NaN  \n",
       "1       NaN  \n",
       "2       NaN  \n",
       "3       NaN  \n",
       "4       NaN  \n",
       "..      ...  \n",
       "64      NaN  \n",
       "65      NaN  \n",
       "66      NaN  \n",
       "67      NaN  \n",
       "68      NaN  \n",
       "\n",
       "[69 rows x 17 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hyper_parms_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "bd7cbaeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write to csv excluding some uninteresting columns\n",
    "(hyper_parms_df.drop(columns=['activation','tuner/epochs',\"tuner/initial_epoch\",\"tuner/trial_id\"])\n",
    " .sort_values([\"metric\",\"exp_name\",\"crossval_id\"])\n",
    " .to_csv(tables_dir/\"best_hyperparms.csv\",index=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "6c3c7bb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a latex table of the hyperpaameters of the best NDCG models\n",
    "tmp = hyper_parms_df.query('metric==\"overall_NDCG\" & exp_name==\"AllCodes\"').loc[:,[\"crossval_id\",\"batchnorm\",\"num_layers\",\"units\",\"dropout\"]]\n",
    "# Rename columns for publication\n",
    "tmp.columns = [\"Cross-validation ID\",\"Batch Normalization\",\"Number of Layers\",\"Number of Units\",\"Dropout Rate\"]\n",
    "# Export to latex table\n",
    "tmp.set_index(\"Cross-validation ID\",drop=True).style.format(precision=2).to_latex(tables_dir/\"AllCodes_best_NDCG_hyperparams.tex\",environment='void',hrules=True,column_format='crrrr')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pmsi_tf_cpu",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}